From 946cc8d29820de59b964897789e9f0b8d145b347 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 14:26:21 +0800 Subject: [PATCH 01/30] feat(cua): ask before moving the system pointer for a session --- docs/config-catalog.md | 4 +- docs/config-catalog.zh.md | 4 +- .../cua-hover-ax-only/guide.i18n.yaml | 4 +- .../v0.2.2-rc.1/cua-hover-ax-only/guide.md | 4 +- .../v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md | 4 +- .../experimental/cua-macos/README.i18n.yaml | 8 +- packages/experimental/cua-macos/README.md | 6 +- packages/experimental/cua-macos/README.zh.md | 6 +- .../cua-macos/native/README.i18n.yaml | 4 +- .../experimental/cua-macos/native/README.md | 2 +- .../cua-macos/native/README.zh.md | 2 +- .../native/Sources/CuaNative/AXHover.swift | 7 -- .../native/Sources/CuaNative/AXWorker.swift | 29 ++++---- .../CuaNative/NativeProtocol.generated.swift | 2 +- .../CuaNative/RequestCoordinator.swift | 3 +- .../native/Sources/CuaNative/Runtime.swift | 7 +- .../cua-macos/native/generate-protocol.ts | 2 +- .../cua-macos/native/protocol.schema.json | 4 +- .../cua-macos/native/tests/e2e.py | 4 +- .../cua-macos/native/tests/electron_e2e.py | 6 +- .../tests/swift/SharedHelperTests.swift | 10 +-- .../native/tests/swift/SystemHoverTests.swift | 14 ---- .../experimental/cua-macos/src/guidance.ts | 2 +- packages/experimental/cua-macos/src/index.ts | 9 +-- .../cua-macos/src/shared-native.ts | 10 +-- .../cua-macos/tests/guidance.spec.ts | 10 +-- .../tests/protocol-generation.spec.ts | 2 +- .../cua-macos/tests/provider-channel.spec.ts | 6 +- .../tests/provider-lifecycle.spec.ts | 73 ++++++++++++++++++- .../cua-macos/tests/shared-native.spec.ts | 10 +-- packages/experimental/cua/README.i18n.yaml | 4 +- packages/experimental/cua/README.md | 2 +- packages/experimental/cua/README.zh.md | 2 +- packages/experimental/cua/src/api.ts | 5 +- .../cua/src/client/ApplicationApproval.tsx | 10 +-- packages/experimental/cua/src/guidance.ts | 1 + packages/experimental/cua/src/index.ts | 9 ++- packages/experimental/cua/src/locales.ts | 25 +++++++ .../experimental/cua/src/mouse-control.ts | 56 ++++++++++++++ .../experimental/cua/src/native-schema.ts | 4 +- packages/experimental/cua/src/protocol.ts | 9 ++- packages/experimental/cua/src/types.ts | 5 ++ packages/experimental/cua/tests/api.spec.ts | 8 ++ .../application-approval.client.spec.tsx | 14 ++++ .../session/cua-native/replay.override.json | 66 +++++++++++++++++ .../cua-native/workspace/native-helper.mjs | 14 ++-- 46 files changed, 346 insertions(+), 146 deletions(-) create mode 100644 packages/experimental/cua/src/mouse-control.ts diff --git a/docs/config-catalog.md b/docs/config-catalog.md index dede6981f9..ed075c348a 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1144,15 +1144,13 @@ export interface Config { - `inject`: `computerUse` · `tools` · `systemPrompt` · `subprocess` · `agents` - `refs`: [`CuaConfig`](../packages/experimental/cua/src/index.ts) -- `source`: [`packages/experimental/cua-macos/src/index.ts:33`](../packages/experimental/cua-macos/src/index.ts) +- `source`: [`packages/experimental/cua-macos/src/index.ts:32`](../packages/experimental/cua-macos/src/index.ts) ```ts config-catalog /** Native preparation, source-build signing, operation bounds, and profile-local application grants. */ export interface Config extends CuaConfig { /** Interval in milliseconds between permission and position checks while the native guide is open. */ permissionPollIntervalMs: number - /** Allow mouseMove to move the visible system pointer when AX hover is unavailable. Off by default to avoid interrupting the user. */ - systemPointerHover: boolean /** * Source-build signing: `ad-hoc` (default) binds grants to the code hash; * an object selects an existing certificate in an explicitly configured keychain. diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index 9c142b9b43..f01692f549 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -1146,15 +1146,13 @@ export interface Config { - `inject`: `computerUse` · `tools` · `systemPrompt` · `subprocess` · `agents` - `refs`: [`CuaConfig`](../packages/experimental/cua/src/index.ts) -- `source`: [`packages/experimental/cua-macos/src/index.ts:33`](../packages/experimental/cua-macos/src/index.ts) +- `source`: [`packages/experimental/cua-macos/src/index.ts:32`](../packages/experimental/cua-macos/src/index.ts) ```ts config-catalog /** Native preparation, source-build signing, operation bounds, and profile-local application grants. */ export interface Config extends CuaConfig { /** Interval in milliseconds between permission and position checks while the native guide is open. */ permissionPollIntervalMs: number - /** Allow mouseMove to move the visible system pointer when AX hover is unavailable. Off by default to avoid interrupting the user. */ - systemPointerHover: boolean /** * Source-build signing: `ad-hoc` (default) binds grants to the code hash; * an object selects an existing certificate in an explicitly configured keychain. diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.i18n.yaml b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.i18n.yaml index 9a48f7428c..cbce06f055 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.i18n.yaml +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.i18n.yaml @@ -12,5 +12,5 @@ en: 55c3070b24b4ad39 zh: 2555d127b0f4e4a0 /macos-hover-keeps-the-system-pointer-in-place-by-default/migration: - en: 1f0ab7e8741e821f - zh: f59adc3a23ebcb43 + en: bf3f886ad344f12a + zh: c877a616d4e147fb diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.md index f7191cc254..a7d03dc869 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.md @@ -14,5 +14,5 @@ On macOS, `win.mouseMove(target)` uses an enabled element or ancestor that adver ## Migration 1. For an available control, use its advertised action or click it directly. Use browser tools for web content that requires page hover. -2. To restore visible pointer movement, set `systemPointerHover: true` in the `config` of the macOS provider entry (`id: cua`) in your profile's `cordis.patch.yml`. Preserve the entry's other settings. If you set `helperPath`, rebuild the custom helper so it accepts `systemPointerHover` in the connection handshake. Restart the profile. -3. Call `win.mouseMove` on a foreground, unobscured control without AX alternate UI. The opt-in result reports `hoverRoute: "system-pointer"` and `systemPointerMoved: true`. Keep the default to avoid disturbing the user's cursor. +2. Remove `systemPointerHover` from the macOS provider entry in your profile’s `cordis.patch.yml`. If you set `helperPath`, rebuild that helper to support the per-request `useSystemPointer` field. Restart the profile. +3. When the task requires a physical hover, call `cua.requestMouseControl()`. After the user approves, `win.mouseMove` on an unobscured foreground target returns `hoverRoute: "system-pointer"` and `systemPointerMoved: true`. This grant expires when DSH restarts. diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md index 713ae43fb3..17064261b3 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md @@ -14,5 +14,5 @@ description: "macOS 的 mouseMove 默认要求控件支持 AX 备用界面，否 ## 迁移 1. 对已经可用的控件，可执行它声明的动作或直接点击。需要网页悬停时可用浏览器工具。 -2. 要恢复可见光标移动，在 profile 的 `cordis.patch.yml` 中，给 macOS 提供方条目（`id: cua`）的 `config` 添加 `systemPointerHover: true`。保留该条目的其他设置。如果指定了 `helperPath`，先重新构建自定义 helper，使其在建立连接时能读取 `systemPointerHover`，然后重启 profile。 -3. 对前台未受遮挡且不提供 AX 备用界面的控件调用 `win.mouseMove`。启用配置时，结果为 `hoverRoute: "system-pointer"` 和 `systemPointerMoved: true`。要保留用户的光标位置，使用默认配置。 +2. 从 profile 的 `cordis.patch.yml` 中移除 macOS 提供方配置项 `systemPointerHover`。如果设置了 `helperPath`，请重新构建该 helper，使其支持按请求读取 `useSystemPointer` 字段，然后重启 profile。 +3. 任务确实需要移动鼠标时，调用 `cua.requestMouseControl()`。用户同意后，对前台未受遮挡的目标调用 `win.mouseMove`，结果为 `hoverRoute: "system-pointer"` 和 `systemPointerMoved: true`。重启 DSH 后授权失效。 diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index 0d83c6e538..25ab23c779 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -15,11 +15,11 @@ en: 1777e7bf6785f986 zh: e3daac1074ebb74d /deepseek-ai-dsh-experimental-cua-macos/use-this-package: - en: 533ed4dd11af5126 - zh: 03036e8c949d6aca + en: 7fdb7c0462275417 + zh: 0a4a9562f4a82945 /deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment: - en: b734a0907747580e - zh: 8110c5951b70e100 + en: 2391a460e295cfca + zh: 1889a903efb6142d /deepseek-ai-dsh-experimental-cua-macos/understand-the-implementation: en: 4e82b85a5aa72eee zh: 4e004b7a205d8688 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 46b035112e..3190f8561d 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -64,7 +64,6 @@ Source builds default to `sourceSigning: ad-hoc`, without creating certificates | `buildTimeoutMs` | `300000` | Maximum native preparation time. | | `timeoutMs` | `30000` | Per-operation deadline; cancellation can leave partial input. | | `connectTimeoutMs` | `5000` | Deadline for connecting to or starting the shared helper. | -| `systemPointerHover` | `false` | Allow `mouseMove` to move the visible cursor when AX hover is unavailable. | | `permissionPollIntervalMs` | `1000` | Interval in milliseconds between permission and position checks while the native guide is open. | | `screenshotTimeoutMs` | `5000` | Screenshot deadline including queue time, capped by `timeoutMs`. | | `axTimeoutMs` | `3000` | Deadline per Accessibility system call, configured independently of the complete operation. | @@ -115,7 +114,7 @@ Use the `cua` API to discover and select the application (`listApps`, `getState` `getWindow` reads its initial tree silently. An owned window without an AX surface returns an empty snapshot and remains available for screenshots and coordinate input. Within one API instance, the facade keeps one current accessibility snapshot per application handle, shared by every app and window object for that handle, including repeated selections. Reading any window replaces that snapshot; element operations use the latest snapshot ID, so use IDs from the latest tree and do not assume an older ID still identifies the same control. A failed read publishes no partial snapshot; read again without repeating the preceding input. Window objects own screenshots and pointer input. Keyboard methods stay on `app` because input follows the app’s current focus. -`mouseMove(target)` defaults to AX hover. An enabled target or ancestor must advertise both `AXShowAlternateUI` and `AXShowDefaultUI`. Moving into its descendants keeps the UI open, and leaving restores it. Without an AX route the helper returns `HOVER_UNAVAILABLE` before sending input. Click a visible control or use `performAction` with an advertised action. For browser content without these AX actions, use browser tools. Enable `systemPointerHover` only when moving the user’s visible cursor is acceptable. That route requires a foreground, unobscured target and leaves the cursor at the destination. +`mouseMove(target)` defaults to AX hover. An enabled target or ancestor must advertise both `AXShowAlternateUI` and `AXShowDefaultUI`. Moving into its descendants keeps the UI open, and leaving restores it. Without an AX route the helper returns `HOVER_UNAVAILABLE` before sending input. Click a visible control or use `performAction` with an advertised action. For browser content without these AX actions, use browser tools. If these alternatives cannot complete the task, `cua.requestMouseControl()` asks for this session until DSH restarts. On approval, every subsequent hover moves the system pointer to the foreground, unobscured target and leaves it there. `paste(text, { format: "text" | "html" })` temporarily publishes the requested clipboard format and invokes the application’s enabled standard Paste menu command. It keeps the temporary format until a pasteboard reader requests its data, then restores every captured original format while it still owns the clipboard, preserving a later copy. Paste operations share a cross-process lock and cancellation joins helper cleanup. A forced process exit can prevent restoration. Unsupported Paste commands fail without typing fallback; `typeText` remains separate key-event input. @@ -186,6 +185,7 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. +- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Returns true only on approval. Avoid asking unless the task requires a physical hover. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -210,7 +210,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only systemPointerHover: true permits moving the visible cursor. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured. +- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, mouseMove uses the system pointer for this session until DSH restarts. If the user declines, continue without mouse control. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index 341796b8c4..87c9fb046f 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -64,7 +64,6 @@ pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis. | `buildTimeoutMs` | `300000` | 原生组件准备的最长时间。 | | `timeoutMs` | `30000` | 单次操作截止时间，取消后可能已发生部分输入。 | | `connectTimeoutMs` | `5000` | 连接或启动共享 helper 的截止时间。 | -| `systemPointerHover` | `false` | AX 悬停不可用时，允许 `mouseMove` 移动可见光标。 | | `permissionPollIntervalMs` | `1000` | 原生引导打开期间检查权限和窗口位置的间隔，单位为毫秒。 | | `screenshotTimeoutMs` | `5000` | 截图总期限，包含排队时间，且不超过 `timeoutMs`。 | | `axTimeoutMs` | `3000` | 每次 Accessibility 系统调用的截止时间，与完整操作截止独立配置。 | @@ -115,7 +114,7 @@ void await window.getScreenshot() `getWindow` 静默读取初始树。已确认归属但没有 AX 界面的窗口返回空快照，仍可截图和发送坐标输入。在单个 API 实例内，门面按应用句柄保存一份当前辅助功能快照，该句柄的所有应用对象和窗口对象共享它，包括重复选择。读取任一窗口都会替换该快照；元素操作使用最新快照 ID，因此应使用最新树中的元素 ID，不应假定旧 ID 仍指向同一控件。读取失败不会发布部分快照；可以重新读取，但不要重复此前的输入。窗口对象负责截图和指针输入。键盘方法保留在 `app` 上，因为输入跟随应用的当前焦点。 -`mouseMove(target)` 默认通过 AX 悬停。目标或其祖先必须已启用，并同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI`。进入子控件会保持展开，离开后恢复默认界面。如果没有可用的 AX 路径，helper 返回 `HOVER_UNAVAILABLE`，不发送输入。可点击已经可见的控件，或用 `performAction` 执行控件声明的动作。浏览器内容缺少这些 AX 动作时，请使用浏览器工具。只有允许移动用户可见光标时才开启 `systemPointerHover`。这条路径要求目标在前台且未被遮挡，操作后光标停在目标位置。 +`mouseMove(target)` 默认通过 AX 悬停。目标或其祖先必须已启用，并同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI`。进入子控件会保持展开，离开后恢复默认界面。如果没有可用的 AX 路径，helper 返回 `HOVER_UNAVAILABLE`，不发送输入。可点击已经可见的控件，或用 `performAction` 执行控件声明的动作。浏览器内容缺少这些 AX 动作时，请使用浏览器工具。只有其他操作都无法完成任务时，才调用 `cua.requestMouseControl()` 询问用户。用户同意后，本会话后续悬停会把系统光标移到前台未受遮挡的目标并停在那里，重启 DSH 后授权失效。 `paste(text, { format: "text" | "html" })` 临时提供指定格式的剪贴板内容，并调用应用已启用的标准 Paste 菜单命令。临时格式保留到剪贴板读取方请求其数据，然后只在仍持有剪贴板时恢复保存的全部原格式，保留期间产生的新复制内容。粘贴操作共用跨进程锁，取消会等待辅助进程清理。强制终止进程可能阻止恢复。应用不支持 Paste 命令时明确失败，不自动改成逐字输入；`typeText` 仍单独提供按键事件输入。 @@ -186,6 +185,7 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. +- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Returns true only on approval. Avoid asking unless the task requires a physical hover. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -210,7 +210,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only systemPointerHover: true permits moving the visible cursor. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured. +- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, mouseMove uses the system pointer for this session until DSH restarts. If the user declines, continue without mouse control. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/packages/experimental/cua-macos/native/README.i18n.yaml b/packages/experimental/cua-macos/native/README.i18n.yaml index 8d43fbd6b8..b858a45026 100644 --- a/packages/experimental/cua-macos/native/README.i18n.yaml +++ b/packages/experimental/cua-macos/native/README.i18n.yaml @@ -21,8 +21,8 @@ en: 88a59904ca4ebedc zh: 14f2b666d7382145 /dsh-cua-native-helper/observation-and-input: - en: 2d380e942a4057e4 - zh: fe7c9339ec63a6e5 + en: c7b28877014797fa + zh: 069271b6be75c8c1 /dsh-cua-native-helper/verification: en: 49f4cce2a4cc3114 zh: e9d65c92ef35fe24 diff --git a/packages/experimental/cua-macos/native/README.md b/packages/experimental/cua-macos/native/README.md index bc80110951..107904886c 100644 --- a/packages/experimental/cua-macos/native/README.md +++ b/packages/experimental/cua-macos/native/README.md @@ -121,7 +121,7 @@ Tree observation and action discovery treat a generic AX failure reading one ele `paste` publishes one plain-text or HTML clipboard format, dispatches the application's enabled standard Paste menu command, and waits for a clipboard data request. It then restores the preserved formats only while its ownership marker and change count still match. The Host serializes its pastes across helpers with a file lock. maxClipboardBytes bounds both input and preserved contents; unsupported menus and oversized data fail before dispatch. A later copy is preserved. Cancellation restores owned clipboard data when possible; forced termination cannot guarantee restoration, and a data request does not prove the target completed editing. After dispatch, `pasteReadTimeoutMs` bounds waiting for clipboard consumption. Expiry returns `PASTE_NOT_CONSUMED`, closes the promised data provider, and restores the original clipboard only while still owned. The failure does not prove the app will never process its queued Paste command; inspect the target before deciding whether to retry. -`mouseMove` reveals or restores alternate UI through an enabled element or ancestor advertising both `AXShowAlternateUI` and `AXShowDefaultUI`. Without an AX route it returns `HOVER_UNAVAILABLE` before sending input. The macOS provider sets `systemPointerHover` in the connection handshake. Diagnostic clients that omit this setting may opt in per request with `allowSystemPointer: true`. The default is false. Only true permits posting a system pointer move after checking the foreground process and window hit. +`mouseMove` reveals or restores alternate UI through an enabled element or ancestor advertising both `AXShowAlternateUI` and `AXShowDefaultUI`. Without an AX route it returns `HOVER_UNAVAILABLE` before sending input. After session approval, the Host sends `useSystemPointer: true` on each `mouseMove` to select the system pointer directly. The opt-in system pointer route returns `FOREGROUND_REQUIRED` if the target is obscured or belongs to a background app. The result reports the route and whether the visible cursor moved in `hoverRoute` and `systemPointerMoved`. It never presses a button or restores the cursor after a read. Later input may interrupt the hover. Clicks send their button events at the target without an off-window priming click. diff --git a/packages/experimental/cua-macos/native/README.zh.md b/packages/experimental/cua-macos/native/README.zh.md index 0927a82bce..1834f216c6 100644 --- a/packages/experimental/cua-macos/native/README.zh.md +++ b/packages/experimental/cua-macos/native/README.zh.md @@ -121,7 +121,7 @@ AppKit 在主事件循环上运行。阻塞式 socket 接收与握手使用后 `paste` 提供一种纯文本或 HTML 剪贴板格式，调用应用已启用的标准 Paste 菜单命令，并等待剪贴板数据请求。随后仅在所有权标记和修改计数仍匹配时恢复原有格式。Host 通过文件锁串行化不同 helper 的粘贴。maxClipboardBytes 同时限制输入和保留内容，不支持的菜单和过大的数据在分派前失败。后续复制会被保留。取消会尽可能恢复仍持有的剪贴板数据，强制结束不保证恢复，数据请求也不证明目标已完成编辑。 分派完成后，`pasteReadTimeoutMs` 限制等待剪贴板读取的时间。到期返回 `PASTE_NOT_CONSUMED`，关闭延迟数据提供方，并仅在仍持有剪贴板时恢复原内容。该失败不能证明应用以后不会处理已排队的 Paste 命令；决定是否重试前先检查目标内容。 -默认情况下，`mouseMove` 通过同一窗口内启用的控件或祖先声明的 `AXShowAlternateUI` 和 `AXShowDefaultUI` 展开或恢复界面。缺少这条路径时返回 `HOVER_UNAVAILABLE`，不发送输入。macOS 提供方在连接握手中设置 `systemPointerHover`。未在握手中指定该配置的诊断客户端，可以在单次请求中传入 `allowSystemPointer: true`。默认值为 false。只有值为 true 才允许移动全局光标，且须检查前台进程和鼠标命中的窗口。 +默认情况下，`mouseMove` 通过同一窗口内启用的控件或祖先声明的 `AXShowAlternateUI` 和 `AXShowDefaultUI` 展开或恢复界面。缺少这条路径时返回 `HOVER_UNAVAILABLE`，不发送输入。用户授权后，Host 在本会话的每次 `mouseMove` 请求中传入 `useSystemPointer: true`，直接使用系统光标。 开启后，系统指针路线会检查前台应用和坐标命中的窗口，不满足条件时返回 `FOREGROUND_REQUIRED`。返回值 `hoverRoute` 和 `systemPointerMoved` 说明走了哪条路线以及是否移动了可见光标。悬停不会按下鼠标，也不会在读取后还原光标。后续输入可能中断悬停。点击只在目标点发送按键事件，不先在窗口外点击。 diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/AXHover.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/AXHover.swift index 17b142d113..959034ff4f 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/AXHover.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/AXHover.swift @@ -2,13 +2,6 @@ import ApplicationServices final class AXHover { - /// Only diagnostic clients without a connection policy may opt in per request. - static func requireSystemPointer(connectionSetting: Bool?, requestSetting: Bool?) throws { - guard connectionSetting ?? requestSetting ?? false else { - throw NativeFailure(code: "HOVER_UNAVAILABLE", message: "No enabled element at this target or in its ancestors offers both AXShowAlternateUI and AXShowDefaultUI. System-pointer hover is disabled; this provider has no other pointer-free hover route. No input was sent and the pointer did not move. Use click on an available control or performAction with an action from listActions.") - } - } - private var revealed: (element: AXUIElement, windowID: UInt32)? /// Entering revealed descendants preserves their parent UI; leaving restores it once. diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift index c4cc6066c8..c44b4078af 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift @@ -78,7 +78,6 @@ actor AXWorker { private let axTimeout: Float private let textEventInterval: Double private let maxClickHoldMs: Int - private let systemPointerHover: Bool? private let sessionState: @Sendable () -> Bool? private var access: AXAccess? private var keyboard: KeyboardInput? @@ -86,13 +85,12 @@ actor AXWorker { private var hovers: [String: AXHover] = [:] private var closed = false - init(cancellation: CancellationLatch, axTimeout: Float, textEventInterval: Double, maxClickHoldMs: Int, systemPointerHover: Bool? = nil, + init(cancellation: CancellationLatch, axTimeout: Float, textEventInterval: Double, maxClickHoldMs: Int, sessionState: @escaping @Sendable () -> Bool? = NativePlatform.sessionLocked) { self.cancellation = cancellation self.axTimeout = axTimeout self.textEventInterval = textEventInterval self.maxClickHoldMs = maxClickHoldMs - self.systemPointerHover = systemPointerHover self.sessionState = sessionState } private func identifier(_ kind: String) throws -> String { @@ -106,13 +104,13 @@ actor AXWorker { access = created return created } - private func moveUsingAX(_ element: AXUIElement?, windowID: UInt32, handle: String, params: RequestParameters) throws -> Bool { - let requestSetting = try params.optionalBoolean("allowSystemPointer") + private func moveUsingAX(_ element: AXUIElement?, windowID: UInt32, handle: String) throws -> DriverOutcome { let hover = hovers[handle] ?? AXHover() hovers[handle] = hover - if try hover.move(to: element, in: windowID, access: ax()) { return true } - try AXHover.requireSystemPointer(connectionSetting: systemPointerHover, requestSetting: requestSetting) - return false + guard try hover.move(to: element, in: windowID, access: ax()) else { + throw NativeFailure(code: "HOVER_UNAVAILABLE", message: "No enabled element at this target or in its ancestors offers both AXShowAlternateUI and AXShowDefaultUI. No input was sent and the pointer did not move. Use click on an available control or performAction with an action from listActions.") + } + return .result(Self.mouseMoveResult(route: "accessibility", pointerMoved: false)) } private func inputs() -> (KeyboardInput, WindowPointerInput) { if let keyboard, let pointer { return (keyboard, pointer) } @@ -351,6 +349,7 @@ actor AXWorker { try ax().selectText(element(handle, params), text: params.string("text"), prefix: params.optionalString("prefix"), suffix: params.optionalString("suffix"), mode: params.optionalString("selectionType") ?? "text") case "click", "mouseMove": + let useSystemPointer = try params.optionalBoolean("useSystemPointer") ?? false let holdMs = params.contains("holdMs") ? try params.integer("holdMs", in: 0...maxClickHoldMs) : nil let button = try params.optionalString("mouseButton") ?? "left" let count = params.contains("clickCount") ? try params.integer("clickCount", in: 1...2) : 1 @@ -370,9 +369,7 @@ actor AXWorker { } if method == "mouseMove" { _ = try NativePlatform.ownedWindow(identity.pid, enclosing.id) - if try moveUsingAX(selected, windowID: enclosing.id, handle: handle, params: params) { - return .result(Self.mouseMoveResult(route: "accessibility", pointerMoved: false)) - } + if !useSystemPointer { return try moveUsingAX(selected, windowID: enclosing.id, handle: handle) } } else { try ax().requireEnabled(selected) } @@ -389,12 +386,14 @@ actor AXWorker { let route = WindowRoute(owner: identity, windowID: window.windowID, bounds: window.bounds) if method == "mouseMove" { try WindowPointerInput.validate(point, in: route) - if elementWindow == nil { + if !useSystemPointer { var hit = try ax().element(at: CGPoint(x: window.bounds.x + point.x, y: window.bounds.y + point.y), owner: identity) if let target = hit, try ax().windowID(target) != window.windowID { hit = nil } - if try moveUsingAX(hit, windowID: window.windowID, handle: handle, params: params) { - return .result(Self.mouseMoveResult(route: "accessibility", pointerMoved: false)) - } + return try moveUsingAX(hit, windowID: window.windowID, handle: handle) + } + if let hover = hovers[handle] { + _ = try hover.move(to: nil, in: window.windowID, access: ax()) + hovers.removeValue(forKey: handle) } return .mouseMove(route, point) } diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift index 0da4e5cb96..a39965362b 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift @@ -14,7 +14,7 @@ enum NativeProtocol { "listActions": ["appHandle", "elementId", "snapshotId", "windowId"], "listApps": [], "listWindows": ["appHandle"], - "mouseMove": ["allowSystemPointer", "appHandle", "elementId", "snapshotId", "windowId", "x", "y"], + "mouseMove": ["appHandle", "elementId", "snapshotId", "useSystemPointer", "windowId", "x", "y"], "openPermissionSettings": ["permission", "settingsMessage"], "paste": ["appHandle", "format", "maxClipboardBytes", "maxDepth", "maxElements", "pasteReadTimeoutMs", "text", "windowId"], "performAction": ["action", "appHandle", "argument", "elementId", "snapshotId", "windowId"], diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift index 000a69e848..3a778cb8ae 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift @@ -42,8 +42,7 @@ import Darwin self.onClose = onClose self.onFatal = onFatal worker = AXWorker(cancellation: cancellation, axTimeout: configuration.axTimeoutSeconds, - textEventInterval: configuration.textEventIntervalSeconds, maxClickHoldMs: configuration.maxClickHoldMs, - systemPointerHover: configuration.systemPointerHover) + textEventInterval: configuration.textEventIntervalSeconds, maxClickHoldMs: configuration.maxClickHoldMs) input = JSONLInput(limit: configuration.maxRequestBytes, cancellation: cancellation, channel: channel) self.writer = writer } diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/Runtime.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/Runtime.swift index c31a07ebef..d61f0803ab 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/Runtime.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/Runtime.swift @@ -21,7 +21,6 @@ struct HelperConfiguration: Sendable { var shutdownGraceSeconds: Double = 3 var textEventIntervalSeconds: Double = 0.00025 var maxClickHoldMs = 5000 - var systemPointerHover: Bool? var timeoutSeconds: Double = 30 var screenshotTimeoutSeconds: Double = 5 init(arguments: [String]) throws { @@ -65,7 +64,7 @@ struct HelperConfiguration: Sendable { guard let fields = decoded.object, Set(fields.keys) == ["type", "protocol", "runtimeKey", "config"], fields["type"] == .string("hello"), fields["protocol"] == .number(1), fields["runtimeKey"] == .string(runtimeKey), let config = fields["config"]?.object, - Set(config.keys).subtracting(["systemPointerHover"]) == ["maxRequestBytes", "maxResponseBytes", "axTimeoutMs", "textEventIntervalMs", "maxClickHoldMs", "screenshotTimeoutMs", "timeoutMs", "graceMs"] else { + Set(config.keys) == ["maxRequestBytes", "maxResponseBytes", "axTimeoutMs", "textEventIntervalMs", "maxClickHoldMs", "screenshotTimeoutMs", "timeoutMs", "graceMs"] else { throw NativeFailure(code: "NATIVE_PROTOCOL", message: "Expected shared helper protocol 1 hello for this runtime") } func number(_ name: String) throws -> String { @@ -83,10 +82,6 @@ struct HelperConfiguration: Sendable { "--screenshot-timeout-ms", number("screenshotTimeoutMs"), "--timeout-ms", number("timeoutMs"), "--shutdown-grace-ms", number("graceMs"), ]) - if let setting = config["systemPointerHover"] { - guard case .bool(let allowed) = setting else { throw RequestParameters.invalid("systemPointerHover must be a boolean") } - systemPointerHover = allowed - } } } diff --git a/packages/experimental/cua-macos/native/generate-protocol.ts b/packages/experimental/cua-macos/native/generate-protocol.ts index 398999cc2c..c643dfb830 100644 --- a/packages/experimental/cua-macos/native/generate-protocol.ts +++ b/packages/experimental/cua-macos/native/generate-protocol.ts @@ -10,7 +10,7 @@ import { PermissionGuideParameters, PermissionSettingsParameters } from '../src/ */ export function renderNativeProtocol(): { json: string; swift: string } { const { fields, enums, json } = nativeProtocol({ - mouseMove: z.object({ allowSystemPointer: z.boolean().optional() }), + mouseMove: z.object({ useSystemPointer: z.boolean().optional() }), requestPermission: PermissionGuideParameters, openPermissionSettings: PermissionSettingsParameters, }) const quoted = (values: Iterable) => [...values].sort().map(value => JSON.stringify(value)).join(', ') diff --git a/packages/experimental/cua-macos/native/protocol.schema.json b/packages/experimental/cua-macos/native/protocol.schema.json index b0d74a84dc..818394ad1b 100644 --- a/packages/experimental/cua-macos/native/protocol.schema.json +++ b/packages/experimental/cua-macos/native/protocol.schema.json @@ -489,7 +489,7 @@ "minLength": 1, "description": "Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot." }, - "allowSystemPointer": { + "useSystemPointer": { "type": "boolean" } }, @@ -524,7 +524,7 @@ "type": "number", "minimum": 0 }, - "allowSystemPointer": { + "useSystemPointer": { "type": "boolean" } }, diff --git a/packages/experimental/cua-macos/native/tests/e2e.py b/packages/experimental/cua-macos/native/tests/e2e.py index 2a16b54b49..c5b385189a 100644 --- a/packages/experimental/cua-macos/native/tests/e2e.py +++ b/packages/experimental/cua-macos/native/tests/e2e.py @@ -236,7 +236,7 @@ def main(): assert (after["cursorChanges"], after["hoverVisible"]) == (before["cursorChanges"], before["hoverVisible"]), after def mouse_move(target): - result = call("mouseMove", target if args.alternate_hover else {**target, "allowSystemPointer": True}) + result = call("mouseMove", target if args.alternate_hover else {**target, "useSystemPointer": True}) if not args.alternate_hover: assert result["hoverRoute"] == "system-pointer" and result["systemPointerMoved"] is True, result if args.alternate_hover: @@ -271,7 +271,7 @@ def main(): mouse_move(target) wait_state(lambda actual: actual["hoverVisible"], "Input-transparent overlay blocked hover") before = cycle_overlay(2) - call("mouseMove", {**target, "allowSystemPointer": True}, "FOREGROUND_REQUIRED") + call("mouseMove", {**target, "useSystemPointer": True}, "FOREGROUND_REQUIRED") after = wait_state(lambda actual: actual["cursorSamples"] >= before["cursorSamples"] + 2, "Pointer observation did not advance") assert after["cursorChanges"] == before["cursorChanges"], "Rejected hover moved the system pointer" diff --git a/packages/experimental/cua-macos/native/tests/electron_e2e.py b/packages/experimental/cua-macos/native/tests/electron_e2e.py index 403bb21eba..d8acffbe9d 100644 --- a/packages/experimental/cua-macos/native/tests/electron_e2e.py +++ b/packages/experimental/cua-macos/native/tests/electron_e2e.py @@ -136,7 +136,7 @@ def main(): call('mouseMove', {**owner, 'snapshotId': state['snapshotId'], 'elementId': trigger['elementId'], 'x': 0}, 'INVALID_ARGUMENT') hover_target = {**owner, 'snapshotId': state['snapshotId'], 'elementId': trigger['elementId']} call('mouseMove', hover_target, 'HOVER_UNAVAILABLE') - call('mouseMove', {**hover_target, 'allowSystemPointer': True}) + call('mouseMove', {**hover_target, 'useSystemPointer': True}) actual = wait_state(lambda actual: actual['popupOpen'], 'Hover did not reveal delayed information') assert actual['pointerDowns'] == baseline and actual['lastMove'] == {'buttons': 0, 'trusted': True}, actual state = observe() @@ -151,13 +151,13 @@ def main(): target = {**owner, 'windowId': window_id, 'x': bounds[0] + bounds[2] / 2 - window['bounds']['x'], 'y': bounds[1] + bounds[3] / 2 - window['bounds']['y']} - call('mouseMove', {**target, 'allowSystemPointer': True}) + call('mouseMove', {**target, 'useSystemPointer': True}) state = observe() assert 'Hover information' in state['tree'], 'Moving into the popup dismissed it' call('click', target) actual = wait_state(lambda actual: actual['popupClicks'] == expected_clicks, 'Popup action did not receive its click') assert actual['popupOpen'] and actual['pointerDowns'] == baseline + 1 and actual['popupClickTrusted'], actual - call('mouseMove', {**owner, 'windowId': window_id, 'x': 10, 'y': 40, 'allowSystemPointer': True}) + call('mouseMove', {**owner, 'windowId': window_id, 'x': 10, 'y': 40, 'useSystemPointer': True}) wait_state(lambda actual: not actual['popupOpen'], 'Leaving the hover region did not close it') hover_sequence() helper.eof() diff --git a/packages/experimental/cua-macos/native/tests/swift/SharedHelperTests.swift b/packages/experimental/cua-macos/native/tests/swift/SharedHelperTests.swift index 52f7f0f224..c0cd744ec4 100644 --- a/packages/experimental/cua-macos/native/tests/swift/SharedHelperTests.swift +++ b/packages/experimental/cua-macos/native/tests/swift/SharedHelperTests.swift @@ -33,14 +33,6 @@ private func hello(overrides: [String: JSONValue] = [:]) throws -> Data { } } -@Test func readsSystemPointerHoverFromEachConnection() throws { - #expect(try HelperConfiguration(hello: hello(), runtimeKey: runtimeKey).systemPointerHover == nil) - for allowed in [true, false] { - let configuration = try HelperConfiguration(hello: hello(overrides: ["systemPointerHover": .bool(allowed)]), runtimeKey: runtimeKey) - #expect(configuration.systemPointerHover == allowed) - } -} - @Test func resolvesClickHoldLimitFromHelperArguments() throws { #expect(try HelperConfiguration(arguments: []).maxClickHoldMs == 5000) for limit in [0, 7000] { @@ -56,7 +48,7 @@ private func hello(overrides: [String: JSONValue] = [:]) throws -> Data { ("axTimeoutMs", .number(0)), ("textEventIntervalMs", .number(-1)), ("screenshotTimeoutMs", .number(0)), ("timeoutMs", .string("8000")), ("graceMs", .number(-1)), ("maxClickHoldMs", .number(-1)), ("maxClickHoldMs", .number(0.5)), - ("maxClickHoldMs", .number(2147483648)), ("unexpected", .bool(true)), ("systemPointerHover", .string("true")), + ("maxClickHoldMs", .number(2147483648)), ("unexpected", .bool(true)), ("systemPointerHover", .bool(true)), ]) func refusesInvalidSharedConfiguration(_ name: String, _ value: JSONValue) throws { #expect(throws: NativeFailure.self) { try HelperConfiguration(hello: hello(overrides: [name: value]), runtimeKey: runtimeKey) } diff --git a/packages/experimental/cua-macos/native/tests/swift/SystemHoverTests.swift b/packages/experimental/cua-macos/native/tests/swift/SystemHoverTests.swift index df48871c4c..38f82ac4a8 100644 --- a/packages/experimental/cua-macos/native/tests/swift/SystemHoverTests.swift +++ b/packages/experimental/cua-macos/native/tests/swift/SystemHoverTests.swift @@ -3,20 +3,6 @@ import Testing import CoreGraphics @testable import CuaNative -@Test func unavailableAXHoverHonorsConnectionPolicyBeforeRequestFlags() throws { - for (connection, request): (Bool?, Bool?) in [(nil, nil), (nil, false), (false, nil), (false, true)] { - do { - try AXHover.requireSystemPointer(connectionSetting: connection, requestSetting: request) - Issue.record("An unapproved request was allowed to move the pointer") - } catch let failure as NativeFailure { - #expect(failure.code == "HOVER_UNAVAILABLE") - #expect(failure.message.contains("No input was sent")) - } - } - try AXHover.requireSystemPointer(connectionSetting: true, requestSetting: nil) - try AXHover.requireSystemPointer(connectionSetting: nil, requestSetting: true) -} - @Test func systemHoverRequiresTheForegroundUnobscuredTarget() throws { let owner = ProcessIdentity(pid: 42, startedSeconds: 1, startedMicroseconds: 0) let bounds = WindowBounds(x: 100, y: 100, width: 400, height: 300) diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index f8779651a9..5a340f045f 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -5,7 +5,7 @@ import { COMMON_GUIDANCE } from '@deepseek-ai/dsh-cua/guidance' export const PLATFORM_GUIDANCE = 'macOS: accessibility tree operations are available at app or window level. Simulated keyboard input is available at app level. Simulated mouse input is available at window level. For pointer input on a sheet, select its window ID with app.getWindow(id). For menu items outside a window, use app.invoke(id). findApps searches /Applications, /System/Applications, and ~/Applications. Use the Computer Use permission page to grant the helper application Accessibility and Screen Recording permissions in System Settings. Native window and AX bounds use macOS points, while delivered screenshots carry their own pixel dimensions. show does not switch Spaces, and Stage Manager can present reduced window geometry; explain any required foreground step before calling app.activate() or performing an advertised AXRaise action.' const methodDetails = new Map([ - ['mouseMove', 'Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only systemPointerHover: true permits moving the visible cursor. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured.'], + ['mouseMove', 'Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, mouseMove uses the system pointer for this session until DSH restarts. If the user declines, continue without mouse control. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured.'], ['click', 'click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying.'], ]) diff --git a/packages/experimental/cua-macos/src/index.ts b/packages/experimental/cua-macos/src/index.ts index 4b07e9c8b7..dff6dd6997 100644 --- a/packages/experimental/cua-macos/src/index.ts +++ b/packages/experimental/cua-macos/src/index.ts @@ -4,7 +4,6 @@ import { fileURLToPath } from 'node:url' import type { Context } from '@deepseek-ai/cordis' import Schema from '@deepseek-ai/schemastery' import { installCua, type CuaConfig, type CuaStartupCapabilities } from '@deepseek-ai/dsh-cua' -import type { NativeChannelFactory } from '@deepseek-ai/dsh-cua/native-client' import { CuaConfigFields } from '@deepseek-ai/dsh-cua/config' import { createPermissionParameters } from './permission-guide.ts' import { nativeHostArchitecture, prepareNativeExecutable, type SourceSigning } from './native-build.ts' @@ -16,7 +15,7 @@ const CAPABILITIES: CuaStartupCapabilities = { platform: 'macos', methods: [ 'activate', 'click', 'drag', 'findApps', 'getApp', 'getCapabilities', 'getState', 'invoke', - 'listActions', 'listApps', 'listWindows', 'mouseMove', 'paste', 'performAction', 'pressKey', 'screenshot', 'scroll', + 'listActions', 'listApps', 'listWindows', 'mouseMove', 'paste', 'performAction', 'pressKey', 'requestMouseControl', 'screenshot', 'scroll', 'selectText', 'setValue', 'show', 'typeText', ], appLevelAccessibility: true, @@ -33,8 +32,6 @@ export const inject = ['computerUse', 'tools', 'systemPrompt', 'subprocess', 'ag export interface Config extends CuaConfig { /** Interval in milliseconds between permission and position checks while the native guide is open. */ permissionPollIntervalMs: number - /** Allow mouseMove to move the visible system pointer when AX hover is unavailable. Off by default to avoid interrupting the user. */ - systemPointerHover: boolean /** * Source-build signing: `ad-hoc` (default) binds grants to the code hash; * an object selects an existing certificate in an explicitly configured keychain. @@ -45,7 +42,6 @@ export interface Config extends CuaConfig { /** Validate shared operation bounds and macOS source signing before preparing the helper. */ export const Config = Schema.object({ ...CuaConfigFields, - systemPointerHover: Schema.boolean().default(false), permissionPollIntervalMs: Schema.number().step(1).min(100).max(2 ** 31 - 1).default(1000), sourceSigning: Schema.union([ Schema.const('ad-hoc'), @@ -64,8 +60,7 @@ export const Config = Schema.object({ * @returns Completion after native preparation and tool registration. */ export async function apply(ctx: Context, config: Config): Promise { - const createChannel: NativeChannelFactory = (owner, native) => createSharedNativeChannel(owner, native, config.systemPointerHover) - const backend = { createChannel, guidance: GUIDANCE, capabilities: CAPABILITIES, + const backend = { createChannel: createSharedNativeChannel, guidance: GUIDANCE, capabilities: CAPABILITIES, permissionParameters: createPermissionParameters(config.permissionPollIntervalMs) } if (config.helperPath !== undefined) { if (!isAbsolute(config.helperPath)) throw new Error('dsh-cua helperPath must be absolute') diff --git a/packages/experimental/cua-macos/src/shared-native.ts b/packages/experimental/cua-macos/src/shared-native.ts index 7be638a96b..e76e1f0440 100644 --- a/packages/experimental/cua-macos/src/shared-native.ts +++ b/packages/experimental/cua-macos/src/shared-native.ts @@ -56,8 +56,7 @@ class SharedNativeChannel implements NativeChannel { private cleanupTimer: ReturnType | undefined private cleanupFailure: Error | undefined - constructor(private readonly owner: { subprocess: Pick }, private readonly config: NativeClientConfig, - private readonly systemPointerHover: boolean) { + constructor(private readonly owner: { subprocess: Pick }, private readonly config: NativeClientConfig) { this.stdin = new Writable({ write: (chunk: Buffer, encoding, callback) => { void this.ready.promise.then((socket) => { @@ -216,7 +215,7 @@ class SharedNativeChannel implements NativeChannel { screenshotTimeoutMs, timeoutMs, graceMs } = this.config socket.write(JSON.stringify({ type: 'hello', protocol: 1, runtimeKey, config: { maxRequestBytes, maxResponseBytes, axTimeoutMs, textEventIntervalMs, maxClickHoldMs, - screenshotTimeoutMs, timeoutMs, graceMs, systemPointerHover: this.systemPointerHover } }) + '\n') + screenshotTimeoutMs, timeoutMs, graceMs } }) + '\n') }) const receive = (chunk: Buffer): void => { const newline = chunk.indexOf(10) @@ -288,9 +287,8 @@ class SharedNativeChannel implements NativeChannel { * Prepare or join the native process; writes wait for identity-checked readiness. * @param owner - Provider owning the short-lived native startup command. * @param config - Native runtime identity, operation limits and connection deadlines. - * @param systemPointerHover - Whether this connection may move the visible pointer when AX hover is unavailable. * @returns One independently disposable Agent connection, without operation replay. */ -export function createSharedNativeChannel(owner: { subprocess: Pick }, config: NativeClientConfig, systemPointerHover: boolean): NativeChannel { - return new SharedNativeChannel(owner, config, systemPointerHover) +export function createSharedNativeChannel(owner: { subprocess: Pick }, config: NativeClientConfig): NativeChannel { + return new SharedNativeChannel(owner, config) } diff --git a/packages/experimental/cua-macos/tests/guidance.spec.ts b/packages/experimental/cua-macos/tests/guidance.spec.ts index c4a6aeec7d..2db5c24eb6 100644 --- a/packages/experimental/cua-macos/tests/guidance.spec.ts +++ b/packages/experimental/cua-macos/tests/guidance.spec.ts @@ -4,7 +4,7 @@ import type { CuaApi, CuaApp, CuaSurface, CuaWindow } from '@deepseek-ai/dsh-cua import { COMMON_GUIDANCE } from '@deepseek-ai/dsh-cua/guidance' import { GUIDANCE, PLATFORM_GUIDANCE } from '../src/guidance.ts' -const apiNames: Array = ['getCapabilities', 'listApps', 'findApps', 'getState', 'getApp'] +const apiNames: Array = ['getCapabilities', 'requestMouseControl', 'listApps', 'findApps', 'getState', 'getApp'] const accessibilityNames: Array> = [ 'getAccessibilityTree', 'listActions', 'invoke', 'performAction', 'setValue', 'selectText', ] @@ -28,12 +28,12 @@ it('states the macOS accessibility, keyboard, and mouse levels', () => { expect(PLATFORM_GUIDANCE).toContain('Simulated keyboard input is available at app level') expect(PLATFORM_GUIDANCE).toContain('Simulated mouse input is available at window level') const hover = GUIDANCE.split('\n').find(line => line.startsWith('- mouseMove(')) - for (const term of ['AXShowAlternateUI', 'AXShowDefaultUI', 'HOVER_UNAVAILABLE', 'systemPointerHover']) expect(hover).toContain(term) + for (const term of ['AXShowAlternateUI', 'AXShowDefaultUI', 'HOVER_UNAVAILABLE', 'The question interrupts the user']) expect(hover).toContain(term) const click = GUIDANCE.split('\n').find(line => line.startsWith('- click(')) expect(click).toContain('await app.getWindow(windowId), then call win.click(target)') expect(click).toContain('use app.invoke(id)') for (const step of ['call app.activate() first', 'outside the window', 'screenshot pixels', 'before typing or retrying']) expect(click).toContain(step) - for (const platformDetail of ['macOS', 'AXShowAlternateUI', 'AXRaise', 'HOVER_UNAVAILABLE', 'systemPointerHover', 'mouse-down']) { + for (const platformDetail of ['macOS', 'AXShowAlternateUI', 'AXRaise', 'HOVER_UNAVAILABLE', 'The question interrupts the user', 'mouse-down']) { expect(COMMON_GUIDANCE).not.toContain(platformDetail) } }) @@ -50,8 +50,8 @@ it('enumerates every available method once in its catalog entry', () => { for (const catalog of [COMMON_GUIDANCE, GUIDANCE]) { for (const signature of signatures) expect(catalog.split('\n').filter(line => line.startsWith(`- ${signature}`))).toHaveLength(1) } - expect(signatures).toHaveLength(24) - expect(new Set(signatures.map(signature => signature.slice(signature.indexOf('.') + 1, -1))).size).toBe(23) + expect(signatures).toHaveLength(25) + expect(new Set(signatures.map(signature => signature.slice(signature.indexOf('.') + 1, -1))).size).toBe(24) expect(COMMON_GUIDANCE).toContain('app.appHandle') expect(COMMON_GUIDANCE).toContain('window.windowId') }) diff --git a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts index aefc6fbee6..87afd7eed9 100644 --- a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts +++ b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts @@ -26,7 +26,7 @@ describe('native protocol generation', () => { { method: 'drag', params: { appHandle: 'app', windowId: 1, fromX: 1, fromY: 1, toX: 2, toY: 2, steps: 1 } }, { method: 'paste', params: { appHandle: 'app', text: 'text', maxClipboardBytes: 1024 } }, { method: 'paste', params: { appHandle: 'app', text: 'text', pasteReadTimeoutMs: 1000 } }, - { method: 'mouseMove', params: { appHandle: 'app', windowId: 1, x: 0, y: 0, allowSystemPointer: true } }, + { method: 'mouseMove', params: { appHandle: 'app', windowId: 1, x: 0, y: 0, useSystemPointer: true } }, { method: 'findApps', params: { query: 'Editor', maxAppMatches: 1 } }, { method: 'launchApp', params: { appHandle: 'pending' } }, { method: 'applicationIcons', params: { paths: ['/Applications/Fixture.app'] } }, diff --git a/packages/experimental/cua-macos/tests/provider-channel.spec.ts b/packages/experimental/cua-macos/tests/provider-channel.spec.ts index 02d9fc0fe6..1b9ac97c44 100644 --- a/packages/experimental/cua-macos/tests/provider-channel.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-channel.spec.ts @@ -14,10 +14,10 @@ vi.mock('@deepseek-ai/dsh-cua', async importOriginal => ({ })) vi.mock('../src/shared-native.ts', () => ({ createSharedNativeChannel: vi.fn() })) -it.each([false, true])('forwards systemPointerHover=%s and native limits to each channel', async (systemPointerHover) => { +it('forwards native limits to each channel', async () => { const ctx = new Context() onTestFinished(() => ctx.fiber.dispose()) - const config = Config({ helperPath: resolve('fixture-helper'), systemPointerHover }) + const config = Config({ helperPath: resolve('fixture-helper') }) const channel: NativeChannel = { stdin: new PassThrough(), stdout: new PassThrough(), done: Promise.resolve(), close() {}, waitForClose: async () => {}, @@ -32,5 +32,5 @@ it.each([false, true])('forwards systemPointerHover=%s and native limits to each const owner = { subprocess: { spawn: vi.fn() } } const native = { ...config, helperPath: resolve('fixture-helper'), cwd: process.cwd() } expect(backend.createChannel(owner, native)).toBe(channel) - expect(createSharedNativeChannel).toHaveBeenCalledExactlyOnceWith(owner, native, systemPointerHover) + expect(createSharedNativeChannel).toHaveBeenCalledExactlyOnceWith(owner, native) }) diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index 48b9eb99e3..f975862baa 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -21,7 +21,7 @@ import { CuaConfig } from '@deepseek-ai/dsh-cua' import { createConfigProjector } from '@deepseek-ai/dsh-app-boot/src/config-schema/projector.ts' import LocalAttachmentStore from '@deepseek-ai/dsh-attachment-local' import FileSystem from '@deepseek-ai/dsh-fs-local' -import UserQuestions from '@deepseek-ai/dsh-user-questions' +import UserQuestions, { UserQuestionError } from '@deepseek-ai/dsh-user-questions' import NodeRuntime from '@deepseek-ai/dsh-ptc-runtime-node' import Sandbox from '@deepseek-ai/dsh-sandbox-local' import SandboxPolicy from '@deepseek-ai/dsh-sandbox-policy' @@ -61,7 +61,7 @@ vi.mock('@deepseek-ai/dsh-cua/native-client', async importOriginal => ({ })) const config = { - permissionPollIntervalMs: 1000, systemPointerHover: false, + permissionPollIntervalMs: 1000, agentPreset: false, helperPath: resolve('fixture-helper'), sourceSigning: 'ad-hoc', buildTimeoutMs: 300_000, timeoutMs: 10_000, connectTimeoutMs: 5000, screenshotTimeoutMs: 5000, axTimeoutMs: 3000, maxRequestBytes: 1024, maxResponseBytes: 4096, maxClipboardBytes: 8192, pasteReadTimeoutMs: 1000, maxAppMatches: 20, graceMs: 10, textEventIntervalMs: 8, maxClickHoldMs: 7000, maxElements: 64, maxDepth: 8, dragDurationMs: 200, dragSteps: 10, @@ -71,7 +71,6 @@ const { helperPath: _helperPath, ...automaticConfig } = config it('defaults to ad-hoc signing and requires a keychain path for certificate signing', () => { expect(NativeProvider.Config({}).sourceSigning).toBe('ad-hoc') - expect(NativeProvider.Config({}).systemPointerHover).toBe(false) expect(NativeProvider.Config({ sourceSigning: 'ad-hoc' }).sourceSigning).toBe('ad-hoc') for (const sourceSigning of [{ identity: 'Apple Development: Example', keychain: '/keychains/custom.keychain-db' }, { identity: 'ABC123', keychain: '/keychains/custom.keychain-db' }]) { expect(NativeProvider.Config({ sourceSigning }).sourceSigning).toEqual(sourceSigning) @@ -199,7 +198,7 @@ it('merges startup capabilities into runtime native fields', async () => { expect(result.value).toEqual({ ...runtime, platform: 'macos', appLevelAccessibility: true, keyboard: ['app'], browser: false, coordinateSpace: 'screenshot-pixels', methods: [ 'activate', 'click', 'drag', 'findApps', 'getApp', 'getCapabilities', 'getState', 'invoke', - 'listActions', 'listApps', 'listWindows', 'mouseMove', 'paste', 'performAction', 'pressKey', 'screenshot', 'scroll', + 'listActions', 'listApps', 'listWindows', 'mouseMove', 'paste', 'performAction', 'pressKey', 'requestMouseControl', 'screenshot', 'scroll', 'selectText', 'setValue', 'show', 'typeText', ] }) expect(native.request).toHaveBeenCalledExactlyOnceWith('getCapabilities', {}, expect.any(AbortSignal), ToolCallId('lifecycle-1')) @@ -878,3 +877,69 @@ it('routes permission actions through the macOS provider with the Client languag permission: 'screenRecording', settingsMessage: 'Open System Settings > Privacy & Security > Screen Recording manually.', }, expect.any(AbortSignal)) }) + +it('moves only for the consenting session, without granting application access or retaining consent after reload', async () => { + const f = await setup() + await f.ctx.plugin(UserQuestions) + const ask = vi.fn((id: string) => ({ answers: [{ id, selected: ['Allow for this session'] }] })) + f.ctx.on('user-questions/request', async ({ questions }) => ask(questions[0]!.id)) + const target = { appHandle: 'app-one', snapshotId: 'snapshot', elementId: '0' } + native.request.mockResolvedValueOnce({ appHandle: 'app-one', name: 'Fixture', running: true, + authorization: { id: 'a'.repeat(64), name: 'Fixture', persistable: false } }) + expect((await f.execute('getApp', { selector: 'Fixture' })).isError).toBe(false) + const move = async (useSystemPointer: boolean) => { + expect((await f.execute('mouseMove', target)).isError).toBe(false) + expect(native.request).toHaveBeenLastCalledWith('mouseMove', { ...target, useSystemPointer }, expect.any(AbortSignal), expect.any(String)) + } + await move(false) + expect((await f.execute('mouseMove', { ...target, useSystemPointer: true })).isError).toBe(true) + for (let i = 0; i < 2; i++) expect((await f.execute('requestMouseControl')).value).toBe(true) + await move(true) + const other = await f.ctx.agents.create({ sessionId: SessionId('other-mouse-session') }) + const otherCall = (method: string, params: Record) => executeToolFixture(f.ctx, { + agent: other.agent, signal: new AbortController().signal, callId: ToolCallId(method), name: 'cua', arguments: { method, params }, + }) + expect((await otherCall('requestMouseControl', {})).value).toBe(true) + expect((await otherCall('mouseMove', target)).isError).toBe(true) + await f.provider.dispose() + await f.ctx.plugin(NativeProvider, config) + expect((await f.execute('requestMouseControl')).value).toBe(true) + expect(ask.mock.calls.map(([id]) => id)).toEqual(['cua-app-access', 'cua-mouse-control', 'cua-mouse-control', 'cua-mouse-control']) +}) + +it.each(['Deny', 'custom', 'cancel'] as const)('remembers a %s answer without moving the pointer or asking again', async (answer) => { + const f = await setup() + await f.ctx.plugin(UserQuestions) + const ask = vi.fn(() => { + if (answer === 'cancel') throw new UserQuestionError('User dismissed the request', 'ASK_CANCELLED') + return { answers: [{ id: 'cua-mouse-control', selected: [answer === 'custom' ? 'Allow for this session' : answer], + ...(answer === 'custom' ? { custom: 'maybe later' } : {}) }] } + }) + f.ctx.on('user-questions/request', async () => ask()) + expect((await f.execute('requestMouseControl')).value).toBe(false) + expect((await f.execute('requestMouseControl')).value).toBe(false) + expect(ask).toHaveBeenCalledTimes(1) + expect(native.request).not.toHaveBeenCalled() +}) + +it('drops late approval after cancellation and admits only one pending question', async () => { + const f = await setup() + expect((await f.execute('requestMouseControl')).isError).toBe(true) + await f.ctx.plugin(UserQuestions) + const entered = Promise.withResolvers() + const release = Promise.withResolvers() + f.ctx.on('user-questions/request', async () => { + entered.resolve(undefined) + await release.promise + return { answers: [{ id: 'cua-mouse-control', selected: ['Allow for this session'] }] } + }) + const controller = new AbortController() + const first = f.execute('requestMouseControl', {}, controller.signal) + await entered.promise + expect((await f.execute('requestMouseControl')).isError).toBe(true) + controller.abort() + release.resolve(undefined) + await Promise.allSettled([first]) + expect((await f.execute('requestMouseControl')).value).toBe(true) + expect(native.request).not.toHaveBeenCalled() +}) diff --git a/packages/experimental/cua-macos/tests/shared-native.spec.ts b/packages/experimental/cua-macos/tests/shared-native.spec.ts index 871475fb53..86c5437ccf 100644 --- a/packages/experimental/cua-macos/tests/shared-native.spec.ts +++ b/packages/experimental/cua-macos/tests/shared-native.spec.ts @@ -133,8 +133,8 @@ async function fixture(options: { timeoutMs: 30_000, graceMs: options.graceMs ?? 1000, axTimeoutMs: 3000, maxRequestBytes: 65536, maxResponseBytes: 1048576, textEventIntervalMs: 0.5, maxClickHoldMs: 7000, } - function open(systemPointerHover = false): NativeChannel { - const channel = createSharedNativeChannel({ subprocess: { spawn } }, config, systemPointerHover) + function open(): NativeChannel { + const channel = createSharedNativeChannel({ subprocess: { spawn } }, config) channels.push(channel) return channel } @@ -172,7 +172,7 @@ describe.skipIf(process.platform === 'win32')('shared native Unix connection', ( it('keeps concurrent Agent scopes independent and leaves other sockets alive when one closes', async () => { const f = await fixture() const first = f.open() - const second = f.open(true) + const second = f.open() const firstReply = nextBytes(first) const secondReply = nextBytes(second) await Promise.all([write(first, 'one\n'), write(second, 'two\n')]) @@ -191,9 +191,7 @@ describe.skipIf(process.platform === 'win32')('shared native Unix connection', ( maxRequestBytes: 65536, maxResponseBytes: 1048576, textEventIntervalMs: 0.5, maxClickHoldMs: 7000, } expect(f.hellos).toHaveLength(2) - for (const systemPointerHover of [true, false]) { - expect(f.hellos).toContainEqual(expect.objectContaining({ type: 'hello', protocol: 1, config: { ...limits, systemPointerHover } })) - } + for (const hello of f.hellos) expect(hello).toMatchObject({ type: 'hello', protocol: 1, config: limits }) }) it('waits for both launcher outcome and managed range release before sending any bytes', async () => { diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index f4abb13c43..b72fd91584 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -15,8 +15,8 @@ en: 5db4d4fea25493b9 zh: 65ab64fc24eb0d09 /deepseek-ai-dsh-cua/use-this-package: - en: ba1f8e6ae51f69d7 - zh: 8f4fde9857bae319 + en: a55ba012aedca751 + zh: fcddea780c0ec58e /deepseek-ai-dsh-cua/use-this-package/permission-controls: en: d92d5948a7941e8d zh: d43491ae6521b87e diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index c95e4ae8ec..39a3e522d8 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -41,7 +41,7 @@ Observation methods emit and return their result. Use `void await app.getAccessi Application consent precedes launch, observation, capture, and input. Session access is stored by conversation and application identity and survives DSH restart and Computer Use disablement. Other conversations require their own consent. Saved grants require a platform-provided reusable application identity. Refusal and revocation close affected Agent connections, so their old objects require fresh selection. No input is replayed after cancellation, timeout, or reconnect. -Actions return a `dispatched` boolean; `mouseMove` also reports `hoverRoute` and `systemPointerMoved`, and `show()` adds `windowId` and `foregroundChanged`. Read fresh state to verify the result. Screenshots and pointer actions are available only on explicit window objects. Coordinate actions default to pixels in the latest returned screenshot; `relative: true` accepts integers from 0 to 999 after a successful window observation. Values are divided by 1000 and scaled to the window dimensions, then clamped inside the window. The Host resolves attachment normalization and the selected model route’s image projection before returning the screenshot, then converts input to the provider’s native window coordinates. Resizing invalidates the screenshot mapping; `SCREENSHOT_REQUIRED` requires a new screenshot for absolute coordinates. Resizing rejects old coordinates with `WINDOW_CHANGED`; movement alone preserves window-local coordinates. +Use `cua.requestMouseControl()` only when the task requires moving the user’s pointer. Approval applies to `mouseMove` in this session until DSH restarts. Actions return a `dispatched` boolean; `mouseMove` also reports `hoverRoute` and `systemPointerMoved`, and `show()` adds `windowId` and `foregroundChanged`. Read fresh state to verify the result. Screenshots and pointer actions are available only on explicit window objects. Coordinate actions default to pixels in the latest returned screenshot; `relative: true` accepts integers from 0 to 999 after a successful window observation. Values are divided by 1000 and scaled to the window dimensions, then clamped inside the window. The Host resolves attachment normalization and the selected model route’s image projection before returning the screenshot, then converts input to the provider’s native window coordinates. Resizing invalidates the screenshot mapping; `SCREENSHOT_REQUIRED` requires a new screenshot for absolute coordinates. Resizing rejects old coordinates with `WINDOW_CHANGED`; movement alone preserves window-local coordinates. `click(target, { holdMs: 800 })` holds the mouse button for 800 milliseconds before release. `holdMs` applies only to `click`; it accepts nonnegative integers and applies to each press of a double-click; zero releases without an additional hold. The provider config `maxClickHoldMs` limits explicit holds and defaults to 5000 ms. Explicit holds use pointer input; omission preserves ordinary click behavior. The operation deadline includes the hold, and cancellation or timeout releases the button. Received input cannot be undone; forced helper termination cannot guarantee release. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index c1c5ed863c..c2933dc0d1 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -41,7 +41,7 @@ kind: "package-reference" 应用授权发生在启动、观察、截图和输入之前。会话许可按对话和应用身份持久化，重启 DSH 或关闭 Computer Use 后仍可复用，其他对话需要单独授权。长期许可要求平台提供可复用的应用身份。拒绝或撤销会关闭受影响的 Agent 连接，因此旧对象必须重新选择目标。取消、超时或重新连接后均不重放输入。 -动作返回 `dispatched` 布尔值；`mouseMove` 还会返回 `hoverRoute` 和 `systemPointerMoved`，`show()` 还会返回 `windowId` 和 `foregroundChanged`。请读取新状态以验证结果。截图和指针操作只能通过显式窗口对象使用。坐标操作默认使用最近返回截图中的像素；成功观察窗口后，传入 `relative: true` 可使用 0～999 的整数坐标。数值除以 1000 后按窗口尺寸缩放，并限制在窗口内。Host 在返回截图前解析附件归一化和当前模型的图片缩放规则，再将输入换算为提供方的原生窗口坐标。窗口尺寸变化会使截图映射失效，绝对坐标遇到 `SCREENSHOT_REQUIRED` 时需要重新截图。窗口缩放后以 `WINDOW_CHANGED` 拒绝旧坐标；单纯移动窗口不改变窗口内坐标。 +只有任务确实需要移动用户的鼠标时，才调用 `cua.requestMouseControl()` 询问用户。授权仅适用于当前会话的 `mouseMove`，重启 DSH 后失效。动作返回 `dispatched` 布尔值；`mouseMove` 还会返回 `hoverRoute` 和 `systemPointerMoved`，`show()` 还会返回 `windowId` 和 `foregroundChanged`。请读取新状态以验证结果。截图和指针操作只能通过显式窗口对象使用。坐标操作默认使用最近返回截图中的像素；成功观察窗口后，传入 `relative: true` 可使用 0～999 的整数坐标。数值除以 1000 后按窗口尺寸缩放，并限制在窗口内。Host 在返回截图前解析附件归一化和当前模型的图片缩放规则，再将输入换算为提供方的原生窗口坐标。窗口尺寸变化会使截图映射失效，绝对坐标遇到 `SCREENSHOT_REQUIRED` 时需要重新截图。窗口缩放后以 `WINDOW_CHANGED` 拒绝旧坐标；单纯移动窗口不改变窗口内坐标。 `click(target, { holdMs: 800 })` 按住鼠标按钮 800 毫秒后松开。`holdMs` 仅适用于 `click`；它接受非负整数，双击时应用于每次按下；0 表示按下后立即松开，不额外等待。提供方配置 `maxClickHoldMs` 限制显式按住时长，默认 5000 毫秒。指定时长后使用鼠标事件；省略时保留普通点击行为。按住时间计入操作超时，取消或超时会松开按钮。已接收的输入无法撤销；辅助进程被强制终止时无法保证松开。 diff --git a/packages/experimental/cua/src/api.ts b/packages/experimental/cua/src/api.ts index 22039c6591..930db84195 100644 --- a/packages/experimental/cua/src/api.ts +++ b/packages/experimental/cua/src/api.ts @@ -1,6 +1,6 @@ /** Object facade over logged native operations; its caller owns evaluator persistence. */ import { Buffer } from 'node:buffer' -import type { z } from 'zod' +import { z } from 'zod' import type { JsonValue } from '@deepseek-ai/dsh-util-values' import { ActionNamesResult, ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from './protocol.ts' import type { CuaApi, CuaApp, CuaAppHandle, CuaCapabilities, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaOperation, CuaScreenshot, CuaSurface, CuaWindow } from './types.ts' @@ -51,6 +51,9 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): const objects = createCuaObjects(call, sink) const api: CuaApi = { + async requestMouseControl() { + return z.boolean().parse(await call('requestMouseControl', {})) + }, async getCapabilities(options) { const result = CapabilitiesResult.parse(await call('getCapabilities', {})) capabilities = result diff --git a/packages/experimental/cua/src/client/ApplicationApproval.tsx b/packages/experimental/cua/src/client/ApplicationApproval.tsx index a82a115fc4..f762d5d624 100644 --- a/packages/experimental/cua/src/client/ApplicationApproval.tsx +++ b/packages/experimental/cua/src/client/ApplicationApproval.tsx @@ -5,18 +5,18 @@ import type {} from '@deepseek-ai/dsh-client-ui-user-questions/client' import type {} from '@deepseek-ai/dsh-client-ui-approval/client' /** - * Select only the provider's fixed application question; other interactions keep their renderer. + * Select the provider’s application and mouse-control questions; other interactions keep their renderer. * @param pending The session's currently elected interaction. - * @returns Its application decision, or null for another interaction. + * @returns Its Computer Use decision, or null for another interaction. */ export function selectApplicationApproval(pending: ComposerChainProps['pendingInteraction']) { if (pending?.kind !== 'question' || pending.questions.length !== 1) return null const item = pending.questions[0] as (typeof pending.questions)[number] - if (item.id !== 'cua-app-access' || item.intent?.kind !== 'approval') return null + if ((item.id !== 'cua-app-access' && item.id !== 'cua-mouse-control') || item.intent?.kind !== 'approval') return null return { pending, item, intent: item.intent } } -/** Composer bindings for an application decision and caller-owned localized copy. */ +/** Composer bindings for a Computer Use decision and caller-owned localized copy. */ export type ApplicationApprovalProps = PropsRuntime<'conversation.composer'> & PropsRenderFactories & PropsLocale<'cuaPermissions'> & { matched: NonNullable> resolveText: (text: { en: string; [locale: string]: string }) => string @@ -32,7 +32,7 @@ export function ApplicationApproval({ matched: { pending, item, intent }, resolv const answer = (label: string) => pending.answer({ answers: [{ id: item.id, selected: [label] }] }) return renderFactorySlot('conversation.approval.card', { requestKey: pending.key, - warning: { label: t('approvalRisks'), title: t('approvalRiskTitle'), detail: t('approvalRiskNotice') }, + ...(item.id === 'cua-app-access' ? { warning: { label: t('approvalRisks'), title: t('approvalRiskTitle'), detail: t('approvalRiskNotice') } } : {}), title: intent.display === undefined ? item.question : resolveText(intent.display.question), ...(item.detail === undefined ? {} : { description: intent.display?.detail === undefined ? item.detail : resolveText(intent.display.detail), diff --git a/packages/experimental/cua/src/guidance.ts b/packages/experimental/cua/src/guidance.ts index 745ccfe9be..5f0abd1dac 100644 --- a/packages/experimental/cua/src/guidance.ts +++ b/packages/experimental/cua/src/guidance.ts @@ -8,6 +8,7 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. +- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Returns true only on approval. Avoid asking unless the task requires a physical hover. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index d6c75f46bc..40cb0bf1cd 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -19,6 +19,7 @@ import type { CuaApplicationGrant, CuaApplicationId, CuaSystemPermission } from import type { CuaAppHandle, CuaOperation, CuaStartupCapabilities } from './types.ts' import { ApplicationIcon, AppResult, NativeScreenshotResult, Request, ScreenshotResult, StateResult, WindowsResult } from './protocol.ts' import { CuaConfigFields } from './config.ts' +import { MouseControl } from './mouse-control.ts' import { CuaCoordinates } from './coordinates.ts' import { createScope, type Scope } from '@deepseek-ai/dsh-scope' import type {} from '@deepseek-ai/dsh-agent-preset-registry' @@ -156,6 +157,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) permissionParameters: backend.permissionParameters, store: new CuaPermissionStore(ctx, config.allowedApps.map(grant => ({ ...grant, id: grant.id as CuaApplicationId })), config.timeoutMs), }) + const mouseControl = new MouseControl(ctx) const clients = new Map() const lifetime = new AbortController() const pending = new Map, Agent>() @@ -220,6 +222,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) ? { ...params, maxElements: config.maxElements, maxDepth: config.maxDepth } : method === 'drag' ? { ...params, durationMs: config.dragDurationMs, steps: config.dragSteps } : params + if (method === 'mouseMove') nativeParams.useSystemPointer = mouseControl.allowed(agent) const requestNative = () => client.request(method, nativeParams, signal, execution.callId) if (method === 'screenshot') { const { attachments, target } = await screenshotStore(ctx, execution) @@ -306,6 +309,10 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) const signal = AbortSignal.any([execution.signal, lifetime.signal, owner.signal]) signal.throwIfAborted() const operation = (async () => { + if (request.method === 'requestMouseControl') { + if (!backend.capabilities.methods.includes(request.method)) throw new NativeError('UNSUPPORTED_OPERATION', 'This provider does not support system mouse control') + return mouseControl.request(agent, signal) + } const previous = clients.get(agent) if (previous?.disposed) { try { await previous.dispose() } catch (error) { @@ -333,7 +340,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) const nativeClient = client const dispatch = async (signal: AbortSignal) => { const scoped = { ...execution, agent, signal, nativeClient, coordinates: owner.coordinates } - const result = await callNative(Request.parse(request), scoped) + const result = await callNative(request, scoped) signal.throwIfAborted() return result } diff --git a/packages/experimental/cua/src/locales.ts b/packages/experimental/cua/src/locales.ts index 499ce34339..3833690158 100644 --- a/packages/experimental/cua/src/locales.ts +++ b/packages/experimental/cua/src/locales.ts @@ -103,3 +103,28 @@ export const launchAccessQuestion = { ru: 'Разрешить запуск? Доступ к окнам запрашивается отдельно.', pt: 'Permitir iniciar esta entrada? O acesso às janelas é solicitado separadamente.', } + +/** Scope and interruption shown before the user grants mouse control. */ +export const mouseControlDisplay = { + question: { + en: 'Allow the agent to move your mouse pointer?', + zh: '允许智能体移动你的鼠标指针吗？', + ja: 'エージェントがマウスポインタを動かすことを許可しますか？', + es: '¿Permitir que el agente mueva el puntero del ratón?', + fr: 'Autoriser l’agent à déplacer le pointeur de la souris ?', + de: 'Darf der Agent den Mauszeiger bewegen?', + ru: 'Разрешить агенту перемещать указатель мыши?', + pt: 'Permitir que o agente mova o ponteiro do mouse?', + }, + detail: { + en: 'Hovering will move your pointer and leave it at the target, even while you are using the mouse. This permission applies only to this session and expires when DSH restarts.', + zh: '智能体悬停时会移动你的鼠标指针，并将它留在目标位置，即使你正在使用鼠标。授权仅限当前会话，重启 DSH 后失效。', + ja: 'ホバーすると、マウスを使用中でもポインタが移動し、移動先に残ります。許可はこのセッションに限り、DSH を再起動すると失効します。', + es: 'Al pasar sobre un elemento, el puntero se moverá y quedará allí aunque estés usando el ratón. El permiso solo se aplica a esta sesión y caduca al reiniciar DSH.', + fr: 'Lors d’un survol, le pointeur se déplace et reste sur la cible, même si vous utilisez la souris. Cette autorisation vaut pour cette session et expire au redémarrage de DSH.', + de: 'Beim Überfahren eines Ziels bewegt sich der Zeiger und bleibt dort, auch wenn Sie die Maus verwenden. Die Erlaubnis gilt nur für diese Sitzung und erlischt beim Neustart von DSH.', + ru: 'При наведении указатель переместится и останется на цели, даже если вы пользуетесь мышью. Разрешение действует только в этом сеансе до перезапуска DSH.', + pt: 'Ao passar sobre um elemento, o ponteiro se moverá e ficará no destino, mesmo enquanto você usa o mouse. A permissão vale apenas para esta sessão e expira ao reiniciar o DSH.', + }, + approve: applicationAccessDisplay.approve, +} diff --git a/packages/experimental/cua/src/mouse-control.ts b/packages/experimental/cua/src/mouse-control.ts new file mode 100644 index 0000000000..306cd43a2f --- /dev/null +++ b/packages/experimental/cua/src/mouse-control.ts @@ -0,0 +1,56 @@ +/** Human decisions about the system pointer live in the provider process, keyed by Session. */ +import type { Context } from '@deepseek-ai/cordis' +import type { Agent } from '@deepseek-ai/dsh-agent' +import { UserQuestionError } from '@deepseek-ai/dsh-user-questions' +import { mouseControlDisplay } from './locales.ts' +import { NativeError } from './native-client.ts' + +/** Mouse control applies only to mouseMove and never grants access to an application. */ +export class MouseControl { + private readonly decisions = new Map() + + constructor(private readonly ctx: Context) {} + + /** @param agent Calling Agent. @returns Whether this Session explicitly approved moving the system pointer. */ + allowed(agent: Agent): boolean { + return this.decisions.get(agent.id) === true + } + + /** + * Ask once for the Session. Simultaneous requests cannot open additional questions. + * @param agent Live Agent that owns the Session. + * @param signal Tool lifetime. An aborted tool cannot grant permission. + * @returns True only when the user chose the offered approval. + */ + async request(agent: Agent, signal: AbortSignal): Promise { + const decision = this.decisions.get(agent.id) + if (decision === 'asking') throw new NativeError('MOUSE_CONTROL_PENDING', 'A mouse control request is already awaiting an answer in this session') + if (decision !== undefined) return decision + const questions = this.ctx.get('userQuestions') + if (!questions) throw new NativeError('MOUSE_APPROVAL_UNAVAILABLE', 'No human answerer is available for mouse control') + this.decisions.set(agent.id, 'asking') + const id = 'cua-mouse-control' + const approve = mouseControlDisplay.approve.en + const reject = 'Deny' + try { + const result = await questions.ask({ agent, signal, questions: [{ + id, question: mouseControlDisplay.question.en, detail: mouseControlDisplay.detail.en, + options: [{ label: approve }, { label: reject }], + intent: { kind: 'approval', approve, reject, display: mouseControlDisplay }, + }] }) + signal.throwIfAborted() + const [answer] = result.answers + const granted = result.answers.length === 1 && answer?.id === id && !answer.custom?.trim() + && answer.selected.length === 1 && answer.selected[0] === approve + this.decisions.set(agent.id, granted) + return granted + } catch (error) { + if (error instanceof UserQuestionError && error.code === 'ASK_CANCELLED') { + this.decisions.set(agent.id, false) + return false + } + this.decisions.delete(agent.id) + throw error + } + } +} diff --git a/packages/experimental/cua/src/native-schema.ts b/packages/experimental/cua/src/native-schema.ts index a6122ed71e..9cf8435001 100644 --- a/packages/experimental/cua/src/native-schema.ts +++ b/packages/experimental/cua/src/native-schema.ts @@ -1,7 +1,7 @@ /** Build-time native admission metadata derived from the shared operation definitions. */ import { z } from 'zod' import type { JSONSchema } from 'zod/v4/core' -import { NativeOnlyOperations, NativeResourceParameters, Request } from './protocol.ts' +import { NativeOnlyOperations, NativeResourceParameters, NativeToolOperations, type Request } from './protocol.ts' type Schema = JSONSchema.JSONSchema @@ -22,7 +22,7 @@ export function nativeProtocol(platformFields: Partial z.toJSONSchema(schema)) const fields = new Map>() const enums = new Map>>() diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index 538c95cfe6..161da04ee7 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -34,8 +34,8 @@ const imageBase64 = z.string().min(1).refine(value => Buffer.from(value, 'base64 message: 'Expected canonical base64 image data', }) -/** Model-submittable operation parameters; Host resource fields and private continuations are separate. */ -export const Request = z.discriminatedUnion('method', [ +/** Operations shared by the model and native helper, before Host parameters are added. */ +export const NativeToolOperations = [ operation('getCapabilities', {}), operation('listApps', {}), operation('findApps', { query: z.string().min(1, 'Expected a non-empty search string').regex(/\S/, 'Expected a non-empty search string') }), operation('getApp', { selector: z.string().min(1) }), @@ -77,7 +77,10 @@ export const Request = z.discriminatedUnion('method', [ fromX: relativePoint, fromY: relativePoint, toX: relativePoint, toY: relativePoint }).strict(), ]) }).strict(), operation('screenshot', window), -]) +] as const + +/** Model-submittable operations. The Host answers mouse-control requests without dispatching native input. */ +export const Request = z.discriminatedUnion('method', [...NativeToolOperations, operation('requestMouseControl', {})]) /** Resource parameters supplied by the Host, never accepted from model tool arguments. */ export const NativeResourceParameters = { diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index b84663f9e2..541dcdd070 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -246,6 +246,11 @@ export interface CuaApp extends CuaSurface { } /** Native app API; its caller owns global bindings and variable persistence. */ export interface CuaApi { + /** + * Ask to move the user's pointer with mouseMove for this session until restart. + * @returns True only after explicit approval. A refusal is remembered for the session. + */ + requestMouseControl(): Promise /** * Inspect permissions without requesting them. * @param options - output options. diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index cbaa419951..053010af02 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -890,3 +890,11 @@ it('expands menu contents and details only for the requested observation', async await app.getAccessibilityTree({ includeMenus: false, details: false }) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'getState', params: { appHandle: selected.appHandle, includeMenus: false, details: false } }) }) + +it('requests mouse control without dispatching a move and returns the decision', async () => { + const dispatch = vi.fn().mockResolvedValueOnce(false).mockResolvedValueOnce(true) + const cua = createCuaApi(dispatch) + expect(await cua.requestMouseControl()).toBe(false) + expect(await cua.requestMouseControl()).toBe(true) + expect(dispatch.mock.calls).toEqual(Array(2).fill([{ method: 'requestMouseControl', params: {} }])) +}) diff --git a/packages/experimental/cua/tests/application-approval.client.spec.tsx b/packages/experimental/cua/tests/application-approval.client.spec.tsx index f9ec74239c..b6bc9d3d7c 100644 --- a/packages/experimental/cua/tests/application-approval.client.spec.tsx +++ b/packages/experimental/cua/tests/application-approval.client.spec.tsx @@ -6,6 +6,7 @@ import { afterEach, expect, it, vi } from 'vitest' import type { ApprovalCardInput } from '@deepseek-ai/dsh-client-ui-approval/client' import { PendingQuestion, createWaterfallRequest } from '@deepseek-ai/dsh-client-ui-user-questions/src/client/contract/slots.ts' import { ApplicationApproval, selectApplicationApproval, type ApplicationApprovalProps } from '../src/client/ApplicationApproval.tsx' +import { mouseControlDisplay } from '../src/locales.ts' import { en, zh } from '../src/client/locales.ts' type AskUserQuestionItem = PendingQuestion['questions'][number] @@ -84,3 +85,16 @@ it('uses the original question cancellation and refuses late approval', async () expect(card.canAnswer()).toBe(false) await expect(card.onAllow()).rejects.toThrow('no channel accepts an answer yet') }) + +it('shows the mouse interruption and expiry with only session approval and refusal', () => { + const display = mouseControlDisplay + const pending = new PendingQuestion('s1' as SessionId, [{ id: 'cua-mouse-control', question: display.question.en, + detail: display.detail.en, intent: { kind: 'approval', approve: display.approve.en, reject: 'Deny', display } }]) + const factory = vi.fn((_name: string, _input: ApprovalCardInput) => null) + render( text.zh } as ApplicationApprovalProps} />) + const card = factory.mock.calls[0]![1] + expect(card).toMatchObject({ title: display.question.zh, description: display.detail.zh, allowLabel: display.approve.zh }) + expect(card.onAlways).toBeUndefined() + expect(card.warning).toBeUndefined() +}) diff --git a/snapshots/session/cua-native/replay.override.json b/snapshots/session/cua-native/replay.override.json index b43a6e20f0..e6bf49a2c3 100644 --- a/snapshots/session/cua-native/replay.override.json +++ b/snapshots/session/cua-native/replay.override.json @@ -1418,6 +1418,72 @@ } ] }, + { + "kind": "chunks", + "chunks": [ + { + "type": "block-start", + "index": 0, + "blockType": "tool-call" + }, + { + "type": "block-end", + "index": 0, + "block": { + "type": "tool-call", + "id": "cua-pointer-request", + "name": "cua", + "arguments": "{\"method\":\"requestMouseControl\",\"params\":{}}" + } + }, + { + "type": "usage", + "usage": { + "inputTokens": 3, + "outputTokens": 3 + } + }, + { + "type": "finish", + "reason": { + "kind": "tool-calls" + } + } + ] + }, + { + "kind": "chunks", + "chunks": [ + { + "type": "block-start", + "index": 0, + "blockType": "tool-call" + }, + { + "type": "block-end", + "index": 0, + "block": { + "type": "tool-call", + "id": "cua-system-hover", + "name": "cua", + "arguments": "{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}" + } + }, + { + "type": "usage", + "usage": { + "inputTokens": 3, + "outputTokens": 3 + } + }, + { + "type": "finish", + "reason": { + "kind": "tool-calls" + } + } + ] + }, { "kind": "chunks", "chunks": [ diff --git a/snapshots/session/cua-native/workspace/native-helper.mjs b/snapshots/session/cua-native/workspace/native-helper.mjs index 1e148d7ab5..e9a972a7eb 100755 --- a/snapshots/session/cua-native/workspace/native-helper.mjs +++ b/snapshots/session/cua-native/workspace/native-helper.mjs @@ -47,7 +47,6 @@ function serve() { const server = createServer({ allowHalfOpen: true }, socket => { sockets.add(socket) let connected = false - let systemPointerHover const connectionId = ++connection const input = createInterface({ input: socket }) const scopedRecord = (event, data = {}) => record(event, { connection: connectionId, ...data }) @@ -59,7 +58,6 @@ function serve() { socket.destroy() return } - systemPointerHover = request.config.systemPointerHover connected = true hadClient = true clearTimeout(startupDeadline) @@ -73,7 +71,7 @@ function serve() { } let response try { - const result = call(request.id, request.method, request.params, systemPointerHover) + const result = call(request.id, request.method, request.params) if (result === undefined) return response = { id: request.id, result } } catch (error) { @@ -124,7 +122,7 @@ function createApplicationState(socket, record, application, captureBroker, conn let observationChanged = false let malformedScreenshot = false function refuse(code, message) { throw Object.assign(new Error(message), { code }) } - function call(requestId, method, params, systemPointerHover) { + function call(requestId, method, params) { record('call', { requestId, method, params }) if (method === 'applicationIcons') return Object.fromEntries(params.paths.map(path => [path, `data:image/png;base64,${png}`])) if (method === 'openPermissionSettings') return { opened: true } @@ -204,11 +202,9 @@ function createApplicationState(socket, record, application, captureBroker, conn let systemPointerMoved = false if (method === 'mouseMove') { const alternate = params.elementId === '0' || (params.x >= 20 && params.x <= 40) - if (!alternate && !application.hovering) { - if (!(systemPointerHover ?? params.allowSystemPointer)) refuse('HOVER_UNAVAILABLE', 'No AX alternate UI at this target. No input was sent and the pointer did not move.') - systemPointerMoved = true - } - application.hovering = alternate + systemPointerMoved = params.useSystemPointer === true + if (!systemPointerMoved && !alternate && !application.hovering) refuse('HOVER_UNAVAILABLE', 'No AX alternate UI at this target. No input was sent and the pointer did not move.') + application.hovering = !systemPointerMoved && alternate } else if (method === 'click' && params.windowId === 8) application.visualCount++ else if (method === 'click') application.count++ From fb1b204385873d0a93c259e415536e593ff45558 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 14:28:59 +0800 Subject: [PATCH 02/30] test(cua): record consent and pointer routing through a session --- .../experimental/cua/src/mouse-control.ts | 6 +- .../application-approval.client.spec.tsx | 7 +- snapshots/session/cua-native/session.v4.jsonl | 178 +++++++++--------- .../cua-native/system-prompt.expected.md | 3 +- .../cua-native/tool-schemas.expected.json | 19 ++ .../session/cua-ptc/system-prompt.expected.md | 3 +- .../system-prompt.expected.md | 3 +- 7 files changed, 129 insertions(+), 90 deletions(-) diff --git a/packages/experimental/cua/src/mouse-control.ts b/packages/experimental/cua/src/mouse-control.ts index 306cd43a2f..947482766f 100644 --- a/packages/experimental/cua/src/mouse-control.ts +++ b/packages/experimental/cua/src/mouse-control.ts @@ -11,7 +11,11 @@ export class MouseControl { constructor(private readonly ctx: Context) {} - /** @param agent Calling Agent. @returns Whether this Session explicitly approved moving the system pointer. */ + /** + * Select the native route without prompting the user. + * @param agent Calling Agent. + * @returns Whether this Session explicitly approved moving the system pointer. + */ allowed(agent: Agent): boolean { return this.decisions.get(agent.id) === true } diff --git a/packages/experimental/cua/tests/application-approval.client.spec.tsx b/packages/experimental/cua/tests/application-approval.client.spec.tsx index b6bc9d3d7c..13e79b8faa 100644 --- a/packages/experimental/cua/tests/application-approval.client.spec.tsx +++ b/packages/experimental/cua/tests/application-approval.client.spec.tsx @@ -6,7 +6,6 @@ import { afterEach, expect, it, vi } from 'vitest' import type { ApprovalCardInput } from '@deepseek-ai/dsh-client-ui-approval/client' import { PendingQuestion, createWaterfallRequest } from '@deepseek-ai/dsh-client-ui-user-questions/src/client/contract/slots.ts' import { ApplicationApproval, selectApplicationApproval, type ApplicationApprovalProps } from '../src/client/ApplicationApproval.tsx' -import { mouseControlDisplay } from '../src/locales.ts' import { en, zh } from '../src/client/locales.ts' type AskUserQuestionItem = PendingQuestion['questions'][number] @@ -87,7 +86,11 @@ it('uses the original question cancellation and refuses late approval', async () }) it('shows the mouse interruption and expiry with only session approval and refusal', () => { - const display = mouseControlDisplay + const display = { + question: { en: 'Move pointer?', zh: '允许移动鼠标指针吗？' }, + detail: { en: 'Until restart', zh: '重启后失效' }, + approve: { en: 'Allow for this session', zh: '本会话允许' }, + } const pending = new PendingQuestion('s1' as SessionId, [{ id: 'cua-mouse-control', question: display.question.en, detail: display.detail.en, intent: { kind: 'approval', approve: display.approve.en, reject: 'Deny', display } }]) const factory = vi.fn((_name: string, _input: ApprovalCardInput) => null) diff --git a/snapshots/session/cua-native/session.v4.jsonl b/snapshots/session/cua-native/session.v4.jsonl index 10a71dc794..7f49b59d88 100644 --- a/snapshots/session/cua-native/session.v4.jsonl +++ b/snapshots/session/cua-native/session.v4.jsonl @@ -12,286 +12,296 @@ {"type":"request/header","data":{"header":{"config":{"provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"tools":"{{tools}}"},"reason":"initial"}} {"type":"request/context","data":{"provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"}} {"type":"session/title","data":{"title":"Inspect CUA capabilities, select the","messageSeqs":[8],"source":{"kind":"fallback"}}} -{"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:4}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949731596,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949731597,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}}},{"type":"chunk","time":1793949731597,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949731597,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:4}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032889225,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032889225,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}}},{"type":"chunk","time":1794032889225,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032889225,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":1,"callId":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}} -{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-1"},"toolCallId":"cua-native-1","content":[{"type":"text","text":"{\"sessionLocked\":false,\"permissions\":{\"accessibility\":true,\"screenRecording\":true},\"platform\":\"macos\",\"methods\":[\"activate\",\"click\",\"drag\",\"findApps\",\"getApp\",\"getCapabilities\",\"getState\",\"invoke\",\"listActions\",\"listApps\",\"listWindows\",\"mouseMove\",\"paste\",\"performAction\",\"pressKey\",\"screenshot\",\"scroll\",\"selectText\",\"setValue\",\"show\",\"typeText\"],\"appLevelAccessibility\":true,\"keyboard\":[\"app\"],\"browser\":false,\"coordinateSpace\":\"screenshot-pixels\"}"}],"isError":false,"id":"{{message:5}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"} +{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-1"},"toolCallId":"cua-native-1","content":[{"type":"text","text":"{\"sessionLocked\":false,\"permissions\":{\"accessibility\":true,\"screenRecording\":true},\"platform\":\"macos\",\"methods\":[\"activate\",\"click\",\"drag\",\"findApps\",\"getApp\",\"getCapabilities\",\"getState\",\"invoke\",\"listActions\",\"listApps\",\"listWindows\",\"mouseMove\",\"paste\",\"performAction\",\"pressKey\",\"requestMouseControl\",\"screenshot\",\"scroll\",\"selectText\",\"setValue\",\"show\",\"typeText\"],\"appLevelAccessibility\":true,\"keyboard\":[\"app\"],\"browser\":false,\"coordinateSpace\":\"screenshot-pixels\"}"}],"isError":false,"id":"{{message:5}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":1}} {"type":"step/start","data":{"turn":1,"step":2}} -{"type":"assistant/message","data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-unauthorized-getState","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:6}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732188,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732188,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-unauthorized-getState","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1793949732188,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732188,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-unauthorized-getState","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:6}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032889794,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032889794,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-unauthorized-getState","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1794032889794,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032889794,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":2,"callId":"cua-unauthorized-getState","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}"}} {"type":"tool/result","data":{"turn":1,"step":2,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-unauthorized-getState"},"toolCallId":"cua-unauthorized-getState","content":[{"type":"text","text":"Error: APP_ACCESS_REQUIRED: Select the application with getApp and obtain user consent before reading or controlling it"}],"isError":true,"id":"{{message:7}}"}},"sourceEventSeqs":[19],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":2}} {"type":"step/start","data":{"turn":1,"step":3}} -{"type":"assistant/message","data":{"turn":1,"step":3,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-unauthorized-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:8}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732206,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732206,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-unauthorized-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949732206,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732206,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":3,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-unauthorized-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:8}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032889811,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032889811,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-unauthorized-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032889811,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032889811,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":3,"callId":"cua-unauthorized-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} {"type":"tool/result","data":{"turn":1,"step":3,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-unauthorized-screenshot"},"toolCallId":"cua-unauthorized-screenshot","content":[{"type":"text","text":"Error: APP_ACCESS_REQUIRED: Select the application with getApp and obtain user consent before reading or controlling it"}],"isError":true,"id":"{{message:9}}"}},"sourceEventSeqs":[24],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":3}} {"type":"step/start","data":{"turn":1,"step":4}} -{"type":"assistant/message","data":{"turn":1,"step":4,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-unauthorized-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementId\":\"0\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:10}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732224,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732224,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-unauthorized-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementId\":\"0\"}}"}}},{"type":"chunk","time":1793949732224,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732224,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":4,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-unauthorized-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementId\":\"0\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:10}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032889830,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032889830,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-unauthorized-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementId\":\"0\"}}"}}},{"type":"chunk","time":1794032889830,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032889830,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":4,"callId":"cua-unauthorized-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementId\":\"0\"}}"}} {"type":"tool/result","data":{"turn":1,"step":4,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-unauthorized-click"},"toolCallId":"cua-unauthorized-click","content":[{"type":"text","text":"Error: APP_ACCESS_REQUIRED: Select the application with getApp and obtain user consent before reading or controlling it"}],"isError":true,"id":"{{message:11}}"}},"sourceEventSeqs":[29],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":4}} {"type":"step/start","data":{"turn":1,"step":5}} -{"type":"assistant/message","data":{"turn":1,"step":5,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-2","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"test.fixture\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:12}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732241,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732241,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-2","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"test.fixture\"}}"}}},{"type":"chunk","time":1793949732241,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732241,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":5,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-2","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"test.fixture\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:12}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032889846,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032889846,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-2","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"test.fixture\"}}"}}},{"type":"chunk","time":1794032889846,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032889846,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":5,"callId":"cua-native-2","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"test.fixture\"}}"}} {"type":"tool/result","data":{"turn":1,"step":5,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-2"},"toolCallId":"cua-native-2","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"name\":\"Fixture\",\"bundleId\":\"test.fixture\",\"running\":true}"}],"isError":false,"id":"{{message:13}}"}},"sourceEventSeqs":[34],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":5}} {"type":"step/start","data":{"turn":1,"step":6}} -{"type":"assistant/message","data":{"turn":1,"step":6,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-3","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"includeMenus\":true,\"details\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:14}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732276,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732276,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-3","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"includeMenus\":true,\"details\":true}}"}}},{"type":"chunk","time":1793949732276,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732276,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":6,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-3","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"includeMenus\":true,\"details\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:14}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032889881,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032889881,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-3","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"includeMenus\":true,\"details\":true}}"}}},{"type":"chunk","time":1794032889881,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032889881,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":6,"callId":"cua-native-3","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"includeMenus\":true,\"details\":true}}"}} {"type":"tool/result","data":{"turn":1,"step":6,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-3"},"toolCallId":"cua-native-3","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-1\",\"tree\":\"AX scope=window offsets=UTF-16 bounds=global-points\\n- [0] AXButton \\\"Increment\\\" id=\\\"increment\\\" actions=[\\\"AXPress\\\"] bounds=[60,80,100,20]\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 0\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\n - [3] AXMenuItem \\\"Fixture command\\\"\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:15}}"}},"sourceEventSeqs":[39],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":6}} {"type":"step/start","data":{"turn":1,"step":7}} -{"type":"assistant/message","data":{"turn":1,"step":7,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementId\":\"0\",\"holdMs\":800}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:16}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732298,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732298,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementId\":\"0\",\"holdMs\":800}}"}}},{"type":"chunk","time":1793949732298,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732298,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":7,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementId\":\"0\",\"holdMs\":800}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:16}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032889899,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032889899,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementId\":\"0\",\"holdMs\":800}}"}}},{"type":"chunk","time":1794032889899,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032889899,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":7,"callId":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementId\":\"0\",\"holdMs\":800}}"}} {"type":"tool/result","data":{"turn":1,"step":7,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-4"},"toolCallId":"cua-native-4","content":[{"type":"text","text":"{\"dispatched\":true}"}],"isError":false,"id":"{{message:17}}"}},"sourceEventSeqs":[44],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":7}} {"type":"step/start","data":{"turn":1,"step":8}} -{"type":"assistant/message","data":{"turn":1,"step":8,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-5","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:18}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732316,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732316,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-5","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949732316,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732316,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":8,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-5","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:18}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032889915,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032889915,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-5","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032889915,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032889915,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":8,"callId":"cua-native-5","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} {"type":"tool/result","data":{"turn":1,"step":8,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-5"},"toolCallId":"cua-native-5","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-2\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 1\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:19}}"}},"sourceEventSeqs":[49],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":8}} {"type":"step/start","data":{"turn":1,"step":9}} -{"type":"assistant/message","data":{"turn":1,"step":9,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-6","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:20}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732333,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732333,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-6","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949732333,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732333,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":9,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-6","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:20}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032889932,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032889932,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-6","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032889932,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032889932,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":9,"callId":"cua-native-6","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} {"type":"tool/result","data":{"turn":1,"step":9,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-6"},"toolCallId":"cua-native-6","content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"isError":false,"id":"{{message:21}}"}},"sourceEventSeqs":[54],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":9}} {"type":"step/start","data":{"turn":1,"step":10}} -{"type":"assistant/message","data":{"turn":1,"step":10,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-7","name":"cua","arguments":"{\"method\":\"listWindows\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:22}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732405,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732405,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-7","name":"cua","arguments":"{\"method\":\"listWindows\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1793949732405,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732405,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":10,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-7","name":"cua","arguments":"{\"method\":\"listWindows\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:22}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890010,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890010,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-7","name":"cua","arguments":"{\"method\":\"listWindows\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1794032890010,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890010,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":10,"callId":"cua-native-7","name":"cua","arguments":"{\"method\":\"listWindows\",\"params\":{\"appHandle\":\"fixture-app\"}}"}} {"type":"tool/result","data":{"turn":1,"step":10,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-7"},"toolCallId":"cua-native-7","content":[{"type":"text","text":"[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true},{\"windowId\":8,\"appName\":\"Fixture\",\"title\":\"AXless canvas\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}]"}],"isError":false,"id":"{{message:23}}"}},"sourceEventSeqs":[59],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":10}} {"type":"step/start","data":{"turn":1,"step":11}} -{"type":"assistant/message","data":{"turn":1,"step":11,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-8","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:24}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732424,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732424,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-8","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1793949732424,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732424,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":11,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-8","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:24}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890027,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890027,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-8","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1794032890027,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890027,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":11,"callId":"cua-native-8","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}} {"type":"tool/result","data":{"turn":1,"step":11,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-8"},"toolCallId":"cua-native-8","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"tree\":\"\",\"windows\":[{\"windowId\":8,\"appName\":\"Fixture\",\"title\":\"AXless canvas\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:25}}"}},"sourceEventSeqs":[64],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":11}} {"type":"step/start","data":{"turn":1,"step":12}} -{"type":"assistant/message","data":{"turn":1,"step":12,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-9","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:26}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732445,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732445,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-9","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1793949732445,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732445,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":12,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-9","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:26}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890044,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890044,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-9","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1794032890044,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890044,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":12,"callId":"cua-native-9","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}} {"type":"tool/result","data":{"turn":1,"step":12,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-9"},"toolCallId":"cua-native-9","content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":8,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"isError":false,"id":"{{message:27}}"}},"sourceEventSeqs":[69],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":12}} {"type":"step/start","data":{"turn":1,"step":13}} -{"type":"assistant/message","data":{"turn":1,"step":13,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-10","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":0,\"y\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:28}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732481,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732481,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-10","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":0,\"y\":0}}"}}},{"type":"chunk","time":1793949732481,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732481,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":13,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-10","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":0,\"y\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:28}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890077,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890077,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-10","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":0,\"y\":0}}"}}},{"type":"chunk","time":1794032890077,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890077,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":13,"callId":"cua-native-10","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":0,\"y\":0}}"}} {"type":"tool/result","data":{"turn":1,"step":13,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-10"},"toolCallId":"cua-native-10","content":[{"type":"text","text":"{\"dispatched\":true}"}],"isError":false,"id":"{{message:29}}"}},"sourceEventSeqs":[74],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":13}} {"type":"step/start","data":{"turn":1,"step":14}} -{"type":"assistant/message","data":{"turn":1,"step":14,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-11","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:30}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732502,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732502,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-11","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1793949732502,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732502,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":14,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-11","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:30}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890097,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890097,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-11","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1794032890097,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890097,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":14,"callId":"cua-native-11","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}} {"type":"tool/result","data":{"turn":1,"step":14,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-11"},"toolCallId":"cua-native-11","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"tree\":\"\",\"windows\":[{\"windowId\":8,\"appName\":\"Fixture\",\"title\":\"AXless canvas\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:31}}"}},"sourceEventSeqs":[79],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":14}} {"type":"step/start","data":{"turn":1,"step":15}} -{"type":"assistant/message","data":{"turn":1,"step":15,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-show","name":"cua","arguments":"{\"method\":\"show\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:32}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732523,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732523,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-show","name":"cua","arguments":"{\"method\":\"show\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949732523,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732523,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":15,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-show","name":"cua","arguments":"{\"method\":\"show\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:32}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890116,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890116,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-show","name":"cua","arguments":"{\"method\":\"show\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890116,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890116,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":15,"callId":"cua-native-show","name":"cua","arguments":"{\"method\":\"show\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} {"type":"tool/result","data":{"turn":1,"step":15,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-show"},"toolCallId":"cua-native-show","content":[{"type":"text","text":"{\"dispatched\":true,\"windowId\":7,\"foregroundChanged\":false}"}],"isError":false,"id":"{{message:33}}"}},"sourceEventSeqs":[84],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":15}} {"type":"step/start","data":{"turn":1,"step":16}} -{"type":"assistant/message","data":{"turn":1,"step":16,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:34}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732546,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732546,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1793949732546,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732546,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":16,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:34}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890134,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890134,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1794032890134,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890134,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":16,"callId":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}} {"type":"tool/result","data":{"turn":1,"step":16,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-activate-first"},"toolCallId":"cua-native-activate-first","content":[{"type":"text","text":"{\"dispatched\":true,\"foregroundChanged\":true,\"foreground\":{\"requestId\":\"cua-native-activate-first\",\"targetPid\":9,\"before\":{\"frontPid\":42,\"focusedWindowId\":7,\"frontWindowId\":99,\"frontWindowPid\":42},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":true}}"}],"isError":false,"id":"{{message:35}}"}},"sourceEventSeqs":[89],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":16}} {"type":"step/start","data":{"turn":1,"step":17}} -{"type":"assistant/message","data":{"turn":1,"step":17,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:36}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732573,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732573,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1793949732573,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732573,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":17,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:36}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890151,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890151,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1794032890151,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890151,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":17,"callId":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}} {"type":"tool/result","data":{"turn":1,"step":17,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-activate-repeat"},"toolCallId":"cua-native-activate-repeat","content":[{"type":"text","text":"{\"dispatched\":true,\"foregroundChanged\":false,\"foreground\":{\"requestId\":\"cua-native-activate-repeat\",\"targetPid\":9,\"before\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":false}}"}],"isError":false,"id":"{{message:37}}"}},"sourceEventSeqs":[94],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":17}} {"type":"step/start","data":{"turn":1,"step":18}} -{"type":"assistant/message","data":{"turn":1,"step":18,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-transition","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"Return\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:38}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732598,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732598,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-transition","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"Return\"}}"}}},{"type":"chunk","time":1793949732598,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732598,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":18,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-transition","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"Return\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:38}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890168,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890168,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-transition","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"Return\"}}"}}},{"type":"chunk","time":1794032890168,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890168,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":18,"callId":"cua-native-transition","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"Return\"}}"}} {"type":"tool/result","data":{"turn":1,"step":18,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-transition"},"toolCallId":"cua-native-transition","content":[{"type":"text","text":"{\"dispatched\":true}"}],"isError":false,"id":"{{message:39}}"}},"sourceEventSeqs":[99],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":18}} {"type":"step/start","data":{"turn":1,"step":19}} -{"type":"assistant/message","data":{"turn":1,"step":19,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-stale-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:40}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732622,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732622,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-stale-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949732622,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732622,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":19,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-stale-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:40}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890186,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890186,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-stale-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890186,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890186,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":19,"callId":"cua-native-stale-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} {"type":"tool/result","data":{"turn":1,"step":19,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-stale-read"},"toolCallId":"cua-native-stale-read","content":[{"type":"text","text":"Error: STALE_SNAPSHOT: Accessibility tree changed during observation; read fresh getAccessibilityTree"}],"isError":true,"id":"{{message:41}}"}},"sourceEventSeqs":[104],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":19}} {"type":"step/start","data":{"turn":1,"step":20}} -{"type":"assistant/message","data":{"turn":1,"step":20,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-stale-index","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-2\",\"elementId\":\"0\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:42}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732643,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732643,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-stale-index","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-2\",\"elementId\":\"0\"}}"}}},{"type":"chunk","time":1793949732643,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732643,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":20,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-stale-index","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-2\",\"elementId\":\"0\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:42}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890203,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890203,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-stale-index","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-2\",\"elementId\":\"0\"}}"}}},{"type":"chunk","time":1794032890203,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890203,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":20,"callId":"cua-native-stale-index","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-2\",\"elementId\":\"0\"}}"}} {"type":"tool/result","data":{"turn":1,"step":20,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-stale-index"},"toolCallId":"cua-native-stale-index","content":[{"type":"text","text":"Error: STALE_SNAPSHOT: Read fresh state before acting"}],"isError":true,"id":"{{message:43}}"}},"sourceEventSeqs":[109],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":20}} {"type":"step/start","data":{"turn":1,"step":21}} -{"type":"assistant/message","data":{"turn":1,"step":21,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0,\"holdMs\":800}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:44}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732663,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732663,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0,\"holdMs\":800}}"}}},{"type":"chunk","time":1793949732663,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732663,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":21,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0,\"holdMs\":800}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:44}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890219,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890219,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0,\"holdMs\":800}}"}}},{"type":"chunk","time":1794032890219,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890219,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":21,"callId":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0,\"holdMs\":800}}"}} {"type":"tool/result","data":{"turn":1,"step":21,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-other-window"},"toolCallId":"cua-native-other-window","content":[{"type":"text","text":"{\"dispatched\":true}"}],"isError":false,"id":"{{message:45}}"}},"sourceEventSeqs":[114],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":21}} {"type":"step/start","data":{"turn":1,"step":22}} -{"type":"assistant/message","data":{"turn":1,"step":22,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-fresh-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:46}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732687,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732687,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-fresh-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949732687,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732687,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":22,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-fresh-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:46}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890236,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890236,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-fresh-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890236,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890236,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":22,"callId":"cua-native-fresh-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} {"type":"tool/result","data":{"turn":1,"step":22,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-fresh-read"},"toolCallId":"cua-native-fresh-read","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-5\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 1\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:47}}"}},"sourceEventSeqs":[119],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":22}} {"type":"step/start","data":{"turn":1,"step":23}} -{"type":"assistant/message","data":{"turn":1,"step":23,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-fresh-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-5\",\"elementId\":\"0\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:48}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732710,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732710,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-fresh-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-5\",\"elementId\":\"0\"}}"}}},{"type":"chunk","time":1793949732710,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732710,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":23,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-fresh-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-5\",\"elementId\":\"0\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:48}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890253,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890253,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-fresh-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-5\",\"elementId\":\"0\"}}"}}},{"type":"chunk","time":1794032890253,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890253,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":23,"callId":"cua-native-fresh-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-5\",\"elementId\":\"0\"}}"}} {"type":"tool/result","data":{"turn":1,"step":23,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-fresh-click"},"toolCallId":"cua-native-fresh-click","content":[{"type":"text","text":"{\"dispatched\":true}"}],"isError":false,"id":"{{message:49}}"}},"sourceEventSeqs":[124],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":23}} {"type":"step/start","data":{"turn":1,"step":24}} -{"type":"assistant/message","data":{"turn":1,"step":24,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:50}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732733,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732733,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949732733,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732733,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":24,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:50}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890269,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890269,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890269,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890269,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":24,"callId":"cua-native-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} {"type":"tool/result","data":{"turn":1,"step":24,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-verify"},"toolCallId":"cua-native-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-6\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 2\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:51}}"}},"sourceEventSeqs":[129],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":24}} {"type":"step/start","data":{"turn":1,"step":25}} -{"type":"assistant/message","data":{"turn":1,"step":25,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:52}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732756,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732756,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}}},{"type":"chunk","time":1793949732756,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732756,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":25,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:52}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890285,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890285,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}}},{"type":"chunk","time":1794032890285,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890285,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":25,"callId":"cua-native-geometry-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}} {"type":"tool/result","data":{"turn":1,"step":25,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-observe"},"toolCallId":"cua-native-geometry-observe","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":9,\"snapshotId\":\"fixture-snapshot-7\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 2\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":9,\"appName\":\"Fixture\",\"title\":\"Transformed window\",\"bounds\":{\"x\":16,\"y\":512,\"width\":131,\"height\":145},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:53}}"}},"sourceEventSeqs":[134],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":25}} {"type":"step/start","data":{"turn":1,"step":26}} -{"type":"assistant/message","data":{"turn":1,"step":26,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:54}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732777,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732777,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}}},{"type":"chunk","time":1793949732777,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732777,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":26,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:54}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890301,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890301,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}}},{"type":"chunk","time":1794032890301,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890301,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":26,"callId":"cua-native-geometry-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}} {"type":"tool/result","data":{"turn":1,"step":26,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-capture"},"toolCallId":"cua-native-geometry-capture","content":[{"type":"text","text":"Error: WINDOW_CHANGED: Window display and accessibility bounds disagree. Read fresh state; if unchanged, use AX actions or explicitly restore the window before capture or coordinate input"}],"isError":true,"id":"{{message:55}}"}},"sourceEventSeqs":[139],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":26}} {"type":"step/start","data":{"turn":1,"step":27}} -{"type":"assistant/message","data":{"turn":1,"step":27,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"snapshotId\":\"fixture-snapshot-7\",\"x\":20,\"y\":20,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:56}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732798,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732798,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"snapshotId\":\"fixture-snapshot-7\",\"x\":20,\"y\":20,\"relative\":true}}"}}},{"type":"chunk","time":1793949732798,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732798,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":27,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"snapshotId\":\"fixture-snapshot-7\",\"x\":20,\"y\":20,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:56}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890319,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890319,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"snapshotId\":\"fixture-snapshot-7\",\"x\":20,\"y\":20,\"relative\":true}}"}}},{"type":"chunk","time":1794032890319,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890319,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":27,"callId":"cua-native-geometry-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"snapshotId\":\"fixture-snapshot-7\",\"x\":20,\"y\":20,\"relative\":true}}"}} {"type":"tool/result","data":{"turn":1,"step":27,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-click"},"toolCallId":"cua-native-geometry-click","content":[{"type":"text","text":"Error: WINDOW_CHANGED: Window display and accessibility bounds disagree. Read fresh state; if unchanged, use AX actions or explicitly restore the window before capture or coordinate input"}],"isError":true,"id":"{{message:57}}"}},"sourceEventSeqs":[144],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":27}} {"type":"step/start","data":{"turn":1,"step":28}} -{"type":"assistant/message","data":{"turn":1,"step":28,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-scroll","name":"cua","arguments":"{\"method\":\"scroll\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"x\":20,\"y\":20,\"deltaLinesX\":0,\"deltaLinesY\":-1,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:58}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732822,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732822,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-scroll","name":"cua","arguments":"{\"method\":\"scroll\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"x\":20,\"y\":20,\"deltaLinesX\":0,\"deltaLinesY\":-1,\"relative\":true}}"}}},{"type":"chunk","time":1793949732822,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732822,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":28,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-scroll","name":"cua","arguments":"{\"method\":\"scroll\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"x\":20,\"y\":20,\"deltaLinesX\":0,\"deltaLinesY\":-1,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:58}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890336,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890336,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-scroll","name":"cua","arguments":"{\"method\":\"scroll\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"x\":20,\"y\":20,\"deltaLinesX\":0,\"deltaLinesY\":-1,\"relative\":true}}"}}},{"type":"chunk","time":1794032890336,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890336,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":28,"callId":"cua-native-geometry-scroll","name":"cua","arguments":"{\"method\":\"scroll\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"x\":20,\"y\":20,\"deltaLinesX\":0,\"deltaLinesY\":-1,\"relative\":true}}"}} {"type":"tool/result","data":{"turn":1,"step":28,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-scroll"},"toolCallId":"cua-native-geometry-scroll","content":[{"type":"text","text":"Error: WINDOW_CHANGED: Window display and accessibility bounds disagree. Read fresh state; if unchanged, use AX actions or explicitly restore the window before capture or coordinate input"}],"isError":true,"id":"{{message:59}}"}},"sourceEventSeqs":[149],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":28}} {"type":"step/start","data":{"turn":1,"step":29}} -{"type":"assistant/message","data":{"turn":1,"step":29,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-drag","name":"cua","arguments":"{\"method\":\"drag\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"fromX\":20,\"fromY\":20,\"toX\":30,\"toY\":30,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:60}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732845,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732845,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-drag","name":"cua","arguments":"{\"method\":\"drag\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"fromX\":20,\"fromY\":20,\"toX\":30,\"toY\":30,\"relative\":true}}"}}},{"type":"chunk","time":1793949732845,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732845,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":29,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-drag","name":"cua","arguments":"{\"method\":\"drag\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"fromX\":20,\"fromY\":20,\"toX\":30,\"toY\":30,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:60}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890353,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890353,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-drag","name":"cua","arguments":"{\"method\":\"drag\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"fromX\":20,\"fromY\":20,\"toX\":30,\"toY\":30,\"relative\":true}}"}}},{"type":"chunk","time":1794032890353,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890353,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":29,"callId":"cua-native-geometry-drag","name":"cua","arguments":"{\"method\":\"drag\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"fromX\":20,\"fromY\":20,\"toX\":30,\"toY\":30,\"relative\":true}}"}} {"type":"tool/result","data":{"turn":1,"step":29,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-drag"},"toolCallId":"cua-native-geometry-drag","content":[{"type":"text","text":"Error: WINDOW_CHANGED: Window display and accessibility bounds disagree. Read fresh state; if unchanged, use AX actions or explicitly restore the window before capture or coordinate input"}],"isError":true,"id":"{{message:61}}"}},"sourceEventSeqs":[154],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":29}} {"type":"step/start","data":{"turn":1,"step":30}} -{"type":"assistant/message","data":{"turn":1,"step":30,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-ax","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-7\",\"elementId\":\"0\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:62}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732870,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732870,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-ax","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-7\",\"elementId\":\"0\"}}"}}},{"type":"chunk","time":1793949732870,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732870,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":30,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-ax","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-7\",\"elementId\":\"0\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:62}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890372,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890372,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-ax","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-7\",\"elementId\":\"0\"}}"}}},{"type":"chunk","time":1794032890372,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890372,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":30,"callId":"cua-native-geometry-ax","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-7\",\"elementId\":\"0\"}}"}} {"type":"tool/result","data":{"turn":1,"step":30,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-ax"},"toolCallId":"cua-native-geometry-ax","content":[{"type":"text","text":"{\"dispatched\":true}"}],"isError":false,"id":"{{message:63}}"}},"sourceEventSeqs":[159],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":30}} {"type":"step/start","data":{"turn":1,"step":31}} -{"type":"assistant/message","data":{"turn":1,"step":31,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:64}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732900,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732900,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}}},{"type":"chunk","time":1793949732900,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732900,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":31,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:64}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890389,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890389,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}}},{"type":"chunk","time":1794032890389,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890389,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":31,"callId":"cua-native-geometry-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}} {"type":"tool/result","data":{"turn":1,"step":31,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-verify"},"toolCallId":"cua-native-geometry-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":9,\"snapshotId\":\"fixture-snapshot-8\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 3\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":9,\"appName\":\"Fixture\",\"title\":\"Transformed window\",\"bounds\":{\"x\":16,\"y\":512,\"width\":131,\"height\":145},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:65}}"}},"sourceEventSeqs":[164],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":31}} {"type":"step/start","data":{"turn":1,"step":32}} -{"type":"assistant/message","data":{"turn":1,"step":32,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-busy-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F11\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:66}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732924,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732924,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-busy-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F11\"}}"}}},{"type":"chunk","time":1793949732924,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732924,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":32,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-busy-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F11\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:66}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890407,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890407,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-busy-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F11\"}}"}}},{"type":"chunk","time":1794032890407,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890407,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":32,"callId":"cua-capture-busy-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F11\"}}"}} {"type":"tool/result","data":{"turn":1,"step":32,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-capture-busy-arm"},"toolCallId":"cua-capture-busy-arm","content":[{"type":"text","text":"{\"dispatched\":true}"}],"isError":false,"id":"{{message:67}}"}},"sourceEventSeqs":[169],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":32}} {"type":"step/start","data":{"turn":1,"step":33}} -{"type":"assistant/message","data":{"turn":1,"step":33,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-busy","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:68}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732947,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732947,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-busy","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949732947,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732947,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":33,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-busy","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:68}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890424,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890424,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-busy","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890424,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890424,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":33,"callId":"cua-capture-busy","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} {"type":"tool/result","data":{"turn":1,"step":33,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-capture-busy"},"toolCallId":"cua-capture-busy","content":[{"type":"text","text":"Error: CAPTURE_BUSY: A previous system capture is still pending; other native operations remain available\nScreenshot diagnostics:\n[cua-screenshot] {\"requestId\":\"cua-capture-busy\",\"stage\":\"helper.failed\",\"elapsedMs\":1,\"errorCode\":\"CAPTURE_BUSY\"}"}],"isError":true,"id":"{{message:69}}"}},"sourceEventSeqs":[174],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":33}} {"type":"step/start","data":{"turn":1,"step":34}} -{"type":"assistant/message","data":{"turn":1,"step":34,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-busy-ax","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:70}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732969,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732969,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-busy-ax","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949732969,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732969,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":34,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-busy-ax","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:70}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890441,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890441,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-busy-ax","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890441,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890441,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":34,"callId":"cua-capture-busy-ax","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} {"type":"tool/result","data":{"turn":1,"step":34,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-capture-busy-ax"},"toolCallId":"cua-capture-busy-ax","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-9\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 3\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:71}}"}},"sourceEventSeqs":[179],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":34}} {"type":"step/start","data":{"turn":1,"step":35}} -{"type":"assistant/message","data":{"turn":1,"step":35,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-callback","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F10\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:72}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732988,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732988,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-callback","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F10\"}}"}}},{"type":"chunk","time":1793949732988,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732988,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":35,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-callback","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F10\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:72}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890457,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890457,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-callback","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F10\"}}"}}},{"type":"chunk","time":1794032890457,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890457,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":35,"callId":"cua-capture-callback","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F10\"}}"}} {"type":"tool/result","data":{"turn":1,"step":35,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-capture-callback"},"toolCallId":"cua-capture-callback","content":[{"type":"text","text":"{\"dispatched\":true}"}],"isError":false,"id":"{{message:73}}"}},"sourceEventSeqs":[184],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":35}} {"type":"step/start","data":{"turn":1,"step":36}} -{"type":"assistant/message","data":{"turn":1,"step":36,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-recovered","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:74}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733007,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733007,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-recovered","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949733007,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733007,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":36,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-recovered","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:74}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890475,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890475,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-recovered","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890475,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890475,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":36,"callId":"cua-capture-recovered","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} {"type":"tool/result","data":{"turn":1,"step":36,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-capture-recovered"},"toolCallId":"cua-capture-recovered","content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"isError":false,"id":"{{message:75}}"}},"sourceEventSeqs":[189],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":36}} {"type":"step/start","data":{"turn":1,"step":37}} -{"type":"assistant/message","data":{"turn":1,"step":37,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-enter","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-9\",\"elementId\":\"0\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:76}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733041,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733041,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-enter","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-9\",\"elementId\":\"0\"}}"}}},{"type":"chunk","time":1793949733041,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733041,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":37,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-enter","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-9\",\"elementId\":\"0\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:76}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890537,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890537,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-enter","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-9\",\"elementId\":\"0\"}}"}}},{"type":"chunk","time":1794032890537,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890537,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":37,"callId":"cua-hover-enter","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-9\",\"elementId\":\"0\"}}"}} {"type":"tool/result","data":{"turn":1,"step":37,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-enter"},"toolCallId":"cua-hover-enter","content":[{"type":"text","text":"{\"dispatched\":true,\"hoverRoute\":\"accessibility\",\"systemPointerMoved\":false}"}],"isError":false,"id":"{{message:77}}"}},"sourceEventSeqs":[194],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":37}} {"type":"step/start","data":{"turn":1,"step":38}} -{"type":"assistant/message","data":{"turn":1,"step":38,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:78}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733060,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733060,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949733060,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733060,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":38,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:78}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890554,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890554,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890554,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890554,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":38,"callId":"cua-hover-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} {"type":"tool/result","data":{"turn":1,"step":38,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-observe"},"toolCallId":"cua-hover-observe","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-10\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 3\\\"\\n- AXStaticText \\\"Hover information\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:79}}"}},"sourceEventSeqs":[199],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":38}} {"type":"step/start","data":{"turn":1,"step":39}} -{"type":"assistant/message","data":{"turn":1,"step":39,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:80}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733087,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733087,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949733087,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733087,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":39,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:80}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890576,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890576,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890576,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890576,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":39,"callId":"cua-hover-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} {"type":"tool/result","data":{"turn":1,"step":39,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-capture"},"toolCallId":"cua-hover-capture","content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"isError":false,"id":"{{message:81}}"}},"sourceEventSeqs":[204],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":39}} {"type":"step/start","data":{"turn":1,"step":40}} -{"type":"assistant/message","data":{"turn":1,"step":40,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-popup","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:82}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733130,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733130,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-popup","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1793949733130,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733130,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":40,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-popup","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:82}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890612,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890612,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-popup","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1794032890612,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890612,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":40,"callId":"cua-hover-popup","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}} {"type":"tool/result","data":{"turn":1,"step":40,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-popup"},"toolCallId":"cua-hover-popup","content":[{"type":"text","text":"{\"dispatched\":true,\"hoverRoute\":\"accessibility\",\"systemPointerMoved\":false}"}],"isError":false,"id":"{{message:83}}"}},"sourceEventSeqs":[209],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":40}} {"type":"step/start","data":{"turn":1,"step":41}} -{"type":"assistant/message","data":{"turn":1,"step":41,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-10\",\"x\":100,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:84}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733147,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733147,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-10\",\"x\":100,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1793949733147,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733147,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":41,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-10\",\"x\":100,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:84}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890629,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890629,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-10\",\"x\":100,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1794032890629,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890629,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":41,"callId":"cua-hover-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-10\",\"x\":100,\"y\":200,\"relative\":true}}"}} {"type":"tool/result","data":{"turn":1,"step":41,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-click"},"toolCallId":"cua-hover-click","content":[{"type":"text","text":"{\"dispatched\":true}"}],"isError":false,"id":"{{message:85}}"}},"sourceEventSeqs":[214],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":41}} {"type":"step/start","data":{"turn":1,"step":42}} -{"type":"assistant/message","data":{"turn":1,"step":42,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-exit","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:86}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733164,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733164,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-exit","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1793949733164,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733164,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":42,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-exit","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:86}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890645,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890645,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-exit","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1794032890645,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890646,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":42,"callId":"cua-hover-exit","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"}} {"type":"tool/result","data":{"turn":1,"step":42,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-exit"},"toolCallId":"cua-hover-exit","content":[{"type":"text","text":"{\"dispatched\":true,\"hoverRoute\":\"accessibility\",\"systemPointerMoved\":false}"}],"isError":false,"id":"{{message:87}}"}},"sourceEventSeqs":[219],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":42}} {"type":"step/start","data":{"turn":1,"step":43}} -{"type":"assistant/message","data":{"turn":1,"step":43,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-unavailable","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:88}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733183,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733183,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-unavailable","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1793949733183,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733183,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":43,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-unavailable","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:88}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890662,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890662,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-unavailable","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1794032890662,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890662,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":43,"callId":"cua-hover-unavailable","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"}} {"type":"tool/result","data":{"turn":1,"step":43,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-unavailable"},"toolCallId":"cua-hover-unavailable","content":[{"type":"text","text":"Error: HOVER_UNAVAILABLE: No AX alternate UI at this target. No input was sent and the pointer did not move."}],"isError":true,"id":"{{message:89}}"}},"sourceEventSeqs":[224],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":43}} {"type":"step/start","data":{"turn":1,"step":44}} -{"type":"assistant/message","data":{"turn":1,"step":44,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:90}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733201,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733201,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949733201,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733201,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":44,"callId":"cua-hover-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":44,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-verify"},"toolCallId":"cua-hover-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-11\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:91}}"}},"sourceEventSeqs":[229],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":44,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-pointer-request","name":"cua","arguments":"{\"method\":\"requestMouseControl\",\"params\":{}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:90}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890679,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890679,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-pointer-request","name":"cua","arguments":"{\"method\":\"requestMouseControl\",\"params\":{}}"}}},{"type":"chunk","time":1794032890679,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890679,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":44,"callId":"cua-pointer-request","name":"cua","arguments":"{\"method\":\"requestMouseControl\",\"params\":{}}"}} +{"type":"tool/result","data":{"turn":1,"step":44,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-pointer-request"},"toolCallId":"cua-pointer-request","content":[{"type":"text","text":"true"}],"isError":false,"id":"{{message:91}}"}},"sourceEventSeqs":[229],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":44}} {"type":"step/start","data":{"turn":1,"step":45}} -{"type":"assistant/message","data":{"turn":1,"step":45,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-background","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F9\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:92}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733220,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733220,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-background","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F9\"}}"}}},{"type":"chunk","time":1793949733220,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733220,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":45,"callId":"cua-paste-background","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F9\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":45,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-background"},"toolCallId":"cua-paste-background","content":[{"type":"text","text":"{\"dispatched\":true}"}],"isError":false,"id":"{{message:93}}"}},"sourceEventSeqs":[234],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":45,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-system-hover","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:92}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890696,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890696,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-system-hover","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1794032890696,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890696,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":45,"callId":"cua-system-hover","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}} +{"type":"tool/result","data":{"turn":1,"step":45,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-system-hover"},"toolCallId":"cua-system-hover","content":[{"type":"text","text":"{\"dispatched\":true,\"hoverRoute\":\"system-pointer\",\"systemPointerMoved\":true}"}],"isError":false,"id":"{{message:93}}"}},"sourceEventSeqs":[234],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":45}} {"type":"step/start","data":{"turn":1,"step":46}} -{"type":"assistant/message","data":{"turn":1,"step":46,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-unread","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:94}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733238,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733238,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-unread","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}}},{"type":"chunk","time":1793949733238,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733238,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":46,"callId":"cua-paste-unread","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":46,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-unread"},"toolCallId":"cua-paste-unread","content":[{"type":"text","text":"Error: PASTE_NOT_CONSUMED: Paste command was dispatched, but clipboard data was not requested within the read deadline. Bring the target app to the foreground and focus its text input. Inspect the contents before deciding whether to retry; a delayed paste may still run."}],"isError":true,"id":"{{message:95}}"}},"sourceEventSeqs":[239],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":46,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:94}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890712,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890712,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890712,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890712,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":46,"callId":"cua-hover-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":46,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-verify"},"toolCallId":"cua-hover-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-11\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:95}}"}},"sourceEventSeqs":[239],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":46}} {"type":"step/start","data":{"turn":1,"step":47}} -{"type":"assistant/message","data":{"turn":1,"step":47,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-inspect","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:96}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733260,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733260,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-inspect","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949733260,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733260,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":47,"callId":"cua-paste-inspect","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":47,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-inspect"},"toolCallId":"cua-paste-inspect","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-12\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:97}}"}},"sourceEventSeqs":[244],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":47,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-background","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F9\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:96}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890729,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890729,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-background","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F9\"}}"}}},{"type":"chunk","time":1794032890729,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890730,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":47,"callId":"cua-paste-background","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F9\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":47,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-background"},"toolCallId":"cua-paste-background","content":[{"type":"text","text":"{\"dispatched\":true}"}],"isError":false,"id":"{{message:97}}"}},"sourceEventSeqs":[244],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":47}} {"type":"step/start","data":{"turn":1,"step":48}} -{"type":"assistant/message","data":{"turn":1,"step":48,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:98}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733280,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733280,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1793949733280,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733280,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":48,"callId":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":48,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-activate"},"toolCallId":"cua-paste-activate","content":[{"type":"text","text":"{\"dispatched\":true,\"foregroundChanged\":true,\"foreground\":{\"requestId\":\"cua-paste-activate\",\"targetPid\":9,\"before\":{\"frontPid\":42,\"focusedWindowId\":7,\"frontWindowId\":99,\"frontWindowPid\":42},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":true}}"}],"isError":false,"id":"{{message:99}}"}},"sourceEventSeqs":[249],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":48,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-unread","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:98}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890746,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890746,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-unread","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}}},{"type":"chunk","time":1794032890746,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890747,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":48,"callId":"cua-paste-unread","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":48,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-unread"},"toolCallId":"cua-paste-unread","content":[{"type":"text","text":"Error: PASTE_NOT_CONSUMED: Paste command was dispatched, but clipboard data was not requested within the read deadline. Bring the target app to the foreground and focus its text input. Inspect the contents before deciding whether to retry; a delayed paste may still run."}],"isError":true,"id":"{{message:99}}"}},"sourceEventSeqs":[249],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":48}} {"type":"step/start","data":{"turn":1,"step":49}} -{"type":"assistant/message","data":{"turn":1,"step":49,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:100}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733300,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733300,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949733300,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733300,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":49,"callId":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":49,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-focus"},"toolCallId":"cua-paste-focus","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-13\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:101}}"}},"sourceEventSeqs":[254],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":49,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-inspect","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:100}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890765,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890765,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-inspect","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890765,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890765,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":49,"callId":"cua-paste-inspect","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":49,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-inspect"},"toolCallId":"cua-paste-inspect","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-12\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:101}}"}},"sourceEventSeqs":[254],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":49}} {"type":"step/start","data":{"turn":1,"step":50}} -{"type":"assistant/message","data":{"turn":1,"step":50,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-consumed","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:102}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733320,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733320,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-consumed","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}}},{"type":"chunk","time":1793949733320,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733320,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":50,"callId":"cua-paste-consumed","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":50,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-consumed"},"toolCallId":"cua-paste-consumed","content":[{"type":"text","text":"{\"dispatched\":true}"}],"isError":false,"id":"{{message:103}}"}},"sourceEventSeqs":[259],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":50,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:102}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890781,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890781,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1794032890781,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890781,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":50,"callId":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":50,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-activate"},"toolCallId":"cua-paste-activate","content":[{"type":"text","text":"{\"dispatched\":true,\"foregroundChanged\":true,\"foreground\":{\"requestId\":\"cua-paste-activate\",\"targetPid\":9,\"before\":{\"frontPid\":42,\"focusedWindowId\":7,\"frontWindowId\":99,\"frontWindowPid\":42},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":true}}"}],"isError":false,"id":"{{message:103}}"}},"sourceEventSeqs":[259],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":50}} {"type":"step/start","data":{"turn":1,"step":51}} -{"type":"assistant/message","data":{"turn":1,"step":51,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:104}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733340,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733340,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949733340,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733340,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":51,"callId":"cua-paste-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":51,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-verify"},"toolCallId":"cua-paste-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-14\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"fixture paste\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:105}}"}},"sourceEventSeqs":[264],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":51,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:104}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890797,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890797,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890797,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890797,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":51,"callId":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":51,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-focus"},"toolCallId":"cua-paste-focus","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-13\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:105}}"}},"sourceEventSeqs":[264],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":51}} {"type":"step/start","data":{"turn":1,"step":52}} -{"type":"assistant/message","data":{"turn":1,"step":52,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-debug-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F12\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:106}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733361,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733361,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-debug-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F12\"}}"}}},{"type":"chunk","time":1793949733361,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733361,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":52,"callId":"cua-debug-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F12\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":52,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-debug-arm"},"toolCallId":"cua-debug-arm","content":[{"type":"text","text":"{\"dispatched\":true}"}],"isError":false,"id":"{{message:107}}"}},"sourceEventSeqs":[269],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":52,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-consumed","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:106}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890813,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890813,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-consumed","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}}},{"type":"chunk","time":1794032890813,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890813,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":52,"callId":"cua-paste-consumed","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":52,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-consumed"},"toolCallId":"cua-paste-consumed","content":[{"type":"text","text":"{\"dispatched\":true}"}],"isError":false,"id":"{{message:107}}"}},"sourceEventSeqs":[269],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":52}} {"type":"step/start","data":{"turn":1,"step":53}} -{"type":"assistant/message","data":{"turn":1,"step":53,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-debug-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:108}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733379,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733379,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-debug-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1793949733379,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733379,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":53,"callId":"cua-debug-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":53,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-debug-screenshot"},"toolCallId":"cua-debug-screenshot","content":[{"type":"text","text":"Error: NATIVE_PROTOCOL: Helper returned an invalid response; acquire fresh application handles\nScreenshot diagnostics:\n[cua-screenshot] {\"requestId\":\"cua-debug-screenshot\",\"stage\":\"capture-image.begin\",\"elapsedMs\":1}"}],"isError":true,"id":"{{message:109}}"}},"sourceEventSeqs":[274],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":53,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:108}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890830,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890830,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890830,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890830,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":53,"callId":"cua-paste-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":53,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-verify"},"toolCallId":"cua-paste-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-14\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"fixture paste\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:109}}"}},"sourceEventSeqs":[274],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":53}} {"type":"step/start","data":{"turn":1,"step":54}} -{"type":"assistant/message","data":{"turn":1,"step":54,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-consent-denied","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Denied Fixture\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:110}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733403,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733403,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-consent-denied","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Denied Fixture\"}}"}}},{"type":"chunk","time":1793949733403,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733403,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":54,"callId":"cua-consent-denied","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Denied Fixture\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":54,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-consent-denied"},"toolCallId":"cua-consent-denied","content":[{"type":"text","text":"Error: APP_ACCESS_DENIED: The user denied access to Denied Fixture. Do not retry or use another tool to bypass this decision."}],"isError":true,"id":"{{message:111}}"}},"sourceEventSeqs":[279],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":54,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-debug-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F12\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:110}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890846,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890846,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-debug-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F12\"}}"}}},{"type":"chunk","time":1794032890846,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890846,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":54,"callId":"cua-debug-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F12\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":54,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-debug-arm"},"toolCallId":"cua-debug-arm","content":[{"type":"text","text":"{\"dispatched\":true}"}],"isError":false,"id":"{{message:111}}"}},"sourceEventSeqs":[279],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":54}} {"type":"step/start","data":{"turn":1,"step":55}} -{"type":"assistant/message","data":{"turn":1,"step":55,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-consent-revocable","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Revocable Fixture\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:112}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733577,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733577,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-consent-revocable","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Revocable Fixture\"}}"}}},{"type":"chunk","time":1793949733577,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733577,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":55,"callId":"cua-consent-revocable","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Revocable Fixture\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":55,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-consent-revocable"},"toolCallId":"cua-consent-revocable","content":[{"type":"text","text":"{\"appHandle\":\"revoked-app\",\"name\":\"Revocable Fixture\",\"running\":true}"}],"isError":false,"id":"{{message:113}}"}},"sourceEventSeqs":[284],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":55,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-debug-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:112}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890862,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890862,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-debug-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890862,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890862,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":55,"callId":"cua-debug-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":55,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-debug-screenshot"},"toolCallId":"cua-debug-screenshot","content":[{"type":"text","text":"Error: NATIVE_PROTOCOL: Helper returned an invalid response; acquire fresh application handles\nScreenshot diagnostics:\n[cua-screenshot] {\"requestId\":\"cua-debug-screenshot\",\"stage\":\"capture-image.begin\",\"elapsedMs\":1}"}],"isError":true,"id":"{{message:113}}"}},"sourceEventSeqs":[284],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":55}} {"type":"step/start","data":{"turn":1,"step":56}} -{"type":"assistant/message","data":{"turn":1,"step":56,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-consent-revoked","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"revoked-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:114}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733745,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949733745,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-consent-revoked","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"revoked-app\"}}"}}},{"type":"chunk","time":1793949733745,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733745,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":56,"callId":"cua-consent-revoked","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"revoked-app\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":56,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-consent-revoked"},"toolCallId":"cua-consent-revoked","content":[{"type":"text","text":"Error: APP_ACCESS_REVOKED: Application access was revoked; stop using this application"}],"isError":true,"id":"{{message:115}}"}},"sourceEventSeqs":[289],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":56,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-consent-denied","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Denied Fixture\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:114}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890881,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890881,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-consent-denied","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Denied Fixture\"}}"}}},{"type":"chunk","time":1794032890881,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890881,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":56,"callId":"cua-consent-denied","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Denied Fixture\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":56,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-consent-denied"},"toolCallId":"cua-consent-denied","content":[{"type":"text","text":"Error: APP_ACCESS_DENIED: The user denied access to Denied Fixture. Do not retry or use another tool to bypass this decision."}],"isError":true,"id":"{{message:115}}"}},"sourceEventSeqs":[289],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":56}} {"type":"step/start","data":{"turn":1,"step":57}} -{"type":"assistant/message","data":{"turn":1,"step":57,"message":{"role":"assistant","content":[{"type":"text","text":"Observed Count: 4 after hovering, observing and clicking the revealed control. Moving away dismissed the hover information. Geometry conflicts rejected capture and coordinate gestures; the advertised AX button action remained available. No automatic activation or input retry was requested."}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:116}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949733763,"chunk":{"type":"block-start","index":0,"blockType":"text"}},{"type":"chunk","time":1793949733763,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"Observed Count: 4 after hovering, observing and clicking the revealed control. Moving away dismissed the hover information. Geometry conflicts rejected capture and coordinate gestures; the advertised AX button action remained available. No automatic activation or input retry was requested."}}},{"type":"chunk","time":1793949733763,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949733763,"chunk":{"type":"finish","reason":{"kind":"stop"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":57,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-consent-revocable","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Revocable Fixture\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:116}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032891018,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032891018,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-consent-revocable","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Revocable Fixture\"}}"}}},{"type":"chunk","time":1794032891018,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032891018,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":57,"callId":"cua-consent-revocable","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Revocable Fixture\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":57,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-consent-revocable"},"toolCallId":"cua-consent-revocable","content":[{"type":"text","text":"{\"appHandle\":\"revoked-app\",\"name\":\"Revocable Fixture\",\"running\":true}"}],"isError":false,"id":"{{message:117}}"}},"sourceEventSeqs":[294],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":57}} +{"type":"step/start","data":{"turn":1,"step":58}} +{"type":"assistant/message","data":{"turn":1,"step":58,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-consent-revoked","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"revoked-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:118}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032891165,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032891165,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-consent-revoked","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"revoked-app\"}}"}}},{"type":"chunk","time":1794032891165,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032891165,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":58,"callId":"cua-consent-revoked","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"revoked-app\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":58,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-consent-revoked"},"toolCallId":"cua-consent-revoked","content":[{"type":"text","text":"Error: APP_ACCESS_REVOKED: Application access was revoked; stop using this application"}],"isError":true,"id":"{{message:119}}"}},"sourceEventSeqs":[299],"surfaceOp":"append"} +{"type":"step/end","data":{"turn":1,"step":58}} +{"type":"step/start","data":{"turn":1,"step":59}} +{"type":"assistant/message","data":{"turn":1,"step":59,"message":{"role":"assistant","content":[{"type":"text","text":"Observed Count: 4 after hovering, observing and clicking the revealed control. Moving away dismissed the hover information. Geometry conflicts rejected capture and coordinate gestures; the advertised AX button action remained available. No automatic activation or input retry was requested."}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:120}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032891180,"chunk":{"type":"block-start","index":0,"blockType":"text"}},{"type":"chunk","time":1794032891180,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"Observed Count: 4 after hovering, observing and clicking the revealed control. Moving away dismissed the hover information. Geometry conflicts rejected capture and coordinate gestures; the advertised AX button action remained available. No automatic activation or input retry was requested."}}},{"type":"chunk","time":1794032891180,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032891180,"chunk":{"type":"finish","reason":{"kind":"stop"}}}]},"surfaceOp":"append"} +{"type":"step/end","data":{"turn":1,"step":59}} {"type":"turn/end","data":{"turn":1,"reason":{"kind":"completed"}}} diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index 0b9c1b29f6..e7132c8c0f 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -43,6 +43,7 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. +- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Returns true only on approval. Avoid asking unless the task requires a physical hover. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -67,7 +68,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only systemPointerHover: true permits moving the visible cursor. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured. +- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, mouseMove uses the system pointer for this session until DSH restarts. If the user declines, continue without mouse control. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/snapshots/session/cua-native/tool-schemas.expected.json b/snapshots/session/cua-native/tool-schemas.expected.json index 33f3636750..ed8a14779d 100644 --- a/snapshots/session/cua-native/tool-schemas.expected.json +++ b/snapshots/session/cua-native/tool-schemas.expected.json @@ -1413,6 +1413,25 @@ "params" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "method": { + "type": "string", + "const": "requestMouseControl" + }, + "params": { + "type": "object", + "properties": {}, + "additionalProperties": false + } + }, + "required": [ + "method", + "params" + ], + "additionalProperties": false } ], "$defs": { diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index 587436227a..45154f9179 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -45,6 +45,7 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. +- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Returns true only on approval. Avoid asking unless the task requires a physical hover. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -69,7 +70,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only systemPointerHover: true permits moving the visible cursor. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured. +- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, mouseMove uses the system pointer for this session until DSH restarts. If the user declines, continue without mouse control. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/snapshots/session/ptc-script-retained/system-prompt.expected.md b/snapshots/session/ptc-script-retained/system-prompt.expected.md index dd88fa8d9e..b1769c75dc 100644 --- a/snapshots/session/ptc-script-retained/system-prompt.expected.md +++ b/snapshots/session/ptc-script-retained/system-prompt.expected.md @@ -45,6 +45,7 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. +- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Returns true only on approval. Avoid asking unless the task requires a physical hover. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -69,7 +70,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only systemPointerHover: true permits moving the visible cursor. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured. +- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, mouseMove uses the system pointer for this session until DSH restarts. If the user declines, continue without mouse control. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. From 409b0b1ab9be8466f954bec85f1265e786175fb1 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 14:33:30 +0800 Subject: [PATCH 03/30] fix(cua): keep physical hover specific to supporting providers --- .../2026-10-13-cua-continuous-hover.i18n.yaml | 6 ++--- .../2026-10-13-cua-continuous-hover.md | 4 ++-- .../2026-10-13-cua-continuous-hover.zh.md | 2 +- ...cua-hover-routes-and-coordinates.i18n.yaml | 4 ++-- ...-10-13-cua-hover-routes-and-coordinates.md | 2 +- ...-13-cua-hover-routes-and-coordinates.zh.md | 2 +- .../tests/provider-lifecycle.spec.ts | 23 ++++++++++++++++++- packages/experimental/cua/src/index.ts | 6 ++--- 8 files changed, 35 insertions(+), 14 deletions(-) diff --git a/.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.i18n.yaml b/.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.i18n.yaml index 04a78f70d3..1dbc38b5ae 100644 --- a/.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.i18n.yaml @@ -9,10 +9,10 @@ en: cdc04139c8d69415 zh: a3fe3e78ada08d95 /agent-note-continuous-cua-hover-without-cross-call-ownership/decision: - en: 125c860c1c568960 - zh: 8a2ece1101358200 + en: 193acf8c23672457 + zh: e614884d52585b16 /agent-note-continuous-cua-hover-without-cross-call-ownership/alternatives-considered: - en: 18d917ee2e909b93 + en: df67e547bce2f289 zh: 5b5ad0530f138cda /agent-note-continuous-cua-hover-without-cross-call-ownership/consequences: en: 714789d0a472a308 diff --git a/.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.md b/.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.md index 2cce428cff..29adedf224 100644 --- a/.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.md +++ b/.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.md @@ -10,7 +10,7 @@ Hover interactions span movement, delayed content, observation and input inside ## Decision -An explicit Window exposes `mouseMove` for coordinates or an observed element ID. The result reports `hoverRoute` and `systemPointerMoved`. It never presses a mouse button; disabled controls can use system-pointer movement with `systemPointerHover` enabled. Completion, AX reads and screenshots do not restore a previous pointer position. Successive calls can visit intermediate points before entering a popup. Clicks deliver button events at their target without an off-window priming click. The [input-routing decision](2026-10-13-cua-hover-routes-and-coordinates.md) selects AX alternate UI before allowing opt-in system movement with foreground and mouse-hit checks. It never activates or raises the app implicitly. Other input or window changes can race that check. +An explicit Window exposes `mouseMove` for coordinates or an observed element ID. The result reports `hoverRoute` and `systemPointerMoved`. It never presses a mouse button. With session approval, system-pointer movement also works on disabled controls. Completion, AX reads and screenshots do not restore a previous pointer position. Successive calls can visit intermediate points before entering a popup. Clicks deliver button events at their target without an off-window priming click. The [input-routing decision](2026-10-13-cua-hover-routes-and-coordinates.md) requires foreground and mouse-hit checks for system movement. It never activates or raises the app implicitly. Other input or window changes can race that check. The [shared helper](2026-10-09-cua-shared-native-helper.md) serializes complete individual operations. It grants no application ownership across calls. Earlier arrival, a retained app object and Session lifetime do not establish task priority. Waiting for the model does not reserve the application. Explicit task coordination determines any required execution order. @@ -20,7 +20,7 @@ The [shared helper](2026-10-09-cua-shared-native-helper.md) serializes complete **Click or drag as hover** introduces button input that can activate a control, select text or dismiss a popup. Hover requires a separate unpressed movement operation. -**PID-directed virtual movement** can display Chromium hover content after prior interaction, but a fresh window can ignore it, and later pointer checks can dismiss the content. Native AppKit tracking also requires real pointer movement in the measured fixture. The system-pointer fallback avoids reporting this unreliable event route as continuous hover. +**PID-directed virtual movement** can display Chromium hover content after prior interaction, but a fresh window can ignore it, and later pointer checks can dismiss the content. Native AppKit tracking also requires real pointer movement in the measured fixture. The approved system-pointer route avoids reporting this unreliable event route as continuous hover. ## Consequences diff --git a/.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.zh.md b/.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.zh.md index cc2457a70a..22668bd215 100644 --- a/.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.zh.md +++ b/.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.zh.md @@ -10,7 +10,7 @@ Status: implemented ## 决策 -显式窗口提供 `mouseMove`，接受窗口坐标或已观察元素的字符串 ID，结果用 `hoverRoute` 和 `systemPointerMoved` 报告执行路径。悬停不按下鼠标按钮，启用 `systemPointerHover` 后可通过系统鼠标悬停禁用控件。调用结束、AX 读取和截图不恢复先前的鼠标位置。进入浮层前可连续调用并经过中间位置。点击只在目标位置发送按键事件，不先在窗口外点击。 [输入路径决定](2026-10-13-cua-hover-routes-and-coordinates.zh.md)优先选择 AX 替代界面，只有启用该配置时才回退到经过前台和鼠标命中检查的系统移动，不隐式激活或抬起应用。其他输入或窗口变化仍可能与检查交错。 +显式窗口提供 `mouseMove`，接受窗口坐标或已观察元素的字符串 ID，结果用 `hoverRoute` 和 `systemPointerMoved` 报告执行路径。悬停不按下鼠标按钮，用户授权当前会话移动鼠标后，也可悬停在禁用的控件上。调用结束、AX 读取和截图不恢复先前的鼠标位置。进入浮层前可连续调用并经过中间位置。点击只在目标位置发送按键事件，不先在窗口外点击。 [输入路径决定](2026-10-13-cua-hover-routes-and-coordinates.zh.md)要求移动系统鼠标前检查前台应用和鼠标命中的窗口，不隐式激活或抬起应用。其他输入或窗口变化仍可能与检查交错。 [共享 helper](2026-10-09-cua-shared-native-helper.zh.md)串行执行完整的单次操作，不分配跨调用的应用所有权。先发起请求、持有 app 对象和 Session 寿命均不代表任务优先级。等待模型期间不占用应用。需要确定执行顺序时，由明确的任务协调决定。 diff --git a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.i18n.yaml b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.i18n.yaml index 9775baf3a4..f6546d2ce8 100644 --- a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.i18n.yaml @@ -9,8 +9,8 @@ en: f96bf1ebc0719fb0 zh: e9a1144f0e76572d /agent-note-background-hover-and-screenshot-coordinates/decision: - en: bdf4d3386521ce00 - zh: 96db93177ad96a76 + en: 9e06d78eba10770d + zh: 25e985aa49f75dd8 /agent-note-background-hover-and-screenshot-coordinates/alternatives-considered: en: 5fc4beaf79761c50 zh: 4948333b0020be28 diff --git a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.md b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.md index c385525731..a1f0a7fac1 100644 --- a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.md +++ b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.md @@ -10,7 +10,7 @@ Hover-only controls can sometimes be revealed without moving the user's pointer. ## Decision -Native hover first uses a same-window element or ancestor advertising both `AXShowAlternateUI` and `AXShowDefaultUI`. Entering its revealed descendants preserves the parent UI; leaving restores it. An unavailable route returns `HOVER_UNAVAILABLE` by default. Enabling `systemPointerHover` permits the real system pointer fallback. An AX action failure propagates because received effects may be uncertain. The result records the route and whether the system pointer moved. System movement requires the selected foreground app and AppKit's mouse hit test, which excludes input-transparent overlays. No route grants [cross-call ownership](2026-10-13-cua-continuous-hover.md). +By default, native hover uses a same-window element or ancestor advertising both `AXShowAlternateUI` and `AXShowDefaultUI`. Entering its revealed descendants preserves the parent UI; leaving restores it. An unavailable route returns `HOVER_UNAVAILABLE` by default. After the user approves `cua.requestMouseControl()`, each `mouseMove` in that session uses the system pointer until DSH restarts. An AX action failure propagates because received effects may be uncertain. The result records the route and whether the system pointer moved. System movement requires the selected foreground app and AppKit's mouse hit test, which excludes input-transparent overlays. No route grants [cross-call ownership](2026-10-13-cua-continuous-hover.md). Model-facing coordinate input defaults to pixels in the returned screenshot. `relative: true` accepts integer thousandths from 0 to 999 for `click`, `mouseMove`, `scroll` and `drag`; it does not apply to opaque element IDs. The Host owns the mapping to native window points. An application-wide observation refreshes its inventory; a scoped observation preserves other windows' image mappings. Window resizing invalidates the old image, while movement preserves local coordinates. Relative endpoints remain inside the window and do not depend on screenshot resolution. diff --git a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.zh.md b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.zh.md index 0d3db32400..f079ff736f 100644 --- a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.zh.md +++ b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.zh.md @@ -10,7 +10,7 @@ Status: implemented ## 决策 -原生悬停优先寻找同一窗口内同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI` 的元素或祖先。进入其展开区域的子控件时保持父控件展开，离开后恢复。没有可用路径时默认返回 `HOVER_UNAVAILABLE`。启用 `systemPointerHover` 后才允许移动系统光标。AX 动作失败直接返回，因为已产生的效果可能不确定。结果记录实际路径及是否移动系统鼠标。系统移动要求所选应用处于前台，并使用 AppKit 的鼠标命中查询排除不接收输入的覆盖窗口。所有路径都遵守[跨调用不分配所有权](2026-10-13-cua-continuous-hover.zh.md)的决定。 +默认情况下，原生悬停会寻找同一窗口内同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI` 的元素或祖先。进入其展开区域的子控件时保持父控件展开，离开后恢复。没有可用路径时默认返回 `HOVER_UNAVAILABLE`。用户同意 `cua.requestMouseControl()` 后，本会话的每次 `mouseMove` 都使用系统光标，重启 DSH 后授权失效。AX 动作失败直接返回，因为已产生的效果可能不确定。结果记录实际路径及是否移动系统鼠标。系统移动要求所选应用处于前台，并使用 AppKit 的鼠标命中查询排除不接收输入的覆盖窗口。所有路径都遵守[跨调用不分配所有权](2026-10-13-cua-continuous-hover.zh.md)的决定。 面向模型的坐标默认使用返回截图中的像素。`click`、`mouseMove`、`scroll` 和 `drag` 传入 `relative: true` 时接受 0～999 的千分比整数，仅用于坐标目标。Host 负责换算原生窗口 point。应用级观察刷新窗口清单，单窗口观察保留其他窗口的图片映射。窗口尺寸变化使旧图片失效，移动窗口保留局部坐标。相对坐标的端点保持在窗口内，不随截图分辨率改变。 diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index f975862baa..922078987a 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -17,7 +17,7 @@ import * as StorageDomain from '@deepseek-ai/dsh-storage-domain' import Hmr from '@deepseek-ai/dsh-hmr' import { mountAgentLoopTestDependencies, mountAgentLoopTestHarness } from '@deepseek-ai/dsh-agent-loop-testkit' import ComputerUseRegistry from '@deepseek-ai/dsh-computer-use' -import { CuaConfig } from '@deepseek-ai/dsh-cua' +import { CuaConfig, installCua } from '@deepseek-ai/dsh-cua' import { createConfigProjector } from '@deepseek-ai/dsh-app-boot/src/config-schema/projector.ts' import LocalAttachmentStore from '@deepseek-ai/dsh-attachment-local' import FileSystem from '@deepseek-ai/dsh-fs-local' @@ -927,8 +927,10 @@ it('drops late approval after cancellation and admits only one pending question' expect((await f.execute('requestMouseControl')).isError).toBe(true) await f.ctx.plugin(UserQuestions) const entered = Promise.withResolvers() + let count = 0 const release = Promise.withResolvers() f.ctx.on('user-questions/request', async () => { + count++ entered.resolve(undefined) await release.promise return { answers: [{ id: 'cua-mouse-control', selected: ['Allow for this session'] }] } @@ -941,5 +943,24 @@ it('drops late approval after cancellation and admits only one pending question' release.resolve(undefined) await Promise.allSettled([first]) expect((await f.execute('requestMouseControl')).value).toBe(true) + expect(count).toBe(2) expect(native.request).not.toHaveBeenCalled() }) + +it('keeps provider-specific pointer flags and questions off providers without mouse control', async () => { + const f = await setup() + await f.provider.dispose() + await f.ctx.plugin({ name: 'fixture-window-provider', inject: NativeProvider.inject, apply(ctx: Context) { + installCua(ctx, CuaConfig(config), { + helperPath: config.helperPath, createChannel() { throw new Error('Unexpected native channel') }, + guidance: '', permissionParameters: () => ({}), + capabilities: { platform: 'fixture', methods: ['getApp', 'mouseMove'], appLevelAccessibility: true, + keyboard: ['app'], coordinateSpace: 'screenshot-pixels', browser: false }, + }) + } }) + expect((await f.execute('requestMouseControl')).isError).toBe(true) + await selectApplication(f) + const target = { appHandle: 'app-one', snapshotId: 'snapshot', elementId: '0' } + expect((await f.execute('mouseMove', target)).isError).toBe(false) + expect(native.request).toHaveBeenLastCalledWith('mouseMove', target, expect.any(AbortSignal), expect.any(String)) +}) diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index 40cb0bf1cd..763e7bc0ec 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -157,7 +157,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) permissionParameters: backend.permissionParameters, store: new CuaPermissionStore(ctx, config.allowedApps.map(grant => ({ ...grant, id: grant.id as CuaApplicationId })), config.timeoutMs), }) - const mouseControl = new MouseControl(ctx) + const mouseControl = backend.capabilities.methods.includes('requestMouseControl') ? new MouseControl(ctx) : undefined const clients = new Map() const lifetime = new AbortController() const pending = new Map, Agent>() @@ -222,7 +222,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) ? { ...params, maxElements: config.maxElements, maxDepth: config.maxDepth } : method === 'drag' ? { ...params, durationMs: config.dragDurationMs, steps: config.dragSteps } : params - if (method === 'mouseMove') nativeParams.useSystemPointer = mouseControl.allowed(agent) + if (method === 'mouseMove' && mouseControl) nativeParams.useSystemPointer = mouseControl.allowed(agent) const requestNative = () => client.request(method, nativeParams, signal, execution.callId) if (method === 'screenshot') { const { attachments, target } = await screenshotStore(ctx, execution) @@ -310,7 +310,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) signal.throwIfAborted() const operation = (async () => { if (request.method === 'requestMouseControl') { - if (!backend.capabilities.methods.includes(request.method)) throw new NativeError('UNSUPPORTED_OPERATION', 'This provider does not support system mouse control') + if (!mouseControl) throw new NativeError('UNSUPPORTED_OPERATION', 'This provider does not support system mouse control') return mouseControl.request(agent, signal) } const previous = clients.get(agent) From 3af5496e14cb9b6b99d1b1e47faa46e0cd1f45df Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 16:04:36 +0800 Subject: [PATCH 04/30] fix(cua): reserve mouse permission requests for the main task --- ...cua-hover-routes-and-coordinates.i18n.yaml | 4 +- ...-10-13-cua-hover-routes-and-coordinates.md | 2 +- ...-13-cua-hover-routes-and-coordinates.zh.md | 2 +- .../cua-hover-ax-only/guide.i18n.yaml | 8 +-- .../v0.2.2-rc.1/cua-hover-ax-only/guide.md | 4 +- .../v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md | 4 +- packages/experimental/cua-macos/README.md | 4 +- packages/experimental/cua-macos/README.zh.md | 4 +- .../native/Sources/CuaNative/AXWorker.swift | 4 -- .../cua-macos/native/tests/Fixture.swift | 3 + .../cua-macos/native/tests/e2e.py | 12 +++- .../experimental/cua-macos/src/guidance.ts | 2 +- .../tests/provider-lifecycle.spec.ts | 57 ++++++++++++++++--- packages/experimental/cua/src/guidance.ts | 2 +- packages/experimental/cua/src/locales.ts | 16 +++--- .../experimental/cua/src/mouse-control.ts | 14 +++-- packages/experimental/cua/src/types.ts | 5 +- 17 files changed, 99 insertions(+), 48 deletions(-) diff --git a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.i18n.yaml b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.i18n.yaml index f6546d2ce8..7681ea3630 100644 --- a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.i18n.yaml @@ -9,8 +9,8 @@ en: f96bf1ebc0719fb0 zh: e9a1144f0e76572d /agent-note-background-hover-and-screenshot-coordinates/decision: - en: 9e06d78eba10770d - zh: 25e985aa49f75dd8 + en: 93631d45c7ecfb4c + zh: 77a08aafb1115ba1 /agent-note-background-hover-and-screenshot-coordinates/alternatives-considered: en: 5fc4beaf79761c50 zh: 4948333b0020be28 diff --git a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.md b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.md index a1f0a7fac1..1f5c18ce74 100644 --- a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.md +++ b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.md @@ -10,7 +10,7 @@ Hover-only controls can sometimes be revealed without moving the user's pointer. ## Decision -By default, native hover uses a same-window element or ancestor advertising both `AXShowAlternateUI` and `AXShowDefaultUI`. Entering its revealed descendants preserves the parent UI; leaving restores it. An unavailable route returns `HOVER_UNAVAILABLE` by default. After the user approves `cua.requestMouseControl()`, each `mouseMove` in that session uses the system pointer until DSH restarts. An AX action failure propagates because received effects may be uncertain. The result records the route and whether the system pointer moved. System movement requires the selected foreground app and AppKit's mouse hit test, which excludes input-transparent overlays. No route grants [cross-call ownership](2026-10-13-cua-continuous-hover.md). +By default, native hover uses a same-window element or ancestor advertising both `AXShowAlternateUI` and `AXShowDefaultUI`. Entering its revealed descendants preserves the parent UI; leaving restores it. An unavailable route returns `HOVER_UNAVAILABLE` by default. After the user approves `cua.requestMouseControl()`, each `mouseMove` in that session uses the system pointer until DSH restarts. This lets a caller reach pointer-tracking controls even when the same target also advertises AX alternate UI. The Host passes the session decision with each move. Physical movement leaves the visibility of previously revealed controls to the application, so switching routes cannot dismiss a target before dispatch. An AX action failure propagates because received effects may be uncertain. The result records the route and whether the system pointer moved. System movement requires the selected foreground app and AppKit's mouse hit test, which excludes input-transparent overlays. No route grants [cross-call ownership](2026-10-13-cua-continuous-hover.md). Model-facing coordinate input defaults to pixels in the returned screenshot. `relative: true` accepts integer thousandths from 0 to 999 for `click`, `mouseMove`, `scroll` and `drag`; it does not apply to opaque element IDs. The Host owns the mapping to native window points. An application-wide observation refreshes its inventory; a scoped observation preserves other windows' image mappings. Window resizing invalidates the old image, while movement preserves local coordinates. Relative endpoints remain inside the window and do not depend on screenshot resolution. diff --git a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.zh.md b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.zh.md index f079ff736f..d1a14ee9e4 100644 --- a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.zh.md +++ b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.zh.md @@ -10,7 +10,7 @@ Status: implemented ## 决策 -默认情况下，原生悬停会寻找同一窗口内同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI` 的元素或祖先。进入其展开区域的子控件时保持父控件展开，离开后恢复。没有可用路径时默认返回 `HOVER_UNAVAILABLE`。用户同意 `cua.requestMouseControl()` 后，本会话的每次 `mouseMove` 都使用系统光标，重启 DSH 后授权失效。AX 动作失败直接返回，因为已产生的效果可能不确定。结果记录实际路径及是否移动系统鼠标。系统移动要求所选应用处于前台，并使用 AppKit 的鼠标命中查询排除不接收输入的覆盖窗口。所有路径都遵守[跨调用不分配所有权](2026-10-13-cua-continuous-hover.zh.md)的决定。 +默认情况下，原生悬停会寻找同一窗口内同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI` 的元素或祖先。进入其展开区域的子控件时保持父控件展开，离开后恢复。没有可用路径时默认返回 `HOVER_UNAVAILABLE`。用户同意 `cua.requestMouseControl()` 后，本会话的每次 `mouseMove` 都使用系统光标，重启 DSH 后授权失效。有些控件既支持 AX 备用界面，又跟踪真实鼠标。Host 在每次悬停请求中传入本会话的授权结果，让调用方能够触发后者。切换到系统鼠标后，由应用决定是否收起先前展开的控件，以免派发前目标消失。AX 动作失败直接返回，因为已产生的效果可能不确定。结果记录实际路径及是否移动系统鼠标。系统移动要求所选应用处于前台，并使用 AppKit 的鼠标命中查询排除不接收输入的覆盖窗口。所有路径都遵守[跨调用不分配所有权](2026-10-13-cua-continuous-hover.zh.md)的决定。 面向模型的坐标默认使用返回截图中的像素。`click`、`mouseMove`、`scroll` 和 `drag` 传入 `relative: true` 时接受 0～999 的千分比整数，仅用于坐标目标。Host 负责换算原生窗口 point。应用级观察刷新窗口清单，单窗口观察保留其他窗口的图片映射。窗口尺寸变化使旧图片失效，移动窗口保留局部坐标。相对坐标的端点保持在窗口内，不随截图分辨率改变。 diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.i18n.yaml b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.i18n.yaml index cbce06f055..2f886f19df 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.i18n.yaml +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.i18n.yaml @@ -9,8 +9,8 @@ en: 6833a7851486f10d zh: 22983f326fb725ce /macos-hover-keeps-the-system-pointer-in-place-by-default/change: - en: 55c3070b24b4ad39 - zh: 2555d127b0f4e4a0 + en: 2369a7daf049311a + zh: 2ad1bfb579be4426 /macos-hover-keeps-the-system-pointer-in-place-by-default/migration: - en: bf3f886ad344f12a - zh: c877a616d4e147fb + en: 7c89c67c9acefd25 + zh: 54a0064c30af043c diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.md index a7d03dc869..733a8a7061 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.md @@ -9,10 +9,10 @@ English | [中文](guide.zh.md) ## Change -On macOS, `win.mouseMove(target)` uses an enabled element or ancestor that advertises both `AXShowAlternateUI` and `AXShowDefaultUI`. Without that route, previous versions moved the visible system cursor. The default now returns `HOVER_UNAVAILABLE` before sending input. +On macOS, `win.mouseMove(target)` uses an enabled element or ancestor that advertises both `AXShowAlternateUI` and `AXShowDefaultUI`. Without that route, previous versions moved the visible system cursor. The default now returns `HOVER_UNAVAILABLE` before sending input. Moving the pointer requires the user’s approval for the current session. ## Migration 1. For an available control, use its advertised action or click it directly. Use browser tools for web content that requires page hover. -2. Remove `systemPointerHover` from the macOS provider entry in your profile’s `cordis.patch.yml`. If you set `helperPath`, rebuild that helper to support the per-request `useSystemPointer` field. Restart the profile. +2. If your profile sets `helperPath`, rebuild the custom helper to support the per-request `useSystemPointer` field and restart the profile. 3. When the task requires a physical hover, call `cua.requestMouseControl()`. After the user approves, `win.mouseMove` on an unobscured foreground target returns `hoverRoute: "system-pointer"` and `systemPointerMoved: true`. This grant expires when DSH restarts. diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md index 17064261b3..84ac6cbe2b 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md @@ -9,10 +9,10 @@ description: "macOS 的 mouseMove 默认要求控件支持 AX 备用界面，否 ## 变更 -在 macOS 上，`win.mouseMove(target)` 会查找已启用且同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI` 的控件或祖先。找不到时，旧版会移动系统光标，新版默认在发送输入前返回 `HOVER_UNAVAILABLE`。 +在 macOS 上，`win.mouseMove(target)` 会查找已启用且同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI` 的控件或祖先。找不到时，旧版会移动系统光标，新版默认在发送输入前返回 `HOVER_UNAVAILABLE`。移动系统光标需要用户单独授权当前会话。 ## 迁移 1. 对已经可用的控件，可执行它声明的动作或直接点击。需要网页悬停时可用浏览器工具。 -2. 从 profile 的 `cordis.patch.yml` 中移除 macOS 提供方配置项 `systemPointerHover`。如果设置了 `helperPath`，请重新构建该 helper，使其支持按请求读取 `useSystemPointer` 字段，然后重启 profile。 +2. 如果 profile 设置了 `helperPath`，请重新构建该 helper，使其支持按请求读取 `useSystemPointer` 字段，然后重启 profile。 3. 任务确实需要移动鼠标时，调用 `cua.requestMouseControl()`。用户同意后，对前台未受遮挡的目标调用 `win.mouseMove`，结果为 `hoverRoute: "system-pointer"` 和 `systemPointerMoved: true`。重启 DSH 后授权失效。 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 3190f8561d..8e269f5529 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -185,7 +185,7 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. -- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Returns true only on approval. Avoid asking unless the task requires a physical hover. +- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Only the main task may ask, and only when a physical hover is necessary. A false result is a refusal for this session until restart. If the user dismisses the request, continue without mouse control unless they ask to reconsider. Await the outstanding request before calling again. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -210,7 +210,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, mouseMove uses the system pointer for this session until DSH restarts. If the user declines, continue without mouse control. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured. +- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, every mouseMove requires the target app to be in front and the point unobscured, including targets that previously supported background AX hover. If mouseMove returns FOREGROUND_REQUIRED, explain the necessary foreground step before activating the app, then observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index 87c9fb046f..e68fa5b402 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -185,7 +185,7 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. -- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Returns true only on approval. Avoid asking unless the task requires a physical hover. +- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Only the main task may ask, and only when a physical hover is necessary. A false result is a refusal for this session until restart. If the user dismisses the request, continue without mouse control unless they ask to reconsider. Await the outstanding request before calling again. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -210,7 +210,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, mouseMove uses the system pointer for this session until DSH restarts. If the user declines, continue without mouse control. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured. +- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, every mouseMove requires the target app to be in front and the point unobscured, including targets that previously supported background AX hover. If mouseMove returns FOREGROUND_REQUIRED, explain the necessary foreground step before activating the app, then observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift index c44b4078af..7602eb30d4 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift @@ -391,10 +391,6 @@ actor AXWorker { if let target = hit, try ax().windowID(target) != window.windowID { hit = nil } return try moveUsingAX(hit, windowID: window.windowID, handle: handle) } - if let hover = hovers[handle] { - _ = try hover.move(to: nil, in: window.windowID, access: ax()) - hovers.removeValue(forKey: handle) - } return .mouseMove(route, point) } var clickRoute = route diff --git a/packages/experimental/cua-macos/native/tests/Fixture.swift b/packages/experimental/cua-macos/native/tests/Fixture.swift index 930f7df5d2..d7f5bc21a8 100644 --- a/packages/experimental/cua-macos/native/tests/Fixture.swift +++ b/packages/experimental/cua-macos/native/tests/Fixture.swift @@ -37,6 +37,7 @@ class DragPad: NSView { } } final class AlternateHoverPad: DragPad { + private(set) var defaultUIRequests = 0 override func isAccessibilityElement() -> Bool { true } override func accessibilityRole() -> NSAccessibility.Role? { .group } override func accessibilityLabel() -> String? { "Alternate hover controls" } @@ -44,6 +45,7 @@ final class AlternateHoverPad: DragPad { hoverButton.isHidden = false; changed?(); return true } override func accessibilityPerformShowDefaultUI() -> Bool { + defaultUIRequests += 1 hoverButton.isHidden = true; changed?(); return true } } @@ -212,6 +214,7 @@ final class Fixture: NSObject, NSApplicationDelegate, NSTextFieldDelegate { "rowCount": rowCount, "primaryWindowId": window.windowNumber, "dragDownCount": dragPad.dragDownCount, "dragCount": dragPad.dragCount, "dragPoint": [dragPad.lastPoint.x, dragPad.lastPoint.y], "hoverVisible": !dragPad.hoverButton.isHidden, "hoverClicks": dragPad.hoverClicks, + "defaultUIRequests": (dragPad as? AlternateHoverPad)?.defaultUIRequests ?? 0, "cursorSamples": cursorSamples, "cursorChanges": cursorChanges, "overlayMode": overlayMode, "overlayWindowId": inputOverlay?.windowNumber ?? 0, "overlayOnScreen": overlayOnScreen, "hoverHitWindow": hoverHitWindow, diff --git a/packages/experimental/cua-macos/native/tests/e2e.py b/packages/experimental/cua-macos/native/tests/e2e.py index c5b385189a..5eeeaac59b 100644 --- a/packages/experimental/cua-macos/native/tests/e2e.py +++ b/packages/experimental/cua-macos/native/tests/e2e.py @@ -220,7 +220,7 @@ def main(): return result def check_hover(): - if not args.alternate_hover: + if not args.alternate_hover or args.input_overlays: call("activate", owner) current = call("getState", primary_params) window_element = next(element for element in current["elements"] if element["role"] == "AXWindow") @@ -275,7 +275,17 @@ def main(): after = wait_state(lambda actual: actual["cursorSamples"] >= before["cursorSamples"] + 2, "Pointer observation did not advance") assert after["cursorChanges"] == before["cursorChanges"], "Rejected hover moved the system pointer" + if args.alternate_hover: + assert before["hoverVisible"] and after["hoverVisible"], "Rejected physical hover dismissed AX controls" + assert after["defaultUIRequests"] == before["defaultUIRequests"], after cycle_overlay(0) + if args.alternate_hover: + state = call("getState", primary_params) + child = button_target(state, "Use native hover details") + result = call("mouseMove", {**child, "windowId": primary["windowId"], "useSystemPointer": True}) + assert result["hoverRoute"] == "system-pointer" and result["systemPointerMoved"], result + current = json.loads(state_path.read_text()) + assert current["hoverVisible"] and current["defaultUIRequests"] == before["defaultUIRequests"], current mouse_move({**owner, "windowId": primary["windowId"], "x": 10, "y": 40}) wait_state(lambda actual: not actual["hoverVisible"], "Hover did not exit after removing the overlay") diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index 5a340f045f..85272e3371 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -5,7 +5,7 @@ import { COMMON_GUIDANCE } from '@deepseek-ai/dsh-cua/guidance' export const PLATFORM_GUIDANCE = 'macOS: accessibility tree operations are available at app or window level. Simulated keyboard input is available at app level. Simulated mouse input is available at window level. For pointer input on a sheet, select its window ID with app.getWindow(id). For menu items outside a window, use app.invoke(id). findApps searches /Applications, /System/Applications, and ~/Applications. Use the Computer Use permission page to grant the helper application Accessibility and Screen Recording permissions in System Settings. Native window and AX bounds use macOS points, while delivered screenshots carry their own pixel dimensions. show does not switch Spaces, and Stage Manager can present reduced window geometry; explain any required foreground step before calling app.activate() or performing an advertised AXRaise action.' const methodDetails = new Map([ - ['mouseMove', 'Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, mouseMove uses the system pointer for this session until DSH restarts. If the user declines, continue without mouse control. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured.'], + ['mouseMove', 'Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, every mouseMove requires the target app to be in front and the point unobscured, including targets that previously supported background AX hover. If mouseMove returns FOREGROUND_REQUIRED, explain the necessary foreground step before activating the app, then observe the target again.'], ['click', 'click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying.'], ]) diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index 922078987a..fe32ca0774 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -884,8 +884,9 @@ it('moves only for the consenting session, without granting application access o const ask = vi.fn((id: string) => ({ answers: [{ id, selected: ['Allow for this session'] }] })) f.ctx.on('user-questions/request', async ({ questions }) => ask(questions[0]!.id)) const target = { appHandle: 'app-one', snapshotId: 'snapshot', elementId: '0' } - native.request.mockResolvedValueOnce({ appHandle: 'app-one', name: 'Fixture', running: true, - authorization: { id: 'a'.repeat(64), name: 'Fixture', persistable: false } }) + const application = { appHandle: 'app-one', name: 'Fixture', running: true, + authorization: { id: 'a'.repeat(64), name: 'Fixture', persistable: false } } + native.request.mockResolvedValueOnce(application) expect((await f.execute('getApp', { selector: 'Fixture' })).isError).toBe(false) const move = async (useSystemPointer: boolean) => { expect((await f.execute('mouseMove', target)).isError).toBe(false) @@ -899,19 +900,24 @@ it('moves only for the consenting session, without granting application access o const otherCall = (method: string, params: Record) => executeToolFixture(f.ctx, { agent: other.agent, signal: new AbortController().signal, callId: ToolCallId(method), name: 'cua', arguments: { method, params }, }) - expect((await otherCall('requestMouseControl', {})).value).toBe(true) - expect((await otherCall('mouseMove', target)).isError).toBe(true) + expect(JSON.stringify((await otherCall('mouseMove', target)).content)).toContain('APP_ACCESS_REQUIRED') + native.request.mockResolvedValueOnce(application) + expect((await otherCall('getApp', { selector: 'Fixture' })).isError).toBe(false) + ask.mockImplementationOnce(id => ({ answers: [{ id, selected: ['Deny'] }] })) + expect((await otherCall('requestMouseControl', {})).value).toBe(false) + expect((await otherCall('mouseMove', target)).isError).toBe(false) + expect(native.request).toHaveBeenLastCalledWith('mouseMove', { ...target, useSystemPointer: false }, expect.any(AbortSignal), expect.any(String)) + await move(true) await f.provider.dispose() await f.ctx.plugin(NativeProvider, config) expect((await f.execute('requestMouseControl')).value).toBe(true) - expect(ask.mock.calls.map(([id]) => id)).toEqual(['cua-app-access', 'cua-mouse-control', 'cua-mouse-control', 'cua-mouse-control']) + expect(ask.mock.calls.map(([id]) => id)).toEqual(['cua-app-access', 'cua-mouse-control', 'cua-app-access', 'cua-mouse-control', 'cua-mouse-control']) }) -it.each(['Deny', 'custom', 'cancel'] as const)('remembers a %s answer without moving the pointer or asking again', async (answer) => { +it.each(['Deny', 'custom'] as const)('remembers a %s answer without moving the pointer or asking again', async (answer) => { const f = await setup() await f.ctx.plugin(UserQuestions) const ask = vi.fn(() => { - if (answer === 'cancel') throw new UserQuestionError('User dismissed the request', 'ASK_CANCELLED') return { answers: [{ id: 'cua-mouse-control', selected: [answer === 'custom' ? 'Allow for this session' : answer], ...(answer === 'custom' ? { custom: 'maybe later' } : {}) }] } }) @@ -922,6 +928,39 @@ it.each(['Deny', 'custom', 'cancel'] as const)('remembers a %s answer without mo expect(native.request).not.toHaveBeenCalled() }) +it('allows a fresh request after dismissal without authorizing the first one', async () => { + const f = await setup() + await f.ctx.plugin(UserQuestions) + const ask = vi.spyOn(f.ctx.userQuestions, 'ask') + .mockRejectedValueOnce(new UserQuestionError('User dismissed the request', 'ASK_CANCELLED')) + .mockResolvedValueOnce({ answers: [{ id: 'cua-mouse-control', selected: ['Allow for this session'] }] }) + const dismissed = await f.execute('requestMouseControl') + expect(dismissed.isError).toBe(true) + expect(JSON.stringify(dismissed.content)).toContain('MOUSE_APPROVAL_CANCELLED') + expect((await f.execute('requestMouseControl')).value).toBe(true) + expect(ask).toHaveBeenCalledTimes(2) + expect(native.request).not.toHaveBeenCalled() +}) + +it('refuses delegated mouse control before asking the user even when the parent has approval', async () => { + const f = await setup() + await f.ctx.plugin(UserQuestions) + const ask = vi.spyOn(f.ctx.userQuestions, 'ask').mockResolvedValue({ + answers: [{ id: 'cua-mouse-control', selected: ['Allow for this session'] }], + }) + expect((await f.execute('requestMouseControl')).value).toBe(true) + ask.mockClear() + const { agent } = await f.ctx.agents.create({ sessionId: SessionId('delegated-mouse-session'), + meta: { origin: 'subagent', parentSession: f.handle.agent.id } }) + const refused = await executeToolFixture(f.ctx, { agent, signal: new AbortController().signal, + callId: ToolCallId('delegated-mouse-control'), name: 'cua', arguments: { method: 'requestMouseControl', params: {} } }) + expect(refused.isError).toBe(true) + expect(JSON.stringify(refused.content)).toContain('MOUSE_CONTROL_UNAVAILABLE') + expect(JSON.stringify(refused.content)).toContain('report that step in your final result for the main task') + expect(ask).not.toHaveBeenCalled() + expect(native.request).not.toHaveBeenCalled() +}) + it('drops late approval after cancellation and admits only one pending question', async () => { const f = await setup() expect((await f.execute('requestMouseControl')).isError).toBe(true) @@ -938,7 +977,7 @@ it('drops late approval after cancellation and admits only one pending question' const controller = new AbortController() const first = f.execute('requestMouseControl', {}, controller.signal) await entered.promise - expect((await f.execute('requestMouseControl')).isError).toBe(true) + expect(JSON.stringify((await f.execute('requestMouseControl')).content)).toContain('MOUSE_CONTROL_PENDING') controller.abort() release.resolve(undefined) await Promise.allSettled([first]) @@ -958,7 +997,7 @@ it('keeps provider-specific pointer flags and questions off providers without mo keyboard: ['app'], coordinateSpace: 'screenshot-pixels', browser: false }, }) } }) - expect((await f.execute('requestMouseControl')).isError).toBe(true) + expect(JSON.stringify((await f.execute('requestMouseControl')).content)).toContain('UNSUPPORTED_OPERATION') await selectApplication(f) const target = { appHandle: 'app-one', snapshotId: 'snapshot', elementId: '0' } expect((await f.execute('mouseMove', target)).isError).toBe(false) diff --git a/packages/experimental/cua/src/guidance.ts b/packages/experimental/cua/src/guidance.ts index 5f0abd1dac..5787dd1667 100644 --- a/packages/experimental/cua/src/guidance.ts +++ b/packages/experimental/cua/src/guidance.ts @@ -8,7 +8,7 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. -- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Returns true only on approval. Avoid asking unless the task requires a physical hover. +- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Only the main task may ask, and only when a physical hover is necessary. A false result is a refusal for this session until restart. If the user dismisses the request, continue without mouse control unless they ask to reconsider. Await the outstanding request before calling again. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. diff --git a/packages/experimental/cua/src/locales.ts b/packages/experimental/cua/src/locales.ts index 3833690158..7c2cba49eb 100644 --- a/packages/experimental/cua/src/locales.ts +++ b/packages/experimental/cua/src/locales.ts @@ -117,14 +117,14 @@ export const mouseControlDisplay = { pt: 'Permitir que o agente mova o ponteiro do mouse?', }, detail: { - en: 'Hovering will move your pointer and leave it at the target, even while you are using the mouse. This permission applies only to this session and expires when DSH restarts.', - zh: '智能体悬停时会移动你的鼠标指针，并将它留在目标位置，即使你正在使用鼠标。授权仅限当前会话，重启 DSH 后失效。', - ja: 'ホバーすると、マウスを使用中でもポインタが移動し、移動先に残ります。許可はこのセッションに限り、DSH を再起動すると失効します。', - es: 'Al pasar sobre un elemento, el puntero se moverá y quedará allí aunque estés usando el ratón. El permiso solo se aplica a esta sesión y caduca al reiniciar DSH.', - fr: 'Lors d’un survol, le pointeur se déplace et reste sur la cible, même si vous utilisez la souris. Cette autorisation vaut pour cette session et expire au redémarrage de DSH.', - de: 'Beim Überfahren eines Ziels bewegt sich der Zeiger und bleibt dort, auch wenn Sie die Maus verwenden. Die Erlaubnis gilt nur für diese Sitzung und erlischt beim Neustart von DSH.', - ru: 'При наведении указатель переместится и останется на цели, даже если вы пользуетесь мышью. Разрешение действует только в этом сеансе до перезапуска DSH.', - pt: 'Ao passar sobre um elemento, o ponteiro se moverá e ficará no destino, mesmo enquanto você usa o mouse. A permissão vale apenas para esta sessão e expira ao reiniciar o DSH.', + en: 'Hovering will move your pointer and leave it at the target, even while you are using the mouse. This permission applies only to this session until DSH restarts. If you decline, this session will not ask again before restart.', + zh: '智能体悬停时会移动你的鼠标指针，并将它留在目标位置，即使你正在使用鼠标。授权仅限当前会话，重启 DSH 后失效。如果拒绝，重启前本会话不会再次询问。', + ja: 'ホバーすると、マウスを使用中でもポインタが移動し、移動先に残ります。許可はこのセッションに限り、DSH を再起動すると失効します。拒否すると、再起動までこのセッションでは再度確認しません。', + es: 'Al pasar sobre un elemento, el puntero se moverá y quedará allí aunque estés usando el ratón. El permiso solo se aplica a esta sesión y caduca al reiniciar DSH. Si lo deniegas, esta sesión no volverá a preguntar hasta que reinicies DSH.', + fr: 'Lors d’un survol, le pointeur se déplace et reste sur la cible, même si vous utilisez la souris. Cette autorisation vaut pour cette session et expire au redémarrage de DSH. Si vous refusez, cette session ne redemandera pas avant le redémarrage.', + de: 'Beim Überfahren eines Ziels bewegt sich der Zeiger und bleibt dort, auch wenn Sie die Maus verwenden. Die Erlaubnis gilt nur für diese Sitzung und erlischt beim Neustart von DSH. Wenn Sie ablehnen, fragt diese Sitzung bis zum Neustart nicht erneut.', + ru: 'При наведении указатель переместится и останется на цели, даже если вы пользуетесь мышью. Разрешение действует только в этом сеансе до перезапуска DSH. Если вы откажете, запрос в этом сеансе не повторится до перезапуска.', + pt: 'Ao passar sobre um elemento, o ponteiro se moverá e ficará no destino, mesmo enquanto você usa o mouse. A permissão vale apenas para esta sessão e expira ao reiniciar o DSH. Se você recusar, esta sessão não perguntará novamente até a reinicialização.', }, approve: applicationAccessDisplay.approve, } diff --git a/packages/experimental/cua/src/mouse-control.ts b/packages/experimental/cua/src/mouse-control.ts index 947482766f..595cef0f0b 100644 --- a/packages/experimental/cua/src/mouse-control.ts +++ b/packages/experimental/cua/src/mouse-control.ts @@ -21,14 +21,17 @@ export class MouseControl { } /** - * Ask once for the Session. Simultaneous requests cannot open additional questions. + * Ask the main task for a Session decision. Simultaneous requests cannot open additional questions. * @param agent Live Agent that owns the Session. * @param signal Tool lifetime. An aborted tool cannot grant permission. * @returns True only when the user chose the offered approval. */ async request(agent: Agent, signal: AbortSignal): Promise { + if (agent.session.header.origin === 'subagent') { + throw new NativeError('MOUSE_CONTROL_UNAVAILABLE', "Delegated tasks cannot move the user's pointer. Use AX hover, click, or performAction. If the task requires a physical hover, report that step in your final result for the main task.") + } const decision = this.decisions.get(agent.id) - if (decision === 'asking') throw new NativeError('MOUSE_CONTROL_PENDING', 'A mouse control request is already awaiting an answer in this session') + if (decision === 'asking') throw new NativeError('MOUSE_CONTROL_PENDING', 'Wait for the pending mouse control request in this session before continuing') if (decision !== undefined) return decision const questions = this.ctx.get('userQuestions') if (!questions) throw new NativeError('MOUSE_APPROVAL_UNAVAILABLE', 'No human answerer is available for mouse control') @@ -49,11 +52,10 @@ export class MouseControl { this.decisions.set(agent.id, granted) return granted } catch (error) { - if (error instanceof UserQuestionError && error.code === 'ASK_CANCELLED') { - this.decisions.set(agent.id, false) - return false - } this.decisions.delete(agent.id) + if (error instanceof UserQuestionError && error.code === 'ASK_CANCELLED') { + throw new NativeError('MOUSE_APPROVAL_CANCELLED', 'The mouse control request was dismissed. Continue without mouse control unless the user asks to reconsider', { cause: error }) + } throw error } } diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index 541dcdd070..77923d64be 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -247,8 +247,9 @@ export interface CuaApp extends CuaSurface { /** Native app API; its caller owns global bindings and variable persistence. */ export interface CuaApi { /** - * Ask to move the user's pointer with mouseMove for this session until restart. - * @returns True only after explicit approval. A refusal is remembered for the session. + * Ask to move the user's pointer with mouseMove for this session until restart. Only the main task can ask. + * @returns True on approval, false on refusal. Both decisions last until restart for this session. + * @throws If the user dismisses the question without deciding or another request is pending. */ requestMouseControl(): Promise /** From 30ccf8a8ed447c44a98397e7a384249f77e80214 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 16:05:53 +0800 Subject: [PATCH 05/30] test(cua): refresh mouse approval guidance snapshots --- snapshots/session/cua-native/system-prompt.expected.md | 4 ++-- snapshots/session/cua-ptc/system-prompt.expected.md | 4 ++-- .../session/ptc-script-retained/system-prompt.expected.md | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index e7132c8c0f..2fd5c3df3d 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -43,7 +43,7 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. -- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Returns true only on approval. Avoid asking unless the task requires a physical hover. +- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Only the main task may ask, and only when a physical hover is necessary. A false result is a refusal for this session until restart. If the user dismisses the request, continue without mouse control unless they ask to reconsider. Await the outstanding request before calling again. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -68,7 +68,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, mouseMove uses the system pointer for this session until DSH restarts. If the user declines, continue without mouse control. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured. +- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, every mouseMove requires the target app to be in front and the point unobscured, including targets that previously supported background AX hover. If mouseMove returns FOREGROUND_REQUIRED, explain the necessary foreground step before activating the app, then observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index 45154f9179..b1fae81bcf 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -45,7 +45,7 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. -- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Returns true only on approval. Avoid asking unless the task requires a physical hover. +- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Only the main task may ask, and only when a physical hover is necessary. A false result is a refusal for this session until restart. If the user dismisses the request, continue without mouse control unless they ask to reconsider. Await the outstanding request before calling again. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -70,7 +70,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, mouseMove uses the system pointer for this session until DSH restarts. If the user declines, continue without mouse control. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured. +- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, every mouseMove requires the target app to be in front and the point unobscured, including targets that previously supported background AX hover. If mouseMove returns FOREGROUND_REQUIRED, explain the necessary foreground step before activating the app, then observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/snapshots/session/ptc-script-retained/system-prompt.expected.md b/snapshots/session/ptc-script-retained/system-prompt.expected.md index b1769c75dc..8780b966c7 100644 --- a/snapshots/session/ptc-script-retained/system-prompt.expected.md +++ b/snapshots/session/ptc-script-retained/system-prompt.expected.md @@ -45,7 +45,7 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. -- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Returns true only on approval. Avoid asking unless the task requires a physical hover. +- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Only the main task may ask, and only when a physical hover is necessary. A false result is a refusal for this session until restart. If the user dismisses the request, continue without mouse control unless they ask to reconsider. Await the outstanding request before calling again. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -70,7 +70,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, mouseMove uses the system pointer for this session until DSH restarts. If the user declines, continue without mouse control. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured. +- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, every mouseMove requires the target app to be in front and the point unobscured, including targets that previously supported background AX hover. If mouseMove returns FOREGROUND_REQUIRED, explain the necessary foreground step before activating the app, then observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. From ed5d4d23de059c10663ae7bbee96caa68fabe387 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 16:10:38 +0800 Subject: [PATCH 06/30] test(cua): use the shared channel fixture for providers without pointer control --- .../experimental/cua-macos/tests/provider-lifecycle.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index 23925fe847..52202a0774 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -1083,7 +1083,7 @@ it('keeps provider-specific pointer flags and questions off providers without mo await f.provider.dispose() await f.ctx.plugin({ name: 'fixture-window-provider', inject: NativeProvider.inject, apply(ctx: Context) { installCua(ctx, CuaConfig(config), { - helperPath: config.helperPath, createChannel() { throw new Error('Unexpected native channel') }, + helperPath: config.helperPath, createChannel: native.connect, guidance: '', permissionParameters: () => ({}), capabilities: { platform: 'fixture', methods: ['getApp', 'mouseMove'], appLevelAccessibility: true, keyboard: ['app'], coordinateSpace: 'screenshot-pixels', browser: false }, From d8d6081f44844ec56487cd6029c471b8435256ec Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Sat, 7 Nov 2026 16:22:21 +0800 Subject: [PATCH 07/30] perf(ui-sidebar-files): give the navigator's covering rules a subject The two rules that hide a row's body under a filled navigator started with `:has(`, so the browser treated every element as a possible subject and restyled the whole document after any DOM change. Both rows that host the navigator now carry data-files-navigator-row, which the rules name. --- .../client/ui-sidebar-documentpreview/README.i18n.yaml | 4 ++-- packages/client/ui-sidebar-documentpreview/README.md | 2 +- packages/client/ui-sidebar-documentpreview/README.zh.md | 2 +- .../src/client/PreviewFrame.tsx | 2 +- .../tests/text-preview.client.spec.tsx | 6 ++++-- .../client/ui-sidebar-files/src/client/FileNavigator.tsx | 6 +++++- .../ui-sidebar-files/src/client/FilesBody.module.css | 8 +++++--- packages/client/ui-sidebar-files/src/client/FilesBody.tsx | 2 +- .../ui-sidebar-files/tests/files-body.client.spec.tsx | 2 ++ 9 files changed, 22 insertions(+), 12 deletions(-) diff --git a/packages/client/ui-sidebar-documentpreview/README.i18n.yaml b/packages/client/ui-sidebar-documentpreview/README.i18n.yaml index 99d60b54c9..91373281f1 100644 --- a/packages/client/ui-sidebar-documentpreview/README.i18n.yaml +++ b/packages/client/ui-sidebar-documentpreview/README.i18n.yaml @@ -15,8 +15,8 @@ en: 0dd8c3a22e28aacf zh: c27c11bf4b5cceba /deepseek-ai-dsh-client-ui-sidebar-documentpreview/what-it-registers: - en: 5686f49d98437f49 - zh: 40c154f8cc7b8d14 + en: 691249b09395eae1 + zh: e2ae49df2ebe130e /deepseek-ai-dsh-client-ui-sidebar-documentpreview/addresses: en: 57942270e23d3a8e zh: 6617124e652b3f67 diff --git a/packages/client/ui-sidebar-documentpreview/README.md b/packages/client/ui-sidebar-documentpreview/README.md index 4e87ae3aed..6453a874e8 100644 --- a/packages/client/ui-sidebar-documentpreview/README.md +++ b/packages/client/ui-sidebar-documentpreview/README.md @@ -31,7 +31,7 @@ Preview files in the right Sidebar and choose among registered renderers. Markdo Image drag actions arrive through the image entry’s inject face from [ui-file-drag](../ui-file-drag/README.md). - **The type** — `ctx.sidebarRightTabs.register(...)` with id `@deepseek-ai/dsh-client-ui-sidebar-documentpreview` (this implementation's identity in the tab system, and the key its body registers under), kind `text`, a whole-address file-resource pattern, band `fallback`. `canOpen` accepts only Session addresses, whose paths may be relative or absolute; bare `absolute` addresses are not claimed. A type registered at the `extension` or `builtin` band for a narrower pattern (say `*.png`) takes those addresses; other supported files land here. The whole address is the content identity, so two files with one name in different directories, or one path under two sessions, are two tabs; the decoded basename is the tab title, and the keyed `sidebar.right.pane.tab.title` seat places its extension-specific `FileTypeIcon` before that title. -- **The body** — the keyed `sidebar.right.pane.tab` seat under the type's id. Its fixed header shows the Host's absolute path when available, otherwise the requested path; the shared [`PathLabel`](../ui-primitives/README.md#component-catalog) subdues directories, preserves a clipped path’s trailing characters with a left-edge fade, and exposes the full value on hover. A dropdown appears when multiple supported renderers are available. Plain text is offered only for text-compatible sources; a single renderer shows no viewer control. A known binary container suffix without a registered renderer shows the file-type icon and an unsupported-preview message under the path header, without issuing a read. Two Session-scoped list child slots hand the previewed file to other plugins once its metadata reports a Host path, both with `absolutePath` as the owner prop: `sidebar.right.tab.document.actions` renders after the header's own controls and before the navigator toggle, at the same position in every state that shows the header so it is not remounted when the state changes, and `sidebar.right.tab.document.unpreviewable` renders in the unsupported empty state and in the empty state of a readable file the preview cannot render (`not-text`, `too-large`), where Retry would otherwise stand; a carrier or unclassified read failure keeps Retry, and a missing or non-regular path offers only its explanation. The shipped occupant of both is [`ui-open-in-app`](../ui-open-in-app/README.md). The preview directly renders [`ui-sidebar-files`](../ui-sidebar-files/README.md#the-navigator)'s Session-scoped Factories: `sidebar.files.navigator` after the body in its content row with `{ useTabInfo, page: false, onOpen }`, where `onOpen(address)` calls `tab.actions.openResource(address, { replaceTab: true })`, and `sidebar.files.navigator.toggle` at the header's end with `{ useTabInfo }`. Both stay mounted from loading to loaded in docked and floating panes; unregistered Factories render nothing. The navigator owns its width within the row's positioning box and stacking context, narrowing or overlaying the body without covering the floating pane's resize handle. A renderer layer handling Escape calls `preventDefault()` to keep the navigator from also closing. A wrap toggle appears only when the selected renderer declares `wrap: true`; its glyph describes the mode the click selects, and the per-tab preference starts on. Reload stays in this header, not the Sidebar's tab strip. Automatic refresh defaults to enabled; its separate toggle is hidden, with state, labels, styles, and toggle logic retained. The body reaches every pane edge; each renderer owns its content inset and may own an inner scrollport. This intentionally differs from the file tree's 2px right-side scrollbar offset: previews keep the full pane width so edge-to-edge HTML and code scrollports end at the pane edge. +- **The body** — the keyed `sidebar.right.pane.tab` seat under the type's id. Its fixed header shows the Host's absolute path when available, otherwise the requested path; the shared [`PathLabel`](../ui-primitives/README.md#component-catalog) subdues directories, preserves a clipped path’s trailing characters with a left-edge fade, and exposes the full value on hover. A dropdown appears when multiple supported renderers are available. Plain text is offered only for text-compatible sources; a single renderer shows no viewer control. A known binary container suffix without a registered renderer shows the file-type icon and an unsupported-preview message under the path header, without issuing a read. Two Session-scoped list child slots hand the previewed file to other plugins once its metadata reports a Host path, both with `absolutePath` as the owner prop: `sidebar.right.tab.document.actions` renders after the header's own controls and before the navigator toggle, at the same position in every state that shows the header so it is not remounted when the state changes, and `sidebar.right.tab.document.unpreviewable` renders in the unsupported empty state and in the empty state of a readable file the preview cannot render (`not-text`, `too-large`), where Retry would otherwise stand; a carrier or unclassified read failure keeps Retry, and a missing or non-regular path offers only its explanation. The shipped occupant of both is [`ui-open-in-app`](../ui-open-in-app/README.md). The preview directly renders [`ui-sidebar-files`](../ui-sidebar-files/README.md#the-navigator)'s Session-scoped Factories: `sidebar.files.navigator` after the body in its content row, which carries `data-files-navigator-row`, with `{ useTabInfo, page: false, onOpen }`, where `onOpen(address)` calls `tab.actions.openResource(address, { replaceTab: true })`, and `sidebar.files.navigator.toggle` at the header's end with `{ useTabInfo }`. Both stay mounted from loading to loaded in docked and floating panes; unregistered Factories render nothing. The navigator owns its width within the row's positioning box and stacking context, narrowing or overlaying the body without covering the floating pane's resize handle. A renderer layer handling Escape calls `preventDefault()` to keep the navigator from also closing. A wrap toggle appears only when the selected renderer declares `wrap: true`; its glyph describes the mode the click selects, and the per-tab preference starts on. Reload stays in this header, not the Sidebar's tab strip. Automatic refresh defaults to enabled; its separate toggle is hidden, with state, labels, styles, and toggle logic retained. The body reaches every pane edge; each renderer owns its content inset and may own an inner scrollport. This intentionally differs from the file tree's 2px right-side scrollbar offset: previews keep the full pane width so edge-to-edge HTML and code scrollports end at the pane edge. - **Shared loading and view state**, session-scoped and bucketed by tab id. The store holds accumulated pages or complete bytes, read and observed versions, pending Resource changes, automatic-refresh state, loading/failure state, renderer choice, scroll offset, wrap, and the answered navigation revision. The ordinary inject face calls Remote readers and writes through declared store actions. Reloads and loading-mode changes retire older requests; the tab's abort signal forgets its state. Document implementations register metadata with `ctx.documentPreviews.register({ id, extensions, binaryExtensions?, priority, title, loading, wrap?, sourceLines? })` and a body under the same `id` in the keyed, Session-scoped `sidebar.right.tab.document` child slot. `binaryExtensions` lists suffixes in `extensions` that cannot be read as text and omit the plain-text option. `sourceLines` declares that a `text-pages` body exposes source-line anchors for [navigation](#navigation). Own both registrations with effects and wait for the child slot through `ctx.slots.inject`. Bodies receive `resourceAddress`, `content`, `wrap`, `scrollportRef`, `addResource`, `setResources`, and the standard `useTabInfo`/`useResource` hooks. Dependency membership belongs to a tab-owned `ResourceGroup`; replacing dependencies always retains the root Resource. An inner scrolling element attaches `scrollportRef`; unmounting restores the shared body as scroll owner. The registry retains all matching alternatives: `extension` (the default) ranks above `builtin`, then longer suffixes rank first, then registration order. The dropdown preserves a selected implementation while it remains available. HTML, SVG, and unmatched extensions retain the plain-text fallback independently of loading mode. diff --git a/packages/client/ui-sidebar-documentpreview/README.zh.md b/packages/client/ui-sidebar-documentpreview/README.zh.md index f2996fd813..3e846d1f63 100644 --- a/packages/client/ui-sidebar-documentpreview/README.zh.md +++ b/packages/client/ui-sidebar-documentpreview/README.zh.md @@ -31,7 +31,7 @@ kind: "package-reference" 图片拖放回调通过图片入口的注入接口从 [ui-file-drag](../ui-file-drag/README.zh.md) 获取。 - **类型** —— `ctx.sidebarRightTabs.register(...)`，id 为 `@deepseek-ai/dsh-client-ui-sidebar-documentpreview`（这个实现在 tab 系统中的身份，也是其正文注册所用的键），kind `text`，匹配完整文件资源地址的 pattern，档位 `fallback`。`canOpen` 只接受 Session 地址，其中路径可为相对或绝对路径；不认领裸 `absolute` 地址。在 `extension` 或 `builtin` 档以更窄 pattern（比如 `*.png`）注册的类型接走那些地址；其他受支持文件落到这里。整个地址就是内容身份，所以不同目录下同名的两个文件、或同一路径在两个会话之下，是两个 tab；解码后的 basename 是 tab 标题，keyed slot `sidebar.right.pane.tab.title` 会在标题前放置按扩展名选择的 `FileTypeIcon`。 -- **正文** —— keyed slot `sidebar.right.pane.tab`，键为类型的 id。固定头部在可用时显示 Host 的绝对路径，否则显示请求路径；共享的 [`PathLabel`](../ui-primitives/README.zh.md#component-catalog) 弱化目录颜色，以左侧渐隐保留被裁剪路径的尾部字符，并在悬停时展示完整值。有多个受支持的渲染器时才显示下拉菜单。仅文本兼容的源文件提供纯文本选项；只有一个渲染器时不显示查看器控件。已知的二进制容器后缀没有注册渲染器时，在路径头部下方显示文件类型图标和不支持预览的说明，并且不会发起读取。两个 Session 作用域的 list 子 slot 在文件元数据报出 Host 路径后把当前文件交给其他插件，owner prop 为 `absolutePath`：`sidebar.right.tab.document.actions` 在每个显示头部的状态里渲染在头部自有控件之后、导航栏开关之前，位置在各状态间相同，因此状态切换时不会重新挂载；`sidebar.right.tab.document.unpreviewable` 渲染在不支持预览的空态里，以及可读但本预览无法渲染的文件（`not-text`、`too-large`）的空态里，占据原本放重试的位置；载体或未分类的读取失败仍保留重试，不存在或非普通文件的路径只给说明。两者已发布的占位者都是 [`ui-open-in-app`](../ui-open-in-app/README.zh.md)。预览直接渲染 [`ui-sidebar-files`](../ui-sidebar-files/README.zh.md#the-navigator) 的 Session 作用域 Factory：`sidebar.files.navigator` 位于内容行的正文之后，传入 `{ useTabInfo, page: false, onOpen }`，其中 `onOpen(address)` 调用 `tab.actions.openResource(address, { replaceTab: true })`；`sidebar.files.navigator.toggle` 位于头部末端，传入 `{ useTabInfo }`。两者在停靠与浮动分栏中从加载到完成保持挂载；未注册的 Factory 不绘制内容。导航栏在内容行的定位框与层叠上下文内决定宽度，让正文变窄或覆盖正文，不遮挡浮窗缩放手柄。自行处理 Escape 的渲染器浮层需调用 `preventDefault()`，避免同时关闭导航栏。仅当所选渲染器声明 `wrap: true` 时显示换行开关；图标表示点击后切换到的模式，该偏好按 tab 保存，初始开启。重新载入仍在此头部，不放入 Sidebar 的 tab 条。自动刷新默认开启；独立开关暂时隐藏，状态、文案、样式与切换逻辑保留。正文贴合格的每条边，各渲染器自行提供内容留白，并可拥有内部滚动区。这与文件树右侧预留 2px 滚动条间距的布局有意不同：Preview 使用格的完整宽度，使贴边 HTML 与代码滚动区终止于格的边缘。 +- **正文** —— keyed slot `sidebar.right.pane.tab`，键为类型的 id。固定头部在可用时显示 Host 的绝对路径，否则显示请求路径；共享的 [`PathLabel`](../ui-primitives/README.zh.md#component-catalog) 弱化目录颜色，以左侧渐隐保留被裁剪路径的尾部字符，并在悬停时展示完整值。有多个受支持的渲染器时才显示下拉菜单。仅文本兼容的源文件提供纯文本选项；只有一个渲染器时不显示查看器控件。已知的二进制容器后缀没有注册渲染器时，在路径头部下方显示文件类型图标和不支持预览的说明，并且不会发起读取。两个 Session 作用域的 list 子 slot 在文件元数据报出 Host 路径后把当前文件交给其他插件，owner prop 为 `absolutePath`：`sidebar.right.tab.document.actions` 在每个显示头部的状态里渲染在头部自有控件之后、导航栏开关之前，位置在各状态间相同，因此状态切换时不会重新挂载；`sidebar.right.tab.document.unpreviewable` 渲染在不支持预览的空态里，以及可读但本预览无法渲染的文件（`not-text`、`too-large`）的空态里，占据原本放重试的位置；载体或未分类的读取失败仍保留重试，不存在或非普通文件的路径只给说明。两者已发布的占位者都是 [`ui-open-in-app`](../ui-open-in-app/README.zh.md)。预览直接渲染 [`ui-sidebar-files`](../ui-sidebar-files/README.zh.md#the-navigator) 的 Session 作用域 Factory：`sidebar.files.navigator` 位于内容行的正文之后（该行带有 `data-files-navigator-row`），传入 `{ useTabInfo, page: false, onOpen }`，其中 `onOpen(address)` 调用 `tab.actions.openResource(address, { replaceTab: true })`；`sidebar.files.navigator.toggle` 位于头部末端，传入 `{ useTabInfo }`。两者在停靠与浮动分栏中从加载到完成保持挂载；未注册的 Factory 不绘制内容。导航栏在内容行的定位框与层叠上下文内决定宽度，让正文变窄或覆盖正文，不遮挡浮窗缩放手柄。自行处理 Escape 的渲染器浮层需调用 `preventDefault()`，避免同时关闭导航栏。仅当所选渲染器声明 `wrap: true` 时显示换行开关；图标表示点击后切换到的模式，该偏好按 tab 保存，初始开启。重新载入仍在此头部，不放入 Sidebar 的 tab 条。自动刷新默认开启；独立开关暂时隐藏，状态、文案、样式与切换逻辑保留。正文贴合格的每条边，各渲染器自行提供内容留白，并可拥有内部滚动区。这与文件树右侧预留 2px 滚动条间距的布局有意不同：Preview 使用格的完整宽度，使贴边 HTML 与代码滚动区终止于格的边缘。 - **共享加载与视图状态**，会话作用域、按 tab id 分桶。store 持有累计页或完整字节、读取与观察版本、待处理 Resource 变化、自动刷新状态、加载/失败状态、渲染器选择、滚动位置、换行和已响应的导航 revision。普通 inject face 调用 Remote 读取，并经声明的 store action 写入。重新载入和加载模式变化会淘汰旧请求；tab 的中止信号清理其状态。 文档实现在 `ctx.documentPreviews.register({ id, extensions, binaryExtensions?, priority, title, loading, wrap?, sourceLines? })` 注册元数据，并以相同 `id` 向 keyed、Session 作用域的子 slot `sidebar.right.tab.document` 注册正文。`binaryExtensions` 列出 `extensions` 中不可按文本阅读的后缀，这些后缀不提供纯文本选项。`sourceLines` 声明 `text-pages` 正文为[导航](#navigation)提供源码行锚点。两处注册都由 effect 持有，通过 `ctx.slots.inject` 等待子 slot。正文接收 `resourceAddress`、`content`、`wrap`、`scrollportRef`、`addResource`、`setResources` 和标准 `useTabInfo`/`useResource` 钩子。依赖成员由 tab 持有的 `ResourceGroup` 管理；替换依赖始终保留根 Resource。内部滚动元素挂载 `scrollportRef`；卸载时恢复共享正文的滚动职责。注册表保留所有匹配备选：`extension`（默认）优先于 `builtin`，随后按更长的后缀、再按注册顺序排列。所选实现仍可用时，下拉选择保持不变。HTML、SVG 和未匹配的扩展名保留纯文本回退，与加载方式无关。 diff --git a/packages/client/ui-sidebar-documentpreview/src/client/PreviewFrame.tsx b/packages/client/ui-sidebar-documentpreview/src/client/PreviewFrame.tsx index 6ee191a3fe..5b529fffee 100644 --- a/packages/client/ui-sidebar-documentpreview/src/client/PreviewFrame.tsx +++ b/packages/client/ui-sidebar-documentpreview/src/client/PreviewFrame.tsx @@ -73,7 +73,7 @@ export function PreviewFrame({ {handoff} -
+
{children}
diff --git a/packages/client/ui-sidebar-documentpreview/tests/text-preview.client.spec.tsx b/packages/client/ui-sidebar-documentpreview/tests/text-preview.client.spec.tsx index fb24a887b4..8e0c2b089c 100644 --- a/packages/client/ui-sidebar-documentpreview/tests/text-preview.client.spec.tsx +++ b/packages/client/ui-sidebar-documentpreview/tests/text-preview.client.spec.tsx @@ -861,6 +861,8 @@ describe('TextPreview — workspace navigator Factories', () => { return { toggle: header?.lastElementChild?.getAttribute('data-navigator-factory'), navigator: row?.lastElementChild?.getAttribute('data-navigator-factory'), + // The navigator's filled layout hides the body of the row this attribute names. + navigatorRow: row?.hasAttribute('data-files-navigator-row'), } } @@ -876,7 +878,7 @@ describe('TextPreview — workspace navigator Factories', () => { ]) expect(view.container.querySelector('[data-textpreview-state="loading"]')).not.toBeNull() expect(seats(view.container)).toEqual({ - toggle: 'sidebar.files.navigator.toggle', navigator: 'sidebar.files.navigator', + toggle: 'sidebar.files.navigator.toggle', navigator: 'sidebar.files.navigator', navigatorRow: true, }) const navigator = view.container.querySelector('[data-textpreview-content] > [data-navigator-factory]')! const toggle = view.container.querySelector('[data-textpreview-path]')?.parentElement?.lastElementChild @@ -890,7 +892,7 @@ describe('TextPreview — workspace navigator Factories', () => { await settle() expect(view.container.querySelector('[data-textpreview-state="text"]')).not.toBeNull() expect(seats(view.container)).toEqual({ - toggle: 'sidebar.files.navigator.toggle', navigator: 'sidebar.files.navigator', + toggle: 'sidebar.files.navigator.toggle', navigator: 'sidebar.files.navigator', navigatorRow: true, }) expect(view.container.querySelector('[data-textpreview-content] > [data-navigator-factory]')).toBe(navigator) expect(view.container.querySelector('[data-textpreview-path]')?.parentElement?.lastElementChild).toBe(toggle) diff --git a/packages/client/ui-sidebar-files/src/client/FileNavigator.tsx b/packages/client/ui-sidebar-files/src/client/FileNavigator.tsx index dfe53b637f..dac8bb7f88 100644 --- a/packages/client/ui-sidebar-files/src/client/FileNavigator.tsx +++ b/packages/client/ui-sidebar-files/src/client/FileNavigator.tsx @@ -41,7 +41,11 @@ export interface FileNavigatorInput { declare module '@deepseek-ai/dsh-client-ui-slots' { interface SlotFactoryMap { - /** One pane's workspace tree with its width and show/hide state. */ + /** + * One pane's workspace tree with its width and show/hide state. The host renders it as a direct child of + * the content row it shares with the body, and that row carries `data-files-navigator-row`, which the + * filled layout needs to hide the covered body. + */ 'sidebar.files.navigator': { scope: 'session' props: FileNavigatorInput diff --git a/packages/client/ui-sidebar-files/src/client/FilesBody.module.css b/packages/client/ui-sidebar-files/src/client/FilesBody.module.css index 4bddc11bb1..315d5afae8 100644 --- a/packages/client/ui-sidebar-files/src/client/FilesBody.module.css +++ b/packages/client/ui-sidebar-files/src/client/FilesBody.module.css @@ -74,14 +74,16 @@ border-left: 0; } -/* Covered content stays laid out but cannot receive keyboard focus. */ -:has(> .navigator.fill) > :not(.navigator) { +/* Covered content stays laid out but cannot receive keyboard focus. The rule + names its row through `data-files-navigator-row`: a `:has()` without a + subject makes the browser restyle the whole document after any DOM change. */ +[data-files-navigator-row]:has(> .navigator.fill) > :not(.navigator) { visibility: hidden; } /* navigator-layout `FULLSCREEN_QUERY`: every layout fills the row. */ @media (width < 768px) { - :has(> .navigator:not(.wholePage)) > :not(.navigator) { + [data-files-navigator-row]:has(> .navigator:not(.wholePage)) > :not(.navigator) { visibility: hidden; } diff --git a/packages/client/ui-sidebar-files/src/client/FilesBody.tsx b/packages/client/ui-sidebar-files/src/client/FilesBody.tsx index f7f5ae385f..0f6b26d7f6 100644 --- a/packages/client/ui-sidebar-files/src/client/FilesBody.tsx +++ b/packages/client/ui-sidebar-files/src/client/FilesBody.tsx @@ -113,7 +113,7 @@ export function FilesBody(props: FilesBodyProps): ReactNode { {renderSlot('sidebar.right.tab.files.actions', { absolutePath: cwd })}
-
{navigator}
+
{navigator}
) } diff --git a/packages/client/ui-sidebar-files/tests/files-body.client.spec.tsx b/packages/client/ui-sidebar-files/tests/files-body.client.spec.tsx index a8ef76a271..2cd0aa3d5a 100644 --- a/packages/client/ui-sidebar-files/tests/files-body.client.spec.tsx +++ b/packages/client/ui-sidebar-files/tests/files-body.client.spec.tsx @@ -56,6 +56,8 @@ describe('FilesBody', () => { expect([...header.querySelectorAll('button:not([data-files-auto-refresh])')].map(button => button.getAttribute('aria-label'))) .toEqual([zh.reload, zh['search.placeholder']]) const content = page.querySelector('[data-files-content]')! + // The filled layout's rule finds the row the navigator covers by this attribute. + expect(content.hasAttribute('data-files-navigator-row')).toBe(true) expect(content.children).toHaveLength(1) const navigator = content.firstElementChild! expect(navigator.getAttribute('data-files-navigator')).toBe(PANE) From 9b456b4e9353e02027b40818ccd26f6cfc5f2285 Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Sat, 7 Nov 2026 16:22:24 +0800 Subject: [PATCH 08/30] perf(forge): compare from the recorded base while confirming the head Status snapshots carry the base commit the host recorded with the head (baseSha). ForgeService.diff passes it to the provider when the latest status names the requested head, and forge-github starts the compare read from it while its GraphQL read confirms the head; a stale head or another base cancels and joins the early read. --- docs/subsystems/forge.md | 13 +++-- docs/subsystems/forge.zh.md | 13 +++-- .../forge-github/README.i18n.yaml | 8 +-- packages/experimental/forge-github/README.md | 4 +- .../experimental/forge-github/README.zh.md | 4 +- .../experimental/forge-github/src/graphql.ts | 4 +- .../experimental/forge-github/src/provider.ts | 44 +++++++++++----- .../forge-github/tests/fixtures.ts | 1 + .../forge-github/tests/graphql.spec.ts | 1 + .../forge-github/tests/provider.spec.ts | 51 ++++++++++++++++++- .../tests/loader-composition.spec.ts | 1 + packages/experimental/forge/README.i18n.yaml | 4 +- packages/experimental/forge/README.md | 2 +- packages/experimental/forge/README.zh.md | 2 +- packages/experimental/forge/src/index.ts | 7 ++- packages/experimental/forge/src/types.ts | 10 +++- .../experimental/forge/tests/watch.spec.ts | 16 ++++++ .../extensions/tool-cordis/src/api-catalog.ts | 6 +-- 18 files changed, 152 insertions(+), 39 deletions(-) diff --git a/docs/subsystems/forge.md b/docs/subsystems/forge.md index b5156c101f..208c6a5d9f 100644 --- a/docs/subsystems/forge.md +++ b/docs/subsystems/forge.md @@ -71,9 +71,10 @@ interface ForgeProvider { /** * Read the changes of a change request at head commit `headSha` as the host computes them, as one unified * patch with `a/` and `b/` path prefixes. A patch is returned only when it describes `headSha`; once the - * head is another commit, the read resolves to a `stale` failure. + * head is another commit, the read resolves to a `stale` failure. `base`, when given, is the `baseSha` the + * latest status recorded with `headSha`; a provider may start reading from it before it confirms the head. */ - diff(ref: ChangeRequestRef, headSha: string, signal: AbortSignal): Promise + diff(ref: ChangeRequestRef, headSha: string, signal: AbortSignal, base?: string): Promise /** Read the text of `path` at `commit` in the change request's repository. */ fileText(ref: ChangeRequestRef, commit: string, path: string, signal: AbortSignal): Promise /** Build the web page of `path` at `commit` in the change request's repository. */ @@ -199,6 +200,11 @@ interface ChangeRequestSnapshot { readonly headSha: string /** Branch the change request merges into. */ readonly baseBranch: string + /** + * Base commit the host recorded with `headSha`, whose merge base with the head starts the change request's diff; + * omitted when the provider does not report it. + */ + readonly baseSha?: string /** Lines the change request adds. */ readonly additions: number /** Lines the change request deletes. */ @@ -907,7 +913,8 @@ async fetchChecks(ref: ChangeRequestRef, signal: AbortSignal): Promise + diff(ref: ChangeRequestRef, headSha: string, signal: AbortSignal, base?: string): Promise /** Read the text of `path` at `commit` in the change request's repository. */ fileText(ref: ChangeRequestRef, commit: string, path: string, signal: AbortSignal): Promise /** Build the web page of `path` at `commit` in the change request's repository. */ @@ -199,6 +200,11 @@ interface ChangeRequestSnapshot { readonly headSha: string /** Branch the change request merges into. */ readonly baseBranch: string + /** + * Base commit the host recorded with `headSha`, whose merge base with the head starts the change request's diff; + * omitted when the provider does not report it. + */ + readonly baseSha?: string /** Lines the change request adds. */ readonly additions: number /** Lines the change request deletes. */ @@ -907,7 +913,8 @@ async fetchChecks(ref: ChangeRequestRef, signal: AbortSignal): Promise --repo //`. Merging calls `PUT repos///pulls//merge` through `gh api` with the method and the expected head SHA, so GitHub refuses once new commits arrived, which is reported as `stale`. Unlike `gh pr merge`, it never enables auto-merge, never joins a merge queue, and never deletes the head branch. A refused action reports GitHub's message, and a diagnostic that mentions authentication becomes `unauthenticated`. -`diff` reads `headRefOid` and `baseRefOid` in one GraphQL query and returns `stale` when the head is not the requested commit, which must be a full 40- or 64-digit hexadecimal commit. Otherwise it reads `repos///compare/...` through `gh api --allow-escape-sequences` with the diff media type, the patch from the merge base of both commits, so neither end follows a moving branch. For every open and merged pull request measured, this patch is byte-identical to `gh pr diff`, which takes no commit argument. Unlike the pull request diff, the compare endpoint returns diffs over 300 files, so only output over `maxOutputBytes` or GitHub's own refusal makes such a diff `too-large`. `--allow-escape-sequences` keeps terminal escape sequences that file contents hold, which `gh` 2.97 and newer otherwise refuse to print; an older `gh` rejects the option with `unknown flag` before sending a request and prints such sequences unchanged, so that rejection repeats the read without it. `fileText` reads `repos///contents/?ref=` through `gh api` with the raw media type and accepts only a full 40- or 64-digit hexadecimal commit. `fileUrl` returns `https://///blob//`. Both paths percent-encode each segment. `commits` reads a pull request's commits through GraphQL, 100 per page with the owner, name, number, and cursor as variables, until `maxCommits`; every page must report the head and commit count of the first one and a cursor no earlier page returned, else the listing is `unavailable`. `compare` reads the patch from the merge base of two full commits to the second through the same compare request as `diff`. A refused read becomes `not-found` for HTTP 404, `unauthenticated` for a diagnostic that mentions authentication, `too-large` for GitHub's HTTP 406 refusal or `too_large` error, which a diff or file over its size limit draws, and `unavailable` otherwise, with GitHub's message. A read whose output exceeds `maxOutputBytes` becomes `too-large`, and one that outlives `timeoutMs` becomes `unavailable`. +`diff` reads `headRefOid` and `baseRefOid` in one GraphQL query and returns `stale` when the head is not the requested commit, which must be a full 40- or 64-digit hexadecimal commit. Otherwise it reads `repos///compare/...` through `gh api --allow-escape-sequences` with the diff media type, the patch from the merge base of both commits, so neither end follows a moving branch. Each status snapshot carries `baseRefOid` as `baseSha`; given it, `diff` starts this comparison while the GraphQL query confirms the head, and a stale head or another base commit cancels the early comparison and waits for it to end before `diff` returns. For every open and merged pull request measured, this patch is byte-identical to `gh pr diff`, which takes no commit argument. Unlike the pull request diff, the compare endpoint returns diffs over 300 files, so only output over `maxOutputBytes` or GitHub's own refusal makes such a diff `too-large`. `--allow-escape-sequences` keeps terminal escape sequences that file contents hold, which `gh` 2.97 and newer otherwise refuse to print; an older `gh` rejects the option with `unknown flag` before sending a request and prints such sequences unchanged, so that rejection repeats the read without it. `fileText` reads `repos///contents/?ref=` through `gh api` with the raw media type and accepts only a full 40- or 64-digit hexadecimal commit. `fileUrl` returns `https://///blob//`. Both paths percent-encode each segment. `commits` reads a pull request's commits through GraphQL, 100 per page with the owner, name, number, and cursor as variables, until `maxCommits`; every page must report the head and commit count of the first one and a cursor no earlier page returned, else the listing is `unavailable`. `compare` reads the patch from the merge base of two full commits to the second through the same compare request as `diff`. A refused read becomes `not-found` for HTTP 404, `unauthenticated` for a diagnostic that mentions authentication, `too-large` for GitHub's HTTP 406 refusal or `too_large` error, which a diff or file over its size limit draws, and `unavailable` otherwise, with GitHub's message. A read whose output exceeds `maxOutputBytes` becomes `too-large`, and one that outlives `timeoutMs` becomes `unavailable`. Review submission checks the current pull request head, then posts a single inline comment or one completed review through the REST API with an explicit `commit_id`. Multiline comments carry their side and inclusive start and end lines. Review decisions are Comment, Approve, and Request changes. JSON request bodies travel on stdin. The provider never retries a mutation: a timeout, unexpected response, or server error after dispatch returns an unknown outcome that the caller must reconcile on GitHub. A commit arriving between the check and the write can make the explicitly addressed review outdated; GitHub does not offer an atomic head precondition on these endpoints. @@ -100,7 +100,7 @@ None. Provider output reaches only host-side watchers. - **Batch deadline** — continuation pages of a pull request with many review threads lengthen its host's polling batch; a batch that outlives `ctx.forge`'s `fetchTimeoutMs` reports `unavailable` for every pull request in it, not only for the one with many threads. - **Rewritten base branch** — the diff starts from the merge base of the head and the base commit GitHub recorded at the pull request's last update. After the base branch is force-pushed, that merge base can differ from the one GitHub's pull request view uses, so the patch can include or omit commits until the pull request is updated. - **No fork discovery** — head lookups exclude pull requests whose head is in another repository. -- **Rate limits** — every poll costs one GraphQL query per host, plus one query per further page of 100 review threads, and a complete check read one more query per further page of 100 check records; every diff read costs one GraphQL query and one REST request. All count against the signed-in account’s rate limit. +- **Rate limits** — every poll costs one GraphQL query per host, plus one query per further page of 100 review threads, and a complete check read one more query per further page of 100 check records; every diff read costs one GraphQL query and one REST request, and a second REST request when the base commit changed after the latest status. All count against the signed-in account’s rate limit. - **No merge queue** — a base branch that requires a merge queue refuses a direct merge, and the provider never queues one. diff --git a/packages/experimental/forge-github/README.zh.md b/packages/experimental/forge-github/README.zh.md index 205a8f0439..0851001833 100644 --- a/packages/experimental/forge-github/README.zh.md +++ b/packages/experimental/forge-github/README.zh.md @@ -46,7 +46,7 @@ kind: "package-reference" pull request 处于打开状态、不是草稿、GitHub 报告其合并状态为 `CLEAN`、`HAS_HOOKS` 或 `UNSTABLE`，且已登录用户具有 write、maintain 或 admin 权限时，`mergeReady` 为真。`mergeMethods` 列出仓库允许的压缩合并、合并提交与变基合并方式，用户默认的合并方式排在最前。标记为可评审运行 `gh pr ready --repo //`。合并经 `gh api` 调用 `PUT repos///pulls//merge`，带上合并方式与预期的 head SHA，因此有新提交到达后 GitHub 会拒绝，报告为 `stale`。与 `gh pr merge` 不同，它从不启用自动合并、从不加入合并队列，也从不删除 head 分支。被拒绝的操作报告 GitHub 的原因，提到身份验证的诊断转为 `unauthenticated`。 -`diff` 在一次 GraphQL 查询中读取 `headRefOid` 与 `baseRefOid`，head 不是请求的提交时返回 `stale`；请求的提交必须是完整的 40 位或 64 位十六进制提交。否则经 `gh api --allow-escape-sequences` 以 diff 媒体类型读取 `repos///compare/...`，即两个提交的合并基准到 head 的补丁，两端都不跟随移动的分支。实测的所有 open 与已合并 pull request 上，该补丁与不接受提交参数的 `gh pr diff` 逐字节一致。与 pull request diff 不同，compare 接口会返回超过 300 个文件的 diff，因此这类 diff 只有在输出超过 `maxOutputBytes` 或 GitHub 自身拒绝时才为 `too-large`。`--allow-escape-sequences` 保留文件内容中的终端转义序列；否则 `gh` 2.97 及以上版本拒绝输出。更早的 `gh` 在发出请求前就以 `unknown flag` 拒绝该选项，且原样输出这类序列，因此遇到这一拒绝时去掉该选项重读一次。`fileText` 经 `gh api` 以 raw 媒体类型读取 `repos///contents/?ref=`，只接受完整的 40 位或 64 位十六进制提交。`fileUrl` 返回 `https://///blob//`。两者的路径都逐段做百分号编码。`commits` 经 GraphQL 读取拉取请求的提交，每页 100 个，owner、name、number 与游标作为变量传入，直到 `maxCommits`；每一页报告的 head 与提交总数必须与第一页相同，游标也不能与之前任何一页重复，否则返回 `unavailable`。`compare` 用与 `diff` 相同的 compare 请求读取两个完整提交的合并基准到第二个提交的补丁。被拒绝的读取在 HTTP 404 时为 `not-found`，诊断提到身份验证时为 `unauthenticated`，GitHub 以 HTTP 406 拒绝或报告 `too_large` 错误时（diff 或文件超过其大小上限即会如此）为 `too-large`，其余为 `unavailable`，并带有 GitHub 的原因。输出超过 `maxOutputBytes` 的读取为 `too-large`，超过 `timeoutMs` 的为 `unavailable`。 +`diff` 在一次 GraphQL 查询中读取 `headRefOid` 与 `baseRefOid`，head 不是请求的提交时返回 `stale`；请求的提交必须是完整的 40 位或 64 位十六进制提交。否则经 `gh api --allow-escape-sequences` 以 diff 媒体类型读取 `repos///compare/...`，即两个提交的合并基准到 head 的补丁，两端都不跟随移动的分支。每个状态快照把 `baseRefOid` 作为 `baseSha` 携带；拿到它时，`diff` 在 GraphQL 查询确认 head 的同时就开始这次比较，head 已过时或基准提交不同时会取消提前开始的比较并等它结束，然后才返回。实测的所有 open 与已合并 pull request 上，该补丁与不接受提交参数的 `gh pr diff` 逐字节一致。与 pull request diff 不同，compare 接口会返回超过 300 个文件的 diff，因此这类 diff 只有在输出超过 `maxOutputBytes` 或 GitHub 自身拒绝时才为 `too-large`。`--allow-escape-sequences` 保留文件内容中的终端转义序列；否则 `gh` 2.97 及以上版本拒绝输出。更早的 `gh` 在发出请求前就以 `unknown flag` 拒绝该选项，且原样输出这类序列，因此遇到这一拒绝时去掉该选项重读一次。`fileText` 经 `gh api` 以 raw 媒体类型读取 `repos///contents/?ref=`，只接受完整的 40 位或 64 位十六进制提交。`fileUrl` 返回 `https://///blob//`。两者的路径都逐段做百分号编码。`commits` 经 GraphQL 读取拉取请求的提交，每页 100 个，owner、name、number 与游标作为变量传入，直到 `maxCommits`；每一页报告的 head 与提交总数必须与第一页相同，游标也不能与之前任何一页重复，否则返回 `unavailable`。`compare` 用与 `diff` 相同的 compare 请求读取两个完整提交的合并基准到第二个提交的补丁。被拒绝的读取在 HTTP 404 时为 `not-found`，诊断提到身份验证时为 `unauthenticated`，GitHub 以 HTTP 406 拒绝或报告 `too_large` 错误时（diff 或文件超过其大小上限即会如此）为 `too-large`，其余为 `unavailable`，并带有 GitHub 的原因。输出超过 `maxOutputBytes` 的读取为 `too-large`，超过 `timeoutMs` 的为 `unavailable`。 提交评审时先检查 PR 当前的 head，再通过 REST API 发布单条行内评论或一次完整评审，并显式指定 `commit_id`。多行评论携带所属侧及包含端点的起止行号。评审结论为“评论”“批准”或“请求更改”。JSON 请求正文通过标准输入传递。provider 不会重试写操作；发送后的超时、异常响应或服务端错误会返回结果不明，由调用方到 GitHub 核对。检查与写入之间到来的新提交可能让已明确定位到旧提交的评审过时；GitHub 的这些接口不提供原子的 head 前置条件。 @@ -100,7 +100,7 @@ pull request 处于打开状态、不是草稿、GitHub 报告其合并状态为 - **批次时限**——评审讨论很多的 pull request 的续页会拉长其主机的轮询批次；批次超过 `ctx.forge` 的 `fetchTimeoutMs` 时，批次内每个 pull request 都报告 `unavailable`，而不只是讨论很多的那个。 - **基准分支被改写**——diff 从 head 与 GitHub 在 pull request 最后一次更新时记录的基准提交的合并基准开始。基准分支被强制推送后，该合并基准可能不同于 GitHub pull request 页面使用的合并基准，因此在 pull request 更新前，补丁可能多出或缺少提交。 - **不发现 fork**——按 head 分支查找时，不包含 head 位于其他仓库的 pull request。 -- **速率限制**——每次轮询为每个主机消耗一次 GraphQL 查询，评审讨论每多一页（100 个）再消耗一次；完整检查读取时检查记录每多一页（100 条）再消耗一次；每次 diff 读取消耗一次 GraphQL 查询与一次 REST 请求。这些都计入已登录账号的速率配额。 +- **速率限制**——每次轮询为每个主机消耗一次 GraphQL 查询，评审讨论每多一页（100 个）再消耗一次；完整检查读取时检查记录每多一页（100 条）再消耗一次；每次 diff 读取消耗一次 GraphQL 查询与一次 REST 请求；最新状态之后基准提交有变化时再多一次 REST 请求。这些都计入已登录账号的速率配额。 - **不支持合并队列**——要求合并队列的基准分支会拒绝直接合并，提供方也不会把变更请求加入队列。 diff --git a/packages/experimental/forge-github/src/graphql.ts b/packages/experimental/forge-github/src/graphql.ts index 75922515eb..92bcc4fbb2 100644 --- a/packages/experimental/forge-github/src/graphql.ts +++ b/packages/experimental/forge-github/src/graphql.ts @@ -32,7 +32,7 @@ const THREAD_FIELDS = 'pageInfo { hasNextPage endCursor } nodes { isResolved }' function pullRequestFields(threads: number): string { return `id number url title state isDraft updatedAt closedAt author { login } - headRefName headRefOid baseRefName additions deletions mergeable mergeStateStatus reviewDecision + headRefName headRefOid baseRefName baseRefOid additions deletions mergeable mergeStateStatus reviewDecision reviewThreads(first: ${threads}) { ${THREAD_FIELDS} } commits(last: 1) { nodes { commit { statusCheckRollup { id state contexts(first: ${PAGE_SIZE}) { ${CHECK_FIELDS} @@ -132,6 +132,7 @@ const pullRequestSchema = z.object({ headRefName: z.string(), headRefOid: z.string(), baseRefName: z.string(), + baseRefOid: z.string().refine(isCommit), additions: z.number().int().nonnegative(), deletions: z.number().int().nonnegative(), mergeable: z.string(), @@ -466,6 +467,7 @@ function toSnapshot( headBranch: node.headRefName, headSha: node.headRefOid, baseBranch: node.baseRefName, + baseSha: node.baseRefOid, additions: node.additions, deletions: node.deletions, mergeability: mergeability(node.mergeable), diff --git a/packages/experimental/forge-github/src/provider.ts b/packages/experimental/forge-github/src/provider.ts index 67e963e3d9..d94a277bb8 100644 --- a/packages/experimental/forge-github/src/provider.ts +++ b/packages/experimental/forge-github/src/provider.ts @@ -159,24 +159,44 @@ export class GitHubProvider implements ForgeProvider { * current head and the base commit GitHub recorded with it; a head other * than `headSha` returns `stale`. Otherwise the compare endpoint reads the * patch from the merge base of that base commit and `headSha`, as the pull - * request's own diff does, with both ends fixed. + * request's own diff does, with both ends fixed. Given the `base` the latest + * status recorded with `headSha`, the comparison from it runs while the + * GraphQL read confirms the head; a stale head or another base commit + * cancels that comparison and waits for it to end before the read returns. */ - async diff(ref: ChangeRequestRef, headSha: string, signal: AbortSignal): Promise { + async diff(ref: ChangeRequestRef, headSha: string, signal: AbortSignal, base?: string): Promise { const { host, owner, name } = checkedRepository(ref) const failure = this.hostFailure(host) if (failure !== undefined) return failure if (!isCommit(headSha)) throw new Error(`forge-github: invalid head commit ${JSON.stringify(headSha)}`) - const output = await this.read([ - 'api', 'graphql', '--hostname', host, - '-f', `query=${PULL_REQUEST_COMMITS_QUERY}`, '-f', `owner=${owner}`, '-f', `name=${name}`, '-F', `number=${ref.number}`, - ], signal) - if (typeof output !== 'string') return output - const commits = mapPullRequestCommitsResponse(parseJson(output)) - if (commits === undefined) return { code: 'unavailable', message: 'unexpected GitHub response to the pull request commit read' } - if (commits.head.toLowerCase() !== headSha.toLowerCase()) { - return { code: 'stale', message: `the pull request head moved from ${headSha} to ${commits.head}` } + if (base !== undefined && !isCommit(base)) throw new Error(`forge-github: invalid base commit ${JSON.stringify(base)}`) + const compare = async (from: string, operation: AbortSignal): Promise => + await readComparison(async (args, inner) => await this.read(args, inner), ref.repository, from, headSha, operation) + const speculation = new AbortController() + const early = base === undefined ? undefined : { base, patch: compare(base, AbortSignal.any([signal, speculation.signal])) } + // The early comparison is observed here, so a cancelled or unused one never rejects unhandled. + const settled = early?.patch.then(() => undefined, () => undefined) + const discard = async (): Promise => { + speculation.abort() + await settled + } + try { + const output = await this.read([ + 'api', 'graphql', '--hostname', host, + '-f', `query=${PULL_REQUEST_COMMITS_QUERY}`, '-f', `owner=${owner}`, '-f', `name=${name}`, '-F', `number=${ref.number}`, + ], signal) + if (typeof output !== 'string') return output + const commits = mapPullRequestCommitsResponse(parseJson(output)) + if (commits === undefined) return { code: 'unavailable', message: 'unexpected GitHub response to the pull request commit read' } + if (commits.head.toLowerCase() !== headSha.toLowerCase()) { + return { code: 'stale', message: `the pull request head moved from ${headSha} to ${commits.head}` } + } + if (early?.base.toLowerCase() === commits.base.toLowerCase()) return await early.patch + await discard() + return await compare(commits.base, signal) + } finally { + await discard() } - return await readComparison(async (args, operation) => await this.read(args, operation), ref.repository, commits.base, headSha, signal) } /** Read one file at a commit through the contents API in its raw form. */ diff --git a/packages/experimental/forge-github/tests/fixtures.ts b/packages/experimental/forge-github/tests/fixtures.ts index 66f2f47652..bc5d1297b0 100644 --- a/packages/experimental/forge-github/tests/fixtures.ts +++ b/packages/experimental/forge-github/tests/fixtures.ts @@ -54,6 +54,7 @@ export function pullRequestNode(number: number, overrides: Record { headBranch: 'topic-1', headSha: 'f'.repeat(40), baseBranch: 'master', + baseSha: 'b'.repeat(40), additions: 128, deletions: 14, mergeability: 'mergeable', diff --git a/packages/experimental/forge-github/tests/provider.spec.ts b/packages/experimental/forge-github/tests/provider.spec.ts index 1993c8cee6..d923d7425e 100644 --- a/packages/experimental/forge-github/tests/provider.spec.ts +++ b/packages/experimental/forge-github/tests/provider.spec.ts @@ -1,5 +1,5 @@ import { changeRequestKey } from '@deepseek-ai/dsh-experimental-forge' -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { GhOutputLimitError, GhTimeoutError, GhUnavailableError } from '../src/gh.ts' import type { GhOutput, GhRunner } from '../src/gh.ts' import { PULL_REQUEST_COMMITS_QUERY, THREADS_QUERY } from '../src/graphql.ts' @@ -446,6 +446,55 @@ describe('GitHub provider reads', () => { expect(gh.calls[1]?.at(-1)).toBe(`repos/octo/demo/compare/${base}...${head.toUpperCase()}`) }) + it('starts the comparison from the base the status recorded while it confirms the head', async () => { + const confirmed = Promise.withResolvers() + const gh = fakeGh(args => args.includes(`query=${PULL_REQUEST_COMMITS_QUERY}`) + ? confirmed.promise : output('diff --git a/x b/x\n')) + + const read = provider(gh.run).diff(pull(7), head, signal, base) + await vi.waitFor(() => { expect(gh.calls).toHaveLength(2) }) + expect(gh.calls.map(call => call.at(-1))).toContain(`repos/octo/demo/compare/${base}...${head}`) + confirmed.resolve(output({ data: { repository: { pullRequest: { headRefOid: head, baseRefOid: base.toUpperCase() } } } })) + + await expect(read).resolves.toBe('diff --git a/x b/x\n') + expect(gh.calls).toHaveLength(2) + }) + + it('cancels the early comparison and waits for it when the head moved, the base differs, or the head read fails', async () => { + const moved = 'e'.repeat(40) + for (const [answer, expected, compares] of [ + [output({ data: { repository: { pullRequest: { headRefOid: pushed, baseRefOid: base } } } }), + { code: 'stale', message: `the pull request head moved from ${head} to ${pushed}` }, [base]], + [output({ data: { repository: { pullRequest: { headRefOid: head, baseRefOid: moved } } } }), 'diff --git a/y b/y\n', [base, moved]], + [output('', 1, 'gh: Something went wrong (HTTP 502)'), { code: 'unavailable', message: 'gh: Something went wrong (HTTP 502)' }, [base]], + ] as const) { + const calls: string[][] = [] + const cancelled: string[] = [] + const run: GhRunner = async (args, operation) => { + calls.push([...args]) + if (args.includes(`query=${PULL_REQUEST_COMMITS_QUERY}`)) return answer + const range = String(args.at(-1)) + if (range.includes(`${moved}...`)) return output('diff --git a/y b/y\n') + // The early comparison runs until the provider cancels it. + await new Promise((resolve) => { operation.addEventListener('abort', resolve, { once: true }) }) + cancelled.push(range) + throw operation.reason + } + + await expect(provider(run).diff(pull(7), head, signal, base)).resolves.toEqual(expected) + expect(calls.map(call => String(call.at(-1))).filter(target => target.includes('/compare/')) + .map(target => target.split('/compare/')[1]?.split('...')[0])).toEqual(compares) + expect(cancelled).toEqual([`repos/octo/demo/compare/${base}...${head}`]) + } + }) + + it('refuses a base that is not a full commit object name before starting gh', async () => { + const gh = pinned() + + await expect(provider(gh.run).diff(pull(7), head, signal, 'HEAD~1')).rejects.toThrow('forge-github: invalid base commit "HEAD~1"') + expect(gh.calls).toEqual([]) + }) + it('reports a head that moved past the requested commit as stale without reading the diff', async () => { const gh = pinned({ head: pushed, base }) diff --git a/packages/experimental/forge-session/tests/loader-composition.spec.ts b/packages/experimental/forge-session/tests/loader-composition.spec.ts index 486e4f0635..f403e347a4 100644 --- a/packages/experimental/forge-session/tests/loader-composition.spec.ts +++ b/packages/experimental/forge-session/tests/loader-composition.spec.ts @@ -80,6 +80,7 @@ const node = (number) => ({ headRefName: number === 7 ? 'feat' : 'other', headRefOid: 'a'.repeat(40), baseRefName: 'main', + baseRefOid: 'e'.repeat(40), additions: 1, deletions: 0, mergeable: 'MERGEABLE', diff --git a/packages/experimental/forge/README.i18n.yaml b/packages/experimental/forge/README.i18n.yaml index ce02168f4c..19fb0423c8 100644 --- a/packages/experimental/forge/README.i18n.yaml +++ b/packages/experimental/forge/README.i18n.yaml @@ -15,8 +15,8 @@ en: d152484eb41ac6b4 zh: 2f8a621b991260a1 /deepseek-ai-dsh-experimental-forge/use-this-package: - en: 66ce801db0de565f - zh: 884973fad2ad2540 + en: 4b29b898061bcc4e + zh: fd37c2a8b06e7aaf /deepseek-ai-dsh-experimental-forge/understand-the-implementation: en: e193cb547c47069b zh: faa517522402ea41 diff --git a/packages/experimental/forge/README.md b/packages/experimental/forge/README.md index 8727e8cd2b..b316d57a78 100644 --- a/packages/experimental/forge/README.md +++ b/packages/experimental/forge/README.md @@ -36,7 +36,7 @@ Mount the registry with at least one provider package, such as the GitHub provid `markReady(ref, signal)` marks a draft ready for review, and `merge(ref, { method, headSha }, signal)` merges a change request only while its head is still `headSha`. Both delegate to the provider of the reference's forge, throw when none is registered, and resolve to undefined once the host accepted the change or to a `ForgeActionFailure`: `unauthenticated`, `stale` when the head moved past `headSha`, or `failed` with the host's reason. After either settles, a watched change request is polled at once. Snapshots carry `mergeReady`, whether the host would merge the change request now for the signed-in user, and `mergeMethods`, the methods the repository accepts with that user's usual one first. -`diff(ref, headSha, signal)` reads the change request's unified patch at head commit `headSha` as the host computes it, with `a/` and `b/` path prefixes, and `fileText(ref, commit, path, signal)` reads one file of its repository at a commit. Both delegate to the provider of the reference's forge, throw when none is registered, and resolve to the text or to a `ForgeFailure`, which is `too-large` when the host or the provider refuses an answer over its size limit. A provider returns a patch only when it describes `headSha`; once the head is another commit, `diff` resolves to `stale`. The first stale read at a head commit polls a watched change request at once so its next status names the current head; later stale reads at that commit wait for the regular poll, which bounds reads while the host's answers briefly disagree. Snapshots carry `unresolvedThreads` with `threadsTruncated`, which marks the count as a lower bound when the provider counted only some review threads. `fileUrl(ref, commit, path)` returns the web page of that file at that commit. +`diff(ref, headSha, signal)` reads the change request's unified patch at head commit `headSha` as the host computes it, with `a/` and `b/` path prefixes, and `fileText(ref, commit, path, signal)` reads one file of its repository at a commit. Both delegate to the provider of the reference's forge, throw when none is registered, and resolve to the text or to a `ForgeFailure`, which is `too-large` when the host or the provider refuses an answer over its size limit. A provider returns a patch only when it describes `headSha`; once the head is another commit, `diff` resolves to `stale`. The first stale read at a head commit polls a watched change request at once so its next status names the current head; later stale reads at that commit wait for the regular poll, which bounds reads while the host's answers briefly disagree. When the latest status of a watched change request names `headSha` and carries `baseSha`, the base commit the host recorded with that head, `diff` passes it to the provider, which may start reading from it before it confirms the head. Snapshots carry `unresolvedThreads` with `threadsTruncated`, which marks the count as a lower bound when the provider counted only some review threads. `fileUrl(ref, commit, path)` returns the web page of that file at that commit. Providers may implement `commits` and `compare` together. `commits(ref, signal)` lists a change request's commits oldest first, each with its full object name, first parent, title, author, and author time, at one head commit and with a `truncated` flag; `compare(ref, base, head, signal)` reads the unified patch from commit `base` to commit `head`. The registry returns undefined from `commits` and an `unavailable` failure from `compare` when the provider lacks either method, so a consumer can offer commit views only where both exist. diff --git a/packages/experimental/forge/README.zh.md b/packages/experimental/forge/README.zh.md index 168012126c..e931eec159 100644 --- a/packages/experimental/forge/README.zh.md +++ b/packages/experimental/forge/README.zh.md @@ -36,7 +36,7 @@ kind: "package-reference" `markReady(ref, signal)` 把草稿标记为可评审；`merge(ref, { method, headSha }, signal)` 只在 head 仍是 `headSha` 时合并变更请求。两者都委托给引用所属 forge 的提供方，没有注册提供方时抛错；主机接受时 resolve 为 undefined，否则为 `ForgeActionFailure`：`unauthenticated`、head 已越过 `headSha` 时的 `stale`，或带主机原因的 `failed`。两者结束后，被监听的变更请求会立即重新轮询。快照带有 `mergeReady`（主机现在是否会为已登录用户合并该变更请求）与 `mergeMethods`（仓库接受的合并方式，该用户常用的方式排在最前）。 -`diff(ref, headSha, signal)` 读取托管平台计算出的变更请求在 head 提交 `headSha` 处的统一补丁，路径带 `a/` 与 `b/` 前缀；`fileText(ref, commit, path, signal)` 读取其仓库中某个提交下的一个文件。两者都委托给引用所属 forge 的提供方，没有注册提供方时抛错，resolve 为文本或 `ForgeFailure`；托管平台或提供方因结果超过其大小上限而拒绝时，失败为 `too-large`。提供方只在补丁描述 `headSha` 时返回补丁；head 已是其他提交时，`diff` resolve 为 `stale`。同一 head 提交的第一次 stale 读取会让被 watch 的变更请求立即重新轮询，使下一次状态给出当前 head；之后在该提交上的 stale 读取等待常规轮询，从而在托管平台的应答短暂不一致时限制读取次数。快照携带 `unresolvedThreads` 与 `threadsTruncated`；提供方只统计了部分评审讨论时，后者表示该计数只是下限。`fileUrl(ref, commit, path)` 返回该文件在该提交下的网页地址。 +`diff(ref, headSha, signal)` 读取托管平台计算出的变更请求在 head 提交 `headSha` 处的统一补丁，路径带 `a/` 与 `b/` 前缀；`fileText(ref, commit, path, signal)` 读取其仓库中某个提交下的一个文件。两者都委托给引用所属 forge 的提供方，没有注册提供方时抛错，resolve 为文本或 `ForgeFailure`；托管平台或提供方因结果超过其大小上限而拒绝时，失败为 `too-large`。提供方只在补丁描述 `headSha` 时返回补丁；head 已是其他提交时，`diff` resolve 为 `stale`。同一 head 提交的第一次 stale 读取会让被 watch 的变更请求立即重新轮询，使下一次状态给出当前 head；之后在该提交上的 stale 读取等待常规轮询，从而在托管平台的应答短暂不一致时限制读取次数。被 watch 的变更请求的最新状态给出 `headSha` 并携带 `baseSha`（托管平台随该 head 记录的基准提交）时，`diff` 把它传给提供方，提供方可以在确认 head 之前就从它开始读取。快照携带 `unresolvedThreads` 与 `threadsTruncated`；提供方只统计了部分评审讨论时，后者表示该计数只是下限。`fileUrl(ref, commit, path)` 返回该文件在该提交下的网页地址。 提供方可以同时实现 `commits` 与 `compare`。`commits(ref, signal)` 在一个 head 提交上按从旧到新列出变更请求的提交，每个提交带完整对象名、第一个父提交、标题、作者与作者时间，并带 `truncated` 标志；`compare(ref, base, head, signal)` 读取从提交 `base` 到提交 `head` 的统一补丁。提供方缺少其中任一方法时，注册表的 `commits` 返回 undefined，`compare` 返回 `unavailable` 失败，因此使用方只在两者都存在时提供按提交查看。 diff --git a/packages/experimental/forge/src/index.ts b/packages/experimental/forge/src/index.ts index f52f77e818..4040c1f836 100644 --- a/packages/experimental/forge/src/index.ts +++ b/packages/experimental/forge/src/index.ts @@ -242,7 +242,8 @@ export class ForgeService extends Service { * computes them. The first `stale` failure at a head commit polls a watched * change request again at once, so its next status names the current head; * later stale reads at the same commit wait for the regular poll, which - * bounds the reads while the host's answers disagree. + * bounds the reads while the host's answers disagree. When the latest status of a watched change request names + * `headSha`, its `baseSha` goes to the provider, which may then read the patch while it confirms the head. * @param ref - change request served by a registered provider. * @param headSha - head commit the patch must describe, usually the head of the latest status. * @param signal - cancellation of the read. @@ -250,8 +251,10 @@ export class ForgeService extends Service { * @throws when no provider serves the change request's forge, or with `signal.reason` when it aborts. */ async diff(ref: ChangeRequestRef, headSha: string, signal: AbortSignal): Promise { + const latest = this.watches.get(changeRequestKey(ref))?.latest + const base = latest !== undefined && 'snapshot' in latest && latest.snapshot.headSha === headSha ? latest.snapshot.baseSha : undefined const result = await this.invoke(this.registration(ref.repository.forge), signal, - (provider, operation) => provider.diff(ref, headSha, operation)) + (provider, operation) => provider.diff(ref, headSha, operation, base)) const watch = this.watches.get(changeRequestKey(ref)) if (typeof result !== 'string' && result.code === 'stale' && watch !== undefined && watch.staleHead !== headSha) { watch.staleHead = headSha diff --git a/packages/experimental/forge/src/types.ts b/packages/experimental/forge/src/types.ts index 18c8b02e6d..b691744ec1 100644 --- a/packages/experimental/forge/src/types.ts +++ b/packages/experimental/forge/src/types.ts @@ -79,6 +79,11 @@ export interface ChangeRequestSnapshot { readonly headSha: string /** Branch the change request merges into. */ readonly baseBranch: string + /** + * Base commit the host recorded with `headSha`, whose merge base with the head starts the change request's diff; + * omitted when the provider does not report it. + */ + readonly baseSha?: string /** Lines the change request adds. */ readonly additions: number /** Lines the change request deletes. */ @@ -244,9 +249,10 @@ export interface ForgeProvider { /** * Read the changes of a change request at head commit `headSha` as the host computes them, as one unified * patch with `a/` and `b/` path prefixes. A patch is returned only when it describes `headSha`; once the - * head is another commit, the read resolves to a `stale` failure. + * head is another commit, the read resolves to a `stale` failure. `base`, when given, is the `baseSha` the + * latest status recorded with `headSha`; a provider may start reading from it before it confirms the head. */ - diff(ref: ChangeRequestRef, headSha: string, signal: AbortSignal): Promise + diff(ref: ChangeRequestRef, headSha: string, signal: AbortSignal, base?: string): Promise /** Read the text of `path` at `commit` in the change request's repository. */ fileText(ref: ChangeRequestRef, commit: string, path: string, signal: AbortSignal): Promise /** Build the web page of `path` at `commit` in the change request's repository. */ diff --git a/packages/experimental/forge/tests/watch.spec.ts b/packages/experimental/forge/tests/watch.spec.ts index 4537d82670..0988bb3ad1 100644 --- a/packages/experimental/forge/tests/watch.spec.ts +++ b/packages/experimental/forge/tests/watch.spec.ts @@ -223,6 +223,22 @@ describe('forge watch', () => { expect(provider.fetches).toHaveLength(3) }) + it('passes the base commit of the latest status to a diff read at its head only', async () => { + const provider = new FakeProvider() + provider.results.set(key, snapshot(ref(1), { baseSha: 'e'.repeat(40) })) + const context = await setup(provider) + collect(context) + await vi.advanceTimersByTimeAsync(0) + const diff = vi.spyOn(provider, 'diff') + const signal = new AbortController().signal + + await context.forge.diff(ref(1), 'a'.repeat(40), signal) + await context.forge.diff(ref(1), 'f'.repeat(40), signal) + await context.forge.diff(ref(2), 'a'.repeat(40), signal) + + expect(diff.mock.calls.map(call => call[3])).toEqual(['e'.repeat(40), undefined, undefined]) + }) + it('polls a watched change request at once after a diff read finds a newer head, and only then', async () => { const provider = new FakeProvider() provider.results.set(key, snapshot(ref(1))) diff --git a/packages/extensions/tool-cordis/src/api-catalog.ts b/packages/extensions/tool-cordis/src/api-catalog.ts index de37043f87..690a1a7476 100644 --- a/packages/extensions/tool-cordis/src/api-catalog.ts +++ b/packages/extensions/tool-cordis/src/api-catalog.ts @@ -1271,7 +1271,7 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ }, { signature: 'async diff(ref: ChangeRequestRef, headSha: string, signal: AbortSignal): Promise', - description: 'Read the changes of a change request at one head commit as its host computes them. The first `stale` failure at a head commit polls a watched change request again at once, so its next status names the current head; later stale reads at the same commit wait for the regular poll, which bounds the reads while the host\'s answers disagree.', + description: 'Read the changes of a change request at one head commit as its host computes them. The first `stale` failure at a head commit polls a watched change request again at once, so its next status names the current head; later stale reads at the same commit wait for the regular poll, which bounds the reads while the host\'s answers disagree. When the latest status of a watched change request names `headSha`, its `baseSha` goes to the provider, which may then read the patch while it confirms the head.', parameters: [{ name: 'ref', description: 'change request served by a registered provider.' }, { name: 'headSha', description: 'head commit the patch must describe, usually the head of the latest status.' }, { name: 'signal', description: 'cancellation of the read.' }], returns: 'one unified patch with `a/` and `b/` path prefixes, or why the host did not provide it at `headSha`.', throws: ['when no provider serves the change request\'s forge, or with `signal.reason` when it aborts.'], @@ -6027,7 +6027,7 @@ export const TYPE_API: readonly TypeApiEntry[] = [ }, { name: 'ChangeRequestSnapshot', - declaration: 'export interface ChangeRequestSnapshot {\n readonly ref: ChangeRequestRef;\n readonly url: string;\n readonly title: string;\n readonly state: ChangeRequestState;\n readonly draft: boolean;\n readonly author: string | null;\n readonly headBranch: string;\n readonly headSha: string;\n readonly baseBranch: string;\n readonly additions: number;\n readonly deletions: number;\n readonly mergeability: Mergeability;\n readonly mergeReady: boolean;\n readonly mergeMethods: readonly MergeMethod[];\n readonly reviewDecision: ReviewDecision;\n readonly checks: readonly ChangeRequestCheck[];\n readonly checksRollup: ChecksRollup;\n readonly checksTruncated: boolean;\n readonly unresolvedThreads: number;\n readonly threadsTruncated: boolean;\n readonly updatedAt: string;\n readonly closedAt: string | null;\n}', + declaration: 'export interface ChangeRequestSnapshot {\n readonly ref: ChangeRequestRef;\n readonly url: string;\n readonly title: string;\n readonly state: ChangeRequestState;\n readonly draft: boolean;\n readonly author: string | null;\n readonly headBranch: string;\n readonly headSha: string;\n readonly baseBranch: string;\n readonly baseSha?: string;\n readonly additions: number;\n readonly deletions: number;\n readonly mergeability: Mergeability;\n readonly mergeReady: boolean;\n readonly mergeMethods: readonly MergeMethod[];\n readonly reviewDecision: ReviewDecision;\n readonly checks: readonly ChangeRequestCheck[];\n readonly checksRollup: ChecksRollup;\n readonly checksTruncated: boolean;\n readonly unresolvedThreads: number;\n readonly threadsTruncated: boolean;\n readonly updatedAt: string;\n readonly closedAt: string | null;\n}', }, { name: 'ChangeRequestState', @@ -6731,7 +6731,7 @@ export const TYPE_API: readonly TypeApiEntry[] = [ }, { name: 'ForgeProvider', - declaration: 'export interface ForgeProvider {\n readonly forge: string;\n submitReview?(ref: ChangeRequestRef, request: ForgeReviewRequest, signal: AbortSignal): Promise;\n repositoryFromRemote(url: string): ForgeRepository | undefined;\n changeRequestFromUrl(url: string): ChangeRequestRef | undefined;\n changeRequestUrl(ref: ChangeRequestRef): string;\n fetch(refs: readonly ChangeRequestRef[], signal: AbortSignal): Promise>;\n fetchChecks?(ref: ChangeRequestRef, signal: AbortSignal): Promise;\n findByHead(repository: ForgeRepository, branch: string, signal: AbortSignal): Promise;\n readiness(host: string, signal: AbortSignal): Promise;\n markReady(ref: ChangeRequestRef, signal: AbortSignal): Promise;\n merge(ref: ChangeRequestRef, request: MergeRequest, signal: AbortSignal): Promise;\n diff(ref: ChangeRequestRef, headSha: string, signal: AbortSignal): Promise;\n fileText(ref: ChangeRequestRef, commit: string, path: string, signal: AbortSignal): Promise;\n fileUrl(ref: ChangeRequestRef, commit: string, path: string): string;\n commits?(ref: ChangeRequestRef, signal: AbortSignal): Promise;\n compare?(ref: ChangeRequestRef, base: string, head: string, s /* …truncated — full shape in source */', + declaration: 'export interface ForgeProvider {\n readonly forge: string;\n submitReview?(ref: ChangeRequestRef, request: ForgeReviewRequest, signal: AbortSignal): Promise;\n repositoryFromRemote(url: string): ForgeRepository | undefined;\n changeRequestFromUrl(url: string): ChangeRequestRef | undefined;\n changeRequestUrl(ref: ChangeRequestRef): string;\n fetch(refs: readonly ChangeRequestRef[], signal: AbortSignal): Promise>;\n fetchChecks?(ref: ChangeRequestRef, signal: AbortSignal): Promise;\n findByHead(repository: ForgeRepository, branch: string, signal: AbortSignal): Promise;\n readiness(host: string, signal: AbortSignal): Promise;\n markReady(ref: ChangeRequestRef, signal: AbortSignal): Promise;\n merge(ref: ChangeRequestRef, request: MergeRequest, signal: AbortSignal): Promise;\n diff(ref: ChangeRequestRef, headSha: string, signal: AbortSignal, base?: string): Promise;\n fileText(ref: ChangeRequestRef, commit: string, path: string, signal: AbortSignal): Promise;\n fileUrl(ref: ChangeRequestRef, commit: string, path: string): string;\n commits?(ref: ChangeRequestRef, signal: AbortSignal): Promise;\n compare?(ref: ChangeRequestRef, base: string, /* …truncated — full shape in source */', }, { name: 'ForgeReadiness', From 1754e2ea9843460fbb1546007567166a9cc42009 Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Sat, 7 Nov 2026 16:22:26 +0800 Subject: [PATCH 09/30] perf(forge-panel): draw only what changes and what the reader reaches - Read a change request's patch and .gitattributes together. - Publish a panel state only when it differs from the published one. - File sections receive stable values and redraw only when theirs change; tab visibility reaches only the menus that need it. - An expanded file draws its lines once its section comes within reach of the scrollport, and highlights them in idle time in bounded fragments, again only when its segments or its language's grammar change. - A jumped-to file stays aligned while the sections around it draw. --- apps/web/tests/forge-panel.e2e.ts | 4 +- .../client-ui-forge/README.i18n.yaml | 8 +- .../experimental/client-ui-forge/README.md | 8 +- .../experimental/client-ui-forge/README.zh.md | 8 +- .../src/client/ChangesTab.module.css | 6 + .../client-ui-forge/src/client/ChangesTab.tsx | 134 +++++++--- .../client-ui-forge/src/client/DiffFile.tsx | 229 +++++++++++++----- .../src/client/section-reach.ts | 86 +++++++ .../experimental/client-ui-forge/src/index.ts | 41 +++- .../experimental/client-ui-forge/src/panel.ts | 3 + .../tests/changes-tab.client.spec.tsx | 128 +++++++++- .../tests/controller.host.spec.ts | 64 ++++- .../tests/diff-review.client.spec.tsx | 7 +- .../client-ui-forge/tests/harness.ts | 4 +- .../tests/section-reach.client.spec.ts | 112 +++++++++ 15 files changed, 718 insertions(+), 124 deletions(-) create mode 100644 packages/experimental/client-ui-forge/src/client/section-reach.ts create mode 100644 packages/experimental/client-ui-forge/tests/section-reach.client.spec.ts diff --git a/apps/web/tests/forge-panel.e2e.ts b/apps/web/tests/forge-panel.e2e.ts index b732d7dea5..c7020636d8 100644 --- a/apps/web/tests/forge-panel.e2e.ts +++ b/apps/web/tests/forge-panel.e2e.ts @@ -141,7 +141,7 @@ if (args[0] === 'api' && args[1] === 'graphql') { const pulls = { 7: { id: 'pull-7', number: 7, url: 'https://github.com/octo/demo/pull/7', title: 'Add the pull request panel', state: 'OPEN', isDraft: false, - updatedAt: '2026-10-09T00:00:00Z', closedAt: null, author: { login: 'octo' }, headRefName: 'feat', headRefOid: 'a'.repeat(40), + updatedAt: '2026-10-09T00:00:00Z', closedAt: null, author: { login: 'octo' }, headRefName: 'feat', headRefOid: 'a'.repeat(40), baseRefOid: BASE, baseRefName: 'main', additions: 128, deletions: 14, mergeable: 'MERGEABLE', mergeStateStatus: 'BLOCKED', reviewDecision: 'REVIEW_REQUIRED', reviewThreads: { pageInfo: { hasNextPage: false, endCursor: null }, nodes: [{ isResolved: false }] }, commits: { nodes: [{ commit: { statusCheckRollup: { id: 'rollup-7', state: 'FAILURE', contexts: { @@ -152,7 +152,7 @@ if (args[0] === 'api' && args[1] === 'graphql') { 9: { id: 'pull-9', number: 9, url: 'https://github.com/octo/demo/pull/9', title: 'Add the merge button', state: state.mergedAt === null ? 'OPEN' : 'MERGED', isDraft: !state.ready, updatedAt: '2026-10-09T00:00:00Z', closedAt: state.mergedAt, author: { login: 'octo' }, - headRefName: 'feat/merge', headRefOid: heads()[9], baseRefName: 'main', additions: 30, deletions: 2, mergeable: 'MERGEABLE', + headRefName: 'feat/merge', headRefOid: heads()[9], baseRefName: 'main', baseRefOid: BASE, additions: 30, deletions: 2, mergeable: 'MERGEABLE', mergeStateStatus: state.ready ? 'CLEAN' : 'BLOCKED', reviewDecision: 'APPROVED', reviewThreads: { pageInfo: { hasNextPage: false, endCursor: null }, nodes: [] }, commits: { nodes: [{ commit: { statusCheckRollup: { id: 'rollup-9', state: 'SUCCESS', contexts: { diff --git a/packages/experimental/client-ui-forge/README.i18n.yaml b/packages/experimental/client-ui-forge/README.i18n.yaml index 5b25741184..a096484521 100644 --- a/packages/experimental/client-ui-forge/README.i18n.yaml +++ b/packages/experimental/client-ui-forge/README.i18n.yaml @@ -18,8 +18,8 @@ en: 05ee6d9ba0b8d2c7 zh: 98c988e8f957511f /deepseek-ai-dsh-experimental-client-ui-forge/understand-the-implementation: - en: 7495fe1e0d0ee643 - zh: f820a5790ae6e2a7 + en: 0afedb76112dfcb8 + zh: 52d5ddc7a4f9d31c /deepseek-ai-dsh-experimental-client-ui-forge/further-exploration: en: fa4d8c579ffc49ff zh: 5856ed7469e4585b @@ -30,8 +30,8 @@ en: 2bc25a99e17789c4 zh: 4bec723fbf249d28 /deepseek-ai-dsh-experimental-client-ui-forge/known-limitations-and-deferred-work: - en: 5e29ac36aad15d02 - zh: e46339bb637a69ef + en: 8f8ed6efb2fb7872 + zh: 489730ecefad078b /deepseek-ai-dsh-experimental-client-ui-forge/known-limitations-and-deferred-work/dev-note: en: e67f633dc6fe7773 zh: 3bb0c20c93f16f98 diff --git a/packages/experimental/client-ui-forge/README.md b/packages/experimental/client-ui-forge/README.md index 27ff8a01c3..62f8212ad2 100644 --- a/packages/experimental/client-ui-forge/README.md +++ b/packages/experimental/client-ui-forge/README.md @@ -58,7 +58,7 @@ All files opens a searchable directory tree with the selected file scrolled to i
Implementation internals — click to expand -`ForgePanelController` keeps one `SessionPanel` per followed Session and disposes it when the last follower leaves. A panel starts with the Session's working directory and bindings, re-inspects after each `working-directory/change`, `tool/result`, and `sandbox/mode` event, and publishes a new `ForgePanelState` for every change. Each change request keeps the latest snapshot a poll returned while later polls fail. A failed parallel Git read cancels and joins its siblings. Inspections run one at a time with one trailing rerun, and a result for a directory the Session has since left is dropped. The code host comes from the remotes in order: the upstream's remote, then `origin`, then the rest. When an inspection finds an upstream branch on a remote a provider recognizes, the panel asks `ctx.forgeSession.discoverBranch` to bind that branch's open change requests, once per remote and branch until a refresh. A sign-in probe runs when the recognized host changes and on refresh, within `readinessTimeoutMs`; a refresh settles once the inspection and the probe it started have ended, and poll results arrive later. +`ForgePanelController` keeps one `SessionPanel` per followed Session and disposes it when the last follower leaves. A panel starts with the Session's working directory and bindings, re-inspects after each `working-directory/change`, `tool/result`, and `sandbox/mode` event, and publishes a new `ForgePanelState` only when the state differs from the published one, so a reinspection that finds the same checkout reaches no follower. Each change request keeps the latest snapshot a poll returned while later polls fail. A failed parallel Git read cancels and joins its siblings. Inspections run one at a time with one trailing rerun, and a result for a directory the Session has since left is dropped. The code host comes from the remotes in order: the upstream's remote, then `origin`, then the rest. When an inspection finds an upstream branch on a remote a provider recognizes, the panel asks `ctx.forgeSession.discoverBranch` to bind that branch's open change requests, once per remote and branch until a refresh. A sign-in probe runs when the recognized host changes and on refresh, within `readinessTimeoutMs`; a refresh settles once the inspection and the probe it started have ended, and poll results arrive later. Inspection runs `rev-parse --show-toplevel`, `rev-parse --path-format=absolute --git-common-dir`, `rev-parse --path-format=absolute --git-dir`, `symbolic-ref --short HEAD`, `rev-parse --short HEAD`, `rev-parse --abbrev-ref @{upstream}`, and `remote -v`; `branches` runs `symbolic-ref` and `for-each-ref --count=`; `diffBases` finds the default fork point as `localDiff` does below, runs `for-each-ref --sort=-committerdate` over `refs/heads/` and `refs/remotes/` within `maxDiffBytes`, leaves out symbolic refs such as `origin/HEAD`, and keeps at most `maxBranches` branches of each kind whose names contain the query. Inspection then sizes the `branch` scope for the local changes card: it finds the fork point, runs `diff --numstat -z -M` against it with the same helper and submodule restrictions as `localDiff` below, and lists untracked files. Every command uses the Session's resolved sandbox mode: `read-only` and `workspace-write` require confinement, while `danger-full-access` runs without a sandbox backend. Commands disable optional index writes, hooks, the filesystem monitor, prompts, and lazy object fetches, remove inherited Git selectors and secret-bearing variables, and use their own `gitTimeoutMs` deadline and `maxOutputBytes` limit per stream. The top-level, Git, and common Git directories Git prints are resolved through `ctx.fs` into the canonical process paths that the working-directory service records; a checkout whose Git directory differs from the common one is a linked worktree. The worktree option is usable under the `danger-full-access` sandbox mode, never under `read-only`, and under `workspace-write` only when both directories lie inside the canonical workspace root of the Session's sandbox policy. @@ -69,11 +69,11 @@ Restoration records `restoring` before mutation and `restored` only after succes Each inspection also decides whether the worktree is missing: the Session's directory lies inside the created worktree, `ctx.fs.lstat` finds no directory there, and `show-ref` in the source repository tells whether the branch still exists. -`localDiff` takes the fork point from the created worktree's `baseCommit` when the Session works in it, else, for each remote in the code host order, from `symbolic-ref --quiet --short refs/remotes//HEAD` and `merge-base HEAD`; without one the `branch` scope refuses with `forgePanel/no-fork-point`. A `branch` scope that names a base ref under `refs/heads/` or `refs/remotes/` instead takes `merge-base HEAD` with it once `show-ref --verify` finds the ref, refusing with `forgePanel/base-missing` when the ref does not exist and with `forgePanel/no-fork-point` when HEAD shares no history with it; another ref name is refused before Git runs. It runs `diff --no-ext-diff --no-textconv --submodule=short --ignore-submodules=dirty --no-color -M` with fixed `a/` and `b/` prefixes and `core.quotePath=false` against the fork point, against HEAD (the empty tree before the first commit), against the index, or with `--cached` against HEAD. Before patches and statistics, Git configuration is read to apply command-local clean/process filter overrides. A failed or truncated configuration read stops the comparison: `localDiff` reports `forgePanel/filter-config-failed`, with a retry after correcting Git configuration or the output limit; failed statistics leave the local changes card hidden. Filter enumeration uses `maxDiffBytes` for patches and `maxOutputBytes` for statistics. Untracked files, at most `maxUntrackedFiles` with the rest counted as omitted, come from `ls-files --others --exclude-standard -z` and are read through `ctx.fs` as new files, binary when their first 8,000 bytes hold a NUL byte or they are not UTF-8. A patch, file list, or file over `maxDiffBytes` refuses with `forgePanel/diff-too-large`, and a directory outside a checkout with `forgePanel/not-a-checkout`. The Host parses patches with jsdiff into files and hunks, so the browser receives no raw patch. It gives each file a digest of its paths, status, hunks, and `old mode` and `new mode` lines, plus the blob names of a binary change or the content of an untracked binary file, and a layer from the `dsh-review-layer` and `linguist-generated` lines of the root `.gitattributes`, later lines overriding earlier ones and a value other than a layer name leaving the layer unspecified, with patterns matched by Git's wildmatch rules in [`src/wildmatch.ts`](src/wildmatch.ts) and negated patterns skipped as Git skips them, then from picomatch matches, with dot files included, of `testPaths`, `interfacePaths`, and the built-in rules in [`src/layers.ts`](src/layers.ts), with `.gitattributes` read from the index for Staged, the working tree for the other local scopes, the last chosen commit for chosen commits, and the head or last chosen commit through `fileText` for a change request, where a missing file reads as none. `localCommits` lists the commits between the fork point and HEAD with `log --no-show-signature --topo-order --reverse`, the newest `maxCommits` of them, so the list ends with HEAD's commit. A Branch scope with chosen commits compares the first one's first parent, or the empty tree for a root commit, with the last, refusing commits that are not full object names, not listed, or out of order with `forgePanel/commits-missing`, and a chosen commit whose first parent is not the commit listed before it with `forgePanel/commits-not-linear`; `localText` reads that last commit through `show`. `localText` reads the new side of one file, from the index through `show :` for Staged and through `ctx.fs` otherwise, and refuses a path that leaves the checkout. `changeRequestDiff` and `changeRequestText` act only on a visible binding a provider serves, else they refuse with `forgePanel/not-bound`; they read the head commit of the binding's latest status through the provider's `diff` and `fileText`, refuse a `too-large` failure with `forgePanel/diff-too-large`, a `stale` failure, meaning the head moved past that status, with `forgePanel/stale-head`, and refuse with `forgePanel/diff-failed` before a status was read or when the host refuses otherwise. A `stale` diff polls the change request at once, once per head commit, and is not cached, so no patch is ever kept under a head it does not describe. The Host keeps the patches `changeRequestDiff` read with their attributes, one per change request at its head commit and base branch, and drops the least recently read once they exceed `maxCachedDiffBytes`; a later read at the same head and base answers from memory for any Session, and a failed read is not kept. `changeRequestCommits` returns the provider's `commits` list, or null when the provider does not read commits; `changeRequestDiff` with chosen commits finds them in that list, refuses them as the Branch scope does, and reads `compare` from the first one's parent to the last, uncached. The commits the checkout has beyond the head come from `symbolic-ref`, `cat-file -e`, and `rev-list --count`, and are null when the checked-out branch is not the head branch, the head commit is absent, or Git fails. +`localDiff` takes the fork point from the created worktree's `baseCommit` when the Session works in it, else, for each remote in the code host order, from `symbolic-ref --quiet --short refs/remotes//HEAD` and `merge-base HEAD`; without one the `branch` scope refuses with `forgePanel/no-fork-point`. A `branch` scope that names a base ref under `refs/heads/` or `refs/remotes/` instead takes `merge-base HEAD` with it once `show-ref --verify` finds the ref, refusing with `forgePanel/base-missing` when the ref does not exist and with `forgePanel/no-fork-point` when HEAD shares no history with it; another ref name is refused before Git runs. It runs `diff --no-ext-diff --no-textconv --submodule=short --ignore-submodules=dirty --no-color -M` with fixed `a/` and `b/` prefixes and `core.quotePath=false` against the fork point, against HEAD (the empty tree before the first commit), against the index, or with `--cached` against HEAD. Before patches and statistics, Git configuration is read to apply command-local clean/process filter overrides. A failed or truncated configuration read stops the comparison: `localDiff` reports `forgePanel/filter-config-failed`, with a retry after correcting Git configuration or the output limit; failed statistics leave the local changes card hidden. Filter enumeration uses `maxDiffBytes` for patches and `maxOutputBytes` for statistics. Untracked files, at most `maxUntrackedFiles` with the rest counted as omitted, come from `ls-files --others --exclude-standard -z` and are read through `ctx.fs` as new files, binary when their first 8,000 bytes hold a NUL byte or they are not UTF-8. A patch, file list, or file over `maxDiffBytes` refuses with `forgePanel/diff-too-large`, and a directory outside a checkout with `forgePanel/not-a-checkout`. The Host parses patches with jsdiff into files and hunks, so the browser receives no raw patch. It gives each file a digest of its paths, status, hunks, and `old mode` and `new mode` lines, plus the blob names of a binary change or the content of an untracked binary file, and a layer from the `dsh-review-layer` and `linguist-generated` lines of the root `.gitattributes`, later lines overriding earlier ones and a value other than a layer name leaving the layer unspecified, with patterns matched by Git's wildmatch rules in [`src/wildmatch.ts`](src/wildmatch.ts) and negated patterns skipped as Git skips them, then from picomatch matches, with dot files included, of `testPaths`, `interfacePaths`, and the built-in rules in [`src/layers.ts`](src/layers.ts), with `.gitattributes` read from the index for Staged, the working tree for the other local scopes, the last chosen commit for chosen commits, and the head or last chosen commit through `fileText` for a change request, where a missing file reads as none. `localCommits` lists the commits between the fork point and HEAD with `log --no-show-signature --topo-order --reverse`, the newest `maxCommits` of them, so the list ends with HEAD's commit. A Branch scope with chosen commits compares the first one's first parent, or the empty tree for a root commit, with the last, refusing commits that are not full object names, not listed, or out of order with `forgePanel/commits-missing`, and a chosen commit whose first parent is not the commit listed before it with `forgePanel/commits-not-linear`; `localText` reads that last commit through `show`. `localText` reads the new side of one file, from the index through `show :` for Staged and through `ctx.fs` otherwise, and refuses a path that leaves the checkout. `changeRequestDiff` and `changeRequestText` act only on a visible binding a provider serves, else they refuse with `forgePanel/not-bound`; they read the head commit of the binding's latest status through the provider's `diff` and `fileText`, a patch together with its `.gitattributes`, where the first failure cancels the other read and waits for it to end, refuse a `too-large` failure with `forgePanel/diff-too-large`, a `stale` failure, meaning the head moved past that status, with `forgePanel/stale-head`, and refuse with `forgePanel/diff-failed` before a status was read or when the host refuses otherwise. A `stale` diff polls the change request at once, once per head commit, and is not cached, so no patch is ever kept under a head it does not describe. The Host keeps the patches `changeRequestDiff` read with their attributes, one per change request at its head commit and base branch, and drops the least recently read once they exceed `maxCachedDiffBytes`; a later read at the same head and base answers from memory for any Session, and a failed read is not kept. `changeRequestCommits` returns the provider's `commits` list, or null when the provider does not read commits; `changeRequestDiff` with chosen commits finds them in that list, refuses them as the Branch scope does, and reads `compare` from the first one's parent to the last, uncached. The commits the checkout has beyond the head come from `symbolic-ref`, `cat-file -e`, and `rev-list --count`, and are null when the checked-out branch is not the head branch, the head commit is absent, or Git fails. `changeRequestStates` streams, for the Session list, the last known number, title, state, draft flag, checks rollup, review decision, and mergeability of every change request that a listed Session the caller may read binds: a `baseline` frame, then `changed` frames. One run serves every stream. Cancellation is observed before initialization and between frames; the last stream and plugin disposal await initialization and accepted writes, then close the run’s storage domain. A later run waits for that closure before reopening the domain and reading its stored states. Binding updates received during listing override its initial snapshot. It lists Sessions through `sessionQuery` without loading them, takes the visible bindings of a loaded Session from `ctx.forgeSession` and of another from its cached `forgeChangeRequests` summary, follows binding changes of loaded Sessions, and subscribes to each bound change request through `ctx.forge.watch`; closed requests use the idle polling cadence, and merged requests remain subscribed for explicit refresh. It stores every changed state in the `forge_panel_change_requests` storage domain under the SHA-256 digest of its URL and drops unbound stored states only when every listed Session has known bindings; a failed write is logged and the state stays in memory. The stream fails when the Host lacks the Session query, projection cache, storage, or Session store service. -The browser entry mounts the Remote, then the UI. One stream per Session feeds the tab, the header button, the New Session option, and the creation card; it opens with the first subscriber, closes with the last, reopens after a transport reset, and keeps the last state while the Session scope is retained. Releasing that scope closes the stream and removes its cached state. The New Session option fills `conversation.hero.options` and the worktree card `conversation.input.dock`. The worktree card shows a running creation, with Cancel calling `stopCreation` while a message waits for it, or the failed or stopped attempt a message waits after, with Retry calling `retryWorktree` and Continue without a worktree calling `cancelWorktree`. The tab shortens paths with the Host home from the Remote's Host facts, read again after each transport reset. The tab reports its visibility to the header button's pressed state. The header button collapses a docked tab or closes its floating tab. The toast lives in a `shell.overlay` entry so it outlives the tab. One stream of change request states feeds the Session row icons in `sidebar.session.row.trailing` and the hover card section in `sidebar.session.row.hover` while any of them is mounted; both take a Session's worktree and bindings from its Session list summary values. The change request card takes a change request, its snapshot, and optional actions, so another surface can reuse it. The changes view is the `forge-changes` resource tab type: its addresses, `dsh-resource://forge-changes//local` and `dsh-resource://forge-changes//change/`, name the source, and a navigation may name a file to scroll to. The tab type enables `keepMounted`, retaining visited bodies and their owning Session references until their tab records end. Its keyed panel hook subscribes only for the visible body; other panel surfaces may keep the shared Session stream open. Its body keeps the scope, base branch, chosen commits, lens, test file switch, and layout choices of each tab in a slot store, keeps viewed marks per Session in `localStorage` under `dsh.forge-viewed.v1.`, at most 20 sources per Session and 50 Sessions, reads through the Remote, and highlights lines with the shared code highlighter; a chip title shows its name with the changes icon. +The browser entry mounts the Remote, then the UI. One stream per Session feeds the tab, the header button, the New Session option, and the creation card; it opens with the first subscriber, closes with the last, reopens after a transport reset, and keeps the last state while the Session scope is retained. Releasing that scope closes the stream and removes its cached state. The New Session option fills `conversation.hero.options` and the worktree card `conversation.input.dock`. The worktree card shows a running creation, with Cancel calling `stopCreation` while a message waits for it, or the failed or stopped attempt a message waits after, with Retry calling `retryWorktree` and Continue without a worktree calling `cancelWorktree`. The tab shortens paths with the Host home from the Remote's Host facts, read again after each transport reset. The tab reports its visibility to the header button's pressed state. The header button collapses a docked tab or closes its floating tab. The toast lives in a `shell.overlay` entry so it outlives the tab. One stream of change request states feeds the Session row icons in `sidebar.session.row.trailing` and the hover card section in `sidebar.session.row.hover` while any of them is mounted; both take a Session's worktree and bindings from its Session list summary values. The change request card takes a change request, its snapshot, and optional actions, so another surface can reuse it. The changes view is the `forge-changes` resource tab type: its addresses, `dsh-resource://forge-changes//local` and `dsh-resource://forge-changes//change/`, name the source, and a navigation may name a file to scroll to. The tab type enables `keepMounted`, retaining visited bodies and their owning Session references until their tab records end. Its keyed panel hook subscribes only for the visible body; other panel surfaces may keep the shared Session stream open. Its body keeps the scope, base branch, chosen commits, lens, test file switch, and layout choices of each tab in a slot store, keeps viewed marks per Session in `localStorage` under `dsh.forge-viewed.v1.`, at most 20 sources per Session and 50 Sessions, reads through the Remote, and passes each file section values that keep their identity across its renders, so a render redraws only the sections whose values changed. An expanded file draws its lines once its section first comes within 1200px of the scrollport, keeps them drawn after that, and until then holds a 22px row per drawn line; the lines are highlighted with the shared code highlighter when the page is next idle, again only when the file's segments change or its language's grammar finishes loading; a chip title shows its name with the changes icon. | Source | Responsibility | |---|---| @@ -92,6 +92,7 @@ The browser entry mounts the Remote, then the UI. One stream per Session feeds t | [`src/client/ChangesTab.tsx`](src/client/ChangesTab.tsx) | Changes view | | [`src/client/BaseMenu.tsx`](src/client/BaseMenu.tsx) | Base branch menu of the Branch scope | | [`src/client/DiffFile.tsx`](src/client/DiffFile.tsx) | One file of the changes view | +| [`src/client/section-reach.ts`](src/client/section-reach.ts) | When file sections draw their lines, and idle-time highlighting | | [`src/client/ReviewControls.tsx`](src/client/ReviewControls.tsx) | Lens, test file switch, and viewed progress | | [`src/client/CommitMenu.tsx`](src/client/CommitMenu.tsx) | Commit picker | | [`src/client/viewed-files.ts`](src/client/viewed-files.ts) | Viewed marks kept in the browser | @@ -139,6 +140,7 @@ None for a new worktree, as its notice belongs to the first request of the Sessi - **Changes scopes** — deferred: a scope for the last turn. The Branch scope's Default needs a remote whose default branch HEAD shares history with, or a worktree the Session created; otherwise a base branch must be chosen. - **Very many branches** — the base branch menu reads every ref before it filters, so a search can reach any branch. Refs over `maxDiffBytes`, about 165,000 at the default, or a listing over `gitTimeoutMs`, leave the menu saying that branches could not be read; Default keeps working. - **Large changes** — a local patch over `maxDiffBytes` shows no files, and a change request diff over the size limit of the host or its provider shows no files and offers only View in browser. Without wrapping, the side-by-side view cuts long lines at the column edge. +- **Find in page over long diffs** — an expanded file draws its lines only once the reader scrolls near it, so the browser's find in page does not match lines of files further down until they are drawn; a page-wide search of the diff needs a find control of the view's own. - **Viewed files on the code host** — viewed marks of a change request stay in this browser and are not read from or written to GitHub's viewed state (`markFileAsViewed`) until the provider's review operations exist; another browser or device starts without them. - **Review layers** — the built-in rules cover common conventions only, so a repository records others in `.gitattributes`; a path cannot show Rust's inline `#[cfg(test)]` modules or which declarations a language exports, and the Interface layer keeps whole files, so finding changed exports needs a language analyzer that a later lens extension point can supply. `linguist-generated` is read only from the root `.gitattributes`, without nested `.gitattributes` files or attribute macros. - **Chosen commits** — a change request lists at most `maxCommits` commits of the provider, the oldest 250 for GitHub, while the Branch scope lists its newest `maxCommits`; a run must follow first parents, so commits of a merged branch can be shown one at a time but not together with the branch's own; a commit range is read from the host on every view; comments on chosen commits can only be added to the input, as a GitHub review addresses the head. diff --git a/packages/experimental/client-ui-forge/README.zh.md b/packages/experimental/client-ui-forge/README.zh.md index 4aca1d2c38..bc4c12efd9 100644 --- a/packages/experimental/client-ui-forge/README.zh.md +++ b/packages/experimental/client-ui-forge/README.zh.md @@ -58,7 +58,7 @@ diff 按文件树的深度优先顺序排列：同层目录在前、文件在后
实现细节——点击展开 -`ForgePanelController` 为每个被关注的 Session 保留一个 `SessionPanel`，最后一个关注方离开时销毁它。面板从 Session 的工作目录与绑定开始，在每个 `working-directory/change`、`tool/result` 与 `sandbox/mode` 事件之后重新检查，每次变化都发布新的 `ForgePanelState`。每个变更请求在后续轮询失败时保留最近一次轮询得到的快照。并行 Git 读取失败时会取消并等待同批读取结束。检查一次只运行一个，最多再追加一次；Session 已离开的目录的结果会被丢弃。代码托管平台按远端顺序识别：先是上游所在的远端，然后是 `origin`，最后是其余远端。检查发现提供方识别的远端上有上游分支时，面板调用 `ctx.forgeSession.discoverBranch` 绑定该分支打开状态的变更请求；同一远端与分支在刷新前只查找一次。识别出的托管平台变化时以及刷新时，会在 `readinessTimeoutMs` 内检查登录状态；一次刷新在它触发的检查与登录状态检查结束后完成，轮询结果之后才到达。 +`ForgePanelController` 为每个被关注的 Session 保留一个 `SessionPanel`，最后一个关注方离开时销毁它。面板从 Session 的工作目录与绑定开始，在每个 `working-directory/change`、`tool/result` 与 `sandbox/mode` 事件之后重新检查，只在状态与已发布的不同时发布新的 `ForgePanelState`，所以检查结果与原来相同时不会通知任何关注方。每个变更请求在后续轮询失败时保留最近一次轮询得到的快照。并行 Git 读取失败时会取消并等待同批读取结束。检查一次只运行一个，最多再追加一次；Session 已离开的目录的结果会被丢弃。代码托管平台按远端顺序识别：先是上游所在的远端，然后是 `origin`，最后是其余远端。检查发现提供方识别的远端上有上游分支时，面板调用 `ctx.forgeSession.discoverBranch` 绑定该分支打开状态的变更请求；同一远端与分支在刷新前只查找一次。识别出的托管平台变化时以及刷新时，会在 `readinessTimeoutMs` 内检查登录状态；一次刷新在它触发的检查与登录状态检查结束后完成，轮询结果之后才到达。 检查运行 `rev-parse --show-toplevel`、`rev-parse --path-format=absolute --git-common-dir`、`rev-parse --path-format=absolute --git-dir`、`symbolic-ref --short HEAD`、`rev-parse --short HEAD`、`rev-parse --abbrev-ref @{upstream}` 与 `remote -v`；`branches` 运行 `symbolic-ref` 与 `for-each-ref --count=`；`diffBases` 像下文的 `localDiff` 一样找出默认分叉点，在 `maxDiffBytes` 以内对 `refs/heads/` 与 `refs/remotes/` 运行 `for-each-ref --sort=-committerdate`，排除 `origin/HEAD` 等符号引用，每类最多保留 `maxBranches` 个名称包含查询文字的分支。之后，检查会为本地变更卡片统计 `branch` 范围的规模：找出分叉点，对它运行 `diff --numstat -z -M`，采用与下文 `localDiff` 相同的辅助程序和子模块限制，并列出未跟踪文件。每个命令都使用 Session 解析出的沙箱模式：`read-only` 与 `workspace-write` 需要沙箱，`danger-full-access` 无需沙箱后端。命令禁用可选的索引写入、钩子、文件系统监视器、提示与对象的延迟获取，移除继承的 Git 选择器与携带密钥的变量，并各自受 `gitTimeoutMs` 时限与每个输出流的 `maxOutputBytes` 上限约束。Git 打印的顶层目录、Git 目录与公共 Git 目录经 `ctx.fs` 解析为工作目录服务记录的规范进程路径；Git 目录与公共 Git 目录不同的检出是链接 worktree。工作树选项在 `danger-full-access` 沙箱模式下可用，在 `read-only` 下不可用，在 `workspace-write` 下只有两个目录都位于 Session 沙箱策略的规范工作区根内时才可用。 @@ -69,11 +69,11 @@ worktree 请求是一条 `plugin:client-ui-forge/worktree` 记录，由 `forgeWo 每次检查还会判断 worktree 是否缺失：Session 的目录位于所创建的 worktree 内，`ctx.fs.lstat` 在那里找不到目录，再由源仓库中的 `show-ref` 判断分支是否还在。 -Session 在所创建的 worktree 中工作时，`localDiff` 以它的 `baseCommit` 为分叉点；否则按代码托管远端的顺序，对每个远端依次运行 `symbolic-ref --quiet --short refs/remotes//HEAD` 与 `merge-base HEAD` 找出分叉点；找不到时 `branch` 范围以 `forgePanel/no-fork-point` 拒绝。`branch` 范围指定了 `refs/heads/` 或 `refs/remotes/` 下的基准引用时，`show-ref --verify` 找到该引用后改用 `merge-base HEAD` 与它的结果；引用不存在时以 `forgePanel/base-missing` 拒绝，HEAD 与它没有共同历史时以 `forgePanel/no-fork-point` 拒绝；其他引用名在运行 Git 之前即被拒绝。它以固定的 `a/`、`b/` 前缀和 `core.quotePath=false` 运行 `diff --no-ext-diff --no-textconv --submodule=short --ignore-submodules=dirty --no-color -M`，分别与分叉点、HEAD（第一次提交之前为空树）、暂存区比较，或带 `--cached` 与 HEAD 比较。读取补丁与统计前会查询 Git 配置，应用命令级 clean/process filter 覆盖。配置读取失败或被截断时停止比较：`localDiff` 报告 `forgePanel/filter-config-failed`，修正 Git 配置或读取上限后可重试；统计失败时本地变更卡片保持隐藏。过滤器枚举在读取补丁时使用 `maxDiffBytes` 上限，统计时使用 `maxOutputBytes` 上限。未跟踪文件由 `ls-files --others --exclude-standard -z` 列出，最多 `maxUntrackedFiles` 个，其余计为未列出；它们经 `ctx.fs` 读取并作为新文件加入，前 8,000 字节含 NUL 字节或不是 UTF-8 时视为二进制。补丁、文件列表或文件超过 `maxDiffBytes` 时以 `forgePanel/diff-too-large` 拒绝，目录不在检出中时以 `forgePanel/not-a-checkout` 拒绝。宿主用 jsdiff 把补丁解析为文件与 hunk，浏览器不会收到原始补丁。Host 为每个文件计算由路径、状态、hunk 以及 `old mode` 与 `new mode` 行组成的摘要（二进制改动加上 blob 名，未跟踪的二进制文件加上内容），并先按根目录 `.gitattributes` 中的 `dsh-review-layer` 与 `linguist-generated` 行确定分层（后面的行覆盖前面的行，不是层名的值视为未指定；模式按 [`src/wildmatch.ts`](src/wildmatch.ts) 中 Git 的 wildmatch 规则匹配，取反模式与 Git 一样跳过），再用 picomatch（包含点文件）依次匹配 `testPaths`、`interfacePaths` 与 [`src/layers.ts`](src/layers.ts) 中的内置规则。`.gitattributes` 的读取来源：“已暂存”读暂存区，其他本地范围读工作树，所选提交读最后一个提交，变更请求经 `fileText` 读 head 或最后一个所选提交，文件不存在时按没有处理。`localCommits` 用 `log --no-show-signature --topo-order --reverse` 列出分叉点到 HEAD 之间最新的 `maxCommits` 个提交，因此列表以 HEAD 的提交结尾。带所选提交的“分支”范围比较第一个提交的第一个父提交（根提交用空树）与最后一个提交；不是完整对象名、不在列表中或顺序颠倒的提交以 `forgePanel/commits-missing` 拒绝，第一父提交不是列表中前一个提交的所选提交以 `forgePanel/commits-not-linear` 拒绝；`localText` 经 `show` 读取那个最后的提交。`localText` 读取一个文件的新版本：“已暂存”经 `show :` 从暂存区读取，其余经 `ctx.fs` 读取；离开检出的路径会被拒绝。`changeRequestDiff` 与 `changeRequestText` 只作用于该 Session 中由提供方服务的可见绑定，否则以 `forgePanel/not-bound` 拒绝；它们经提供方的 `diff` 与 `fileText` 读取绑定最新状态中的 head 提交，`too-large` 失败以 `forgePanel/diff-too-large` 拒绝，`stale` 失败（head 已越过该状态）以 `forgePanel/stale-head` 拒绝，尚未读到状态或托管平台以其他原因拒绝时以 `forgePanel/diff-failed` 拒绝。`stale` 的 diff 会立即重新轮询该变更请求（每个 head 提交一次）且不缓存，因此不会有补丁被存放在它不描述的 head 下。Host 保留 `changeRequestDiff` 读到的补丁及其属性，每个变更请求一份，按 head 提交与目标分支区分，总量超过 `maxCachedDiffBytes` 时丢弃最久未读的；之后在同一 head 与目标分支下的读取不论来自哪个 Session 都从内存返回，读取失败的结果不保留。`changeRequestCommits` 返回提供方 `commits` 的列表，提供方不读取提交时返回 null；带所选提交的 `changeRequestDiff` 在该列表中找到它们，按与“分支”范围相同的规则拒绝，并从第一个提交的父提交到最后一个提交读取 `compare`，不缓存。本地分支比 head 多出的提交数由 `symbolic-ref`、`cat-file -e` 与 `rev-list --count` 得出；当前检出分支不是源分支、本地没有 head 提交或 Git 失败时为 null。 +Session 在所创建的 worktree 中工作时，`localDiff` 以它的 `baseCommit` 为分叉点；否则按代码托管远端的顺序，对每个远端依次运行 `symbolic-ref --quiet --short refs/remotes//HEAD` 与 `merge-base HEAD` 找出分叉点；找不到时 `branch` 范围以 `forgePanel/no-fork-point` 拒绝。`branch` 范围指定了 `refs/heads/` 或 `refs/remotes/` 下的基准引用时，`show-ref --verify` 找到该引用后改用 `merge-base HEAD` 与它的结果；引用不存在时以 `forgePanel/base-missing` 拒绝，HEAD 与它没有共同历史时以 `forgePanel/no-fork-point` 拒绝；其他引用名在运行 Git 之前即被拒绝。它以固定的 `a/`、`b/` 前缀和 `core.quotePath=false` 运行 `diff --no-ext-diff --no-textconv --submodule=short --ignore-submodules=dirty --no-color -M`，分别与分叉点、HEAD（第一次提交之前为空树）、暂存区比较，或带 `--cached` 与 HEAD 比较。读取补丁与统计前会查询 Git 配置，应用命令级 clean/process filter 覆盖。配置读取失败或被截断时停止比较：`localDiff` 报告 `forgePanel/filter-config-failed`，修正 Git 配置或读取上限后可重试；统计失败时本地变更卡片保持隐藏。过滤器枚举在读取补丁时使用 `maxDiffBytes` 上限，统计时使用 `maxOutputBytes` 上限。未跟踪文件由 `ls-files --others --exclude-standard -z` 列出，最多 `maxUntrackedFiles` 个，其余计为未列出；它们经 `ctx.fs` 读取并作为新文件加入，前 8,000 字节含 NUL 字节或不是 UTF-8 时视为二进制。补丁、文件列表或文件超过 `maxDiffBytes` 时以 `forgePanel/diff-too-large` 拒绝，目录不在检出中时以 `forgePanel/not-a-checkout` 拒绝。宿主用 jsdiff 把补丁解析为文件与 hunk，浏览器不会收到原始补丁。Host 为每个文件计算由路径、状态、hunk 以及 `old mode` 与 `new mode` 行组成的摘要（二进制改动加上 blob 名，未跟踪的二进制文件加上内容），并先按根目录 `.gitattributes` 中的 `dsh-review-layer` 与 `linguist-generated` 行确定分层（后面的行覆盖前面的行，不是层名的值视为未指定；模式按 [`src/wildmatch.ts`](src/wildmatch.ts) 中 Git 的 wildmatch 规则匹配，取反模式与 Git 一样跳过），再用 picomatch（包含点文件）依次匹配 `testPaths`、`interfacePaths` 与 [`src/layers.ts`](src/layers.ts) 中的内置规则。`.gitattributes` 的读取来源：“已暂存”读暂存区，其他本地范围读工作树，所选提交读最后一个提交，变更请求经 `fileText` 读 head 或最后一个所选提交，文件不存在时按没有处理。`localCommits` 用 `log --no-show-signature --topo-order --reverse` 列出分叉点到 HEAD 之间最新的 `maxCommits` 个提交，因此列表以 HEAD 的提交结尾。带所选提交的“分支”范围比较第一个提交的第一个父提交（根提交用空树）与最后一个提交；不是完整对象名、不在列表中或顺序颠倒的提交以 `forgePanel/commits-missing` 拒绝，第一父提交不是列表中前一个提交的所选提交以 `forgePanel/commits-not-linear` 拒绝；`localText` 经 `show` 读取那个最后的提交。`localText` 读取一个文件的新版本：“已暂存”经 `show :` 从暂存区读取，其余经 `ctx.fs` 读取；离开检出的路径会被拒绝。`changeRequestDiff` 与 `changeRequestText` 只作用于该 Session 中由提供方服务的可见绑定，否则以 `forgePanel/not-bound` 拒绝；它们经提供方的 `diff` 与 `fileText` 读取绑定最新状态中的 head 提交，补丁与它的 `.gitattributes` 同时读取，先失败的一方会取消另一方并等它结束，`too-large` 失败以 `forgePanel/diff-too-large` 拒绝，`stale` 失败（head 已越过该状态）以 `forgePanel/stale-head` 拒绝，尚未读到状态或托管平台以其他原因拒绝时以 `forgePanel/diff-failed` 拒绝。`stale` 的 diff 会立即重新轮询该变更请求（每个 head 提交一次）且不缓存，因此不会有补丁被存放在它不描述的 head 下。Host 保留 `changeRequestDiff` 读到的补丁及其属性，每个变更请求一份，按 head 提交与目标分支区分，总量超过 `maxCachedDiffBytes` 时丢弃最久未读的；之后在同一 head 与目标分支下的读取不论来自哪个 Session 都从内存返回，读取失败的结果不保留。`changeRequestCommits` 返回提供方 `commits` 的列表，提供方不读取提交时返回 null；带所选提交的 `changeRequestDiff` 在该列表中找到它们，按与“分支”范围相同的规则拒绝，并从第一个提交的父提交到最后一个提交读取 `compare`，不缓存。本地分支比 head 多出的提交数由 `symbolic-ref`、`cat-file -e` 与 `rev-list --count` 得出；当前检出分支不是源分支、本地没有 head 提交或 Git 失败时为 null。 `changeRequestStates` 为会话列表推送调用方可读的、已列出的 Session 所绑定的每个变更请求的最近编号、标题、状态、草稿标记、检查汇总、评审结论与可合并性：先是一个 `baseline` 帧，之后是 `changed` 帧。所有流共用一次运行。初始化之前和帧之间均响应取消；最后一条流结束及插件卸载会等待初始化与已接受的写入完成，再关闭本次运行的存储域。后续运行会等待该域关闭后重新打开，并读取已存状态。列举期间收到的绑定更新优先于初始快照。它经 `sessionQuery` 列出 Session 而不加载它们，已加载 Session 的可见绑定取自 `ctx.forgeSession`，其他 Session 的取自缓存的 `forgeChangeRequests` 摘要；它跟随已加载 Session 的绑定变化，并经 `ctx.forge.watch` 订阅每个已绑定的变更请求；已关闭的按空闲间隔轮询，已合并的保留订阅以接收显式刷新结果。每个变化了的状态以其 URL 的 SHA-256 摘要为键存入 `forge_panel_change_requests` 存储域，仅在所有已列出 Session 的绑定均已知时删除无人绑定的已存状态；写入失败时记录日志，状态仍保留在内存中。宿主缺少 Session 查询、投影缓存、存储或 Session 存储服务时，流以失败结束。 -浏览器入口先挂载 Remote，再挂载界面。每个 Session 一条流，同时供给标签页、标题栏按钮、新会话选项与创建卡片；第一个订阅方出现时打开，最后一个离开时关闭，传输重置后重新打开，在 Session scope 保留期间保存最后的状态。释放该 scope 时关闭流并移除缓存状态。新会话选项填入 `conversation.hero.options`，worktree 卡片填入 `conversation.input.dock`。worktree 卡片显示正在进行的创建，有消息在等待时“取消”调用 `stopCreation`；或显示消息等待时失败或被停止的那次尝试，“重试”调用 `retryWorktree`，“不使用工作树”调用 `cancelWorktree`。标签页用 Remote 宿主信息中的宿主主目录缩短路径，每次传输重置后重新读取。标签页把自己是否可见报告给标题栏按钮的按下状态。标题栏按钮收起停靠的标签页，或关闭其浮动标签页。toast 位于 `shell.overlay` 条目中，因此比标签页存在得更久。只要 `sidebar.session.row.trailing` 中的会话行图标或 `sidebar.session.row.hover` 中的悬浮卡片部分有一个已挂载，就由同一条变更请求状态流供给它们；两者都从会话列表的摘要值中读取 Session 的 worktree 与绑定。变更请求卡片接收变更请求、它的快照与可选操作，其他界面可以复用。变更视图是 `forge-changes` 资源标签页类型：它的地址 `dsh-resource://forge-changes//local` 与 `dsh-resource://forge-changes//change/` 指明来源，导航参数可以指定要滚动到的文件。标签页类型启用 `keepMounted`，将访问过的正文及其所属 Session 引用保留至标签页记录结束。它的 keyed panel hook 仅为可见正文订阅；其他面板界面仍可能保持共享 Session 流开启。它的正文用 slot store 按标签页保存范围、基准分支、所选提交、透镜、测试文件开关与布局选择，按 Session 把已审阅标记保存在 `localStorage` 的 `dsh.forge-viewed.v1.` 下（每个 Session 最多 20 个来源，最多 50 个 Session），经 Remote 读取，并用共享的代码高亮器高亮各行；标签页标题显示变更图标与视图名称。 +浏览器入口先挂载 Remote，再挂载界面。每个 Session 一条流，同时供给标签页、标题栏按钮、新会话选项与创建卡片；第一个订阅方出现时打开，最后一个离开时关闭，传输重置后重新打开，在 Session scope 保留期间保存最后的状态。释放该 scope 时关闭流并移除缓存状态。新会话选项填入 `conversation.hero.options`，worktree 卡片填入 `conversation.input.dock`。worktree 卡片显示正在进行的创建，有消息在等待时“取消”调用 `stopCreation`；或显示消息等待时失败或被停止的那次尝试，“重试”调用 `retryWorktree`，“不使用工作树”调用 `cancelWorktree`。标签页用 Remote 宿主信息中的宿主主目录缩短路径，每次传输重置后重新读取。标签页把自己是否可见报告给标题栏按钮的按下状态。标题栏按钮收起停靠的标签页，或关闭其浮动标签页。toast 位于 `shell.overlay` 条目中，因此比标签页存在得更久。只要 `sidebar.session.row.trailing` 中的会话行图标或 `sidebar.session.row.hover` 中的悬浮卡片部分有一个已挂载，就由同一条变更请求状态流供给它们；两者都从会话列表的摘要值中读取 Session 的 worktree 与绑定。变更请求卡片接收变更请求、它的快照与可选操作，其他界面可以复用。变更视图是 `forge-changes` 资源标签页类型：它的地址 `dsh-resource://forge-changes//local` 与 `dsh-resource://forge-changes//change/` 指明来源，导航参数可以指定要滚动到的文件。标签页类型启用 `keepMounted`，将访问过的正文及其所属 Session 引用保留至标签页记录结束。它的 keyed panel hook 仅为可见正文订阅；其他面板界面仍可能保持共享 Session 流开启。它的正文用 slot store 按标签页保存范围、基准分支、所选提交、透镜、测试文件开关与布局选择，按 Session 把已审阅标记保存在 `localStorage` 的 `dsh.forge-viewed.v1.` 下（每个 Session 最多 20 个来源，最多 50 个 Session），经 Remote 读取，并且传给每个文件段的值在多次渲染之间保持同一标识，所以一次渲染只重绘值有变化的文件段。展开的文件在它的段第一次进入滚动区域 1200px 范围内时才绘制各行，此后一直保留；在此之前按每个绘制行 22px 占位。各行在页面下次空闲时用共享的代码高亮器高亮，之后只在文件的分段变化或该语言的语法加载完成时重新高亮；标签页标题显示变更图标与视图名称。 | 源码 | 职责 | |---|---| @@ -92,6 +92,7 @@ Session 在所创建的 worktree 中工作时，`localDiff` 以它的 `baseCommi | [`src/client/ChangesTab.tsx`](src/client/ChangesTab.tsx) | 变更视图 | | [`src/client/BaseMenu.tsx`](src/client/BaseMenu.tsx) | “分支”范围的基准分支菜单 | | [`src/client/DiffFile.tsx`](src/client/DiffFile.tsx) | 变更视图中的一个文件 | +| [`src/client/section-reach.ts`](src/client/section-reach.ts) | 文件段何时绘制各行，以及空闲时高亮 | | [`src/client/ReviewControls.tsx`](src/client/ReviewControls.tsx) | 分层透镜、测试文件开关与已审阅进度 | | [`src/client/CommitMenu.tsx`](src/client/CommitMenu.tsx) | 提交选择器 | | [`src/client/viewed-files.ts`](src/client/viewed-files.ts) | 保存在浏览器中的已审阅标记 | @@ -139,6 +140,7 @@ Session 在所创建的 worktree 中工作时，`localDiff` 以它的 `baseCommi - **变更范围**——待办：上一轮的范围。“分支”范围的“默认”需要一个默认分支与 HEAD 有共同历史的远端，或 Session 创建的 worktree；否则必须选择基准分支。 - **分支极多**——基准分支菜单在筛选前读取全部引用，这样搜索才能找到任意分支。引用超过 `maxDiffBytes`（默认约 165,000 个）或列出时间超过 `gitTimeoutMs` 时，菜单提示无法读取分支；“默认”仍可使用。 - **大型变更**——本地补丁超过 `maxDiffBytes` 时不显示任何文件；超过托管平台或其提供方大小上限的变更请求 diff 不显示任何文件，只提供“在浏览器中查看”。不自动换行时，左右对比会在栏边截断长行。 +- **长 diff 中的页内查找**——展开的文件要等读者滚动到附近才绘制各行，所以浏览器的页内查找在更靠后的文件绘制之前匹配不到它们的行；要在整个 diff 中查找，需要视图自己的查找控件。 - **代码托管平台上的已审阅状态**——变更请求的已审阅标记只保存在本浏览器中，在提供方评审接口完成之前，不读取也不写入 GitHub 的已审阅状态（`markFileAsViewed`）；换浏览器或设备后没有这些标记。 - **分层**——内置规则只覆盖常见约定，其他约定由仓库写在 `.gitattributes` 中；按路径无法识别 Rust 的内联 `#[cfg(test)]` 模块，也无法判断一种语言导出了哪些声明，“接口”层保留整个文件，找出变更的导出需要语言分析器，可由之后的透镜扩展点提供。`linguist-generated` 只读取根目录的 `.gitattributes`，不读子目录中的 `.gitattributes`，也不展开属性宏。 - **所选提交**——变更请求最多列出提供方的 `maxCommits` 个提交（GitHub 为最早的 250 个），“分支”范围则列出最新的 `maxCommits` 个；范围必须沿第一父提交相连，因此已合并分支的提交只能逐个查看，不能与本分支的提交一起查看；每次查看提交范围都从托管平台读取；所选提交上的评论只能添加到输入框，因为 GitHub 评审针对 head。 diff --git a/packages/experimental/client-ui-forge/src/client/ChangesTab.module.css b/packages/experimental/client-ui-forge/src/client/ChangesTab.module.css index f664c4b2fa..a4ef2ab3e3 100644 --- a/packages/experimental/client-ui-forge/src/client/ChangesTab.module.css +++ b/packages/experimental/client-ui-forge/src/client/ChangesTab.module.css @@ -521,6 +521,12 @@ font: var(--dsw-font-markdown-code-block); } +/* An expanded file the reader has not scrolled near yet: one 22px row (`.line`) per drawn line, so the scroll + range and the positions of the files below stay close to their drawn values. */ +.unreached { + height: calc(var(--forge-rows) * 22px); +} + /* One intrinsic-width track paints every unwrapped unified row across the scrolled width. */ .code:not([data-wrap]):not([data-split]) { display: grid; diff --git a/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx b/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx index f57bb937bb..8902be11e2 100644 --- a/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx +++ b/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx @@ -5,6 +5,8 @@ * layout. The view reads the changes when it opens, on each navigation, on * refresh, and when the change request's head or the checkout's `branch` * scope summary changes; a read keeps the previous changes on screen until it settles. + * File sections receive values that keep their identity across renders, so a + * render of the view redraws only the sections whose own values changed. */ import { @@ -39,7 +41,8 @@ import type { ChangesLens, createChangesStore } from './changes-store.ts' import { CommitMenu } from './CommitMenu.tsx' import { HideTestsSwitch, LensControl, LensMenu, LENSES, ViewedProgress } from './ReviewControls.tsx' import { viewedStateOf, type ViewedFiles, type ViewedMarks } from './viewed-files.ts' -import { Counts, DiffFile, type DiffFileAction } from './DiffFile.tsx' +import { Counts, DiffFile, type DiffFileAction, type DiffFileHandlers } from './DiffFile.tsx' +import { SectionReach, SectionReachContext, ViewVisibleContext } from './section-reach.ts' import { FileTreePane, type FileTreePosition } from './FileTreePane.tsx' import { changedFileTree, filesInTreeOrder } from './file-tree.ts' import { ChangedFilesMenu, type ChangedFilesProps, type ChangedFilesState } from './ChangedFiles.tsx' @@ -145,6 +148,12 @@ const OPEN_LINES = 5000 const NO_GAPS: ReadonlySet = new Set() +/** How long, in milliseconds, a jumped-to file stays at the top while the sections around it draw. */ +const JUMP_PIN_MS = 1000 + +/** Input that scrolls the file list, which ends a jump's pin. */ +const READER_SCROLL = ['wheel', 'touchstart', 'pointerdown', 'keydown'] as const + const SCOPE_LABEL: Record = { branch: 'diff.scope.branch', uncommitted: 'diff.scope.uncommitted', @@ -564,7 +573,7 @@ export function ChangesTab({ const files = useMemo(() => allFiles.filter(file => shownBy(file, lens, hideTests)), [allFiles, lens, hideTests]) const lensCounts = useMemo(() => Object.fromEntries(LENSES.map(item => [item, allFiles.filter(file => shownBy(file, item, hideTests)).length])) as Record, [allFiles, hideTests]) - const shownSource = data === null ? null : reviewSourceOf(data) + const shownSource = useMemo(() => data === null ? null : reviewSourceOf(data), [data]) const viewedKey = shownSource === null ? '' : viewedSourceOf(shownSource) const marks = useSyncExternalStore(viewedFiles.subscribe, () => viewedFiles.marksOf(viewedKey)) // The centered loader marks a first read; the toolbar's glyph turns only while shown changes are read again. @@ -582,9 +591,44 @@ export function ChangesTab({ const expandedByDefault = (file: ForgeDiffFile): boolean => initial.has(file.path) && viewedStateOf(marks, file.path, file.digest) !== 'viewed' const isExpanded = (file: ForgeDiffFile): boolean => chosen[file.path] ?? expandedByDefault(file) const sections = useRef(new Map()) - const scrollport = useRef(null) + const scrollport = useRef(null) + const reach = useMemo(() => new SectionReach(), []) + const attachScrollport = useCallback((element: HTMLDivElement | null) => { + scrollport.current = element + reach.attach(element) + }, [reach]) // A short final file cannot align with the viewport top; retain its jump until the reader scrolls away. const jumpPosition = useRef<{ readonly data: ChangesData; readonly top: number } | null>(null) + const releasePin = useRef<(() => void) | null>(null) + /** + * Keep a jumped-to section at the top of the scrollport for {@link JUMP_PIN_MS}: the sections around it draw their + * lines after the jump, which moves it. Scrolling input from the reader, another jump, or unmounting ends the pin. + */ + const pinJump = (port: HTMLElement, section: HTMLElement, owner: ChangesData): void => { + releasePin.current?.() + const until = performance.now() + JUMP_PIN_MS + let frame = 0 + const release = (): void => { + cancelAnimationFrame(frame) + for (const type of READER_SCROLL) port.removeEventListener(type, release) + if (releasePin.current === release) releasePin.current = null + } + const keep = (now: number): void => { + if (now > until) { + release() + return + } + const offset = section.getBoundingClientRect().top - port.getBoundingClientRect().top + if (Math.abs(offset) > 1 && port.scrollTop < port.scrollHeight - port.clientHeight - 1) { + section.scrollIntoView({ block: 'start' }) + jumpPosition.current = { data: owner, top: port.scrollTop } + } + frame = requestAnimationFrame(keep) + } + for (const type of READER_SCROLL) port.addEventListener(type, release, { passive: true }) + releasePin.current = release + frame = requestAnimationFrame(keep) + } useLayoutEffect(() => { if (!tab.visible || wanted === null || data === null) return // A file the lens or the test switch hides is shown before the view scrolls to it. @@ -594,10 +638,13 @@ export function ChangesTab({ if (hideTests && hidden.layer === 'test') actions.toggledTests(tab.id) return } - sections.current.get(wanted)?.scrollIntoView({ block: 'start' }) + const section = sections.current.get(wanted) + section?.scrollIntoView({ block: 'start' }) if (scrollport.current !== null) jumpPosition.current = { data, top: scrollport.current.scrollTop } + if (section !== undefined && scrollport.current !== null) pinJump(scrollport.current, section, data) setWanted(null) }, [tab.visible, wanted, data, allFiles, lens, hideTests, actions, tab.id]) + useEffect(() => () => { releasePin.current?.() }, []) const refresh = (): void => { setAttempt(value => value + 1) } const root = state !== null && state.checkout.state === 'ready' ? state.checkout.checkout.root : null // A chosen base branch that is gone or shares no history with HEAD offers the automatic fork point instead. @@ -749,6 +796,54 @@ export function ChangesTab({ markViewed(data, file, viewedStateOf(marks, file.path, file.digest) !== 'viewed') } + // What the section handlers and comment actions read when they run; replaced on every render. + const live = useRef({ data, expandedByDefault, expand, markViewed, fileActions, publishReview, report, t }) + live.current = { data, expandedByDefault, expand, markViewed, fileActions, publishReview, report, t } + const handlers = useMemo((): DiffFileHandlers => ({ + toggle: (file) => { + const byDefault = live.current.expandedByDefault(file) + setChosen(value => ({ ...value, [file.path]: !(value[file.path] ?? byDefault) })) + }, + reveal: (path) => { setRevealed(value => new Set([...value, path])) }, + expand: (file, index) => { + const { data: owner, expand: open } = live.current + /* v8 ignore next -- sections render only while changes are loaded */ + if (owner !== null) open(owner, file, index) + }, + view: (file, viewed) => { + const { data: owner, markViewed: mark } = live.current + /* v8 ignore next -- sections render only while changes are loaded */ + if (owner !== null) mark(owner, file, viewed) + }, + actions: (file) => { + const { data: owner, fileActions: list } = live.current + /* v8 ignore next -- sections render only while changes are loaded */ + return owner === null ? [] : list(owner, file) + }, + section: (path, element) => { + if (element === null) sections.current.delete(path) + else sections.current.set(path, element) + }, + }), []) + const reviewStarted = reviewState.comments.length > 0 + const publishingBlocked = reviewState.uncertain !== null + // Comments on a commit range or on local changes go to the input only; a whole change request also takes GitHub comments. + const publishable = data?.kind === 'change' && data.commits === null + const fileReview = useMemo((): ReviewActions | undefined => shownSource === null ? undefined : { + source: shownSource, + addToInput: addReviewToInput, + records: queue, openLink, reviewStarted, + onQuoted: (selection) => { if (!queue.recordAgent(selection)) live.current.report('review.saveFailed') }, + publishingBlocked, + ...publishable ? { + queue: selection => queue.add(selection), + publish: selection => selection.source.kind === 'change' + ? live.current.publishReview(selection.source.url, { kind: 'comment', headSha: selection.source.headSha, comment: selection.comment }, + url => queue.published(selection, url)) + : Promise.resolve(live.current.t('review.stale')), + } : {}, + }, [shownSource, addReviewToInput, queue, openLink, reviewStarted, publishingBlocked, publishable]) + let body: ReactNode if (data === null) { if (state === null && snapshot?.disconnected) { @@ -782,7 +877,7 @@ export function ChangesTab({ body =
{orphanRecords}
{t('lens.empty')}
} else { body = ( -
{ if (jumpPosition.current?.top !== event.currentTarget.scrollTop) jumpPosition.current = null @@ -792,40 +887,19 @@ export function ChangesTab({ return ( 0, - onQuoted: (selection) => { if (!queue.recordAgent(selection)) report('review.saveFailed') }, - publishingBlocked: reviewState.uncertain !== null, - ...data.kind === 'local' || data.commits !== null ? {} : { - queue: selection => queue.add(selection), - publish: selection => selection.source.kind === 'change' - ? publishReview(selection.source.url, { kind: 'comment', headSha: selection.source.headSha, comment: selection.comment }, - url => queue.published(selection, url)) - : Promise.resolve(t('review.stale')), - }, - }} + review={fileReview} expanded={isExpanded(file)} - onToggle={() => { setChosen(value => ({ ...value, [path]: !(value[path] ?? expandedByDefault(file)) })) }} large={drawnLines(file) > LARGE_FILE_LINES} revealed={revealed.has(path)} - onReveal={() => { setRevealed(value => new Set([...value, path])) }} text={current.texts[path]} open={current.open[path] ?? NO_GAPS} - onExpand={(index) => { expand(data, file, index) }} split={split} wrap={wrap} badge={badgeOf(file, data, t)} tooltip={pathTooltipOf(file, t)} - actions={fileActions(data, file)} viewed={viewedStateOf(marks, path, file.digest)} - onViewed={(viewed) => { markViewed(data, file, viewed) }} - sectionRef={(element) => { - if (element === null) sections.current.delete(path) - else sections.current.set(path, element) - }} + handlers={handlers} t={t} /> ) @@ -940,7 +1014,9 @@ export function ChangesTab({ {data?.kind === 'local' && data.omitted > 0 && {t('diff.omitted', { count: data.omitted })}} {reviewForm}
- {body} + + {body} + {allFiles.length > 0 && }
diff --git a/packages/experimental/client-ui-forge/src/client/DiffFile.tsx b/packages/experimental/client-ui-forge/src/client/DiffFile.tsx index ad46c66132..bd0b9ac500 100644 --- a/packages/experimental/client-ui-forge/src/client/DiffFile.tsx +++ b/packages/experimental/client-ui-forge/src/client/DiffFile.tsx @@ -1,10 +1,12 @@ /** * One file of the changes view: a sticky header with the path, badges, line * counts, and file actions, then its hunks unified or side by side with the - * unchanged lines between them collapsed into rows that expand them. + * unchanged lines between them collapsed into rows that expand them. A section + * draws its lines once it comes within reach of the scrollport and highlights + * them when the page is next idle; it redraws only when its own props change. */ -import { useCallback, useEffect, useMemo, useRef, useState, type CSSProperties, type ReactNode } from 'react' +import { memo, useCallback, useContext, useEffect, useMemo, useRef, useState, type CSSProperties, type ReactNode } from 'react' import clsx from 'clsx' import { flushSync } from 'react-dom' import { @@ -14,7 +16,7 @@ import { } from '@deepseek-ai/dsh-client-ui-primitives' import type { PropsLocale } from '@deepseek-ai/dsh-client-ui-slots' import type { ForgeDiffFile } from '../types.ts' -import { segmentsOf, splitRows, type DiffRow, type SplitCell, type SplitRow } from './diff-rows.ts' +import { drawnLines, segmentsOf, splitRows, type DiffRow, type DiffSegment, type SplitCell, type SplitRow } from './diff-rows.ts' import { codeSideAt, readCodeSelection } from './code-selection.ts' import { reviewMatches, selectReviewLines, type ReviewSelection } from './review.ts' import { ReviewThread, useFileReview } from './ReviewThread.tsx' @@ -24,6 +26,7 @@ import reviewCss from './ReviewComposer.module.css' import { FileLabel } from './FileLabel.tsx' import type { ForgePanelKey, NS } from './locales.ts' import type { ViewedState } from './viewed-files.ts' +import { SectionReachContext, ViewVisibleContext, whenIdle } from './section-reach.ts' import css from './ChangesTab.module.css' type Translate = PropsLocale['t'] @@ -39,30 +42,41 @@ export interface DiffFileAction { /** New side of a file: its lines once read, `loading` while a read runs, or undefined before any. */ export type DiffFileText = readonly string[] | 'loading' | undefined +/** + * What every file section of one changes view calls. The view passes the same + * object on every render, so a section redraws only when its own values change. + */ +export interface DiffFileHandlers { + /** Expand or collapse a file. */ + readonly toggle: (file: ForgeDiffFile) => void + /** Show a large file anyway. */ + readonly reveal: (path: string) => void + /** Expand one gap of a file. */ + readonly expand: (file: ForgeDiffFile, index: number) => void + /** Mark a file viewed, which also collapses it, or clear the mark. */ + readonly view: (file: ForgeDiffFile, viewed: boolean) => void + /** Rows of a file's menu, read whenever the menu draws. */ + readonly actions: (file: ForgeDiffFile) => readonly DiffFileAction[] + /** Receive a file's section element, which scrolling to the file targets, or null once it unmounts. */ + readonly section: (path: string, element: HTMLElement | null) => void +} + /** Props of one file section. */ export interface DiffFileProps { - /** Whether the owning tab is visible and may show temporary menus. */ - readonly visible: boolean /** Comment destinations and the source being reviewed. */ - readonly review?: ReviewActions + readonly review?: ReviewActions | undefined /** The file and its hunks. */ readonly file: ForgeDiffFile /** Whether the section shows its lines. */ readonly expanded: boolean - /** Expand or collapse the section. */ - readonly onToggle: () => void /** Whether the file shows the large-file notice instead of its lines until revealed. */ readonly large: boolean /** Whether the user revealed a large file. */ readonly revealed: boolean - /** Show a large file anyway. */ - readonly onReveal: () => void /** The file's new side, which supplies collapsed unchanged lines. */ readonly text: DiffFileText /** Indexes of the expanded gaps. */ readonly open: ReadonlySet - /** Expand one gap. */ - readonly onExpand: (index: number) => void /** Whether deletions and additions show in two columns; a file with only additions or only deletions keeps one. */ readonly split: boolean /** Whether long lines wrap. */ @@ -71,14 +85,10 @@ export interface DiffFileProps { readonly badge: string /** Hover text of the path: both paths of a renamed file, else the path. */ readonly tooltip: string - /** Rows of the file's menu. */ - readonly actions: readonly DiffFileAction[] /** Whether the reader marked this change viewed, or marked an earlier change of the file. */ readonly viewed: ViewedState - /** Mark the file viewed, which also collapses it, or clear the mark. */ - readonly onViewed: (viewed: boolean) => void - /** Receives the section element, which scrolling to the file targets. */ - readonly sectionRef: (element: HTMLElement | null) => void + /** Actions shared by every section of the view. */ + readonly handlers: DiffFileHandlers /** Forge panel copy. */ readonly t: Translate } @@ -103,26 +113,87 @@ export function Counts({ file, t }: { const SIGN: Record = { add: '+', del: '-', context: '' } -/** Highlighted tokens of one side's lines by line number, or undefined while the grammar is unavailable. */ -type SideSpans = ReadonlyMap | undefined +/** Highlighted tokens of one side's lines by line number. */ +type SideSpans = Map -/** A segment ready to draw: collapsed lines, or rows with the tokens of each side. */ -type Prepared = - | { readonly kind: 'gap'; readonly index: number; readonly count: number | null } - | { readonly kind: 'rows'; readonly rows: readonly DiffRow[]; readonly old: SideSpans; readonly new: SideSpans } +/** Highlighted tokens of a file's drawn lines on each side; a line without an entry draws as plain text. */ +interface FileSpans { + readonly old: SideSpans + readonly new: SideSpans +} -/** Highlight one side of a run of rows as one fragment, so tokens spanning lines keep their state. */ -function sideSpans(rows: readonly DiffRow[], side: 'old' | 'new', highlighter: CodeHighlighter): SideSpans { - const numbered: { readonly no: number; readonly text: string }[] = [] - for (const row of rows) { - const no = row[side] - if (no !== null) numbered.push({ no, text: row.text }) +/** Lines highlighted as one fragment; a longer run splits here, where a token spanning the split restarts. */ +const HIGHLIGHT_CHUNK_LINES = 100 + +/** Numbered lines of one side that are highlighted together. */ +interface HighlightChunk { + readonly side: 'old' | 'new' + readonly lines: readonly { readonly no: number; readonly text: string }[] +} + +/** Split each run of rows of each side into fragments of at most {@link HIGHLIGHT_CHUNK_LINES} lines. */ +function highlightChunks(segments: readonly DiffSegment[]): HighlightChunk[] { + const chunks: HighlightChunk[] = [] + for (const segment of segments) { + if (segment.kind === 'gap') continue + for (const side of ['old', 'new'] as const) { + const lines: { readonly no: number; readonly text: string }[] = [] + for (const row of segment.rows) { + const no = row[side] + if (no !== null) lines.push({ no, text: row.text }) + } + for (let at = 0; at < lines.length; at += HIGHLIGHT_CHUNK_LINES) { + chunks.push({ side, lines: lines.slice(at, at + HIGHLIGHT_CHUNK_LINES) }) + } + } } - if (numbered.length === 0) return new Map() - const highlighted = highlighter(numbered.map(line => line.text).join('\n')) - if (highlighted === undefined) return undefined + return chunks +} + +/** Highlight one fragment, so tokens spanning its lines keep their state, into `spans`. */ +function addChunkSpans(spans: FileSpans, chunk: HighlightChunk, highlighter: CodeHighlighter): void { + const highlighted = highlighter(chunk.lines.map(line => line.text).join('\n')) + /* v8 ignore next -- a grammar that reported ready stays registered, so its fragments always tokenize */ + if (highlighted === undefined) return // Shiki omits a final empty token row, so a final empty line gets no tokens. - return new Map(numbered.map((line, index) => [line.no, highlighted[index] ?? []])) + for (const [index, line] of chunk.lines.entries()) spans[chunk.side].set(line.no, highlighted[index] ?? []) +} + +/** + * Highlight a file's drawn lines in idle time, once its language's grammar is ready: each idle period highlights + * fragments while it lasts, at least one, and the lines take their tokens when the last fragment is done. Until + * then, the lines keep the tokens of the previous pass, by line number, and new lines draw as plain text. A grammar + * that loads for another language starts no pass. + */ +function useFileSpans(path: string, segments: readonly DiffSegment[]): FileSpans | undefined { + const highlighter = useCodeHighlighter(languageForPath(path)) + // The highlighter changes whenever any grammar loads; the empty fragment tells whether this language's is ready. + const ready = highlighter('') !== undefined + const current = useRef(highlighter) + current.current = highlighter + const [spans, setSpans] = useState(undefined) + useEffect(() => { + if (!ready) return + const chunks = highlightChunks(segments) + const pass: FileSpans = { old: new Map(), new: new Map() } + let next = 0 + const step = (deadline: IdleDeadline | undefined): void => { + for (;;) { + const chunk = chunks[next] + if (chunk === undefined) { + setSpans(pass) + return + } + addChunkSpans(pass, chunk, current.current) + next++ + if ((deadline?.timeRemaining() ?? 0) <= 0) break + } + cancel = whenIdle(step) + } + let cancel = whenIdle(step) + return () => { cancel() } + }, [segments, ready]) + return spans } function Code({ text, spans, old, next }: { @@ -143,16 +214,20 @@ function pairKind(pair: SplitRow): 'context' | 'change' { return pair.left !== null && pair.left.kind === 'context' ? 'context' : 'change' } +/** The composer of a new comment, shown with its menus only while the changes tab is visible. */ +function VisibleComposer(props: Omit[0], 'visible'>): ReactNode { + return +} + /** The hunks of one file with their collapsed and expanded unchanged lines. */ -function Lines({ file, text, open, onExpand, split, wrap, review, visible, comments, t }: Pick & { readonly comments: FileReviewState }): ReactNode { +function Lines({ file, text, open, handlers, split, wrap, review, comments, t }: Pick & { readonly comments: FileReviewState }): ReactNode { const lines = typeof text === 'object' ? text : undefined const columns = split && file.additions > 0 && file.deletions > 0 const singleSide = file.status === 'added' ? 'R' : file.status === 'deleted' ? 'L' : null let maximum = Math.max(1, lines?.length ?? 0, (lines?.length ?? 0) + file.deletions - file.additions) for (const hunk of file.hunks) maximum = Math.max(maximum, hunk.oldStart + hunk.oldLines - 1, hunk.newStart + hunk.newLines - 1) const style: CSSProperties & { '--forge-number-width': string } = { '--forge-number-width': `calc(${Math.max(2, String(maximum).length)}ch + 12px)` } - const highlighter = useCodeHighlighter(languageForPath(file.path)) const [selection, setSelection] = useState(null) const root = useRef(null) const origin = useRef<{ side: 'L' | 'R'; line: number } | null>(null) @@ -210,9 +285,8 @@ function Lines({ file, text, open, onExpand, split, wrap, review, visible, comme window.removeEventListener('pointercancel', finish) } }, [selecting, nativeSelection]) - const prepared = useMemo(() => segmentsOf(file, lines, open).map((segment): Prepared => segment.kind === 'gap' - ? segment - : { ...segment, old: sideSpans(segment.rows, 'old', highlighter), new: sideSpans(segment.rows, 'new', highlighter) }), [file, lines, open, highlighter]) + const segments = useMemo(() => segmentsOf(file, lines, open), [file, lines, open]) + const spans = useFileSpans(file.path, segments) // Returns whether the range was accepted; a range across hunks or outside them keeps the previous selection. const choose = (side: 'L' | 'R', line: number, extend: boolean): boolean => { if (review === undefined || editing) return false @@ -286,15 +360,15 @@ function Lines({ file, text, open, onExpand, split, wrap, review, visible, comme )} } - const cell = (value: SplitCell | null, spans: SideSpans, side: 'L' | 'R'): ReactNode => value === null + const cell = (value: SplitCell | null, side: 'L' | 'R'): ReactNode => value === null ? : - {number(side, value.no, true)} + {number(side, value.no, true)} const rows: ReactNode[] = [] let inserted = false - const editor = review === undefined || selection === null || !editing ? null : { const native = window.getSelection() if (root.current?.contains(native?.anchorNode ?? null) === true) native?.removeAllRanges() @@ -315,10 +389,10 @@ function Lines({ file, text, open, onExpand, split, wrap, review, visible, comme } const endsAt = (old: number | null, next: number | null): boolean => selection !== null && (selection.comment.side === 'L' ? old : next) === selection.comment.endLine - for (const [at, segment] of prepared.entries()) { + for (const [at, segment] of segments.entries()) { if (segment.kind === 'gap') { rows.push() @@ -326,7 +400,7 @@ function Lines({ file, text, open, onExpand, split, wrap, review, visible, comme for (const [index, pair] of splitRows(segment.rows).entries()) { rows.push(
{ extendAdd(event.buttons, pair.left?.no ?? null, pair.right?.no ?? null) }}> - {cell(pair.left, segment.old, 'L')}{cell(pair.right, segment.new, 'R')} + {cell(pair.left, 'L')}{cell(pair.right, 'R')}
) if (!inserted && endsAt(pair.left?.no ?? null, pair.right?.no ?? null)) { rows.push(editor) @@ -341,7 +415,7 @@ function Lines({ file, text, open, onExpand, split, wrap, review, visible, comme rows.push(
{ extendAdd(event.buttons, row.old, row.new) }}> {singleSide !== 'R' && number('L', row.old, row.kind === 'del' && !labeled)}{singleSide !== 'L' && number('R', row.new, row.kind !== 'del' && !labeled)}{SIGN[row.kind]} - +
) if (!inserted && endsAt(row.old, row.new)) { rows.push(editor) @@ -374,8 +448,13 @@ function Lines({ file, text, open, onExpand, split, wrap, review, visible, comme }}>{rows} } +/** Whether a file's body draws its hunks line by line rather than a note or a collapsed strip. */ +function drawsLines(file: ForgeDiffFile, large: boolean, revealed: boolean): boolean { + return !file.binary && !file.oversized && file.hunks.length > 0 && (!large || revealed) +} + function Body(props: DiffFileProps): ReactNode { - const { file, large, revealed, onReveal, review, t } = props + const { file, large, revealed, handlers, review, t } = props const comments = useFileReview(review, file.path) const records = review === undefined ? null : comments.threads.map(thread => ) @@ -385,7 +464,7 @@ function Body(props: DiffFileProps): ReactNode { if (large && !revealed) { // A large file starts as one collapsed strip, drawn and opened like the collapsed unchanged lines. return ( - <>{records} @@ -394,12 +473,20 @@ function Body(props: DiffFileProps): ReactNode { return } -/** File actions live only while their owning tab is visible. */ -function FileMenu({ name, actions, t }: { +/** Props of a file's menu. */ +interface FileMenuProps { readonly name: string - readonly actions: readonly DiffFileAction[] + readonly file: ForgeDiffFile + readonly handlers: DiffFileHandlers readonly t: Translate -}): ReactNode { +} + +/** File actions live only while their owning tab is visible, so hiding the tab closes the menu. */ +function FileMenu(props: FileMenuProps): ReactNode { + return useContext(ViewVisibleContext) ? : null +} + +function OpenableFileMenu({ name, file, handlers, t }: FileMenuProps): ReactNode { const [menuOpen, setMenuOpen] = useState(false) return ( )} > - {actions.map(action => ( + {handlers.actions(file).map(action => ( { @@ -438,13 +525,28 @@ function FileMenu({ name, actions, t }: { } /** - * Render one file of a diff. - * @param props - the file, its section and gap state, layout choices, menu actions, and copy. + * Render one file of a diff. Until the section first comes within reach of the scrollport, an expanded file that + * draws lines shows a box of about their height instead. + * @param props - the file, its section and gap state, layout choices, shared actions, and copy. * @returns the file section. */ -export function DiffFile(props: DiffFileProps): ReactNode { - const { file, expanded, onToggle, badge, tooltip, actions, viewed, onViewed, sectionRef, t } = props +function DiffFileSection(props: DiffFileProps): ReactNode { + const { file, expanded, large, revealed, badge, tooltip, viewed, handlers, t } = props const name = file.path.slice(file.path.lastIndexOf('/') + 1) + const reach = useContext(SectionReachContext) + const element = useRef(null) + const [reached, setReached] = useState(false) + const waiting = expanded && !reached && reach !== null && drawsLines(file, large, revealed) + useEffect(() => { + /* v8 ignore next -- React attaches the section before it runs effects */ + if (!waiting || element.current === null) return + return reach.observe(element.current, () => { setReached(true) }) + }, [waiting, reach]) + const sectionRef = useCallback((node: HTMLElement | null) => { + element.current = node + handlers.section(file.path, node) + }, [handlers, file.path]) + const placeholder: CSSProperties & { '--forge-rows': number } = { '--forge-rows': drawnLines(file) } return (
@@ -454,7 +556,7 @@ export function DiffFile(props: DiffFileProps): ReactNode { className={css.fileToggle} aria-expanded={expanded} aria-label={t(expanded ? 'change.collapse' : 'change.expand', { name })} - onClick={onToggle} + onClick={() => { handlers.toggle(file) }} > @@ -468,12 +570,15 @@ export function DiffFile(props: DiffFileProps): ReactNode { {!file.binary && } - {props.visible && } + - {expanded && } + {expanded && (waiting ?
: )}
) } + +/** One file of a diff, redrawn only when its props change. */ +export const DiffFile = memo(DiffFileSection) diff --git a/packages/experimental/client-ui-forge/src/client/section-reach.ts b/packages/experimental/client-ui-forge/src/client/section-reach.ts new file mode 100644 index 0000000000..d926bd1ee2 --- /dev/null +++ b/packages/experimental/client-ui-forge/src/client/section-reach.ts @@ -0,0 +1,86 @@ +/** + * Reach of the changes view's file sections: a section's lines mount once the + * section first comes within a margin of the files scrollport, and stay + * mounted, so long diffs draw only what the reader approaches. + */ + +import { createContext } from 'react' + +/** Distance beyond the scrollport's edges within which a section counts as reached. */ +const REACH_MARGIN = '1200px 0px' + +const noop = (): void => {} + +/** + * Watches the file sections of one changes view against its scrollport. Without + * `IntersectionObserver` every section is reached at once. + */ +export class SectionReach { + private observer: IntersectionObserver | undefined + private readonly waiting = new Map void>() + + /** + * Measure sections against `root`, or stop measuring when it unmounts. Sections that are still waiting move to the new root. + * @param root - the files scrollport, or null. + */ + attach(root: HTMLElement | null): void { + this.observer?.disconnect() + this.observer = undefined + if (root === null || typeof IntersectionObserver === 'undefined') return + const observer = new IntersectionObserver((entries) => { + for (const entry of entries) { + if (!entry.isIntersecting) continue + const reached = this.waiting.get(entry.target) + /* v8 ignore next -- the observer reports only sections still waiting, which it stops watching on release */ + if (reached === undefined) continue + this.waiting.delete(entry.target) + observer.unobserve(entry.target) + reached() + } + }, { root, rootMargin: REACH_MARGIN }) + this.observer = observer + for (const element of this.waiting.keys()) observer.observe(element) + } + + /** + * Call `reached` once when `element` comes within reach. + * @param element - a file section inside the scrollport. + * @param reached - runs once, from the observer's callback or at once without `IntersectionObserver`. + * @returns the disposer that stops watching before the section is reached. + */ + observe(element: Element, reached: () => void): () => void { + if (typeof IntersectionObserver === 'undefined') { + reached() + return noop + } + this.waiting.set(element, reached) + this.observer?.observe(element) + return () => { + this.waiting.delete(element) + this.observer?.unobserve(element) + } + } +} + +/** The reach of the enclosing changes view; null outside one, where every section counts as reached. */ +export const SectionReachContext = createContext(null) + +/** Whether the enclosing changes tab is visible and may show temporary menus; true outside one. */ +export const ViewVisibleContext = createContext(true) + +/** Longest wait, in milliseconds, before deferred highlighting runs although the page stays busy. */ +const IDLE_TIMEOUT_MS = 500 + +/** + * Run `task` when the page is next idle, or after other queued work where the browser lacks `requestIdleCallback`. + * @param task - work that may wait for a frame to finish; it receives the idle period, or undefined without one. + * @returns the canceller, which has no effect after `task` ran. + */ +export function whenIdle(task: (deadline: IdleDeadline | undefined) => void): () => void { + if (typeof requestIdleCallback === 'function') { + const id = requestIdleCallback(task, { timeout: IDLE_TIMEOUT_MS }) + return () => { cancelIdleCallback(id) } + } + const id = setTimeout(() => { task(undefined) }, 0) + return () => { clearTimeout(id) } +} diff --git a/packages/experimental/client-ui-forge/src/index.ts b/packages/experimental/client-ui-forge/src/index.ts index 708eb757f4..b6dc034eac 100644 --- a/packages/experimental/client-ui-forge/src/index.ts +++ b/packages/experimental/client-ui-forge/src/index.ts @@ -418,9 +418,7 @@ export class ForgePanelController extends TypertRemoteService { const version = JSON.stringify([headSha, baseBranch]) let read = this.patches.get(key, version) if (read === undefined) { - const patch = await this.home.forge.diff(ref, headSha, signal) - if (typeof patch !== 'string') throw readError(patch) - read = { patch, attributes: await this.attributesAt(ref, headSha, signal) } + read = await this.patchAt(ref, headSha, async operation => await this.home.forge.diff(ref, headSha, operation), signal) this.patches.set(key, version, read) } const files = this.layers.classify(parseDiff(read.patch), read.attributes) @@ -683,13 +681,44 @@ export class ForgePanelController extends TypertRemoteService { if (start === 'not-linear') throw new RemoteError('forgePanel/commits-not-linear', 'The chosen commits are not one run of first parents', {}) const parent = start.parent if (parent === null) throw new RemoteError('forgePanel/diff-failed', 'The first chosen commit has no parent to compare with', {}) - const patch = await this.home.forge.compare(ref, parent, commits.to, signal) - if (typeof patch !== 'string') throw readError(patch) - const files = this.layers.classify(parseDiff(patch), await this.attributesAt(ref, commits.to, signal)) + const { patch, attributes } = await this.patchAt( + ref, commits.to, async operation => await this.home.forge.compare(ref, parent, commits.to, operation), signal, + ) + const files = this.layers.classify(parseDiff(patch), attributes) const fileUrls = Object.fromEntries(files.map(file => [file.path, this.home.forge.fileUrl(ref, commits.to, file.path)])) return { headSha: commits.to, files, localAhead: null, fileUrls } } + /** + * Read a patch that ends at `commit` together with the root `.gitattributes` at that commit. The first failure + * cancels the other read and waits for it to end before it passes on. + * @param ref - change request whose repository holds the commit. + * @param commit - commit the patch ends at. + * @param readPatch - reads the patch under the given cancellation. + * @param signal - request cancellation. + * @returns the patch and the attributes text, empty when the commit has none. + * @throws the Remote error for a refused patch or attributes read, or `signal.reason` when it aborts. + */ + private async patchAt( + ref: ChangeRequestRef, commit: string, readPatch: (signal: AbortSignal) => Promise, signal: AbortSignal, + ): Promise<{ readonly patch: string; readonly attributes: string }> { + const sibling = new AbortController() + const operation = AbortSignal.any([signal, sibling.signal]) + const patch = readPatch(operation).then((result) => { + if (typeof result !== 'string') throw readError(result) + return result + }) + const attributes = this.attributesAt(ref, commit, operation) + try { + const [text, rules] = await Promise.all([patch, attributes]) + return { patch: text, attributes: rules } + } catch (error) { + sibling.abort() + await Promise.allSettled([patch, attributes]) + throw error + } + } + /** The root `.gitattributes` of a change request's repository at a commit, empty when the commit has none. */ private async attributesAt(ref: ChangeRequestRef, commit: string, signal: AbortSignal): Promise { const attributes = await this.home.forge.fileText(ref, commit, ATTRIBUTES_PATH, signal) diff --git a/packages/experimental/client-ui-forge/src/panel.ts b/packages/experimental/client-ui-forge/src/panel.ts index 0c21c934c4..ead8fecc97 100644 --- a/packages/experimental/client-ui-forge/src/panel.ts +++ b/packages/experimental/client-ui-forge/src/panel.ts @@ -1,5 +1,6 @@ /** Live panel state of one followed Session, shared by all of its followers. */ +import { isDeepStrictEqual } from 'node:util' import type { Context } from '@deepseek-ai/cordis' import type { ChangeRequestSnapshot, ForgeReadiness, ForgeRepository } from '@deepseek-ai/dsh-experimental-forge' import type { SessionChangeRequest } from '@deepseek-ai/dsh-experimental-forge-session' @@ -230,6 +231,8 @@ export class SessionPanel { } private publish(next: ForgePanelState): void { + // A reinspection after a tool result usually finds the same checkout; equal states reach no follower. + if (isDeepStrictEqual(next, this.current)) return this.current = next for (const notify of [...this.followers]) notify() } diff --git a/packages/experimental/client-ui-forge/tests/changes-tab.client.spec.tsx b/packages/experimental/client-ui-forge/tests/changes-tab.client.spec.tsx index a4b277e34c..f1d3e62ef1 100644 --- a/packages/experimental/client-ui-forge/tests/changes-tab.client.spec.tsx +++ b/packages/experimental/client-ui-forge/tests/changes-tab.client.spec.tsx @@ -1,7 +1,7 @@ // @vitest-environment jsdom /** The changes tab: reading each source and scope, the toolbar, file sections, unchanged lines, notices, and failures. */ import { useSyncExternalStore } from 'react' -import { act, cleanup, fireEvent, render, within } from '@testing-library/react' +import { act, cleanup, fireEvent, render, waitFor, within } from '@testing-library/react' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { PaneId, SidebarRightTabActions, SidebarRightTabInfo, TabId } from '@deepseek-ai/dsh-client-ui-sidebar-right/client' import { createSnapshotStore } from '@deepseek-ai/dsh-client-store' @@ -713,9 +713,10 @@ describe('ChangesTab base branch', () => { }) it('records review comments under the compared ref and names the base the Host reports', async () => { - const renderDiff = DiffFileModule.DiffFile + // The memoized section draws through its inner component, which the spy observes. + const renderDiff = DiffFileModule.DiffFile.type let source: ReviewActions['source'] | undefined - vi.spyOn(DiffFileModule, 'DiffFile').mockImplementation((props) => { + vi.spyOn(DiffFileModule.DiffFile, 'type').mockImplementation((props) => { source = props.review?.source return renderDiff(props) }) @@ -789,6 +790,72 @@ describe('ChangesTab base branch', () => { }) }) +describe('ChangesTab drawing work', () => { + it('draws the lines of an expanded file once its section comes within reach of the scrollport', async () => { + const watched = new Map() + let root: Element | Document | null | undefined + vi.stubGlobal('IntersectionObserver', class { + constructor(readonly report: IntersectionObserverCallback, options?: IntersectionObserverInit) { root = options?.root } + observe(element: Element): void { watched.set(element, this.report) } + unobserve(element: Element): void { watched.delete(element) } + disconnect(): void { watched.clear() } + }) + try { + const m = mount() + await settle() + const app = m.section('src/app.ts') + expect(root).toBe(m.view.container.querySelector('[data-forge-changes-scroll]')) + expect(app.querySelector('[data-forge-diff-unreached]')).not.toBeNull() + expect(rowTexts(app)).toEqual([]) + + act(() => { + // The double reports only the entries, which is all the view reads. + Reflect.apply(watched.get(app)!, undefined, [[{ target: app, isIntersecting: true }], undefined]) + }) + expect(app.querySelector('[data-forge-diff-unreached]')).toBeNull() + expect(rowTexts(app).length).toBeGreaterThan(0) + expect(watched.has(app)).toBe(false) + } finally { + vi.unstubAllGlobals() + } + }) + + it('highlights a file in idle periods, as many fragments as each period allows, then draws the tokens at once', async () => { + // Idle periods with this much time left, in turn; afterwards each period has none. + const remaining = [5] + const periods: number[] = [] + vi.stubGlobal('requestIdleCallback', (task: IdleRequestCallback) => setTimeout(() => { + const left = remaining.shift() ?? 0 + periods.push(left) + task({ didTimeout: false, timeRemaining: () => left }) + }, 0)) + vi.stubGlobal('cancelIdleCallback', (id: number) => { clearTimeout(id) }) + try { + const m = mount() + await settle() + const app = m.section('src/app.ts') + await waitFor(() => { expect(app.querySelectorAll('[data-diff-code]').length).toBeGreaterThan(0) }) + // The first period highlights both sides of the file's one run and still has time; the second finds nothing left. + expect(periods.slice(0, 2)).toEqual([5, 0]) + } finally { + // Unmounting cancels pending passes through the stubbed canceller. + cleanup() + vi.unstubAllGlobals() + } + }) + + it('leaves file sections undrawn while the reader filters the file list', async () => { + // The memoized section draws through its inner component, which the spy counts. + const draw = vi.spyOn(DiffFileModule.DiffFile, 'type') + const m = mount() + await settle() + const drawn = draw.mock.calls.length + fireEvent.change(m.view.getAllByRole('textbox', { name: 'Filter files…' })[0]!, { target: { value: 'app' } }) + await settle() + expect(draw.mock.calls.length).toBe(drawn) + }) +}) + describe('ChangesTab layout', () => { it('draws lines side by side, then in one column, and wraps them by choice', async () => { const m = mount() @@ -800,7 +867,8 @@ describe('ChangesTab layout', () => { expect([...app.querySelectorAll('[data-diff-pair]')].map(pair => pair.getAttribute('data-diff-pair'))).toEqual([ 'context', 'change', 'change', 'context', 'context', 'change', ]) - expect(app.querySelectorAll('[data-diff-code]').length).toBeGreaterThan(0) + // Highlighting waits for the page to be idle. + await waitFor(() => { expect(app.querySelectorAll('[data-diff-code]').length).toBeGreaterThan(0) }) expect(m.section('docs/gone.txt').querySelector('[data-diff-code]')).toBeNull() expect(rowTexts(m.section('docs/gone.txt'))).toEqual(['bye']) expect(rowTexts(m.section('src/new.ts'))).toEqual(['export const added = 1', '']) @@ -887,6 +955,53 @@ describe('ChangesTab layout', () => { expect(scrolled).toHaveBeenCalledWith({ block: 'start' }) }) + it('keeps a jumped-to file at the top while sections around it draw, until the reader scrolls or a second passes', async () => { + const m = mount() + await settle() + const { viewport } = fileViewport(m) + Object.defineProperty(viewport, 'scrollHeight', { configurable: true, value: 1200 }) + Object.defineProperty(viewport, 'clientHeight', { configurable: true, value: 300 }) + const target = m.section('src/new.ts') + // Sections above the target draw after the jump and push it down by this much until it scrolls again. + let shift = 0 + vi.spyOn(target, 'getBoundingClientRect').mockImplementation(() => new DOMRect(0, shift, 600, 200)) + scrolled.mockImplementation(() => { shift = 0 }) + vi.useFakeTimers({ toFake: ['requestAnimationFrame', 'cancelAnimationFrame', 'performance'] }) + try { + const jump = (): void => { + fireEvent.click(m.view.getByRole('button', { name: 'All 6 files' })) + fireEvent.click(m.view.getByRole('menuitem', { name: 'src/new.ts' })) + } + jump() + expect(scrolled).toHaveBeenCalledOnce() + shift = 80 + act(() => { vi.advanceTimersToNextFrame() }) + expect(scrolled).toHaveBeenCalledTimes(2) + act(() => { vi.advanceTimersToNextFrame() }) + expect(scrolled).toHaveBeenCalledTimes(2) + + // At the bottom of the list nothing can move the file higher. + viewport.scrollTop = 900 + shift = 40 + act(() => { vi.advanceTimersToNextFrame() }) + expect(scrolled).toHaveBeenCalledTimes(2) + viewport.scrollTop = 0 + + fireEvent.wheel(viewport) + act(() => { vi.advanceTimersToNextFrame() }) + expect(scrolled).toHaveBeenCalledTimes(2) + + jump() + expect(scrolled).toHaveBeenCalledTimes(3) + act(() => { vi.advanceTimersByTime(1100) }) + shift = 80 + act(() => { vi.advanceTimersToNextFrame() }) + expect(scrolled).toHaveBeenCalledTimes(3) + } finally { + vi.useRealTimers() + } + }) + it('scrolls once to the file each navigation names when its changes arrive', async () => { const m = mount({ params: { path: 'docs/gone.txt' } }) expect(scrolled).not.toHaveBeenCalled() @@ -1323,9 +1438,10 @@ describe('line review actions', () => { expect(tree.getByRole('button', { name: 'src/app.ts' })).toBeTruthy() }) it('refuses a local reference passed to the PR publication callback', async () => { - const renderDiff = DiffFileModule.DiffFile + // The memoized section draws through its inner component, which the spy observes. + const renderDiff = DiffFileModule.DiffFile.type let publish: ReviewActions['publish'] - vi.spyOn(DiffFileModule, 'DiffFile').mockImplementation((props) => { + vi.spyOn(DiffFileModule.DiffFile, 'type').mockImplementation((props) => { publish = props.review?.publish return renderDiff(props) }) diff --git a/packages/experimental/client-ui-forge/tests/controller.host.spec.ts b/packages/experimental/client-ui-forge/tests/controller.host.spec.ts index 51f6ac0d66..92b5a6bdbc 100644 --- a/packages/experimental/client-ui-forge/tests/controller.host.spec.ts +++ b/packages/experimental/client-ui-forge/tests/controller.host.spec.ts @@ -306,6 +306,8 @@ describe('forgePanel.follow', () => { expect(moved.readiness).toEqual({ state: 'unauthenticated', message: 'sign in' }) expect(opened.provider.probes).toEqual(['fake.test', 'other.test']) + // A new commit gives the next inspection a state of its own to publish. + await git(repo, 'commit', '--allow-empty', '-m', 'again') shellTurn(opened.session, 2, [{ callId: 'again', command: 'ls', output: '' }]) await follower.next(state => state.checkout.state === 'ready') expect(follower.states.at(-1)?.readiness).toEqual({ state: 'unauthenticated', message: 'sign in' }) @@ -726,6 +728,41 @@ describe('forgePanel change request diffs', () => { .rejects.toMatchObject({ code: 'forgePanel/diff-failed', message: 'down' }) }) + it('reads the patch and the .gitattributes together and cancels the other read when one fails', async () => { + const { repo } = await project() + const opened = await open(repo) + shellTurn(opened.session, 1, [{ callId: 'create', command: 'gh pr create --fill', output: `${pullUrl(7)}\n` }]) + await reader(opened).until(state => polled(state, pullUrl(7))) + const read = async (): Promise => await opened.ctx.forgePanel.changeRequestDiff(opened.session, pullUrl(7), null, signal) + const cancelled: string[] = [] + const held = (name: string) => async (operation: AbortSignal): Promise => { + await new Promise((resolve) => { operation.addEventListener('abort', resolve, { once: true }) }) + cancelled.push(name) + throw operation.reason + } + + // Failed reads are not kept, so each case reads again. + const diff = vi.spyOn(opened.provider, 'diff').mockImplementation(async (_ref, _head, operation) => await held('patch')(operation)) + const text = vi.spyOn(opened.provider, 'fileText').mockResolvedValue({ code: 'unavailable', message: 'attributes down' }) + await expect(read()).rejects.toMatchObject({ code: 'forgePanel/diff-failed', message: 'attributes down' }) + diff.mockResolvedValue({ code: 'unavailable', message: 'patch down' }) + text.mockImplementation(async (_ref, _commit, _path, operation) => await held('attributes')(operation)) + await expect(read()).rejects.toMatchObject({ code: 'forgePanel/diff-failed', message: 'patch down' }) + + // The patch read finishes only once the attributes read has started, so reading them one after the other never ends. + const attributesStarted = Promise.withResolvers() + diff.mockImplementation(async () => { + await attributesStarted.promise + return PATCH + }) + text.mockImplementation(async () => { + attributesStarted.resolve(undefined) + return 'src/** linguist-generated\n' + }) + await expect(read()).resolves.toMatchObject({ files: [{ path: 'src/a.ts', layer: 'test' }] }) + expect(cancelled).toEqual(['patch', 'attributes']) + }) + it('lists the commits of a change request and reads consecutive ones from the first one\'s parent', async () => { const { repo } = await project() const opened = await open(repo) @@ -1726,13 +1763,12 @@ describe('forgePanel actions', () => { expect(settled).toBe(false) inspection.release() await inspected - await follower.next(() => true) const probe = Promise.withResolvers() opened.provider.readinessImpl = () => probe.promise settled = false const probed = opened.ctx.forgePanel.refresh(opened.session, signal).then(() => { settled = true }) - await follower.next(() => true) + await vi.waitFor(() => { expect(opened.provider.probes).toHaveLength(3) }) await settle() expect(settled).toBe(false) probe.resolve({ state: 'ready', message: null }) @@ -1740,6 +1776,24 @@ describe('forgePanel actions', () => { expect(opened.provider.probes).toHaveLength(3) }) + it('publishes nothing when a reinspection and a sign-in probe find what the panel already shows', async () => { + const { repo } = await project() + const opened = await open(repo) + const follower = reader(opened) + await follower.until(state => ready(state) && state.readiness !== null) + await settle() + const published = follower.states.length + const inspections = opened.sandbox.calls.length + + shellTurn(opened.session, 1, [{ callId: 'list', command: 'ls', output: '' }]) + await opened.ctx.forgePanel.refresh(opened.session, signal) + await settle() + + expect(opened.sandbox.calls.length).toBeGreaterThan(inspections) + expect(opened.provider.probes).toHaveLength(2) + expect(follower.states).toHaveLength(published) + }) + it('refreshes a Session outside a checkout without polling change requests that no provider serves', async () => { const opened = await open(await plain()) shellTurn(opened.session, 1, [{ callId: 'create', command: 'gh pr create --fill', output: 'https://unserved.test/a/b/pull/3\n' }]) @@ -1749,8 +1803,10 @@ describe('forgePanel actions', () => { await opened.ctx.forgePanel.refresh(opened.session, signal) - const refreshed = await follower.next(state => state.checkout.state === 'none') - expect(refreshed.changeRequests).toEqual([{ url: 'https://unserved.test/a/b/pull/3', origin: 'created', ref: null, update: null, snapshot: null }]) + // The refresh found the same state, which the panel already published. + const refreshed = follower.states.at(-1) + expect(refreshed?.checkout.state).toBe('none') + expect(refreshed?.changeRequests).toEqual([{ url: 'https://unserved.test/a/b/pull/3', origin: 'created', ref: null, update: null, snapshot: null }]) expect(opened.sandbox.calls.length).toBeGreaterThan(inspections) expect(opened.provider.fetches).toEqual([]) expect(opened.provider.probes).toEqual([]) diff --git a/packages/experimental/client-ui-forge/tests/diff-review.client.spec.tsx b/packages/experimental/client-ui-forge/tests/diff-review.client.spec.tsx index 6abac1781d..33f82ad73c 100644 --- a/packages/experimental/client-ui-forge/tests/diff-review.client.spec.tsx +++ b/packages/experimental/client-ui-forge/tests/diff-review.client.spec.tsx @@ -20,9 +20,10 @@ function mount(overrides: Partial = {}) { const addToInput = vi.fn().mockReturnValue(true) const records = new ReviewQueue('diff-review') const review: ReviewActions = { source, addToInput, records, onQuoted: (item) => { records.recordAgent(item) } } - const props: DiffFileProps = { file, visible: true, expanded: true, onToggle: vi.fn(), large: false, revealed: false, onReveal: vi.fn(), - text: undefined, open: new Set(), onExpand: vi.fn(), split: true, wrap: true, badge: '', tooltip: 'a.txt', actions: [], - viewed: 'unviewed', onViewed: vi.fn(), sectionRef: vi.fn(), review, t, ...overrides } + const handlers = { toggle: vi.fn(), reveal: vi.fn(), expand: vi.fn(), view: vi.fn(), actions: () => [], section: vi.fn() } + const props: DiffFileProps = { file, expanded: true, large: false, revealed: false, + text: undefined, open: new Set(), split: true, wrap: true, badge: '', tooltip: 'a.txt', + viewed: 'unviewed', handlers, review, t, ...overrides } const view = render() const line = (name: string) => view.getByRole('button', { name: `Select line ${name}` }) const code = (side: 'L' | 'R', no: number): HTMLElement => { diff --git a/packages/experimental/client-ui-forge/tests/harness.ts b/packages/experimental/client-ui-forge/tests/harness.ts index 961aded76f..5eeaaa6e98 100644 --- a/packages/experimental/client-ui-forge/tests/harness.ts +++ b/packages/experimental/client-ui-forge/tests/harness.ts @@ -347,13 +347,13 @@ export class FakeProvider implements ForgeProvider { return Promise.resolve(this.actionResult) } - diff(ref: ChangeRequestRef, headSha: string): Promise { + diff(ref: ChangeRequestRef, headSha: string, _signal: AbortSignal): Promise { this.diffs.push(ref.number) this.diffHeads.push(headSha) return Promise.resolve(this.patches.get(ref.number) ?? { code: 'not-found', message: `no diff of #${ref.number}` }) } - fileText(_ref: ChangeRequestRef, commit: string, path: string): Promise { + fileText(_ref: ChangeRequestRef, commit: string, path: string, _signal: AbortSignal): Promise { return Promise.resolve(this.texts.get(`${commit}:${path}`) ?? { code: 'not-found', message: `no ${path} at ${commit}` }) } diff --git a/packages/experimental/client-ui-forge/tests/section-reach.client.spec.ts b/packages/experimental/client-ui-forge/tests/section-reach.client.spec.ts new file mode 100644 index 0000000000..3533147a0c --- /dev/null +++ b/packages/experimental/client-ui-forge/tests/section-reach.client.spec.ts @@ -0,0 +1,112 @@ +// @vitest-environment jsdom +/** File sections reach the scrollport once, and deferred highlighting waits for idle time. */ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { SectionReach, whenIdle } from '../src/client/section-reach.ts' + +/** An IntersectionObserver double that records its options and the elements it watches. */ +class FakeObserver { + static readonly made: FakeObserver[] = [] + readonly watched = new Set() + disconnected = false + constructor(readonly report: IntersectionObserverCallback, readonly options: IntersectionObserverInit | undefined) { + FakeObserver.made.push(this) + } + observe(element: Element): void { this.watched.add(element) } + unobserve(element: Element): void { this.watched.delete(element) } + disconnect(): void { + this.disconnected = true + this.watched.clear() + } + /** Report each element as intersecting or not. */ + fire(entries: readonly (readonly [Element, boolean])[]): void { + // The production callback reads only the entries, so the double passes itself as the observer. + Reflect.apply(this.report, undefined, [entries.map(([target, isIntersecting]) => ({ target, isIntersecting })), this]) + } +} + +afterEach(() => { + vi.unstubAllGlobals() + FakeObserver.made.length = 0 +}) + +describe('SectionReach', () => { + it('reaches every section at once where the browser has no IntersectionObserver', () => { + vi.stubGlobal('IntersectionObserver', undefined) + const reach = new SectionReach() + reach.attach(document.createElement('div')) + const reached = vi.fn() + const stop = reach.observe(document.createElement('section'), reached) + expect(reached).toHaveBeenCalledOnce() + stop() + }) + + it('reaches a section once it intersects the scrollport margin and moves waiting sections to a new scrollport', () => { + vi.stubGlobal('IntersectionObserver', FakeObserver) + const reach = new SectionReach() + const near = document.createElement('section') + const far = document.createElement('section') + const gone = document.createElement('section') + const reachedNear = vi.fn() + const reachedFar = vi.fn() + reach.observe(near, reachedNear) + reach.observe(far, reachedFar) + const stopGone = reach.observe(gone, vi.fn()) + expect(FakeObserver.made).toHaveLength(0) + + const root = document.createElement('div') + reach.attach(root) + const first = FakeObserver.made[0]! + expect(first.options).toEqual({ root, rootMargin: '1200px 0px' }) + expect([...first.watched]).toEqual([near, far, gone]) + stopGone() + expect([...first.watched]).toEqual([near, far]) + + first.fire([[near, true], [far, false]]) + expect(reachedNear).toHaveBeenCalledOnce() + expect(reachedFar).not.toHaveBeenCalled() + expect([...first.watched]).toEqual([far]) + + const other = document.createElement('div') + reach.attach(other) + expect(first.disconnected).toBe(true) + const second = FakeObserver.made[1]! + expect(second.options?.root).toBe(other) + expect([...second.watched]).toEqual([far]) + second.fire([[far, true]]) + expect(reachedFar).toHaveBeenCalledOnce() + + reach.attach(null) + expect(second.disconnected).toBe(true) + expect(FakeObserver.made).toHaveLength(2) + }) +}) + +describe('whenIdle', () => { + it('waits for an idle period, at most the timeout, and can be cancelled', () => { + const request = vi.fn<(task: () => void, options: { timeout: number }) => number>(() => 7) + const cancel = vi.fn() + vi.stubGlobal('requestIdleCallback', request) + vi.stubGlobal('cancelIdleCallback', cancel) + const task = vi.fn() + const stop = whenIdle(task) + expect(request).toHaveBeenCalledWith(task, { timeout: 500 }) + stop() + expect(cancel).toHaveBeenCalledWith(7) + }) + + it('runs after queued work where the browser has no requestIdleCallback', () => { + vi.useFakeTimers() + try { + vi.stubGlobal('requestIdleCallback', undefined) + const ran = vi.fn() + whenIdle(ran) + const cancelled = vi.fn() + whenIdle(cancelled)() + vi.runAllTimers() + expect(ran).toHaveBeenCalledOnce() + expect(cancelled).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) +}) From c51ecb2bff643e0c756c13c28ecdccc75e0a1c69 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 16:31:24 +0800 Subject: [PATCH 10/30] feat(cua): request system pointer access on the target mouse move --- ...cua-hover-routes-and-coordinates.i18n.yaml | 4 +- ...-10-13-cua-hover-routes-and-coordinates.md | 2 +- ...-13-cua-hover-routes-and-coordinates.zh.md | 2 +- docs/config-catalog.md | 2 + docs/config-catalog.zh.md | 2 + .../cua-hover-ax-only/guide.i18n.yaml | 8 +- .../v0.2.2-rc.1/cua-hover-ax-only/guide.md | 4 +- .../v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md | 4 +- .../experimental/cua-macos/README.i18n.yaml | 8 +- packages/experimental/cua-macos/README.md | 6 +- packages/experimental/cua-macos/README.zh.md | 6 +- .../cua-macos/native/README.i18n.yaml | 4 +- .../experimental/cua-macos/native/README.md | 2 +- .../cua-macos/native/README.zh.md | 2 +- .../experimental/cua-macos/src/guidance.ts | 2 +- packages/experimental/cua-macos/src/index.ts | 7 +- .../cua-macos/tests/guidance.spec.ts | 6 +- .../tests/protocol-generation.spec.ts | 1 - .../tests/provider-lifecycle.spec.ts | 231 ++++++++++-------- packages/experimental/cua/README.i18n.yaml | 4 +- packages/experimental/cua/README.md | 2 +- packages/experimental/cua/README.zh.md | 2 +- packages/experimental/cua/src/api.ts | 14 +- .../cua/src/application-access.ts | 13 +- packages/experimental/cua/src/guidance.ts | 3 +- packages/experimental/cua/src/index.ts | 34 ++- packages/experimental/cua/src/locales.ts | 16 +- .../experimental/cua/src/mouse-control.ts | 15 +- .../experimental/cua/src/native-schema.ts | 4 +- packages/experimental/cua/src/protocol.ts | 14 +- packages/experimental/cua/src/types.ts | 11 +- packages/experimental/cua/tests/api.spec.ts | 17 +- .../session/cua-native/replay.override.json | 4 +- 33 files changed, 263 insertions(+), 193 deletions(-) diff --git a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.i18n.yaml b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.i18n.yaml index 7681ea3630..4450e114ec 100644 --- a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.i18n.yaml @@ -9,8 +9,8 @@ en: f96bf1ebc0719fb0 zh: e9a1144f0e76572d /agent-note-background-hover-and-screenshot-coordinates/decision: - en: 93631d45c7ecfb4c - zh: 77a08aafb1115ba1 + en: 42f40e78c2fff8a8 + zh: b7c5fa5fe538ed77 /agent-note-background-hover-and-screenshot-coordinates/alternatives-considered: en: 5fc4beaf79761c50 zh: 4948333b0020be28 diff --git a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.md b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.md index 1f5c18ce74..c46dc02060 100644 --- a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.md +++ b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.md @@ -10,7 +10,7 @@ Hover-only controls can sometimes be revealed without moving the user's pointer. ## Decision -By default, native hover uses a same-window element or ancestor advertising both `AXShowAlternateUI` and `AXShowDefaultUI`. Entering its revealed descendants preserves the parent UI; leaving restores it. An unavailable route returns `HOVER_UNAVAILABLE` by default. After the user approves `cua.requestMouseControl()`, each `mouseMove` in that session uses the system pointer until DSH restarts. This lets a caller reach pointer-tracking controls even when the same target also advertises AX alternate UI. The Host passes the session decision with each move. Physical movement leaves the visibility of previously revealed controls to the application, so switching routes cannot dismiss a target before dispatch. An AX action failure propagates because received effects may be uncertain. The result records the route and whether the system pointer moved. System movement requires the selected foreground app and AppKit's mouse hit test, which excludes input-transparent overlays. No route grants [cross-call ownership](2026-10-13-cua-continuous-hover.md). +By default, native hover uses a same-window element or ancestor advertising both `AXShowAlternateUI` and `AXShowDefaultUI`. Entering its revealed descendants preserves the parent UI; leaving restores it. An unavailable route returns `HOVER_UNAVAILABLE` by default. With `requireSystemPointerApproval` enabled, the first explicit `mouseMove(target, { useSystemPointer: true })` asks the user. Approval applies to this move and later ordinary moves in the live session. Closing the session or restarting DSH clears it. Without the approval requirement only an explicit request selects real input. This lets a caller reach pointer-tracking controls even when the same target also advertises AX alternate UI. The Host passes the session decision with each move. Physical movement leaves the visibility of previously revealed controls to the application, so switching routes cannot dismiss a target before dispatch. An AX action failure propagates because received effects may be uncertain. The result records the route and whether the system pointer moved. System movement requires the selected foreground app and AppKit's mouse hit test, which excludes input-transparent overlays. No route grants [cross-call ownership](2026-10-13-cua-continuous-hover.md). Model-facing coordinate input defaults to pixels in the returned screenshot. `relative: true` accepts integer thousandths from 0 to 999 for `click`, `mouseMove`, `scroll` and `drag`; it does not apply to opaque element IDs. The Host owns the mapping to native window points. An application-wide observation refreshes its inventory; a scoped observation preserves other windows' image mappings. Window resizing invalidates the old image, while movement preserves local coordinates. Relative endpoints remain inside the window and do not depend on screenshot resolution. diff --git a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.zh.md b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.zh.md index d1a14ee9e4..f4c5c39d69 100644 --- a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.zh.md +++ b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.zh.md @@ -10,7 +10,7 @@ Status: implemented ## 决策 -默认情况下，原生悬停会寻找同一窗口内同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI` 的元素或祖先。进入其展开区域的子控件时保持父控件展开，离开后恢复。没有可用路径时默认返回 `HOVER_UNAVAILABLE`。用户同意 `cua.requestMouseControl()` 后，本会话的每次 `mouseMove` 都使用系统光标，重启 DSH 后授权失效。有些控件既支持 AX 备用界面，又跟踪真实鼠标。Host 在每次悬停请求中传入本会话的授权结果，让调用方能够触发后者。切换到系统鼠标后，由应用决定是否收起先前展开的控件，以免派发前目标消失。AX 动作失败直接返回，因为已产生的效果可能不确定。结果记录实际路径及是否移动系统鼠标。系统移动要求所选应用处于前台，并使用 AppKit 的鼠标命中查询排除不接收输入的覆盖窗口。所有路径都遵守[跨调用不分配所有权](2026-10-13-cua-continuous-hover.zh.md)的决定。 +默认情况下，原生悬停会寻找同一窗口内同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI` 的元素或祖先。进入其展开区域的子控件时保持父控件展开，离开后恢复。没有可用路径时默认返回 `HOVER_UNAVAILABLE`。启用 `requireSystemPointerApproval` 时，首次显式调用 `mouseMove(target, { useSystemPointer: true })` 会询问用户。同意后，本次移动和本会话后续普通悬停使用系统光标。关闭会话或重启 DSH 后授权失效。免询问配置下，只有显式请求才会移动系统光标。有些控件既支持 AX 备用界面，又跟踪真实鼠标。Host 在每次悬停请求中传入本会话的授权结果，让调用方能够触发后者。切换到系统鼠标后，由应用决定是否收起先前展开的控件，以免派发前目标消失。AX 动作失败直接返回，因为已产生的效果可能不确定。结果记录实际路径及是否移动系统鼠标。系统移动要求所选应用处于前台，并使用 AppKit 的鼠标命中查询排除不接收输入的覆盖窗口。所有路径都遵守[跨调用不分配所有权](2026-10-13-cua-continuous-hover.zh.md)的决定。 面向模型的坐标默认使用返回截图中的像素。`click`、`mouseMove`、`scroll` 和 `drag` 传入 `relative: true` 时接受 0～999 的千分比整数，仅用于坐标目标。Host 负责换算原生窗口 point。应用级观察刷新窗口清单，单窗口观察保留其他窗口的图片映射。窗口尺寸变化使旧图片失效，移动窗口保留局部坐标。相对坐标的端点保持在窗口内，不随截图分辨率改变。 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 07e8c52196..26a136abb0 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1163,6 +1163,8 @@ export interface Config { export interface Config extends CuaConfig { /** Interval in milliseconds between permission and position checks while the native guide is open. */ permissionPollIntervalMs: number + /** Ask before a main task may move the system pointer. When false only explicitly requested moves use it. */ + requireSystemPointerApproval: boolean /** * Source-build signing: `ad-hoc` (default) binds grants to the code hash; * an object selects an existing certificate in an explicitly configured keychain. diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index 8308df3e6f..c7dfeff61e 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -1165,6 +1165,8 @@ export interface Config { export interface Config extends CuaConfig { /** Interval in milliseconds between permission and position checks while the native guide is open. */ permissionPollIntervalMs: number + /** Ask before a main task may move the system pointer. When false only explicitly requested moves use it. */ + requireSystemPointerApproval: boolean /** * Source-build signing: `ad-hoc` (default) binds grants to the code hash; * an object selects an existing certificate in an explicitly configured keychain. diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.i18n.yaml b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.i18n.yaml index 2f886f19df..1a804c0a53 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.i18n.yaml +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.i18n.yaml @@ -9,8 +9,8 @@ en: 6833a7851486f10d zh: 22983f326fb725ce /macos-hover-keeps-the-system-pointer-in-place-by-default/change: - en: 2369a7daf049311a - zh: 2ad1bfb579be4426 + en: e6d2e56ccb50e46a + zh: a788fc19e0771a46 /macos-hover-keeps-the-system-pointer-in-place-by-default/migration: - en: 7c89c67c9acefd25 - zh: 54a0064c30af043c + en: b1070d660b90318c + zh: 2f2ad1b8897dc044 diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.md index 733a8a7061..e38e026f1c 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.md @@ -9,10 +9,10 @@ English | [中文](guide.zh.md) ## Change -On macOS, `win.mouseMove(target)` uses an enabled element or ancestor that advertises both `AXShowAlternateUI` and `AXShowDefaultUI`. Without that route, previous versions moved the visible system cursor. The default now returns `HOVER_UNAVAILABLE` before sending input. Moving the pointer requires the user’s approval for the current session. +On macOS, `win.mouseMove(target)` uses an enabled element or ancestor that advertises both `AXShowAlternateUI` and `AXShowDefaultUI`. Without that route, previous versions moved the visible system cursor. The default now returns `HOVER_UNAVAILABLE` before sending input. By default, explicitly requesting the system pointer asks the user for this session. ## Migration 1. For an available control, use its advertised action or click it directly. Use browser tools for web content that requires page hover. 2. If your profile sets `helperPath`, rebuild the custom helper to support the per-request `useSystemPointer` field and restart the profile. -3. When the task requires a physical hover, call `cua.requestMouseControl()`. After the user approves, `win.mouseMove` on an unobscured foreground target returns `hoverRoute: "system-pointer"` and `systemPointerMoved: true`. This grant expires when DSH restarts. +3. When the task requires physical movement, call `win.mouseMove(target, { useSystemPointer: true })` on an unobscured foreground target. After approval this move reports `systemPointerMoved: true`. Closing the session or restarting DSH clears approval. diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md index 84ac6cbe2b..d5111faf05 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md @@ -9,10 +9,10 @@ description: "macOS 的 mouseMove 默认要求控件支持 AX 备用界面，否 ## 变更 -在 macOS 上，`win.mouseMove(target)` 会查找已启用且同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI` 的控件或祖先。找不到时，旧版会移动系统光标，新版默认在发送输入前返回 `HOVER_UNAVAILABLE`。移动系统光标需要用户单独授权当前会话。 +在 macOS 上，`win.mouseMove(target)` 会查找已启用且同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI` 的控件或祖先。找不到时，旧版会移动系统光标，新版默认在发送输入前返回 `HOVER_UNAVAILABLE`。默认配置下，显式请求移动系统光标时会询问用户是否授权当前会话。 ## 迁移 1. 对已经可用的控件，可执行它声明的动作或直接点击。需要网页悬停时可用浏览器工具。 2. 如果 profile 设置了 `helperPath`，请重新构建该 helper，使其支持按请求读取 `useSystemPointer` 字段，然后重启 profile。 -3. 任务确实需要移动鼠标时，调用 `cua.requestMouseControl()`。用户同意后，对前台未受遮挡的目标调用 `win.mouseMove`，结果为 `hoverRoute: "system-pointer"` 和 `systemPointerMoved: true`。重启 DSH 后授权失效。 +3. 任务确实需要移动鼠标时，在前台未受遮挡的目标上调用 `win.mouseMove(target, { useSystemPointer: true })`。用户同意后，本次移动返回 `systemPointerMoved: true`。关闭会话或重启 DSH 后需要重新授权。 diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index 7cefb93644..45d7cd7ff8 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -15,11 +15,11 @@ en: 1777e7bf6785f986 zh: e3daac1074ebb74d /deepseek-ai-dsh-experimental-cua-macos/use-this-package: - en: c81e5fbf8ff7e110 - zh: f70adab1257ed26d + en: e51a6c3625873430 + zh: 0a6998897532a05f /deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment: - en: 2391a460e295cfca - zh: 1889a903efb6142d + en: b87eede533256337 + zh: 15c6b1323a60c5b6 /deepseek-ai-dsh-experimental-cua-macos/understand-the-implementation: en: 4e82b85a5aa72eee zh: 4e004b7a205d8688 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 31793059c9..dddb02c2ef 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -60,6 +60,7 @@ Source builds default to `sourceSigning: ad-hoc`, without creating certificates | `agentPreset` | `ptc` | Register tools and guidance only for Agents bound to this preset; `false` registers globally. | | `allowedApps` | `[]` | Legacy path/signature grants, imported once for removal; they do not authorize new bundle-ID identities. | | `helperPath` | package-owned executable | Absolute executable inside the signed helper `.app`; a bare binary fails with `NATIVE_APPLICATION_REQUIRED`. | +| `requireSystemPointerApproval` | `true` | Ask on the first explicit request to move the system pointer. With `false`, only explicitly requested moves use it. | | `sourceSigning` | `ad-hoc` | Source signing: `ad-hoc`, or `{ identity, keychain }` for an existing certificate and keychain path. Ignored for installed packages and `helperPath`. | | `buildTimeoutMs` | `300000` | Maximum native preparation time. | | `timeoutMs` | `30000` | Per-operation deadline; cancellation can leave partial input. | @@ -116,7 +117,7 @@ Use the `cua` API to discover and select the application (`listApps`, `getState` `getWindow` reads its initial tree silently. An owned window without an AX surface returns an empty snapshot and remains available for screenshots and coordinate input. Within one API instance, the facade keeps one current accessibility snapshot per application handle, shared by every app and window object for that handle, including repeated selections. Reading any window replaces that snapshot; element operations use the latest snapshot ID, so use IDs from the latest tree and do not assume an older ID still identifies the same control. A failed read publishes no partial snapshot; read again without repeating the preceding input. Window objects own screenshots and pointer input. Keyboard methods stay on `app` because input follows the app’s current focus. -`mouseMove(target)` defaults to AX hover. An enabled target or ancestor must advertise both `AXShowAlternateUI` and `AXShowDefaultUI`. Moving into its descendants keeps the UI open, and leaving restores it. Without an AX route the helper returns `HOVER_UNAVAILABLE` before sending input. Click a visible control or use `performAction` with an advertised action. For browser content without these AX actions, use browser tools. If these alternatives cannot complete the task, `cua.requestMouseControl()` asks for this session until DSH restarts. On approval, every subsequent hover moves the system pointer to the foreground, unobscured target and leaves it there. +Ordinary `mouseMove(target)` defaults to AX hover. Without both `AXShowAlternateUI` and `AXShowDefaultUI` on an enabled target or ancestor, it returns `HOVER_UNAVAILABLE` without prompting. Only when the task requires physical movement should the main task call `win.mouseMove(target, { useSystemPointer: true })`. On approval the Host rechecks the window and point, dispatches this move, and uses the visible pointer for later ordinary moves. Refusal sends no input and suppresses further questions. Closing the session or restarting DSH clears the decision. Explicit `useSystemPointer: false` selects AX for one call. `paste(text, { format: "text" | "html" })` temporarily publishes the requested clipboard format and invokes the application’s enabled standard Paste menu command. It keeps the temporary format until a pasteboard reader requests its data, then restores every captured original format while it still owns the clipboard, preserving a later copy. Paste operations share a cross-process lock and cancellation joins helper cleanup. A forced process exit can prevent restoration. Unsupported Paste commands fail without typing fallback; `typeText` remains separate key-event input. @@ -187,7 +188,6 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. -- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Only the main task may ask, and only when a physical hover is necessary. A false result is a refusal for this session until restart. If the user dismisses the request, continue without mouse control unless they ask to reconsider. Await the outstanding request before calling again. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -212,7 +212,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, every mouseMove requires the target app to be in front and the point unobscured, including targets that previously supported background AX hover. If mouseMove returns FOREGROUND_REQUIRED, explain the necessary foreground step before activating the app, then observe the target again. +- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user; approval moves their pointer to this target immediately and makes subsequent ordinary moves use it until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. Without an approval requirement only explicitly requested moves use the system pointer. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index bb9e052a8c..17fd51d708 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -60,6 +60,7 @@ pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis. | `agentPreset` | `ptc` | 仅为绑定该 preset 的 Agent 注册工具和指引；`false` 表示全局注册。 | | `allowedApps` | `[]` | 旧路径和签名许可，只导入 profile 存储一次供撤销，不授予新 Bundle ID 身份访问权限。 | | `helperPath` | 包内可执行文件 | 已签名 helper `.app` 内可执行文件的绝对路径；裸二进制会触发 `NATIVE_APPLICATION_REQUIRED`。 | +| `requireSystemPointerApproval` | `true` | 主任务首次显式请求移动系统指针时询问用户。设为 `false` 时仅本次显式移动免询问。 | | `sourceSigning` | `ad-hoc` | 源码签名：`ad-hoc` 临时签名，或 `{ identity, keychain }` 指定已有证书及钥匙串路径。安装版和 `helperPath` 忽略此项。 | | `buildTimeoutMs` | `300000` | 原生组件准备的最长时间。 | | `timeoutMs` | `30000` | 单次操作截止时间，取消后可能已发生部分输入。 | @@ -116,7 +117,7 @@ void await window.getScreenshot() `getWindow` 静默读取初始树。已确认归属但没有 AX 界面的窗口返回空快照，仍可截图和发送坐标输入。在单个 API 实例内，门面按应用句柄保存一份当前辅助功能快照，该句柄的所有应用对象和窗口对象共享它，包括重复选择。读取任一窗口都会替换该快照；元素操作使用最新快照 ID，因此应使用最新树中的元素 ID，不应假定旧 ID 仍指向同一控件。读取失败不会发布部分快照；可以重新读取，但不要重复此前的输入。窗口对象负责截图和指针输入。键盘方法保留在 `app` 上，因为输入跟随应用的当前焦点。 -`mouseMove(target)` 默认通过 AX 悬停。目标或其祖先必须已启用，并同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI`。进入子控件会保持展开，离开后恢复默认界面。如果没有可用的 AX 路径，helper 返回 `HOVER_UNAVAILABLE`，不发送输入。可点击已经可见的控件，或用 `performAction` 执行控件声明的动作。浏览器内容缺少这些 AX 动作时，请使用浏览器工具。只有其他操作都无法完成任务时，才调用 `cua.requestMouseControl()` 询问用户。用户同意后，本会话后续悬停会把系统光标移到前台未受遮挡的目标并停在那里，重启 DSH 后授权失效。 +普通 `mouseMove(target)` 默认通过辅助功能悬停。如果目标或其祖先没有同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI`，则返回 `HOVER_UNAVAILABLE`，不会弹窗。只有任务确实需要移动用户的鼠标时，主任务才调用 `win.mouseMove(target, { useSystemPointer: true })`。用户允许后，Host 重新检查窗口和目标位置，执行这一次移动，此后本会话的普通悬停也使用系统指针。拒绝后不移动，也不再提问。关闭会话或重启 DSH 后决定失效。传入 `useSystemPointer: false` 可在本次调用中使用辅助功能。 `paste(text, { format: "text" | "html" })` 临时提供指定格式的剪贴板内容，并调用应用已启用的标准 Paste 菜单命令。临时格式保留到剪贴板读取方请求其数据，然后只在仍持有剪贴板时恢复保存的全部原格式，保留期间产生的新复制内容。粘贴操作共用跨进程锁，取消会等待辅助进程清理。强制终止进程可能阻止恢复。应用不支持 Paste 命令时明确失败，不自动改成逐字输入；`typeText` 仍单独提供按键事件输入。 @@ -187,7 +188,6 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. -- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Only the main task may ask, and only when a physical hover is necessary. A false result is a refusal for this session until restart. If the user dismisses the request, continue without mouse control unless they ask to reconsider. Await the outstanding request before calling again. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -212,7 +212,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, every mouseMove requires the target app to be in front and the point unobscured, including targets that previously supported background AX hover. If mouseMove returns FOREGROUND_REQUIRED, explain the necessary foreground step before activating the app, then observe the target again. +- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user; approval moves their pointer to this target immediately and makes subsequent ordinary moves use it until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. Without an approval requirement only explicitly requested moves use the system pointer. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/packages/experimental/cua-macos/native/README.i18n.yaml b/packages/experimental/cua-macos/native/README.i18n.yaml index 16dc733352..cae002d8da 100644 --- a/packages/experimental/cua-macos/native/README.i18n.yaml +++ b/packages/experimental/cua-macos/native/README.i18n.yaml @@ -21,8 +21,8 @@ en: 2a69d89b7fc95998 zh: 084575bb31fb40dd /dsh-cua-native-helper/observation-and-input: - en: 88ad57b61e873f32 - zh: ff8a89697a6c0cc5 + en: 7e63469ddd8c4e72 + zh: dbc87b8d8953c7e7 /dsh-cua-native-helper/verification: en: c87945aeec587bf5 zh: e5d69e4927e2fed3 diff --git a/packages/experimental/cua-macos/native/README.md b/packages/experimental/cua-macos/native/README.md index 65ad850f18..16d1e102f0 100644 --- a/packages/experimental/cua-macos/native/README.md +++ b/packages/experimental/cua-macos/native/README.md @@ -123,7 +123,7 @@ Tree observation and action discovery treat a generic AX failure reading one ele `paste` publishes one plain-text or HTML clipboard format, dispatches the application's enabled standard Paste menu command, and waits for a clipboard data request. It then restores the preserved formats only while its ownership marker and change count still match. The Host serializes its pastes across helpers with a file lock. maxClipboardBytes bounds both input and preserved contents; unsupported menus and oversized data fail before dispatch. A later copy is preserved. Cancellation restores owned clipboard data when possible; forced termination cannot guarantee restoration, and a data request does not prove the target completed editing. After dispatch, `pasteReadTimeoutMs` bounds waiting for clipboard consumption. Expiry returns `PASTE_NOT_CONSUMED`, closes the promised data provider, and restores the original clipboard only while still owned. The failure does not prove the app will never process its queued Paste command; inspect the target before deciding whether to retry. -`mouseMove` reveals or restores alternate UI through an enabled element or ancestor advertising both `AXShowAlternateUI` and `AXShowDefaultUI`. Without an AX route it returns `HOVER_UNAVAILABLE` before sending input. After session approval, the Host sends `useSystemPointer: true` on each `mouseMove` to select the system pointer directly. +`mouseMove` reveals or restores alternate UI through an enabled element or ancestor advertising both `AXShowAlternateUI` and `AXShowDefaultUI`. Without an AX route it returns `HOVER_UNAVAILABLE` before sending input. The Host resolves the requested route against session approval or the configured exemption before sending `useSystemPointer` to the helper. The opt-in system pointer route returns `FOREGROUND_REQUIRED` if the target is obscured or belongs to a background app. The result reports the route and whether the visible cursor moved in `hoverRoute` and `systemPointerMoved`. It never presses a button or restores the cursor after a read. Later input may interrupt the hover. Clicks send their button events at the target without an off-window priming click. diff --git a/packages/experimental/cua-macos/native/README.zh.md b/packages/experimental/cua-macos/native/README.zh.md index 8dc1427c7e..7acb51fd17 100644 --- a/packages/experimental/cua-macos/native/README.zh.md +++ b/packages/experimental/cua-macos/native/README.zh.md @@ -123,7 +123,7 @@ AppKit 在主事件循环上运行。阻塞式 socket 接收与握手使用后 `paste` 提供一种纯文本或 HTML 剪贴板格式，调用应用已启用的标准 Paste 菜单命令，并等待剪贴板数据请求。随后仅在所有权标记和修改计数仍匹配时恢复原有格式。Host 通过文件锁串行化不同 helper 的粘贴。maxClipboardBytes 同时限制输入和保留内容，不支持的菜单和过大的数据在分派前失败。后续复制会被保留。取消会尽可能恢复仍持有的剪贴板数据，强制结束不保证恢复，数据请求也不证明目标已完成编辑。 分派完成后，`pasteReadTimeoutMs` 限制等待剪贴板读取的时间。到期返回 `PASTE_NOT_CONSUMED`，关闭延迟数据提供方，并仅在仍持有剪贴板时恢复原内容。该失败不能证明应用以后不会处理已排队的 Paste 命令；决定是否重试前先检查目标内容。 -默认情况下，`mouseMove` 通过同一窗口内启用的控件或祖先声明的 `AXShowAlternateUI` 和 `AXShowDefaultUI` 展开或恢复界面。缺少这条路径时返回 `HOVER_UNAVAILABLE`，不发送输入。用户授权后，Host 在本会话的每次 `mouseMove` 请求中传入 `useSystemPointer: true`，直接使用系统光标。 +默认情况下，`mouseMove` 通过同一窗口内启用的控件或祖先声明的 `AXShowAlternateUI` 和 `AXShowDefaultUI` 展开或恢复界面。缺少这条路径时返回 `HOVER_UNAVAILABLE`，不发送输入。Host 根据本会话的授权或免询问配置决定本次悬停的路径，再向 helper 传入 `useSystemPointer`。 开启后，系统指针路线会检查前台应用和坐标命中的窗口，不满足条件时返回 `FOREGROUND_REQUIRED`。返回值 `hoverRoute` 和 `systemPointerMoved` 说明走了哪条路线以及是否移动了可见光标。悬停不会按下鼠标，也不会在读取后还原光标。后续输入可能中断悬停。点击只在目标点发送按键事件，不先在窗口外点击。 diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index b1ab6ad168..63a81e0b40 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -6,7 +6,7 @@ export const PLATFORM_GUIDANCE = 'macOS: accessibility tree operations are avail const methodDetails = new Map([ ['getAccessibilityTree', 'If the tree reports maxElements or maxDepth truncation, select a narrower window or ask the user to raise the named macOS provider setting and read again. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current.'], - ['mouseMove', 'Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, every mouseMove requires the target app to be in front and the point unobscured, including targets that previously supported background AX hover. If mouseMove returns FOREGROUND_REQUIRED, explain the necessary foreground step before activating the app, then observe the target again.'], + ['mouseMove', 'Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user; approval moves their pointer to this target immediately and makes subsequent ordinary moves use it until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. Without an approval requirement only explicitly requested moves use the system pointer. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again.'], ['click', 'click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying.'], ]) diff --git a/packages/experimental/cua-macos/src/index.ts b/packages/experimental/cua-macos/src/index.ts index 757cc11050..26142f654c 100644 --- a/packages/experimental/cua-macos/src/index.ts +++ b/packages/experimental/cua-macos/src/index.ts @@ -16,7 +16,7 @@ const CAPABILITIES: CuaStartupCapabilities = { platform: 'macos', methods: [ 'activate', 'click', 'drag', 'findApps', 'getApp', 'getCapabilities', 'getState', 'invoke', - 'listActions', 'listApps', 'listWindows', 'mouseMove', 'paste', 'performAction', 'pressKey', 'requestMouseControl', 'screenshot', 'scroll', + 'listActions', 'listApps', 'listWindows', 'mouseMove', 'paste', 'performAction', 'pressKey', 'screenshot', 'scroll', 'selectText', 'sendKeyEvent', 'setValue', 'show', 'typeText', ], appLevelAccessibility: true, @@ -33,6 +33,8 @@ export const inject = ['computerUse', 'tools', 'systemPrompt', 'subprocess', 'ag export interface Config extends CuaConfig { /** Interval in milliseconds between permission and position checks while the native guide is open. */ permissionPollIntervalMs: number + /** Ask before a main task may move the system pointer. When false only explicitly requested moves use it. */ + requireSystemPointerApproval: boolean /** * Source-build signing: `ad-hoc` (default) binds grants to the code hash; * an object selects an existing certificate in an explicitly configured keychain. @@ -43,6 +45,7 @@ export interface Config extends CuaConfig { /** Validate shared operation bounds and macOS source signing before preparing the helper. */ export const Config = Schema.object({ ...CuaConfigFields, + requireSystemPointerApproval: Schema.boolean().default(true), permissionPollIntervalMs: Schema.number().step(1).min(100).max(2 ** 31 - 1).default(1000), sourceSigning: Schema.union([ Schema.const('ad-hoc'), @@ -62,7 +65,7 @@ export const Config = Schema.object({ */ export async function apply(ctx: Context, config: Config): Promise { const backend = { createChannel: createSharedNativeChannel, sendKeyEvent: installHeldKeys(ctx), - guidance: GUIDANCE, capabilities: CAPABILITIES, + guidance: GUIDANCE, capabilities: CAPABILITIES, requireSystemPointerApproval: config.requireSystemPointerApproval, permissionParameters: createPermissionParameters(config.permissionPollIntervalMs) } if (config.helperPath !== undefined) { if (!isAbsolute(config.helperPath)) throw new Error('dsh-cua helperPath must be absolute') diff --git a/packages/experimental/cua-macos/tests/guidance.spec.ts b/packages/experimental/cua-macos/tests/guidance.spec.ts index a0f6ac2339..7f0cb9ea86 100644 --- a/packages/experimental/cua-macos/tests/guidance.spec.ts +++ b/packages/experimental/cua-macos/tests/guidance.spec.ts @@ -4,7 +4,7 @@ import type { CuaApi, CuaApp, CuaSurface, CuaWindow } from '@deepseek-ai/dsh-cua import { COMMON_GUIDANCE } from '@deepseek-ai/dsh-cua/guidance' import { GUIDANCE, PLATFORM_GUIDANCE } from '../src/guidance.ts' -const apiNames: Array = ['getCapabilities', 'requestMouseControl', 'listApps', 'findApps', 'getState', 'getApp'] +const apiNames: Array = ['getCapabilities', 'listApps', 'findApps', 'getState', 'getApp'] const accessibilityNames: Array> = [ 'getAccessibilityTree', 'listActions', 'invoke', 'performAction', 'setValue', 'selectText', ] @@ -56,8 +56,8 @@ it('enumerates every available method once in its catalog entry', () => { for (const catalog of [COMMON_GUIDANCE, GUIDANCE]) { for (const signature of signatures) expect(catalog.split('\n').filter(line => line.startsWith(`- ${signature}`))).toHaveLength(1) } - expect(signatures).toHaveLength(25) - expect(new Set(signatures.map(signature => signature.slice(signature.indexOf('.') + 1, -1))).size).toBe(24) + expect(signatures).toHaveLength(24) + expect(new Set(signatures.map(signature => signature.slice(signature.indexOf('.') + 1, -1))).size).toBe(23) expect(COMMON_GUIDANCE).toContain('app.appHandle') expect(COMMON_GUIDANCE).toContain('window.windowId') }) diff --git a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts index c5ca6b90c0..36c6dd94c9 100644 --- a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts +++ b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts @@ -33,7 +33,6 @@ describe('native protocol generation', () => { { method: 'drag', params: { appHandle: 'app', windowId: 1, fromX: 1, fromY: 1, toX: 2, toY: 2, steps: 1 } }, { method: 'paste', params: { appHandle: 'app', text: 'text', maxClipboardBytes: 1024 } }, { method: 'paste', params: { appHandle: 'app', text: 'text', pasteReadTimeoutMs: 1000 } }, - { method: 'mouseMove', params: { appHandle: 'app', windowId: 1, x: 0, y: 0, useSystemPointer: true } }, { method: 'findApps', params: { query: 'Editor', maxAppMatches: 1 } }, { method: 'releaseHeldKeys', params: { runId: 'execution' } }, { method: 'launchApp', params: { appHandle: 'pending' } }, diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index 52202a0774..ea69351f0d 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -67,7 +67,7 @@ vi.mock('@deepseek-ai/dsh-cua/native-client', async importOriginal => ({ })) const config = { - permissionPollIntervalMs: 1000, + permissionPollIntervalMs: 1000, requireSystemPointerApproval: true, agentPreset: false, helperPath: resolve('fixture-helper'), sourceSigning: 'ad-hoc', buildTimeoutMs: 300_000, timeoutMs: 10_000, connectTimeoutMs: 5000, screenshotTimeoutMs: 5000, axTimeoutMs: 3000, maxRequestBytes: 1024, maxResponseBytes: 4096, maxClipboardBytes: 8192, pasteReadTimeoutMs: 1000, maxAppMatches: 20, graceMs: 10, textEventIntervalMs: 8, maxClickHoldMs: 7000, maxElements: 64, maxDepth: 8, dragDurationMs: 200, dragSteps: 10, @@ -77,6 +77,7 @@ const { helperPath: _helperPath, ...automaticConfig } = config it('defaults to ad-hoc signing and requires a keychain path for certificate signing', () => { expect(NativeProvider.Config({}).sourceSigning).toBe('ad-hoc') + expect(NativeProvider.Config({}).requireSystemPointerApproval).toBe(true) expect(NativeProvider.Config({ sourceSigning: 'ad-hoc' }).sourceSigning).toBe('ad-hoc') for (const sourceSigning of [{ identity: 'Apple Development: Example', keychain: '/keychains/custom.keychain-db' }, { identity: 'ABC123', keychain: '/keychains/custom.keychain-db' }]) { expect(NativeProvider.Config({ sourceSigning }).sourceSigning).toEqual(sourceSigning) @@ -290,7 +291,7 @@ it('merges startup capabilities into runtime native fields', async () => { expect(result.value).toEqual({ ...runtime, platform: 'macos', appLevelAccessibility: true, keyboard: ['app'], browser: false, coordinateSpace: 'screenshot-pixels', methods: [ 'activate', 'click', 'drag', 'findApps', 'getApp', 'getCapabilities', 'getState', 'invoke', - 'listActions', 'listApps', 'listWindows', 'mouseMove', 'paste', 'performAction', 'pressKey', 'requestMouseControl', 'screenshot', 'scroll', + 'listActions', 'listApps', 'listWindows', 'mouseMove', 'paste', 'performAction', 'pressKey', 'screenshot', 'scroll', 'selectText', 'sendKeyEvent', 'setValue', 'show', 'typeText', ] }) expect(native.request).toHaveBeenCalledExactlyOnceWith('getCapabilities', {}, expect.any(AbortSignal), ToolCallId('lifecycle-1')) @@ -970,111 +971,149 @@ it('routes permission actions through the macOS provider with the Client languag }, expect.any(AbortSignal)) }) -it('moves only for the consenting session, without granting application access or retaining consent after reload', async () => { +const hoverTarget = { appHandle: 'app-one', snapshotId: 'snapshot', elementId: '0' } +const physicalHover = { ...hoverTarget, useSystemPointer: true } + +async function hoverFixture(requireApproval = true) { const f = await setup() - await f.ctx.plugin(UserQuestions) - const ask = vi.fn((id: string) => ({ answers: [{ id, selected: ['Allow for this session'] }] })) - f.ctx.on('user-questions/request', async ({ questions }) => ask(questions[0]!.id)) - const target = { appHandle: 'app-one', snapshotId: 'snapshot', elementId: '0' } - const application = { appHandle: 'app-one', name: 'Fixture', running: true, - authorization: { id: 'a'.repeat(64), name: 'Fixture', persistable: false } } - native.request.mockResolvedValueOnce(application) - expect((await f.execute('getApp', { selector: 'Fixture' })).isError).toBe(false) - const move = async (useSystemPointer: boolean) => { - expect((await f.execute('mouseMove', target)).isError).toBe(false) - expect(native.request).toHaveBeenLastCalledWith('mouseMove', { ...target, useSystemPointer }, expect.any(AbortSignal), expect.any(String)) + if (!requireApproval) { + await f.provider.dispose() + await f.ctx.plugin(NativeProvider, { ...config, requireSystemPointerApproval: false }) } - await move(false) - expect((await f.execute('mouseMove', { ...target, useSystemPointer: true })).isError).toBe(true) - for (let i = 0; i < 2; i++) expect((await f.execute('requestMouseControl')).value).toBe(true) - await move(true) - const other = await f.ctx.agents.create({ sessionId: SessionId('other-mouse-session') }) - const otherCall = (method: string, params: Record) => executeToolFixture(f.ctx, { - agent: other.agent, signal: new AbortController().signal, callId: ToolCallId(method), name: 'cua', arguments: { method, params }, + const questions = await f.ctx.plugin(UserQuestions) + const questionId = 'cua-mouse-control' + const approved = { answers: [{ id: questionId, selected: ['Allow for this session'] }] } + const ask = vi.fn(async () => approved) + f.ctx.on('user-questions/request', async ({ questions }) => questions[0]!.id === questionId ? ask() + : { answers: [{ id: 'cua-app-access', selected: ['Allow for this session'] }] }) + let index = 0 + const call = (agent: typeof f.handle.agent, method: string, params: Record) => executeToolFixture(f.ctx, { + agent, signal: new AbortController().signal, callId: ToolCallId(`hover-${++index}`), name: 'cua', arguments: { method, params }, }) - expect(JSON.stringify((await otherCall('mouseMove', target)).content)).toContain('APP_ACCESS_REQUIRED') - native.request.mockResolvedValueOnce(application) - expect((await otherCall('getApp', { selector: 'Fixture' })).isError).toBe(false) - ask.mockImplementationOnce(id => ({ answers: [{ id, selected: ['Deny'] }] })) - expect((await otherCall('requestMouseControl', {})).value).toBe(false) - expect((await otherCall('mouseMove', target)).isError).toBe(false) - expect(native.request).toHaveBeenLastCalledWith('mouseMove', { ...target, useSystemPointer: false }, expect.any(AbortSignal), expect.any(String)) - await move(true) + const select = async (agent = f.handle.agent) => { + native.request.mockResolvedValueOnce({ appHandle: 'app-one', name: 'Fixture', running: true, + authorization: { id: 'a'.repeat(64), name: 'Fixture', persistable: false } }) + expect((await call(agent, 'getApp', { selector: 'Fixture' })).isError).toBe(false) + } + await select() + native.request.mockClear() + return { ...f, ask, approved, call, select, questions } +} + +it('asks only on an explicit move and dispatches that same move after approval', async () => { + const f = await hoverFixture() + native.request.mockRejectedValueOnce(new Error('HOVER_UNAVAILABLE')) + expect(JSON.stringify((await f.execute('mouseMove', hoverTarget)).content)).toContain('HOVER_UNAVAILABLE') + expect(f.ask).not.toHaveBeenCalled() + const entered: PromiseWithResolvers = Promise.withResolvers() + const answer = Promise.withResolvers() + f.ask.mockImplementationOnce(() => { entered.resolve(); return answer.promise }) + onTestFinished(() => { answer.resolve(f.approved) }) + const pending = f.execute('mouseMove', physicalHover) + await entered.promise + expect(native.request).toHaveBeenCalledTimes(1) + expect(JSON.stringify((await f.execute('mouseMove', physicalHover)).content)).toContain('MOUSE_CONTROL_PENDING') + answer.resolve(f.approved) + expect((await pending).isError).toBe(false) + for (const [params, physical] of [ + [hoverTarget, true], [{ ...hoverTarget, useSystemPointer: false }, false], [physicalHover, true], + ] as const) { + expect((await f.execute('mouseMove', params)).isError).toBe(false) + expect(native.request).toHaveBeenLastCalledWith('mouseMove', { ...hoverTarget, useSystemPointer: physical }, expect.any(AbortSignal), expect.any(String)) + } + expect(native.request.mock.calls[1]?.[1]).toEqual(physicalHover) + expect(f.ask).toHaveBeenCalledTimes(1) +}) + +it('keeps refusals and grants separate and forgets a closed session on reopen', async () => { + const f = await hoverFixture() + expect((await f.execute('mouseMove', physicalHover)).isError).toBe(false) + const { agent } = await f.ctx.agents.create({ sessionId: SessionId('other-mouse-session') }) + expect(JSON.stringify((await f.call(agent, 'mouseMove', physicalHover)).content)).toContain('APP_ACCESS_REQUIRED') + await f.select(agent) + f.ask.mockResolvedValueOnce({ answers: [{ id: 'cua-mouse-control', selected: ['Deny'] }] }) + native.request.mockClear() + for (let i = 0; i < 2; i++) expect(JSON.stringify((await f.call(agent, 'mouseMove', physicalHover)).content)).toContain('MOUSE_CONTROL_DENIED') + expect(native.request).not.toHaveBeenCalled() + expect((await f.call(agent, 'mouseMove', hoverTarget)).isError).toBe(false) + expect(native.request).toHaveBeenLastCalledWith('mouseMove', { ...hoverTarget, useSystemPointer: false }, expect.any(AbortSignal), expect.any(String)) + expect((await f.execute('mouseMove', hoverTarget)).isError).toBe(false) + expect(native.request).toHaveBeenLastCalledWith('mouseMove', physicalHover, expect.any(AbortSignal), expect.any(String)) + const sessionId = f.handle.agent.id + await f.handle.dispose() + const reopened = await f.ctx.agents.create({ sessionId }) + await f.select(reopened.agent) + expect((await f.call(reopened.agent, 'mouseMove', hoverTarget)).isError).toBe(false) + expect(native.request).toHaveBeenLastCalledWith('mouseMove', { ...hoverTarget, useSystemPointer: false }, expect.any(AbortSignal), expect.any(String)) + expect((await f.call(reopened.agent, 'mouseMove', physicalHover)).isError).toBe(false) + expect(f.ask).toHaveBeenCalledTimes(3) +}) + +it('rechecks the selection after a pending approval and does not dispatch revoked or stale input', async () => { + const f = await hoverFixture() + f.ask.mockImplementationOnce(async () => { + await f.ctx.cuaAccess.revoke('a'.repeat(64) as Parameters[0]) + return f.approved + }) + expect((await f.execute('mouseMove', physicalHover)).isError).toBe(true) + expect(native.request).not.toHaveBeenCalled() +}) + +it('revalidates window coordinates after the user answers', async () => { + const f = await hoverFixture() + native.request.mockResolvedValueOnce([{ windowId: 7, appName: 'Fixture', title: 'Window', bounds: { x: 0, y: 0, width: 100, height: 100 }, + isOnScreen: true, onCurrentSpace: true }]) + await f.execute('listWindows', { appHandle: 'app-one' }) + f.ask.mockImplementationOnce(async () => { + native.request.mockResolvedValueOnce([]) + await f.execute('listWindows', { appHandle: 'app-one' }) + return f.approved + }) + const result = await f.execute('mouseMove', { appHandle: 'app-one', windowId: 7, x: 500, y: 500, relative: true, useSystemPointer: true }) + expect(JSON.stringify(result.content)).toContain('STALE_SNAPSHOT') + expect(native.request.mock.calls.map(([name]) => name)).toEqual(['listWindows', 'listWindows']) +}) + +it('drops dismissal, aborted answers and missing human answerers without allowing input', async () => { + const f = await hoverFixture() + f.ask.mockRejectedValueOnce(new UserQuestionError('dismissed', 'ASK_CANCELLED')) + expect(JSON.stringify((await f.execute('mouseMove', physicalHover)).content)).toContain('MOUSE_APPROVAL_CANCELLED') + const controller = new AbortController() + f.ask.mockImplementationOnce(async () => { controller.abort(); return f.approved }) + await Promise.allSettled([f.execute('mouseMove', physicalHover, controller.signal)]) + expect(native.request).not.toHaveBeenCalled() + expect((await f.execute('mouseMove', physicalHover)).isError).toBe(false) + expect(f.ask).toHaveBeenCalledTimes(3) await f.provider.dispose() await f.ctx.plugin(NativeProvider, config) - expect((await f.execute('requestMouseControl')).value).toBe(true) - expect(ask.mock.calls.map(([id]) => id)).toEqual(['cua-app-access', 'cua-mouse-control', 'cua-app-access', 'cua-mouse-control', 'cua-mouse-control']) + await f.select() + await f.questions.dispose() + expect(JSON.stringify((await f.execute('mouseMove', physicalHover)).content)).toContain('MOUSE_APPROVAL_UNAVAILABLE') }) -it.each(['Deny', 'custom'] as const)('remembers a %s answer without moving the pointer or asking again', async (answer) => { - const f = await setup() - await f.ctx.plugin(UserQuestions) - const ask = vi.fn(() => { - return { answers: [{ id: 'cua-mouse-control', selected: [answer === 'custom' ? 'Allow for this session' : answer], - ...(answer === 'custom' ? { custom: 'maybe later' } : {}) }] } - }) - f.ctx.on('user-questions/request', async () => ask()) - expect((await f.execute('requestMouseControl')).value).toBe(false) - expect((await f.execute('requestMouseControl')).value).toBe(false) - expect(ask).toHaveBeenCalledTimes(1) - expect(native.request).not.toHaveBeenCalled() +it('allows only the explicitly requested physical move when approval is disabled', async () => { + const f = await hoverFixture(false) + for (const [target, expected] of [[hoverTarget, false], [physicalHover, true], [hoverTarget, false]] as const) { + expect((await f.execute('mouseMove', target)).isError).toBe(false) + expect(native.request).toHaveBeenLastCalledWith('mouseMove', { ...hoverTarget, useSystemPointer: expected }, expect.any(AbortSignal), expect.any(String)) + } + expect(f.ask).not.toHaveBeenCalled() }) -it('allows a fresh request after dismissal without authorizing the first one', async () => { - const f = await setup() - await f.ctx.plugin(UserQuestions) - const ask = vi.spyOn(f.ctx.userQuestions, 'ask') - .mockRejectedValueOnce(new UserQuestionError('User dismissed the request', 'ASK_CANCELLED')) - .mockResolvedValueOnce({ answers: [{ id: 'cua-mouse-control', selected: ['Allow for this session'] }] }) - const dismissed = await f.execute('requestMouseControl') - expect(dismissed.isError).toBe(true) - expect(JSON.stringify(dismissed.content)).toContain('MOUSE_APPROVAL_CANCELLED') - expect((await f.execute('requestMouseControl')).value).toBe(true) - expect(ask).toHaveBeenCalledTimes(2) - expect(native.request).not.toHaveBeenCalled() -}) - -it('refuses delegated mouse control before asking the user even when the parent has approval', async () => { - const f = await setup() - await f.ctx.plugin(UserQuestions) - const ask = vi.spyOn(f.ctx.userQuestions, 'ask').mockResolvedValue({ - answers: [{ id: 'cua-mouse-control', selected: ['Allow for this session'] }], - }) - expect((await f.execute('requestMouseControl')).value).toBe(true) - ask.mockClear() +it.each([true, false])('rejects a delegated pointer move before asking or dispatching with approval=%s', async (approval) => { + const f = await hoverFixture(approval) const { agent } = await f.ctx.agents.create({ sessionId: SessionId('delegated-mouse-session'), meta: { origin: 'subagent', parentSession: f.handle.agent.id } }) - const refused = await executeToolFixture(f.ctx, { agent, signal: new AbortController().signal, - callId: ToolCallId('delegated-mouse-control'), name: 'cua', arguments: { method: 'requestMouseControl', params: {} } }) - expect(refused.isError).toBe(true) + // This case starts with application access already granted to the child. + const selectAnswer = vi.spyOn(f.ctx.userQuestions, 'ask') + .mockResolvedValueOnce({ answers: [{ id: 'cua-app-access', selected: ['Allow for this session'] }] }) + await f.select(agent) + selectAnswer.mockClear() + native.request.mockClear() + const refused = await f.call(agent, 'mouseMove', physicalHover) expect(JSON.stringify(refused.content)).toContain('MOUSE_CONTROL_UNAVAILABLE') - expect(JSON.stringify(refused.content)).toContain('report that step in your final result for the main task') - expect(ask).not.toHaveBeenCalled() - expect(native.request).not.toHaveBeenCalled() -}) - -it('drops late approval after cancellation and admits only one pending question', async () => { - const f = await setup() - expect((await f.execute('requestMouseControl')).isError).toBe(true) - await f.ctx.plugin(UserQuestions) - const entered = Promise.withResolvers() - let count = 0 - const release = Promise.withResolvers() - f.ctx.on('user-questions/request', async () => { - count++ - entered.resolve(undefined) - await release.promise - return { answers: [{ id: 'cua-mouse-control', selected: ['Allow for this session'] }] } - }) - const controller = new AbortController() - const first = f.execute('requestMouseControl', {}, controller.signal) - await entered.promise - expect(JSON.stringify((await f.execute('requestMouseControl')).content)).toContain('MOUSE_CONTROL_PENDING') - controller.abort() - release.resolve(undefined) - await Promise.allSettled([first]) - expect((await f.execute('requestMouseControl')).value).toBe(true) - expect(count).toBe(2) + expect(selectAnswer).not.toHaveBeenCalled() + expect(f.ask).not.toHaveBeenCalled() expect(native.request).not.toHaveBeenCalled() }) @@ -1089,8 +1128,8 @@ it('keeps provider-specific pointer flags and questions off providers without mo keyboard: ['app'], coordinateSpace: 'screenshot-pixels', browser: false }, }) } }) - expect(JSON.stringify((await f.execute('requestMouseControl')).content)).toContain('UNSUPPORTED_OPERATION') await selectApplication(f) + expect(JSON.stringify((await f.execute('mouseMove', physicalHover)).content)).toContain('UNSUPPORTED_OPERATION') const target = { appHandle: 'app-one', snapshotId: 'snapshot', elementId: '0' } expect((await f.execute('mouseMove', target)).isError).toBe(false) expect(native.request).toHaveBeenLastCalledWith('mouseMove', target, expect.any(AbortSignal), expect.any(String)) diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index b72fd91584..62318feee3 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -15,8 +15,8 @@ en: 5db4d4fea25493b9 zh: 65ab64fc24eb0d09 /deepseek-ai-dsh-cua/use-this-package: - en: a55ba012aedca751 - zh: fcddea780c0ec58e + en: 5e95c138cb107385 + zh: b126c56fb7d35538 /deepseek-ai-dsh-cua/use-this-package/permission-controls: en: d92d5948a7941e8d zh: d43491ae6521b87e diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index 39a3e522d8..c3d2ce62c8 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -41,7 +41,7 @@ Observation methods emit and return their result. Use `void await app.getAccessi Application consent precedes launch, observation, capture, and input. Session access is stored by conversation and application identity and survives DSH restart and Computer Use disablement. Other conversations require their own consent. Saved grants require a platform-provided reusable application identity. Refusal and revocation close affected Agent connections, so their old objects require fresh selection. No input is replayed after cancellation, timeout, or reconnect. -Use `cua.requestMouseControl()` only when the task requires moving the user’s pointer. Approval applies to `mouseMove` in this session until DSH restarts. Actions return a `dispatched` boolean; `mouseMove` also reports `hoverRoute` and `systemPointerMoved`, and `show()` adds `windowId` and `foregroundChanged`. Read fresh state to verify the result. Screenshots and pointer actions are available only on explicit window objects. Coordinate actions default to pixels in the latest returned screenshot; `relative: true` accepts integers from 0 to 999 after a successful window observation. Values are divided by 1000 and scaled to the window dimensions, then clamped inside the window. The Host resolves attachment normalization and the selected model route’s image projection before returning the screenshot, then converts input to the provider’s native window coordinates. Resizing invalidates the screenshot mapping; `SCREENSHOT_REQUIRED` requires a new screenshot for absolute coordinates. Resizing rejects old coordinates with `WINDOW_CHANGED`; movement alone preserves window-local coordinates. +On supported providers the main task can request a physical move with `win.mouseMove(target, { useSystemPointer: true })`. Actions return a `dispatched` boolean; `mouseMove` also reports `hoverRoute` and `systemPointerMoved`, and `show()` adds `windowId` and `foregroundChanged`. Read fresh state to verify the result. Screenshots and pointer actions are available only on explicit window objects. Coordinate actions default to pixels in the latest returned screenshot; `relative: true` accepts integers from 0 to 999 after a successful window observation. Values are divided by 1000 and scaled to the window dimensions, then clamped inside the window. The Host resolves attachment normalization and the selected model route’s image projection before returning the screenshot, then converts input to the provider’s native window coordinates. Resizing invalidates the screenshot mapping; `SCREENSHOT_REQUIRED` requires a new screenshot for absolute coordinates. Resizing rejects old coordinates with `WINDOW_CHANGED`; movement alone preserves window-local coordinates. `click(target, { holdMs: 800 })` holds the mouse button for 800 milliseconds before release. `holdMs` applies only to `click`; it accepts nonnegative integers and applies to each press of a double-click; zero releases without an additional hold. The provider config `maxClickHoldMs` limits explicit holds and defaults to 5000 ms. Explicit holds use pointer input; omission preserves ordinary click behavior. The operation deadline includes the hold, and cancellation or timeout releases the button. Received input cannot be undone; forced helper termination cannot guarantee release. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index c2933dc0d1..bde2ccd1a3 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -41,7 +41,7 @@ kind: "package-reference" 应用授权发生在启动、观察、截图和输入之前。会话许可按对话和应用身份持久化，重启 DSH 或关闭 Computer Use 后仍可复用，其他对话需要单独授权。长期许可要求平台提供可复用的应用身份。拒绝或撤销会关闭受影响的 Agent 连接，因此旧对象必须重新选择目标。取消、超时或重新连接后均不重放输入。 -只有任务确实需要移动用户的鼠标时，才调用 `cua.requestMouseControl()` 询问用户。授权仅适用于当前会话的 `mouseMove`，重启 DSH 后失效。动作返回 `dispatched` 布尔值；`mouseMove` 还会返回 `hoverRoute` 和 `systemPointerMoved`，`show()` 还会返回 `windowId` 和 `foregroundChanged`。请读取新状态以验证结果。截图和指针操作只能通过显式窗口对象使用。坐标操作默认使用最近返回截图中的像素；成功观察窗口后，传入 `relative: true` 可使用 0～999 的整数坐标。数值除以 1000 后按窗口尺寸缩放，并限制在窗口内。Host 在返回截图前解析附件归一化和当前模型的图片缩放规则，再将输入换算为提供方的原生窗口坐标。窗口尺寸变化会使截图映射失效，绝对坐标遇到 `SCREENSHOT_REQUIRED` 时需要重新截图。窗口缩放后以 `WINDOW_CHANGED` 拒绝旧坐标；单纯移动窗口不改变窗口内坐标。 +在支持系统鼠标的提供方上，主任务可通过 `win.mouseMove(target, { useSystemPointer: true })` 请求真实移动。动作返回 `dispatched` 布尔值；`mouseMove` 还会返回 `hoverRoute` 和 `systemPointerMoved`，`show()` 还会返回 `windowId` 和 `foregroundChanged`。请读取新状态以验证结果。截图和指针操作只能通过显式窗口对象使用。坐标操作默认使用最近返回截图中的像素；成功观察窗口后，传入 `relative: true` 可使用 0～999 的整数坐标。数值除以 1000 后按窗口尺寸缩放，并限制在窗口内。Host 在返回截图前解析附件归一化和当前模型的图片缩放规则，再将输入换算为提供方的原生窗口坐标。窗口尺寸变化会使截图映射失效，绝对坐标遇到 `SCREENSHOT_REQUIRED` 时需要重新截图。窗口缩放后以 `WINDOW_CHANGED` 拒绝旧坐标；单纯移动窗口不改变窗口内坐标。 `click(target, { holdMs: 800 })` 按住鼠标按钮 800 毫秒后松开。`holdMs` 仅适用于 `click`；它接受非负整数，双击时应用于每次按下；0 表示按下后立即松开，不额外等待。提供方配置 `maxClickHoldMs` 限制显式按住时长，默认 5000 毫秒。指定时长后使用鼠标事件；省略时保留普通点击行为。按住时间计入操作超时，取消或超时会松开按钮。已接收的输入无法撤销；辅助进程被强制终止时无法保证松开。 diff --git a/packages/experimental/cua/src/api.ts b/packages/experimental/cua/src/api.ts index 930db84195..a33f4ca9a7 100644 --- a/packages/experimental/cua/src/api.ts +++ b/packages/experimental/cua/src/api.ts @@ -1,6 +1,6 @@ /** Object facade over logged native operations; its caller owns evaluator persistence. */ import { Buffer } from 'node:buffer' -import { z } from 'zod' +import type { z } from 'zod' import type { JsonValue } from '@deepseek-ai/dsh-util-values' import { ActionNamesResult, ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from './protocol.ts' import type { CuaApi, CuaApp, CuaAppHandle, CuaCapabilities, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaOperation, CuaScreenshot, CuaSurface, CuaWindow } from './types.ts' @@ -51,9 +51,6 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): const objects = createCuaObjects(call, sink) const api: CuaApi = { - async requestMouseControl() { - return z.boolean().parse(await call('requestMouseControl', {})) - }, async getCapabilities(options) { const result = CapabilitiesResult.parse(await call('getCapabilities', {})) capabilities = result @@ -203,12 +200,15 @@ function createCuaObjects(call: ReturnType, sink: CuaObser return screenshot }, async mouseMove(targetPoint, options) { - const relative = options?.relative === undefined ? {} : { relative: options.relative } + const movement = { + ...options?.relative === undefined ? {} : { relative: options.relative }, + ...options?.useSystemPointer === undefined ? {} : { useSystemPointer: options.useSystemPointer }, + } if (typeof targetPoint === 'string') { - return HoverResult.parse(await base.call('mouseMove', { ...base.elementParams(), elementId: targetPoint, ...relative })) + return HoverResult.parse(await base.call('mouseMove', { ...base.elementParams(), elementId: targetPoint, ...movement })) } const [x, y] = targetPoint - return HoverResult.parse(await base.call('mouseMove', { ...coordinateParams(appHandle, windowId), x, y, ...relative })) + return HoverResult.parse(await base.call('mouseMove', { ...coordinateParams(appHandle, windowId), x, y, ...movement })) }, async click(targetPoint, options) { const mouseButton = options?.mouseButton ?? 'left' diff --git a/packages/experimental/cua/src/application-access.ts b/packages/experimental/cua/src/application-access.ts index 02066411e1..32bc31b811 100644 --- a/packages/experimental/cua/src/application-access.ts +++ b/packages/experimental/cua/src/application-access.ts @@ -231,16 +231,17 @@ export class ApplicationAccess { * @param handle Selected application. * @param signal Calling tool lifetime. * @param dispatch One native operation, with revocation cancellation. + * @param prepare Optional interactive preparation outside the application lock. Dispatch rechecks access after it settles. * @returns The native result, with no implicit retry after consent or cancellation. */ async run(client: NativeClient, handle: CuaAppHandle, signal: AbortSignal, - dispatch: (signal: AbortSignal) => Promise): Promise { + dispatch: (signal: AbortSignal) => Promise, prepare?: (signal: AbortSignal) => Promise): Promise { const state = this.clients.get(client) const selected = state?.handles.get(handle) if (!state || !selected) throw new NativeError('APP_ACCESS_REQUIRED', 'Select the application with getApp and obtain user consent before reading or controlling it') if (selected.lifetime.signal.aborted) throw new NativeError('APP_ACCESS_REVOKED', 'Application access was revoked; stop using this application') const lifetime = AbortSignal.any([signal, this.lifetime.signal, selected.lifetime.signal]) - return this.options.store.withApplication(selected.identity.id, lifetime, async () => { + const requireAccess = async () => { const permitted = await this.options.store.withState(lifetime, ({ state: permissions }) => this.permitted(selected, permissions.applications[selected.identity.id] ?? this.empty())) if (!permitted) { @@ -248,6 +249,14 @@ export class ApplicationAccess { throw new NativeError('APP_ACCESS_REVOKED', 'Application access was revoked; stop using this application') } if (client.disposed) throw new NativeError('NATIVE_CLOSED', 'Helper is closed; acquire fresh application handles') + } + if (prepare) { + await requireAccess() + await prepare(lifetime) + lifetime.throwIfAborted() + } + return this.options.store.withApplication(selected.identity.id, lifetime, async () => { + await requireAccess() return this.withOperation(state.agent, client, selected.identity, lifetime, dispatch) }) } diff --git a/packages/experimental/cua/src/guidance.ts b/packages/experimental/cua/src/guidance.ts index a729901370..d11a575e37 100644 --- a/packages/experimental/cua/src/guidance.ts +++ b/packages/experimental/cua/src/guidance.ts @@ -8,7 +8,6 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. -- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Only the main task may ask, and only when a physical hover is necessary. A false result is a refusal for this session until restart. If the user dismisses the request, continue without mouse control unless they ask to reconsider. Await the outstanding request before calling again. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -33,7 +32,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. +- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index 410225856a..24f216fb78 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -127,6 +127,8 @@ export interface CuaBackend { createChannel: NativeChannelFactory guidance: string capabilities: CuaStartupCapabilities + /** Whether explicit system-pointer moves require consent. Omit for providers without this route. */ + requireSystemPointerApproval?: boolean /** Dispatch a directional key through the provider's execution-scoped input owner, after application authorization. */ sendKeyEvent?: (client: NativeClient, params: Record, execution: Readonly) => Promise @@ -160,7 +162,8 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) permissionParameters: backend.permissionParameters, store: new CuaPermissionStore(ctx, config.allowedApps.map(grant => ({ ...grant, id: grant.id as CuaApplicationId })), config.timeoutMs), }) - const mouseControl = backend.capabilities.methods.includes('requestMouseControl') ? new MouseControl(ctx) : undefined + const mouseControl = backend.requireSystemPointerApproval === undefined + ? undefined : new MouseControl(ctx, backend.requireSystemPointerApproval) const clients = new Map() const lifetime = new AbortController() const pending = new Map, Agent>() @@ -209,7 +212,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) for (const result of [...await permissions, ...results, ...releases]) if (result.status === 'rejected') errors.push(result.reason) if (errors.length > 0) throw new AggregateError(errors, 'dsh-cua helper cleanup failed') } - const callNative = async (request: CuaOperation, execution: Readonly) => { + const callNative = async (request: CuaOperation, execution: Readonly, pointerRoute?: boolean) => { const { method, params: parsedParams } = request const params = execution.coordinates.parameters(request) const agent = execution.agent @@ -226,7 +229,10 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) ? { ...params, maxElements: config.maxElements, maxDepth: config.maxDepth } : method === 'drag' ? { ...params, durationMs: config.dragDurationMs, steps: config.dragSteps } : params - if (method === 'mouseMove' && mouseControl) nativeParams.useSystemPointer = mouseControl.allowed(agent) + if (method === 'mouseMove') { + delete nativeParams.useSystemPointer + if (mouseControl) nativeParams.useSystemPointer = pointerRoute ?? mouseControl.allowed(agent) + } const requestNative = () => client.request(method, nativeParams, signal, execution.callId) if (method === 'screenshot') { const { attachments, target } = await screenshotStore(ctx, execution) @@ -313,10 +319,6 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) const signal = AbortSignal.any([execution.signal, lifetime.signal, owner.signal]) signal.throwIfAborted() const operation = (async () => { - if (request.method === 'requestMouseControl') { - if (!mouseControl) throw new NativeError('UNSUPPORTED_OPERATION', 'This provider does not support system mouse control') - return mouseControl.request(agent, signal) - } const previous = clients.get(agent) if (previous?.disposed) { try { await previous.dispose() } catch (error) { @@ -342,14 +344,28 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) owner.coordinates = new CuaCoordinates() } const nativeClient = client + let pointerRoute: boolean | undefined + let prepare: ((signal: AbortSignal) => Promise) | undefined + if (request.method === 'mouseMove') { + if (request.params.useSystemPointer === true) { + prepare = async (signal) => { + if (!mouseControl) throw new NativeError('UNSUPPORTED_OPERATION', 'This provider does not support system mouse control') + owner.coordinates.parameters(request) + if (!await mouseControl.request(agent, signal)) { + throw new NativeError('MOUSE_CONTROL_DENIED', 'The user declined system-pointer movement for this session. No input was sent. Use pointer-free actions.') + } + pointerRoute = true + } + } else pointerRoute = request.params.useSystemPointer + } const dispatch = async (signal: AbortSignal) => { const scoped = { ...execution, agent, signal, nativeClient, coordinates: owner.coordinates } - const result = await callNative(request, scoped) + const result = await callNative(request, scoped, pointerRoute) signal.throwIfAborted() return result } return 'appHandle' in request.params - ? access.run(nativeClient, request.params.appHandle as CuaAppHandle, signal, dispatch) + ? access.run(nativeClient, request.params.appHandle as CuaAppHandle, signal, dispatch, prepare) : dispatch(signal) })() pending.set(operation, agent) diff --git a/packages/experimental/cua/src/locales.ts b/packages/experimental/cua/src/locales.ts index 7c2cba49eb..fbd9b7ac43 100644 --- a/packages/experimental/cua/src/locales.ts +++ b/packages/experimental/cua/src/locales.ts @@ -117,14 +117,14 @@ export const mouseControlDisplay = { pt: 'Permitir que o agente mova o ponteiro do mouse?', }, detail: { - en: 'Hovering will move your pointer and leave it at the target, even while you are using the mouse. This permission applies only to this session until DSH restarts. If you decline, this session will not ask again before restart.', - zh: '智能体悬停时会移动你的鼠标指针，并将它留在目标位置，即使你正在使用鼠标。授权仅限当前会话，重启 DSH 后失效。如果拒绝，重启前本会话不会再次询问。', - ja: 'ホバーすると、マウスを使用中でもポインタが移動し、移動先に残ります。許可はこのセッションに限り、DSH を再起動すると失効します。拒否すると、再起動までこのセッションでは再度確認しません。', - es: 'Al pasar sobre un elemento, el puntero se moverá y quedará allí aunque estés usando el ratón. El permiso solo se aplica a esta sesión y caduca al reiniciar DSH. Si lo deniegas, esta sesión no volverá a preguntar hasta que reinicies DSH.', - fr: 'Lors d’un survol, le pointeur se déplace et reste sur la cible, même si vous utilisez la souris. Cette autorisation vaut pour cette session et expire au redémarrage de DSH. Si vous refusez, cette session ne redemandera pas avant le redémarrage.', - de: 'Beim Überfahren eines Ziels bewegt sich der Zeiger und bleibt dort, auch wenn Sie die Maus verwenden. Die Erlaubnis gilt nur für diese Sitzung und erlischt beim Neustart von DSH. Wenn Sie ablehnen, fragt diese Sitzung bis zum Neustart nicht erneut.', - ru: 'При наведении указатель переместится и останется на цели, даже если вы пользуетесь мышью. Разрешение действует только в этом сеансе до перезапуска DSH. Если вы откажете, запрос в этом сеансе не повторится до перезапуска.', - pt: 'Ao passar sobre um elemento, o ponteiro se moverá e ficará no destino, mesmo enquanto você usa o mouse. A permissão vale apenas para esta sessão e expira ao reiniciar o DSH. Se você recusar, esta sessão não perguntará novamente até a reinicialização.', + en: 'Allowing moves your pointer to this target now. Later hovers can also move it. If you decline, this session will not ask again. The decision expires when you close the session or restart DSH.', + zh: '允许后，你的鼠标指针会立即移到目标位置，本会话后续悬停也会移动指针。如果拒绝，本会话不会再询问。关闭会话或重启 DSH 后，这个决定失效。', + ja: '許可するとポインタが今すぐ移動し、以後のホバーでも移動します。拒否すると、このセッションでは再度確認しません。セッションを閉じるか DSH を再起動すると、この選択は失効します。', + es: 'Si aceptas, el puntero se moverá ahora al destino y también podrá moverse al pasar sobre otros elementos. Si rechazas, no se volverá a preguntar. La decisión caduca al cerrar la sesión o reiniciar DSH.', + fr: 'Si vous acceptez, le pointeur se déplace immédiatement sur la cible, puis lors des survols suivants. Si vous refusez, la question ne sera plus posée. Ce choix expire à la fermeture de la session ou au redémarrage de DSH.', + de: 'Bei Zustimmung bewegt sich der Zeiger sofort zum Ziel und auch bei späteren Mausbewegungen. Bei Ablehnung wird nicht erneut gefragt. Die Entscheidung gilt, bis Sie die Sitzung schließen oder DSH neu starten.', + ru: 'При разрешении указатель сразу переместится на цель и сможет перемещаться при следующих наведениях. При отказе вопрос не повторится. Решение действует до закрытия сеанса или перезапуска DSH.', + pt: 'Se permitir, o ponteiro irá para o alvo agora e poderá se mover nas próximas ações. Se recusar, a pergunta não será repetida. A decisão expira ao encerrar a sessão ou reiniciar o DSH.', }, approve: applicationAccessDisplay.approve, } diff --git a/packages/experimental/cua/src/mouse-control.ts b/packages/experimental/cua/src/mouse-control.ts index 595cef0f0b..8482554943 100644 --- a/packages/experimental/cua/src/mouse-control.ts +++ b/packages/experimental/cua/src/mouse-control.ts @@ -7,9 +7,9 @@ import { NativeError } from './native-client.ts' /** Mouse control applies only to mouseMove and never grants access to an application. */ export class MouseControl { - private readonly decisions = new Map() + private readonly decisions = new WeakMap() - constructor(private readonly ctx: Context) {} + constructor(private readonly ctx: Context, private readonly requireApproval: boolean) {} /** * Select the native route without prompting the user. @@ -17,7 +17,7 @@ export class MouseControl { * @returns Whether this Session explicitly approved moving the system pointer. */ allowed(agent: Agent): boolean { - return this.decisions.get(agent.id) === true + return this.decisions.get(agent) === true } /** @@ -30,12 +30,13 @@ export class MouseControl { if (agent.session.header.origin === 'subagent') { throw new NativeError('MOUSE_CONTROL_UNAVAILABLE', "Delegated tasks cannot move the user's pointer. Use AX hover, click, or performAction. If the task requires a physical hover, report that step in your final result for the main task.") } - const decision = this.decisions.get(agent.id) + if (!this.requireApproval) return true + const decision = this.decisions.get(agent) if (decision === 'asking') throw new NativeError('MOUSE_CONTROL_PENDING', 'Wait for the pending mouse control request in this session before continuing') if (decision !== undefined) return decision const questions = this.ctx.get('userQuestions') if (!questions) throw new NativeError('MOUSE_APPROVAL_UNAVAILABLE', 'No human answerer is available for mouse control') - this.decisions.set(agent.id, 'asking') + this.decisions.set(agent, 'asking') const id = 'cua-mouse-control' const approve = mouseControlDisplay.approve.en const reject = 'Deny' @@ -49,10 +50,10 @@ export class MouseControl { const [answer] = result.answers const granted = result.answers.length === 1 && answer?.id === id && !answer.custom?.trim() && answer.selected.length === 1 && answer.selected[0] === approve - this.decisions.set(agent.id, granted) + this.decisions.set(agent, granted) return granted } catch (error) { - this.decisions.delete(agent.id) + this.decisions.delete(agent) if (error instanceof UserQuestionError && error.code === 'ASK_CANCELLED') { throw new NativeError('MOUSE_APPROVAL_CANCELLED', 'The mouse control request was dismissed. Continue without mouse control unless the user asks to reconsider', { cause: error }) } diff --git a/packages/experimental/cua/src/native-schema.ts b/packages/experimental/cua/src/native-schema.ts index 72487f2108..5bb061383f 100644 --- a/packages/experimental/cua/src/native-schema.ts +++ b/packages/experimental/cua/src/native-schema.ts @@ -1,7 +1,7 @@ /** Build-time native admission metadata derived from the shared operation definitions. */ import { z } from 'zod' import type { JSONSchema } from 'zod/v4/core' -import { NativeOnlyOperations, NativeResourceParameters, NativeToolOperations, type Request } from './protocol.ts' +import { NativeOnlyOperations, NativeResourceParameters, Request } from './protocol.ts' type Schema = JSONSchema.JSONSchema @@ -26,7 +26,7 @@ export function nativeProtocol( json: string } { const resources = new Map(Object.entries({ ...NativeResourceParameters, ...platformFields })) - const schemas = [...NativeToolOperations, ...NativeOnlyOperations, ...privateOperations] + const schemas = [...Request.options, ...NativeOnlyOperations, ...privateOperations] .map(schema => z.toJSONSchema(schema)) const fields = new Map>() const enums = new Map>>() diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index 161da04ee7..d3f0d6a0b6 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -34,8 +34,10 @@ const imageBase64 = z.string().min(1).refine(value => Buffer.from(value, 'base64 message: 'Expected canonical base64 image data', }) -/** Operations shared by the model and native helper, before Host parameters are added. */ -export const NativeToolOperations = [ +const mouseMove = { useSystemPointer: z.boolean().optional().describe('Request use of the system pointer. The Host checks permission before dispatch.') } + +/** Model-submittable operations, before the Host applies permission decisions and resource limits. */ +export const Request = z.discriminatedUnion('method', [ operation('getCapabilities', {}), operation('listApps', {}), operation('findApps', { query: z.string().min(1, 'Expected a non-empty search string').regex(/\S/, 'Expected a non-empty search string') }), operation('getApp', { selector: z.string().min(1) }), @@ -50,7 +52,8 @@ export const NativeToolOperations = [ operation('performAction', { ...snapshot, action: z.string().min(1), argument: jsonValue.optional() }), operation('invoke', snapshot), z.object({ method: z.literal('mouseMove'), params: z.union([ - z.object(snapshot).strict(), z.object(absoluteCoordinate).strict(), z.object(relativeCoordinate).strict(), + z.object({ ...snapshot, ...mouseMove }).strict(), z.object({ ...absoluteCoordinate, ...mouseMove }).strict(), + z.object({ ...relativeCoordinate, ...mouseMove }).strict(), ]) }).strict(), z.object({ method: z.literal('click'), params: z.union([ z.object({ ...snapshot, ...click }).strict(), @@ -77,10 +80,7 @@ export const NativeToolOperations = [ fromX: relativePoint, fromY: relativePoint, toX: relativePoint, toY: relativePoint }).strict(), ]) }).strict(), operation('screenshot', window), -] as const - -/** Model-submittable operations. The Host answers mouse-control requests without dispatching native input. */ -export const Request = z.discriminatedUnion('method', [...NativeToolOperations, operation('requestMouseControl', {})]) +]) /** Resource parameters supplied by the Host, never accepted from model tool arguments. */ export const NativeResourceParameters = { diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index 77923d64be..7c596c87d4 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -196,10 +196,11 @@ export interface CuaWindow extends CuaSurface { * Returning and subsequent observations do not restore the previous pointer position. * Other input and application changes can interrupt the interaction; observe before continuing. * @param target - current element id or screenshot-local [x,y]. - * @param options - coordinate mode; applies only to [x,y]. + * @param options - Coordinate mode for points and optional system-pointer request. False explicitly selects pointer-free hover. * @returns dispatch evidence and the selected route; observe the resulting UI after any UI delay. */ - mouseMove(target: CuaElementId | readonly [number, number], options?: CuaCoordinateOptions): Promise + mouseMove(target: CuaElementId | readonly [number, number], + options?: CuaCoordinateOptions & { useSystemPointer?: boolean }): Promise /** * Click a current element id or screenshot-local point. * Cancellation or timeout releases the held button; received input cannot be undone. The operation deadline includes holdMs. @@ -246,12 +247,6 @@ export interface CuaApp extends CuaSurface { } /** Native app API; its caller owns global bindings and variable persistence. */ export interface CuaApi { - /** - * Ask to move the user's pointer with mouseMove for this session until restart. Only the main task can ask. - * @returns True on approval, false on refusal. Both decisions last until restart for this session. - * @throws If the user dismisses the question without deciding or another request is pending. - */ - requestMouseControl(): Promise /** * Inspect permissions without requesting them. * @param options - output options. diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index 053010af02..50eb722493 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -891,10 +891,15 @@ it('expands menu contents and details only for the requested observation', async expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'getState', params: { appHandle: selected.appHandle, includeMenus: false, details: false } }) }) -it('requests mouse control without dispatching a move and returns the decision', async () => { - const dispatch = vi.fn().mockResolvedValueOnce(false).mockResolvedValueOnce(true) - const cua = createCuaApi(dispatch) - expect(await cua.requestMouseControl()).toBe(false) - expect(await cua.requestMouseControl()).toBe(true) - expect(dispatch.mock.calls).toEqual(Array(2).fill([{ method: 'requestMouseControl', params: {} }])) +it('passes explicit system-pointer intent for both mouseMove target forms', async () => { + const f = fixture() + const win = await (await f.api.getApp('Editor')).getWindow(23) + await win.mouseMove(element('0'), { useSystemPointer: true }) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'mouseMove', params: { + appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', elementId: '0', useSystemPointer: true, + } }) + await win.mouseMove([250, 750], { relative: true, useSystemPointer: false }) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'mouseMove', params: { + appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 250, y: 750, relative: true, useSystemPointer: false, + } }) }) diff --git a/snapshots/session/cua-native/replay.override.json b/snapshots/session/cua-native/replay.override.json index e6bf49a2c3..eb110a4715 100644 --- a/snapshots/session/cua-native/replay.override.json +++ b/snapshots/session/cua-native/replay.override.json @@ -1431,9 +1431,9 @@ "index": 0, "block": { "type": "tool-call", - "id": "cua-pointer-request", + "id": "cua-explicit-pointer-move", "name": "cua", - "arguments": "{\"method\":\"requestMouseControl\",\"params\":{}}" + "arguments": "{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true,\"useSystemPointer\":true}}" } }, { From def08e19a63ddabb7ccf4604717b4cc1ad282358 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 16:33:10 +0800 Subject: [PATCH 11/30] test(cua): record approval on the requested mouse move --- snapshots/session/cua-native/session.v4.jsonl | 8 ++--- .../cua-native/system-prompt.expected.md | 3 +- .../cua-native/tool-schemas.expected.json | 31 +++++++------------ .../session/cua-ptc/system-prompt.expected.md | 3 +- .../system-prompt.expected.md | 3 +- 5 files changed, 19 insertions(+), 29 deletions(-) diff --git a/snapshots/session/cua-native/session.v4.jsonl b/snapshots/session/cua-native/session.v4.jsonl index 2dc362b2af..8e343139cc 100644 --- a/snapshots/session/cua-native/session.v4.jsonl +++ b/snapshots/session/cua-native/session.v4.jsonl @@ -14,7 +14,7 @@ {"type":"session/title","data":{"title":"Inspect CUA capabilities, select the","messageSeqs":[8],"source":{"kind":"fallback"}}} {"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:4}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032889225,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032889225,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}}},{"type":"chunk","time":1794032889225,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032889225,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":1,"callId":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}} -{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-1"},"toolCallId":"cua-native-1","content":[{"type":"text","text":"{\"sessionLocked\":false,\"permissions\":{\"accessibility\":true,\"screenRecording\":true},\"platform\":\"macos\",\"methods\":[\"activate\",\"click\",\"drag\",\"findApps\",\"getApp\",\"getCapabilities\",\"getState\",\"invoke\",\"listActions\",\"listApps\",\"listWindows\",\"mouseMove\",\"paste\",\"performAction\",\"pressKey\",\"requestMouseControl\",\"screenshot\",\"scroll\",\"selectText\",\"sendKeyEvent\",\"setValue\",\"show\",\"typeText\"],\"appLevelAccessibility\":true,\"keyboard\":[\"app\"],\"browser\":false,\"coordinateSpace\":\"screenshot-pixels\"}"}],"isError":false,"id":"{{message:5}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"} +{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-1"},"toolCallId":"cua-native-1","content":[{"type":"text","text":"{\"sessionLocked\":false,\"permissions\":{\"accessibility\":true,\"screenRecording\":true},\"platform\":\"macos\",\"methods\":[\"activate\",\"click\",\"drag\",\"findApps\",\"getApp\",\"getCapabilities\",\"getState\",\"invoke\",\"listActions\",\"listApps\",\"listWindows\",\"mouseMove\",\"paste\",\"performAction\",\"pressKey\",\"screenshot\",\"scroll\",\"selectText\",\"sendKeyEvent\",\"setValue\",\"show\",\"typeText\"],\"appLevelAccessibility\":true,\"keyboard\":[\"app\"],\"browser\":false,\"coordinateSpace\":\"screenshot-pixels\"}"}],"isError":false,"id":"{{message:5}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":1}} {"type":"step/start","data":{"turn":1,"step":2}} {"type":"assistant/message","data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-unauthorized-getState","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:6}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032889794,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032889794,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-unauthorized-getState","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1794032889794,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032889794,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} @@ -227,9 +227,9 @@ {"type":"tool/result","data":{"turn":1,"step":43,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-unavailable"},"toolCallId":"cua-hover-unavailable","content":[{"type":"text","text":"Error: HOVER_UNAVAILABLE: No AX alternate UI at this target. No input was sent and the pointer did not move."}],"isError":true,"id":"{{message:89}}"}},"sourceEventSeqs":[224],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":43}} {"type":"step/start","data":{"turn":1,"step":44}} -{"type":"assistant/message","data":{"turn":1,"step":44,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-pointer-request","name":"cua","arguments":"{\"method\":\"requestMouseControl\",\"params\":{}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:90}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890679,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890679,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-pointer-request","name":"cua","arguments":"{\"method\":\"requestMouseControl\",\"params\":{}}"}}},{"type":"chunk","time":1794032890679,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890679,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":44,"callId":"cua-pointer-request","name":"cua","arguments":"{\"method\":\"requestMouseControl\",\"params\":{}}"}} -{"type":"tool/result","data":{"turn":1,"step":44,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-pointer-request"},"toolCallId":"cua-pointer-request","content":[{"type":"text","text":"true"}],"isError":false,"id":"{{message:91}}"}},"sourceEventSeqs":[229],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":44,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-explicit-pointer-move","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true,\"useSystemPointer\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:90}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890679,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890679,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-explicit-pointer-move","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true,\"useSystemPointer\":true}}"}}},{"type":"chunk","time":1794032890679,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890679,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":44,"callId":"cua-explicit-pointer-move","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true,\"useSystemPointer\":true}}"}} +{"type":"tool/result","data":{"turn":1,"step":44,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-explicit-pointer-move"},"toolCallId":"cua-explicit-pointer-move","content":[{"type":"text","text":"{\"dispatched\":true,\"hoverRoute\":\"system-pointer\",\"systemPointerMoved\":true}"}],"isError":false,"id":"{{message:91}}"}},"sourceEventSeqs":[229],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":44}} {"type":"step/start","data":{"turn":1,"step":45}} {"type":"assistant/message","data":{"turn":1,"step":45,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-system-hover","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:92}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890696,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890696,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-system-hover","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1794032890696,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890696,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index 87c08e7425..037bde95e5 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -43,7 +43,6 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. -- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Only the main task may ask, and only when a physical hover is necessary. A false result is a refusal for this session until restart. If the user dismisses the request, continue without mouse control unless they ask to reconsider. Await the outstanding request before calling again. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -68,7 +67,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, every mouseMove requires the target app to be in front and the point unobscured, including targets that previously supported background AX hover. If mouseMove returns FOREGROUND_REQUIRED, explain the necessary foreground step before activating the app, then observe the target again. +- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user; approval moves their pointer to this target immediately and makes subsequent ordinary moves use it until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. Without an approval requirement only explicitly requested moves use the system pointer. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/snapshots/session/cua-native/tool-schemas.expected.json b/snapshots/session/cua-native/tool-schemas.expected.json index 77b7653284..b9bb435797 100644 --- a/snapshots/session/cua-native/tool-schemas.expected.json +++ b/snapshots/session/cua-native/tool-schemas.expected.json @@ -484,6 +484,10 @@ "type": "string", "minLength": 1, "description": "Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot." + }, + "useSystemPointer": { + "description": "Request use of the system pointer. The Host checks permission before dispatch.", + "type": "boolean" } }, "required": [ @@ -521,6 +525,10 @@ "description": "Omitted or false uses pixels in the latest screenshot returned by this tool.", "type": "boolean", "const": false + }, + "useSystemPointer": { + "description": "Request use of the system pointer. The Host checks permission before dispatch.", + "type": "boolean" } }, "required": [ @@ -561,6 +569,10 @@ "type": "boolean", "const": true, "description": "Use integer thousandths of the window width and height, from 0 to 999." + }, + "useSystemPointer": { + "description": "Request use of the system pointer. The Host checks permission before dispatch.", + "type": "boolean" } }, "required": [ @@ -1413,25 +1425,6 @@ "params" ], "additionalProperties": false - }, - { - "type": "object", - "properties": { - "method": { - "type": "string", - "const": "requestMouseControl" - }, - "params": { - "type": "object", - "properties": {}, - "additionalProperties": false - } - }, - "required": [ - "method", - "params" - ], - "additionalProperties": false } ], "$defs": { diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index 2155b8c9ea..d5580e4964 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -45,7 +45,6 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. -- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Only the main task may ask, and only when a physical hover is necessary. A false result is a refusal for this session until restart. If the user dismisses the request, continue without mouse control unless they ask to reconsider. Await the outstanding request before calling again. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -70,7 +69,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, every mouseMove requires the target app to be in front and the point unobscured, including targets that previously supported background AX hover. If mouseMove returns FOREGROUND_REQUIRED, explain the necessary foreground step before activating the app, then observe the target again. +- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user; approval moves their pointer to this target immediately and makes subsequent ordinary moves use it until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. Without an approval requirement only explicitly requested moves use the system pointer. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/snapshots/session/ptc-script-retained/system-prompt.expected.md b/snapshots/session/ptc-script-retained/system-prompt.expected.md index ed2cfda2e8..3e0d591989 100644 --- a/snapshots/session/ptc-script-retained/system-prompt.expected.md +++ b/snapshots/session/ptc-script-retained/system-prompt.expected.md @@ -45,7 +45,6 @@ Signatures show resolved return values after await. ActionResult = { dispatched: Setup and discovery: - cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. -- cua.requestMouseControl() -> boolean — ask the user to allow mouseMove to move their system pointer for this session until DSH restarts. Only the main task may ask, and only when a physical hover is necessary. A false result is a refusal for this session until restart. If the user dismisses the request, continue without mouse control unless they ask to reconsider. Await the outstanding request before calling again. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -70,7 +69,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only request mouse control if the task cannot be completed with those alternatives. The question interrupts the user, and moving their pointer can disrupt their work. After approval, every mouseMove requires the target app to be in front and the point unobscured, including targets that previously supported background AX hover. If mouseMove returns FOREGROUND_REQUIRED, explain the necessary foreground step before activating the app, then observe the target again. +- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user; approval moves their pointer to this target immediately and makes subsequent ordinary moves use it until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. Without an approval requirement only explicitly requested moves use the system pointer. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. From 6f088b13e7d383a69202f28eaca354efea6d4e76 Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Sat, 7 Nov 2026 16:42:40 +0800 Subject: [PATCH 12/30] perf(forge-panel): highlight only after scrolling settles Highlighting waits until the page is idle and the scrollport has been still for 200 ms, so a slow grammar's first tokenization never interrupts a scroll. Adds the Agent Note recording the changes view decisions. --- ...orge-changes-view-responsiveness.i18n.yaml | 19 ++++++++++ ...11-07-forge-changes-view-responsiveness.md | 33 +++++++++++++++++ ...07-forge-changes-view-responsiveness.zh.md | 33 +++++++++++++++++ .../client-ui-forge/README.i18n.yaml | 4 +-- .../experimental/client-ui-forge/README.md | 2 +- .../experimental/client-ui-forge/README.zh.md | 2 +- .../client-ui-forge/src/client/ChangesTab.tsx | 3 +- .../client-ui-forge/src/client/DiffFile.tsx | 17 +++++---- .../src/client/section-reach.ts | 30 ++++++++++++++++ .../tests/diff-review.client.spec.tsx | 8 ++++- .../tests/section-reach.client.spec.ts | 36 +++++++++++++++++++ 11 files changed, 174 insertions(+), 13 deletions(-) create mode 100644 .agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.i18n.yaml create mode 100644 .agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md create mode 100644 .agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.zh.md diff --git a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.i18n.yaml new file mode 100644 index 0000000000..85b111788b --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.i18n.yaml @@ -0,0 +1,19 @@ +# Bilingual-pair consistency record for 2026-11-07-forge-changes-view-responsiveness.md (docs/i18n/README.md): per heading +# section, a hash of its English and Chinese blocks outside code blocks and generated regions. +# After editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md +/agent-note-forge-changes-view-responsiveness: + en: e809ffea0d880a4b + zh: 6725e6a9bd1c3bd1 +/agent-note-forge-changes-view-responsiveness/problem: + en: 77fab27f3d4a4a59 + zh: 46e0fa8b9ef978a7 +/agent-note-forge-changes-view-responsiveness/decision: + en: 8a0a9ef23a83ce2b + zh: 840d67e79ae39ff1 +/agent-note-forge-changes-view-responsiveness/alternatives-considered: + en: 9d12ccf1d236676d + zh: 0ea63c5c16c62de7 +/agent-note-forge-changes-view-responsiveness/consequences: + en: 4087fff405624d95 + zh: 2f0e577eea4daf40 diff --git a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md new file mode 100644 index 0000000000..197552f117 --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md @@ -0,0 +1,33 @@ +# Agent Note: Forge changes view responsiveness + +Status: implemented + +English | [中文](2026-11-07-forge-changes-view-responsiveness.zh.md) + +## Problem + +The forge changes view of a large change request froze the browser. Opening #7985 (366 files, 37,563 drawn lines) kept the main thread busy for about five seconds, "expand all" for fifteen, and every later state change, such as a poll result, a tool result, a file jump, or a keystroke in the file filter, re-rendered every file section. While the view was visible, the whole app slowed down: two subject-less `:has()` rules in `ui-sidebar-files` made each DOM change restyle the entire document, and the view's hundred thousand nodes made each restyle cost 170–260 ms, so a streaming Agent turn kept the main thread saturated. + +## Decision + +**Scoped `:has()` rules.** The navigator's covering rules name their row through `data-files-navigator-row`, which both hosts of `sidebar.files.navigator` set, so a DOM change outside such a row schedules no document-wide invalidation. + +**Sections draw what changed and what the reader reaches.** `ChangesTab` passes each memoized `DiffFile` values whose identity survives its renders: one handler object, one comment-actions object, and per-file strings and booleans. Tab visibility reaches only the menus and the composer through a context. An expanded file holds a 22 px row per drawn line until its section first comes within 1200 px of the scrollport, then draws its lines and keeps them. A jump keeps its target aligned for one second while nearby sections draw. + +**Idle-time highlighting.** Lines draw as plain text and take their tokens once the page is idle and the scrollport has been still for 200 ms, so tokenization never interrupts a scroll in progress. A pass highlights 100-line fragments while each idle period lasts and runs again only when the file's segments change or its own language's grammar loads; the shared highlighter's identity, which changes on every grammar load, no longer triggers one. + +**Fewer serial reads.** A panel state equal to the published one reaches no follower. `changeRequestDiff` reads the patch and `.gitattributes` together. Snapshots carry the base commit as `baseSha`; `ForgeService.diff` passes it to the provider, and forge-github starts the compare read while its GraphQL read confirms the head. + +## Alternatives considered + +**`content-visibility: auto` alone** cut style work as much as scoping the rules did, but kept the React render and Shiki tokenization of every expanded file at open, about 3 s of script, and its placeholder sizes would need estimates for every collapsed and expanded section. + +**Full virtualization that also unmounts far sections** would bound the DOM after long scrolls, but a section holds a comment draft and a text selection, which unmounting loses. + +**Highlighting in a Web Worker** removes tokenization from the main thread entirely, including a heavy grammar's first use, but the shared highlighter in `ui-primitives` owns Shiki; moving it is a separate change for every consumer. + +**A provider-private cache of base commits** would avoid the optional `baseSha` field, but it adds retained state without an owner that knows when a change request stops being watched. + +## Consequences + +Opening #7985 now takes 0.5 s of main-thread work instead of 5.2 s, and file jumps, filtering, polling, and Agent turns no longer produce long tasks. In exchange, the browser's find in page does not see lines of files the reader has not approached, highlighting restarts at 100-line fragment boundaries, so a multi-line comment or string crossing one can be mis-colored, and the first use of a slow grammar, such as C++, still produces a 400–550 ms long task while scrolling. A diff read costs a second compare request when the base commit changed after the latest status. diff --git a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.zh.md b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.zh.md new file mode 100644 index 0000000000..80be6bf6fa --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.zh.md @@ -0,0 +1,33 @@ +# Agent Note: Forge 变更视图的响应速度 + +Status: implemented + +[English](2026-11-07-forge-changes-view-responsiveness.md) | 中文 + +## Problem + +大型变更请求的 forge 变更视图会让浏览器卡住。打开 #7985（366 个文件、37,563 个绘制行）时主线程忙约 5 秒，“全部展开”约 15 秒；之后的每次状态变化，例如轮询结果、工具结果、跳转文件或在文件筛选框中按键，都会重渲染所有文件段。视图可见时整个应用都变慢：`ui-sidebar-files` 中两条没有主体的 `:has()` 规则让每次 DOM 变化都重算整个文档的样式，而视图的十万个节点让每次重算耗时 170–260 ms，所以 Agent 流式输出的回合会让主线程一直饱和。 + +## Decision + +**限定 `:has()` 规则。** 导航器的遮盖规则通过 `data-files-navigator-row` 指定所在行，`sidebar.files.navigator` 的两个宿主都会设置它，因此这类行之外的 DOM 变化不再触发全文档失效。 + +**文件段只绘制变化的部分和读者接近的部分。** `ChangesTab` 传给每个 memo 化的 `DiffFile` 的值在多次渲染之间保持同一标识：一个处理函数对象、一个评论操作对象，以及每个文件的字符串和布尔值。标签页可见性通过 context 只传给菜单和评论编辑器。展开的文件在它的段第一次进入滚动区域 1200 px 范围之前，按每个绘制行 22 px 占位，之后绘制各行并一直保留。跳转后的一秒内，目标文件在周围文件段绘制时保持对齐。 + +**空闲时高亮。** 各行先以纯文本绘制，在页面空闲且滚动区域静止 200 ms 后得到 token，因此分词不会打断正在进行的滚动。一次高亮在每个空闲时段内处理多个 100 行的片段，只有文件的分段变化或该语言自己的语法加载完成时才重新运行；共享高亮器的标识在任何语法加载时都会变化，不再触发重新高亮。 + +**减少串行读取。** 与已发布状态相同的面板状态不会通知任何关注方。`changeRequestDiff` 同时读取补丁和 `.gitattributes`。快照把基准提交作为 `baseSha` 携带；`ForgeService.diff` 把它传给提供方，forge-github 在 GraphQL 读取确认 head 的同时就开始 compare 读取。 + +## Alternatives considered + +**只用 `content-visibility: auto`** 减少的样式工作与限定规则相当，但打开时仍要对每个展开文件做 React 渲染和 Shiki 分词，约 3 秒脚本时间，而且每个折叠和展开的段都需要估计占位尺寸。 + +**同时卸载远处文件段的完整虚拟化** 可以在长距离滚动后限制 DOM 规模，但文件段持有评论草稿和文本选区，卸载会丢失它们。 + +**在 Web Worker 中高亮** 能把分词完全移出主线程，包括重型语法的首次使用，但 Shiki 由 `ui-primitives` 中的共享高亮器持有；迁移它是影响所有使用方的另一项改动。 + +**由提供方私有缓存基准提交** 可以省去可选的 `baseSha` 字段，但会引入保留状态，而且没有一个知道变更请求何时不再被 watch 的所有者。 + +## Consequences + +打开 #7985 的主线程工作从 5.2 秒降到 0.5 秒，跳转文件、筛选、轮询和 Agent 回合不再产生长任务。代价是：浏览器的页内查找看不到读者尚未接近的文件中的行；高亮在 100 行片段边界处重新开始，跨越边界的多行注释或字符串可能着色错误；慢语法（例如 C++）首次使用时，滚动中仍会出现 400–550 ms 的长任务。最新状态之后基准提交有变化时，一次 diff 读取会多一次 compare 请求。 diff --git a/packages/experimental/client-ui-forge/README.i18n.yaml b/packages/experimental/client-ui-forge/README.i18n.yaml index 5bb4a25d1e..d2e078a9fc 100644 --- a/packages/experimental/client-ui-forge/README.i18n.yaml +++ b/packages/experimental/client-ui-forge/README.i18n.yaml @@ -18,8 +18,8 @@ en: c7e1b7d87ccccd2e zh: 42d864b918a6175c /deepseek-ai-dsh-experimental-client-ui-forge/understand-the-implementation: - en: 0afedb76112dfcb8 - zh: 52d5ddc7a4f9d31c + en: d0756f4b236e1e6f + zh: 9a02f6fa80118c5a /deepseek-ai-dsh-experimental-client-ui-forge/further-exploration: en: fa4d8c579ffc49ff zh: 5856ed7469e4585b diff --git a/packages/experimental/client-ui-forge/README.md b/packages/experimental/client-ui-forge/README.md index ddbd23b094..bc116fbb1d 100644 --- a/packages/experimental/client-ui-forge/README.md +++ b/packages/experimental/client-ui-forge/README.md @@ -73,7 +73,7 @@ Each inspection also decides whether the worktree is missing: the Session's dire `changeRequestStates` streams, for the Session list, the last known number, title, state, draft flag, checks rollup, review decision, and mergeability of every change request that a listed Session the caller may read binds: a `baseline` frame, then `changed` frames. One run serves every stream. Cancellation is observed before initialization and between frames; the last stream and plugin disposal await initialization and accepted writes, then close the run’s storage domain. A later run waits for that closure before reopening the domain and reading its stored states. Binding updates received during listing override its initial snapshot. It lists Sessions through `sessionQuery` without loading them, takes the visible bindings of a loaded Session from `ctx.forgeSession` and of another from its cached `forgeChangeRequests` summary, follows binding changes of loaded Sessions, and subscribes to each bound change request through `ctx.forge.watch`; closed requests use the idle polling cadence, and merged requests remain subscribed for explicit refresh. It stores every changed state in the `forge_panel_change_requests` storage domain under the SHA-256 digest of its URL and drops unbound stored states only when every listed Session has known bindings; a failed write is logged and the state stays in memory. The stream fails when the Host lacks the Session query, projection cache, storage, or Session store service. -The browser entry mounts the Remote, then the UI. One stream per Session feeds the tab, the header button, the New Session option, and the creation card; it opens with the first subscriber, closes with the last, reopens after a transport reset, and keeps the last state while the Session scope is retained. Releasing that scope closes the stream and removes its cached state. The New Session option fills `conversation.hero.options` and the worktree card `conversation.input.dock`. The worktree card shows a running creation, with Cancel calling `stopCreation` while a message waits for it, or the failed or stopped attempt a message waits after, with Retry calling `retryWorktree` and Continue without a worktree calling `cancelWorktree`. The tab shortens paths with the Host home from the Remote's Host facts, read again after each transport reset. The tab reports its visibility to the header button's pressed state. The header button collapses a docked tab or closes its floating tab. The toast lives in a `shell.overlay` entry so it outlives the tab. One stream of change request states feeds the Session row icons in `sidebar.session.row.trailing` and the hover card section in `sidebar.session.row.hover` while any of them is mounted; both take a Session's worktree and bindings from its Session list summary values. The change request card takes a change request, its snapshot, and optional actions, so another surface can reuse it. The changes view is the `forge-changes` resource tab type: its addresses, `dsh-resource://forge-changes//local` and `dsh-resource://forge-changes//change/`, name the source, and a navigation may name a file to scroll to. The tab type enables `keepMounted`, retaining visited bodies and their owning Session references until their tab records end. Its keyed panel hook subscribes only for the visible body; other panel surfaces may keep the shared Session stream open. Its body keeps the scope, base branch, chosen commits, lens, test file switch, and layout choices of each tab in a slot store, keeps viewed marks per Session in `localStorage` under `dsh.forge-viewed.v1.`, at most 20 sources per Session and 50 Sessions, reads through the Remote, and passes each file section values that keep their identity across its renders, so a render redraws only the sections whose values changed. An expanded file draws its lines once its section first comes within 1200px of the scrollport, keeps them drawn after that, and until then holds a 22px row per drawn line; the lines are highlighted with the shared code highlighter when the page is next idle, again only when the file's segments change or its language's grammar finishes loading; a chip title shows its name with the changes icon. +The browser entry mounts the Remote, then the UI. One stream per Session feeds the tab, the header button, the New Session option, and the creation card; it opens with the first subscriber, closes with the last, reopens after a transport reset, and keeps the last state while the Session scope is retained. Releasing that scope closes the stream and removes its cached state. The New Session option fills `conversation.hero.options` and the worktree card `conversation.input.dock`. The worktree card shows a running creation, with Cancel calling `stopCreation` while a message waits for it, or the failed or stopped attempt a message waits after, with Retry calling `retryWorktree` and Continue without a worktree calling `cancelWorktree`. The tab shortens paths with the Host home from the Remote's Host facts, read again after each transport reset. The tab reports its visibility to the header button's pressed state. The header button collapses a docked tab or closes its floating tab. The toast lives in a `shell.overlay` entry so it outlives the tab. One stream of change request states feeds the Session row icons in `sidebar.session.row.trailing` and the hover card section in `sidebar.session.row.hover` while any of them is mounted; both take a Session's worktree and bindings from its Session list summary values. The change request card takes a change request, its snapshot, and optional actions, so another surface can reuse it. The changes view is the `forge-changes` resource tab type: its addresses, `dsh-resource://forge-changes//local` and `dsh-resource://forge-changes//change/`, name the source, and a navigation may name a file to scroll to. The tab type enables `keepMounted`, retaining visited bodies and their owning Session references until their tab records end. Its keyed panel hook subscribes only for the visible body; other panel surfaces may keep the shared Session stream open. Its body keeps the scope, base branch, chosen commits, lens, test file switch, and layout choices of each tab in a slot store, keeps viewed marks per Session in `localStorage` under `dsh.forge-viewed.v1.`, at most 20 sources per Session and 50 Sessions, reads through the Remote, and passes each file section values that keep their identity across its renders, so a render redraws only the sections whose values changed. An expanded file draws its lines once its section first comes within 1200px of the scrollport, keeps them drawn after that, and until then holds a 22px row per drawn line; the lines are highlighted with the shared code highlighter in 100-line fragments when the page is idle and the scrollport has been still for 200 ms, again only when the file's segments change or its language's grammar finishes loading; a chip title shows its name with the changes icon. | Source | Responsibility | |---|---| diff --git a/packages/experimental/client-ui-forge/README.zh.md b/packages/experimental/client-ui-forge/README.zh.md index bd8de3f4bf..81f2bed6f0 100644 --- a/packages/experimental/client-ui-forge/README.zh.md +++ b/packages/experimental/client-ui-forge/README.zh.md @@ -73,7 +73,7 @@ Session 在所创建的 worktree 中工作时，`localDiff` 以它的 `baseCommi `changeRequestStates` 为会话列表推送调用方可读的、已列出的 Session 所绑定的每个变更请求的最近编号、标题、状态、草稿标记、检查汇总、评审结论与可合并性：先是一个 `baseline` 帧，之后是 `changed` 帧。所有流共用一次运行。初始化之前和帧之间均响应取消；最后一条流结束及插件卸载会等待初始化与已接受的写入完成，再关闭本次运行的存储域。后续运行会等待该域关闭后重新打开，并读取已存状态。列举期间收到的绑定更新优先于初始快照。它经 `sessionQuery` 列出 Session 而不加载它们，已加载 Session 的可见绑定取自 `ctx.forgeSession`，其他 Session 的取自缓存的 `forgeChangeRequests` 摘要；它跟随已加载 Session 的绑定变化，并经 `ctx.forge.watch` 订阅每个已绑定的变更请求；已关闭的按空闲间隔轮询，已合并的保留订阅以接收显式刷新结果。每个变化了的状态以其 URL 的 SHA-256 摘要为键存入 `forge_panel_change_requests` 存储域，仅在所有已列出 Session 的绑定均已知时删除无人绑定的已存状态；写入失败时记录日志，状态仍保留在内存中。宿主缺少 Session 查询、投影缓存、存储或 Session 存储服务时，流以失败结束。 -浏览器入口先挂载 Remote，再挂载界面。每个 Session 一条流，同时供给标签页、标题栏按钮、新会话选项与创建卡片；第一个订阅方出现时打开，最后一个离开时关闭，传输重置后重新打开，在 Session scope 保留期间保存最后的状态。释放该 scope 时关闭流并移除缓存状态。新会话选项填入 `conversation.hero.options`，worktree 卡片填入 `conversation.input.dock`。worktree 卡片显示正在进行的创建，有消息在等待时“取消”调用 `stopCreation`；或显示消息等待时失败或被停止的那次尝试，“重试”调用 `retryWorktree`，“不使用工作树”调用 `cancelWorktree`。标签页用 Remote 宿主信息中的宿主主目录缩短路径，每次传输重置后重新读取。标签页把自己是否可见报告给标题栏按钮的按下状态。标题栏按钮收起停靠的标签页，或关闭其浮动标签页。toast 位于 `shell.overlay` 条目中，因此比标签页存在得更久。只要 `sidebar.session.row.trailing` 中的会话行图标或 `sidebar.session.row.hover` 中的悬浮卡片部分有一个已挂载，就由同一条变更请求状态流供给它们；两者都从会话列表的摘要值中读取 Session 的 worktree 与绑定。变更请求卡片接收变更请求、它的快照与可选操作，其他界面可以复用。变更视图是 `forge-changes` 资源标签页类型：它的地址 `dsh-resource://forge-changes//local` 与 `dsh-resource://forge-changes//change/` 指明来源，导航参数可以指定要滚动到的文件。标签页类型启用 `keepMounted`，将访问过的正文及其所属 Session 引用保留至标签页记录结束。它的 keyed panel hook 仅为可见正文订阅；其他面板界面仍可能保持共享 Session 流开启。它的正文用 slot store 按标签页保存范围、基准分支、所选提交、透镜、测试文件开关与布局选择，按 Session 把已审阅标记保存在 `localStorage` 的 `dsh.forge-viewed.v1.` 下（每个 Session 最多 20 个来源，最多 50 个 Session），经 Remote 读取，并且传给每个文件段的值在多次渲染之间保持同一标识，所以一次渲染只重绘值有变化的文件段。展开的文件在它的段第一次进入滚动区域 1200px 范围内时才绘制各行，此后一直保留；在此之前按每个绘制行 22px 占位。各行在页面下次空闲时用共享的代码高亮器高亮，之后只在文件的分段变化或该语言的语法加载完成时重新高亮；标签页标题显示变更图标与视图名称。 +浏览器入口先挂载 Remote，再挂载界面。每个 Session 一条流，同时供给标签页、标题栏按钮、新会话选项与创建卡片；第一个订阅方出现时打开，最后一个离开时关闭，传输重置后重新打开，在 Session scope 保留期间保存最后的状态。释放该 scope 时关闭流并移除缓存状态。新会话选项填入 `conversation.hero.options`，worktree 卡片填入 `conversation.input.dock`。worktree 卡片显示正在进行的创建，有消息在等待时“取消”调用 `stopCreation`；或显示消息等待时失败或被停止的那次尝试，“重试”调用 `retryWorktree`，“不使用工作树”调用 `cancelWorktree`。标签页用 Remote 宿主信息中的宿主主目录缩短路径，每次传输重置后重新读取。标签页把自己是否可见报告给标题栏按钮的按下状态。标题栏按钮收起停靠的标签页，或关闭其浮动标签页。toast 位于 `shell.overlay` 条目中，因此比标签页存在得更久。只要 `sidebar.session.row.trailing` 中的会话行图标或 `sidebar.session.row.hover` 中的悬浮卡片部分有一个已挂载，就由同一条变更请求状态流供给它们；两者都从会话列表的摘要值中读取 Session 的 worktree 与绑定。变更请求卡片接收变更请求、它的快照与可选操作，其他界面可以复用。变更视图是 `forge-changes` 资源标签页类型：它的地址 `dsh-resource://forge-changes//local` 与 `dsh-resource://forge-changes//change/` 指明来源，导航参数可以指定要滚动到的文件。标签页类型启用 `keepMounted`，将访问过的正文及其所属 Session 引用保留至标签页记录结束。它的 keyed panel hook 仅为可见正文订阅；其他面板界面仍可能保持共享 Session 流开启。它的正文用 slot store 按标签页保存范围、基准分支、所选提交、透镜、测试文件开关与布局选择，按 Session 把已审阅标记保存在 `localStorage` 的 `dsh.forge-viewed.v1.` 下（每个 Session 最多 20 个来源，最多 50 个 Session），经 Remote 读取，并且传给每个文件段的值在多次渲染之间保持同一标识，所以一次渲染只重绘值有变化的文件段。展开的文件在它的段第一次进入滚动区域 1200px 范围内时才绘制各行，此后一直保留；在此之前按每个绘制行 22px 占位。各行在页面空闲且滚动区域静止 200 ms 后，用共享的代码高亮器按 100 行一片高亮，之后只在文件的分段变化或该语言的语法加载完成时重新高亮；标签页标题显示变更图标与视图名称。 | 源码 | 职责 | |---|---| diff --git a/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx b/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx index 8902be11e2..c47a16f9a6 100644 --- a/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx +++ b/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx @@ -611,7 +611,7 @@ export function ChangesTab({ const release = (): void => { cancelAnimationFrame(frame) for (const type of READER_SCROLL) port.removeEventListener(type, release) - if (releasePin.current === release) releasePin.current = null + releasePin.current = null } const keep = (now: number): void => { if (now > until) { @@ -880,6 +880,7 @@ export function ChangesTab({
{ + reach.scrolled() if (jumpPosition.current?.top !== event.currentTarget.scrollTop) jumpPosition.current = null }}> {files.map((file) => { diff --git a/packages/experimental/client-ui-forge/src/client/DiffFile.tsx b/packages/experimental/client-ui-forge/src/client/DiffFile.tsx index cb6c3ad121..db48c61457 100644 --- a/packages/experimental/client-ui-forge/src/client/DiffFile.tsx +++ b/packages/experimental/client-ui-forge/src/client/DiffFile.tsx @@ -160,12 +160,13 @@ function addChunkSpans(spans: FileSpans, chunk: HighlightChunk, highlighter: Cod } /** - * Highlight a file's drawn lines in idle time, once its language's grammar is ready: each idle period highlights - * fragments while it lasts, at least one, and the lines take their tokens when the last fragment is done. Until - * then, the lines keep the tokens of the previous pass, by line number, and new lines draw as plain text. A grammar - * that loads for another language starts no pass. + * Highlight a file's drawn lines in idle time while the scrollport is still, once its language's grammar is ready: + * each idle period highlights fragments while it lasts, at least one, and the lines take their tokens when the last + * fragment is done. Until then, the lines keep the tokens of the previous pass, by line number, and new lines draw as + * plain text. A grammar that loads for another language starts no pass. */ function useFileSpans(path: string, segments: readonly DiffSegment[]): FileSpans | undefined { + const reach = useContext(SectionReachContext) const highlighter = useCodeHighlighter(languageForPath(path)) // The highlighter changes whenever any grammar loads; the empty fragment tells whether this language's is ready. const ready = highlighter('') !== undefined @@ -188,11 +189,13 @@ function useFileSpans(path: string, segments: readonly DiffSegment[]): FileSpans next++ if ((deadline?.timeRemaining() ?? 0) <= 0) break } - cancel = whenIdle(step) + cancel = later(step) } - let cancel = whenIdle(step) + // Outside a changes view nothing scrolls the sections, so idle time alone gates the pass. + const later = reach === null ? whenIdle : (task: (deadline: IdleDeadline | undefined) => void) => reach.whenSettled(task) + let cancel = later(step) return () => { cancel() } - }, [segments, ready]) + }, [segments, ready, reach]) return spans } diff --git a/packages/experimental/client-ui-forge/src/client/section-reach.ts b/packages/experimental/client-ui-forge/src/client/section-reach.ts index d926bd1ee2..1cb3997531 100644 --- a/packages/experimental/client-ui-forge/src/client/section-reach.ts +++ b/packages/experimental/client-ui-forge/src/client/section-reach.ts @@ -9,6 +9,9 @@ import { createContext } from 'react' /** Distance beyond the scrollport's edges within which a section counts as reached. */ const REACH_MARGIN = '1200px 0px' +/** Time, in milliseconds, the scrollport stays still before deferred highlighting runs. */ +const SCROLL_QUIET_MS = 200 + const noop = (): void => {} /** @@ -18,6 +21,33 @@ const noop = (): void => {} export class SectionReach { private observer: IntersectionObserver | undefined private readonly waiting = new Map void>() + private scrolledAt = Number.NEGATIVE_INFINITY + + /** Record that the scrollport scrolled, which holds back work started through {@link whenSettled}. */ + scrolled(): void { + this.scrolledAt = performance.now() + } + + /** + * Run `task` once the page is idle and the scrollport has been still for {@link SCROLL_QUIET_MS}, so work that + * can wait never interrupts a scroll in progress. + * @param task - deferrable work; it receives the idle period, or undefined without one. + * @returns the canceller, which has no effect after `task` ran. + */ + whenSettled(task: (deadline: IdleDeadline | undefined) => void): () => void { + let cancel = noop + const attempt = (deadline: IdleDeadline | undefined): void => { + const wait = this.scrolledAt + SCROLL_QUIET_MS - performance.now() + if (wait <= 0) { + task(deadline) + return + } + const id = setTimeout(() => { cancel = whenIdle(attempt) }, wait) + cancel = () => { clearTimeout(id) } + } + cancel = whenIdle(attempt) + return () => { cancel() } + } /** * Measure sections against `root`, or stop measuring when it unmounts. Sections that are still waiting move to the new root. diff --git a/packages/experimental/client-ui-forge/tests/diff-review.client.spec.tsx b/packages/experimental/client-ui-forge/tests/diff-review.client.spec.tsx index 33f82ad73c..71bc95a914 100644 --- a/packages/experimental/client-ui-forge/tests/diff-review.client.spec.tsx +++ b/packages/experimental/client-ui-forge/tests/diff-review.client.spec.tsx @@ -1,6 +1,6 @@ // @vitest-environment jsdom /** Diff gestures, browser clipboard output, and retained comments at their captured lines. */ -import { act, cleanup, fireEvent, render, within } from '@testing-library/react' +import { act, cleanup, fireEvent, render, waitFor, within } from '@testing-library/react' import { afterEach, expect, it, vi } from 'vitest' import { makeTranslate } from '@deepseek-ai/dsh-client-test-runtime' import { DiffFile, type DiffFileProps } from '../src/client/DiffFile.tsx' @@ -249,3 +249,9 @@ it('places left and right records in unified rows and retains outdated records f expect(m.container.querySelectorAll('[data-forge-review-thread]')).toHaveLength(3) expect(within(m.container).getByRole('button', { name: /changed lines/ })).toBeTruthy() }) + +it('highlights a section drawn outside a changes view once the page is idle', async () => { + const m = mount({ file: { ...file, path: 'a.ts' } }) + expect(m.container.querySelector('[data-diff-code]')).toBeNull() + await waitFor(() => { expect(m.container.querySelectorAll('[data-diff-code]').length).toBeGreaterThan(0) }) +}) diff --git a/packages/experimental/client-ui-forge/tests/section-reach.client.spec.ts b/packages/experimental/client-ui-forge/tests/section-reach.client.spec.ts index 3533147a0c..f33a2422ec 100644 --- a/packages/experimental/client-ui-forge/tests/section-reach.client.spec.ts +++ b/packages/experimental/client-ui-forge/tests/section-reach.client.spec.ts @@ -81,6 +81,42 @@ describe('SectionReach', () => { }) }) +describe('SectionReach.whenSettled', () => { + it('waits for the scrollport to stay still before running deferred work, and can be cancelled while it waits', () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] }) + try { + vi.stubGlobal('requestIdleCallback', undefined) + const reach = new SectionReach() + const settled = vi.fn() + reach.whenSettled(settled) + vi.advanceTimersByTime(0) + expect(settled).toHaveBeenCalledOnce() + + reach.scrolled() + const held = vi.fn() + reach.whenSettled(held) + vi.advanceTimersByTime(0) + expect(held).not.toHaveBeenCalled() + vi.advanceTimersByTime(100) + reach.scrolled() + vi.advanceTimersByTime(100) + expect(held).not.toHaveBeenCalled() + vi.advanceTimersByTime(200) + expect(held).toHaveBeenCalledOnce() + + reach.scrolled() + const cancelled = vi.fn() + const stop = reach.whenSettled(cancelled) + vi.advanceTimersByTime(0) + stop() + vi.advanceTimersByTime(500) + expect(cancelled).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) +}) + describe('whenIdle', () => { it('waits for an idle period, at most the timeout, and can be cancelled', () => { const request = vi.fn<(task: () => void, options: { timeout: number }) => number>(() => 7) From c683f402397f57827ef7d11d6d648af617ca1ee1 Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Sat, 7 Nov 2026 17:01:10 +0800 Subject: [PATCH 13/30] test(forge-panel): scroll to a file before reading its lines An expanded file now draws its lines once the reader approaches it, and a jump to the last file settles at the end of the list after it draws. Regenerates the config catalog line reference. --- apps/web/tests/forge-panel.e2e.ts | 4 ++++ docs/config-catalog.md | 2 +- docs/config-catalog.zh.md | 2 +- packages/experimental/forge/tests/fake-provider.ts | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/web/tests/forge-panel.e2e.ts b/apps/web/tests/forge-panel.e2e.ts index bd72c7d8ec..f7820e556d 100644 --- a/apps/web/tests/forge-panel.e2e.ts +++ b/apps/web/tests/forge-panel.e2e.ts @@ -1175,6 +1175,8 @@ it.skipIf(MODE === 'record')('retains a wrapped code line and tree width across return { path: element.getAttribute('data-forge-diff-file'), line: line.dataset.newLine, offset: line.getBoundingClientRect().top - top, scrollTop: viewport.scrollTop } }) + // An expanded file draws its lines once the reader scrolls near it. + await file.scrollIntoViewIfNeeded() await code.waitFor() for (const width of [240, 320]) { if (width === 320) { @@ -1252,6 +1254,8 @@ it.skipIf(MODE === 'record')('uses horizontal arrows to navigate files while pre expect(await scroll.evaluate(element => document.activeElement === element)).toBe(true) await page.keyboard.press('ArrowRight') await expect.poll(() => tree.getByRole('button', { name: 'README.md', exact: true }).getAttribute('aria-current')).toBe('true') + // The last file cannot reach the top, so the jump settles at the end of the list once the file has drawn its lines. + await expect.poll(() => scroll.evaluate(element => element.scrollHeight - element.clientHeight - element.scrollTop)).toBeLessThan(1) const lastPosition = await scroll.evaluate(element => element.scrollTop) const lastBox = await c.boundingBox(), viewport = await scroll.boundingBox() if (lastBox === null || viewport === null) throw new Error('The last file and scroll viewport must be rendered') diff --git a/docs/config-catalog.md b/docs/config-catalog.md index d6284aee3c..79ceea0f2d 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1188,7 +1188,7 @@ export type SourceSigning = 'ad-hoc' | { ## `@deepseek-ai/dsh-experimental-forge` -- `source`: [`packages/experimental/forge/src/types.ts:273`](../packages/experimental/forge/src/types.ts) +- `source`: [`packages/experimental/forge/src/types.ts:279`](../packages/experimental/forge/src/types.ts) ```ts config-catalog /** Poll cadence and fetch deadline for watched change requests. */ diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index 70e75f8142..93f845deab 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -1190,7 +1190,7 @@ export type SourceSigning = 'ad-hoc' | { ## `@deepseek-ai/dsh-experimental-forge` -- `source`: [`packages/experimental/forge/src/types.ts:273`](../packages/experimental/forge/src/types.ts) +- `source`: [`packages/experimental/forge/src/types.ts:279`](../packages/experimental/forge/src/types.ts) ```ts config-catalog /** Poll cadence and fetch deadline for watched change requests. */ diff --git a/packages/experimental/forge/tests/fake-provider.ts b/packages/experimental/forge/tests/fake-provider.ts index d82f6e7b65..4c035071b7 100644 --- a/packages/experimental/forge/tests/fake-provider.ts +++ b/packages/experimental/forge/tests/fake-provider.ts @@ -111,7 +111,7 @@ export class FakeProvider implements ForgeProvider { return this.actionResult } - async diff(target: ChangeRequestRef, headSha: string): Promise { + async diff(target: ChangeRequestRef, headSha: string, _signal: AbortSignal, _base?: string): Promise { this.actions.push(`diff ${target.number} ${headSha}`) return this.readResult } From 0356074ec52fa48f7ca921f457e9a644bc06838f Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 17:07:35 +0800 Subject: [PATCH 14/30] refactor(cua): let providers own hover options and request named permissions --- docs/config-catalog.md | 2 +- docs/config-catalog.zh.md | 2 +- .../experimental/cua-macos/src/guidance.ts | 1 + packages/experimental/cua-macos/src/index.ts | 18 +++++- .../cua-macos/src/mouse-control-locales.ts | 23 +++++++ .../cua-macos/src/mouse-control.ts | 51 +++++++++++++++ .../tests/protocol-generation.spec.ts | 10 +++ .../tests/provider-lifecycle.spec.ts | 29 +++++++-- packages/experimental/cua/README.i18n.yaml | 8 +-- packages/experimental/cua/README.md | 4 +- packages/experimental/cua/README.zh.md | 4 +- packages/experimental/cua/src/api.ts | 23 ++++--- .../cua/src/client/ApplicationApproval.tsx | 4 +- packages/experimental/cua/src/guidance.ts | 2 +- packages/experimental/cua/src/index.ts | 39 ++++------- packages/experimental/cua/src/locales.ts | 25 -------- .../experimental/cua/src/mouse-control.ts | 63 ------------------ packages/experimental/cua/src/protocol.ts | 12 ++-- .../cua/src/session-permissions.ts | 64 +++++++++++++++++++ packages/experimental/cua/src/types.ts | 5 +- packages/experimental/cua/tests/api.spec.ts | 21 ++++-- .../application-approval.client.spec.tsx | 2 +- 22 files changed, 255 insertions(+), 157 deletions(-) create mode 100644 packages/experimental/cua-macos/src/mouse-control-locales.ts create mode 100644 packages/experimental/cua-macos/src/mouse-control.ts delete mode 100644 packages/experimental/cua/src/mouse-control.ts create mode 100644 packages/experimental/cua/src/session-permissions.ts diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 26a136abb0..ced0f149e5 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1156,7 +1156,7 @@ export interface Config { - `inject`: `computerUse` · `tools` · `systemPrompt` · `subprocess` · `agents` - `refs`: [`CuaConfig`](../packages/experimental/cua/src/index.ts) -- `source`: [`packages/experimental/cua-macos/src/index.ts:33`](../packages/experimental/cua-macos/src/index.ts) +- `source`: [`packages/experimental/cua-macos/src/index.ts:48`](../packages/experimental/cua-macos/src/index.ts) ```ts config-catalog /** Native preparation, source-build signing, operation bounds, and profile-local application grants. */ diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index c7dfeff61e..e91a7e3d53 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -1158,7 +1158,7 @@ export interface Config { - `inject`: `computerUse` · `tools` · `systemPrompt` · `subprocess` · `agents` - `refs`: [`CuaConfig`](../packages/experimental/cua/src/index.ts) -- `source`: [`packages/experimental/cua-macos/src/index.ts:33`](../packages/experimental/cua-macos/src/index.ts) +- `source`: [`packages/experimental/cua-macos/src/index.ts:48`](../packages/experimental/cua-macos/src/index.ts) ```ts config-catalog /** Native preparation, source-build signing, operation bounds, and profile-local application grants. */ diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index 63a81e0b40..38f0328499 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -16,6 +16,7 @@ export const GUIDANCE = `${PLATFORM_GUIDANCE} ${COMMON_GUIDANCE.replace(/^- ([\w.]+)\(.*$/gm, (entry, method: string) => { if (method === 'pressKey') return `${entry} - sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.` + if (method === 'mouseMove') entry = entry.replace('relative?: boolean', 'relative?: boolean, useSystemPointer?: boolean') const detail = methodDetails.get(method) return detail ? `${entry} ${detail}` : entry })}` diff --git a/packages/experimental/cua-macos/src/index.ts b/packages/experimental/cua-macos/src/index.ts index 26142f654c..bf4359155c 100644 --- a/packages/experimental/cua-macos/src/index.ts +++ b/packages/experimental/cua-macos/src/index.ts @@ -4,14 +4,29 @@ import { fileURLToPath } from 'node:url' import type { Context } from '@deepseek-ai/cordis' import Schema from '@deepseek-ai/schemastery' import { installCua, type CuaConfig, type CuaStartupCapabilities } from '@deepseek-ai/dsh-cua' +import type { CuaCoordinateOptions, CuaElementId, CuaHoverResult } from '@deepseek-ai/dsh-cua/api' import { CuaConfigFields } from '@deepseek-ai/dsh-cua/config' import { createPermissionParameters } from './permission-guide.ts' import { nativeHostArchitecture, prepareNativeExecutable, type SourceSigning } from './native-build.ts' import { createSharedNativeChannel } from './shared-native.ts' +import { mouseMovePreparation, MacRequest } from './mouse-control.ts' import { installHeldKeys } from './held-keys.ts' import { GUIDANCE } from './guidance.ts' export { GUIDANCE } from './guidance.ts' +declare module '@deepseek-ai/dsh-cua/api' { + interface CuaWindow { + /** + * Request the macOS system pointer or explicitly select accessibility hover for this move. + * @param target Current element id or screenshot-local point. + * @param options Coordinate mode and requested macOS pointer behavior. + * @returns Dispatch evidence and the selected hover route. + */ + mouseMove(target: CuaElementId | readonly [number, number], + options: CuaCoordinateOptions & { useSystemPointer: boolean }): Promise + } +} + const CAPABILITIES: CuaStartupCapabilities = { platform: 'macos', methods: [ @@ -65,7 +80,8 @@ export const Config = Schema.object({ */ export async function apply(ctx: Context, config: Config): Promise { const backend = { createChannel: createSharedNativeChannel, sendKeyEvent: installHeldKeys(ctx), - guidance: GUIDANCE, capabilities: CAPABILITIES, requireSystemPointerApproval: config.requireSystemPointerApproval, + guidance: GUIDANCE, capabilities: CAPABILITIES, requestSchema: MacRequest, + prepareMouseMove: mouseMovePreparation(ctx, config.requireSystemPointerApproval), permissionParameters: createPermissionParameters(config.permissionPollIntervalMs) } if (config.helperPath !== undefined) { if (!isAbsolute(config.helperPath)) throw new Error('dsh-cua helperPath must be absolute') diff --git a/packages/experimental/cua-macos/src/mouse-control-locales.ts b/packages/experimental/cua-macos/src/mouse-control-locales.ts new file mode 100644 index 0000000000..8d1f8352d6 --- /dev/null +++ b/packages/experimental/cua-macos/src/mouse-control-locales.ts @@ -0,0 +1,23 @@ +/** Scope and interruption shown before the user grants mouse control. */ +export const mouseControlPrompt = { + question: { + en: 'Allow the agent to move your mouse pointer?', + zh: '允许智能体移动你的鼠标指针吗？', + ja: 'エージェントがマウスポインタを動かすことを許可しますか？', + es: '¿Permitir que el agente mueva el puntero del ratón?', + fr: 'Autoriser l’agent à déplacer le pointeur de la souris ?', + de: 'Darf der Agent den Mauszeiger bewegen?', + ru: 'Разрешить агенту перемещать указатель мыши?', + pt: 'Permitir que o agente mova o ponteiro do mouse?', + }, + detail: { + en: 'Allowing moves your pointer to this target now. Later hovers can also move it. If you decline, this session will not ask again. The decision expires when you close the session or restart DSH.', + zh: '允许后，你的鼠标指针会立即移到目标位置，本会话后续悬停也会移动指针。如果拒绝，本会话不会再询问。关闭会话或重启 DSH 后，这个决定失效。', + ja: '許可するとポインタが今すぐ移動し、以後のホバーでも移動します。拒否すると、このセッションでは再度確認しません。セッションを閉じるか DSH を再起動すると、この選択は失効します。', + es: 'Si aceptas, el puntero se moverá ahora al destino y también podrá moverse al pasar sobre otros elementos. Si rechazas, no se volverá a preguntar. La decisión caduca al cerrar la sesión o reiniciar DSH.', + fr: 'Si vous acceptez, le pointeur se déplace immédiatement sur la cible, puis lors des survols suivants. Si vous refusez, la question ne sera plus posée. Ce choix expire à la fermeture de la session ou au redémarrage de DSH.', + de: 'Bei Zustimmung bewegt sich der Zeiger sofort zum Ziel und auch bei späteren Mausbewegungen. Bei Ablehnung wird nicht erneut gefragt. Die Entscheidung gilt, bis Sie die Sitzung schließen oder DSH neu starten.', + ru: 'При разрешении указатель сразу переместится на цель и сможет перемещаться при следующих наведениях. При отказе вопрос не повторится. Решение действует до закрытия сеанса или перезапуска DSH.', + pt: 'Se permitir, o ponteiro irá para o alvo agora e poderá se mover nas próximas ações. Se recusar, a pergunta não será repetida. A decisão expira ao encerrar a sessão ou reiniciar o DSH.', + }, +} diff --git a/packages/experimental/cua-macos/src/mouse-control.ts b/packages/experimental/cua-macos/src/mouse-control.ts new file mode 100644 index 0000000000..1aa75186b0 --- /dev/null +++ b/packages/experimental/cua-macos/src/mouse-control.ts @@ -0,0 +1,51 @@ +/** macOS system-pointer options and session approval for physical hover. */ +import type { Context } from '@deepseek-ai/cordis' +import { CuaSessionPermissions, type CuaBackend } from '@deepseek-ai/dsh-cua' +import { NativeError } from '@deepseek-ai/dsh-cua/native-client' +import { MouseMoveTargets, Request } from '@deepseek-ai/dsh-cua/protocol' +import { UserQuestionError } from '@deepseek-ai/dsh-user-questions' +import { z } from 'zod' +import { mouseControlPrompt } from './mouse-control-locales.ts' + +const pointerOptions = { useSystemPointer: z.boolean().optional().describe('Request use of the system pointer. The Host checks permission before dispatch.') } +const mouseMove = z.strictObject({ method: z.literal('mouseMove'), params: z.union(MouseMoveTargets.map(target => target.extend(pointerOptions))) }) + +/** macOS advertises and validates its additional option for element and coordinate moves. */ +export const MacRequest = z.discriminatedUnion('method', Request.options.map(op => op.shape.method.value === 'mouseMove' ? mouseMove : op)) + +/** + * Select the native route on the provider's parsed request, before coordinates are rechecked and dispatched. + * @param ctx Provider context owning this set of Session decisions. + * @param requireApproval Whether physical moves need the user's approval. + * @returns Preparation for mouseMove with an optional user interaction. + */ +export function mouseMovePreparation(ctx: Context, requireApproval: boolean): NonNullable { + const permissions = new CuaSessionPermissions(ctx) + const permission = 'system-pointer' + return (agent, params) => { + if (params.useSystemPointer !== true) { + params.useSystemPointer = params.useSystemPointer === false ? false : permissions.allowed(agent, permission) + return undefined + } + if (agent.session.header.origin === 'subagent') { + throw new NativeError('MOUSE_CONTROL_UNAVAILABLE', "Delegated tasks cannot move the user's pointer. Use AX hover, click, or performAction. If the task requires a physical hover, report that step in your final result for the main task.") + } + if (!requireApproval) return undefined + params.useSystemPointer = false + return async (signal) => { + try { + if (!await permissions.request(agent, permission, mouseControlPrompt, signal)) { + throw new NativeError('MOUSE_CONTROL_DENIED', 'The user declined system-pointer movement for this session. No input was sent. Use pointer-free actions.') + } + } catch (error) { + if (error instanceof UserQuestionError) { + if (error.code === 'APPROVAL_PENDING') throw new NativeError('MOUSE_CONTROL_PENDING', error.message, { cause: error }) + if (error.code === 'NO_PROVIDER') throw new NativeError('MOUSE_APPROVAL_UNAVAILABLE', error.message, { cause: error }) + if (error.code === 'ASK_CANCELLED') throw new NativeError('MOUSE_APPROVAL_CANCELLED', 'The mouse control request was dismissed. Continue without mouse control unless the user asks to reconsider', { cause: error }) + } + throw error + } + params.useSystemPointer = true + } + } +} diff --git a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts index 36c6dd94c9..76d3bbff12 100644 --- a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts +++ b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts @@ -4,6 +4,7 @@ import { Ajv2020 } from 'ajv/dist/2020.js' import { describe, expect, it } from 'vitest' import { renderNativeProtocol } from '../native/generate-protocol.ts' import { Request } from '@deepseek-ai/dsh-cua/protocol' +import { MacRequest } from '../src/mouse-control.ts' describe('native protocol generation', () => { it('keeps native JSON Schema and Swift admission tables current', async () => { @@ -28,6 +29,15 @@ describe('native protocol generation', () => { expect(validate({ id: 'shortcut', ...request })).toBe(false) }) + it('admits the pointer option only in the macOS model schema and only as a boolean', () => { + const target = { appHandle: 'app', windowId: 1, x: 0, y: 0 } + const request = { method: 'mouseMove', params: { ...target, useSystemPointer: true } } + expect(Request.safeParse(request).success).toBe(false) + expect(MacRequest.safeParse(request).success).toBe(true) + expect(MacRequest.safeParse({ ...request, params: { ...target, useSystemPointer: 'true' } }).success).toBe(false) + expect(MacRequest.safeParse({ ...request, params: { ...target, arbitraryField: true } }).success).toBe(false) + }) + it.each([ { method: 'getState', params: { appHandle: 'app', maxElements: 1 } }, { method: 'drag', params: { appHandle: 'app', windowId: 1, fromX: 1, fromY: 1, toX: 2, toY: 2, steps: 1 } }, diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index ea69351f0d..9b8cf11595 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -17,7 +17,7 @@ import * as StorageDomain from '@deepseek-ai/dsh-storage-domain' import Hmr from '@deepseek-ai/dsh-hmr' import { mountAgentLoopTestDependencies, mountAgentLoopTestHarness } from '@deepseek-ai/dsh-agent-loop-testkit' import ComputerUseRegistry from '@deepseek-ai/dsh-computer-use' -import { CuaConfig, installCua } from '@deepseek-ai/dsh-cua' +import { CuaConfig, CuaSessionPermissions, installCua } from '@deepseek-ai/dsh-cua' import { createConfigProjector } from '@deepseek-ai/dsh-app-boot/src/config-schema/projector.ts' import LocalAttachmentStore from '@deepseek-ai/dsh-attachment-local' import FileSystem from '@deepseek-ai/dsh-fs-local' @@ -981,7 +981,7 @@ async function hoverFixture(requireApproval = true) { await f.ctx.plugin(NativeProvider, { ...config, requireSystemPointerApproval: false }) } const questions = await f.ctx.plugin(UserQuestions) - const questionId = 'cua-mouse-control' + const questionId = 'cua-session-permission:system-pointer' const approved = { answers: [{ id: questionId, selected: ['Allow for this session'] }] } const ask = vi.fn(async () => approved) f.ctx.on('user-questions/request', async ({ questions }) => questions[0]!.id === questionId ? ask() @@ -1031,7 +1031,7 @@ it('keeps refusals and grants separate and forgets a closed session on reopen', const { agent } = await f.ctx.agents.create({ sessionId: SessionId('other-mouse-session') }) expect(JSON.stringify((await f.call(agent, 'mouseMove', physicalHover)).content)).toContain('APP_ACCESS_REQUIRED') await f.select(agent) - f.ask.mockResolvedValueOnce({ answers: [{ id: 'cua-mouse-control', selected: ['Deny'] }] }) + f.ask.mockResolvedValueOnce({ answers: [{ id: 'cua-session-permission:system-pointer', selected: ['Deny'] }] }) native.request.mockClear() for (let i = 0; i < 2; i++) expect(JSON.stringify((await f.call(agent, 'mouseMove', physicalHover)).content)).toContain('MOUSE_CONTROL_DENIED') expect(native.request).not.toHaveBeenCalled() @@ -1129,8 +1129,29 @@ it('keeps provider-specific pointer flags and questions off providers without mo }) } }) await selectApplication(f) - expect(JSON.stringify((await f.execute('mouseMove', physicalHover)).content)).toContain('UNSUPPORTED_OPERATION') + expect(JSON.stringify((await f.execute('mouseMove', physicalHover)).content)).toContain('useSystemPointer') const target = { appHandle: 'app-one', snapshotId: 'snapshot', elementId: '0' } expect((await f.execute('mouseMove', target)).isError).toBe(false) expect(native.request).toHaveBeenLastCalledWith('mouseMove', target, expect.any(AbortSignal), expect.any(String)) }) + +it('separates a provider’s permission decisions and presents its question for each capability', async () => { + const f = await setup() + await f.ctx.plugin(UserQuestions) + const permissions = new CuaSessionPermissions(f.ctx) + const signal = new AbortController().signal + const ask = vi.spyOn(f.ctx.userQuestions, 'ask').mockImplementation(async ({ questions }) => { + const q = questions[0]! + return { answers: [{ id: q.id, selected: [q.question === 'Read status?' ? 'Allow for this session' : 'Deny'] }] } + }) + const read = { question: { en: 'Read status?' }, detail: { en: 'Only for this session' } } + const write = { question: { en: 'Change status?' }, detail: { en: 'Sets the current status' } } + const agent = f.handle.agent + await expect(permissions.request(agent, 'read-status', read, signal)).resolves.toBe(true) + await expect(permissions.request(agent, 'write-status', write, signal)).resolves.toBe(false) + expect(permissions.allowed(agent, 'read-status')).toBe(true) + expect(permissions.allowed(agent, 'write-status')).toBe(false) + await expect(permissions.request(agent, 'write-status', write, signal)).resolves.toBe(false) + await expect(permissions.request(agent, 'read-status', read, signal)).resolves.toBe(true) + expect(ask.mock.calls.map(([request]) => request.questions[0]!.detail)).toEqual([read.detail.en, write.detail.en]) +}) diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index 62318feee3..1cc1591c92 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -6,8 +6,8 @@ en: ed2539c6d12f814a zh: 55a6c9d7f0d21fb7 /deepseek-ai-dsh-cua: - en: 00f8b591b74dfba6 - zh: e928ab13be2eebfa + en: e419f97a55d5c315 + zh: ebe3dce308d9cd97 /deepseek-ai-dsh-cua/summary: en: 2ee2024dfc558d86 zh: c3b5661be12923c4 @@ -15,8 +15,8 @@ en: 5db4d4fea25493b9 zh: 65ab64fc24eb0d09 /deepseek-ai-dsh-cua/use-this-package: - en: 5e95c138cb107385 - zh: b126c56fb7d35538 + en: ba1f8e6ae51f69d7 + zh: 8f4fde9857bae319 /deepseek-ai-dsh-cua/use-this-package/permission-controls: en: d92d5948a7941e8d zh: d43491ae6521b87e diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index c3d2ce62c8..83e36c76f8 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -9,6 +9,8 @@ English | [中文](README.zh.md) Providers install the app/window object API with `installCua`; callers connect through `connectCua`. The `cua` API discovers applications, and `CuaApp` objects activate the selected application and enumerate its windows. `CuaApp` and `CuaWindow` both extend `CuaSurface`; explicit window objects own screenshots and pointer input. The macOS provider supplies native execution for this interface. +`CuaSessionPermissions.request(agent, permission, prompt, signal)` asks for one named permission using the provider’s localized question and detail. A live Session caches each permission separately. Providers declare additional tool fields with their request schema and can prepare a move before dispatch. + ## Summary Inspect and operate applications through retained objects: select an application, read its accessibility tree, capture explicit windows, and dispatch authorized input. Platform providers supply native execution; this library owns the object API, request validation, application consent, and image admission. The external PTC runtime owns variable persistence. @@ -41,7 +43,7 @@ Observation methods emit and return their result. Use `void await app.getAccessi Application consent precedes launch, observation, capture, and input. Session access is stored by conversation and application identity and survives DSH restart and Computer Use disablement. Other conversations require their own consent. Saved grants require a platform-provided reusable application identity. Refusal and revocation close affected Agent connections, so their old objects require fresh selection. No input is replayed after cancellation, timeout, or reconnect. -On supported providers the main task can request a physical move with `win.mouseMove(target, { useSystemPointer: true })`. Actions return a `dispatched` boolean; `mouseMove` also reports `hoverRoute` and `systemPointerMoved`, and `show()` adds `windowId` and `foregroundChanged`. Read fresh state to verify the result. Screenshots and pointer actions are available only on explicit window objects. Coordinate actions default to pixels in the latest returned screenshot; `relative: true` accepts integers from 0 to 999 after a successful window observation. Values are divided by 1000 and scaled to the window dimensions, then clamped inside the window. The Host resolves attachment normalization and the selected model route’s image projection before returning the screenshot, then converts input to the provider’s native window coordinates. Resizing invalidates the screenshot mapping; `SCREENSHOT_REQUIRED` requires a new screenshot for absolute coordinates. Resizing rejects old coordinates with `WINDOW_CHANGED`; movement alone preserves window-local coordinates. +Actions return a `dispatched` boolean; `mouseMove` also reports `hoverRoute` and `systemPointerMoved`, and `show()` adds `windowId` and `foregroundChanged`. Read fresh state to verify the result. Screenshots and pointer actions are available only on explicit window objects. Coordinate actions default to pixels in the latest returned screenshot; `relative: true` accepts integers from 0 to 999 after a successful window observation. Values are divided by 1000 and scaled to the window dimensions, then clamped inside the window. The Host resolves attachment normalization and the selected model route’s image projection before returning the screenshot, then converts input to the provider’s native window coordinates. Resizing invalidates the screenshot mapping; `SCREENSHOT_REQUIRED` requires a new screenshot for absolute coordinates. Resizing rejects old coordinates with `WINDOW_CHANGED`; movement alone preserves window-local coordinates. `click(target, { holdMs: 800 })` holds the mouse button for 800 milliseconds before release. `holdMs` applies only to `click`; it accepts nonnegative integers and applies to each press of a double-click; zero releases without an additional hold. The provider config `maxClickHoldMs` limits explicit holds and defaults to 5000 ms. Explicit holds use pointer input; omission preserves ordinary click behavior. The operation deadline includes the hold, and cancellation or timeout releases the button. Received input cannot be undone; forced helper termination cannot guarantee release. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index bde2ccd1a3..53b30f0f82 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -9,6 +9,8 @@ kind: "package-reference" 各提供方通过 `installCua` 安装应用/窗口对象 API；调用方通过 `connectCua` 连接。`cua` API 负责发现应用，`CuaApp` 对象负责激活选定的应用并枚举其窗口。`CuaApp` 与 `CuaWindow` 都扩展 `CuaSurface`；显式窗口对象负责截图和指针输入。macOS 提供方为此接口提供原生执行。 +提供方通过 `CuaSessionPermissions.request(agent, permission, prompt, signal)` 申请指定权限，并传入本地化的问题和说明。每个会话分别记录各项权限的决定。提供方通过请求 schema 声明额外字段，可在派发前处理悬停请求。 + ## 概述 通过保留对象检查并操作应用：选择应用、读取辅助功能树、截取显式窗口并派发已授权输入。平台提供方负责原生执行；本库负责对象 API、请求校验、应用授权和图像接纳。变量持久性由外部 PTC 运行时负责。 @@ -41,7 +43,7 @@ kind: "package-reference" 应用授权发生在启动、观察、截图和输入之前。会话许可按对话和应用身份持久化，重启 DSH 或关闭 Computer Use 后仍可复用，其他对话需要单独授权。长期许可要求平台提供可复用的应用身份。拒绝或撤销会关闭受影响的 Agent 连接，因此旧对象必须重新选择目标。取消、超时或重新连接后均不重放输入。 -在支持系统鼠标的提供方上，主任务可通过 `win.mouseMove(target, { useSystemPointer: true })` 请求真实移动。动作返回 `dispatched` 布尔值；`mouseMove` 还会返回 `hoverRoute` 和 `systemPointerMoved`，`show()` 还会返回 `windowId` 和 `foregroundChanged`。请读取新状态以验证结果。截图和指针操作只能通过显式窗口对象使用。坐标操作默认使用最近返回截图中的像素；成功观察窗口后，传入 `relative: true` 可使用 0～999 的整数坐标。数值除以 1000 后按窗口尺寸缩放，并限制在窗口内。Host 在返回截图前解析附件归一化和当前模型的图片缩放规则，再将输入换算为提供方的原生窗口坐标。窗口尺寸变化会使截图映射失效，绝对坐标遇到 `SCREENSHOT_REQUIRED` 时需要重新截图。窗口缩放后以 `WINDOW_CHANGED` 拒绝旧坐标；单纯移动窗口不改变窗口内坐标。 +动作返回 `dispatched` 布尔值；`mouseMove` 还会返回 `hoverRoute` 和 `systemPointerMoved`，`show()` 还会返回 `windowId` 和 `foregroundChanged`。请读取新状态以验证结果。截图和指针操作只能通过显式窗口对象使用。坐标操作默认使用最近返回截图中的像素；成功观察窗口后，传入 `relative: true` 可使用 0～999 的整数坐标。数值除以 1000 后按窗口尺寸缩放，并限制在窗口内。Host 在返回截图前解析附件归一化和当前模型的图片缩放规则，再将输入换算为提供方的原生窗口坐标。窗口尺寸变化会使截图映射失效，绝对坐标遇到 `SCREENSHOT_REQUIRED` 时需要重新截图。窗口缩放后以 `WINDOW_CHANGED` 拒绝旧坐标；单纯移动窗口不改变窗口内坐标。 `click(target, { holdMs: 800 })` 按住鼠标按钮 800 毫秒后松开。`holdMs` 仅适用于 `click`；它接受非负整数，双击时应用于每次按下；0 表示按下后立即松开，不额外等待。提供方配置 `maxClickHoldMs` 限制显式按住时长，默认 5000 毫秒。指定时长后使用鼠标事件；省略时保留普通点击行为。按住时间计入操作超时，取消或超时会松开按钮。已接收的输入无法撤销；辅助进程被强制终止时无法保证松开。 diff --git a/packages/experimental/cua/src/api.ts b/packages/experimental/cua/src/api.ts index a33f4ca9a7..5dee39030f 100644 --- a/packages/experimental/cua/src/api.ts +++ b/packages/experimental/cua/src/api.ts @@ -2,7 +2,7 @@ import { Buffer } from 'node:buffer' import type { z } from 'zod' import type { JsonValue } from '@deepseek-ai/dsh-util-values' -import { ActionNamesResult, ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from './protocol.ts' +import { ActionNamesResult, ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, MouseMoveTargets, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from './protocol.ts' import type { CuaApi, CuaApp, CuaAppHandle, CuaCapabilities, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaOperation, CuaScreenshot, CuaSurface, CuaWindow } from './types.ts' export type * from './types.ts' @@ -24,9 +24,15 @@ function requestIssueMessage(issues: readonly z.core.$ZodIssue[], parent: readon } function validatedCall(dispatch: CuaDispatch) { - return (method: CuaOperation['method'], params: Record) => { + return (method: CuaOperation['method'], params: Record, options?: object) => { const request = Request.safeParse({ method, params }) if (!request.success) throw new Error(`${method} ${requestIssueMessage(request.error.issues)}`) + if (options) { + // The facade owns the target and coordinates. The Host validates additional options against its provider schema. + const extra = Object.entries(options).filter(([key, value]) => + value !== undefined && !MouseMoveTargets.some(target => Object.hasOwn(target.shape, key))) + Object.assign(request.data.params, Object.fromEntries(extra)) + } return dispatch(request.data) } } @@ -109,8 +115,8 @@ function createCuaObjects(call: ReturnType, sink: CuaObser requireAccessibility: () => void, requireKeyboard: () => void, ) { - const scopedCall = (method: CuaOperation['method'], params: Record) => - call(method, paramsFor(params)) + const scopedCall = (method: CuaOperation['method'], params: Record, options?: object) => + call(method, paramsFor(params), options) const action = async (method: CuaOperation['method'], params: Record) => ActionResult.parse(await scopedCall(method, params)) const elementParams = (): Record => ({ snapshotId: currentSnapshot(target.appHandle) }) @@ -200,15 +206,12 @@ function createCuaObjects(call: ReturnType, sink: CuaObser return screenshot }, async mouseMove(targetPoint, options) { - const movement = { - ...options?.relative === undefined ? {} : { relative: options.relative }, - ...options?.useSystemPointer === undefined ? {} : { useSystemPointer: options.useSystemPointer }, - } + const relative = options?.relative === undefined ? {} : { relative: options.relative } if (typeof targetPoint === 'string') { - return HoverResult.parse(await base.call('mouseMove', { ...base.elementParams(), elementId: targetPoint, ...movement })) + return HoverResult.parse(await base.call('mouseMove', { ...base.elementParams(), elementId: targetPoint, ...relative }, options)) } const [x, y] = targetPoint - return HoverResult.parse(await base.call('mouseMove', { ...coordinateParams(appHandle, windowId), x, y, ...movement })) + return HoverResult.parse(await base.call('mouseMove', { ...coordinateParams(appHandle, windowId), x, y, ...relative }, options)) }, async click(targetPoint, options) { const mouseButton = options?.mouseButton ?? 'left' diff --git a/packages/experimental/cua/src/client/ApplicationApproval.tsx b/packages/experimental/cua/src/client/ApplicationApproval.tsx index f762d5d624..c55f3ead50 100644 --- a/packages/experimental/cua/src/client/ApplicationApproval.tsx +++ b/packages/experimental/cua/src/client/ApplicationApproval.tsx @@ -5,14 +5,14 @@ import type {} from '@deepseek-ai/dsh-client-ui-user-questions/client' import type {} from '@deepseek-ai/dsh-client-ui-approval/client' /** - * Select the provider’s application and mouse-control questions; other interactions keep their renderer. + * Select application access and provider Session permission questions; other interactions keep their renderer. * @param pending The session's currently elected interaction. * @returns Its Computer Use decision, or null for another interaction. */ export function selectApplicationApproval(pending: ComposerChainProps['pendingInteraction']) { if (pending?.kind !== 'question' || pending.questions.length !== 1) return null const item = pending.questions[0] as (typeof pending.questions)[number] - if ((item.id !== 'cua-app-access' && item.id !== 'cua-mouse-control') || item.intent?.kind !== 'approval') return null + if ((item.id !== 'cua-app-access' && !item.id.startsWith('cua-session-permission:')) || item.intent?.kind !== 'approval') return null return { pending, item, intent: item.intent } } diff --git a/packages/experimental/cua/src/guidance.ts b/packages/experimental/cua/src/guidance.ts index d11a575e37..d8d7d46660 100644 --- a/packages/experimental/cua/src/guidance.ts +++ b/packages/experimental/cua/src/guidance.ts @@ -32,7 +32,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. +- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index 24f216fb78..568e7b83a8 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -19,7 +19,6 @@ import type { CuaApplicationGrant, CuaApplicationId, CuaSystemPermission } from import type { CuaAppHandle, CuaOperation, CuaStartupCapabilities } from './types.ts' import { ApplicationIcon, AppResult, NativeScreenshotResult, Request, ScreenshotResult, StateResult, WindowsResult } from './protocol.ts' import { CuaConfigFields } from './config.ts' -import { MouseControl } from './mouse-control.ts' import { CuaCoordinates } from './coordinates.ts' import { createScope, type Scope } from '@deepseek-ai/dsh-scope' import type {} from '@deepseek-ai/dsh-agent-preset-registry' @@ -30,6 +29,7 @@ import type {} from '@deepseek-ai/dsh-computer-use' import type {} from '@deepseek-ai/dsh-ptc-runtime' import type {} from '@deepseek-ai/cordis-plugin-loader' +export { CuaSessionPermissions } from './session-permissions.ts' export { CuaAccess } from './access.ts' export type { CuaStartupCapabilities } from './types.ts' @@ -127,8 +127,10 @@ export interface CuaBackend { createChannel: NativeChannelFactory guidance: string capabilities: CuaStartupCapabilities - /** Whether explicit system-pointer moves require consent. Omit for providers without this route. */ - requireSystemPointerApproval?: boolean + /** Model request schema including this provider's operation options. Defaults to the shared schema. */ + requestSchema?: z.ZodType + /** Resolve provider options on the parsed copy. An optional interactive step runs before authorized dispatch. */ + prepareMouseMove?: (agent: Agent, params: Record) => ((signal: AbortSignal) => Promise) | undefined /** Dispatch a directional key through the provider's execution-scoped input owner, after application authorization. */ sendKeyEvent?: (client: NativeClient, params: Record, execution: Readonly) => Promise @@ -162,8 +164,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) permissionParameters: backend.permissionParameters, store: new CuaPermissionStore(ctx, config.allowedApps.map(grant => ({ ...grant, id: grant.id as CuaApplicationId })), config.timeoutMs), }) - const mouseControl = backend.requireSystemPointerApproval === undefined - ? undefined : new MouseControl(ctx, backend.requireSystemPointerApproval) + const requestSchema = backend.requestSchema ?? Request const clients = new Map() const lifetime = new AbortController() const pending = new Map, Agent>() @@ -212,7 +213,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) for (const result of [...await permissions, ...results, ...releases]) if (result.status === 'rejected') errors.push(result.reason) if (errors.length > 0) throw new AggregateError(errors, 'dsh-cua helper cleanup failed') } - const callNative = async (request: CuaOperation, execution: Readonly, pointerRoute?: boolean) => { + const callNative = async (request: CuaOperation, execution: Readonly) => { const { method, params: parsedParams } = request const params = execution.coordinates.parameters(request) const agent = execution.agent @@ -229,10 +230,6 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) ? { ...params, maxElements: config.maxElements, maxDepth: config.maxDepth } : method === 'drag' ? { ...params, durationMs: config.dragDurationMs, steps: config.dragSteps } : params - if (method === 'mouseMove') { - delete nativeParams.useSystemPointer - if (mouseControl) nativeParams.useSystemPointer = pointerRoute ?? mouseControl.allowed(agent) - } const requestNative = () => client.request(method, nativeParams, signal, execution.callId) if (method === 'screenshot') { const { attachments, target } = await screenshotStore(ctx, execution) @@ -308,10 +305,10 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) const tool: ToolDefinition = { name: 'cua', description: 'Inspect desktop access, select an application and window, read AX state, capture screenshots or dispatch input. Use appHandle from getApp; verify actions by reading fresh state.', - parameters: { ...z.record(z.string(), z.json()).parse(z.toJSONSchema(Request)), type: 'object' }, + parameters: { ...z.record(z.string(), z.json()).parse(z.toJSONSchema(requestSchema)), type: 'object' }, output: { schema: {}, render: renderResult }, async execute(args, execution) { - const request = Request.parse(args) + const request = requestSchema.parse(args) const agent = executionAgent(execution) execution.signal.throwIfAborted() lifetime.signal.throwIfAborted() @@ -344,23 +341,11 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) owner.coordinates = new CuaCoordinates() } const nativeClient = client - let pointerRoute: boolean | undefined - let prepare: ((signal: AbortSignal) => Promise) | undefined - if (request.method === 'mouseMove') { - if (request.params.useSystemPointer === true) { - prepare = async (signal) => { - if (!mouseControl) throw new NativeError('UNSUPPORTED_OPERATION', 'This provider does not support system mouse control') - owner.coordinates.parameters(request) - if (!await mouseControl.request(agent, signal)) { - throw new NativeError('MOUSE_CONTROL_DENIED', 'The user declined system-pointer movement for this session. No input was sent. Use pointer-free actions.') - } - pointerRoute = true - } - } else pointerRoute = request.params.useSystemPointer - } + const prepare = request.method === 'mouseMove' ? backend.prepareMouseMove?.(agent, request.params) : undefined + if (prepare) owner.coordinates.parameters(request) const dispatch = async (signal: AbortSignal) => { const scoped = { ...execution, agent, signal, nativeClient, coordinates: owner.coordinates } - const result = await callNative(request, scoped, pointerRoute) + const result = await callNative(request, scoped) signal.throwIfAborted() return result } diff --git a/packages/experimental/cua/src/locales.ts b/packages/experimental/cua/src/locales.ts index fbd9b7ac43..499ce34339 100644 --- a/packages/experimental/cua/src/locales.ts +++ b/packages/experimental/cua/src/locales.ts @@ -103,28 +103,3 @@ export const launchAccessQuestion = { ru: 'Разрешить запуск? Доступ к окнам запрашивается отдельно.', pt: 'Permitir iniciar esta entrada? O acesso às janelas é solicitado separadamente.', } - -/** Scope and interruption shown before the user grants mouse control. */ -export const mouseControlDisplay = { - question: { - en: 'Allow the agent to move your mouse pointer?', - zh: '允许智能体移动你的鼠标指针吗？', - ja: 'エージェントがマウスポインタを動かすことを許可しますか？', - es: '¿Permitir que el agente mueva el puntero del ratón?', - fr: 'Autoriser l’agent à déplacer le pointeur de la souris ?', - de: 'Darf der Agent den Mauszeiger bewegen?', - ru: 'Разрешить агенту перемещать указатель мыши?', - pt: 'Permitir que o agente mova o ponteiro do mouse?', - }, - detail: { - en: 'Allowing moves your pointer to this target now. Later hovers can also move it. If you decline, this session will not ask again. The decision expires when you close the session or restart DSH.', - zh: '允许后，你的鼠标指针会立即移到目标位置，本会话后续悬停也会移动指针。如果拒绝，本会话不会再询问。关闭会话或重启 DSH 后，这个决定失效。', - ja: '許可するとポインタが今すぐ移動し、以後のホバーでも移動します。拒否すると、このセッションでは再度確認しません。セッションを閉じるか DSH を再起動すると、この選択は失効します。', - es: 'Si aceptas, el puntero se moverá ahora al destino y también podrá moverse al pasar sobre otros elementos. Si rechazas, no se volverá a preguntar. La decisión caduca al cerrar la sesión o reiniciar DSH.', - fr: 'Si vous acceptez, le pointeur se déplace immédiatement sur la cible, puis lors des survols suivants. Si vous refusez, la question ne sera plus posée. Ce choix expire à la fermeture de la session ou au redémarrage de DSH.', - de: 'Bei Zustimmung bewegt sich der Zeiger sofort zum Ziel und auch bei späteren Mausbewegungen. Bei Ablehnung wird nicht erneut gefragt. Die Entscheidung gilt, bis Sie die Sitzung schließen oder DSH neu starten.', - ru: 'При разрешении указатель сразу переместится на цель и сможет перемещаться при следующих наведениях. При отказе вопрос не повторится. Решение действует до закрытия сеанса или перезапуска DSH.', - pt: 'Se permitir, o ponteiro irá para o alvo agora e poderá se mover nas próximas ações. Se recusar, a pergunta não será repetida. A decisão expira ao encerrar a sessão ou reiniciar o DSH.', - }, - approve: applicationAccessDisplay.approve, -} diff --git a/packages/experimental/cua/src/mouse-control.ts b/packages/experimental/cua/src/mouse-control.ts deleted file mode 100644 index 8482554943..0000000000 --- a/packages/experimental/cua/src/mouse-control.ts +++ /dev/null @@ -1,63 +0,0 @@ -/** Human decisions about the system pointer live in the provider process, keyed by Session. */ -import type { Context } from '@deepseek-ai/cordis' -import type { Agent } from '@deepseek-ai/dsh-agent' -import { UserQuestionError } from '@deepseek-ai/dsh-user-questions' -import { mouseControlDisplay } from './locales.ts' -import { NativeError } from './native-client.ts' - -/** Mouse control applies only to mouseMove and never grants access to an application. */ -export class MouseControl { - private readonly decisions = new WeakMap() - - constructor(private readonly ctx: Context, private readonly requireApproval: boolean) {} - - /** - * Select the native route without prompting the user. - * @param agent Calling Agent. - * @returns Whether this Session explicitly approved moving the system pointer. - */ - allowed(agent: Agent): boolean { - return this.decisions.get(agent) === true - } - - /** - * Ask the main task for a Session decision. Simultaneous requests cannot open additional questions. - * @param agent Live Agent that owns the Session. - * @param signal Tool lifetime. An aborted tool cannot grant permission. - * @returns True only when the user chose the offered approval. - */ - async request(agent: Agent, signal: AbortSignal): Promise { - if (agent.session.header.origin === 'subagent') { - throw new NativeError('MOUSE_CONTROL_UNAVAILABLE', "Delegated tasks cannot move the user's pointer. Use AX hover, click, or performAction. If the task requires a physical hover, report that step in your final result for the main task.") - } - if (!this.requireApproval) return true - const decision = this.decisions.get(agent) - if (decision === 'asking') throw new NativeError('MOUSE_CONTROL_PENDING', 'Wait for the pending mouse control request in this session before continuing') - if (decision !== undefined) return decision - const questions = this.ctx.get('userQuestions') - if (!questions) throw new NativeError('MOUSE_APPROVAL_UNAVAILABLE', 'No human answerer is available for mouse control') - this.decisions.set(agent, 'asking') - const id = 'cua-mouse-control' - const approve = mouseControlDisplay.approve.en - const reject = 'Deny' - try { - const result = await questions.ask({ agent, signal, questions: [{ - id, question: mouseControlDisplay.question.en, detail: mouseControlDisplay.detail.en, - options: [{ label: approve }, { label: reject }], - intent: { kind: 'approval', approve, reject, display: mouseControlDisplay }, - }] }) - signal.throwIfAborted() - const [answer] = result.answers - const granted = result.answers.length === 1 && answer?.id === id && !answer.custom?.trim() - && answer.selected.length === 1 && answer.selected[0] === approve - this.decisions.set(agent, granted) - return granted - } catch (error) { - this.decisions.delete(agent) - if (error instanceof UserQuestionError && error.code === 'ASK_CANCELLED') { - throw new NativeError('MOUSE_APPROVAL_CANCELLED', 'The mouse control request was dismissed. Continue without mouse control unless the user asks to reconsider', { cause: error }) - } - throw error - } - } -} diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index d3f0d6a0b6..1fd9d6cd41 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -34,9 +34,12 @@ const imageBase64 = z.string().min(1).refine(value => Buffer.from(value, 'base64 message: 'Expected canonical base64 image data', }) -const mouseMove = { useSystemPointer: z.boolean().optional().describe('Request use of the system pointer. The Host checks permission before dispatch.') } +/** Shared hover target variants. Providers may add options without replacing a target or its coordinate mode. */ +export const MouseMoveTargets = [ + z.object(snapshot).strict(), z.object(absoluteCoordinate).strict(), z.object(relativeCoordinate).strict(), +] as const -/** Model-submittable operations, before the Host applies permission decisions and resource limits. */ +/** Model-submittable operation parameters; Host resource fields and private continuations are separate. */ export const Request = z.discriminatedUnion('method', [ operation('getCapabilities', {}), operation('listApps', {}), operation('findApps', { query: z.string().min(1, 'Expected a non-empty search string').regex(/\S/, 'Expected a non-empty search string') }), @@ -51,10 +54,7 @@ export const Request = z.discriminatedUnion('method', [ operation('listActions', snapshot), operation('performAction', { ...snapshot, action: z.string().min(1), argument: jsonValue.optional() }), operation('invoke', snapshot), - z.object({ method: z.literal('mouseMove'), params: z.union([ - z.object({ ...snapshot, ...mouseMove }).strict(), z.object({ ...absoluteCoordinate, ...mouseMove }).strict(), - z.object({ ...relativeCoordinate, ...mouseMove }).strict(), - ]) }).strict(), + z.object({ method: z.literal('mouseMove'), params: z.union(MouseMoveTargets) }).strict(), z.object({ method: z.literal('click'), params: z.union([ z.object({ ...snapshot, ...click }).strict(), z.object({ ...absoluteCoordinate, ...click }).strict(), diff --git a/packages/experimental/cua/src/session-permissions.ts b/packages/experimental/cua/src/session-permissions.ts new file mode 100644 index 0000000000..f34133cb1c --- /dev/null +++ b/packages/experimental/cua/src/session-permissions.ts @@ -0,0 +1,64 @@ +/** Provider-owned permissions remembered separately for each live Session and requested capability. */ +import type { Context } from '@deepseek-ai/cordis' +import type { Agent } from '@deepseek-ai/dsh-agent' +import { UserQuestionError } from '@deepseek-ai/dsh-user-questions' +import { applicationAccessDisplay } from './locales.ts' + +type Localized = { en: string; [locale: string]: string } + +/** A provider owns the lifetime and names of the permissions it requests. */ +export class CuaSessionPermissions { + private readonly decisions = new WeakMap>() + + constructor(private readonly ctx: Context) {} + + /** + * Read a permission without asking the user. + * @param agent Live Session owner. + * @param permission Provider-defined permission name. + * @returns Whether the user approved this permission in this Session. + */ + allowed(agent: Agent, permission: string): boolean { + return this.decisions.get(agent)?.get(permission) === true + } + + /** + * Remember the user's choice only for this permission in this live Session. + * @param agent Live Session owner. + * @param permission Provider-defined permission name shared by calls requesting the same access. + * @param prompt Localized question and scope displayed before the user decides. + * @param signal Calling operation lifetime. Aborting or dismissing the question leaves the permission undecided. + * @returns True only when the user selects session approval. + */ + async request(agent: Agent, permission: string, prompt: { question: Localized; detail: Localized }, + signal: AbortSignal): Promise { + signal.throwIfAborted() + const decisions = this.decisions.get(agent) ?? new Map() + const decision = decisions.get(permission) + if (decision === 'asking') throw new UserQuestionError('Wait for the outstanding permission request', 'APPROVAL_PENDING') + if (decision !== undefined) return decision + const questions = this.ctx.get('userQuestions') + if (!questions) throw new UserQuestionError('No human answerer is available for the permission request', 'NO_PROVIDER') + this.decisions.set(agent, decisions) + decisions.set(permission, 'asking') + const id = `cua-session-permission:${permission}` + const display = { ...prompt, approve: applicationAccessDisplay.approve } + const approve = display.approve.en + const reject = 'Deny' + try { + const result = await questions.ask({ agent, signal, questions: [{ + id, question: prompt.question.en, detail: prompt.detail.en, options: [{ label: approve }, { label: reject }], + intent: { kind: 'approval', approve, reject, display }, + }] }) + signal.throwIfAborted() + const [answer] = result.answers + const granted = result.answers.length === 1 && answer?.id === id && !answer.custom?.trim() + && answer.selected.length === 1 && answer.selected[0] === approve + decisions.set(permission, granted) + return granted + } catch (error) { + decisions.delete(permission) + throw error + } + } +} diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index 7c596c87d4..b84663f9e2 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -196,11 +196,10 @@ export interface CuaWindow extends CuaSurface { * Returning and subsequent observations do not restore the previous pointer position. * Other input and application changes can interrupt the interaction; observe before continuing. * @param target - current element id or screenshot-local [x,y]. - * @param options - Coordinate mode for points and optional system-pointer request. False explicitly selects pointer-free hover. + * @param options - coordinate mode; applies only to [x,y]. * @returns dispatch evidence and the selected route; observe the resulting UI after any UI delay. */ - mouseMove(target: CuaElementId | readonly [number, number], - options?: CuaCoordinateOptions & { useSystemPointer?: boolean }): Promise + mouseMove(target: CuaElementId | readonly [number, number], options?: CuaCoordinateOptions): Promise /** * Click a current element id or screenshot-local point. * Cancellation or timeout releases the held button; received input cannot be undone. The operation deadline includes holdMs. diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index 50eb722493..73dbf044cc 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -4,7 +4,7 @@ import { describe, expect, expectTypeOf, it, vi } from 'vitest' import { brandString } from '@deepseek-ai/dsh-brand' import type { JsonValue } from '@deepseek-ai/dsh-util-values' import { createCuaApi } from '../src/api.ts' -import type { CuaApp, CuaCapabilities, CuaElementId, CuaOperation, CuaScreenshot, CuaWindow } from '../src/types.ts' +import type { CuaApp, CuaCapabilities, CuaCoordinateOptions, CuaElementId, CuaOperation, CuaScreenshot, CuaWindow } from '../src/types.ts' /** Element ids are opaque strings; these fixtures use the macOS decimal form. */ const element = (id: string): CuaElementId => brandString(id) @@ -891,15 +891,24 @@ it('expands menu contents and details only for the requested observation', async expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'getState', params: { appHandle: selected.appHandle, includeMenus: false, details: false } }) }) -it('passes explicit system-pointer intent for both mouseMove target forms', async () => { +it('forwards provider options while retaining the selected target and coordinate mode', async () => { const f = fixture() const win = await (await f.api.getApp('Editor')).getWindow(23) - await win.mouseMove(element('0'), { useSystemPointer: true }) + const defaultCoordinates: CuaCoordinateOptions = {} + const elementOptions = { ...defaultCoordinates, trackingMode: 'precise', appHandle: 'another-app', windowId: 99, unused: undefined } + await win.mouseMove(element('0'), elementOptions) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'mouseMove', params: { - appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', elementId: '0', useSystemPointer: true, + appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', elementId: '0', trackingMode: 'precise', } }) - await win.mouseMove([250, 750], { relative: true, useSystemPointer: false }) + await win.getScreenshot({ emit: false }) + const pixelOptions = { relative: false, trackingMode: 'precise', x: 222 } + await win.mouseMove([30, 40], pixelOptions) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'mouseMove', params: { - appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 250, y: 750, relative: true, useSystemPointer: false, + appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: 23, x: 30, y: 40, relative: false, trackingMode: 'precise', + } }) + const relativeOptions = { relative: true, trackingMode: 'coarse' } + await win.mouseMove([250, 750], relativeOptions) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'mouseMove', params: { + appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: 23, x: 250, y: 750, relative: true, trackingMode: 'coarse', } }) }) diff --git a/packages/experimental/cua/tests/application-approval.client.spec.tsx b/packages/experimental/cua/tests/application-approval.client.spec.tsx index 13e79b8faa..48f4cadafd 100644 --- a/packages/experimental/cua/tests/application-approval.client.spec.tsx +++ b/packages/experimental/cua/tests/application-approval.client.spec.tsx @@ -91,7 +91,7 @@ it('shows the mouse interruption and expiry with only session approval and refus detail: { en: 'Until restart', zh: '重启后失效' }, approve: { en: 'Allow for this session', zh: '本会话允许' }, } - const pending = new PendingQuestion('s1' as SessionId, [{ id: 'cua-mouse-control', question: display.question.en, + const pending = new PendingQuestion('s1' as SessionId, [{ id: 'cua-session-permission:example', question: display.question.en, detail: display.detail.en, intent: { kind: 'approval', approve: display.approve.en, reject: 'Deny', display } }]) const factory = vi.fn((_name: string, _input: ApprovalCardInput) => null) render( Date: Sat, 7 Nov 2026 17:09:35 +0800 Subject: [PATCH 15/30] fix(cua): preserve the typed provider request variants --- packages/experimental/cua-macos/src/mouse-control.ts | 5 ++++- .../cua-macos/tests/provider-lifecycle.spec.ts | 4 +++- packages/experimental/cua/README.i18n.yaml | 8 ++++---- packages/experimental/cua/README.md | 4 ++-- packages/experimental/cua/README.zh.md | 4 ++-- packages/experimental/cua/src/index.ts | 2 +- 6 files changed, 16 insertions(+), 11 deletions(-) diff --git a/packages/experimental/cua-macos/src/mouse-control.ts b/packages/experimental/cua-macos/src/mouse-control.ts index 1aa75186b0..dd7957ac38 100644 --- a/packages/experimental/cua-macos/src/mouse-control.ts +++ b/packages/experimental/cua-macos/src/mouse-control.ts @@ -10,8 +10,11 @@ import { mouseControlPrompt } from './mouse-control-locales.ts' const pointerOptions = { useSystemPointer: z.boolean().optional().describe('Request use of the system pointer. The Host checks permission before dispatch.') } const mouseMove = z.strictObject({ method: z.literal('mouseMove'), params: z.union(MouseMoveTargets.map(target => target.extend(pointerOptions))) }) +const [first, ...rest] = Request.options +const macOperation = (op: typeof Request.options[number]) => op.shape.method.value === 'mouseMove' ? mouseMove : op + /** macOS advertises and validates its additional option for element and coordinate moves. */ -export const MacRequest = z.discriminatedUnion('method', Request.options.map(op => op.shape.method.value === 'mouseMove' ? mouseMove : op)) +export const MacRequest = z.discriminatedUnion('method', [macOperation(first), ...rest.map(macOperation)]) /** * Select the native route on the provider's parsed request, before coordinates are rechecked and dispatched. diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index 9b8cf11595..2ba282c1eb 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -1078,12 +1078,14 @@ it('drops dismissal, aborted answers and missing human answerers without allowin const f = await hoverFixture() f.ask.mockRejectedValueOnce(new UserQuestionError('dismissed', 'ASK_CANCELLED')) expect(JSON.stringify((await f.execute('mouseMove', physicalHover)).content)).toContain('MOUSE_APPROVAL_CANCELLED') + f.ask.mockRejectedValueOnce(new UserQuestionError('The answer channel disconnected', 'ASK_ABORTED')) + expect(JSON.stringify((await f.execute('mouseMove', physicalHover)).content)).toContain('The answer channel disconnected') const controller = new AbortController() f.ask.mockImplementationOnce(async () => { controller.abort(); return f.approved }) await Promise.allSettled([f.execute('mouseMove', physicalHover, controller.signal)]) expect(native.request).not.toHaveBeenCalled() expect((await f.execute('mouseMove', physicalHover)).isError).toBe(false) - expect(f.ask).toHaveBeenCalledTimes(3) + expect(f.ask).toHaveBeenCalledTimes(4) await f.provider.dispose() await f.ctx.plugin(NativeProvider, config) await f.select() diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index 1cc1591c92..d4110d36f7 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -6,8 +6,8 @@ en: ed2539c6d12f814a zh: 55a6c9d7f0d21fb7 /deepseek-ai-dsh-cua: - en: e419f97a55d5c315 - zh: ebe3dce308d9cd97 + en: 00f8b591b74dfba6 + zh: e928ab13be2eebfa /deepseek-ai-dsh-cua/summary: en: 2ee2024dfc558d86 zh: c3b5661be12923c4 @@ -21,8 +21,8 @@ en: d92d5948a7941e8d zh: d43491ae6521b87e /deepseek-ai-dsh-cua/understand-the-implementation: - en: 2ab74a1ff120459e - zh: 39b3ef198989f133 + en: cb26e420c2ea222f + zh: b6f17c9b71cea7bf /deepseek-ai-dsh-cua/further-exploration: en: c174fb5a00ea3681 zh: 039340042676881d diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index 83e36c76f8..1871b735a1 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -9,8 +9,6 @@ English | [中文](README.zh.md) Providers install the app/window object API with `installCua`; callers connect through `connectCua`. The `cua` API discovers applications, and `CuaApp` objects activate the selected application and enumerate its windows. `CuaApp` and `CuaWindow` both extend `CuaSurface`; explicit window objects own screenshots and pointer input. The macOS provider supplies native execution for this interface. -`CuaSessionPermissions.request(agent, permission, prompt, signal)` asks for one named permission using the provider’s localized question and detail. A live Session caches each permission separately. Providers declare additional tool fields with their request schema and can prepare a move before dispatch. - ## Summary Inspect and operate applications through retained objects: select an application, read its accessibility tree, capture explicit windows, and dispatch authorized input. Platform providers supply native execution; this library owns the object API, request validation, application consent, and image admission. The external PTC runtime owns variable persistence. @@ -70,6 +68,8 @@ The Host marker has no configurable fields; platform providers own native config ## Understand the implementation +`CuaSessionPermissions.request(agent, permission, prompt, signal)` asks for one named permission using the provider’s localized question and detail. A live Session caches each permission separately. Providers declare additional tool fields with their request schema and can prepare a move before dispatch. + Permission responses identify their platform and report only the state that platform can observe. The shared page renders the controls those observations support. Before platform identification, it shows neutral observation and input rows without system actions. Native providers use the shared build lock and compiler cleanup. Standalone executables are retained by digest. The macOS provider reads its signed application receipt after compilation completes under the same lock.
diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index 53b30f0f82..ae2aaef775 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -9,8 +9,6 @@ kind: "package-reference" 各提供方通过 `installCua` 安装应用/窗口对象 API；调用方通过 `connectCua` 连接。`cua` API 负责发现应用，`CuaApp` 对象负责激活选定的应用并枚举其窗口。`CuaApp` 与 `CuaWindow` 都扩展 `CuaSurface`；显式窗口对象负责截图和指针输入。macOS 提供方为此接口提供原生执行。 -提供方通过 `CuaSessionPermissions.request(agent, permission, prompt, signal)` 申请指定权限，并传入本地化的问题和说明。每个会话分别记录各项权限的决定。提供方通过请求 schema 声明额外字段，可在派发前处理悬停请求。 - ## 概述 通过保留对象检查并操作应用：选择应用、读取辅助功能树、截取显式窗口并派发已授权输入。平台提供方负责原生执行；本库负责对象 API、请求校验、应用授权和图像接纳。变量持久性由外部 PTC 运行时负责。 @@ -70,6 +68,8 @@ Computer Use 详情页的权限卡片、组件列表和来源信息使用 l3 边 ## 理解实现 +提供方通过 `CuaSessionPermissions.request(agent, permission, prompt, signal)` 申请指定权限，并传入本地化的问题和说明。每个会话分别记录各项权限的决定。提供方通过请求 schema 声明额外字段，可在派发前处理悬停请求。 + 权限响应标识所属平台，并且只报告该平台能够观察到的状态。共享页面显示这些观察结果支持的控件。平台尚未确认时显示通用观察和输入项，不提供系统操作。原生提供方共用编译锁和编译进程清理逻辑。独立可执行文件按内容摘要保存。macOS 提供方在锁内等待编译完成，再读取签名应用的构建回执。
diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index 568e7b83a8..71374183d9 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -130,7 +130,7 @@ export interface CuaBackend { /** Model request schema including this provider's operation options. Defaults to the shared schema. */ requestSchema?: z.ZodType /** Resolve provider options on the parsed copy. An optional interactive step runs before authorized dispatch. */ - prepareMouseMove?: (agent: Agent, params: Record) => ((signal: AbortSignal) => Promise) | undefined + prepareMouseMove?: (agent: Agent, params: Record) => ((signal: AbortSignal) => Promise) | undefined /** Dispatch a directional key through the provider's execution-scoped input owner, after application authorization. */ sendKeyEvent?: (client: NativeClient, params: Record, execution: Readonly) => Promise From e9e265b48fec6486ccb8ddf7673059960bf81cac Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 17:17:00 +0800 Subject: [PATCH 16/30] fix(cua): publish session permissions outside the plugin entry --- packages/experimental/cua-macos/src/mouse-control.ts | 3 ++- .../experimental/cua-macos/tests/provider-lifecycle.spec.ts | 3 ++- packages/experimental/cua/README.i18n.yaml | 4 ++-- packages/experimental/cua/README.md | 2 +- packages/experimental/cua/README.zh.md | 2 +- packages/experimental/cua/package.json | 5 +++++ packages/experimental/cua/src/index.ts | 1 - packages/experimental/cua/tests/fixtures/exports.mjs | 2 ++ packages/experimental/cua/tsdown.config.ts | 1 + scripts/check-workspace-constraints.ts | 2 +- tsconfig.base.json | 1 + 11 files changed, 18 insertions(+), 8 deletions(-) diff --git a/packages/experimental/cua-macos/src/mouse-control.ts b/packages/experimental/cua-macos/src/mouse-control.ts index dd7957ac38..6f8e7f8a94 100644 --- a/packages/experimental/cua-macos/src/mouse-control.ts +++ b/packages/experimental/cua-macos/src/mouse-control.ts @@ -1,6 +1,7 @@ /** macOS system-pointer options and session approval for physical hover. */ import type { Context } from '@deepseek-ai/cordis' -import { CuaSessionPermissions, type CuaBackend } from '@deepseek-ai/dsh-cua' +import type { CuaBackend } from '@deepseek-ai/dsh-cua' +import { CuaSessionPermissions } from '@deepseek-ai/dsh-cua/session-permissions' import { NativeError } from '@deepseek-ai/dsh-cua/native-client' import { MouseMoveTargets, Request } from '@deepseek-ai/dsh-cua/protocol' import { UserQuestionError } from '@deepseek-ai/dsh-user-questions' diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index 2ba282c1eb..a5d74e20e8 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -17,7 +17,8 @@ import * as StorageDomain from '@deepseek-ai/dsh-storage-domain' import Hmr from '@deepseek-ai/dsh-hmr' import { mountAgentLoopTestDependencies, mountAgentLoopTestHarness } from '@deepseek-ai/dsh-agent-loop-testkit' import ComputerUseRegistry from '@deepseek-ai/dsh-computer-use' -import { CuaConfig, CuaSessionPermissions, installCua } from '@deepseek-ai/dsh-cua' +import { CuaConfig, installCua } from '@deepseek-ai/dsh-cua' +import { CuaSessionPermissions } from '@deepseek-ai/dsh-cua/session-permissions' import { createConfigProjector } from '@deepseek-ai/dsh-app-boot/src/config-schema/projector.ts' import LocalAttachmentStore from '@deepseek-ai/dsh-attachment-local' import FileSystem from '@deepseek-ai/dsh-fs-local' diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index d4110d36f7..851909295c 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -21,8 +21,8 @@ en: d92d5948a7941e8d zh: d43491ae6521b87e /deepseek-ai-dsh-cua/understand-the-implementation: - en: cb26e420c2ea222f - zh: b6f17c9b71cea7bf + en: edb0b08ad9ee9a73 + zh: da868fc1061516e4 /deepseek-ai-dsh-cua/further-exploration: en: c174fb5a00ea3681 zh: 039340042676881d diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index 1871b735a1..5eea482387 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -68,7 +68,7 @@ The Host marker has no configurable fields; platform providers own native config ## Understand the implementation -`CuaSessionPermissions.request(agent, permission, prompt, signal)` asks for one named permission using the provider’s localized question and detail. A live Session caches each permission separately. Providers declare additional tool fields with their request schema and can prepare a move before dispatch. +Import `CuaSessionPermissions` from `@deepseek-ai/dsh-cua/session-permissions`. Its `request(agent, permission, prompt, signal)` method asks for one named permission using the provider’s localized question and detail. A live Session caches each permission separately. Providers declare additional tool fields with their request schema and can prepare a move before dispatch. Permission responses identify their platform and report only the state that platform can observe. The shared page renders the controls those observations support. Before platform identification, it shows neutral observation and input rows without system actions. Native providers use the shared build lock and compiler cleanup. Standalone executables are retained by digest. The macOS provider reads its signed application receipt after compilation completes under the same lock. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index ae2aaef775..2db56fd1b2 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -68,7 +68,7 @@ Computer Use 详情页的权限卡片、组件列表和来源信息使用 l3 边 ## 理解实现 -提供方通过 `CuaSessionPermissions.request(agent, permission, prompt, signal)` 申请指定权限，并传入本地化的问题和说明。每个会话分别记录各项权限的决定。提供方通过请求 schema 声明额外字段，可在派发前处理悬停请求。 +提供方从 `@deepseek-ai/dsh-cua/session-permissions` 导入 `CuaSessionPermissions`，通过 `request(agent, permission, prompt, signal)` 申请指定权限，并传入本地化的问题和说明。每个会话分别记录各项权限的决定。提供方通过请求 schema 声明额外字段，可在派发前处理悬停请求。 权限响应标识所属平台，并且只报告该平台能够观察到的状态。共享页面显示这些观察结果支持的控件。平台尚未确认时显示通用观察和输入项，不提供系统操作。原生提供方共用编译锁和编译进程清理逻辑。独立可执行文件按内容摘要保存。macOS 提供方在锁内等待编译完成，再读取签名应用的构建回执。 diff --git a/packages/experimental/cua/package.json b/packages/experimental/cua/package.json index 440f321935..5d5c6bc0fb 100644 --- a/packages/experimental/cua/package.json +++ b/packages/experimental/cua/package.json @@ -53,6 +53,10 @@ "types": "./lib/types/client/index.d.ts", "default": "./lib/client.js" }, + "./session-permissions": { + "types": "./lib/types/session-permissions.d.ts", + "default": "./lib/session-permissions.js" + }, "./config": { "types": "./lib/types/config.d.ts", "default": "./lib/config.js" @@ -78,6 +82,7 @@ "lib/api.js", "lib/bridge.js", "lib/native-client.js", + "lib/session-permissions.js", "lib/protocol.js", "lib/config.js", "lib/native-schema.js", diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index 71374183d9..731ae38157 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -29,7 +29,6 @@ import type {} from '@deepseek-ai/dsh-computer-use' import type {} from '@deepseek-ai/dsh-ptc-runtime' import type {} from '@deepseek-ai/cordis-plugin-loader' -export { CuaSessionPermissions } from './session-permissions.ts' export { CuaAccess } from './access.ts' export type { CuaStartupCapabilities } from './types.ts' diff --git a/packages/experimental/cua/tests/fixtures/exports.mjs b/packages/experimental/cua/tests/fixtures/exports.mjs index 19dbd3973e..91b55b76b8 100644 --- a/packages/experimental/cua/tests/fixtures/exports.mjs +++ b/packages/experimental/cua/tests/fixtures/exports.mjs @@ -12,6 +12,8 @@ const guard = registerHooks({ }) const { Context } = await import('@deepseek-ai/cordis') const host = await import('@deepseek-ai/dsh-cua') +const { CuaSessionPermissions } = await import('@deepseek-ai/dsh-cua/session-permissions') +assert.equal(typeof CuaSessionPermissions.prototype.request, 'function') const { createCuaApi } = await import('@deepseek-ai/dsh-cua/api') const { CuaConfigFields } = await import('@deepseek-ai/dsh-cua/config') assert.equal(typeof CuaConfigFields.connectTimeoutMs.default, 'function') diff --git a/packages/experimental/cua/tsdown.config.ts b/packages/experimental/cua/tsdown.config.ts index 29526fddd6..92c9eca9a8 100644 --- a/packages/experimental/cua/tsdown.config.ts +++ b/packages/experimental/cua/tsdown.config.ts @@ -3,6 +3,7 @@ import { clientBundle } from '../../client/tsdown.client.ts' const runtime: UserConfig[] = Object.entries({ 'remote-types': 'lib/types/remote-types.js', api: 'lib/types/api.js', bridge: 'lib/types/bridge.js', + 'session-permissions': 'lib/types/session-permissions.js', 'native-client': 'lib/types/native-client.js', protocol: 'lib/types/protocol.js', config: 'lib/types/config.js', 'native-schema': 'lib/types/native-schema.js', 'native-build': 'lib/types/native-build.js', guidance: 'lib/types/guidance.js', }).map(([name, path]) => ({ diff --git a/scripts/check-workspace-constraints.ts b/scripts/check-workspace-constraints.ts index f759eca7ec..8e75b08fbd 100644 --- a/scripts/check-workspace-constraints.ts +++ b/scripts/check-workspace-constraints.ts @@ -188,7 +188,7 @@ const packageFileExtras: Readonly> = { // The embedded OAR implementation retains its upstream license. '@deepseek-ai/dsh-experimental-subagent-oar': ['LICENSE-OAR'], // CUA publishes its client API and attributed native source for local macOS builds. - '@deepseek-ai/dsh-cua': ['lib/remote-types.js', 'lib/api.js', 'lib/bridge.js', 'lib/native-client.js', 'lib/protocol.js', 'lib/config.js', 'lib/native-schema.js', 'lib/native-build.js', 'lib/guidance.js'], + '@deepseek-ai/dsh-cua': ['lib/remote-types.js', 'lib/api.js', 'lib/bridge.js', 'lib/native-client.js', 'lib/session-permissions.js', 'lib/protocol.js', 'lib/config.js', 'lib/native-schema.js', 'lib/native-build.js', 'lib/guidance.js'], '@deepseek-ai/dsh-experimental-cua-macos': ['native/bin', 'cua-helper.json', 'native/Package.swift', 'native/Info.plist', 'native/DSHComputerUse.icns', 'native/build_app.py', 'native/Sources', 'native/tests/swift', 'native/tests/test_swift.py', 'native/LICENSE.cua-driver', 'native/LICENSE.computer-use-icon', 'native/upstream.json', 'native/README.md', 'native/README.zh.md', 'native/protocol.schema.json'], '@deepseek-ai/dsh-webhook-github': ['examples/github-review/cordis.yml', 'examples/github-review/github-ready-review-rule.mjs'], // Liveshare publishes its built Relay gateway entry and the shared runtime diff --git a/tsconfig.base.json b/tsconfig.base.json index e8ca4f8e44..58c5ce6e40 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -113,6 +113,7 @@ "@deepseek-ai/dsh-experimental-computer-use-cua-driver-mcp": ["./packages/experimental/computer-use-cua-driver-mcp/src/index.ts"], "@deepseek-ai/dsh-experimental-cua-macos": ["./packages/experimental/cua-macos/src/index.ts"], "@deepseek-ai/dsh-cua/config": ["./packages/experimental/cua/src/config.ts"], + "@deepseek-ai/dsh-cua/session-permissions": ["./packages/experimental/cua/src/session-permissions.ts"], "@deepseek-ai/dsh-cua/native-schema": ["./packages/experimental/cua/src/native-schema.ts"], "@deepseek-ai/dsh-cua/native-build": ["./packages/experimental/cua/src/native-build.ts"], "@deepseek-ai/dsh-cua/guidance": ["./packages/experimental/cua/src/guidance.ts"], From 0682926512a4d18018af8e90cb231332f906fe9a Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Sat, 7 Nov 2026 17:23:30 +0800 Subject: [PATCH 17/30] test(forge-panel): wait for the last file to draw before reading the end position --- apps/web/tests/forge-panel.e2e.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/web/tests/forge-panel.e2e.ts b/apps/web/tests/forge-panel.e2e.ts index f7820e556d..51cc42479d 100644 --- a/apps/web/tests/forge-panel.e2e.ts +++ b/apps/web/tests/forge-panel.e2e.ts @@ -1255,6 +1255,7 @@ it.skipIf(MODE === 'record')('uses horizontal arrows to navigate files while pre await page.keyboard.press('ArrowRight') await expect.poll(() => tree.getByRole('button', { name: 'README.md', exact: true }).getAttribute('aria-current')).toBe('true') // The last file cannot reach the top, so the jump settles at the end of the list once the file has drawn its lines. + await c.locator('[data-review-code]').first().waitFor() await expect.poll(() => scroll.evaluate(element => element.scrollHeight - element.clientHeight - element.scrollTop)).toBeLessThan(1) const lastPosition = await scroll.evaluate(element => element.scrollTop) const lastBox = await c.boundingBox(), viewport = await scroll.boundingBox() From bfa829a6487c4a5c21fd3a2d4034ba566dd66bf8 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 17:40:42 +0800 Subject: [PATCH 18/30] fix(cua-macos): require an explicit request for every system-pointer move --- ...13-cua-hover-routes-and-coordinates.i18n.yaml | 4 ++-- ...026-10-13-cua-hover-routes-and-coordinates.md | 2 +- ...-10-13-cua-hover-routes-and-coordinates.zh.md | 2 +- docs/config-catalog.md | 2 +- docs/config-catalog.zh.md | 2 +- .../cua-hover-ax-only/guide.i18n.yaml | 4 ++-- .../v0.2.2-rc.1/cua-hover-ax-only/guide.md | 2 +- .../v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md | 2 +- packages/experimental/cua-macos/README.i18n.yaml | 4 ++-- packages/experimental/cua-macos/README.md | 4 ++-- packages/experimental/cua-macos/README.zh.md | 4 ++-- packages/experimental/cua-macos/src/guidance.ts | 2 +- packages/experimental/cua-macos/src/index.ts | 2 +- .../cua-macos/src/mouse-control-locales.ts | 16 ++++++++-------- .../experimental/cua-macos/src/mouse-control.ts | 2 +- .../cua-macos/tests/provider-lifecycle.spec.ts | 8 +++----- .../experimental/cua/src/session-permissions.ts | 10 ---------- .../session/cua-native/replay.override.json | 2 +- 18 files changed, 31 insertions(+), 43 deletions(-) diff --git a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.i18n.yaml b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.i18n.yaml index 4450e114ec..d510b342e3 100644 --- a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.i18n.yaml @@ -9,8 +9,8 @@ en: f96bf1ebc0719fb0 zh: e9a1144f0e76572d /agent-note-background-hover-and-screenshot-coordinates/decision: - en: 42f40e78c2fff8a8 - zh: b7c5fa5fe538ed77 + en: 36188669cfa9af11 + zh: f1f0cba1ebb18f49 /agent-note-background-hover-and-screenshot-coordinates/alternatives-considered: en: 5fc4beaf79761c50 zh: 4948333b0020be28 diff --git a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.md b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.md index c46dc02060..5d5dc2a785 100644 --- a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.md +++ b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.md @@ -10,7 +10,7 @@ Hover-only controls can sometimes be revealed without moving the user's pointer. ## Decision -By default, native hover uses a same-window element or ancestor advertising both `AXShowAlternateUI` and `AXShowDefaultUI`. Entering its revealed descendants preserves the parent UI; leaving restores it. An unavailable route returns `HOVER_UNAVAILABLE` by default. With `requireSystemPointerApproval` enabled, the first explicit `mouseMove(target, { useSystemPointer: true })` asks the user. Approval applies to this move and later ordinary moves in the live session. Closing the session or restarting DSH clears it. Without the approval requirement only an explicit request selects real input. This lets a caller reach pointer-tracking controls even when the same target also advertises AX alternate UI. The Host passes the session decision with each move. Physical movement leaves the visibility of previously revealed controls to the application, so switching routes cannot dismiss a target before dispatch. An AX action failure propagates because received effects may be uncertain. The result records the route and whether the system pointer moved. System movement requires the selected foreground app and AppKit's mouse hit test, which excludes input-transparent overlays. No route grants [cross-call ownership](2026-10-13-cua-continuous-hover.md). +By default, native hover uses a same-window element or ancestor advertising both `AXShowAlternateUI` and `AXShowDefaultUI`. Entering its revealed descendants preserves the parent UI; leaving restores it. An unavailable route returns `HOVER_UNAVAILABLE` by default. With `requireSystemPointerApproval` enabled, the first explicit `mouseMove(target, { useSystemPointer: true })` asks the user. Each physical move requires an explicit request, even after approval. The live session remembers the decision until the session closes or DSH restarts. This lets a caller reach pointer-tracking controls even when the same target also advertises AX alternate UI. The Host selects real input only for a move that requests it and meets the approval policy. Physical movement leaves the visibility of previously revealed controls to the application, so switching routes cannot dismiss a target before dispatch. An AX action failure propagates because received effects may be uncertain. The result records the route and whether the system pointer moved. System movement requires the selected foreground app and AppKit's mouse hit test, which excludes input-transparent overlays. No route grants [cross-call ownership](2026-10-13-cua-continuous-hover.md). Model-facing coordinate input defaults to pixels in the returned screenshot. `relative: true` accepts integer thousandths from 0 to 999 for `click`, `mouseMove`, `scroll` and `drag`; it does not apply to opaque element IDs. The Host owns the mapping to native window points. An application-wide observation refreshes its inventory; a scoped observation preserves other windows' image mappings. Window resizing invalidates the old image, while movement preserves local coordinates. Relative endpoints remain inside the window and do not depend on screenshot resolution. diff --git a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.zh.md b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.zh.md index f4c5c39d69..bedcc2740d 100644 --- a/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.zh.md +++ b/.agents/notes/implemented/architecture/2026-10-13-cua-hover-routes-and-coordinates.zh.md @@ -10,7 +10,7 @@ Status: implemented ## 决策 -默认情况下，原生悬停会寻找同一窗口内同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI` 的元素或祖先。进入其展开区域的子控件时保持父控件展开，离开后恢复。没有可用路径时默认返回 `HOVER_UNAVAILABLE`。启用 `requireSystemPointerApproval` 时，首次显式调用 `mouseMove(target, { useSystemPointer: true })` 会询问用户。同意后，本次移动和本会话后续普通悬停使用系统光标。关闭会话或重启 DSH 后授权失效。免询问配置下，只有显式请求才会移动系统光标。有些控件既支持 AX 备用界面，又跟踪真实鼠标。Host 在每次悬停请求中传入本会话的授权结果，让调用方能够触发后者。切换到系统鼠标后，由应用决定是否收起先前展开的控件，以免派发前目标消失。AX 动作失败直接返回，因为已产生的效果可能不确定。结果记录实际路径及是否移动系统鼠标。系统移动要求所选应用处于前台，并使用 AppKit 的鼠标命中查询排除不接收输入的覆盖窗口。所有路径都遵守[跨调用不分配所有权](2026-10-13-cua-continuous-hover.zh.md)的决定。 +默认情况下，原生悬停会寻找同一窗口内同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI` 的元素或祖先。进入其展开区域的子控件时保持父控件展开，离开后恢复。没有可用路径时默认返回 `HOVER_UNAVAILABLE`。启用 `requireSystemPointerApproval` 时，首次显式调用 `mouseMove(target, { useSystemPointer: true })` 会询问用户。即使用户已授权，每次移动系统光标仍需显式请求。会话记住授权决定，直到关闭会话或重启 DSH。有些控件既支持 AX 备用界面，又跟踪真实鼠标。只有本次调用显式请求系统鼠标并满足授权条件时，Host 才使用真实鼠标触发跟踪。切换到系统鼠标后，由应用决定是否收起先前展开的控件，以免派发前目标消失。AX 动作失败直接返回，因为已产生的效果可能不确定。结果记录实际路径及是否移动系统鼠标。系统移动要求所选应用处于前台，并使用 AppKit 的鼠标命中查询排除不接收输入的覆盖窗口。所有路径都遵守[跨调用不分配所有权](2026-10-13-cua-continuous-hover.zh.md)的决定。 面向模型的坐标默认使用返回截图中的像素。`click`、`mouseMove`、`scroll` 和 `drag` 传入 `relative: true` 时接受 0～999 的千分比整数，仅用于坐标目标。Host 负责换算原生窗口 point。应用级观察刷新窗口清单，单窗口观察保留其他窗口的图片映射。窗口尺寸变化使旧图片失效，移动窗口保留局部坐标。相对坐标的端点保持在窗口内，不随截图分辨率改变。 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index ced0f149e5..52d8ee4329 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1163,7 +1163,7 @@ export interface Config { export interface Config extends CuaConfig { /** Interval in milliseconds between permission and position checks while the native guide is open. */ permissionPollIntervalMs: number - /** Ask before a main task may move the system pointer. When false only explicitly requested moves use it. */ + /** Ask once per session before the first explicitly requested system-pointer move. */ requireSystemPointerApproval: boolean /** * Source-build signing: `ad-hoc` (default) binds grants to the code hash; diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index e91a7e3d53..73281cb49a 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -1165,7 +1165,7 @@ export interface Config { export interface Config extends CuaConfig { /** Interval in milliseconds between permission and position checks while the native guide is open. */ permissionPollIntervalMs: number - /** Ask before a main task may move the system pointer. When false only explicitly requested moves use it. */ + /** Ask once per session before the first explicitly requested system-pointer move. */ requireSystemPointerApproval: boolean /** * Source-build signing: `ad-hoc` (default) binds grants to the code hash; diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.i18n.yaml b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.i18n.yaml index 1a804c0a53..44d138539c 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.i18n.yaml +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.i18n.yaml @@ -12,5 +12,5 @@ en: e6d2e56ccb50e46a zh: a788fc19e0771a46 /macos-hover-keeps-the-system-pointer-in-place-by-default/migration: - en: b1070d660b90318c - zh: 2f2ad1b8897dc044 + en: 5db40025fe40c374 + zh: a5556c0e0b000d1c diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.md index e38e026f1c..e2e074cac4 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.md @@ -15,4 +15,4 @@ On macOS, `win.mouseMove(target)` uses an enabled element or ancestor that adver 1. For an available control, use its advertised action or click it directly. Use browser tools for web content that requires page hover. 2. If your profile sets `helperPath`, rebuild the custom helper to support the per-request `useSystemPointer` field and restart the profile. -3. When the task requires physical movement, call `win.mouseMove(target, { useSystemPointer: true })` on an unobscured foreground target. After approval this move reports `systemPointerMoved: true`. Closing the session or restarting DSH clears approval. +3. When the task requires physical movement, call `win.mouseMove(target, { useSystemPointer: true })` on an unobscured foreground target. After approval this move reports `systemPointerMoved: true`. Pass `useSystemPointer: true` again for each subsequent physical move. Closing the session or restarting DSH clears approval. diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md index d5111faf05..a26861e5d3 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-hover-ax-only/guide.zh.md @@ -15,4 +15,4 @@ description: "macOS 的 mouseMove 默认要求控件支持 AX 备用界面，否 1. 对已经可用的控件，可执行它声明的动作或直接点击。需要网页悬停时可用浏览器工具。 2. 如果 profile 设置了 `helperPath`，请重新构建该 helper，使其支持按请求读取 `useSystemPointer` 字段，然后重启 profile。 -3. 任务确实需要移动鼠标时，在前台未受遮挡的目标上调用 `win.mouseMove(target, { useSystemPointer: true })`。用户同意后，本次移动返回 `systemPointerMoved: true`。关闭会话或重启 DSH 后需要重新授权。 +3. 任务确实需要移动鼠标时，在前台未受遮挡的目标上调用 `win.mouseMove(target, { useSystemPointer: true })`。用户同意后，本次移动返回 `systemPointerMoved: true`。之后每次移动系统鼠标仍需显式传 `useSystemPointer: true`。关闭会话或重启 DSH 后需要重新授权。 diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index 45d7cd7ff8..f8b32baf91 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -18,8 +18,8 @@ en: e51a6c3625873430 zh: 0a6998897532a05f /deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment: - en: b87eede533256337 - zh: 15c6b1323a60c5b6 + en: 652b0ce8892e522c + zh: 101c20aa18b36c05 /deepseek-ai-dsh-experimental-cua-macos/understand-the-implementation: en: 4e82b85a5aa72eee zh: 4e004b7a205d8688 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index dddb02c2ef..ad896c5764 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -117,7 +117,7 @@ Use the `cua` API to discover and select the application (`listApps`, `getState` `getWindow` reads its initial tree silently. An owned window without an AX surface returns an empty snapshot and remains available for screenshots and coordinate input. Within one API instance, the facade keeps one current accessibility snapshot per application handle, shared by every app and window object for that handle, including repeated selections. Reading any window replaces that snapshot; element operations use the latest snapshot ID, so use IDs from the latest tree and do not assume an older ID still identifies the same control. A failed read publishes no partial snapshot; read again without repeating the preceding input. Window objects own screenshots and pointer input. Keyboard methods stay on `app` because input follows the app’s current focus. -Ordinary `mouseMove(target)` defaults to AX hover. Without both `AXShowAlternateUI` and `AXShowDefaultUI` on an enabled target or ancestor, it returns `HOVER_UNAVAILABLE` without prompting. Only when the task requires physical movement should the main task call `win.mouseMove(target, { useSystemPointer: true })`. On approval the Host rechecks the window and point, dispatches this move, and uses the visible pointer for later ordinary moves. Refusal sends no input and suppresses further questions. Closing the session or restarting DSH clears the decision. Explicit `useSystemPointer: false` selects AX for one call. +Ordinary `mouseMove(target)` defaults to AX hover. Without both `AXShowAlternateUI` and `AXShowDefaultUI` on an enabled target or ancestor, it returns `HOVER_UNAVAILABLE` without prompting. Only when the task requires physical movement should the main task call `win.mouseMove(target, { useSystemPointer: true })`. On approval the Host rechecks the window and point and dispatches this move. Pass `useSystemPointer: true` on every physical move. Later explicit moves reuse the decision until the session closes or DSH restarts. Omitting the option or passing `false` uses AX. Refusal sends no input. `paste(text, { format: "text" | "html" })` temporarily publishes the requested clipboard format and invokes the application’s enabled standard Paste menu command. It keeps the temporary format until a pasteboard reader requests its data, then restores every captured original format while it still owns the clipboard, preserving a later copy. Paste operations share a cross-process lock and cancellation joins helper cleanup. A forced process exit can prevent restoration. Unsupported Paste commands fail without typing fallback; `typeText` remains separate key-event input. @@ -212,7 +212,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user; approval moves their pointer to this target immediately and makes subsequent ordinary moves use it until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. Without an approval requirement only explicitly requested moves use the system pointer. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. +- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user. Approval moves their pointer to this target. Pass useSystemPointer: true on every physical move, even after approval. Omit it or pass false for AX hover. Further explicit moves reuse approval until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. With requireSystemPointerApproval disabled, explicit moves do not prompt. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index 17fd51d708..973613327c 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -117,7 +117,7 @@ void await window.getScreenshot() `getWindow` 静默读取初始树。已确认归属但没有 AX 界面的窗口返回空快照，仍可截图和发送坐标输入。在单个 API 实例内，门面按应用句柄保存一份当前辅助功能快照，该句柄的所有应用对象和窗口对象共享它，包括重复选择。读取任一窗口都会替换该快照；元素操作使用最新快照 ID，因此应使用最新树中的元素 ID，不应假定旧 ID 仍指向同一控件。读取失败不会发布部分快照；可以重新读取，但不要重复此前的输入。窗口对象负责截图和指针输入。键盘方法保留在 `app` 上，因为输入跟随应用的当前焦点。 -普通 `mouseMove(target)` 默认通过辅助功能悬停。如果目标或其祖先没有同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI`，则返回 `HOVER_UNAVAILABLE`，不会弹窗。只有任务确实需要移动用户的鼠标时，主任务才调用 `win.mouseMove(target, { useSystemPointer: true })`。用户允许后，Host 重新检查窗口和目标位置，执行这一次移动，此后本会话的普通悬停也使用系统指针。拒绝后不移动，也不再提问。关闭会话或重启 DSH 后决定失效。传入 `useSystemPointer: false` 可在本次调用中使用辅助功能。 +普通 `mouseMove(target)` 默认通过辅助功能悬停。如果目标或其祖先没有同时声明 `AXShowAlternateUI` 和 `AXShowDefaultUI`，则返回 `HOVER_UNAVAILABLE`，不会弹窗。只有任务确实需要移动用户的鼠标时，主任务才调用 `win.mouseMove(target, { useSystemPointer: true })`。用户允许后，Host 重新检查窗口和目标位置，再执行本次移动。每次移动系统指针都要显式传 `useSystemPointer: true`。省略参数或传 `false` 时使用辅助功能悬停。拒绝时不派发输入。会话记住此次选择，直到关闭会话或重启 DSH。 `paste(text, { format: "text" | "html" })` 临时提供指定格式的剪贴板内容，并调用应用已启用的标准 Paste 菜单命令。临时格式保留到剪贴板读取方请求其数据，然后只在仍持有剪贴板时恢复保存的全部原格式，保留期间产生的新复制内容。粘贴操作共用跨进程锁，取消会等待辅助进程清理。强制终止进程可能阻止恢复。应用不支持 Paste 命令时明确失败，不自动改成逐字输入；`typeText` 仍单独提供按键事件输入。 @@ -212,7 +212,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user; approval moves their pointer to this target immediately and makes subsequent ordinary moves use it until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. Without an approval requirement only explicitly requested moves use the system pointer. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. +- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user. Approval moves their pointer to this target. Pass useSystemPointer: true on every physical move, even after approval. Omit it or pass false for AX hover. Further explicit moves reuse approval until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. With requireSystemPointerApproval disabled, explicit moves do not prompt. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index 38f0328499..c5c1a07381 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -6,7 +6,7 @@ export const PLATFORM_GUIDANCE = 'macOS: accessibility tree operations are avail const methodDetails = new Map([ ['getAccessibilityTree', 'If the tree reports maxElements or maxDepth truncation, select a narrower window or ask the user to raise the named macOS provider setting and read again. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current.'], - ['mouseMove', 'Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user; approval moves their pointer to this target immediately and makes subsequent ordinary moves use it until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. Without an approval requirement only explicitly requested moves use the system pointer. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again.'], + ['mouseMove', 'Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user. Approval moves their pointer to this target. Pass useSystemPointer: true on every physical move, even after approval. Omit it or pass false for AX hover. Further explicit moves reuse approval until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. With requireSystemPointerApproval disabled, explicit moves do not prompt. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again.'], ['click', 'click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying.'], ]) diff --git a/packages/experimental/cua-macos/src/index.ts b/packages/experimental/cua-macos/src/index.ts index bf4359155c..5156e30ab1 100644 --- a/packages/experimental/cua-macos/src/index.ts +++ b/packages/experimental/cua-macos/src/index.ts @@ -48,7 +48,7 @@ export const inject = ['computerUse', 'tools', 'systemPrompt', 'subprocess', 'ag export interface Config extends CuaConfig { /** Interval in milliseconds between permission and position checks while the native guide is open. */ permissionPollIntervalMs: number - /** Ask before a main task may move the system pointer. When false only explicitly requested moves use it. */ + /** Ask once per session before the first explicitly requested system-pointer move. */ requireSystemPointerApproval: boolean /** * Source-build signing: `ad-hoc` (default) binds grants to the code hash; diff --git a/packages/experimental/cua-macos/src/mouse-control-locales.ts b/packages/experimental/cua-macos/src/mouse-control-locales.ts index 8d1f8352d6..24944b062e 100644 --- a/packages/experimental/cua-macos/src/mouse-control-locales.ts +++ b/packages/experimental/cua-macos/src/mouse-control-locales.ts @@ -11,13 +11,13 @@ export const mouseControlPrompt = { pt: 'Permitir que o agente mova o ponteiro do mouse?', }, detail: { - en: 'Allowing moves your pointer to this target now. Later hovers can also move it. If you decline, this session will not ask again. The decision expires when you close the session or restart DSH.', - zh: '允许后，你的鼠标指针会立即移到目标位置，本会话后续悬停也会移动指针。如果拒绝，本会话不会再询问。关闭会话或重启 DSH 后，这个决定失效。', - ja: '許可するとポインタが今すぐ移動し、以後のホバーでも移動します。拒否すると、このセッションでは再度確認しません。セッションを閉じるか DSH を再起動すると、この選択は失効します。', - es: 'Si aceptas, el puntero se moverá ahora al destino y también podrá moverse al pasar sobre otros elementos. Si rechazas, no se volverá a preguntar. La decisión caduca al cerrar la sesión o reiniciar DSH.', - fr: 'Si vous acceptez, le pointeur se déplace immédiatement sur la cible, puis lors des survols suivants. Si vous refusez, la question ne sera plus posée. Ce choix expire à la fermeture de la session ou au redémarrage de DSH.', - de: 'Bei Zustimmung bewegt sich der Zeiger sofort zum Ziel und auch bei späteren Mausbewegungen. Bei Ablehnung wird nicht erneut gefragt. Die Entscheidung gilt, bis Sie die Sitzung schließen oder DSH neu starten.', - ru: 'При разрешении указатель сразу переместится на цель и сможет перемещаться при следующих наведениях. При отказе вопрос не повторится. Решение действует до закрытия сеанса или перезапуска DSH.', - pt: 'Se permitir, o ponteiro irá para o alvo agora e poderá se mover nas próximas ações. Se recusar, a pergunta não será repetida. A decisão expira ao encerrar a sessão ou reiniciar o DSH.', + en: 'Allowing moves your pointer to this target now. Later hovers move it only when the agent explicitly requests pointer movement. Your choice is remembered until you close the session or restart DSH.', + zh: '允许后，指针会立即移到目标位置。之后只有智能体明确请求移动鼠标时才会移动指针。会话会记住你的选择，直到关闭会话或重启 DSH。', + ja: '許可するとポインタが今すぐ移動します。以後は、エージェントが明示的に移動を要求した場合だけポインタが動きます。この選択はセッションを閉じるか DSH を再起動するまで有効です。', + es: 'Si aceptas, el puntero se moverá ahora al destino. Después solo se moverá cuando el agente lo solicite expresamente. Tu decisión se recuerda hasta que cierres la sesión o reinicies DSH.', + fr: 'Si vous acceptez, le pointeur se déplace immédiatement sur la cible. Il ne se déplacera ensuite que si l’agent le demande explicitement. Votre choix est mémorisé jusqu’à la fermeture de la session ou au redémarrage de DSH.', + de: 'Bei Zustimmung bewegt sich der Zeiger jetzt zum Ziel. Danach wird er nur bewegt, wenn der Agent dies ausdrücklich anfordert. Ihre Entscheidung gilt, bis Sie die Sitzung schließen oder DSH neu starten.', + ru: 'При разрешении указатель сразу переместится на цель. При следующих наведениях он будет перемещаться только по явному запросу агента. Выбор сохраняется до закрытия сеанса или перезапуска DSH.', + pt: 'Se permitir, o ponteiro irá para o alvo agora. Depois, só se moverá quando o agente solicitar isso explicitamente. Sua escolha vale até encerrar a sessão ou reiniciar o DSH.', }, } diff --git a/packages/experimental/cua-macos/src/mouse-control.ts b/packages/experimental/cua-macos/src/mouse-control.ts index 6f8e7f8a94..94499ffa8a 100644 --- a/packages/experimental/cua-macos/src/mouse-control.ts +++ b/packages/experimental/cua-macos/src/mouse-control.ts @@ -28,7 +28,7 @@ export function mouseMovePreparation(ctx: Context, requireApproval: boolean): No const permission = 'system-pointer' return (agent, params) => { if (params.useSystemPointer !== true) { - params.useSystemPointer = params.useSystemPointer === false ? false : permissions.allowed(agent, permission) + params.useSystemPointer = false return undefined } if (agent.session.header.origin === 'subagent') { diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index a5d74e20e8..33f87e5866 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -1001,7 +1001,7 @@ async function hoverFixture(requireApproval = true) { return { ...f, ask, approved, call, select, questions } } -it('asks only on an explicit move and dispatches that same move after approval', async () => { +it('requires an explicit system-pointer request on every move and asks only once per session', async () => { const f = await hoverFixture() native.request.mockRejectedValueOnce(new Error('HOVER_UNAVAILABLE')) expect(JSON.stringify((await f.execute('mouseMove', hoverTarget)).content)).toContain('HOVER_UNAVAILABLE') @@ -1017,7 +1017,7 @@ it('asks only on an explicit move and dispatches that same move after approval', answer.resolve(f.approved) expect((await pending).isError).toBe(false) for (const [params, physical] of [ - [hoverTarget, true], [{ ...hoverTarget, useSystemPointer: false }, false], [physicalHover, true], + [hoverTarget, false], [{ ...hoverTarget, useSystemPointer: false }, false], [physicalHover, true], ] as const) { expect((await f.execute('mouseMove', params)).isError).toBe(false) expect(native.request).toHaveBeenLastCalledWith('mouseMove', { ...hoverTarget, useSystemPointer: physical }, expect.any(AbortSignal), expect.any(String)) @@ -1038,7 +1038,7 @@ it('keeps refusals and grants separate and forgets a closed session on reopen', expect(native.request).not.toHaveBeenCalled() expect((await f.call(agent, 'mouseMove', hoverTarget)).isError).toBe(false) expect(native.request).toHaveBeenLastCalledWith('mouseMove', { ...hoverTarget, useSystemPointer: false }, expect.any(AbortSignal), expect.any(String)) - expect((await f.execute('mouseMove', hoverTarget)).isError).toBe(false) + expect((await f.execute('mouseMove', physicalHover)).isError).toBe(false) expect(native.request).toHaveBeenLastCalledWith('mouseMove', physicalHover, expect.any(AbortSignal), expect.any(String)) const sessionId = f.handle.agent.id await f.handle.dispose() @@ -1152,8 +1152,6 @@ it('separates a provider’s permission decisions and presents its question for const agent = f.handle.agent await expect(permissions.request(agent, 'read-status', read, signal)).resolves.toBe(true) await expect(permissions.request(agent, 'write-status', write, signal)).resolves.toBe(false) - expect(permissions.allowed(agent, 'read-status')).toBe(true) - expect(permissions.allowed(agent, 'write-status')).toBe(false) await expect(permissions.request(agent, 'write-status', write, signal)).resolves.toBe(false) await expect(permissions.request(agent, 'read-status', read, signal)).resolves.toBe(true) expect(ask.mock.calls.map(([request]) => request.questions[0]!.detail)).toEqual([read.detail.en, write.detail.en]) diff --git a/packages/experimental/cua/src/session-permissions.ts b/packages/experimental/cua/src/session-permissions.ts index f34133cb1c..f21a61b685 100644 --- a/packages/experimental/cua/src/session-permissions.ts +++ b/packages/experimental/cua/src/session-permissions.ts @@ -12,16 +12,6 @@ export class CuaSessionPermissions { constructor(private readonly ctx: Context) {} - /** - * Read a permission without asking the user. - * @param agent Live Session owner. - * @param permission Provider-defined permission name. - * @returns Whether the user approved this permission in this Session. - */ - allowed(agent: Agent, permission: string): boolean { - return this.decisions.get(agent)?.get(permission) === true - } - /** * Remember the user's choice only for this permission in this live Session. * @param agent Live Session owner. diff --git a/snapshots/session/cua-native/replay.override.json b/snapshots/session/cua-native/replay.override.json index eb110a4715..f93dd05bb5 100644 --- a/snapshots/session/cua-native/replay.override.json +++ b/snapshots/session/cua-native/replay.override.json @@ -1464,7 +1464,7 @@ "index": 0, "block": { "type": "tool-call", - "id": "cua-system-hover", + "id": "cua-ax-hover-after-approval", "name": "cua", "arguments": "{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}" } From bd2b902cc41ffc2b8c99cd6221b8b697a77c20e3 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 17:41:48 +0800 Subject: [PATCH 19/30] test(cua): record pointer-free hover after an approved physical move --- snapshots/session/cua-native/session.v4.jsonl | 14 +++++++------- .../session/cua-native/system-prompt.expected.md | 2 +- .../session/cua-ptc/system-prompt.expected.md | 2 +- .../ptc-script-retained/system-prompt.expected.md | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/snapshots/session/cua-native/session.v4.jsonl b/snapshots/session/cua-native/session.v4.jsonl index 8e343139cc..7482438890 100644 --- a/snapshots/session/cua-native/session.v4.jsonl +++ b/snapshots/session/cua-native/session.v4.jsonl @@ -232,14 +232,14 @@ {"type":"tool/result","data":{"turn":1,"step":44,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-explicit-pointer-move"},"toolCallId":"cua-explicit-pointer-move","content":[{"type":"text","text":"{\"dispatched\":true,\"hoverRoute\":\"system-pointer\",\"systemPointerMoved\":true}"}],"isError":false,"id":"{{message:91}}"}},"sourceEventSeqs":[229],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":44}} {"type":"step/start","data":{"turn":1,"step":45}} -{"type":"assistant/message","data":{"turn":1,"step":45,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-system-hover","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:92}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890696,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890696,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-system-hover","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1794032890696,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890696,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":45,"callId":"cua-system-hover","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}} -{"type":"tool/result","data":{"turn":1,"step":45,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-system-hover"},"toolCallId":"cua-system-hover","content":[{"type":"text","text":"{\"dispatched\":true,\"hoverRoute\":\"system-pointer\",\"systemPointerMoved\":true}"}],"isError":false,"id":"{{message:93}}"}},"sourceEventSeqs":[234],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":45,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-ax-hover-after-approval","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:92}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890696,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890696,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-ax-hover-after-approval","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1794032890696,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890696,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":45,"callId":"cua-ax-hover-after-approval","name":"cua","arguments":"{\"method\":\"mouseMove\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}} +{"type":"tool/result","data":{"turn":1,"step":45,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-ax-hover-after-approval"},"toolCallId":"cua-ax-hover-after-approval","content":[{"type":"text","text":"{\"dispatched\":true,\"hoverRoute\":\"accessibility\",\"systemPointerMoved\":false}"}],"isError":false,"id":"{{message:93}}"}},"sourceEventSeqs":[234],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":45}} {"type":"step/start","data":{"turn":1,"step":46}} {"type":"assistant/message","data":{"turn":1,"step":46,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:94}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890712,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890712,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890712,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890712,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":46,"callId":"cua-hover-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":46,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-verify"},"toolCallId":"cua-hover-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-11\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:95}}"}},"sourceEventSeqs":[239],"surfaceOp":"append"} +{"type":"tool/result","data":{"turn":1,"step":46,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-verify"},"toolCallId":"cua-hover-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-11\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- AXStaticText \\\"Hover information\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:95}}"}},"sourceEventSeqs":[239],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":46}} {"type":"step/start","data":{"turn":1,"step":47}} {"type":"assistant/message","data":{"turn":1,"step":47,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-background","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F9\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:96}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890729,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890729,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-background","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F9\"}}"}}},{"type":"chunk","time":1794032890729,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890730,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} @@ -254,7 +254,7 @@ {"type":"step/start","data":{"turn":1,"step":49}} {"type":"assistant/message","data":{"turn":1,"step":49,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-inspect","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:100}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890765,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890765,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-inspect","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890765,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890765,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":49,"callId":"cua-paste-inspect","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":49,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-inspect"},"toolCallId":"cua-paste-inspect","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-12\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:101}}"}},"sourceEventSeqs":[254],"surfaceOp":"append"} +{"type":"tool/result","data":{"turn":1,"step":49,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-inspect"},"toolCallId":"cua-paste-inspect","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-12\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- AXStaticText \\\"Hover information\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:101}}"}},"sourceEventSeqs":[254],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":49}} {"type":"step/start","data":{"turn":1,"step":50}} {"type":"assistant/message","data":{"turn":1,"step":50,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:102}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890781,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890781,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1794032890781,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890781,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} @@ -264,7 +264,7 @@ {"type":"step/start","data":{"turn":1,"step":51}} {"type":"assistant/message","data":{"turn":1,"step":51,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:104}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890797,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890797,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890797,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890797,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":51,"callId":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":51,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-focus"},"toolCallId":"cua-paste-focus","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-13\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:105}}"}},"sourceEventSeqs":[264],"surfaceOp":"append"} +{"type":"tool/result","data":{"turn":1,"step":51,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-focus"},"toolCallId":"cua-paste-focus","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-13\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- AXStaticText \\\"Hover information\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:105}}"}},"sourceEventSeqs":[264],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":51}} {"type":"step/start","data":{"turn":1,"step":52}} {"type":"assistant/message","data":{"turn":1,"step":52,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-consumed","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:106}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890813,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890813,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-consumed","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}}},{"type":"chunk","time":1794032890813,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890813,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} @@ -274,7 +274,7 @@ {"type":"step/start","data":{"turn":1,"step":53}} {"type":"assistant/message","data":{"turn":1,"step":53,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:108}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890830,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890830,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1794032890830,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890830,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":53,"callId":"cua-paste-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":53,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-verify"},"toolCallId":"cua-paste-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-14\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"fixture paste\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:109}}"}},"sourceEventSeqs":[274],"surfaceOp":"append"} +{"type":"tool/result","data":{"turn":1,"step":53,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-verify"},"toolCallId":"cua-paste-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-14\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"fixture paste\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- AXStaticText \\\"Hover information\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:109}}"}},"sourceEventSeqs":[274],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":53}} {"type":"step/start","data":{"turn":1,"step":54}} {"type":"assistant/message","data":{"turn":1,"step":54,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-debug-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F12\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:110}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032890846,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032890846,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-debug-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F12\"}}"}}},{"type":"chunk","time":1794032890846,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032890846,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index 037bde95e5..a663884957 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -67,7 +67,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user; approval moves their pointer to this target immediately and makes subsequent ordinary moves use it until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. Without an approval requirement only explicitly requested moves use the system pointer. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. +- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user. Approval moves their pointer to this target. Pass useSystemPointer: true on every physical move, even after approval. Omit it or pass false for AX hover. Further explicit moves reuse approval until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. With requireSystemPointerApproval disabled, explicit moves do not prompt. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index d5580e4964..9b9d570d03 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -69,7 +69,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user; approval moves their pointer to this target immediately and makes subsequent ordinary moves use it until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. Without an approval requirement only explicitly requested moves use the system pointer. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. +- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user. Approval moves their pointer to this target. Pass useSystemPointer: true on every physical move, even after approval. Omit it or pass false for AX hover. Further explicit moves reuse approval until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. With requireSystemPointerApproval disabled, explicit moves do not prompt. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. diff --git a/snapshots/session/ptc-script-retained/system-prompt.expected.md b/snapshots/session/ptc-script-retained/system-prompt.expected.md index 3e0d591989..e2fb062dc6 100644 --- a/snapshots/session/ptc-script-retained/system-prompt.expected.md +++ b/snapshots/session/ptc-script-retained/system-prompt.expected.md @@ -69,7 +69,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user; approval moves their pointer to this target immediately and makes subsequent ordinary moves use it until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. Without an approval requirement only explicitly requested moves use the system pointer. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. +- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user. Approval moves their pointer to this target. Pass useSystemPointer: true on every physical move, even after approval. Omit it or pass false for AX hover. Further explicit moves reuse approval until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. With requireSystemPointerApproval disabled, explicit moves do not prompt. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. From 040c93f9b9384de5c012bfdf7a5774dd84a7551e Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Sat, 7 Nov 2026 18:09:06 +0800 Subject: [PATCH 20/30] fix(forge-panel): report a refused patch before a failed .gitattributes read Both reads still run in parallel, and a failed patch read still cancels the attributes read, but the result now waits for both: the patch's failure passes on first, so a too-large or stale patch is never hidden by an attributes failure that happened to end earlier. Also documents the estimated height of undrawn files. --- .../client-ui-forge/README.i18n.yaml | 8 ++--- .../experimental/client-ui-forge/README.md | 3 +- .../experimental/client-ui-forge/README.zh.md | 3 +- .../experimental/client-ui-forge/src/index.ts | 25 ++++++-------- .../tests/controller.host.spec.ts | 34 ++++++++++++++----- 5 files changed, 44 insertions(+), 29 deletions(-) diff --git a/packages/experimental/client-ui-forge/README.i18n.yaml b/packages/experimental/client-ui-forge/README.i18n.yaml index d2e078a9fc..2cbf20d4c3 100644 --- a/packages/experimental/client-ui-forge/README.i18n.yaml +++ b/packages/experimental/client-ui-forge/README.i18n.yaml @@ -18,8 +18,8 @@ en: c7e1b7d87ccccd2e zh: 42d864b918a6175c /deepseek-ai-dsh-experimental-client-ui-forge/understand-the-implementation: - en: d0756f4b236e1e6f - zh: 9a02f6fa80118c5a + en: 0d8e666fc1cd71db + zh: 69f3d10283c2edfc /deepseek-ai-dsh-experimental-client-ui-forge/further-exploration: en: fa4d8c579ffc49ff zh: 5856ed7469e4585b @@ -30,8 +30,8 @@ en: 2bc25a99e17789c4 zh: 4bec723fbf249d28 /deepseek-ai-dsh-experimental-client-ui-forge/known-limitations-and-deferred-work: - en: 8f8ed6efb2fb7872 - zh: 489730ecefad078b + en: 4124d453778daf7b + zh: 33cab3137381942e /deepseek-ai-dsh-experimental-client-ui-forge/known-limitations-and-deferred-work/dev-note: en: e67f633dc6fe7773 zh: 3bb0c20c93f16f98 diff --git a/packages/experimental/client-ui-forge/README.md b/packages/experimental/client-ui-forge/README.md index bc116fbb1d..784a05bfa2 100644 --- a/packages/experimental/client-ui-forge/README.md +++ b/packages/experimental/client-ui-forge/README.md @@ -69,7 +69,7 @@ Restoration records `restoring` before mutation and `restored` only after succes Each inspection also decides whether the worktree is missing: the Session's directory lies inside the created worktree, `ctx.fs.lstat` finds no directory there, and `show-ref` in the source repository tells whether the branch still exists. -`localDiff` takes the fork point from the created worktree's `baseCommit` when the Session works in it, else, for each remote in the code host order, from `symbolic-ref --quiet --short refs/remotes//HEAD` and `merge-base HEAD`; without one the `branch` scope refuses with `forgePanel/no-fork-point`. A `branch` scope that names a base ref under `refs/heads/` or `refs/remotes/` instead takes `merge-base HEAD` with it once `show-ref --verify` finds the ref, refusing with `forgePanel/base-missing` when the ref does not exist and with `forgePanel/no-fork-point` when HEAD shares no history with it; another ref name is refused before Git runs. It runs `diff --no-ext-diff --no-textconv --submodule=short --ignore-submodules=dirty --no-color -M` with fixed `a/` and `b/` prefixes and `core.quotePath=false` against the fork point, against HEAD (the empty tree before the first commit), against the index, or with `--cached` against HEAD. Before patches and statistics, Git configuration is read to apply command-local clean/process filter overrides. A failed or truncated configuration read stops the comparison: `localDiff` reports `forgePanel/filter-config-failed`, with a retry after correcting Git configuration or the output limit; failed statistics leave the local changes card hidden. Filter enumeration uses `maxDiffBytes` for patches and `maxOutputBytes` for statistics. Untracked files, at most `maxUntrackedFiles` with the rest counted as omitted, come from `ls-files --others --exclude-standard -z` and are read through `ctx.fs` as new files, binary when their first 8,000 bytes hold a NUL byte or they are not UTF-8. A patch, file list, or file over `maxDiffBytes` refuses with `forgePanel/diff-too-large`, and a directory outside a checkout with `forgePanel/not-a-checkout`. The Host parses patches with jsdiff into files and hunks, so the browser receives no raw patch. It gives each file a digest of its paths, status, hunks, and `old mode` and `new mode` lines, plus the blob names of a binary change or the content of an untracked binary file, and a layer from the `dsh-review-layer` and `linguist-generated` lines of the root `.gitattributes`, later lines overriding earlier ones and a value other than a layer name leaving the layer unspecified, with patterns matched by Git's wildmatch rules in [`src/wildmatch.ts`](src/wildmatch.ts) and negated patterns skipped as Git skips them, then from picomatch matches, with dot files included, of `testPaths`, `interfacePaths`, and the built-in rules in [`src/layers.ts`](src/layers.ts), with `.gitattributes` read from the index for Staged, the working tree for the other local scopes, the last chosen commit for chosen commits, and the head or last chosen commit through `fileText` for a change request, where a missing file reads as none. `localCommits` lists the commits between the fork point and HEAD with `log --no-show-signature --topo-order --reverse`, the newest `maxCommits` of them, so the list ends with HEAD's commit. A Branch scope with chosen commits compares the first one's first parent, or the empty tree for a root commit, with the last, refusing commits that are not full object names, not listed, or out of order with `forgePanel/commits-missing`, and a chosen commit whose first parent is not the commit listed before it with `forgePanel/commits-not-linear`; `localText` reads that last commit through `show`. `localText` reads the new side of one file, from the index through `show :` for Staged and through `ctx.fs` otherwise, and refuses a path that leaves the checkout. `changeRequestDiff` and `changeRequestText` act only on a visible binding a provider serves, else they refuse with `forgePanel/not-bound`; they read the head commit of the binding's latest status through the provider's `diff` and `fileText`, a patch together with its `.gitattributes`, where the first failure cancels the other read and waits for it to end, refuse a `too-large` failure with `forgePanel/diff-too-large`, a `stale` failure, meaning the head moved past that status, with `forgePanel/stale-head`, and refuse with `forgePanel/diff-failed` before a status was read or when the host refuses otherwise. A `stale` diff polls the change request at once, once per head commit, and is not cached, so no patch is ever kept under a head it does not describe. The Host keeps the patches `changeRequestDiff` read with their attributes, one per change request at its head commit and base branch, and drops the least recently read once they exceed `maxCachedDiffBytes`; a later read at the same head and base answers from memory for any Session, and a failed read is not kept. `changeRequestCommits` returns the provider's `commits` list, or null when the provider does not read commits; `changeRequestDiff` with chosen commits finds them in that list, refuses them as the Branch scope does, and reads `compare` from the first one's parent to the last, uncached. The commits the checkout has beyond the head come from `symbolic-ref`, `cat-file -e`, and `rev-list --count`, and are null when the checked-out branch is not the head branch, the head commit is absent, or Git fails. +`localDiff` takes the fork point from the created worktree's `baseCommit` when the Session works in it, else, for each remote in the code host order, from `symbolic-ref --quiet --short refs/remotes//HEAD` and `merge-base HEAD`; without one the `branch` scope refuses with `forgePanel/no-fork-point`. A `branch` scope that names a base ref under `refs/heads/` or `refs/remotes/` instead takes `merge-base HEAD` with it once `show-ref --verify` finds the ref, refusing with `forgePanel/base-missing` when the ref does not exist and with `forgePanel/no-fork-point` when HEAD shares no history with it; another ref name is refused before Git runs. It runs `diff --no-ext-diff --no-textconv --submodule=short --ignore-submodules=dirty --no-color -M` with fixed `a/` and `b/` prefixes and `core.quotePath=false` against the fork point, against HEAD (the empty tree before the first commit), against the index, or with `--cached` against HEAD. Before patches and statistics, Git configuration is read to apply command-local clean/process filter overrides. A failed or truncated configuration read stops the comparison: `localDiff` reports `forgePanel/filter-config-failed`, with a retry after correcting Git configuration or the output limit; failed statistics leave the local changes card hidden. Filter enumeration uses `maxDiffBytes` for patches and `maxOutputBytes` for statistics. Untracked files, at most `maxUntrackedFiles` with the rest counted as omitted, come from `ls-files --others --exclude-standard -z` and are read through `ctx.fs` as new files, binary when their first 8,000 bytes hold a NUL byte or they are not UTF-8. A patch, file list, or file over `maxDiffBytes` refuses with `forgePanel/diff-too-large`, and a directory outside a checkout with `forgePanel/not-a-checkout`. The Host parses patches with jsdiff into files and hunks, so the browser receives no raw patch. It gives each file a digest of its paths, status, hunks, and `old mode` and `new mode` lines, plus the blob names of a binary change or the content of an untracked binary file, and a layer from the `dsh-review-layer` and `linguist-generated` lines of the root `.gitattributes`, later lines overriding earlier ones and a value other than a layer name leaving the layer unspecified, with patterns matched by Git's wildmatch rules in [`src/wildmatch.ts`](src/wildmatch.ts) and negated patterns skipped as Git skips them, then from picomatch matches, with dot files included, of `testPaths`, `interfacePaths`, and the built-in rules in [`src/layers.ts`](src/layers.ts), with `.gitattributes` read from the index for Staged, the working tree for the other local scopes, the last chosen commit for chosen commits, and the head or last chosen commit through `fileText` for a change request, where a missing file reads as none. `localCommits` lists the commits between the fork point and HEAD with `log --no-show-signature --topo-order --reverse`, the newest `maxCommits` of them, so the list ends with HEAD's commit. A Branch scope with chosen commits compares the first one's first parent, or the empty tree for a root commit, with the last, refusing commits that are not full object names, not listed, or out of order with `forgePanel/commits-missing`, and a chosen commit whose first parent is not the commit listed before it with `forgePanel/commits-not-linear`; `localText` reads that last commit through `show`. `localText` reads the new side of one file, from the index through `show :` for Staged and through `ctx.fs` otherwise, and refuses a path that leaves the checkout. `changeRequestDiff` and `changeRequestText` act only on a visible binding a provider serves, else they refuse with `forgePanel/not-bound`; they read the head commit of the binding's latest status through the provider's `diff` and `fileText`, a patch together with its `.gitattributes`, where a failed patch read cancels the attributes read and both reads end before the result, the patch's failure passing on before the attributes failure, refuse a `too-large` failure with `forgePanel/diff-too-large`, a `stale` failure, meaning the head moved past that status, with `forgePanel/stale-head`, and refuse with `forgePanel/diff-failed` before a status was read or when the host refuses otherwise. A `stale` diff polls the change request at once, once per head commit, and is not cached, so no patch is ever kept under a head it does not describe. The Host keeps the patches `changeRequestDiff` read with their attributes, one per change request at its head commit and base branch, and drops the least recently read once they exceed `maxCachedDiffBytes`; a later read at the same head and base answers from memory for any Session, and a failed read is not kept. `changeRequestCommits` returns the provider's `commits` list, or null when the provider does not read commits; `changeRequestDiff` with chosen commits finds them in that list, refuses them as the Branch scope does, and reads `compare` from the first one's parent to the last, uncached. The commits the checkout has beyond the head come from `symbolic-ref`, `cat-file -e`, and `rev-list --count`, and are null when the checked-out branch is not the head branch, the head commit is absent, or Git fails. `changeRequestStates` streams, for the Session list, the last known number, title, state, draft flag, checks rollup, review decision, and mergeability of every change request that a listed Session the caller may read binds: a `baseline` frame, then `changed` frames. One run serves every stream. Cancellation is observed before initialization and between frames; the last stream and plugin disposal await initialization and accepted writes, then close the run’s storage domain. A later run waits for that closure before reopening the domain and reading its stored states. Binding updates received during listing override its initial snapshot. It lists Sessions through `sessionQuery` without loading them, takes the visible bindings of a loaded Session from `ctx.forgeSession` and of another from its cached `forgeChangeRequests` summary, follows binding changes of loaded Sessions, and subscribes to each bound change request through `ctx.forge.watch`; closed requests use the idle polling cadence, and merged requests remain subscribed for explicit refresh. It stores every changed state in the `forge_panel_change_requests` storage domain under the SHA-256 digest of its URL and drops unbound stored states only when every listed Session has known bindings; a failed write is logged and the state stays in memory. The stream fails when the Host lacks the Session query, projection cache, storage, or Session store service. @@ -140,6 +140,7 @@ None for a new worktree, as its notice belongs to the first request of the Sessi - **Changes scopes** — deferred: a scope for the last turn. The Branch scope's Default needs a remote whose default branch HEAD shares history with, or a worktree the Session created; otherwise a base branch must be chosen. - **Very many branches** — the base branch menu reads every ref before it filters, so a search can reach any branch. Refs over `maxDiffBytes`, about 165,000 at the default, or a listing over `gitTimeoutMs`, leave the menu saying that branches could not be read; Default keeps working. - **Large changes** — a local patch over `maxDiffBytes` shows no files, and a change request diff over the size limit of the host or its provider shows no files and offers only View in browser. Without wrapping, the side-by-side view cuts long lines at the column edge. +- **Undrawn files** — until the reader scrolls near an expanded file, it holds 22 px per drawn line, which leaves out side-by-side pairing, wrapped lines, unchanged-line rows, and review cards, so the scroll range shifts a little as files draw; its saved review comments also appear only once it draws. - **Find in page over long diffs** — an expanded file draws its lines only once the reader scrolls near it, so the browser's find in page does not match lines of files further down until they are drawn; a page-wide search of the diff needs a find control of the view's own. - **Viewed files on the code host** — viewed marks of a change request stay in this browser and are not read from or written to GitHub's viewed state (`markFileAsViewed`) until the provider's review operations exist; another browser or device starts without them. - **Review layers** — the built-in rules cover common conventions only, so a repository records others in `.gitattributes`; a path cannot show Rust's inline `#[cfg(test)]` modules or which declarations a language exports, and the Interface layer keeps whole files, so finding changed exports needs a language analyzer that a later lens extension point can supply. `linguist-generated` is read only from the root `.gitattributes`, without nested `.gitattributes` files or attribute macros. diff --git a/packages/experimental/client-ui-forge/README.zh.md b/packages/experimental/client-ui-forge/README.zh.md index 81f2bed6f0..e4154cc47b 100644 --- a/packages/experimental/client-ui-forge/README.zh.md +++ b/packages/experimental/client-ui-forge/README.zh.md @@ -69,7 +69,7 @@ worktree 请求是一条 `plugin:client-ui-forge/worktree` 记录，由 `forgeWo 每次检查还会判断 worktree 是否缺失：Session 的目录位于所创建的 worktree 内，`ctx.fs.lstat` 在那里找不到目录，再由源仓库中的 `show-ref` 判断分支是否还在。 -Session 在所创建的 worktree 中工作时，`localDiff` 以它的 `baseCommit` 为分叉点；否则按代码托管远端的顺序，对每个远端依次运行 `symbolic-ref --quiet --short refs/remotes//HEAD` 与 `merge-base HEAD` 找出分叉点；找不到时 `branch` 范围以 `forgePanel/no-fork-point` 拒绝。`branch` 范围指定了 `refs/heads/` 或 `refs/remotes/` 下的基准引用时，`show-ref --verify` 找到该引用后改用 `merge-base HEAD` 与它的结果；引用不存在时以 `forgePanel/base-missing` 拒绝，HEAD 与它没有共同历史时以 `forgePanel/no-fork-point` 拒绝；其他引用名在运行 Git 之前即被拒绝。它以固定的 `a/`、`b/` 前缀和 `core.quotePath=false` 运行 `diff --no-ext-diff --no-textconv --submodule=short --ignore-submodules=dirty --no-color -M`，分别与分叉点、HEAD（第一次提交之前为空树）、暂存区比较，或带 `--cached` 与 HEAD 比较。读取补丁与统计前会查询 Git 配置，应用命令级 clean/process filter 覆盖。配置读取失败或被截断时停止比较：`localDiff` 报告 `forgePanel/filter-config-failed`，修正 Git 配置或读取上限后可重试；统计失败时本地变更卡片保持隐藏。过滤器枚举在读取补丁时使用 `maxDiffBytes` 上限，统计时使用 `maxOutputBytes` 上限。未跟踪文件由 `ls-files --others --exclude-standard -z` 列出，最多 `maxUntrackedFiles` 个，其余计为未列出；它们经 `ctx.fs` 读取并作为新文件加入，前 8,000 字节含 NUL 字节或不是 UTF-8 时视为二进制。补丁、文件列表或文件超过 `maxDiffBytes` 时以 `forgePanel/diff-too-large` 拒绝，目录不在检出中时以 `forgePanel/not-a-checkout` 拒绝。宿主用 jsdiff 把补丁解析为文件与 hunk，浏览器不会收到原始补丁。Host 为每个文件计算由路径、状态、hunk 以及 `old mode` 与 `new mode` 行组成的摘要（二进制改动加上 blob 名，未跟踪的二进制文件加上内容），并先按根目录 `.gitattributes` 中的 `dsh-review-layer` 与 `linguist-generated` 行确定分层（后面的行覆盖前面的行，不是层名的值视为未指定；模式按 [`src/wildmatch.ts`](src/wildmatch.ts) 中 Git 的 wildmatch 规则匹配，取反模式与 Git 一样跳过），再用 picomatch（包含点文件）依次匹配 `testPaths`、`interfacePaths` 与 [`src/layers.ts`](src/layers.ts) 中的内置规则。`.gitattributes` 的读取来源：“已暂存”读暂存区，其他本地范围读工作树，所选提交读最后一个提交，变更请求经 `fileText` 读 head 或最后一个所选提交，文件不存在时按没有处理。`localCommits` 用 `log --no-show-signature --topo-order --reverse` 列出分叉点到 HEAD 之间最新的 `maxCommits` 个提交，因此列表以 HEAD 的提交结尾。带所选提交的“分支”范围比较第一个提交的第一个父提交（根提交用空树）与最后一个提交；不是完整对象名、不在列表中或顺序颠倒的提交以 `forgePanel/commits-missing` 拒绝，第一父提交不是列表中前一个提交的所选提交以 `forgePanel/commits-not-linear` 拒绝；`localText` 经 `show` 读取那个最后的提交。`localText` 读取一个文件的新版本：“已暂存”经 `show :` 从暂存区读取，其余经 `ctx.fs` 读取；离开检出的路径会被拒绝。`changeRequestDiff` 与 `changeRequestText` 只作用于该 Session 中由提供方服务的可见绑定，否则以 `forgePanel/not-bound` 拒绝；它们经提供方的 `diff` 与 `fileText` 读取绑定最新状态中的 head 提交，补丁与它的 `.gitattributes` 同时读取，先失败的一方会取消另一方并等它结束，`too-large` 失败以 `forgePanel/diff-too-large` 拒绝，`stale` 失败（head 已越过该状态）以 `forgePanel/stale-head` 拒绝，尚未读到状态或托管平台以其他原因拒绝时以 `forgePanel/diff-failed` 拒绝。`stale` 的 diff 会立即重新轮询该变更请求（每个 head 提交一次）且不缓存，因此不会有补丁被存放在它不描述的 head 下。Host 保留 `changeRequestDiff` 读到的补丁及其属性，每个变更请求一份，按 head 提交与目标分支区分，总量超过 `maxCachedDiffBytes` 时丢弃最久未读的；之后在同一 head 与目标分支下的读取不论来自哪个 Session 都从内存返回，读取失败的结果不保留。`changeRequestCommits` 返回提供方 `commits` 的列表，提供方不读取提交时返回 null；带所选提交的 `changeRequestDiff` 在该列表中找到它们，按与“分支”范围相同的规则拒绝，并从第一个提交的父提交到最后一个提交读取 `compare`，不缓存。本地分支比 head 多出的提交数由 `symbolic-ref`、`cat-file -e` 与 `rev-list --count` 得出；当前检出分支不是源分支、本地没有 head 提交或 Git 失败时为 null。 +Session 在所创建的 worktree 中工作时，`localDiff` 以它的 `baseCommit` 为分叉点；否则按代码托管远端的顺序，对每个远端依次运行 `symbolic-ref --quiet --short refs/remotes//HEAD` 与 `merge-base HEAD` 找出分叉点；找不到时 `branch` 范围以 `forgePanel/no-fork-point` 拒绝。`branch` 范围指定了 `refs/heads/` 或 `refs/remotes/` 下的基准引用时，`show-ref --verify` 找到该引用后改用 `merge-base HEAD` 与它的结果；引用不存在时以 `forgePanel/base-missing` 拒绝，HEAD 与它没有共同历史时以 `forgePanel/no-fork-point` 拒绝；其他引用名在运行 Git 之前即被拒绝。它以固定的 `a/`、`b/` 前缀和 `core.quotePath=false` 运行 `diff --no-ext-diff --no-textconv --submodule=short --ignore-submodules=dirty --no-color -M`，分别与分叉点、HEAD（第一次提交之前为空树）、暂存区比较，或带 `--cached` 与 HEAD 比较。读取补丁与统计前会查询 Git 配置，应用命令级 clean/process filter 覆盖。配置读取失败或被截断时停止比较：`localDiff` 报告 `forgePanel/filter-config-failed`，修正 Git 配置或读取上限后可重试；统计失败时本地变更卡片保持隐藏。过滤器枚举在读取补丁时使用 `maxDiffBytes` 上限，统计时使用 `maxOutputBytes` 上限。未跟踪文件由 `ls-files --others --exclude-standard -z` 列出，最多 `maxUntrackedFiles` 个，其余计为未列出；它们经 `ctx.fs` 读取并作为新文件加入，前 8,000 字节含 NUL 字节或不是 UTF-8 时视为二进制。补丁、文件列表或文件超过 `maxDiffBytes` 时以 `forgePanel/diff-too-large` 拒绝，目录不在检出中时以 `forgePanel/not-a-checkout` 拒绝。宿主用 jsdiff 把补丁解析为文件与 hunk，浏览器不会收到原始补丁。Host 为每个文件计算由路径、状态、hunk 以及 `old mode` 与 `new mode` 行组成的摘要（二进制改动加上 blob 名，未跟踪的二进制文件加上内容），并先按根目录 `.gitattributes` 中的 `dsh-review-layer` 与 `linguist-generated` 行确定分层（后面的行覆盖前面的行，不是层名的值视为未指定；模式按 [`src/wildmatch.ts`](src/wildmatch.ts) 中 Git 的 wildmatch 规则匹配，取反模式与 Git 一样跳过），再用 picomatch（包含点文件）依次匹配 `testPaths`、`interfacePaths` 与 [`src/layers.ts`](src/layers.ts) 中的内置规则。`.gitattributes` 的读取来源：“已暂存”读暂存区，其他本地范围读工作树，所选提交读最后一个提交，变更请求经 `fileText` 读 head 或最后一个所选提交，文件不存在时按没有处理。`localCommits` 用 `log --no-show-signature --topo-order --reverse` 列出分叉点到 HEAD 之间最新的 `maxCommits` 个提交，因此列表以 HEAD 的提交结尾。带所选提交的“分支”范围比较第一个提交的第一个父提交（根提交用空树）与最后一个提交；不是完整对象名、不在列表中或顺序颠倒的提交以 `forgePanel/commits-missing` 拒绝，第一父提交不是列表中前一个提交的所选提交以 `forgePanel/commits-not-linear` 拒绝；`localText` 经 `show` 读取那个最后的提交。`localText` 读取一个文件的新版本：“已暂存”经 `show :` 从暂存区读取，其余经 `ctx.fs` 读取；离开检出的路径会被拒绝。`changeRequestDiff` 与 `changeRequestText` 只作用于该 Session 中由提供方服务的可见绑定，否则以 `forgePanel/not-bound` 拒绝；它们经提供方的 `diff` 与 `fileText` 读取绑定最新状态中的 head 提交，补丁与它的 `.gitattributes` 同时读取，补丁读取失败时取消 `.gitattributes` 的读取，两者都结束后才给出结果，补丁的失败优先于 `.gitattributes` 的失败，`too-large` 失败以 `forgePanel/diff-too-large` 拒绝，`stale` 失败（head 已越过该状态）以 `forgePanel/stale-head` 拒绝，尚未读到状态或托管平台以其他原因拒绝时以 `forgePanel/diff-failed` 拒绝。`stale` 的 diff 会立即重新轮询该变更请求（每个 head 提交一次）且不缓存，因此不会有补丁被存放在它不描述的 head 下。Host 保留 `changeRequestDiff` 读到的补丁及其属性，每个变更请求一份，按 head 提交与目标分支区分，总量超过 `maxCachedDiffBytes` 时丢弃最久未读的；之后在同一 head 与目标分支下的读取不论来自哪个 Session 都从内存返回，读取失败的结果不保留。`changeRequestCommits` 返回提供方 `commits` 的列表，提供方不读取提交时返回 null；带所选提交的 `changeRequestDiff` 在该列表中找到它们，按与“分支”范围相同的规则拒绝，并从第一个提交的父提交到最后一个提交读取 `compare`，不缓存。本地分支比 head 多出的提交数由 `symbolic-ref`、`cat-file -e` 与 `rev-list --count` 得出；当前检出分支不是源分支、本地没有 head 提交或 Git 失败时为 null。 `changeRequestStates` 为会话列表推送调用方可读的、已列出的 Session 所绑定的每个变更请求的最近编号、标题、状态、草稿标记、检查汇总、评审结论与可合并性：先是一个 `baseline` 帧，之后是 `changed` 帧。所有流共用一次运行。初始化之前和帧之间均响应取消；最后一条流结束及插件卸载会等待初始化与已接受的写入完成，再关闭本次运行的存储域。后续运行会等待该域关闭后重新打开，并读取已存状态。列举期间收到的绑定更新优先于初始快照。它经 `sessionQuery` 列出 Session 而不加载它们，已加载 Session 的可见绑定取自 `ctx.forgeSession`，其他 Session 的取自缓存的 `forgeChangeRequests` 摘要；它跟随已加载 Session 的绑定变化，并经 `ctx.forge.watch` 订阅每个已绑定的变更请求；已关闭的按空闲间隔轮询，已合并的保留订阅以接收显式刷新结果。每个变化了的状态以其 URL 的 SHA-256 摘要为键存入 `forge_panel_change_requests` 存储域，仅在所有已列出 Session 的绑定均已知时删除无人绑定的已存状态；写入失败时记录日志，状态仍保留在内存中。宿主缺少 Session 查询、投影缓存、存储或 Session 存储服务时，流以失败结束。 @@ -140,6 +140,7 @@ Session 在所创建的 worktree 中工作时，`localDiff` 以它的 `baseCommi - **变更范围**——待办：上一轮的范围。“分支”范围的“默认”需要一个默认分支与 HEAD 有共同历史的远端，或 Session 创建的 worktree；否则必须选择基准分支。 - **分支极多**——基准分支菜单在筛选前读取全部引用，这样搜索才能找到任意分支。引用超过 `maxDiffBytes`（默认约 165,000 个）或列出时间超过 `gitTimeoutMs` 时，菜单提示无法读取分支；“默认”仍可使用。 - **大型变更**——本地补丁超过 `maxDiffBytes` 时不显示任何文件；超过托管平台或其提供方大小上限的变更请求 diff 不显示任何文件，只提供“在浏览器中查看”。不自动换行时，左右对比会在栏边截断长行。 +- **尚未绘制的文件**——读者滚动到展开的文件附近之前，它按每个绘制行 22 px 占位，没有计入左右对比的配对、自动换行、未变更行按钮和评审卡片，所以文件绘制时滚动范围会略有变化；它已保存的评审评论也要等它绘制后才显示。 - **长 diff 中的页内查找**——展开的文件要等读者滚动到附近才绘制各行，所以浏览器的页内查找在更靠后的文件绘制之前匹配不到它们的行；要在整个 diff 中查找，需要视图自己的查找控件。 - **代码托管平台上的已审阅状态**——变更请求的已审阅标记只保存在本浏览器中，在提供方评审接口完成之前，不读取也不写入 GitHub 的已审阅状态（`markFileAsViewed`）；换浏览器或设备后没有这些标记。 - **分层**——内置规则只覆盖常见约定，其他约定由仓库写在 `.gitattributes` 中；按路径无法识别 Rust 的内联 `#[cfg(test)]` 模块，也无法判断一种语言导出了哪些声明，“接口”层保留整个文件，找出变更的导出需要语言分析器，可由之后的透镜扩展点提供。`linguist-generated` 只读取根目录的 `.gitattributes`，不读子目录中的 `.gitattributes`，也不展开属性宏。 diff --git a/packages/experimental/client-ui-forge/src/index.ts b/packages/experimental/client-ui-forge/src/index.ts index b6dc034eac..6b05219291 100644 --- a/packages/experimental/client-ui-forge/src/index.ts +++ b/packages/experimental/client-ui-forge/src/index.ts @@ -690,8 +690,9 @@ export class ForgePanelController extends TypertRemoteService { } /** - * Read a patch that ends at `commit` together with the root `.gitattributes` at that commit. The first failure - * cancels the other read and waits for it to end before it passes on. + * Read a patch that ends at `commit` together with the root `.gitattributes` at that commit. A failed patch read + * cancels the attributes read. Both reads end before the result: the patch's failure passes on first, and the + * attributes failure only after the patch was read, so the outcome does not depend on which read ends first. * @param ref - change request whose repository holds the commit. * @param commit - commit the patch ends at. * @param readPatch - reads the patch under the given cancellation. @@ -702,21 +703,17 @@ export class ForgePanelController extends TypertRemoteService { private async patchAt( ref: ChangeRequestRef, commit: string, readPatch: (signal: AbortSignal) => Promise, signal: AbortSignal, ): Promise<{ readonly patch: string; readonly attributes: string }> { - const sibling = new AbortController() - const operation = AbortSignal.any([signal, sibling.signal]) - const patch = readPatch(operation).then((result) => { + const unneeded = new AbortController() + const patch = readPatch(signal).then((result) => { if (typeof result !== 'string') throw readError(result) return result }) - const attributes = this.attributesAt(ref, commit, operation) - try { - const [text, rules] = await Promise.all([patch, attributes]) - return { patch: text, attributes: rules } - } catch (error) { - sibling.abort() - await Promise.allSettled([patch, attributes]) - throw error - } + const attributes = this.attributesAt(ref, commit, AbortSignal.any([signal, unneeded.signal])) + void patch.then(undefined, () => { unneeded.abort() }) + const [text, rules] = await Promise.allSettled([patch, attributes]) + if (text.status === 'rejected') throw text.reason + if (rules.status === 'rejected') throw rules.reason + return { patch: text.value, attributes: rules.value } } /** The root `.gitattributes` of a change request's repository at a commit, empty when the commit has none. */ diff --git a/packages/experimental/client-ui-forge/tests/controller.host.spec.ts b/packages/experimental/client-ui-forge/tests/controller.host.spec.ts index 92b5a6bdbc..96257f8c29 100644 --- a/packages/experimental/client-ui-forge/tests/controller.host.spec.ts +++ b/packages/experimental/client-ui-forge/tests/controller.host.spec.ts @@ -728,26 +728,43 @@ describe('forgePanel change request diffs', () => { .rejects.toMatchObject({ code: 'forgePanel/diff-failed', message: 'down' }) }) - it('reads the patch and the .gitattributes together and cancels the other read when one fails', async () => { + it('reads the patch and the .gitattributes together, and reports the patch\'s failure before the attributes failure', async () => { const { repo } = await project() const opened = await open(repo) shellTurn(opened.session, 1, [{ callId: 'create', command: 'gh pr create --fill', output: `${pullUrl(7)}\n` }]) await reader(opened).until(state => polled(state, pullUrl(7))) const read = async (): Promise => await opened.ctx.forgePanel.changeRequestDiff(opened.session, pullUrl(7), null, signal) const cancelled: string[] = [] - const held = (name: string) => async (operation: AbortSignal): Promise => { - await new Promise((resolve) => { operation.addEventListener('abort', resolve, { once: true }) }) - cancelled.push(name) - throw operation.reason + // The patch read ends only after the attributes read has failed, so the attributes failure always comes first. + let attributesFailed = Promise.withResolvers() + const later = (value: T) => async (): Promise => { + await attributesFailed.promise + return value } // Failed reads are not kept, so each case reads again. - const diff = vi.spyOn(opened.provider, 'diff').mockImplementation(async (_ref, _head, operation) => await held('patch')(operation)) - const text = vi.spyOn(opened.provider, 'fileText').mockResolvedValue({ code: 'unavailable', message: 'attributes down' }) + const diff = vi.spyOn(opened.provider, 'diff').mockImplementation(later({ code: 'too-large', message: 'diff over 300 files' })) + const text = vi.spyOn(opened.provider, 'fileText').mockImplementation(async () => { + attributesFailed.resolve(undefined) + return { code: 'unavailable', message: 'attributes down' } + }) + await expect(read()).rejects.toMatchObject({ code: 'forgePanel/diff-too-large', message: 'diff over 300 files' }) + attributesFailed = Promise.withResolvers() + diff.mockImplementation(later({ code: 'stale', message: 'head moved' })) + await expect(read()).rejects.toMatchObject({ code: 'forgePanel/stale-head', message: 'head moved' }) + attributesFailed = Promise.withResolvers() + diff.mockImplementation(later(PATCH)) await expect(read()).rejects.toMatchObject({ code: 'forgePanel/diff-failed', message: 'attributes down' }) + + // A failed patch read cancels the attributes read it no longer needs. diff.mockResolvedValue({ code: 'unavailable', message: 'patch down' }) - text.mockImplementation(async (_ref, _commit, _path, operation) => await held('attributes')(operation)) + text.mockImplementation(async (_ref, _commit, _path, operation) => { + await new Promise((resolve) => { operation.addEventListener('abort', resolve, { once: true }) }) + cancelled.push('attributes') + throw operation.reason + }) await expect(read()).rejects.toMatchObject({ code: 'forgePanel/diff-failed', message: 'patch down' }) + expect(cancelled).toEqual(['attributes']) // The patch read finishes only once the attributes read has started, so reading them one after the other never ends. const attributesStarted = Promise.withResolvers() @@ -760,7 +777,6 @@ describe('forgePanel change request diffs', () => { return 'src/** linguist-generated\n' }) await expect(read()).resolves.toMatchObject({ files: [{ path: 'src/a.ts', layer: 'test' }] }) - expect(cancelled).toEqual(['patch', 'attributes']) }) it('lists the commits of a change request and reads consecutive ones from the first one\'s parent', async () => { From 508b078373310038042d7c0e2dbff7f0ffa297e2 Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Sat, 7 Nov 2026 18:09:08 +0800 Subject: [PATCH 21/30] fix(forge-github): keep the status when GitHub reports no full base commit The base commit only gives the diff read a head start, so a missing or malformed baseRefOid now leaves baseSha out instead of failing the status. --- packages/experimental/forge-github/README.i18n.yaml | 4 ++-- packages/experimental/forge-github/README.md | 2 +- packages/experimental/forge-github/README.zh.md | 2 +- packages/experimental/forge-github/src/graphql.ts | 5 +++-- packages/experimental/forge-github/tests/graphql.spec.ts | 7 +++++++ 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/experimental/forge-github/README.i18n.yaml b/packages/experimental/forge-github/README.i18n.yaml index 9cee871506..ad2931d2cd 100644 --- a/packages/experimental/forge-github/README.i18n.yaml +++ b/packages/experimental/forge-github/README.i18n.yaml @@ -15,8 +15,8 @@ en: d152484eb41ac6b4 zh: 2f8a621b991260a1 /deepseek-ai-dsh-experimental-forge-github/use-this-package: - en: 7cc428526bce20c2 - zh: e58b72d47ad8da87 + en: 937728d83a92e0f0 + zh: ea0805e75f2f4da0 /deepseek-ai-dsh-experimental-forge-github/understand-the-implementation: en: f98e77171cba809f zh: 1b330cb342d69efe diff --git a/packages/experimental/forge-github/README.md b/packages/experimental/forge-github/README.md index 2f7bbf6355..31ed39a9bc 100644 --- a/packages/experimental/forge-github/README.md +++ b/packages/experimental/forge-github/README.md @@ -46,7 +46,7 @@ Unfinished check runs and completed runs with stale, missing, or unrecognized co A pull request is `mergeReady` when it is open, not a draft, GitHub reports its merge state as `CLEAN`, `HAS_HOOKS`, or `UNSTABLE`, and the signed-in user has write, maintain, or admin permission. `mergeMethods` lists the squash, merge-commit, and rebase methods the repository allows, the user's default merge method first. Marking ready runs `gh pr ready --repo //`. Merging calls `PUT repos///pulls//merge` through `gh api` with the method and the expected head SHA, so GitHub refuses once new commits arrived, which is reported as `stale`. Unlike `gh pr merge`, it never enables auto-merge, never joins a merge queue, and never deletes the head branch. A refused action reports GitHub's message, and a diagnostic that mentions authentication becomes `unauthenticated`. -`diff` reads `headRefOid` and `baseRefOid` in one GraphQL query and returns `stale` when the head is not the requested commit, which must be a full 40- or 64-digit hexadecimal commit. Otherwise it reads `repos///compare/...` through `gh api --allow-escape-sequences` with the diff media type, the patch from the merge base of both commits, so neither end follows a moving branch. Each status snapshot carries `baseRefOid` as `baseSha`; given it, `diff` starts this comparison while the GraphQL query confirms the head, and a stale head or another base commit cancels the early comparison and waits for it to end before `diff` returns. For every open and merged pull request measured, this patch is byte-identical to `gh pr diff`, which takes no commit argument. Unlike the pull request diff, the compare endpoint returns diffs over 300 files, so only output over `maxOutputBytes` or GitHub's own refusal makes such a diff `too-large`. `--allow-escape-sequences` keeps terminal escape sequences that file contents hold, which `gh` 2.97 and newer otherwise refuse to print; an older `gh` rejects the option with `unknown flag` before sending a request and prints such sequences unchanged, so that rejection repeats the read without it. `fileText` reads `repos///contents/?ref=` through `gh api` with the raw media type and accepts only a full 40- or 64-digit hexadecimal commit. `fileUrl` returns `https://///blob//`. Both paths percent-encode each segment. `commits` reads a pull request's commits through GraphQL, 100 per page with the owner, name, number, and cursor as variables, until `maxCommits`; every page must report the head and commit count of the first one and a cursor no earlier page returned, else the listing is `unavailable`. `compare` reads the patch from the merge base of two full commits to the second through the same compare request as `diff`. A refused read becomes `not-found` for HTTP 404, `unauthenticated` for a diagnostic that mentions authentication, `too-large` for GitHub's HTTP 406 refusal or `too_large` error, which a diff or file over its size limit draws, and `unavailable` otherwise, with GitHub's message. A read whose output exceeds `maxOutputBytes` becomes `too-large`, and one that outlives `timeoutMs` becomes `unavailable`. +`diff` reads `headRefOid` and `baseRefOid` in one GraphQL query and returns `stale` when the head is not the requested commit, which must be a full 40- or 64-digit hexadecimal commit. Otherwise it reads `repos///compare/...` through `gh api --allow-escape-sequences` with the diff media type, the patch from the merge base of both commits, so neither end follows a moving branch. Each status snapshot carries `baseRefOid` as `baseSha`, and leaves it out when GitHub reports no full commit there; given it, `diff` starts this comparison while the GraphQL query confirms the head, and a stale head or another base commit cancels the early comparison and waits for it to end before `diff` returns. For every open and merged pull request measured, this patch is byte-identical to `gh pr diff`, which takes no commit argument. Unlike the pull request diff, the compare endpoint returns diffs over 300 files, so only output over `maxOutputBytes` or GitHub's own refusal makes such a diff `too-large`. `--allow-escape-sequences` keeps terminal escape sequences that file contents hold, which `gh` 2.97 and newer otherwise refuse to print; an older `gh` rejects the option with `unknown flag` before sending a request and prints such sequences unchanged, so that rejection repeats the read without it. `fileText` reads `repos///contents/?ref=` through `gh api` with the raw media type and accepts only a full 40- or 64-digit hexadecimal commit. `fileUrl` returns `https://///blob//`. Both paths percent-encode each segment. `commits` reads a pull request's commits through GraphQL, 100 per page with the owner, name, number, and cursor as variables, until `maxCommits`; every page must report the head and commit count of the first one and a cursor no earlier page returned, else the listing is `unavailable`. `compare` reads the patch from the merge base of two full commits to the second through the same compare request as `diff`. A refused read becomes `not-found` for HTTP 404, `unauthenticated` for a diagnostic that mentions authentication, `too-large` for GitHub's HTTP 406 refusal or `too_large` error, which a diff or file over its size limit draws, and `unavailable` otherwise, with GitHub's message. A read whose output exceeds `maxOutputBytes` becomes `too-large`, and one that outlives `timeoutMs` becomes `unavailable`. Review submission checks the current pull request head, then posts a single inline comment or one completed review through the REST API with an explicit `commit_id`. Multiline comments carry their side and inclusive start and end lines. Review decisions are Comment, Approve, and Request changes. JSON request bodies travel on stdin. The provider never retries a mutation: a timeout, unexpected response, or server error after dispatch returns an unknown outcome that the caller must reconcile on GitHub. A commit arriving between the check and the write can make the explicitly addressed review outdated; GitHub does not offer an atomic head precondition on these endpoints. diff --git a/packages/experimental/forge-github/README.zh.md b/packages/experimental/forge-github/README.zh.md index 0851001833..d867f19592 100644 --- a/packages/experimental/forge-github/README.zh.md +++ b/packages/experimental/forge-github/README.zh.md @@ -46,7 +46,7 @@ kind: "package-reference" pull request 处于打开状态、不是草稿、GitHub 报告其合并状态为 `CLEAN`、`HAS_HOOKS` 或 `UNSTABLE`，且已登录用户具有 write、maintain 或 admin 权限时，`mergeReady` 为真。`mergeMethods` 列出仓库允许的压缩合并、合并提交与变基合并方式，用户默认的合并方式排在最前。标记为可评审运行 `gh pr ready --repo //`。合并经 `gh api` 调用 `PUT repos///pulls//merge`，带上合并方式与预期的 head SHA，因此有新提交到达后 GitHub 会拒绝，报告为 `stale`。与 `gh pr merge` 不同，它从不启用自动合并、从不加入合并队列，也从不删除 head 分支。被拒绝的操作报告 GitHub 的原因，提到身份验证的诊断转为 `unauthenticated`。 -`diff` 在一次 GraphQL 查询中读取 `headRefOid` 与 `baseRefOid`，head 不是请求的提交时返回 `stale`；请求的提交必须是完整的 40 位或 64 位十六进制提交。否则经 `gh api --allow-escape-sequences` 以 diff 媒体类型读取 `repos///compare/...`，即两个提交的合并基准到 head 的补丁，两端都不跟随移动的分支。每个状态快照把 `baseRefOid` 作为 `baseSha` 携带；拿到它时，`diff` 在 GraphQL 查询确认 head 的同时就开始这次比较，head 已过时或基准提交不同时会取消提前开始的比较并等它结束，然后才返回。实测的所有 open 与已合并 pull request 上，该补丁与不接受提交参数的 `gh pr diff` 逐字节一致。与 pull request diff 不同，compare 接口会返回超过 300 个文件的 diff，因此这类 diff 只有在输出超过 `maxOutputBytes` 或 GitHub 自身拒绝时才为 `too-large`。`--allow-escape-sequences` 保留文件内容中的终端转义序列；否则 `gh` 2.97 及以上版本拒绝输出。更早的 `gh` 在发出请求前就以 `unknown flag` 拒绝该选项，且原样输出这类序列，因此遇到这一拒绝时去掉该选项重读一次。`fileText` 经 `gh api` 以 raw 媒体类型读取 `repos///contents/?ref=`，只接受完整的 40 位或 64 位十六进制提交。`fileUrl` 返回 `https://///blob//`。两者的路径都逐段做百分号编码。`commits` 经 GraphQL 读取拉取请求的提交，每页 100 个，owner、name、number 与游标作为变量传入，直到 `maxCommits`；每一页报告的 head 与提交总数必须与第一页相同，游标也不能与之前任何一页重复，否则返回 `unavailable`。`compare` 用与 `diff` 相同的 compare 请求读取两个完整提交的合并基准到第二个提交的补丁。被拒绝的读取在 HTTP 404 时为 `not-found`，诊断提到身份验证时为 `unauthenticated`，GitHub 以 HTTP 406 拒绝或报告 `too_large` 错误时（diff 或文件超过其大小上限即会如此）为 `too-large`，其余为 `unavailable`，并带有 GitHub 的原因。输出超过 `maxOutputBytes` 的读取为 `too-large`，超过 `timeoutMs` 的为 `unavailable`。 +`diff` 在一次 GraphQL 查询中读取 `headRefOid` 与 `baseRefOid`，head 不是请求的提交时返回 `stale`；请求的提交必须是完整的 40 位或 64 位十六进制提交。否则经 `gh api --allow-escape-sequences` 以 diff 媒体类型读取 `repos///compare/...`，即两个提交的合并基准到 head 的补丁，两端都不跟随移动的分支。每个状态快照把 `baseRefOid` 作为 `baseSha` 携带（GitHub 返回的不是完整提交时省略）；拿到它时，`diff` 在 GraphQL 查询确认 head 的同时就开始这次比较，head 已过时或基准提交不同时会取消提前开始的比较并等它结束，然后才返回。实测的所有 open 与已合并 pull request 上，该补丁与不接受提交参数的 `gh pr diff` 逐字节一致。与 pull request diff 不同，compare 接口会返回超过 300 个文件的 diff，因此这类 diff 只有在输出超过 `maxOutputBytes` 或 GitHub 自身拒绝时才为 `too-large`。`--allow-escape-sequences` 保留文件内容中的终端转义序列；否则 `gh` 2.97 及以上版本拒绝输出。更早的 `gh` 在发出请求前就以 `unknown flag` 拒绝该选项，且原样输出这类序列，因此遇到这一拒绝时去掉该选项重读一次。`fileText` 经 `gh api` 以 raw 媒体类型读取 `repos///contents/?ref=`，只接受完整的 40 位或 64 位十六进制提交。`fileUrl` 返回 `https://///blob//`。两者的路径都逐段做百分号编码。`commits` 经 GraphQL 读取拉取请求的提交，每页 100 个，owner、name、number 与游标作为变量传入，直到 `maxCommits`；每一页报告的 head 与提交总数必须与第一页相同，游标也不能与之前任何一页重复，否则返回 `unavailable`。`compare` 用与 `diff` 相同的 compare 请求读取两个完整提交的合并基准到第二个提交的补丁。被拒绝的读取在 HTTP 404 时为 `not-found`，诊断提到身份验证时为 `unauthenticated`，GitHub 以 HTTP 406 拒绝或报告 `too_large` 错误时（diff 或文件超过其大小上限即会如此）为 `too-large`，其余为 `unavailable`，并带有 GitHub 的原因。输出超过 `maxOutputBytes` 的读取为 `too-large`，超过 `timeoutMs` 的为 `unavailable`。 提交评审时先检查 PR 当前的 head，再通过 REST API 发布单条行内评论或一次完整评审，并显式指定 `commit_id`。多行评论携带所属侧及包含端点的起止行号。评审结论为“评论”“批准”或“请求更改”。JSON 请求正文通过标准输入传递。provider 不会重试写操作；发送后的超时、异常响应或服务端错误会返回结果不明，由调用方到 GitHub 核对。检查与写入之间到来的新提交可能让已明确定位到旧提交的评审过时；GitHub 的这些接口不提供原子的 head 前置条件。 diff --git a/packages/experimental/forge-github/src/graphql.ts b/packages/experimental/forge-github/src/graphql.ts index 92bcc4fbb2..ec97758af6 100644 --- a/packages/experimental/forge-github/src/graphql.ts +++ b/packages/experimental/forge-github/src/graphql.ts @@ -132,7 +132,8 @@ const pullRequestSchema = z.object({ headRefName: z.string(), headRefOid: z.string(), baseRefName: z.string(), - baseRefOid: z.string().refine(isCommit), + // Only the diff read's head start uses the base commit, so a missing or malformed one leaves the rest of the status intact. + baseRefOid: z.unknown().transform(value => typeof value === 'string' && isCommit(value) ? value : undefined), additions: z.number().int().nonnegative(), deletions: z.number().int().nonnegative(), mergeable: z.string(), @@ -467,7 +468,7 @@ function toSnapshot( headBranch: node.headRefName, headSha: node.headRefOid, baseBranch: node.baseRefName, - baseSha: node.baseRefOid, + ...node.baseRefOid === undefined ? {} : { baseSha: node.baseRefOid }, additions: node.additions, deletions: node.deletions, mergeability: mergeability(node.mergeable), diff --git a/packages/experimental/forge-github/tests/graphql.spec.ts b/packages/experimental/forge-github/tests/graphql.spec.ts index 58abd1f67f..36bb799343 100644 --- a/packages/experimental/forge-github/tests/graphql.spec.ts +++ b/packages/experimental/forge-github/tests/graphql.spec.ts @@ -45,6 +45,13 @@ describe('snapshot query', () => { }) describe('snapshot mapping', () => { + it.each([['missing', undefined], ['null', null], ['not a full commit', 'HEAD~1']])( + 'keeps the status and leaves out the base commit when GitHub reports it %s', async (_case, baseRefOid) => { + const snapshot = await single(pullRequestNode(1, { baseRefOid })) + expect(snapshot).not.toHaveProperty('baseSha') + expect(snapshot.headSha).toBe('f'.repeat(40)) + }) + it('maps the pull request fields into a provider-neutral snapshot', async () => { expect((await single(pullRequestNode(1, {}, [checkRun('lint', 'COMPLETED', 'SUCCESS')])))).toEqual({ ref: pull(1), From 1f0eb449a3438862e5307b44045caf52ed1b3870 Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Sat, 7 Nov 2026 18:09:09 +0800 Subject: [PATCH 22/30] fix(forge-panel): end a jump's alignment when the tab hides Also explains why the alignment measures every frame and why the section handlers read values assigned during render. --- .../client-ui-forge/src/client/ChangesTab.tsx | 11 +++++++++-- .../client-ui-forge/tests/changes-tab.client.spec.tsx | 11 ++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx b/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx index 4b4a06a920..cc01ad2a19 100644 --- a/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx +++ b/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx @@ -602,7 +602,9 @@ export function ChangesTab({ const releasePin = useRef<(() => void) | null>(null) /** * Keep a jumped-to section at the top of the scrollport for {@link JUMP_PIN_MS}: the sections around it draw their - * lines after the jump, which moves it. Scrolling input from the reader, another jump, or unmounting ends the pin. + * lines after the jump, which moves it. Scrolling input from the reader, another jump, hiding the tab, or unmounting + * ends the pin. Each frame measures the section once, which covers every cause of movement, such as a section + * drawing its lines, deferred highlighting, or a review card opening, without each one reporting it. */ const pinJump = (port: HTMLElement, section: HTMLElement, owner: ChangesData): void => { releasePin.current?.() @@ -644,6 +646,8 @@ export function ChangesTab({ if (section !== undefined && scrollport.current !== null) pinJump(scrollport.current, section, data) setWanted(null) }, [tab.visible, wanted, data, allFiles, lens, hideTests, actions, tab.id]) + // A hidden tab has nothing to keep in place, and its scrollport measures as empty. + useEffect(() => { if (!tab.visible) releasePin.current?.() }, [tab.visible]) useEffect(() => () => { releasePin.current?.() }, []) const refresh = (): void => { setAttempt(value => value + 1) } const root = state !== null && state.checkout.state === 'ready' ? state.checkout.checkout.root : null @@ -798,7 +802,10 @@ export function ChangesTab({ markViewed(data, file, viewedStateOf(marks, file.path, file.digest) !== 'viewed') } - // What the section handlers and comment actions read when they run; replaced on every render. + // What the section handlers and comment actions read when they run, replaced on every render. The assignment happens + // during render rather than in an effect because the file menus read their actions while this same render draws them; + // an effect would hand them the previous diff's links. A render React discards leaves values derived from the same + // props and state, which the retried render assigns again. const live = useRef({ data, expandedByDefault, expand, markViewed, fileActions, publishReview, report, t }) live.current = { data, expandedByDefault, expand, markViewed, fileActions, publishReview, report, t } const handlers = useMemo((): DiffFileHandlers => ({ diff --git a/packages/experimental/client-ui-forge/tests/changes-tab.client.spec.tsx b/packages/experimental/client-ui-forge/tests/changes-tab.client.spec.tsx index 69121f7e76..230f00809e 100644 --- a/packages/experimental/client-ui-forge/tests/changes-tab.client.spec.tsx +++ b/packages/experimental/client-ui-forge/tests/changes-tab.client.spec.tsx @@ -971,7 +971,7 @@ describe('ChangesTab layout', () => { expect(scrolled).toHaveBeenCalledWith({ block: 'start' }) }) - it('keeps a jumped-to file at the top while sections around it draw, until the reader scrolls or a second passes', async () => { + it('keeps a jumped-to file at the top while sections around it draw, until the reader scrolls, the tab hides, or a second passes', async () => { const m = mount() await settle() const { viewport } = fileViewport(m) @@ -1013,6 +1013,15 @@ describe('ChangesTab layout', () => { shift = 80 act(() => { vi.advanceTimersToNextFrame() }) expect(scrolled).toHaveBeenCalledTimes(3) + + // Hiding the tab ends the pin at once. + shift = 0 + jump() + expect(scrolled).toHaveBeenCalledTimes(4) + act(() => { m.visibility.set(false) }) + shift = 80 + act(() => { vi.advanceTimersToNextFrame() }) + expect(scrolled).toHaveBeenCalledTimes(4) } finally { vi.useRealTimers() } From c5054b5fffb4ec6459421d60727aee550ce99aed Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Sat, 7 Nov 2026 18:09:09 +0800 Subject: [PATCH 23/30] docs(forge): record why the changes view measures pixels and how hosts mark the navigator row --- .../2026-11-07-forge-changes-view-responsiveness.i18n.yaml | 4 ++-- .../bug-fix/2026-11-07-forge-changes-view-responsiveness.md | 2 ++ .../2026-11-07-forge-changes-view-responsiveness.zh.md | 2 ++ packages/client/ui-sidebar-files/README.i18n.yaml | 4 ++-- packages/client/ui-sidebar-files/README.md | 2 +- packages/client/ui-sidebar-files/README.zh.md | 2 +- 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.i18n.yaml index 85b111788b..cc2e9c4b68 100644 --- a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.i18n.yaml +++ b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.i18n.yaml @@ -9,8 +9,8 @@ en: 77fab27f3d4a4a59 zh: 46e0fa8b9ef978a7 /agent-note-forge-changes-view-responsiveness/decision: - en: 8a0a9ef23a83ce2b - zh: 840d67e79ae39ff1 + en: a7127dc65ac6d22c + zh: 8582e17cf52db811 /agent-note-forge-changes-view-responsiveness/alternatives-considered: en: 9d12ccf1d236676d zh: 0ea63c5c16c62de7 diff --git a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md index 197552f117..73583334b2 100644 --- a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md +++ b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md @@ -18,6 +18,8 @@ The forge changes view of a large change request froze the browser. Opening #798 **Fewer serial reads.** A panel state equal to the published one reaches no follower. `changeRequestDiff` reads the patch and `.gitattributes` together. Snapshots carry the base commit as `baseSha`; `ForgeService.diff` passes it to the provider, and forge-github starts the compare read while its GraphQL read confirms the head. +**Pixel measurement.** Two parts measure layout in pixels, which CSS cannot replace. Whether a section is near the scrollport decides whether React renders its lines at all; `content-visibility` only skips layout and paint of nodes that already exist. Keeping a jumped-to section at the top needs its offset after each later layout change, since no CSS property scrolls a container to follow one child. The client packages offer no injected measurement hook, so both read the DOM directly, as the view's file navigation already does: the reach uses one `IntersectionObserver`, which measures off the main thread, and the pin reads two rectangles per frame for at most one second. + ## Alternatives considered **`content-visibility: auto` alone** cut style work as much as scoping the rules did, but kept the React render and Shiki tokenization of every expanded file at open, about 3 s of script, and its placeholder sizes would need estimates for every collapsed and expanded section. diff --git a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.zh.md b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.zh.md index 80be6bf6fa..999ac762fc 100644 --- a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.zh.md +++ b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.zh.md @@ -18,6 +18,8 @@ Status: implemented **减少串行读取。** 与已发布状态相同的面板状态不会通知任何关注方。`changeRequestDiff` 同时读取补丁和 `.gitattributes`。快照把基准提交作为 `baseSha` 携带；`ForgeService.diff` 把它传给提供方，forge-github 在 GraphQL 读取确认 head 的同时就开始 compare 读取。 +**像素测量。** 有两处需要按像素测量布局，CSS 无法替代。文件段是否接近滚动区域，决定 React 是否渲染它的各行；`content-visibility` 只能跳过已有节点的布局与绘制。让跳转后的文件段保持在顶部，需要在之后每次布局变化后读取它的偏移，没有哪个 CSS 属性能让容器跟随某个子元素滚动。客户端包没有可注入的测量 hook，所以这两处像视图已有的文件导航一样直接读取 DOM：接近判断用一个 `IntersectionObserver`，在主线程之外测量；对齐每帧读取两个矩形，最多持续一秒。 + ## Alternatives considered **只用 `content-visibility: auto`** 减少的样式工作与限定规则相当，但打开时仍要对每个展开文件做 React 渲染和 Shiki 分词，约 3 秒脚本时间，而且每个折叠和展开的段都需要估计占位尺寸。 diff --git a/packages/client/ui-sidebar-files/README.i18n.yaml b/packages/client/ui-sidebar-files/README.i18n.yaml index 0dfbf36f96..dd3d34d879 100644 --- a/packages/client/ui-sidebar-files/README.i18n.yaml +++ b/packages/client/ui-sidebar-files/README.i18n.yaml @@ -18,8 +18,8 @@ en: c4b8f3a52f68bd51 zh: 30b580320637572c /deepseek-ai-dsh-client-ui-sidebar-files/the-navigator: - en: 7e3aebc6079b023b - zh: 11911cc59fcec9b1 + en: 3424600b15bde2d2 + zh: 2cf1c99ffef2ad50 /deepseek-ai-dsh-client-ui-sidebar-files/the-tree: en: 3d71e50801761213 zh: e4e9ab19dd22288d diff --git a/packages/client/ui-sidebar-files/README.md b/packages/client/ui-sidebar-files/README.md index 2aea08292d..3991ccec18 100644 --- a/packages/client/ui-sidebar-files/README.md +++ b/packages/client/ui-sidebar-files/README.md @@ -41,7 +41,7 @@ The [registration](src/client/index.ts) assembles the page, navigator, toggle, a ## The navigator -Render `sidebar.files.navigator` with `FileNavigatorInput`: `{ useTabInfo, page, onOpen }`. The required `onOpen: (address: string) => void` callback belongs to the containing plugin; the navigator builds the file address without choosing placement. `page` controls presentation only: `true` always fills the files page, while `false` follows the preview's pane preference and available width. +Render `sidebar.files.navigator` with `FileNavigatorInput`: `{ useTabInfo, page, onOpen }`. The required `onOpen: (address: string) => void` callback belongs to the containing plugin; the navigator builds the file address without choosing placement. `page` controls presentation only: `true` always fills the files page, while `false` follows the preview's pane preference and available width. Render the Factory as a direct child of the row it shares with the body, and mark that row with `data-files-navigator-row`: the layout that fills the row hides the row's other children only under that attribute, so a host without it leaves the covered body reachable by keyboard. The navigator is drawn below the preview's header, which spans the pane, at the right of the row it shares with the body. Beside a preview, its pane's record decides whether it shows: a pane starts with it hidden, and the choice holds for every file tab of the pane until the toggle, Escape, the workspace command, or navigation to a covered preview changes it. A pane without a record of its own, such as a floating panel a tab was dragged out into, starts hidden at the default width. A click outside the tree does not hide it, and opening content in another pane does not hide it either. On the files page the tree fills the page below the header, docked or floating, without a separator, and always shows: the page draws no navigator toggle and Escape leaves it. A file the page opens starts with the destination pane's own choice, hidden for a pane that has made none. diff --git a/packages/client/ui-sidebar-files/README.zh.md b/packages/client/ui-sidebar-files/README.zh.md index 18e9caef84..5c5e776ad2 100644 --- a/packages/client/ui-sidebar-files/README.zh.md +++ b/packages/client/ui-sidebar-files/README.zh.md @@ -41,7 +41,7 @@ kind: "package-reference" ## 导航栏 -用 `FileNavigatorInput` 渲染 `sidebar.files.navigator`：`{ useTabInfo, page, onOpen }`。必填的 `onOpen: (address: string) => void` 回调由容纳它的插件提供；导航栏构造文件地址，不决定落位。`page` 只控制呈现：`true` 时始终铺满文件页，`false` 时遵循预览所在分栏的偏好与可用宽度。 +用 `FileNavigatorInput` 渲染 `sidebar.files.navigator`：`{ useTabInfo, page, onOpen }`。必填的 `onOpen: (address: string) => void` 回调由容纳它的插件提供；导航栏构造文件地址，不决定落位。`page` 只控制呈现：`true` 时始终铺满文件页，`false` 时遵循预览所在分栏的偏好与可用宽度。Factory 要作为它与正文共用的那一行的直接子元素渲染，并给该行加上 `data-files-navigator-row`：铺满整行的布局只在带这个属性的行里隐藏其他子元素，宿主不加时，被盖住的正文仍可用键盘访问。 导航栏画在横跨分栏的预览标题行下方，位于它与正文共用的那一行的右侧。在预览旁，是否显示由其分栏的记录决定：分栏默认隐藏，这一选择对分栏内的每个文件 tab 都有效，直到开关、Escape、工作区命令，或导航到被目录树遮住的预览改变它。自身没有记录的分栏（例如 tab 拖出后所在的浮窗）从隐藏、默认宽度开始。点击目录树以外的地方不会隐藏它；在另一分栏打开内容也不会隐藏它。在文件页上，目录树铺满标题行下方的整页，停靠与浮动时都一样，不显示分隔条，且始终显示：文件页不画导航栏开关，Escape 也不关闭它。文件页打开的文件沿用落定分栏自己的选择，尚未做过选择的分栏为隐藏。 From ac53809d66c89f83943f7d3378024634b80e2ca4 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 18:18:03 +0800 Subject: [PATCH 24/30] refactor(cua): own system-pointer consent in the shared host --- docs/config-catalog.md | 4 +- docs/config-catalog.zh.md | 4 +- packages/experimental/cua-macos/README.md | 6 +- packages/experimental/cua-macos/README.zh.md | 6 +- .../experimental/cua-macos/src/guidance.ts | 3 +- packages/experimental/cua-macos/src/index.ts | 23 +---- .../cua-macos/src/mouse-control.ts | 55 ----------- .../cua-macos/tests/guidance.spec.ts | 6 +- .../tests/protocol-generation.spec.ts | 11 +-- .../tests/provider-lifecycle.spec.ts | 58 ++++++----- packages/experimental/cua/README.i18n.yaml | 4 +- packages/experimental/cua/README.md | 2 +- packages/experimental/cua/README.zh.md | 2 +- packages/experimental/cua/package.json | 5 - .../cua/src/client/ApplicationApproval.tsx | 4 +- packages/experimental/cua/src/config.ts | 1 + packages/experimental/cua/src/guidance.ts | 18 +++- packages/experimental/cua/src/index.ts | 32 ++++--- .../src/mouse-control-locales.ts | 0 .../experimental/cua/src/mouse-control.ts | 63 ++++++++++++ packages/experimental/cua/src/protocol.ts | 95 +++++++++++-------- .../cua/src/session-permissions.ts | 54 ----------- packages/experimental/cua/src/types.ts | 12 ++- packages/experimental/cua/tests/api.spec.ts | 2 +- .../application-approval.client.spec.tsx | 2 +- .../experimental/cua/tests/bridge.spec.ts | 2 +- .../cua/tests/fixtures/exports.mjs | 2 - .../experimental/cua/tests/protocol.spec.ts | 4 +- packages/experimental/cua/tsdown.config.ts | 1 - scripts/check-workspace-constraints.ts | 2 +- tsconfig.base.json | 1 - 31 files changed, 219 insertions(+), 265 deletions(-) delete mode 100644 packages/experimental/cua-macos/src/mouse-control.ts rename packages/experimental/{cua-macos => cua}/src/mouse-control-locales.ts (100%) create mode 100644 packages/experimental/cua/src/mouse-control.ts delete mode 100644 packages/experimental/cua/src/session-permissions.ts diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 52d8ee4329..07e8c52196 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1156,15 +1156,13 @@ export interface Config { - `inject`: `computerUse` · `tools` · `systemPrompt` · `subprocess` · `agents` - `refs`: [`CuaConfig`](../packages/experimental/cua/src/index.ts) -- `source`: [`packages/experimental/cua-macos/src/index.ts:48`](../packages/experimental/cua-macos/src/index.ts) +- `source`: [`packages/experimental/cua-macos/src/index.ts:33`](../packages/experimental/cua-macos/src/index.ts) ```ts config-catalog /** Native preparation, source-build signing, operation bounds, and profile-local application grants. */ export interface Config extends CuaConfig { /** Interval in milliseconds between permission and position checks while the native guide is open. */ permissionPollIntervalMs: number - /** Ask once per session before the first explicitly requested system-pointer move. */ - requireSystemPointerApproval: boolean /** * Source-build signing: `ad-hoc` (default) binds grants to the code hash; * an object selects an existing certificate in an explicitly configured keychain. diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index 73281cb49a..8308df3e6f 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -1158,15 +1158,13 @@ export interface Config { - `inject`: `computerUse` · `tools` · `systemPrompt` · `subprocess` · `agents` - `refs`: [`CuaConfig`](../packages/experimental/cua/src/index.ts) -- `source`: [`packages/experimental/cua-macos/src/index.ts:48`](../packages/experimental/cua-macos/src/index.ts) +- `source`: [`packages/experimental/cua-macos/src/index.ts:33`](../packages/experimental/cua-macos/src/index.ts) ```ts config-catalog /** Native preparation, source-build signing, operation bounds, and profile-local application grants. */ export interface Config extends CuaConfig { /** Interval in milliseconds between permission and position checks while the native guide is open. */ permissionPollIntervalMs: number - /** Ask once per session before the first explicitly requested system-pointer move. */ - requireSystemPointerApproval: boolean /** * Source-build signing: `ad-hoc` (default) binds grants to the code hash; * an object selects an existing certificate in an explicitly configured keychain. diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index ad896c5764..7b5df96219 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -187,7 +187,7 @@ In persistent PTC, use the host-provided bridge initialization once and retain t Signatures show resolved return values after await. ActionResult = { dispatched: boolean, foregroundChanged?: boolean | null }; foregroundChanged is omitted when not sampled and null when the sample is unavailable. JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; numbers must be finite. App is the application object with readonly app.appHandle: string (opaque) and the methods listed below for app level. Window is the window object with readonly window.windowId: number and the methods listed below for window level. Neither object carries pid. Setup and discovery: -- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. +- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -212,7 +212,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user. Approval moves their pointer to this target. Pass useSystemPointer: true on every physical move, even after approval. Omit it or pass false for AX hover. Further explicit moves reuse approval until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. With requireSystemPointerApproval disabled, explicit moves do not prompt. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. +- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without that route, HOVER_UNAVAILABLE is returned before sending input. Physical movement requires an unobscured foreground target. Explain any necessary activation and observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. @@ -227,6 +227,8 @@ Available at app or window level depending on the platform. - sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. + +Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). Prefer an advertised accessibility action, a click on a visible control, or browser tools when possible. The question interrupts the user. Pass true on every physical move, even after approval. Omitting the option or passing false uses the default hover route without prompting. Approval or refusal lasts until this session closes or DSH restarts. A refusal sends no input. If the user dismisses the request, ask again only if they request it. Wait for an outstanding request to finish. Subagents cannot request physical movement. ``` #### Token effect diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index 973613327c..8db5bee329 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -187,7 +187,7 @@ In persistent PTC, use the host-provided bridge initialization once and retain t Signatures show resolved return values after await. ActionResult = { dispatched: boolean, foregroundChanged?: boolean | null }; foregroundChanged is omitted when not sampled and null when the sample is unavailable. JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; numbers must be finite. App is the application object with readonly app.appHandle: string (opaque) and the methods listed below for app level. Window is the window object with readonly window.windowId: number and the methods listed below for window level. Neither object carries pid. Setup and discovery: -- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. +- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -212,7 +212,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user. Approval moves their pointer to this target. Pass useSystemPointer: true on every physical move, even after approval. Omit it or pass false for AX hover. Further explicit moves reuse approval until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. With requireSystemPointerApproval disabled, explicit moves do not prompt. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. +- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without that route, HOVER_UNAVAILABLE is returned before sending input. Physical movement requires an unobscured foreground target. Explain any necessary activation and observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. @@ -227,6 +227,8 @@ Available at app or window level depending on the platform. - sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. + +Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). Prefer an advertised accessibility action, a click on a visible control, or browser tools when possible. The question interrupts the user. Pass true on every physical move, even after approval. Omitting the option or passing false uses the default hover route without prompting. Approval or refusal lasts until this session closes or DSH restarts. A refusal sends no input. If the user dismisses the request, ask again only if they request it. Wait for an outstanding request to finish. Subagents cannot request physical movement. ``` #### Token 影响 diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index c5c1a07381..0428941edb 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -6,7 +6,7 @@ export const PLATFORM_GUIDANCE = 'macOS: accessibility tree operations are avail const methodDetails = new Map([ ['getAccessibilityTree', 'If the tree reports maxElements or maxDepth truncation, select a narrower window or ask the user to raise the named macOS provider setting and read again. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current.'], - ['mouseMove', 'Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user. Approval moves their pointer to this target. Pass useSystemPointer: true on every physical move, even after approval. Omit it or pass false for AX hover. Further explicit moves reuse approval until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. With requireSystemPointerApproval disabled, explicit moves do not prompt. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again.'], + ['mouseMove', 'Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without that route, HOVER_UNAVAILABLE is returned before sending input. Physical movement requires an unobscured foreground target. Explain any necessary activation and observe the target again.'], ['click', 'click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying.'], ]) @@ -16,7 +16,6 @@ export const GUIDANCE = `${PLATFORM_GUIDANCE} ${COMMON_GUIDANCE.replace(/^- ([\w.]+)\(.*$/gm, (entry, method: string) => { if (method === 'pressKey') return `${entry} - sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.` - if (method === 'mouseMove') entry = entry.replace('relative?: boolean', 'relative?: boolean, useSystemPointer?: boolean') const detail = methodDetails.get(method) return detail ? `${entry} ${detail}` : entry })}` diff --git a/packages/experimental/cua-macos/src/index.ts b/packages/experimental/cua-macos/src/index.ts index 5156e30ab1..fb3fada07e 100644 --- a/packages/experimental/cua-macos/src/index.ts +++ b/packages/experimental/cua-macos/src/index.ts @@ -4,31 +4,16 @@ import { fileURLToPath } from 'node:url' import type { Context } from '@deepseek-ai/cordis' import Schema from '@deepseek-ai/schemastery' import { installCua, type CuaConfig, type CuaStartupCapabilities } from '@deepseek-ai/dsh-cua' -import type { CuaCoordinateOptions, CuaElementId, CuaHoverResult } from '@deepseek-ai/dsh-cua/api' import { CuaConfigFields } from '@deepseek-ai/dsh-cua/config' import { createPermissionParameters } from './permission-guide.ts' import { nativeHostArchitecture, prepareNativeExecutable, type SourceSigning } from './native-build.ts' import { createSharedNativeChannel } from './shared-native.ts' -import { mouseMovePreparation, MacRequest } from './mouse-control.ts' import { installHeldKeys } from './held-keys.ts' import { GUIDANCE } from './guidance.ts' export { GUIDANCE } from './guidance.ts' -declare module '@deepseek-ai/dsh-cua/api' { - interface CuaWindow { - /** - * Request the macOS system pointer or explicitly select accessibility hover for this move. - * @param target Current element id or screenshot-local point. - * @param options Coordinate mode and requested macOS pointer behavior. - * @returns Dispatch evidence and the selected hover route. - */ - mouseMove(target: CuaElementId | readonly [number, number], - options: CuaCoordinateOptions & { useSystemPointer: boolean }): Promise - } -} - const CAPABILITIES: CuaStartupCapabilities = { - platform: 'macos', + platform: 'macos', systemPointer: true, methods: [ 'activate', 'click', 'drag', 'findApps', 'getApp', 'getCapabilities', 'getState', 'invoke', 'listActions', 'listApps', 'listWindows', 'mouseMove', 'paste', 'performAction', 'pressKey', 'screenshot', 'scroll', @@ -48,8 +33,6 @@ export const inject = ['computerUse', 'tools', 'systemPrompt', 'subprocess', 'ag export interface Config extends CuaConfig { /** Interval in milliseconds between permission and position checks while the native guide is open. */ permissionPollIntervalMs: number - /** Ask once per session before the first explicitly requested system-pointer move. */ - requireSystemPointerApproval: boolean /** * Source-build signing: `ad-hoc` (default) binds grants to the code hash; * an object selects an existing certificate in an explicitly configured keychain. @@ -60,7 +43,6 @@ export interface Config extends CuaConfig { /** Validate shared operation bounds and macOS source signing before preparing the helper. */ export const Config = Schema.object({ ...CuaConfigFields, - requireSystemPointerApproval: Schema.boolean().default(true), permissionPollIntervalMs: Schema.number().step(1).min(100).max(2 ** 31 - 1).default(1000), sourceSigning: Schema.union([ Schema.const('ad-hoc'), @@ -80,8 +62,7 @@ export const Config = Schema.object({ */ export async function apply(ctx: Context, config: Config): Promise { const backend = { createChannel: createSharedNativeChannel, sendKeyEvent: installHeldKeys(ctx), - guidance: GUIDANCE, capabilities: CAPABILITIES, requestSchema: MacRequest, - prepareMouseMove: mouseMovePreparation(ctx, config.requireSystemPointerApproval), + guidance: GUIDANCE, capabilities: CAPABILITIES, permissionParameters: createPermissionParameters(config.permissionPollIntervalMs) } if (config.helperPath !== undefined) { if (!isAbsolute(config.helperPath)) throw new Error('dsh-cua helperPath must be absolute') diff --git a/packages/experimental/cua-macos/src/mouse-control.ts b/packages/experimental/cua-macos/src/mouse-control.ts deleted file mode 100644 index 94499ffa8a..0000000000 --- a/packages/experimental/cua-macos/src/mouse-control.ts +++ /dev/null @@ -1,55 +0,0 @@ -/** macOS system-pointer options and session approval for physical hover. */ -import type { Context } from '@deepseek-ai/cordis' -import type { CuaBackend } from '@deepseek-ai/dsh-cua' -import { CuaSessionPermissions } from '@deepseek-ai/dsh-cua/session-permissions' -import { NativeError } from '@deepseek-ai/dsh-cua/native-client' -import { MouseMoveTargets, Request } from '@deepseek-ai/dsh-cua/protocol' -import { UserQuestionError } from '@deepseek-ai/dsh-user-questions' -import { z } from 'zod' -import { mouseControlPrompt } from './mouse-control-locales.ts' - -const pointerOptions = { useSystemPointer: z.boolean().optional().describe('Request use of the system pointer. The Host checks permission before dispatch.') } -const mouseMove = z.strictObject({ method: z.literal('mouseMove'), params: z.union(MouseMoveTargets.map(target => target.extend(pointerOptions))) }) - -const [first, ...rest] = Request.options -const macOperation = (op: typeof Request.options[number]) => op.shape.method.value === 'mouseMove' ? mouseMove : op - -/** macOS advertises and validates its additional option for element and coordinate moves. */ -export const MacRequest = z.discriminatedUnion('method', [macOperation(first), ...rest.map(macOperation)]) - -/** - * Select the native route on the provider's parsed request, before coordinates are rechecked and dispatched. - * @param ctx Provider context owning this set of Session decisions. - * @param requireApproval Whether physical moves need the user's approval. - * @returns Preparation for mouseMove with an optional user interaction. - */ -export function mouseMovePreparation(ctx: Context, requireApproval: boolean): NonNullable { - const permissions = new CuaSessionPermissions(ctx) - const permission = 'system-pointer' - return (agent, params) => { - if (params.useSystemPointer !== true) { - params.useSystemPointer = false - return undefined - } - if (agent.session.header.origin === 'subagent') { - throw new NativeError('MOUSE_CONTROL_UNAVAILABLE', "Delegated tasks cannot move the user's pointer. Use AX hover, click, or performAction. If the task requires a physical hover, report that step in your final result for the main task.") - } - if (!requireApproval) return undefined - params.useSystemPointer = false - return async (signal) => { - try { - if (!await permissions.request(agent, permission, mouseControlPrompt, signal)) { - throw new NativeError('MOUSE_CONTROL_DENIED', 'The user declined system-pointer movement for this session. No input was sent. Use pointer-free actions.') - } - } catch (error) { - if (error instanceof UserQuestionError) { - if (error.code === 'APPROVAL_PENDING') throw new NativeError('MOUSE_CONTROL_PENDING', error.message, { cause: error }) - if (error.code === 'NO_PROVIDER') throw new NativeError('MOUSE_APPROVAL_UNAVAILABLE', error.message, { cause: error }) - if (error.code === 'ASK_CANCELLED') throw new NativeError('MOUSE_APPROVAL_CANCELLED', 'The mouse control request was dismissed. Continue without mouse control unless the user asks to reconsider', { cause: error }) - } - throw error - } - params.useSystemPointer = true - } - } -} diff --git a/packages/experimental/cua-macos/tests/guidance.spec.ts b/packages/experimental/cua-macos/tests/guidance.spec.ts index 7f0cb9ea86..54d384a688 100644 --- a/packages/experimental/cua-macos/tests/guidance.spec.ts +++ b/packages/experimental/cua-macos/tests/guidance.spec.ts @@ -34,12 +34,12 @@ it('states the macOS accessibility, keyboard, and mouse levels', () => { expect(pressKey).toBe(COMMON_GUIDANCE.split('\n').find(line => line.startsWith('- pressKey('))) expect(GUIDANCE).toContain('W down, D down, D up, W up') const hover = GUIDANCE.split('\n').find(line => line.startsWith('- mouseMove(')) - for (const term of ['AXShowAlternateUI', 'AXShowDefaultUI', 'HOVER_UNAVAILABLE', 'The question interrupts the user']) expect(hover).toContain(term) + for (const term of ['AXShowAlternateUI', 'AXShowDefaultUI', 'HOVER_UNAVAILABLE']) expect(hover).toContain(term) const click = GUIDANCE.split('\n').find(line => line.startsWith('- click(')) expect(click).toContain('await app.getWindow(windowId), then call win.click(target)') expect(click).toContain('use app.invoke(id)') for (const step of ['call app.activate() first', 'outside the window', 'screenshot pixels', 'before typing or retrying']) expect(click).toContain(step) - for (const platformDetail of ['macOS', 'AXShowAlternateUI', 'AXRaise', 'HOVER_UNAVAILABLE', 'The question interrupts the user', 'mouse-down']) { + for (const platformDetail of ['macOS', 'AXShowAlternateUI', 'AXRaise', 'HOVER_UNAVAILABLE', 'mouse-down']) { expect(COMMON_GUIDANCE).not.toContain(platformDetail) } }) @@ -72,7 +72,7 @@ it('inlines discovery, restoration, screenshot, and mouse-move return records', for (const name of ['cua.getCapabilities', 'cua.listApps', 'cua.findApps', 'cua.getState', 'app.getWindows', 'app.show', 'window.show', 'getScreenshot', 'mouseMove']) { expect(lines.find(line => line.startsWith(`- ${name}(`))).toMatch(/ -> \(?\{ /) } - expect(COMMON_GUIDANCE).toContain("{ platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false }") + expect(COMMON_GUIDANCE).toContain("{ platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false }") expect(COMMON_GUIDANCE).toContain('{ id: string, displayName: string, isRunning: true, pid: number }[]') expect(COMMON_GUIDANCE).toContain('({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[]') expect(COMMON_GUIDANCE).toContain('bounds: { x: number, y: number, width: number, height: number }, isOnScreen: boolean, onCurrentSpace: boolean | null') diff --git a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts index 76d3bbff12..c83c16e77e 100644 --- a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts +++ b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts @@ -3,8 +3,7 @@ import { readFile } from 'node:fs/promises' import { Ajv2020 } from 'ajv/dist/2020.js' import { describe, expect, it } from 'vitest' import { renderNativeProtocol } from '../native/generate-protocol.ts' -import { Request } from '@deepseek-ai/dsh-cua/protocol' -import { MacRequest } from '../src/mouse-control.ts' +import { Request, SystemPointerRequest } from '@deepseek-ai/dsh-cua/protocol' describe('native protocol generation', () => { it('keeps native JSON Schema and Swift admission tables current', async () => { @@ -29,13 +28,13 @@ describe('native protocol generation', () => { expect(validate({ id: 'shortcut', ...request })).toBe(false) }) - it('admits the pointer option only in the macOS model schema and only as a boolean', () => { + it('admits the pointer option only in the system-pointer model schema and only as a boolean', () => { const target = { appHandle: 'app', windowId: 1, x: 0, y: 0 } const request = { method: 'mouseMove', params: { ...target, useSystemPointer: true } } expect(Request.safeParse(request).success).toBe(false) - expect(MacRequest.safeParse(request).success).toBe(true) - expect(MacRequest.safeParse({ ...request, params: { ...target, useSystemPointer: 'true' } }).success).toBe(false) - expect(MacRequest.safeParse({ ...request, params: { ...target, arbitraryField: true } }).success).toBe(false) + expect(SystemPointerRequest.safeParse(request).success).toBe(true) + expect(SystemPointerRequest.safeParse({ ...request, params: { ...target, useSystemPointer: 'true' } }).success).toBe(false) + expect(SystemPointerRequest.safeParse({ ...request, params: { ...target, arbitraryField: true } }).success).toBe(false) }) it.each([ diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index 33f87e5866..343bbbdb33 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -17,8 +17,8 @@ import * as StorageDomain from '@deepseek-ai/dsh-storage-domain' import Hmr from '@deepseek-ai/dsh-hmr' import { mountAgentLoopTestDependencies, mountAgentLoopTestHarness } from '@deepseek-ai/dsh-agent-loop-testkit' import ComputerUseRegistry from '@deepseek-ai/dsh-computer-use' +import { COMMON_GUIDANCE } from '@deepseek-ai/dsh-cua/guidance' import { CuaConfig, installCua } from '@deepseek-ai/dsh-cua' -import { CuaSessionPermissions } from '@deepseek-ai/dsh-cua/session-permissions' import { createConfigProjector } from '@deepseek-ai/dsh-app-boot/src/config-schema/projector.ts' import LocalAttachmentStore from '@deepseek-ai/dsh-attachment-local' import FileSystem from '@deepseek-ai/dsh-fs-local' @@ -289,7 +289,7 @@ it('merges startup capabilities into runtime native fields', async () => { native.request.mockResolvedValueOnce(runtime) const result = await f.execute('getCapabilities') expect(result.isError).toBe(false) - expect(result.value).toEqual({ ...runtime, platform: 'macos', appLevelAccessibility: true, keyboard: ['app'], browser: false, + expect(result.value).toEqual({ ...runtime, platform: 'macos', systemPointer: true, appLevelAccessibility: true, keyboard: ['app'], browser: false, coordinateSpace: 'screenshot-pixels', methods: [ 'activate', 'click', 'drag', 'findApps', 'getApp', 'getCapabilities', 'getState', 'invoke', 'listActions', 'listApps', 'listWindows', 'mouseMove', 'paste', 'performAction', 'pressKey', 'screenshot', 'scroll', @@ -982,7 +982,7 @@ async function hoverFixture(requireApproval = true) { await f.ctx.plugin(NativeProvider, { ...config, requireSystemPointerApproval: false }) } const questions = await f.ctx.plugin(UserQuestions) - const questionId = 'cua-session-permission:system-pointer' + const questionId = 'cua-mouse-control' const approved = { answers: [{ id: questionId, selected: ['Allow for this session'] }] } const ask = vi.fn(async () => approved) f.ctx.on('user-questions/request', async ({ questions }) => questions[0]!.id === questionId ? ask() @@ -1032,7 +1032,7 @@ it('keeps refusals and grants separate and forgets a closed session on reopen', const { agent } = await f.ctx.agents.create({ sessionId: SessionId('other-mouse-session') }) expect(JSON.stringify((await f.call(agent, 'mouseMove', physicalHover)).content)).toContain('APP_ACCESS_REQUIRED') await f.select(agent) - f.ask.mockResolvedValueOnce({ answers: [{ id: 'cua-session-permission:system-pointer', selected: ['Deny'] }] }) + f.ask.mockResolvedValueOnce({ answers: [{ id: 'cua-mouse-control', selected: ['Deny'] }] }) native.request.mockClear() for (let i = 0; i < 2; i++) expect(JSON.stringify((await f.call(agent, 'mouseMove', physicalHover)).content)).toContain('MOUSE_CONTROL_DENIED') expect(native.request).not.toHaveBeenCalled() @@ -1120,39 +1120,35 @@ it.each([true, false])('rejects a delegated pointer move before asking or dispat expect(native.request).not.toHaveBeenCalled() }) -it('keeps provider-specific pointer flags and questions off providers without mouse control', async () => { +it.each([true, false])('uses the declared systemPointer capability for a non-macOS provider (%s)', async (systemPointer) => { const f = await setup() await f.provider.dispose() await f.ctx.plugin({ name: 'fixture-window-provider', inject: NativeProvider.inject, apply(ctx: Context) { installCua(ctx, CuaConfig(config), { - helperPath: config.helperPath, createChannel: native.connect, - guidance: '', permissionParameters: () => ({}), - capabilities: { platform: 'fixture', methods: ['getApp', 'mouseMove'], appLevelAccessibility: true, - keyboard: ['app'], coordinateSpace: 'screenshot-pixels', browser: false }, + helperPath: config.helperPath, createChannel: native.connect, guidance: COMMON_GUIDANCE, + permissionParameters: () => ({}), + capabilities: { platform: 'fixture', methods: ['getApp', 'getCapabilities', 'mouseMove'], systemPointer, + appLevelAccessibility: true, keyboard: ['app'], coordinateSpace: 'screenshot-pixels', browser: false }, }) } }) + native.request.mockResolvedValueOnce({ systemPointer: !systemPointer }) + expect((await f.execute('getCapabilities')).value).toMatchObject({ platform: 'fixture', systemPointer }) + const sections = (await f.ctx.systemPrompt.assemble({ scope: f.handle.agent })).sections + const guidance = sections.find(section => section.name === 'computer-use:dsh-cua')!.text + expect(guidance.includes('useSystemPointer')).toBe(systemPointer) + expect(guidance.includes('The question interrupts the user')).toBe(systemPointer) + expect(JSON.stringify(f.ctx.tools.get('cua', f.handle.agent)!.parameters).includes('useSystemPointer')).toBe(systemPointer) await selectApplication(f) - expect(JSON.stringify((await f.execute('mouseMove', physicalHover)).content)).toContain('useSystemPointer') - const target = { appHandle: 'app-one', snapshotId: 'snapshot', elementId: '0' } - expect((await f.execute('mouseMove', target)).isError).toBe(false) - expect(native.request).toHaveBeenLastCalledWith('mouseMove', target, expect.any(AbortSignal), expect.any(String)) -}) - -it('separates a provider’s permission decisions and presents its question for each capability', async () => { - const f = await setup() - await f.ctx.plugin(UserQuestions) - const permissions = new CuaSessionPermissions(f.ctx) - const signal = new AbortController().signal - const ask = vi.spyOn(f.ctx.userQuestions, 'ask').mockImplementation(async ({ questions }) => { - const q = questions[0]! - return { answers: [{ id: q.id, selected: [q.question === 'Read status?' ? 'Allow for this session' : 'Deny'] }] } + native.request.mockClear() + const answer = vi.spyOn(f.ctx.userQuestions, 'ask').mockResolvedValue({ + answers: [{ id: 'cua-mouse-control', selected: ['Allow for this session'] }], }) - const read = { question: { en: 'Read status?' }, detail: { en: 'Only for this session' } } - const write = { question: { en: 'Change status?' }, detail: { en: 'Sets the current status' } } - const agent = f.handle.agent - await expect(permissions.request(agent, 'read-status', read, signal)).resolves.toBe(true) - await expect(permissions.request(agent, 'write-status', write, signal)).resolves.toBe(false) - await expect(permissions.request(agent, 'write-status', write, signal)).resolves.toBe(false) - await expect(permissions.request(agent, 'read-status', read, signal)).resolves.toBe(true) - expect(ask.mock.calls.map(([request]) => request.questions[0]!.detail)).toEqual([read.detail.en, write.detail.en]) + const physical = await f.execute('mouseMove', physicalHover) + expect(physical.isError).toBe(!systemPointer) + expect(answer).toHaveBeenCalledTimes(systemPointer ? 1 : 0) + if (systemPointer) expect(native.request).toHaveBeenLastCalledWith('mouseMove', physicalHover, expect.any(AbortSignal), expect.any(String)) + else expect(native.request).not.toHaveBeenCalled() + expect((await f.execute('mouseMove', hoverTarget)).isError).toBe(false) + expect(native.request).toHaveBeenLastCalledWith('mouseMove', systemPointer ? { ...hoverTarget, useSystemPointer: false } : hoverTarget, + expect.any(AbortSignal), expect.any(String)) }) diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index 851909295c..40d8ac79c6 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -21,8 +21,8 @@ en: d92d5948a7941e8d zh: d43491ae6521b87e /deepseek-ai-dsh-cua/understand-the-implementation: - en: edb0b08ad9ee9a73 - zh: da868fc1061516e4 + en: f7d47911abb5e2d8 + zh: 227645c9df979455 /deepseek-ai-dsh-cua/further-exploration: en: c174fb5a00ea3681 zh: 039340042676881d diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index 5eea482387..1ee0a555d1 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -68,7 +68,7 @@ The Host marker has no configurable fields; platform providers own native config ## Understand the implementation -Import `CuaSessionPermissions` from `@deepseek-ai/dsh-cua/session-permissions`. Its `request(agent, permission, prompt, signal)` method asks for one named permission using the provider’s localized question and detail. A live Session caches each permission separately. Providers declare additional tool fields with their request schema and can prepare a move before dispatch. +Providers declare `systemPointer: true` to support explicit physical pointer movement. CUA exposes the option in the tool schema and model instructions only for those providers. CUA owns the approval card, live-session decision and delegated-task restriction. A provider receives the per-call pointer mode after CUA checks application access and any required consent. Permission responses identify their platform and report only the state that platform can observe. The shared page renders the controls those observations support. Before platform identification, it shows neutral observation and input rows without system actions. Native providers use the shared build lock and compiler cleanup. Standalone executables are retained by digest. The macOS provider reads its signed application receipt after compilation completes under the same lock. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index 2db56fd1b2..b8592b36eb 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -68,7 +68,7 @@ Computer Use 详情页的权限卡片、组件列表和来源信息使用 l3 边 ## 理解实现 -提供方从 `@deepseek-ai/dsh-cua/session-permissions` 导入 `CuaSessionPermissions`，通过 `request(agent, permission, prompt, signal)` 申请指定权限，并传入本地化的问题和说明。每个会话分别记录各项权限的决定。提供方通过请求 schema 声明额外字段，可在派发前处理悬停请求。 +提供方通过 `systemPointer: true` 声明支持真实鼠标移动。CUA 据此在工具 schema 和模型提示词中加入该选项，负责显示授权卡、记录当前会话的决定并限制子代理。CUA 检查应用访问权限和本次移动所需的授权后，将悬停方式传给提供方。 权限响应标识所属平台，并且只报告该平台能够观察到的状态。共享页面显示这些观察结果支持的控件。平台尚未确认时显示通用观察和输入项，不提供系统操作。原生提供方共用编译锁和编译进程清理逻辑。独立可执行文件按内容摘要保存。macOS 提供方在锁内等待编译完成，再读取签名应用的构建回执。 diff --git a/packages/experimental/cua/package.json b/packages/experimental/cua/package.json index 5d5c6bc0fb..440f321935 100644 --- a/packages/experimental/cua/package.json +++ b/packages/experimental/cua/package.json @@ -53,10 +53,6 @@ "types": "./lib/types/client/index.d.ts", "default": "./lib/client.js" }, - "./session-permissions": { - "types": "./lib/types/session-permissions.d.ts", - "default": "./lib/session-permissions.js" - }, "./config": { "types": "./lib/types/config.d.ts", "default": "./lib/config.js" @@ -82,7 +78,6 @@ "lib/api.js", "lib/bridge.js", "lib/native-client.js", - "lib/session-permissions.js", "lib/protocol.js", "lib/config.js", "lib/native-schema.js", diff --git a/packages/experimental/cua/src/client/ApplicationApproval.tsx b/packages/experimental/cua/src/client/ApplicationApproval.tsx index c55f3ead50..af577d3c2a 100644 --- a/packages/experimental/cua/src/client/ApplicationApproval.tsx +++ b/packages/experimental/cua/src/client/ApplicationApproval.tsx @@ -5,14 +5,14 @@ import type {} from '@deepseek-ai/dsh-client-ui-user-questions/client' import type {} from '@deepseek-ai/dsh-client-ui-approval/client' /** - * Select application access and provider Session permission questions; other interactions keep their renderer. + * Select application access and mouse-control questions; other interactions keep their renderer. * @param pending The session's currently elected interaction. * @returns Its Computer Use decision, or null for another interaction. */ export function selectApplicationApproval(pending: ComposerChainProps['pendingInteraction']) { if (pending?.kind !== 'question' || pending.questions.length !== 1) return null const item = pending.questions[0] as (typeof pending.questions)[number] - if ((item.id !== 'cua-app-access' && !item.id.startsWith('cua-session-permission:')) || item.intent?.kind !== 'approval') return null + if ((item.id !== 'cua-app-access' && item.id !== 'cua-mouse-control') || item.intent?.kind !== 'approval') return null return { pending, item, intent: item.intent } } diff --git a/packages/experimental/cua/src/config.ts b/packages/experimental/cua/src/config.ts index a570d4a93b..4e60c1e363 100644 --- a/packages/experimental/cua/src/config.ts +++ b/packages/experimental/cua/src/config.ts @@ -3,6 +3,7 @@ import Schema from '@deepseek-ai/schemastery' /** Shared configuration fields extended by native platform providers. */ export const CuaConfigFields = { + requireSystemPointerApproval: Schema.boolean().default(true), agentPreset: Schema.union([Schema.string().min(1), Schema.const(false)]).default('ptc'), allowedApps: Schema.array(Schema.object({ id: Schema.string().pattern(/^[a-f0-9]{64}$/).required(), diff --git a/packages/experimental/cua/src/guidance.ts b/packages/experimental/cua/src/guidance.ts index d8d7d46660..5e3f5cfc95 100644 --- a/packages/experimental/cua/src/guidance.ts +++ b/packages/experimental/cua/src/guidance.ts @@ -1,4 +1,6 @@ /** Shared model instructions for native application and window operations. */ +const hoverSignature = 'mouseMove(target: string | [number, number], options?: { relative?: boolean })' + /** Async method catalog with consent and observation rules, grouped by observation, mouse, and keyboard. */ export const COMMON_GUIDANCE = `Use cua to observe and operate native applications. Always use cua.getCapabilities() to confirm whether a specific API is available on the current platform; its reported capabilities take precedence over availability statements in these instructions. All methods are async; observation options default to { emit: true }, and emit: false suppresses sink output only, leaving returned values unchanged. Actions report dispatch evidence, not verified completion. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI actions and ask the user to unlock the desktop session; null or unknown means verify a usable unlocked desktop session first. cua.getApp requests user consent; if access is denied or revoked, stop using that application until the user explicitly permits another request, and never bypass a refusal or switch tools to avoid it. Never replay uncertain input without a fresh observation. @@ -7,7 +9,7 @@ In persistent PTC, use the host-provided bridge initialization once and retain t Signatures show resolved return values after await. ActionResult = { dispatched: boolean, foregroundChanged?: boolean | null }; foregroundChanged is omitted when not sampled and null when the sample is unavailable. JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; numbers must be finite. App is the application object with readonly app.appHandle: string (opaque) and the methods listed below for app level. Window is the window object with readonly window.windowId: number and the methods listed below for window level. Neither object carries pid. Setup and discovery: -- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. +- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -32,7 +34,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. +- ${hoverSignature} -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. @@ -46,3 +48,15 @@ Available at app or window level depending on the platform. - pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name.` + +/** + * Describe the system-pointer option for a provider that declares the capability. + * @param guidance Provider instructions including the shared operation catalog. + * @returns The capability-specific signature and permission instructions. + */ +export function withSystemPointerGuidance(guidance: string): string { + const signature = 'mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean })' + return `${guidance.replace(hoverSignature, signature)} + +Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). Prefer an advertised accessibility action, a click on a visible control, or browser tools when possible. The question interrupts the user. Pass true on every physical move, even after approval. Omitting the option or passing false uses the default hover route without prompting. Approval or refusal lasts until this session closes or DSH restarts. A refusal sends no input. If the user dismisses the request, ask again only if they request it. Wait for an outstanding request to finish. Subagents cannot request physical movement.` +} diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index 731ae38157..43ca13308e 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -13,11 +13,13 @@ import type { ContentBlock } from '@deepseek-ai/dsh-llm' import type { JsonValue } from '@deepseek-ai/dsh-util-values' import { z } from 'zod' import { NativeClient, NativeError, type NativeClientConfig, type NativeChannelFactory } from './native-client.ts' +import { MouseControl } from './mouse-control.ts' +import { withSystemPointerGuidance } from './guidance.ts' import { CuaPermissionStore } from './permission-store.ts' import { ApplicationAuthorizationResult, installCuaAccess } from './access.ts' import type { CuaApplicationGrant, CuaApplicationId, CuaSystemPermission } from './access-types.ts' import type { CuaAppHandle, CuaOperation, CuaStartupCapabilities } from './types.ts' -import { ApplicationIcon, AppResult, NativeScreenshotResult, Request, ScreenshotResult, StateResult, WindowsResult } from './protocol.ts' +import { ApplicationIcon, AppResult, NativeScreenshotResult, Request, SystemPointerRequest, ScreenshotResult, StateResult, WindowsResult } from './protocol.ts' import { CuaConfigFields } from './config.ts' import { CuaCoordinates } from './coordinates.ts' import { createScope, type Scope } from '@deepseek-ai/dsh-scope' @@ -34,6 +36,8 @@ export type { CuaStartupCapabilities } from './types.ts' /** Native executable and per-operation resource bounds. */ export interface CuaConfig { + /** Ask once per live Session before an explicitly requested system-pointer move. */ + requireSystemPointerApproval: boolean /** Register tools for Agents using this preset; defaults to ptc. False enables global registration. */ agentPreset: string | false /** Legacy grants imported once into profile-scoped storage; later edits are ignored. */ @@ -126,10 +130,6 @@ export interface CuaBackend { createChannel: NativeChannelFactory guidance: string capabilities: CuaStartupCapabilities - /** Model request schema including this provider's operation options. Defaults to the shared schema. */ - requestSchema?: z.ZodType - /** Resolve provider options on the parsed copy. An optional interactive step runs before authorized dispatch. */ - prepareMouseMove?: (agent: Agent, params: Record) => ((signal: AbortSignal) => Promise) | undefined /** Dispatch a directional key through the provider's execution-scoped input owner, after application authorization. */ sendKeyEvent?: (client: NativeClient, params: Record, execution: Readonly) => Promise @@ -151,7 +151,11 @@ export interface CuaBackend { */ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend): void { const { helperPath } = backend - const guidance = `${backend.guidance}\n\nMaximum click holdMs: ${config.maxClickHoldMs} ms.` + const systemPointer = backend.capabilities.systemPointer === true + const mouseControl = systemPointer ? new MouseControl(ctx, config.requireSystemPointerApproval) : undefined + const requestSchema = systemPointer ? SystemPointerRequest : Request + const instructions = systemPointer ? withSystemPointerGuidance(backend.guidance) : backend.guidance + const guidance = `${instructions}\n\nMaximum click holdMs: ${config.maxClickHoldMs} ms.` const native: NativeClientConfig = { helperPath, cwd: process.cwd(), timeoutMs: config.timeoutMs, axTimeoutMs: config.axTimeoutMs, connectTimeoutMs: config.connectTimeoutMs, screenshotTimeoutMs: config.screenshotTimeoutMs, @@ -163,7 +167,6 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) permissionParameters: backend.permissionParameters, store: new CuaPermissionStore(ctx, config.allowedApps.map(grant => ({ ...grant, id: grant.id as CuaApplicationId })), config.timeoutMs), }) - const requestSchema = backend.requestSchema ?? Request const clients = new Map() const lifetime = new AbortController() const pending = new Map, Agent>() @@ -212,13 +215,14 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) for (const result of [...await permissions, ...results, ...releases]) if (result.status === 'rejected') errors.push(result.reason) if (errors.length > 0) throw new AggregateError(errors, 'dsh-cua helper cleanup failed') } - const callNative = async (request: CuaOperation, execution: Readonly) => { + const callNative = async (request: CuaOperation, execution: Readonly, useSystemPointer?: boolean) => { const { method, params: parsedParams } = request const params = execution.coordinates.parameters(request) const agent = execution.agent const signal = execution.signal signal.throwIfAborted() const client = execution.nativeClient + if (useSystemPointer !== undefined) params.useSystemPointer = useSystemPointer if (method === 'sendKeyEvent' && backend.sendKeyEvent) return backend.sendKeyEvent(client, params, execution) const nativeParams = method === 'findApps' ? { ...params, maxAppMatches: config.maxAppMatches } @@ -282,7 +286,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) const capabilities = backend.capabilities return { ...runtime, - platform: capabilities.platform, + platform: capabilities.platform, systemPointer, methods: [...capabilities.methods], appLevelAccessibility: capabilities.appLevelAccessibility, keyboard: [...capabilities.keyboard], @@ -340,16 +344,18 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) owner.coordinates = new CuaCoordinates() } const nativeClient = client - const prepare = request.method === 'mouseMove' ? backend.prepareMouseMove?.(agent, request.params) : undefined - if (prepare) owner.coordinates.parameters(request) + const route = request.method === 'mouseMove' + ? mouseControl?.resolve(agent, 'useSystemPointer' in request.params && request.params.useSystemPointer === true) + : undefined + if (route?.prepare) owner.coordinates.parameters(request) const dispatch = async (signal: AbortSignal) => { const scoped = { ...execution, agent, signal, nativeClient, coordinates: owner.coordinates } - const result = await callNative(request, scoped) + const result = await callNative(request, scoped, route?.useSystemPointer) signal.throwIfAborted() return result } return 'appHandle' in request.params - ? access.run(nativeClient, request.params.appHandle as CuaAppHandle, signal, dispatch, prepare) + ? access.run(nativeClient, request.params.appHandle as CuaAppHandle, signal, dispatch, route?.prepare) : dispatch(signal) })() pending.set(operation, agent) diff --git a/packages/experimental/cua-macos/src/mouse-control-locales.ts b/packages/experimental/cua/src/mouse-control-locales.ts similarity index 100% rename from packages/experimental/cua-macos/src/mouse-control-locales.ts rename to packages/experimental/cua/src/mouse-control-locales.ts diff --git a/packages/experimental/cua/src/mouse-control.ts b/packages/experimental/cua/src/mouse-control.ts new file mode 100644 index 0000000000..866defca76 --- /dev/null +++ b/packages/experimental/cua/src/mouse-control.ts @@ -0,0 +1,63 @@ +/** Session approval for explicit physical pointer moves, shared by capable CUA providers. */ +import type { Context } from '@deepseek-ai/cordis' +import type { Agent } from '@deepseek-ai/dsh-agent' +import { UserQuestionError } from '@deepseek-ai/dsh-user-questions' +import { NativeError } from './native-client.ts' +import { applicationAccessDisplay } from './locales.ts' +import { mouseControlPrompt } from './mouse-control-locales.ts' + +/** Access is scoped to the live Session and never grants application access. */ +export class MouseControl { + private readonly decisions = new WeakMap() + + constructor(private readonly ctx: Context, private readonly requireApproval: boolean) {} + + /** + * Select the per-call route and any approval required before dispatch. + * @param agent Agent owning the live Session. + * @param requested Whether the caller explicitly requested system-pointer movement. + * @returns Native pointer mode and an optional preparation that must succeed before input. + */ + resolve(agent: Agent, requested: boolean): { useSystemPointer: boolean; prepare?: (signal: AbortSignal) => Promise } { + if (!requested) return { useSystemPointer: false } + if (agent.session.header.origin === 'subagent') { + throw new NativeError('MOUSE_CONTROL_UNAVAILABLE', "Delegated tasks cannot move the user's pointer. Use AX hover, click, or performAction. If the task requires a physical hover, report that step in your final result for the main task.") + } + if (!this.requireApproval || this.decisions.get(agent) === true) return { useSystemPointer: true } + return { useSystemPointer: true, prepare: signal => this.approve(agent, signal) } + } + + private async approve(agent: Agent, signal: AbortSignal): Promise { + signal.throwIfAborted() + const decision = this.decisions.get(agent) + if (decision === 'asking') throw new NativeError('MOUSE_CONTROL_PENDING', 'Wait for the outstanding mouse control request') + let granted = decision + if (granted === undefined) { + const questions = this.ctx.get('userQuestions') + if (!questions) throw new NativeError('MOUSE_APPROVAL_UNAVAILABLE', 'No human answerer is available for mouse control') + const id = 'cua-mouse-control' + const display = { ...mouseControlPrompt, approve: applicationAccessDisplay.approve } + const approve = display.approve.en + const reject = 'Deny' + this.decisions.set(agent, 'asking') + try { + const result = await questions.ask({ agent, signal, questions: [{ + id, question: display.question.en, detail: display.detail.en, options: [{ label: approve }, { label: reject }], + intent: { kind: 'approval', approve, reject, display }, + }] }) + signal.throwIfAborted() + const [answer] = result.answers + granted = result.answers.length === 1 && answer?.id === id && !answer.custom?.trim() + && answer.selected.length === 1 && answer.selected[0] === approve + this.decisions.set(agent, granted) + } catch (error) { + this.decisions.delete(agent) + if (error instanceof UserQuestionError && error.code === 'ASK_CANCELLED') { + throw new NativeError('MOUSE_APPROVAL_CANCELLED', 'The mouse control request was dismissed. Continue without mouse control unless the user asks to reconsider', { cause: error }) + } + throw error + } + } + if (!granted) throw new NativeError('MOUSE_CONTROL_DENIED', 'The user declined system-pointer movement for this session. No input was sent. Use pointer-free actions.') + } +} diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index 1fd9d6cd41..7684bb4aca 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -39,48 +39,57 @@ export const MouseMoveTargets = [ z.object(snapshot).strict(), z.object(absoluteCoordinate).strict(), z.object(relativeCoordinate).strict(), ] as const -/** Model-submittable operation parameters; Host resource fields and private continuations are separate. */ -export const Request = z.discriminatedUnion('method', [ - operation('getCapabilities', {}), operation('listApps', {}), - operation('findApps', { query: z.string().min(1, 'Expected a non-empty search string').regex(/\S/, 'Expected a non-empty search string') }), - operation('getApp', { selector: z.string().min(1) }), - operation('activate', app), - operation('show', { ...app, windowId: id.optional() }), - operation('listWindows', app), - operation('getState', { ...app, windowId: id.optional(), - includeMenus: z.boolean().optional().describe('Expand menu contents; omitted or false shows menu-bar entries only.'), - details: z.boolean().optional().describe('Include identifiers, global native bounds and complete action names.'), - }), - operation('listActions', snapshot), - operation('performAction', { ...snapshot, action: z.string().min(1), argument: jsonValue.optional() }), - operation('invoke', snapshot), - z.object({ method: z.literal('mouseMove'), params: z.union(MouseMoveTargets) }).strict(), - z.object({ method: z.literal('click'), params: z.union([ - z.object({ ...snapshot, ...click }).strict(), - z.object({ ...absoluteCoordinate, ...click }).strict(), - z.object({ ...relativeCoordinate, ...click }).strict(), - ]) }).strict(), - operation('setValue', { ...snapshot, value: jsonValue }), - operation('selectText', { ...snapshot, text: z.string().min(1), prefix: z.string().optional(), suffix: z.string().optional(), - selectionType: z.enum(['text', 'cursor_before', 'cursor_after']).optional() }), - operation('typeText', { ...app, windowId: id.optional(), text: z.string() }), - operation('paste', { ...app, windowId: id.optional(), text: z.string(), format: z.enum(['text', 'html']).optional() }), - operation('pressKey', { ...app, windowId: id.optional(), key: z.string().min(1), modifiers: z.array(z.string()).optional() }), - operation('sendKeyEvent', { ...app, windowId: id.optional(), key: z.string().min(1), direction: z.enum(['down', 'up']) }), - z.object({ method: z.literal('scroll'), params: z.union([ - z.object({ ...absoluteCoordinate, - deltaLinesX: z.number().int().min(-10).max(10), deltaLinesY: z.number().int().min(-10).max(10) }).strict(), - z.object({ ...relativeCoordinate, - deltaLinesX: z.number().int().min(-10).max(10), deltaLinesY: z.number().int().min(-10).max(10) }).strict(), - ]) }).strict(), - z.object({ method: z.literal('drag'), params: z.union([ - z.object({ ...window, ...pointer, snapshotId: z.string().min(1).optional(), relative: z.literal(false).optional(), - fromX: point, fromY: point, toX: point, toY: point }).strict(), - z.object({ ...window, ...pointer, snapshotId: z.string().min(1).optional(), relative: relativeCoordinate.relative, - fromX: relativePoint, fromY: relativePoint, toX: relativePoint, toY: relativePoint }).strict(), - ]) }).strict(), - operation('screenshot', window), -]) +function requestSchema(hover: T) { + return z.discriminatedUnion('method', [ + operation('getCapabilities', {}), operation('listApps', {}), + operation('findApps', { query: z.string().min(1, 'Expected a non-empty search string').regex(/\S/, 'Expected a non-empty search string') }), + operation('getApp', { selector: z.string().min(1) }), + operation('activate', app), + operation('show', { ...app, windowId: id.optional() }), + operation('listWindows', app), + operation('getState', { ...app, windowId: id.optional(), + includeMenus: z.boolean().optional().describe('Expand menu contents; omitted or false shows menu-bar entries only.'), + details: z.boolean().optional().describe('Include identifiers, global native bounds and complete action names.'), + }), + operation('listActions', snapshot), + operation('performAction', { ...snapshot, action: z.string().min(1), argument: jsonValue.optional() }), + operation('invoke', snapshot), + z.object({ method: z.literal('mouseMove'), params: hover }).strict(), + z.object({ method: z.literal('click'), params: z.union([ + z.object({ ...snapshot, ...click }).strict(), + z.object({ ...absoluteCoordinate, ...click }).strict(), + z.object({ ...relativeCoordinate, ...click }).strict(), + ]) }).strict(), + operation('setValue', { ...snapshot, value: jsonValue }), + operation('selectText', { ...snapshot, text: z.string().min(1), prefix: z.string().optional(), suffix: z.string().optional(), + selectionType: z.enum(['text', 'cursor_before', 'cursor_after']).optional() }), + operation('typeText', { ...app, windowId: id.optional(), text: z.string() }), + operation('paste', { ...app, windowId: id.optional(), text: z.string(), format: z.enum(['text', 'html']).optional() }), + operation('pressKey', { ...app, windowId: id.optional(), key: z.string().min(1), modifiers: z.array(z.string()).optional() }), + operation('sendKeyEvent', { ...app, windowId: id.optional(), key: z.string().min(1), direction: z.enum(['down', 'up']) }), + z.object({ method: z.literal('scroll'), params: z.union([ + z.object({ ...absoluteCoordinate, + deltaLinesX: z.number().int().min(-10).max(10), deltaLinesY: z.number().int().min(-10).max(10) }).strict(), + z.object({ ...relativeCoordinate, + deltaLinesX: z.number().int().min(-10).max(10), deltaLinesY: z.number().int().min(-10).max(10) }).strict(), + ]) }).strict(), + z.object({ method: z.literal('drag'), params: z.union([ + z.object({ ...window, ...pointer, snapshotId: z.string().min(1).optional(), relative: z.literal(false).optional(), + fromX: point, fromY: point, toX: point, toY: point }).strict(), + z.object({ ...window, ...pointer, snapshotId: z.string().min(1).optional(), relative: relativeCoordinate.relative, + fromX: relativePoint, fromY: relativePoint, toX: relativePoint, toY: relativePoint }).strict(), + ]) }).strict(), + operation('screenshot', window), + ]) +} + +/** Portable model operations, including auxiliary hover without a system-pointer option. */ +export const Request = requestSchema(z.union(MouseMoveTargets)) + +/** Model operations for providers that can move the user's system pointer. */ +export const SystemPointerRequest = requestSchema(z.union(MouseMoveTargets.map(target => target.extend({ + useSystemPointer: z.boolean().optional().describe('Request use of the system pointer. The Host checks permission before dispatch.'), +})))) /** Resource parameters supplied by the Host, never accepted from model tool arguments. */ export const NativeResourceParameters = { @@ -139,6 +148,8 @@ export const CapabilitiesResult = z.object({ platform: z.string(), permissions: z.object({ accessibility: z.boolean(), screenRecording: z.boolean() }), /** Whether the provider supports application-level accessibility observations. */ appLevelAccessibility: z.boolean(), + /** Whether mouseMove accepts an explicit request to move the system pointer. */ + systemPointer: z.boolean().default(false), /** Scopes that support keyboard input. */ keyboard: z.array(z.enum(['app', 'window'])), sessionLocked: z.boolean().nullable(), diff --git a/packages/experimental/cua/src/session-permissions.ts b/packages/experimental/cua/src/session-permissions.ts deleted file mode 100644 index f21a61b685..0000000000 --- a/packages/experimental/cua/src/session-permissions.ts +++ /dev/null @@ -1,54 +0,0 @@ -/** Provider-owned permissions remembered separately for each live Session and requested capability. */ -import type { Context } from '@deepseek-ai/cordis' -import type { Agent } from '@deepseek-ai/dsh-agent' -import { UserQuestionError } from '@deepseek-ai/dsh-user-questions' -import { applicationAccessDisplay } from './locales.ts' - -type Localized = { en: string; [locale: string]: string } - -/** A provider owns the lifetime and names of the permissions it requests. */ -export class CuaSessionPermissions { - private readonly decisions = new WeakMap>() - - constructor(private readonly ctx: Context) {} - - /** - * Remember the user's choice only for this permission in this live Session. - * @param agent Live Session owner. - * @param permission Provider-defined permission name shared by calls requesting the same access. - * @param prompt Localized question and scope displayed before the user decides. - * @param signal Calling operation lifetime. Aborting or dismissing the question leaves the permission undecided. - * @returns True only when the user selects session approval. - */ - async request(agent: Agent, permission: string, prompt: { question: Localized; detail: Localized }, - signal: AbortSignal): Promise { - signal.throwIfAborted() - const decisions = this.decisions.get(agent) ?? new Map() - const decision = decisions.get(permission) - if (decision === 'asking') throw new UserQuestionError('Wait for the outstanding permission request', 'APPROVAL_PENDING') - if (decision !== undefined) return decision - const questions = this.ctx.get('userQuestions') - if (!questions) throw new UserQuestionError('No human answerer is available for the permission request', 'NO_PROVIDER') - this.decisions.set(agent, decisions) - decisions.set(permission, 'asking') - const id = `cua-session-permission:${permission}` - const display = { ...prompt, approve: applicationAccessDisplay.approve } - const approve = display.approve.en - const reject = 'Deny' - try { - const result = await questions.ask({ agent, signal, questions: [{ - id, question: prompt.question.en, detail: prompt.detail.en, options: [{ label: approve }, { label: reject }], - intent: { kind: 'approval', approve, reject, display }, - }] }) - signal.throwIfAborted() - const [answer] = result.answers - const granted = result.answers.length === 1 && answer?.id === id && !answer.custom?.trim() - && answer.selected.length === 1 && answer.selected[0] === approve - decisions.set(permission, granted) - return granted - } catch (error) { - decisions.delete(permission) - throw error - } - } -} diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index b84663f9e2..98c7c7a1a2 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -2,7 +2,7 @@ import type { Branded } from '@deepseek-ai/dsh-brand' import type { JsonValue } from '@deepseek-ai/dsh-util-values' import type { z } from 'zod' -import type { ActionResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, WindowsResult } from './protocol.ts' +import type { ActionResult, CapabilitiesResult, FoundAppsResult, HoverResult, SystemPointerRequest, ScreenshotResult, ShowResult, WindowsResult } from './protocol.ts' /** Helper-issued application selection; public app objects hold only this opaque handle. */ export type CuaAppHandle = Branded<'CuaAppHandle'> @@ -11,7 +11,7 @@ export type CuaSnapshotId = Branded<'CuaSnapshotId'> /** Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot. */ export type CuaElementId = Branded<'CuaElementId'> /** One operation sent through an authorized and logged host dispatcher. */ -export type CuaOperation = z.output +export type CuaOperation = z.output type CuaParameters = Extract['params'] /** Dispatch exactly once; a failed action must not be retried implicitly. */ export type CuaDispatch = (operation: CuaOperation) => Promise @@ -49,7 +49,7 @@ export type CuaCapabilities = z.output export type CuaStartupCapabilities = Pick< CuaCapabilities, 'platform' | 'methods' | 'appLevelAccessibility' | 'keyboard' | 'browser' | 'coordinateSpace' -> +> & { systemPointer?: boolean } /** App discovery result accepted by getApp. */ export type CuaAppInfo = Pick, 'id' | 'displayName' | 'isRunning' | 'pid'> /** An installed application match; running matches include a process identifier. */ @@ -196,10 +196,12 @@ export interface CuaWindow extends CuaSurface { * Returning and subsequent observations do not restore the previous pointer position. * Other input and application changes can interrupt the interaction; observe before continuing. * @param target - current element id or screenshot-local [x,y]. - * @param options - coordinate mode; applies only to [x,y]. + * @param options - Coordinate mode for points. Only providers reporting systemPointer accept useSystemPointer. + * True requests physical movement for this call only. * @returns dispatch evidence and the selected route; observe the resulting UI after any UI delay. */ - mouseMove(target: CuaElementId | readonly [number, number], options?: CuaCoordinateOptions): Promise + mouseMove(target: CuaElementId | readonly [number, number], + options?: CuaCoordinateOptions & { useSystemPointer?: boolean }): Promise /** * Click a current element id or screenshot-local point. * Cancellation or timeout releases the held button; received input cannot be undone. The operation deadline includes holdMs. diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index 73dbf044cc..9b9a58c64c 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -13,7 +13,7 @@ const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG const bounds = { x: -640, y: 100, width: 640, height: 480 } const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', bundleId: 'test.editor', running: true } const delivered = { dispatched: true } -const capabilities: CuaCapabilities = { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility: true, keyboard: ['app'], methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false } +const capabilities: CuaCapabilities = { platform: 'macos', systemPointer: true, sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility: true, keyboard: ['app'], methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false } function windowInfo(windowId: number) { return { windowId, appName: 'Editor', title: `Document ${windowId}`, bounds, isOnScreen: true, onCurrentSpace: null } diff --git a/packages/experimental/cua/tests/application-approval.client.spec.tsx b/packages/experimental/cua/tests/application-approval.client.spec.tsx index 48f4cadafd..13e79b8faa 100644 --- a/packages/experimental/cua/tests/application-approval.client.spec.tsx +++ b/packages/experimental/cua/tests/application-approval.client.spec.tsx @@ -91,7 +91,7 @@ it('shows the mouse interruption and expiry with only session approval and refus detail: { en: 'Until restart', zh: '重启后失效' }, approve: { en: 'Allow for this session', zh: '本会话允许' }, } - const pending = new PendingQuestion('s1' as SessionId, [{ id: 'cua-session-permission:example', question: display.question.en, + const pending = new PendingQuestion('s1' as SessionId, [{ id: 'cua-mouse-control', question: display.question.en, detail: display.detail.en, intent: { kind: 'approval', approve: display.approve.en, reject: 'Deny', display } }]) const factory = vi.fn((_name: string, _input: ApprovalCardInput) => null) render( brandString(id) const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC' const capabilities: CuaCapabilities = { - platform: 'macos', permissions: { accessibility: true, screenRecording: true }, appLevelAccessibility: true, + platform: 'macos', systemPointer: true, permissions: { accessibility: true, screenRecording: true }, appLevelAccessibility: true, keyboard: ['app'], sessionLocked: false, methods: ['getCapabilities', 'getApp', 'getState', 'screenshot'], coordinateSpace: 'screenshot-pixels', browser: false, } diff --git a/packages/experimental/cua/tests/fixtures/exports.mjs b/packages/experimental/cua/tests/fixtures/exports.mjs index 91b55b76b8..19dbd3973e 100644 --- a/packages/experimental/cua/tests/fixtures/exports.mjs +++ b/packages/experimental/cua/tests/fixtures/exports.mjs @@ -12,8 +12,6 @@ const guard = registerHooks({ }) const { Context } = await import('@deepseek-ai/cordis') const host = await import('@deepseek-ai/dsh-cua') -const { CuaSessionPermissions } = await import('@deepseek-ai/dsh-cua/session-permissions') -assert.equal(typeof CuaSessionPermissions.prototype.request, 'function') const { createCuaApi } = await import('@deepseek-ai/dsh-cua/api') const { CuaConfigFields } = await import('@deepseek-ai/dsh-cua/config') assert.equal(typeof CuaConfigFields.connectTimeoutMs.default, 'function') diff --git a/packages/experimental/cua/tests/protocol.spec.ts b/packages/experimental/cua/tests/protocol.spec.ts index 50fb329c07..1842ac1f78 100644 --- a/packages/experimental/cua/tests/protocol.spec.ts +++ b/packages/experimental/cua/tests/protocol.spec.ts @@ -179,7 +179,7 @@ it.each([true, false])('requires the reported application-level accessibility ca platform: 'macos', permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility, keyboard: ['app'], sessionLocked: false, methods: [], coordinateSpace: 'screenshot-pixels', browser: false, } - expect(CapabilitiesResult.parse(capabilities)).toEqual(capabilities) + expect(CapabilitiesResult.parse(capabilities)).toEqual({ ...capabilities, systemPointer: false }) const { appLevelAccessibility: _omitted, ...missing } = capabilities expect(CapabilitiesResult.safeParse(missing).success).toBe(false) }) @@ -195,7 +195,7 @@ it.each([ platform: 'windows', permissions: { accessibility: true, screenRecording: true }, appLevelAccessibility: false, keyboard, sessionLocked: false, methods: [], coordinateSpace: 'screenshot-pixels', browser: false, } - expect(CapabilitiesResult.parse(capabilities)).toEqual(capabilities) + expect(CapabilitiesResult.parse(capabilities)).toEqual({ ...capabilities, systemPointer: false }) }) it.each([ diff --git a/packages/experimental/cua/tsdown.config.ts b/packages/experimental/cua/tsdown.config.ts index 92c9eca9a8..29526fddd6 100644 --- a/packages/experimental/cua/tsdown.config.ts +++ b/packages/experimental/cua/tsdown.config.ts @@ -3,7 +3,6 @@ import { clientBundle } from '../../client/tsdown.client.ts' const runtime: UserConfig[] = Object.entries({ 'remote-types': 'lib/types/remote-types.js', api: 'lib/types/api.js', bridge: 'lib/types/bridge.js', - 'session-permissions': 'lib/types/session-permissions.js', 'native-client': 'lib/types/native-client.js', protocol: 'lib/types/protocol.js', config: 'lib/types/config.js', 'native-schema': 'lib/types/native-schema.js', 'native-build': 'lib/types/native-build.js', guidance: 'lib/types/guidance.js', }).map(([name, path]) => ({ diff --git a/scripts/check-workspace-constraints.ts b/scripts/check-workspace-constraints.ts index 8e75b08fbd..f759eca7ec 100644 --- a/scripts/check-workspace-constraints.ts +++ b/scripts/check-workspace-constraints.ts @@ -188,7 +188,7 @@ const packageFileExtras: Readonly> = { // The embedded OAR implementation retains its upstream license. '@deepseek-ai/dsh-experimental-subagent-oar': ['LICENSE-OAR'], // CUA publishes its client API and attributed native source for local macOS builds. - '@deepseek-ai/dsh-cua': ['lib/remote-types.js', 'lib/api.js', 'lib/bridge.js', 'lib/native-client.js', 'lib/session-permissions.js', 'lib/protocol.js', 'lib/config.js', 'lib/native-schema.js', 'lib/native-build.js', 'lib/guidance.js'], + '@deepseek-ai/dsh-cua': ['lib/remote-types.js', 'lib/api.js', 'lib/bridge.js', 'lib/native-client.js', 'lib/protocol.js', 'lib/config.js', 'lib/native-schema.js', 'lib/native-build.js', 'lib/guidance.js'], '@deepseek-ai/dsh-experimental-cua-macos': ['native/bin', 'cua-helper.json', 'native/Package.swift', 'native/Info.plist', 'native/DSHComputerUse.icns', 'native/build_app.py', 'native/Sources', 'native/tests/swift', 'native/tests/test_swift.py', 'native/LICENSE.cua-driver', 'native/LICENSE.computer-use-icon', 'native/upstream.json', 'native/README.md', 'native/README.zh.md', 'native/protocol.schema.json'], '@deepseek-ai/dsh-webhook-github': ['examples/github-review/cordis.yml', 'examples/github-review/github-ready-review-rule.mjs'], // Liveshare publishes its built Relay gateway entry and the shared runtime diff --git a/tsconfig.base.json b/tsconfig.base.json index 58c5ce6e40..e8ca4f8e44 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -113,7 +113,6 @@ "@deepseek-ai/dsh-experimental-computer-use-cua-driver-mcp": ["./packages/experimental/computer-use-cua-driver-mcp/src/index.ts"], "@deepseek-ai/dsh-experimental-cua-macos": ["./packages/experimental/cua-macos/src/index.ts"], "@deepseek-ai/dsh-cua/config": ["./packages/experimental/cua/src/config.ts"], - "@deepseek-ai/dsh-cua/session-permissions": ["./packages/experimental/cua/src/session-permissions.ts"], "@deepseek-ai/dsh-cua/native-schema": ["./packages/experimental/cua/src/native-schema.ts"], "@deepseek-ai/dsh-cua/native-build": ["./packages/experimental/cua/src/native-build.ts"], "@deepseek-ai/dsh-cua/guidance": ["./packages/experimental/cua/src/guidance.ts"], From cbba0c66a970ee88484dd0f3603a41f28acc52c9 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 18:20:34 +0800 Subject: [PATCH 25/30] test(cua): verify pointer capabilities and portable option forwarding --- packages/experimental/cua-macos/README.md | 2 +- packages/experimental/cua-macos/README.zh.md | 2 +- packages/experimental/cua-macos/src/guidance.ts | 2 +- .../cua-macos/tests/protocol-generation.spec.ts | 11 +---------- .../cua-macos/tests/provider-lifecycle.spec.ts | 2 +- packages/experimental/cua/src/guidance.ts | 2 +- packages/experimental/cua/src/protocol.ts | 2 +- packages/experimental/cua/tests/api.spec.ts | 4 ++-- packages/experimental/cua/tests/protocol.spec.ts | 11 ++++++++++- snapshots/session/cua-native/session.v4.jsonl | 2 +- .../session/cua-native/system-prompt.expected.md | 6 ++++-- snapshots/session/cua-ptc/system-prompt.expected.md | 6 ++++-- .../ptc-script-retained/system-prompt.expected.md | 4 ++-- 13 files changed, 30 insertions(+), 26 deletions(-) diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 7b5df96219..e20c639e4c 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -228,7 +228,7 @@ Available at app or window level depending on the platform. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. -Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). Prefer an advertised accessibility action, a click on a visible control, or browser tools when possible. The question interrupts the user. Pass true on every physical move, even after approval. Omitting the option or passing false uses the default hover route without prompting. Approval or refusal lasts until this session closes or DSH restarts. A refusal sends no input. If the user dismisses the request, ask again only if they request it. Wait for an outstanding request to finish. Subagents cannot request physical movement. +Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). Prefer an advertised accessibility action, a click on a visible control, or browser tools when possible. The question interrupts the user. When approval is disabled an explicit move can act immediately. Pass true on every physical move, even after approval. Omitting the option or passing false uses the default hover route without prompting. Approval or refusal lasts until this session closes or DSH restarts. A refusal sends no input. If the user dismisses the request, ask again only if they request it. Wait for an outstanding request to finish. Subagents cannot request physical movement. ``` #### Token effect diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index 8db5bee329..83177e6047 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -228,7 +228,7 @@ Available at app or window level depending on the platform. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. -Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). Prefer an advertised accessibility action, a click on a visible control, or browser tools when possible. The question interrupts the user. Pass true on every physical move, even after approval. Omitting the option or passing false uses the default hover route without prompting. Approval or refusal lasts until this session closes or DSH restarts. A refusal sends no input. If the user dismisses the request, ask again only if they request it. Wait for an outstanding request to finish. Subagents cannot request physical movement. +Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). Prefer an advertised accessibility action, a click on a visible control, or browser tools when possible. The question interrupts the user. When approval is disabled an explicit move can act immediately. Pass true on every physical move, even after approval. Omitting the option or passing false uses the default hover route without prompting. Approval or refusal lasts until this session closes or DSH restarts. A refusal sends no input. If the user dismisses the request, ask again only if they request it. Wait for an outstanding request to finish. Subagents cannot request physical movement. ``` #### Token 影响 diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index 0428941edb..68243fdaab 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -10,7 +10,7 @@ const methodDetails = new Map([ ['click', 'click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying.'], ]) -/** Complete model instructions for this provider. */ +/** Native operation instructions; the shared Host adds guidance for declared capabilities. */ export const GUIDANCE = `${PLATFORM_GUIDANCE} ${COMMON_GUIDANCE.replace(/^- ([\w.]+)\(.*$/gm, (entry, method: string) => { diff --git a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts index c83c16e77e..36c6dd94c9 100644 --- a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts +++ b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts @@ -3,7 +3,7 @@ import { readFile } from 'node:fs/promises' import { Ajv2020 } from 'ajv/dist/2020.js' import { describe, expect, it } from 'vitest' import { renderNativeProtocol } from '../native/generate-protocol.ts' -import { Request, SystemPointerRequest } from '@deepseek-ai/dsh-cua/protocol' +import { Request } from '@deepseek-ai/dsh-cua/protocol' describe('native protocol generation', () => { it('keeps native JSON Schema and Swift admission tables current', async () => { @@ -28,15 +28,6 @@ describe('native protocol generation', () => { expect(validate({ id: 'shortcut', ...request })).toBe(false) }) - it('admits the pointer option only in the system-pointer model schema and only as a boolean', () => { - const target = { appHandle: 'app', windowId: 1, x: 0, y: 0 } - const request = { method: 'mouseMove', params: { ...target, useSystemPointer: true } } - expect(Request.safeParse(request).success).toBe(false) - expect(SystemPointerRequest.safeParse(request).success).toBe(true) - expect(SystemPointerRequest.safeParse({ ...request, params: { ...target, useSystemPointer: 'true' } }).success).toBe(false) - expect(SystemPointerRequest.safeParse({ ...request, params: { ...target, arbitraryField: true } }).success).toBe(false) - }) - it.each([ { method: 'getState', params: { appHandle: 'app', maxElements: 1 } }, { method: 'drag', params: { appHandle: 'app', windowId: 1, fromX: 1, fromY: 1, toX: 2, toY: 2, steps: 1 } }, diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index 343bbbdb33..5e7b5893f6 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -1135,7 +1135,7 @@ it.each([true, false])('uses the declared systemPointer capability for a non-mac expect((await f.execute('getCapabilities')).value).toMatchObject({ platform: 'fixture', systemPointer }) const sections = (await f.ctx.systemPrompt.assemble({ scope: f.handle.agent })).sections const guidance = sections.find(section => section.name === 'computer-use:dsh-cua')!.text - expect(guidance.includes('useSystemPointer')).toBe(systemPointer) + expect(guidance.includes('relative?: boolean, useSystemPointer?: boolean')).toBe(systemPointer) expect(guidance.includes('The question interrupts the user')).toBe(systemPointer) expect(JSON.stringify(f.ctx.tools.get('cua', f.handle.agent)!.parameters).includes('useSystemPointer')).toBe(systemPointer) await selectApplication(f) diff --git a/packages/experimental/cua/src/guidance.ts b/packages/experimental/cua/src/guidance.ts index 5e3f5cfc95..49caa08bf2 100644 --- a/packages/experimental/cua/src/guidance.ts +++ b/packages/experimental/cua/src/guidance.ts @@ -58,5 +58,5 @@ export function withSystemPointerGuidance(guidance: string): string { const signature = 'mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean })' return `${guidance.replace(hoverSignature, signature)} -Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). Prefer an advertised accessibility action, a click on a visible control, or browser tools when possible. The question interrupts the user. Pass true on every physical move, even after approval. Omitting the option or passing false uses the default hover route without prompting. Approval or refusal lasts until this session closes or DSH restarts. A refusal sends no input. If the user dismisses the request, ask again only if they request it. Wait for an outstanding request to finish. Subagents cannot request physical movement.` +Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). Prefer an advertised accessibility action, a click on a visible control, or browser tools when possible. The question interrupts the user. When approval is disabled an explicit move can act immediately. Pass true on every physical move, even after approval. Omitting the option or passing false uses the default hover route without prompting. Approval or refusal lasts until this session closes or DSH restarts. A refusal sends no input. If the user dismisses the request, ask again only if they request it. Wait for an outstanding request to finish. Subagents cannot request physical movement.` } diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index 7684bb4aca..3bedc404d1 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -83,7 +83,7 @@ function requestSchema(hover: T) { ]) } -/** Portable model operations, including auxiliary hover without a system-pointer option. */ +/** Portable model operations, including standard hover without a system-pointer option. */ export const Request = requestSchema(z.union(MouseMoveTargets)) /** Model operations for providers that can move the user's system pointer. */ diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index 9b9a58c64c..f71b958368 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -901,10 +901,10 @@ it('forwards provider options while retaining the selected target and coordinate appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', elementId: '0', trackingMode: 'precise', } }) await win.getScreenshot({ emit: false }) - const pixelOptions = { relative: false, trackingMode: 'precise', x: 222 } + const pixelOptions = { relative: false, useSystemPointer: true, trackingMode: 'precise', x: 222 } await win.mouseMove([30, 40], pixelOptions) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'mouseMove', params: { - appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: 23, x: 30, y: 40, relative: false, trackingMode: 'precise', + appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: 23, x: 30, y: 40, relative: false, useSystemPointer: true, trackingMode: 'precise', } }) const relativeOptions = { relative: true, trackingMode: 'coarse' } await win.mouseMove([250, 750], relativeOptions) diff --git a/packages/experimental/cua/tests/protocol.spec.ts b/packages/experimental/cua/tests/protocol.spec.ts index 1842ac1f78..f259df5fcd 100644 --- a/packages/experimental/cua/tests/protocol.spec.ts +++ b/packages/experimental/cua/tests/protocol.spec.ts @@ -1,7 +1,7 @@ /** Model-authored operation JSON cannot supply host configuration or bypass retained identities. */ import { expect, it } from 'vitest' import type { JsonValue } from '@deepseek-ai/dsh-util-values' -import { CapabilitiesResult, Request, ScreenshotResult } from '../src/protocol.ts' +import { CapabilitiesResult, Request, ScreenshotResult, SystemPointerRequest } from '../src/protocol.ts' const rejectedRequests: Array<{ reason: string; request: JsonValue }> = [ { reason: 'raw process ID instead of a selected app handle', request: { method: 'typeText', params: { pid: 17, text: 'hello' } } }, @@ -217,3 +217,12 @@ it('requires keyboard scopes', () => { } expect(CapabilitiesResult.safeParse(capabilities).success).toBe(false) }) + +it('admits the pointer option only in the system-pointer model schema and only as a boolean', () => { + const target = { appHandle: 'app', windowId: 1, x: 0, y: 0 } + const request = { method: 'mouseMove', params: { ...target, useSystemPointer: true } } + expect(Request.safeParse(request).success).toBe(false) + expect(SystemPointerRequest.safeParse(request).success).toBe(true) + expect(SystemPointerRequest.safeParse({ ...request, params: { ...target, useSystemPointer: 'true' } }).success).toBe(false) + expect(SystemPointerRequest.safeParse({ ...request, params: { ...target, arbitraryField: true } }).success).toBe(false) +}) diff --git a/snapshots/session/cua-native/session.v4.jsonl b/snapshots/session/cua-native/session.v4.jsonl index 7482438890..db5e234368 100644 --- a/snapshots/session/cua-native/session.v4.jsonl +++ b/snapshots/session/cua-native/session.v4.jsonl @@ -14,7 +14,7 @@ {"type":"session/title","data":{"title":"Inspect CUA capabilities, select the","messageSeqs":[8],"source":{"kind":"fallback"}}} {"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:4}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032889225,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032889225,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}}},{"type":"chunk","time":1794032889225,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032889225,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":1,"callId":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}} -{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-1"},"toolCallId":"cua-native-1","content":[{"type":"text","text":"{\"sessionLocked\":false,\"permissions\":{\"accessibility\":true,\"screenRecording\":true},\"platform\":\"macos\",\"methods\":[\"activate\",\"click\",\"drag\",\"findApps\",\"getApp\",\"getCapabilities\",\"getState\",\"invoke\",\"listActions\",\"listApps\",\"listWindows\",\"mouseMove\",\"paste\",\"performAction\",\"pressKey\",\"screenshot\",\"scroll\",\"selectText\",\"sendKeyEvent\",\"setValue\",\"show\",\"typeText\"],\"appLevelAccessibility\":true,\"keyboard\":[\"app\"],\"browser\":false,\"coordinateSpace\":\"screenshot-pixels\"}"}],"isError":false,"id":"{{message:5}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"} +{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-1"},"toolCallId":"cua-native-1","content":[{"type":"text","text":"{\"sessionLocked\":false,\"permissions\":{\"accessibility\":true,\"screenRecording\":true},\"platform\":\"macos\",\"systemPointer\":true,\"methods\":[\"activate\",\"click\",\"drag\",\"findApps\",\"getApp\",\"getCapabilities\",\"getState\",\"invoke\",\"listActions\",\"listApps\",\"listWindows\",\"mouseMove\",\"paste\",\"performAction\",\"pressKey\",\"screenshot\",\"scroll\",\"selectText\",\"sendKeyEvent\",\"setValue\",\"show\",\"typeText\"],\"appLevelAccessibility\":true,\"keyboard\":[\"app\"],\"browser\":false,\"coordinateSpace\":\"screenshot-pixels\"}"}],"isError":false,"id":"{{message:5}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":1}} {"type":"step/start","data":{"turn":1,"step":2}} {"type":"assistant/message","data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-unauthorized-getState","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:6}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794032889794,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794032889794,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-unauthorized-getState","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1794032889794,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794032889794,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index a663884957..9589c20d8e 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -42,7 +42,7 @@ In persistent PTC, use the host-provided bridge initialization once and retain t Signatures show resolved return values after await. ActionResult = { dispatched: boolean, foregroundChanged?: boolean | null }; foregroundChanged is omitted when not sampled and null when the sample is unavailable. JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; numbers must be finite. App is the application object with readonly app.appHandle: string (opaque) and the methods listed below for app level. Window is the window object with readonly window.windowId: number and the methods listed below for window level. Neither object carries pid. Setup and discovery: -- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. +- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -67,7 +67,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user. Approval moves their pointer to this target. Pass useSystemPointer: true on every physical move, even after approval. Omit it or pass false for AX hover. Further explicit moves reuse approval until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. With requireSystemPointerApproval disabled, explicit moves do not prompt. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. +- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without that route, HOVER_UNAVAILABLE is returned before sending input. Physical movement requires an unobscured foreground target. Explain any necessary activation and observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. @@ -83,4 +83,6 @@ Available at app or window level depending on the platform. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. +Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). Prefer an advertised accessibility action, a click on a visible control, or browser tools when possible. The question interrupts the user. When approval is disabled an explicit move can act immediately. Pass true on every physical move, even after approval. Omitting the option or passing false uses the default hover route without prompting. Approval or refusal lasts until this session closes or DSH restarts. A refusal sends no input. If the user dismisses the request, ask again only if they request it. Wait for an outstanding request to finish. Subagents cannot request physical movement. + Maximum click holdMs: 5000 ms. diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index 9b9d570d03..6888ad0481 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -44,7 +44,7 @@ In persistent PTC, use the host-provided bridge initialization once and retain t Signatures show resolved return values after await. ActionResult = { dispatched: boolean, foregroundChanged?: boolean | null }; foregroundChanged is omitted when not sampled and null when the sample is unavailable. JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; numbers must be finite. App is the application object with readonly app.appHandle: string (opaque) and the methods listed below for app level. Window is the window object with readonly window.windowId: number and the methods listed below for window level. Neither object carries pid. Setup and discovery: -- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. +- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -69,7 +69,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user. Approval moves their pointer to this target. Pass useSystemPointer: true on every physical move, even after approval. Omit it or pass false for AX hover. Further explicit moves reuse approval until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. With requireSystemPointerApproval disabled, explicit moves do not prompt. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. +- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without that route, HOVER_UNAVAILABLE is returned before sending input. Physical movement requires an unobscured foreground target. Explain any necessary activation and observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. @@ -85,6 +85,8 @@ Available at app or window level depending on the platform. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. +Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). Prefer an advertised accessibility action, a click on a visible control, or browser tools when possible. The question interrupts the user. When approval is disabled an explicit move can act immediately. Pass true on every physical move, even after approval. Omitting the option or passing false uses the default hover route without prompting. Approval or refusal lasts until this session closes or DSH restarts. A refusal sends no input. If the user dismisses the request, ask again only if they request it. Wait for an outstanding request to finish. Subagents cannot request physical movement. + Maximum click holdMs: 5000 ms. ## Writing code for run_code diff --git a/snapshots/session/ptc-script-retained/system-prompt.expected.md b/snapshots/session/ptc-script-retained/system-prompt.expected.md index e2fb062dc6..c670f443b9 100644 --- a/snapshots/session/ptc-script-retained/system-prompt.expected.md +++ b/snapshots/session/ptc-script-retained/system-prompt.expected.md @@ -44,7 +44,7 @@ In persistent PTC, use the host-provided bridge initialization once and retain t Signatures show resolved return values after await. ActionResult = { dispatched: boolean, foregroundChanged?: boolean | null }; foregroundChanged is omitted when not sampled and null when the sample is unavailable. JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; numbers must be finite. App is the application object with readonly app.appHandle: string (opaque) and the methods listed below for app level. Window is the window object with readonly window.windowId: number and the methods listed below for window level. Neither object carries pid. Setup and discovery: -- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. +- cua.getCapabilities(options?: { emit?: boolean }) -> { platform: string, permissions: { accessibility: boolean, screenRecording: boolean }, appLevelAccessibility: boolean, systemPointer: boolean, keyboard: ('app' | 'window')[], sessionLocked: boolean | null, methods: string[], coordinateSpace: 'screenshot-pixels', browser: false } — inspect reported permissions, sessionLocked, methods, and keyboard scopes without requesting permissions. - cua.listApps(options?: { emit?: boolean }) -> { id: string, displayName: string, isRunning: true, pid: number }[] — enumerate running applications. - cua.findApps(query: string, options?: { emit?: boolean }) -> ({ id: string, displayName: string, path: string } & ({ isRunning: true, pid: number } | { isRunning: false }))[] — search installed applications, including stopped matches; only running matches carry pid. - cua.getState(options?: { emit?: boolean }) -> { apps: { id: string, displayName: string, isRunning: true, pid: number }[], browsers: [] } — summarize running applications; this is not an accessibility observation. @@ -69,7 +69,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. - getScreenshot(options?: { emit?: boolean }) -> { bytes: Uint8Array, mimeType: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif', pixelWidth: number, pixelHeight: number, windowId: number } — capture the selected window as one encoded image; bytes is a JavaScript byte array, and mimeType identifies its encoding, so do not assume PNG. -- mouseMove(target: string | [number, number], options?: { relative?: boolean, useSystemPointer?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. HOVER_UNAVAILABLE sends no input and never opens a question. Use an advertised accessibility action, click a visible control, or use browser tools for web content whenever possible. Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). The question interrupts the user. Approval moves their pointer to this target. Pass useSystemPointer: true on every physical move, even after approval. Omit it or pass false for AX hover. Further explicit moves reuse approval until the session closes or DSH restarts. A refusal sends no input; continue with pointer-free actions. If the question is dismissed, ask again only when the user requests it. Await an outstanding request before making another. With requireSystemPointerApproval disabled, explicit moves do not prompt. Subagents cannot request physical moves. System movement requires an unobscured foreground target; explain any necessary activation and observe the target again. +- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without that route, HOVER_UNAVAILABLE is returned before sending input. Physical movement requires an unobscured foreground target. Explain any necessary activation and observe the target again. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. - drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window. From bafc363d780705ec674abb95eb93917f7d9dbb23 Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Mon, 9 Nov 2026 10:59:09 +0800 Subject: [PATCH 26/30] fix(worktree): reclaim archived checkouts in large repositories Archive cleanup read whole Git listings into the 64 KiB maxOutputBytes buffer, so every clean checkout in a repository with more than about a thousand tracked files settled as failed. - Drop the skip-worktree/assume-unchanged and gitlink listings; removal follows what `git status` reports, as peer agent products do. - Stream `git status` and stop at its first entry, and stream the worktree listing, so neither is bounded by maxOutputBytes; restoring a worktree shares the streamed lookup. - Retain a checkout whose administrative directory holds submodule repositories instead of letting Git refuse the removal. Refs #8151 --- docs/config-catalog.md | 7 +- docs/config-catalog.zh.md | 7 +- .../archive-worktrees/guide.i18n.yaml | 8 +- .../v0.2.2-rc.1/archive-worktrees/guide.md | 4 +- .../v0.2.2-rc.1/archive-worktrees/guide.zh.md | 4 +- .../experimental/worktree/README.i18n.yaml | 4 +- packages/experimental/worktree/README.md | 2 +- packages/experimental/worktree/README.zh.md | 2 +- packages/experimental/worktree/src/index.ts | 32 +++-- packages/experimental/worktree/src/process.ts | 123 ++++++++++++++---- packages/experimental/worktree/src/removal.ts | 81 +++++++----- .../worktree/tests/archive.spec.ts | 38 +++++- .../worktree/tests/removal.spec.ts | 54 ++++++-- .../worktree/tests/worktree.spec.ts | 16 +++ 14 files changed, 290 insertions(+), 92 deletions(-) diff --git a/docs/config-catalog.md b/docs/config-catalog.md index d6284aee3c..678d7387d2 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1561,7 +1561,7 @@ export type PaidTranslationProvider = 'deepseek-account' | 'deepseek-official' - `inject`: `workingDirectory` · `fs` · `subprocess` · `sandbox` · `sandboxPolicy` · `sessionProjections` - `refs`: `Volatile` (`@deepseek-ai/cordis`) -- `source`: [`packages/experimental/worktree/src/index.ts:35`](../packages/experimental/worktree/src/index.ts) +- `source`: [`packages/experimental/worktree/src/index.ts:36`](../packages/experimental/worktree/src/index.ts) ```ts config-catalog /** Worktree location, generated names, executables, subprocess resource limits, and the archive cleanup switch. */ @@ -1580,7 +1580,10 @@ export interface Config { timeoutMs?: number /** Subprocess termination grace in milliseconds. */ graceMs?: number - /** Maximum captured bytes per subprocess output stream. */ + /** + * Maximum captured bytes per subprocess output stream; archive inspection + * streams `git status` and the worktree listing, so it bounds only their stderr. + */ maxOutputBytes?: number } ``` diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index 70e75f8142..b1707d7c32 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -1563,7 +1563,7 @@ export type PaidTranslationProvider = 'deepseek-account' | 'deepseek-official' - `inject`: `workingDirectory` · `fs` · `subprocess` · `sandbox` · `sandboxPolicy` · `sessionProjections` - `refs`: `Volatile` (`@deepseek-ai/cordis`) -- `source`: [`packages/experimental/worktree/src/index.ts:35`](../packages/experimental/worktree/src/index.ts) +- `source`: [`packages/experimental/worktree/src/index.ts:36`](../packages/experimental/worktree/src/index.ts) ```ts config-catalog /** Worktree location, generated names, executables, subprocess resource limits, and the archive cleanup switch. */ @@ -1582,7 +1582,10 @@ export interface Config { timeoutMs?: number /** Subprocess termination grace in milliseconds. */ graceMs?: number - /** Maximum captured bytes per subprocess output stream. */ + /** + * Maximum captured bytes per subprocess output stream; archive inspection + * streams `git status` and the worktree listing, so it bounds only their stderr. + */ maxOutputBytes?: number } ``` diff --git a/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.i18n.yaml b/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.i18n.yaml index b563eb7323..c17fd006c0 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.i18n.yaml +++ b/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.i18n.yaml @@ -9,8 +9,8 @@ en: bc309764ab7b8b98 zh: 20fdfbce76706025 /worktree-cleanup-on-archive/change: - en: 05f27e166f2dbb40 - zh: a320ac6252fd2e38 + en: c16e3db7ee76eee8 + zh: a1db1e346d5af41a /worktree-cleanup-on-archive/migration: - en: c9b2fca77d02e07f - zh: a4228d7f22fb7e35 + en: 05f29b760cb99ae9 + zh: 33f181ac2a582bdf diff --git a/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.md b/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.md index 77912a2e71..9232c6ecdd 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.md @@ -9,13 +9,13 @@ English | [中文](guide.zh.md) ## Change -Archiving a Session previously retained its checkout. The worktree service now records each checkout it creates as a `plugin:worktree/created` record and, when the Session that created one is archived, removes it with a non-force `git worktree remove` when no unarchived Session refers to it, its creator's policy is not read-only, and Git finds no tracked changes, untracked files, hidden index flags, submodules, locks, or detached HEAD. Ignored files such as `node_modules` do not retain it. Branches and commits remain. The service's `retainOnArchive` setting, off by default, keeps every checkout instead. Checkouts created before this release have no such event and are never swept, nor are Sessions archived earlier. See the [worktree service](../../../../packages/experimental/worktree/README.md). +Archiving a Session previously retained its checkout. The worktree service now records each checkout it creates as a `plugin:worktree/created` record and, when the Session that created one is archived, removes it with a non-force `git worktree remove` when no unarchived Session refers to it, its creator's policy is not read-only, and Git finds no tracked changes, untracked files, submodule repositories, locks, or detached HEAD. Ignored files such as `node_modules` and edits to skip-worktree or assume-unchanged files, which `git status` hides, do not retain it; Git discards them. Branches and commits remain. The service's `retainOnArchive` setting, off by default, keeps every checkout instead. Checkouts created before this release have no such event and are never swept, nor are Sessions archived earlier. See the [worktree service](../../../../packages/experimental/worktree/README.md). The Forge bundle mounts one `worktree` service for both the panel and the `create_worktree` tool. The `forge-worktree` row and the `optional-forge-panel` group that isolated a second service are gone; the panel is the top-level `forge-panel` row. ## Migration -1. If a checkout must remain available outside DSH, keep its Session unarchived, lock it with `git worktree lock ` before archiving, or turn on **Keep worktrees when archiving** on the Forge bundle's worktree row of the Plugins page (`retainOnArchive`), which keeps every checkout. Ignored files are not kept. +1. If a checkout must remain available outside DSH, keep its Session unarchived, lock it with `git worktree lock ` before archiving, or turn on **Keep worktrees when archiving** on the Forge bundle's worktree row of the Plugins page (`retainOnArchive`), which keeps every checkout. Ignored files are not kept, nor are edits to skip-worktree or assume-unchanged files: commit those edits, or clear the flags with `git update-index --no-skip-worktree` or `--no-assume-unchanged`, before archiving. 2. To resume an archived Session whose checkout was removed, unarchive it and use **Restore worktree**. The retained branch is checked out at the recorded path. 3. In a profile's `cordis.patch.yml`, move any `forge-worktree` override to the `worktree` row and any `optional-forge-panel` override to the `forge-panel` row; a `disabled: true` on `forge-panel` turns off the panel alone. 4. Confirm the branch and commits are present with `git branch --list` and that **Restore worktree** makes the recorded checkout available. Session logs require no migration. diff --git a/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.zh.md b/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.zh.md index 5edf966298..0d32dded1c 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.zh.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.zh.md @@ -9,13 +9,13 @@ description: "归档 Session 可以删除其干净的工作树并保留分支， ## 变更 -归档 Session 以前会保留检出目录。工作树服务现在把它创建的每个检出目录记录为 `plugin:worktree/created` Session 事件；当创建它的 Session 被归档时，若没有未归档的 Session 引用该目录、创建者的策略不是只读，且 Git 未发现跟踪文件改动、未跟踪文件、隐藏的索引标记、子模块、锁或分离 HEAD，就用不带 force 的 `git worktree remove` 删除它。`node_modules` 之类被忽略的文件不会保留目录。分支和提交仍会保留。服务的 `retainOnArchive` 设置默认关闭，开启后改为保留所有检出目录。本版本之前创建的检出目录没有该事件，不会被扫描；更早归档的 Session 也不会。参见[工作树服务](../../../../packages/experimental/worktree/README.zh.md)。 +归档 Session 以前会保留检出目录。工作树服务现在把它创建的每个检出目录记录为 `plugin:worktree/created` Session 事件；当创建它的 Session 被归档时，若没有未归档的 Session 引用该目录、创建者的策略不是只读，且 Git 未发现跟踪文件改动、未跟踪文件、子模块仓库、锁或分离 HEAD，就用不带 force 的 `git worktree remove` 删除它。`node_modules` 之类被忽略的文件，以及 `git status` 不报告的 skip-worktree 或 assume-unchanged 文件改动，都不会保留目录，Git 会丢弃它们。分支和提交仍会保留。服务的 `retainOnArchive` 设置默认关闭，开启后改为保留所有检出目录。本版本之前创建的检出目录没有该事件，不会被扫描；更早归档的 Session 也不会。参见[工作树服务](../../../../packages/experimental/worktree/README.zh.md)。 Forge bundle 为面板和 `create_worktree` 工具挂载同一个 `worktree` 服务。隔离第二个服务的 `forge-worktree` 行与 `optional-forge-panel` 组已移除；面板是顶层的 `forge-panel` 行。 ## 迁移 -1. 如果检出目录必须供 DSH 之外继续使用，请保持其 Session 未归档，在归档前用 `git worktree lock ` 锁定它，或在插件页 Forge 套件的 worktree 行上开启**归档时不删除工作树**（`retainOnArchive`），它会保留所有检出目录。被忽略的文件不会保留。 +1. 如果检出目录必须供 DSH 之外继续使用，请保持其 Session 未归档，在归档前用 `git worktree lock ` 锁定它，或在插件页 Forge 套件的 worktree 行上开启**归档时不删除工作树**（`retainOnArchive`），它会保留所有检出目录。被忽略的文件不会保留，skip-worktree 或 assume-unchanged 文件的改动也不会：归档前请提交这些改动，或用 `git update-index --no-skip-worktree` 或 `--no-assume-unchanged` 清除标记。 2. 要继续使用已归档且检出目录已删除的 Session，请取消归档并使用**恢复工作树**。保留的分支会在记录的路径重新检出。 3. 在 profile 的 `cordis.patch.yml` 中，把针对 `forge-worktree` 的覆盖迁到 `worktree` 行，把针对 `optional-forge-panel` 的覆盖迁到 `forge-panel` 行；对 `forge-panel` 设置 `disabled: true` 可以只关闭面板。 4. 用 `git branch --list` 确认分支与提交仍然存在，并确认**恢复工作树**使记录的检出目录重新可用。Session 日志无需迁移。 diff --git a/packages/experimental/worktree/README.i18n.yaml b/packages/experimental/worktree/README.i18n.yaml index c964c25be1..788a8dfda7 100644 --- a/packages/experimental/worktree/README.i18n.yaml +++ b/packages/experimental/worktree/README.i18n.yaml @@ -15,8 +15,8 @@ en: d152484eb41ac6b4 zh: 2f8a621b991260a1 /deepseek-ai-dsh-experimental-worktree/use-this-package: - en: c788af94d52dd001 - zh: 200580d1a97048eb + en: 56af10241dc270ee + zh: 2519395ab9641235 /deepseek-ai-dsh-experimental-worktree/understand-the-implementation: en: 6971a7ca9073bef1 zh: b977c28a75c7f963 diff --git a/packages/experimental/worktree/README.md b/packages/experimental/worktree/README.md index 756ea6803f..4c394a6d2a 100644 --- a/packages/experimental/worktree/README.md +++ b/packages/experimental/worktree/README.md @@ -36,7 +36,7 @@ Mount the service alongside `workingDirectory`, `fs`, `subprocess`, `sandbox`, ` `ctx.worktrees.restore(agent, { path, branch, baseCommit, repositoryRoot }, signal?, onPhase?)` checks a removed worktree out again at its recorded directory. It validates the recorded repository root, full commit object name, branch, and canonical target inside the configured pool before mutation. It removes only the missing target's unlocked registration with `git worktree remove`; other worktrees and locked registrations remain unchanged. It then checks out the branch when it still exists, or creates the branch again at `baseCommit` when it does not, and reports which happened as `branchRecreated`. An existing directory is an error. The calling Session's working directory does not change, and changes that were never committed are not recovered. Checkout disables filters as creation does. `onPhase` receives `prepare` before validation, target-registration cleanup, and branch creation, and `checkout` before the files are written; it must not throw. -Every creation appends a `plugin:worktree/created` record to the Session log after the directory change, so the `worktree` projection lists the Session's checkouts with `owned` false for one inside a fork-inherited prefix. When the Workspace archives a Session, the service registers one `scheduled` cleanup for the first checkout that Session created itself and returns; the inspection runs in the background. While `retainOnArchive` is on, archiving registers nothing and every checkout stays; the field is volatile and read at each archive, and the [Forge panel](../client-ui-forge/README.md) edits it as a switch on the bundle's worktree row of the Plugins page. The checkout is retained while any unarchived Session refers to it, by a recorded creation or by a working directory inside it (a Session beneath an archived subagent ancestor counts as archived; cold Sessions are read from the projection cache or their log), and while the creator's sandbox policy is read-only. An eligible checkout is removed with a non-force `git worktree remove` under the creator's policy after a second use check inside the path lock, which also holds `ensure` and `set` into that path through `working-directory/access`; the branch and its commits remain. Tracked changes, untracked files, hidden index flags, submodules, locks, detached HEAD, a mismatched branch, and targets outside the configured pool retain the checkout; ignored files do not, as Git itself discards them. Processes are not consulted: a process still writing there makes Git refuse, and an idle one fails its next command. Unarchiving cancels an unadmitted removal and waits for an admitted one. `cleanupOf(session, revision)` reads the attempt, or waits for a named revision to settle, as `scheduled`, `retained`, `removed`, or `failed`; a failure leaves the Session archived and logs a warning. Checkouts created before this event existed, and Sessions archived earlier, are not swept. +Every creation appends a `plugin:worktree/created` record to the Session log after the directory change, so the `worktree` projection lists the Session's checkouts with `owned` false for one inside a fork-inherited prefix. When the Workspace archives a Session, the service registers one `scheduled` cleanup for the first checkout that Session created itself and returns; the inspection runs in the background. While `retainOnArchive` is on, archiving registers nothing and every checkout stays; the field is volatile and read at each archive, and the [Forge panel](../client-ui-forge/README.md) edits it as a switch on the bundle's worktree row of the Plugins page. The checkout is retained while any unarchived Session refers to it, by a recorded creation or by a working directory inside it (a Session beneath an archived subagent ancestor counts as archived; cold Sessions are read from the projection cache or their log), and while the creator's sandbox policy is read-only. An eligible checkout is removed with a non-force `git worktree remove` under the creator's policy after a second use check inside the path lock, which also holds `ensure` and `set` into that path through `working-directory/access`; the branch and its commits remain. Tracked changes, untracked files, submodule repositories (Git refuses to remove those), locks, detached HEAD, a mismatched branch, and targets outside the configured pool retain the checkout. Ignored files and edits to skip-worktree or assume-unchanged files do not: `git status` reports neither, and Git discards them with the checkout. Inspection reads `git status` only up to its first entry and streams the worktree listing, so neither is bounded by `maxOutputBytes`. Processes are not consulted: a process still writing there makes Git refuse, and an idle one fails its next command. Unarchiving cancels an unadmitted removal and waits for an admitted one. `cleanupOf(session, revision)` reads the attempt, or waits for a named revision to settle, as `scheduled`, `retained`, `removed`, or `failed`; a failure leaves the Session archived and logs a warning. Checkouts created before this event existed, and Sessions archived earlier, are not swept. The default checkout directory is `/.agents/worktrees/`. The returned `repositoryRoot` is the top-level directory of the caller’s current checkout; when it is a linked checkout, the pool is nested inside it. Omitted names join the `dsh-` prefix, the first eight hexadecimal digits of the UUID in the Session id, a hyphen, and four random hexadecimal digits, such as `dsh-6b1f02c4-9a3e`; a Session id without a UUID gets eight random hexadecimal digits after the prefix. `generatedWorktreeName` exports the same rule for callers that name a worktree before creating it. The service writes a `.gitignore` containing `*` and a trailing newline only when its allocation creates the pool; existing files and ignore rules are preserved. Each Git and Node command runs under its own `timeoutMs` deadline, 5 minutes by default, so checking out a large repository on a slow filesystem can finish; cancellation still stops the running command. Relative `directory`, generated `namePrefix`, executable choices, process limits, and `retainOnArchive` are configurable in the [configuration catalog](../../../docs/config-catalog.md#deepseek-aidsh-experimental-worktree). diff --git a/packages/experimental/worktree/README.zh.md b/packages/experimental/worktree/README.zh.md index fa3624a49c..262ef39480 100644 --- a/packages/experimental/worktree/README.zh.md +++ b/packages/experimental/worktree/README.zh.md @@ -36,7 +36,7 @@ kind: "package-reference" `ctx.worktrees.restore(agent, { path, branch, baseCommit, repositoryRoot }, signal?, onPhase?)` 在记录的目录重新检出被删除的 worktree。它在修改前校验记录中的仓库根目录、完整提交对象名、分支和配置池内的规范目标路径。它只通过 `git worktree remove` 移除缺失目标的未锁定登记；其他 worktree 与锁定登记保持不变。随后在分支仍存在时检出该分支，否则在 `baseCommit` 重新创建同名分支，并通过 `branchRecreated` 报告是哪种情况。目录已存在时会报错。调用方 Session 的工作目录不变，从未提交的修改无法恢复。检出与创建时一样禁用过滤器。`onPhase` 在检查目录、清理登记与创建分支之前收到 `prepare`，在写入文件之前收到 `checkout`；它不能抛出异常。 -每次创建都会在目录切换之后向 Session 日志追加一条 `plugin:worktree/created` 事件，因此 `worktree` 投影列出该 Session 的检出目录，位于 fork 继承前缀内的条目 `owned` 为 false。Workspace 归档某个 Session 时，服务为该 Session 自己创建的第一个检出目录登记一条 `scheduled` 清理并立即返回；检查在后台进行。`retainOnArchive` 开启时，归档不登记任何清理，所有检出目录都保留；该字段是 volatile 字段，每次归档时读取，[Forge 面板](../client-ui-forge/README.zh.md)在插件页该套件的 worktree 行上以开关编辑它。只要任一未归档 Session 仍引用该目录——通过记录的创建，或通过位于其中的工作目录（祖先子代理已归档的 Session 视为已归档；冷 Session 从投影缓存或其日志读取）——或创建者的沙箱策略为只读，目录就保留。符合条件的目录在路径锁内再做一次使用检查后，用创建者的策略执行不带 force 的 `git worktree remove`；路径锁同时通过 `working-directory/access` 阻塞进入该路径的 `ensure` 与 `set`；分支及其提交保留。跟踪文件改动、未跟踪文件、隐藏的索引标记、子模块、锁、分离 HEAD、分支不匹配以及位于配置池之外的目标都会保留目录；被忽略的文件不会，因为 Git 自身会丢弃它们。不检查进程：仍在写入的进程会让 Git 拒绝删除，空闲进程的下一条命令会失败。取消归档会取消尚未获准的删除，并等待已获准的删除结束。`cleanupOf(session, revision)` 读取本次尝试，或等待指定修订号结束，状态为 `scheduled`、`retained`、`removed` 或 `failed`；失败时 Session 仍保持归档并记录警告。此事件存在之前创建的检出目录，以及更早归档的 Session，不会被扫描。 +每次创建都会在目录切换之后向 Session 日志追加一条 `plugin:worktree/created` 事件，因此 `worktree` 投影列出该 Session 的检出目录，位于 fork 继承前缀内的条目 `owned` 为 false。Workspace 归档某个 Session 时，服务为该 Session 自己创建的第一个检出目录登记一条 `scheduled` 清理并立即返回；检查在后台进行。`retainOnArchive` 开启时，归档不登记任何清理，所有检出目录都保留；该字段是 volatile 字段，每次归档时读取，[Forge 面板](../client-ui-forge/README.zh.md)在插件页该套件的 worktree 行上以开关编辑它。只要任一未归档 Session 仍引用该目录——通过记录的创建，或通过位于其中的工作目录（祖先子代理已归档的 Session 视为已归档；冷 Session 从投影缓存或其日志读取）——或创建者的沙箱策略为只读，目录就保留。符合条件的目录在路径锁内再做一次使用检查后，用创建者的策略执行不带 force 的 `git worktree remove`；路径锁同时通过 `working-directory/access` 阻塞进入该路径的 `ensure` 与 `set`；分支及其提交保留。跟踪文件改动、未跟踪文件、子模块仓库（Git 拒绝删除这类目录）、锁、分离 HEAD、分支不匹配以及位于配置池之外的目标都会保留目录。被忽略的文件以及对 skip-worktree 或 assume-unchanged 文件的改动不会：`git status` 不报告这两类内容，Git 删除目录时一并丢弃它们。检查只读取 `git status` 的第一条记录，并以流式读取工作树列表，因此两者都不受 `maxOutputBytes` 限制。不检查进程：仍在写入的进程会让 Git 拒绝删除，空闲进程的下一条命令会失败。取消归档会取消尚未获准的删除，并等待已获准的删除结束。`cleanupOf(session, revision)` 读取本次尝试，或等待指定修订号结束，状态为 `scheduled`、`retained`、`removed` 或 `failed`；失败时 Session 仍保持归档并记录警告。此事件存在之前创建的检出目录，以及更早归档的 Session，不会被扫描。 默认检出目录是 `/.agents/worktrees/`。返回的 `repositoryRoot` 是调用方当前检出目录的顶层目录；如果它是链接检出目录，工作树父目录就嵌套在其中。省略名称时，名称由前缀 `dsh-`、Session id 中 UUID 的前 8 位十六进制数、一个连字符和 4 位随机十六进制数组成，例如 `dsh-6b1f02c4-9a3e`；Session id 不含 UUID 时，前缀后接 8 位随机十六进制数。`generatedWorktreeName` 导出同一规则，供在创建前确定名称的调用方使用。仅当服务的分配操作创建了父目录时，服务才会写入内容为 `*` 加换行符的 `.gitignore`；已有文件与忽略规则保持不变。每条 Git 和 Node 命令各自受 `timeoutMs` 时限约束，默认 5 分钟，使大仓库在较慢的文件系统上也能完成检出；取消仍会终止正在运行的命令。相对路径 `directory`、生成名称使用的 `namePrefix`、可执行程序、进程限制与 `retainOnArchive` 可在[配置目录](../../../docs/config-catalog.zh.md#deepseek-aidsh-experimental-worktree)中查阅。 diff --git a/packages/experimental/worktree/src/index.ts b/packages/experimental/worktree/src/index.ts index 3c74e3f73c..683798a5d4 100644 --- a/packages/experimental/worktree/src/index.ts +++ b/packages/experimental/worktree/src/index.ts @@ -15,11 +15,12 @@ import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' import { WorktreeReclamation } from './archive.ts' import { PREPARE_DIRECTORY } from './directory.ts' import { generatedWorktreeName } from './name.ts' -import { runCommand } from './process.ts' +import { runCommand, scanCommand } from './process.ts' import type { ProcessConfig } from './process.ts' import { CREATED_RECORD, worktreeProjection } from './projection.ts' import type { WorktreeProjectionState } from './projection.ts' import { checkoutIdentity, registration, removeCheckout } from './removal.ts' +import type { GitScan } from './removal.ts' import type { CreatedWorktree, CreateWorktreePhase, CreateWorktreeRequest, RestoredWorktree, RestoreWorktreeRequest, WorktreeCleanup, } from './types.ts' @@ -47,7 +48,10 @@ export interface Config { timeoutMs?: number /** Subprocess termination grace in milliseconds. */ graceMs?: number - /** Maximum captured bytes per subprocess output stream. */ + /** + * Maximum captured bytes per subprocess output stream; archive inspection + * streams `git status` and the worktree listing, so it bounds only their stderr. + */ maxOutputBytes?: number } @@ -220,8 +224,9 @@ export class WorktreeService extends Service { ): Promise { const control = AbortSignal.any([this.lifetime.signal, signal]) control.throwIfAborted() - const run = this.gitRunner(this.ctx.sandboxPolicy.resolve({ session }), control) - const operation = removeCheckout(this.ctx, this.config.directory, request, run, control, admit) + const policy = this.ctx.sandboxPolicy.resolve({ session }) + const git = { read: this.gitRunner(policy, control), scan: this.gitScanner(policy, control) } + const operation = removeCheckout(this.ctx, this.config.directory, request, git, control, admit) this.pending.add(operation) try { return await operation } finally { this.pending.delete(operation) } } @@ -296,7 +301,8 @@ export class WorktreeService extends Service { if (identity === null) throw new Error('worktree restoration must use a canonical directory inside the configured pool of the source checkout root') await run(repositoryRoot, ['check-ref-format', `refs/heads/${branch}`]) await run(repositoryRoot, ['rev-parse', '--verify', '--end-of-options', `${baseCommit}^{commit}`]) - const fields = await registration(this.ctx, { path, branch, baseCommit, repositoryRoot }, identity.target, run, signal) + const scan = this.gitScanner(policy, signal) + const fields = await registration(this.ctx, { path, branch, baseCommit, repositoryRoot }, identity.target, scan, signal) if (fields !== undefined) { if (fields.some(field => field === 'locked' || field.startsWith('locked '))) throw new Error(`worktree is locked: ${path}`) if (await this.ctx.fs.lstat(path, undefined, signal) !== undefined) throw new Error(`worktree path already exists: ${path}`) @@ -320,15 +326,23 @@ export class WorktreeService extends Service { } } - /** Confine Git commands to the caller's sandbox policy, with hooks and the filesystem monitor disabled. */ + /** Confine Git commands to the caller's sandbox policy, collecting bounded stdout. */ private gitRunner(policy: SandboxExecutionPolicy, signal: AbortSignal): GitRun { return (directory, args, allowedExitCodes, env) => runCommand( - this.ctx, this.config, policy, directory, - [this.config.gitCommand, '-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', ...args], - signal, allowedExitCodes, env, + this.ctx, this.config, policy, directory, this.gitArgv(args), signal, allowedExitCodes, env, ) } + /** Confine Git commands to the caller's sandbox policy, streaming stdout instead of collecting it. */ + private gitScanner(policy: SandboxExecutionPolicy, signal: AbortSignal): GitScan { + return (directory, args, onField) => scanCommand(this.ctx, this.config, policy, directory, this.gitArgv(args), signal, onField) + } + + /** The configured Git executable with hooks and the filesystem monitor disabled, followed by `args`. */ + private gitArgv(args: string[]): [string, ...string[]] { + return [this.config.gitCommand, '-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', ...args] + } + /** Allocate the checkout directory exclusively inside the pool, creating a self-ignoring pool when it is new. */ private async prepareDirectory( policy: SandboxExecutionPolicy, diff --git a/packages/experimental/worktree/src/process.ts b/packages/experimental/worktree/src/process.ts index 74c8fa1c1b..94846759c1 100644 --- a/packages/experimental/worktree/src/process.ts +++ b/packages/experimental/worktree/src/process.ts @@ -1,8 +1,11 @@ /** Bounded, cancellable commands under the calling Session's existing file policy. */ +import type { Readable } from 'node:stream' import type { Context } from '@deepseek-ai/cordis' import type { SandboxExecutionPolicy } from '@deepseek-ai/dsh-sandbox' -import type { SubprocessCollectedOutputs } from '@deepseek-ai/dsh-subprocess' +import type { + SubprocessCollect, SubprocessCollectedOutputs, SubprocessHandle, SubprocessOutcome, SubprocessOutputReader, +} from '@deepseek-ai/dsh-subprocess' import { deadline } from '@deepseek-ai/dsh-timeout' /** Resolved process resource limits owned by the worktree service. */ @@ -46,39 +49,109 @@ export async function runCommand( env: NodeJS.ProcessEnv = {}, ): Promise<{ stdout: string; exitCode: number }> { using timeout = deadline(signal, config.timeoutMs, 'WORKTREE_TIMEOUT') - timeout.signal.throwIfAborted() - const program = await ctx.subprocess.resolveExecutable(argv[0], undefined, timeout.signal) - const command = [program, ...argv.slice(1)] - const wrapped = policy.mode === 'danger-full-access' - ? command - : (await ctx.sandbox.confine(command, { ...policy, mode: policy.mode }, timeout.signal)).argv - timeout.signal.throwIfAborted() - const child = ctx.subprocess.spawn({ - argv: wrapped, - cwd, - env: { ...GIT_ENV, GIT_TERMINAL_PROMPT: '0', GIT_LFS_SKIP_SMUDGE: '1', GIT_NO_LAZY_FETCH: '1', ...env }, - signal: timeout.signal, - graceMs: config.graceMs, - stdio: { - stdin: 'ignore', - stdout: { maxBytes: config.maxOutputBytes }, - stderr: { maxBytes: config.maxOutputBytes }, - }, - }) + const child = await spawnCommand(ctx, config, policy, cwd, argv, timeout.signal, env, { maxBytes: config.maxOutputBytes }) try { const outcome = await child.done timeout.signal.throwIfAborted() // The spawn spec above collects both streams. const captured = child.collected as Required + const exitCode = allowedExitCode(outcome, captured.stderr, allowedExitCodes) const stdout = captured.stdout.readFrom(0) - const stderr = captured.stderr.readFrom(0) - if (outcome.exitCode === null || !allowedExitCodes.includes(outcome.exitCode)) { - throw new Error(`worktree command failed (${outcome.signal ?? outcome.exitCode}): ${stderr.text.trim()}`) - } if (stdout.lossy) throw new Error('worktree command output exceeds maxOutputBytes') - return { stdout: stdout.text, exitCode: outcome.exitCode } + return { stdout: stdout.text, exitCode } } finally { child.terminate() await child.waitForExit() } } + +/** + * Run an argv as {@link runCommand} does, but pass each NUL-terminated stdout + * field to `onField` as it arrives instead of retaining the output, so the size + * of stdout is unbounded and `maxOutputBytes` limits only stderr. An + * unterminated remainder is ignored. A false return from `onField` terminates + * the command, whose exit status is then not checked; otherwise only status 0 succeeds. + * @param ctx - subprocess and sandbox providers in the same execution world. + * @param config - complete process resource limits. + * @param policy - unchanged calling Session policy. + * @param cwd - canonical process working directory. + * @param argv - executable followed by literal arguments. + * @param signal - operation cancellation. + * @param onField - synchronous consumer of each field in order; false ends the command. + * @returns true when `onField` ended the command, false when it exited successfully first. + */ +export async function scanCommand( + ctx: Context, + config: ProcessConfig, + policy: SandboxExecutionPolicy, + cwd: string, + argv: readonly [string, ...string[]], + signal: AbortSignal, + onField: (field: string) => boolean, +): Promise { + using timeout = deadline(signal, config.timeoutMs, 'WORKTREE_TIMEOUT') + const child = await spawnCommand(ctx, config, policy, cwd, argv, timeout.signal, {}, 'pipe') + try { + // The spawn spec above pipes stdout and collects stderr. + const stdout = child.stdout as Readable + stdout.setEncoding('utf8') + let pending = '' + let stopped = false + for await (const chunk of stdout as AsyncIterable) { + const fields = (pending + chunk).split('\0') + pending = fields.splice(-1).join('') + if (!fields.every(field => onField(field))) { + stopped = true + break + } + } + if (stopped) child.terminate() + const outcome = await child.done + timeout.signal.throwIfAborted() + if (!stopped) allowedExitCode(outcome, child.collected.stderr as SubprocessOutputReader, [0]) + return stopped + } finally { + child.terminate() + await Promise.allSettled([child.done]) + await child.waitForExit() + } +} + +/** Resolve, confine, and start one command whose stderr is collected within `maxOutputBytes`. */ +async function spawnCommand( + ctx: Context, + config: ProcessConfig, + policy: SandboxExecutionPolicy, + cwd: string, + argv: readonly [string, ...string[]], + signal: AbortSignal, + env: NodeJS.ProcessEnv, + stdout: 'pipe' | SubprocessCollect, +): Promise { + signal.throwIfAborted() + const program = await ctx.subprocess.resolveExecutable(argv[0], undefined, signal) + const command = [program, ...argv.slice(1)] + const wrapped = policy.mode === 'danger-full-access' + ? command + : (await ctx.sandbox.confine(command, { ...policy, mode: policy.mode }, signal)).argv + signal.throwIfAborted() + return ctx.subprocess.spawn({ + argv: wrapped, + cwd, + env: { ...GIT_ENV, GIT_TERMINAL_PROMPT: '0', GIT_LFS_SKIP_SMUDGE: '1', GIT_NO_LAZY_FETCH: '1', ...env }, + signal, + graceMs: config.graceMs, + stdio: { stdin: 'ignore', stdout, stderr: { maxBytes: config.maxOutputBytes } }, + }) +} + +/** + * The exit code of a command that exited with an allowed status. + * @throws when the command was signalled or exited with another status, quoting its stderr. + */ +function allowedExitCode(outcome: SubprocessOutcome, stderr: SubprocessOutputReader, allowedExitCodes: readonly number[]): number { + if (outcome.exitCode === null || !allowedExitCodes.includes(outcome.exitCode)) { + throw new Error(`worktree command failed (${outcome.signal ?? outcome.exitCode}): ${stderr.readFrom(0).text.trim()}`) + } + return outcome.exitCode +} diff --git a/packages/experimental/worktree/src/removal.ts b/packages/experimental/worktree/src/removal.ts index f9e4082bf7..bcec543e6c 100644 --- a/packages/experimental/worktree/src/removal.ts +++ b/packages/experimental/worktree/src/removal.ts @@ -3,9 +3,24 @@ import type { Context } from '@deepseek-ai/cordis' import type {} from '@deepseek-ai/dsh-fs' import type { RestoreWorktreeRequest } from './types.ts' -/** Runs one Git command in the owner's confinement and reports its output. */ +/** Runs one Git command in the owner's confinement and reports its complete bounded output. */ export type GitRead = (directory: string, args: string[]) => Promise<{ stdout: string; exitCode: number }> +/** + * Runs one Git command in the owner's confinement and passes each + * NUL-terminated stdout field to `onField` without retaining the output; + * false from `onField` ends the command. Resolves true when `onField` ended it. + */ +export type GitScan = (directory: string, args: string[], onField: (field: string) => boolean) => Promise + +/** The confined Git runners inspection and removal use. */ +export interface GitCommands { + /** Commands whose output stays small, read whole. */ + readonly read: GitRead + /** Commands whose output grows with the repository or its worktrees, read as it arrives. */ + readonly scan: GitScan +} + /** Resolved canonical locations of a recorded checkout that validation confirmed. */ export interface CheckoutIdentity { readonly root: Awaited> @@ -40,22 +55,30 @@ export async function checkoutIdentity( /** * Find the registration of `target` among the source repository's worktrees. + * The listing grows with the number of worktrees and is scanned as it arrives. * @param ctx - execution-world filesystem provider. * @param request - saved checkout identity. * @param target - resolved checkout directory. - * @param run - bounded Git runner with the owner's policy. + * @param scan - streaming Git runner with the owner's policy. * @param signal - cancellation of the lookups. * @returns the porcelain fields of the matching entry, or undefined when none is registered. */ export async function registration( - ctx: Context, request: RestoreWorktreeRequest, target: CheckoutIdentity['target'], run: GitRead, signal: AbortSignal, + ctx: Context, request: RestoreWorktreeRequest, target: CheckoutIdentity['target'], scan: GitScan, signal: AbortSignal, ): Promise { - const registered = await run(request.repositoryRoot, ['worktree', 'list', '--porcelain', '-z']) - for (const entry of registered.stdout.split('\0\0').map(entry => entry.split('\0'))) { - const registeredPath = entry.find(field => field.startsWith('worktree '))?.slice('worktree '.length) - if (registeredPath === undefined) continue - const recorded = await ctx.fs.resolve(registeredPath, { cwd: request.repositoryRoot, signal }) - if (recorded.targetKey === target.targetKey) return entry + const entries: { path: string; fields: string[] }[] = [] + // Each entry of the -z porcelain listing starts with its worktree line and ends with an empty field. + let fields: string[] = [] + await scan(request.repositoryRoot, ['worktree', 'list', '--porcelain', '-z'], (field) => { + if (field.startsWith('worktree ')) { + fields = [field] + entries.push({ path: field.slice('worktree '.length), fields }) + } else if (field.length > 0) fields.push(field) + return true + }) + for (const entry of entries) { + const recorded = await ctx.fs.resolve(entry.path, { cwd: request.repositoryRoot, signal }) + if (recorded.targetKey === target.targetKey) return entry.fields } return undefined } @@ -67,51 +90,51 @@ export async function registration( * @param ctx - execution-world filesystem provider. * @param directory - configured relative worktree pool. * @param request - saved checkout identity. - * @param run - bounded Git runner with the owner's policy. + * @param git - confined Git runners with the owner's policy. * @param signal - cancellation before or during removal. * @param admit - synchronous final admission after inspection; false retains the checkout. * @returns true when removed; false when inspection or admission retains it; Git failures reject. */ export async function removeCheckout( - ctx: Context, directory: string, request: RestoreWorktreeRequest, run: GitRead, signal: AbortSignal, admit: () => boolean, + ctx: Context, directory: string, request: RestoreWorktreeRequest, git: GitCommands, signal: AbortSignal, admit: () => boolean, ): Promise { - if (!await inspectRemoval(ctx, directory, request, run, signal)) return false + if (!await inspectRemoval(ctx, directory, request, git, signal)) return false signal.throwIfAborted() if (!admit()) return false - await run(request.repositoryRoot, ['worktree', 'remove', '--', request.path]) + await git.read(request.repositoryRoot, ['worktree', 'remove', '--', request.path]) return true } /** - * Verify checkout identity and account for every file that Git's non-force - * removal would refuse: tracked changes, untracked files, hidden index flags, - * and submodules. Ignored files are not counted; `git worktree remove` discards them. + * Verify checkout identity and that `git status` reports no tracked changes or + * untracked files; its output is read only up to the first entry. Ignored files + * are not counted, nor are edits to skip-worktree or assume-unchanged files, + * which `git status` hides; `git worktree remove` discards both. A checkout + * whose administrative directory holds submodule repositories is ineligible, + * as Git refuses to remove it. * @param ctx - execution-world filesystem provider. * @param directory - configured relative worktree pool. * @param request - saved checkout identity. - * @param run - bounded Git runner with the owner's policy. + * @param git - confined Git runners with the owner's policy. * @param signal - cancellation of inspection. * @returns false for an ineligible checkout; inspection failures reject. */ export async function inspectRemoval( - ctx: Context, directory: string, request: RestoreWorktreeRequest, run: GitRead, signal: AbortSignal, + ctx: Context, directory: string, request: RestoreWorktreeRequest, git: GitCommands, signal: AbortSignal, ): Promise { const { path, branch } = request - const identity = await checkoutIdentity(ctx, directory, request, run, signal) + const identity = await checkoutIdentity(ctx, directory, request, git.read, signal) if (identity === null) return false if ((await ctx.fs.lstat(path, undefined, signal))?.type !== 'directory') return false - const fields = await registration(ctx, request, identity.target, run, signal) + const fields = await registration(ctx, request, identity.target, git.scan, signal) // The primary registration is excluded even when the configured pool contains it. if (fields === undefined || !fields.includes(`branch refs/heads/${branch}`) || fields.some(field => field === 'detached' || field.startsWith('locked') || field.startsWith('prunable'))) return false - const head = await run(path, ['symbolic-ref', 'HEAD']) + const head = await git.read(path, ['symbolic-ref', 'HEAD']) if (head.stdout.trim() !== `refs/heads/${branch}`) return false - const status = await run(path, ['status', '--porcelain=v1', '-z', '--untracked-files=all', '--ignore-submodules=none']) - if (status.stdout.length !== 0) return false - // Git status deliberately hides skip-worktree and assume-unchanged files. - const flags = await run(path, ['ls-files', '-v', '-z']) - if (flags.stdout.split('\0').some(entry => entry.length > 0 && (entry[0] === 'S' || /^[a-z]/.test(entry)))) return false - const index = await run(path, ['ls-files', '--stage', '-z']) - if (index.stdout.split('\0').some(entry => entry.startsWith('160000 '))) return false - return true + const changed = await git.scan(path, ['status', '--porcelain=v1', '-z', '--untracked-files=all', '--ignore-submodules=none'], () => false) + if (changed) return false + // Git refuses removal whenever this directory exists, populated or not. + const modules = await git.read(path, ['rev-parse', '--git-path', 'modules']) + return await ctx.fs.lstat(modules.stdout.replace(/\r?\n$/, ''), { cwd: path }, signal) === undefined } diff --git a/packages/experimental/worktree/tests/archive.spec.ts b/packages/experimental/worktree/tests/archive.spec.ts index e2b5c8d47a..d8d18c91ed 100644 --- a/packages/experimental/worktree/tests/archive.spec.ts +++ b/packages/experimental/worktree/tests/archive.spec.ts @@ -63,7 +63,7 @@ function agentFor(ctx: Context, session: Session): Agent { return agent } -async function fixture(options: { cache?: boolean; retainOnArchive?: boolean } = {}) { +async function fixture(options: { cache?: boolean; retainOnArchive?: boolean; maxOutputBytes?: number } = {}) { const temporary = await realpath(await mkdtemp(join(tmpdir(), 'dsh-worktree-archive-'))) cleanups.push(() => rm(temporary, { recursive: true, force: true })) const root = await repository(temporary) @@ -87,7 +87,8 @@ async function fixture(options: { cache?: boolean; retainOnArchive?: boolean } = ctx.provide('storageDomain', domain) if (options.cache === true) await ctx.plugin(SessionProjectionCache, { writeEveryEvents: 1_000, writeIntervalMs: 60_000 }) await ctx.plugin(WorkspaceRegistry) - const worktrees = ctx.plugin(WorktreeService, options.retainOnArchive === undefined ? {} : { retainOnArchive: options.retainOnArchive }) + const { cache: _cache, ...config } = options + const worktrees = ctx.plugin(WorktreeService, config) await worktrees const session = ctx.sessions.create(SessionId('archive-owner'), { meta: { cwd: root } }) const agent = agentFor(ctx, session) @@ -137,6 +138,39 @@ describe('archive-triggered worktree cleanup', () => { expect(f.ctx.workspaceRegistry.archivedSessionIds).toContain(f.session.id) }) + it('removes a clean checkout whose file listing exceeds maxOutputBytes', async () => { + const f = await fixture({ maxOutputBytes: 1_024 }) + for (let index = 0; index < 64; index++) { + await writeFile(join(f.worktree.path, `file-${String(index).padStart(3, '0')}-${'x'.repeat(24)}.txt`), `${index}\n`) + } + await git(f.worktree.path, 'add', '.') + await git(f.worktree.path, 'commit', '-m', 'many files') + expect((await git(f.worktree.path, 'ls-files', '-z')).length).toBeGreaterThan(1_024) + await f.ctx.workspaceRegistry.archiveSession(f.session.id) + expect(await settled(f)).toMatchObject({ status: 'removed' }) + await expect(access(f.worktree.path)).rejects.toMatchObject({ code: 'ENOENT' }) + }) + + it('retains a checkout whose status output exceeds maxOutputBytes instead of failing', async () => { + const f = await fixture({ maxOutputBytes: 1_024 }) + for (let index = 0; index < 64; index++) { + await writeFile(join(f.worktree.path, `untracked-${String(index).padStart(3, '0')}-${'x'.repeat(24)}.txt`), `${index}\n`) + } + expect((await git(f.worktree.path, 'status', '--porcelain', '-z', '--untracked-files=all')).length).toBeGreaterThan(1_024) + await f.ctx.workspaceRegistry.archiveSession(f.session.id) + expect(await settled(f)).toMatchObject({ status: 'retained' }) + await access(join(f.worktree.path, 'untracked-000-xxxxxxxxxxxxxxxxxxxxxxxx.txt')) + }) + + it('removes a clean checkout when the worktree listing exceeds maxOutputBytes', async () => { + const f = await fixture({ maxOutputBytes: 1_024 }) + for (let index = 0; index < 8; index++) await git(f.root, 'worktree', 'add', '--detach', join(f.temporary, `extra-${index}`)) + expect((await git(f.root, 'worktree', 'list', '--porcelain', '-z')).length).toBeGreaterThan(1_024) + await f.ctx.workspaceRegistry.archiveSession(f.session.id) + expect(await settled(f)).toMatchObject({ status: 'removed' }) + await expect(access(f.worktree.path)).rejects.toMatchObject({ code: 'ENOENT' }) + }) + it('returns from admission before the inspection runs', async () => { const f = await fixture() const release = Promise.withResolvers() diff --git a/packages/experimental/worktree/tests/removal.spec.ts b/packages/experimental/worktree/tests/removal.spec.ts index d4afb66f85..27be81eca6 100644 --- a/packages/experimental/worktree/tests/removal.spec.ts +++ b/packages/experimental/worktree/tests/removal.spec.ts @@ -4,16 +4,26 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import type { Context } from '@deepseek-ai/cordis' import type { Agent } from '@deepseek-ai/dsh-agent' import { inspectRemoval, removeCheckout } from '../src/removal.ts' -import type { GitRead } from '../src/removal.ts' +import type { GitCommands, GitRead, GitScan } from '../src/removal.ts' import type { RestoreWorktreeRequest } from '../src/types.ts' import { git, harness, repository, testAgent } from './harness.ts' /** A Git runner over the fixture's unconfined git, as the service's danger-full-access runner behaves. */ const run: GitRead = async (directory, args) => ({ stdout: await git(directory, ...args), exitCode: 0 }) +/** Pass each NUL-terminated field of `text` to `onField` as the streaming runner does; true when it stopped early. */ +function scanText(text: string, onField: (field: string) => boolean): boolean { + return !text.split('\0').slice(0, -1).every(field => onField(field)) +} + +/** A streaming runner over the same git; it reads the output whole, then passes its fields in order. */ +const scan: GitScan = async (directory, args, onField) => scanText(await git(directory, ...args), onField) + +const commands: GitCommands = { read: run, scan } + /** The inspection-and-removal step archive cleanup runs, with the pool the harness configures. */ function remove(ctx: Context, _agent: Agent, request: RestoreWorktreeRequest, admit: () => boolean = () => true): Promise { - return removeCheckout(ctx, '.agents/worktrees', request, run, new AbortController().signal, admit) + return removeCheckout(ctx, '.agents/worktrees', request, commands, new AbortController().signal, admit) } const disposers: (() => Promise)[] = [] @@ -48,21 +58,42 @@ describe('safe worktree removal', () => { expect(await git(worktree.path, 'rev-parse', 'HEAD')).toBe(head) }) - it.each(['modified', 'staged', 'untracked', 'assume-unchanged', 'skip-worktree'] as const)( + it.each(['modified', 'staged', 'untracked'] as const)( 'retains %s files', async (kind) => { const { ctx, agent, worktree } = await fixture() - const tracked = join(worktree.path, 'tracked.txt') if (kind === 'untracked') await writeFile(join(worktree.path, 'new.txt'), 'keep') else { - await writeFile(tracked, 'keep\n') + await writeFile(join(worktree.path, 'tracked.txt'), 'keep\n') if (kind === 'staged') await git(worktree.path, 'add', 'tracked.txt') - if (kind === 'assume-unchanged' || kind === 'skip-worktree') await git(worktree.path, 'update-index', `--${kind}`, 'tracked.txt') } expect(await remove(ctx, agent, worktree)).toBe(false) await access(worktree.path) }, ) + it.each(['assume-unchanged', 'skip-worktree'] as const)( + 'removes a checkout whose only edits are to %s files, which git status hides', async (flag) => { + const { ctx, agent, worktree } = await fixture() + await writeFile(join(worktree.path, 'tracked.txt'), 'hidden edit\n') + await git(worktree.path, 'update-index', `--${flag}`, 'tracked.txt') + expect(await git(worktree.path, 'status', '--porcelain')).toBe('') + expect(await remove(ctx, agent, worktree)).toBe(true) + await expect(access(worktree.path)).rejects.toMatchObject({ code: 'ENOENT' }) + }, + ) + + it('retains a checkout holding a submodule repository, which Git refuses to remove', async () => { + const { ctx, agent, worktree, root } = await fixture() + const source = await repository() + disposers.push(() => rm(source, { recursive: true, force: true })) + await git(worktree.path, '-c', 'protocol.file.allow=always', 'submodule', 'add', source, 'nested') + await git(worktree.path, 'commit', '-m', 'submodule') + expect(await git(worktree.path, 'status', '--porcelain')).toBe('') + expect(await remove(ctx, agent, worktree)).toBe(false) + await access(join(worktree.path, 'nested', 'tracked.txt')) + await expect(git(root, 'worktree', 'remove', worktree.path)).rejects.toThrow('working trees containing submodules cannot be moved or removed') + }) + it('removes a checkout that holds only ignored files, as Git does without force', async () => { const { ctx, agent, worktree } = await fixture() await git(worktree.path, 'config', 'core.excludesFile', join(worktree.repositoryRoot, 'exclude')) @@ -94,17 +125,18 @@ describe('safe worktree removal', () => { }, ) - it.each(['repository identity', 'registration', 'checked-out branch', 'submodule'] as const)( + it.each(['repository identity', 'registration', 'checked-out branch'] as const)( 'retains a checkout when the %s changes during inspection', async (changed) => { const { ctx, worktree, root } = await fixture() - const run = vi.fn(async (directory: string, args: string[]) => { + const read = vi.fn(async (directory: string, args: string[]) => { if (changed === 'repository identity' && args[0] === 'rev-parse') return { stdout: worktree.path, exitCode: 0 } - if (changed === 'registration' && args[0] === 'worktree') return { stdout: `worktree ${root}\0branch refs/heads/main\0\0\0`, exitCode: 0 } if (changed === 'checked-out branch' && args[0] === 'symbolic-ref') return { stdout: 'refs/heads/other', exitCode: 0 } - if (changed === 'submodule' && args.includes('--stage')) return { stdout: `160000 ${worktree.baseCommit} 0\tsubmodule\0`, exitCode: 0 } return { stdout: await git(directory, ...args), exitCode: 0 } }) - expect(await inspectRemoval(ctx, '.agents/worktrees', worktree, run, new AbortController().signal)).toBe(false) + const listed: GitScan = async (directory, args, onField) => changed === 'registration' && args[0] === 'worktree' + ? scanText(`worktree ${root}\0branch refs/heads/main\0\0`, onField) + : await scan(directory, args, onField) + expect(await inspectRemoval(ctx, '.agents/worktrees', worktree, { read, scan: listed }, new AbortController().signal)).toBe(false) await access(worktree.path) }, ) diff --git a/packages/experimental/worktree/tests/worktree.spec.ts b/packages/experimental/worktree/tests/worktree.spec.ts index c1beb17c4a..33c9b84f21 100644 --- a/packages/experimental/worktree/tests/worktree.spec.ts +++ b/packages/experimental/worktree/tests/worktree.spec.ts @@ -414,6 +414,22 @@ describe('restoring removed worktrees', () => { expect(ctx.workingDirectory.get(agent.session)).toBe(root) }) + it('finds the stale registration when the worktree listing exceeds maxOutputBytes', async () => { + const root = await repo() + ctx = await harness(root, { maxOutputBytes: 1_024 }) + const agent = testAgent(ctx, root) + const created = await ctx.worktrees.create(agent, { name: 'topic' }) + await ctx.workingDirectory.set(agent, root) + for (let index = 0; index < 8; index++) await git(root, 'worktree', 'add', '--detach', join(root, '.agents/worktrees', `extra-${index}`)) + expect((await git(root, 'worktree', 'list', '--porcelain', '-z')).length).toBeGreaterThan(1_024) + await rm(created.path, { recursive: true, force: true }) + + expect(await ctx.worktrees.restore(agent, created)).toEqual({ path: created.path, branchRecreated: false }) + + expect(await git(created.path, 'symbolic-ref', '--short', 'HEAD')).toBe('topic') + expect(await registered(root)).toContain(gitPath(created.path)) + }) + it('creates a deleted branch again at its base commit, without the commits it had gained', async () => { const root = await repo() ctx = await harness(root) From cfee33035643bafb367f31cba83acd97e959c912 Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Mon, 9 Nov 2026 11:47:05 +0800 Subject: [PATCH 27/30] fix(forge-panel): highlight lines in the render that draws them Deferred idle-time highlighting drew token text, so a file whose content changed at the same path kept showing its previous text until the next pass, and replacing the plain text collapsed a selection made meanwhile. Sections highlight in their drawing render again, as before. File sections now take the tab's visibility as a prop and the reach tracker through their handlers, so no business component reads a React context. --- ...orge-changes-view-responsiveness.i18n.yaml | 12 +- ...11-07-forge-changes-view-responsiveness.md | 10 +- ...07-forge-changes-view-responsiveness.zh.md | 10 +- .../client-ui-forge/README.i18n.yaml | 4 +- .../experimental/client-ui-forge/README.md | 4 +- .../experimental/client-ui-forge/README.zh.md | 4 +- .../client-ui-forge/src/client/ChangesTab.tsx | 16 +- .../client-ui-forge/src/client/DiffFile.tsx | 159 +++++------------- .../src/client/section-reach.ts | 55 ------ .../tests/changes-tab.client.spec.tsx | 29 +--- .../tests/diff-review.client.spec.tsx | 34 +++- .../tests/section-reach.client.spec.ts | 70 +------- 12 files changed, 110 insertions(+), 297 deletions(-) diff --git a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.i18n.yaml index cc2e9c4b68..170d538ff9 100644 --- a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.i18n.yaml +++ b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.i18n.yaml @@ -9,11 +9,11 @@ en: 77fab27f3d4a4a59 zh: 46e0fa8b9ef978a7 /agent-note-forge-changes-view-responsiveness/decision: - en: a7127dc65ac6d22c - zh: 8582e17cf52db811 + en: a138be3ed592e94e + zh: 80963cd20131df3a /agent-note-forge-changes-view-responsiveness/alternatives-considered: - en: 9d12ccf1d236676d - zh: 0ea63c5c16c62de7 + en: 93e305c624bfe144 + zh: e41495d9930801c9 /agent-note-forge-changes-view-responsiveness/consequences: - en: 4087fff405624d95 - zh: 2f0e577eea4daf40 + en: 2d74147e306ed174 + zh: f1369020f27f92c0 diff --git a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md index 73583334b2..3c481c5ea3 100644 --- a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md +++ b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md @@ -12,9 +12,9 @@ The forge changes view of a large change request froze the browser. Opening #798 **Scoped `:has()` rules.** The navigator's covering rules name their row through `data-files-navigator-row`, which both hosts of `sidebar.files.navigator` set, so a DOM change outside such a row schedules no document-wide invalidation. -**Sections draw what changed and what the reader reaches.** `ChangesTab` passes each memoized `DiffFile` values whose identity survives its renders: one handler object, one comment-actions object, and per-file strings and booleans. Tab visibility reaches only the menus and the composer through a context. An expanded file holds a 22 px row per drawn line until its section first comes within 1200 px of the scrollport, then draws its lines and keeps them. A jump keeps its target aligned for one second while nearby sections draw. +**Sections draw what changed and what the reader reaches.** `ChangesTab` passes each memoized `DiffFile` values whose identity survives its renders: one handler object, one comment-actions object, and per-file strings and booleans. Tab visibility is a prop of every section, so a visibility change redraws them all; leaving the view measured +57 ms against reading it from a React context, which the client packages forbid business components to do. The scrollport's reach tracker travels in the handler object. An expanded file holds a 22 px row per drawn line until its section first comes within 1200 px of the scrollport, then draws its lines and keeps them. A jump keeps its target aligned for one second while nearby sections draw. -**Idle-time highlighting.** Lines draw as plain text and take their tokens once the page is idle and the scrollport has been still for 200 ms, so tokenization never interrupts a scroll in progress. A pass highlights 100-line fragments while each idle period lasts and runs again only when the file's segments change or its own language's grammar loads; the shared highlighter's identity, which changes on every grammar load, no longer triggers one. +**Highlighting in the drawing render.** A section highlights its lines in the same render that draws them, as before, so the drawn text is always the file's current text and a native selection keeps its text nodes. **Fewer serial reads.** A panel state equal to the published one reaches no follower. `changeRequestDiff` reads the patch and `.gitattributes` together. Snapshots carry the base commit as `baseSha`; `ForgeService.diff` passes it to the provider, and forge-github starts the compare read while its GraphQL read confirms the head. @@ -26,10 +26,14 @@ The forge changes view of a large change request froze the browser. Opening #798 **Full virtualization that also unmounts far sections** would bound the DOM after long scrolls, but a section holds a comment draft and a text selection, which unmounting loses. +**Deferred idle-time highlighting** drew plain text and applied tokens once the page was idle and the scrollport still. It kept fast scrolling smooth, but tokens carry their own text, so a file whose content changed at the same path showed its previous text until the next pass, and replacing the plain text nodes collapsed a selection made in the meantime. Against it, highlighting while drawing raised the blocking time of scrolling all of #7985 from 22 ms to 1,634 ms and its longest frame gap from 34 ms to 817 ms; other operations stayed within 0.1 s. + +**The CSS Custom Highlight API** would paint token colors over unchanged text nodes and keep deferral without either defect, but it cannot apply the themes' bold and italic, and its cost over tens of thousands of ranges is unmeasured. + **Highlighting in a Web Worker** removes tokenization from the main thread entirely, including a heavy grammar's first use, but the shared highlighter in `ui-primitives` owns Shiki; moving it is a separate change for every consumer. **A provider-private cache of base commits** would avoid the optional `baseSha` field, but it adds retained state without an owner that knows when a change request stops being watched. ## Consequences -Opening #7985 now takes 0.5 s of main-thread work instead of 5.2 s, and file jumps, filtering, polling, and Agent turns no longer produce long tasks. In exchange, the browser's find in page does not see lines of files the reader has not approached, highlighting restarts at 100-line fragment boundaries, so a multi-line comment or string crossing one can be mis-colored, and the first use of a slow grammar, such as C++, still produces a 400–550 ms long task while scrolling. A diff read costs a second compare request when the base commit changed after the latest status. +Opening #7985 now takes 0.5 s of main-thread work instead of 5.2 s, and file jumps, filtering, polling, and Agent turns no longer produce long tasks. In exchange, the browser's find in page does not see lines of files the reader has not approached, and scrolling onto files in a slow grammar's language, such as C++, highlights them during the scroll, up to about 0.8 s of main-thread work when that grammar is first used. A diff read costs a second compare request when the base commit changed after the latest status. diff --git a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.zh.md b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.zh.md index 999ac762fc..6881c5b3ae 100644 --- a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.zh.md +++ b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.zh.md @@ -12,9 +12,9 @@ Status: implemented **限定 `:has()` 规则。** 导航器的遮盖规则通过 `data-files-navigator-row` 指定所在行，`sidebar.files.navigator` 的两个宿主都会设置它，因此这类行之外的 DOM 变化不再触发全文档失效。 -**文件段只绘制变化的部分和读者接近的部分。** `ChangesTab` 传给每个 memo 化的 `DiffFile` 的值在多次渲染之间保持同一标识：一个处理函数对象、一个评论操作对象，以及每个文件的字符串和布尔值。标签页可见性通过 context 只传给菜单和评论编辑器。展开的文件在它的段第一次进入滚动区域 1200 px 范围之前，按每个绘制行 22 px 占位，之后绘制各行并一直保留。跳转后的一秒内，目标文件在周围文件段绘制时保持对齐。 +**文件段只绘制变化的部分和读者接近的部分。** `ChangesTab` 传给每个 memo 化的 `DiffFile` 的值在多次渲染之间保持同一标识：一个处理函数对象、一个评论操作对象，以及每个文件的字符串和布尔值。标签页可见性作为每个文件段的参数传入，可见性变化时全部文件段都会重绘；离开视图实测比从 React context 读取多 57 ms，而客户端包禁止业务组件读取 context。滚动区域的接近跟踪对象放在处理函数对象中传递。展开的文件在它的段第一次进入滚动区域 1200 px 范围之前，按每个绘制行 22 px 占位，之后绘制各行并一直保留。跳转后的一秒内，目标文件在周围文件段绘制时保持对齐。 -**空闲时高亮。** 各行先以纯文本绘制，在页面空闲且滚动区域静止 200 ms 后得到 token，因此分词不会打断正在进行的滚动。一次高亮在每个空闲时段内处理多个 100 行的片段，只有文件的分段变化或该语言自己的语法加载完成时才重新运行；共享高亮器的标识在任何语法加载时都会变化，不再触发重新高亮。 +**在绘制的渲染中高亮。** 文件段与修复前一样，在绘制各行的同一次渲染中完成高亮，所以画出的始终是文件当前的文本，原生选区所在的文本节点也不会被替换。 **减少串行读取。** 与已发布状态相同的面板状态不会通知任何关注方。`changeRequestDiff` 同时读取补丁和 `.gitattributes`。快照把基准提交作为 `baseSha` 携带；`ForgeService.diff` 把它传给提供方，forge-github 在 GraphQL 读取确认 head 的同时就开始 compare 读取。 @@ -26,10 +26,14 @@ Status: implemented **同时卸载远处文件段的完整虚拟化** 可以在长距离滚动后限制 DOM 规模，但文件段持有评论草稿和文本选区，卸载会丢失它们。 +**空闲时延迟高亮** 先画纯文本，等页面空闲且滚动区域静止后再应用 token。它让快速滚动保持流畅，但 token 自带文本，同一路径内容变化后会显示旧文本直到下一轮高亮完成；替换纯文本节点还会清掉期间做出的选区。与它相比，绘制时高亮让滚动整个 #7985 的阻塞时间从 22 ms 增到 1,634 ms，最长帧间隔从 34 ms 增到 817 ms；其他操作的差异在 0.1 秒以内。 + +**CSS Custom Highlight API** 可以在不变的文本节点上绘制 token 颜色，保留延迟高亮又没有上述两个问题，但它无法应用主题中的粗体和斜体，几万个范围的绘制成本也没有测过。 + **在 Web Worker 中高亮** 能把分词完全移出主线程，包括重型语法的首次使用，但 Shiki 由 `ui-primitives` 中的共享高亮器持有；迁移它是影响所有使用方的另一项改动。 **由提供方私有缓存基准提交** 可以省去可选的 `baseSha` 字段，但会引入保留状态，而且没有一个知道变更请求何时不再被 watch 的所有者。 ## Consequences -打开 #7985 的主线程工作从 5.2 秒降到 0.5 秒，跳转文件、筛选、轮询和 Agent 回合不再产生长任务。代价是：浏览器的页内查找看不到读者尚未接近的文件中的行；高亮在 100 行片段边界处重新开始，跨越边界的多行注释或字符串可能着色错误；慢语法（例如 C++）首次使用时，滚动中仍会出现 400–550 ms 的长任务。最新状态之后基准提交有变化时，一次 diff 读取会多一次 compare 请求。 +打开 #7985 的主线程工作从 5.2 秒降到 0.5 秒，跳转文件、筛选、轮询和 Agent 回合不再产生长任务。代价是：浏览器的页内查找看不到读者尚未接近的文件中的行；滚动到慢语法（例如 C++）的文件时会在滚动中完成高亮，该语法首次使用时最多约 0.8 秒的主线程工作。最新状态之后基准提交有变化时，一次 diff 读取会多一次 compare 请求。 diff --git a/packages/experimental/client-ui-forge/README.i18n.yaml b/packages/experimental/client-ui-forge/README.i18n.yaml index 2cbf20d4c3..851fed865d 100644 --- a/packages/experimental/client-ui-forge/README.i18n.yaml +++ b/packages/experimental/client-ui-forge/README.i18n.yaml @@ -18,8 +18,8 @@ en: c7e1b7d87ccccd2e zh: 42d864b918a6175c /deepseek-ai-dsh-experimental-client-ui-forge/understand-the-implementation: - en: 0d8e666fc1cd71db - zh: 69f3d10283c2edfc + en: eaf062b1439ed8b3 + zh: d7c6a68779774825 /deepseek-ai-dsh-experimental-client-ui-forge/further-exploration: en: fa4d8c579ffc49ff zh: 5856ed7469e4585b diff --git a/packages/experimental/client-ui-forge/README.md b/packages/experimental/client-ui-forge/README.md index 784a05bfa2..c4e36aaba3 100644 --- a/packages/experimental/client-ui-forge/README.md +++ b/packages/experimental/client-ui-forge/README.md @@ -73,7 +73,7 @@ Each inspection also decides whether the worktree is missing: the Session's dire `changeRequestStates` streams, for the Session list, the last known number, title, state, draft flag, checks rollup, review decision, and mergeability of every change request that a listed Session the caller may read binds: a `baseline` frame, then `changed` frames. One run serves every stream. Cancellation is observed before initialization and between frames; the last stream and plugin disposal await initialization and accepted writes, then close the run’s storage domain. A later run waits for that closure before reopening the domain and reading its stored states. Binding updates received during listing override its initial snapshot. It lists Sessions through `sessionQuery` without loading them, takes the visible bindings of a loaded Session from `ctx.forgeSession` and of another from its cached `forgeChangeRequests` summary, follows binding changes of loaded Sessions, and subscribes to each bound change request through `ctx.forge.watch`; closed requests use the idle polling cadence, and merged requests remain subscribed for explicit refresh. It stores every changed state in the `forge_panel_change_requests` storage domain under the SHA-256 digest of its URL and drops unbound stored states only when every listed Session has known bindings; a failed write is logged and the state stays in memory. The stream fails when the Host lacks the Session query, projection cache, storage, or Session store service. -The browser entry mounts the Remote, then the UI. One stream per Session feeds the tab, the header button, the New Session option, and the creation card; it opens with the first subscriber, closes with the last, reopens after a transport reset, and keeps the last state while the Session scope is retained. Releasing that scope closes the stream and removes its cached state. The New Session option fills `conversation.hero.options` and the worktree card `conversation.input.dock`. The worktree card shows a running creation, with Cancel calling `stopCreation` while a message waits for it, or the failed or stopped attempt a message waits after, with Retry calling `retryWorktree` and Continue without a worktree calling `cancelWorktree`. The tab shortens paths with the Host home from the Remote's Host facts, read again after each transport reset. The tab reports its visibility to the header button's pressed state. The header button collapses a docked tab or closes its floating tab. The toast lives in a `shell.overlay` entry so it outlives the tab. One stream of change request states feeds the Session row icons in `sidebar.session.row.trailing` and the hover card section in `sidebar.session.row.hover` while any of them is mounted; both take a Session's worktree and bindings from its Session list summary values. The change request card takes a change request, its snapshot, and optional actions, so another surface can reuse it. The changes view is the `forge-changes` resource tab type: its addresses, `dsh-resource://forge-changes//local` and `dsh-resource://forge-changes//change/`, name the source, and a navigation may name a file to scroll to. The tab type enables `keepMounted`, retaining visited bodies and their owning Session references until their tab records end. Its keyed panel hook subscribes only for the visible body; other panel surfaces may keep the shared Session stream open. Its body keeps the scope, base branch, chosen commits, lens, test file switch, and layout choices of each tab in a slot store, keeps viewed marks per Session in `localStorage` under `dsh.forge-viewed.v1.`, at most 20 sources per Session and 50 Sessions, reads through the Remote, and passes each file section values that keep their identity across its renders, so a render redraws only the sections whose values changed. An expanded file draws its lines once its section first comes within 1200px of the scrollport, keeps them drawn after that, and until then holds a 22px row per drawn line; the lines are highlighted with the shared code highlighter in 100-line fragments when the page is idle and the scrollport has been still for 200 ms, again only when the file's segments change or its language's grammar finishes loading; a chip title shows its name with the changes icon. +The browser entry mounts the Remote, then the UI. One stream per Session feeds the tab, the header button, the New Session option, and the creation card; it opens with the first subscriber, closes with the last, reopens after a transport reset, and keeps the last state while the Session scope is retained. Releasing that scope closes the stream and removes its cached state. The New Session option fills `conversation.hero.options` and the worktree card `conversation.input.dock`. The worktree card shows a running creation, with Cancel calling `stopCreation` while a message waits for it, or the failed or stopped attempt a message waits after, with Retry calling `retryWorktree` and Continue without a worktree calling `cancelWorktree`. The tab shortens paths with the Host home from the Remote's Host facts, read again after each transport reset. The tab reports its visibility to the header button's pressed state. The header button collapses a docked tab or closes its floating tab. The toast lives in a `shell.overlay` entry so it outlives the tab. One stream of change request states feeds the Session row icons in `sidebar.session.row.trailing` and the hover card section in `sidebar.session.row.hover` while any of them is mounted; both take a Session's worktree and bindings from its Session list summary values. The change request card takes a change request, its snapshot, and optional actions, so another surface can reuse it. The changes view is the `forge-changes` resource tab type: its addresses, `dsh-resource://forge-changes//local` and `dsh-resource://forge-changes//change/`, name the source, and a navigation may name a file to scroll to. The tab type enables `keepMounted`, retaining visited bodies and their owning Session references until their tab records end. Its keyed panel hook subscribes only for the visible body; other panel surfaces may keep the shared Session stream open. Its body keeps the scope, base branch, chosen commits, lens, test file switch, and layout choices of each tab in a slot store, keeps viewed marks per Session in `localStorage` under `dsh.forge-viewed.v1.`, at most 20 sources per Session and 50 Sessions, reads through the Remote, and passes each file section values that keep their identity across its renders, including the tab's visibility, so a render redraws only the sections whose values changed and a visibility change redraws every section. An expanded file draws its lines, highlighted with the shared code highlighter in the same render, once its section first comes within 1200px of the scrollport, keeps them drawn after that, and until then holds a 22px row per drawn line; a chip title shows its name with the changes icon. | Source | Responsibility | |---|---| @@ -92,7 +92,7 @@ The browser entry mounts the Remote, then the UI. One stream per Session feeds t | [`src/client/ChangesTab.tsx`](src/client/ChangesTab.tsx) | Changes view | | [`src/client/BaseMenu.tsx`](src/client/BaseMenu.tsx) | Base branch menu of the Branch scope | | [`src/client/DiffFile.tsx`](src/client/DiffFile.tsx) | One file of the changes view | -| [`src/client/section-reach.ts`](src/client/section-reach.ts) | When file sections draw their lines, and idle-time highlighting | +| [`src/client/section-reach.ts`](src/client/section-reach.ts) | When file sections draw their lines | | [`src/client/ReviewControls.tsx`](src/client/ReviewControls.tsx) | Lens, test file switch, and viewed progress | | [`src/client/CommitMenu.tsx`](src/client/CommitMenu.tsx) | Commit picker | | [`src/client/viewed-files.ts`](src/client/viewed-files.ts) | Viewed marks kept in the browser | diff --git a/packages/experimental/client-ui-forge/README.zh.md b/packages/experimental/client-ui-forge/README.zh.md index e4154cc47b..ebe5c653ef 100644 --- a/packages/experimental/client-ui-forge/README.zh.md +++ b/packages/experimental/client-ui-forge/README.zh.md @@ -73,7 +73,7 @@ Session 在所创建的 worktree 中工作时，`localDiff` 以它的 `baseCommi `changeRequestStates` 为会话列表推送调用方可读的、已列出的 Session 所绑定的每个变更请求的最近编号、标题、状态、草稿标记、检查汇总、评审结论与可合并性：先是一个 `baseline` 帧，之后是 `changed` 帧。所有流共用一次运行。初始化之前和帧之间均响应取消；最后一条流结束及插件卸载会等待初始化与已接受的写入完成，再关闭本次运行的存储域。后续运行会等待该域关闭后重新打开，并读取已存状态。列举期间收到的绑定更新优先于初始快照。它经 `sessionQuery` 列出 Session 而不加载它们，已加载 Session 的可见绑定取自 `ctx.forgeSession`，其他 Session 的取自缓存的 `forgeChangeRequests` 摘要；它跟随已加载 Session 的绑定变化，并经 `ctx.forge.watch` 订阅每个已绑定的变更请求；已关闭的按空闲间隔轮询，已合并的保留订阅以接收显式刷新结果。每个变化了的状态以其 URL 的 SHA-256 摘要为键存入 `forge_panel_change_requests` 存储域，仅在所有已列出 Session 的绑定均已知时删除无人绑定的已存状态；写入失败时记录日志，状态仍保留在内存中。宿主缺少 Session 查询、投影缓存、存储或 Session 存储服务时，流以失败结束。 -浏览器入口先挂载 Remote，再挂载界面。每个 Session 一条流，同时供给标签页、标题栏按钮、新会话选项与创建卡片；第一个订阅方出现时打开，最后一个离开时关闭，传输重置后重新打开，在 Session scope 保留期间保存最后的状态。释放该 scope 时关闭流并移除缓存状态。新会话选项填入 `conversation.hero.options`，worktree 卡片填入 `conversation.input.dock`。worktree 卡片显示正在进行的创建，有消息在等待时“取消”调用 `stopCreation`；或显示消息等待时失败或被停止的那次尝试，“重试”调用 `retryWorktree`，“不使用工作树”调用 `cancelWorktree`。标签页用 Remote 宿主信息中的宿主主目录缩短路径，每次传输重置后重新读取。标签页把自己是否可见报告给标题栏按钮的按下状态。标题栏按钮收起停靠的标签页，或关闭其浮动标签页。toast 位于 `shell.overlay` 条目中，因此比标签页存在得更久。只要 `sidebar.session.row.trailing` 中的会话行图标或 `sidebar.session.row.hover` 中的悬浮卡片部分有一个已挂载，就由同一条变更请求状态流供给它们；两者都从会话列表的摘要值中读取 Session 的 worktree 与绑定。变更请求卡片接收变更请求、它的快照与可选操作，其他界面可以复用。变更视图是 `forge-changes` 资源标签页类型：它的地址 `dsh-resource://forge-changes//local` 与 `dsh-resource://forge-changes//change/` 指明来源，导航参数可以指定要滚动到的文件。标签页类型启用 `keepMounted`，将访问过的正文及其所属 Session 引用保留至标签页记录结束。它的 keyed panel hook 仅为可见正文订阅；其他面板界面仍可能保持共享 Session 流开启。它的正文用 slot store 按标签页保存范围、基准分支、所选提交、透镜、测试文件开关与布局选择，按 Session 把已审阅标记保存在 `localStorage` 的 `dsh.forge-viewed.v1.` 下（每个 Session 最多 20 个来源，最多 50 个 Session），经 Remote 读取，并且传给每个文件段的值在多次渲染之间保持同一标识，所以一次渲染只重绘值有变化的文件段。展开的文件在它的段第一次进入滚动区域 1200px 范围内时才绘制各行，此后一直保留；在此之前按每个绘制行 22px 占位。各行在页面空闲且滚动区域静止 200 ms 后，用共享的代码高亮器按 100 行一片高亮，之后只在文件的分段变化或该语言的语法加载完成时重新高亮；标签页标题显示变更图标与视图名称。 +浏览器入口先挂载 Remote，再挂载界面。每个 Session 一条流，同时供给标签页、标题栏按钮、新会话选项与创建卡片；第一个订阅方出现时打开，最后一个离开时关闭，传输重置后重新打开，在 Session scope 保留期间保存最后的状态。释放该 scope 时关闭流并移除缓存状态。新会话选项填入 `conversation.hero.options`，worktree 卡片填入 `conversation.input.dock`。worktree 卡片显示正在进行的创建，有消息在等待时“取消”调用 `stopCreation`；或显示消息等待时失败或被停止的那次尝试，“重试”调用 `retryWorktree`，“不使用工作树”调用 `cancelWorktree`。标签页用 Remote 宿主信息中的宿主主目录缩短路径，每次传输重置后重新读取。标签页把自己是否可见报告给标题栏按钮的按下状态。标题栏按钮收起停靠的标签页，或关闭其浮动标签页。toast 位于 `shell.overlay` 条目中，因此比标签页存在得更久。只要 `sidebar.session.row.trailing` 中的会话行图标或 `sidebar.session.row.hover` 中的悬浮卡片部分有一个已挂载，就由同一条变更请求状态流供给它们；两者都从会话列表的摘要值中读取 Session 的 worktree 与绑定。变更请求卡片接收变更请求、它的快照与可选操作，其他界面可以复用。变更视图是 `forge-changes` 资源标签页类型：它的地址 `dsh-resource://forge-changes//local` 与 `dsh-resource://forge-changes//change/` 指明来源，导航参数可以指定要滚动到的文件。标签页类型启用 `keepMounted`，将访问过的正文及其所属 Session 引用保留至标签页记录结束。它的 keyed panel hook 仅为可见正文订阅；其他面板界面仍可能保持共享 Session 流开启。它的正文用 slot store 按标签页保存范围、基准分支、所选提交、透镜、测试文件开关与布局选择，按 Session 把已审阅标记保存在 `localStorage` 的 `dsh.forge-viewed.v1.` 下（每个 Session 最多 20 个来源，最多 50 个 Session），经 Remote 读取，并且传给每个文件段的值（包括标签页可见性）在多次渲染之间保持同一标识，所以一次渲染只重绘值有变化的文件段，可见性变化时重绘全部文件段。展开的文件在它的段第一次进入滚动区域 1200px 范围内时才绘制各行，并在同一次渲染中用共享的代码高亮器高亮，此后一直保留；在此之前按每个绘制行 22px 占位；标签页标题显示变更图标与视图名称。 | 源码 | 职责 | |---|---| @@ -92,7 +92,7 @@ Session 在所创建的 worktree 中工作时，`localDiff` 以它的 `baseCommi | [`src/client/ChangesTab.tsx`](src/client/ChangesTab.tsx) | 变更视图 | | [`src/client/BaseMenu.tsx`](src/client/BaseMenu.tsx) | “分支”范围的基准分支菜单 | | [`src/client/DiffFile.tsx`](src/client/DiffFile.tsx) | 变更视图中的一个文件 | -| [`src/client/section-reach.ts`](src/client/section-reach.ts) | 文件段何时绘制各行，以及空闲时高亮 | +| [`src/client/section-reach.ts`](src/client/section-reach.ts) | 文件段何时绘制各行 | | [`src/client/ReviewControls.tsx`](src/client/ReviewControls.tsx) | 分层透镜、测试文件开关与已审阅进度 | | [`src/client/CommitMenu.tsx`](src/client/CommitMenu.tsx) | 提交选择器 | | [`src/client/viewed-files.ts`](src/client/viewed-files.ts) | 保存在浏览器中的已审阅标记 | diff --git a/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx b/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx index cc01ad2a19..4cec78f68e 100644 --- a/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx +++ b/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx @@ -6,7 +6,8 @@ * refresh, and when the change request's head or the checkout's `branch` * scope summary changes; a read keeps the previous changes on screen until it settles. * File sections receive values that keep their identity across renders, so a - * render of the view redraws only the sections whose own values changed. + * render of the view redraws only the sections whose own values changed, and + * an expanded file draws its lines once its section comes near the scrollport. */ import { @@ -42,7 +43,7 @@ import { CommitMenu } from './CommitMenu.tsx' import { HideTestsSwitch, LensControl, LensMenu, LENSES, ViewedProgress } from './ReviewControls.tsx' import { viewedStateOf, type ViewedFiles, type ViewedMarks } from './viewed-files.ts' import { Counts, DiffFile, type DiffFileAction, type DiffFileHandlers } from './DiffFile.tsx' -import { SectionReach, SectionReachContext, ViewVisibleContext } from './section-reach.ts' +import { SectionReach } from './section-reach.ts' import { FileTreePane, type FileTreePosition } from './FileTreePane.tsx' import { changedFileTree, filesInTreeOrder } from './file-tree.ts' import { ChangedFilesMenu, type ChangedFilesProps, type ChangedFilesState } from './ChangedFiles.tsx' @@ -604,7 +605,7 @@ export function ChangesTab({ * Keep a jumped-to section at the top of the scrollport for {@link JUMP_PIN_MS}: the sections around it draw their * lines after the jump, which moves it. Scrolling input from the reader, another jump, hiding the tab, or unmounting * ends the pin. Each frame measures the section once, which covers every cause of movement, such as a section - * drawing its lines, deferred highlighting, or a review card opening, without each one reporting it. + * drawing its lines or a review card opening, without each one reporting it. */ const pinJump = (port: HTMLElement, section: HTMLElement, owner: ChangesData): void => { releasePin.current?.() @@ -833,7 +834,8 @@ export function ChangesTab({ if (element === null) sections.current.delete(path) else sections.current.set(path, element) }, - }), []) + reach, + }), [reach]) const reviewStarted = reviewState.comments.length > 0 const publishingBlocked = reviewState.uncertain !== null // Comments on a commit range or on local changes go to the input only; a whole change request also takes GitHub comments. @@ -889,7 +891,6 @@ export function ChangesTab({
{ - reach.scrolled() if (jumpPosition.current?.top !== event.currentTarget.scrollTop) jumpPosition.current = null }}> {files.map((file) => { @@ -897,6 +898,7 @@ export function ChangesTab({ return ( 0 && {t('diff.omitted', { count: data.omitted })}} {reviewForm}
- - {body} - + {body} {allFiles.length > 0 && }
diff --git a/packages/experimental/client-ui-forge/src/client/DiffFile.tsx b/packages/experimental/client-ui-forge/src/client/DiffFile.tsx index db48c61457..10a039cc6d 100644 --- a/packages/experimental/client-ui-forge/src/client/DiffFile.tsx +++ b/packages/experimental/client-ui-forge/src/client/DiffFile.tsx @@ -2,11 +2,11 @@ * One file of the changes view: a sticky header with the path, badges, line * counts, and file actions, then its hunks unified or side by side with the * unchanged lines between them collapsed into rows that expand them. A section - * draws its lines once it comes within reach of the scrollport and highlights - * them when the page is next idle; it redraws only when its own props change. + * draws and highlights its lines once it comes within reach of the scrollport; + * it redraws only when its own props change. */ -import { memo, useCallback, useContext, useEffect, useMemo, useRef, useState, type CSSProperties, type ReactNode } from 'react' +import { memo, useCallback, useEffect, useMemo, useRef, useState, type CSSProperties, type ReactNode } from 'react' import clsx from 'clsx' import { flushSync } from 'react-dom' import { @@ -16,7 +16,7 @@ import { } from '@deepseek-ai/dsh-client-ui-primitives' import type { PropsLocale } from '@deepseek-ai/dsh-client-ui-slots' import type { ForgeDiffFile } from '../types.ts' -import { drawnLines, segmentsOf, splitRows, type DiffRow, type DiffSegment, type SplitCell, type SplitRow } from './diff-rows.ts' +import { drawnLines, segmentsOf, splitRows, type DiffRow, type SplitCell, type SplitRow } from './diff-rows.ts' import { codeSideAt, readCodeSelection } from './code-selection.ts' import { reviewMatches, selectReviewLines, type ReviewSelection } from './review.ts' import { ReviewThread, useFileReview } from './ReviewThread.tsx' @@ -26,7 +26,7 @@ import reviewCss from './ReviewComposer.module.css' import { FileLabel } from './FileLabel.tsx' import type { ForgePanelKey, NS } from './locales.ts' import type { ViewedState } from './viewed-files.ts' -import { SectionReachContext, ViewVisibleContext, whenIdle } from './section-reach.ts' +import type { SectionReach } from './section-reach.ts' import css from './ChangesTab.module.css' type Translate = PropsLocale['t'] @@ -59,10 +59,14 @@ export interface DiffFileHandlers { readonly actions: (file: ForgeDiffFile) => readonly DiffFileAction[] /** Receive a file's section element, which scrolling to the file targets, or null once it unmounts. */ readonly section: (path: string, element: HTMLElement | null) => void + /** Tells each expanded section when it comes within reach of the scrollport, which draws its lines. */ + readonly reach: SectionReach } /** Props of one file section. */ export interface DiffFileProps { + /** Whether the owning tab is visible and may show temporary menus. */ + readonly visible: boolean /** Comment destinations and the source being reviewed. */ readonly review?: ReviewActions | undefined /** The file and its hunks. */ @@ -113,90 +117,26 @@ export function Counts({ file, t }: { const SIGN: Record = { add: '+', del: '-', context: '' } -/** Highlighted tokens of one side's lines by line number. */ -type SideSpans = Map +/** Highlighted tokens of one side's lines by line number, or undefined while the grammar is unavailable. */ +type SideSpans = ReadonlyMap | undefined -/** Highlighted tokens of a file's drawn lines on each side; a line without an entry draws as plain text. */ -interface FileSpans { - readonly old: SideSpans - readonly new: SideSpans -} +/** A segment ready to draw: collapsed lines, or rows with the tokens of each side. */ +type Prepared = + | { readonly kind: 'gap'; readonly index: number; readonly count: number | null } + | { readonly kind: 'rows'; readonly rows: readonly DiffRow[]; readonly old: SideSpans; readonly new: SideSpans } -/** Lines highlighted as one fragment; a longer run splits here, where a token spanning the split restarts. */ -const HIGHLIGHT_CHUNK_LINES = 100 - -/** Numbered lines of one side that are highlighted together. */ -interface HighlightChunk { - readonly side: 'old' | 'new' - readonly lines: readonly { readonly no: number; readonly text: string }[] -} - -/** Split each run of rows of each side into fragments of at most {@link HIGHLIGHT_CHUNK_LINES} lines. */ -function highlightChunks(segments: readonly DiffSegment[]): HighlightChunk[] { - const chunks: HighlightChunk[] = [] - for (const segment of segments) { - if (segment.kind === 'gap') continue - for (const side of ['old', 'new'] as const) { - const lines: { readonly no: number; readonly text: string }[] = [] - for (const row of segment.rows) { - const no = row[side] - if (no !== null) lines.push({ no, text: row.text }) - } - for (let at = 0; at < lines.length; at += HIGHLIGHT_CHUNK_LINES) { - chunks.push({ side, lines: lines.slice(at, at + HIGHLIGHT_CHUNK_LINES) }) - } - } +/** Highlight one side of a run of rows as one fragment, so tokens spanning lines keep their state. */ +function sideSpans(rows: readonly DiffRow[], side: 'old' | 'new', highlighter: CodeHighlighter): SideSpans { + const numbered: { readonly no: number; readonly text: string }[] = [] + for (const row of rows) { + const no = row[side] + if (no !== null) numbered.push({ no, text: row.text }) } - return chunks -} - -/** Highlight one fragment, so tokens spanning its lines keep their state, into `spans`. */ -function addChunkSpans(spans: FileSpans, chunk: HighlightChunk, highlighter: CodeHighlighter): void { - const highlighted = highlighter(chunk.lines.map(line => line.text).join('\n')) - /* v8 ignore next -- a grammar that reported ready stays registered, so its fragments always tokenize */ - if (highlighted === undefined) return + if (numbered.length === 0) return new Map() + const highlighted = highlighter(numbered.map(line => line.text).join('\n')) + if (highlighted === undefined) return undefined // Shiki omits a final empty token row, so a final empty line gets no tokens. - for (const [index, line] of chunk.lines.entries()) spans[chunk.side].set(line.no, highlighted[index] ?? []) -} - -/** - * Highlight a file's drawn lines in idle time while the scrollport is still, once its language's grammar is ready: - * each idle period highlights fragments while it lasts, at least one, and the lines take their tokens when the last - * fragment is done. Until then, the lines keep the tokens of the previous pass, by line number, and new lines draw as - * plain text. A grammar that loads for another language starts no pass. - */ -function useFileSpans(path: string, segments: readonly DiffSegment[]): FileSpans | undefined { - const reach = useContext(SectionReachContext) - const highlighter = useCodeHighlighter(languageForPath(path)) - // The highlighter changes whenever any grammar loads; the empty fragment tells whether this language's is ready. - const ready = highlighter('') !== undefined - const current = useRef(highlighter) - current.current = highlighter - const [spans, setSpans] = useState(undefined) - useEffect(() => { - if (!ready) return - const chunks = highlightChunks(segments) - const pass: FileSpans = { old: new Map(), new: new Map() } - let next = 0 - const step = (deadline: IdleDeadline | undefined): void => { - for (;;) { - const chunk = chunks[next] - if (chunk === undefined) { - setSpans(pass) - return - } - addChunkSpans(pass, chunk, current.current) - next++ - if ((deadline?.timeRemaining() ?? 0) <= 0) break - } - cancel = later(step) - } - // Outside a changes view nothing scrolls the sections, so idle time alone gates the pass. - const later = reach === null ? whenIdle : (task: (deadline: IdleDeadline | undefined) => void) => reach.whenSettled(task) - let cancel = later(step) - return () => { cancel() } - }, [segments, ready, reach]) - return spans + return new Map(numbered.map((line, index) => [line.no, highlighted[index] ?? []])) } function Code({ text, spans, old, next }: { @@ -217,20 +157,16 @@ function pairKind(pair: SplitRow): 'context' | 'change' { return pair.left !== null && pair.left.kind === 'context' ? 'context' : 'change' } -/** The composer of a new comment, shown with its menus only while the changes tab is visible. */ -function VisibleComposer(props: Omit[0], 'visible'>): ReactNode { - return -} - /** The hunks of one file with their collapsed and expanded unchanged lines. */ -function Lines({ file, text, open, handlers, split, wrap, review, comments, t }: Pick & { readonly comments: FileReviewState }): ReactNode { +function Lines({ file, text, open, handlers, split, wrap, review, visible, comments, t }: Pick & { readonly comments: FileReviewState }): ReactNode { const lines = typeof text === 'object' ? text : undefined const columns = split && file.additions > 0 && file.deletions > 0 const singleSide = file.status === 'added' ? 'R' : file.status === 'deleted' ? 'L' : null let maximum = Math.max(1, lines?.length ?? 0, (lines?.length ?? 0) + file.deletions - file.additions) for (const hunk of file.hunks) maximum = Math.max(maximum, hunk.oldStart + hunk.oldLines - 1, hunk.newStart + hunk.newLines - 1) const style: CSSProperties & { '--forge-number-width': string } = { '--forge-number-width': `calc(${Math.max(2, String(maximum).length)}ch + 12px)` } + const highlighter = useCodeHighlighter(languageForPath(file.path)) const [selection, setSelection] = useState(null) const root = useRef(null) const origin = useRef<{ side: 'L' | 'R'; line: number } | null>(null) @@ -288,8 +224,9 @@ function Lines({ file, text, open, handlers, split, wrap, review, comments, t }: window.removeEventListener('pointercancel', finish) } }, [selecting, nativeSelection]) - const segments = useMemo(() => segmentsOf(file, lines, open), [file, lines, open]) - const spans = useFileSpans(file.path, segments) + const prepared = useMemo(() => segmentsOf(file, lines, open).map((segment): Prepared => segment.kind === 'gap' + ? segment + : { ...segment, old: sideSpans(segment.rows, 'old', highlighter), new: sideSpans(segment.rows, 'new', highlighter) }), [file, lines, open, highlighter]) // Returns whether the range was accepted; a range across hunks or outside them keeps the previous selection. const choose = (side: 'L' | 'R', line: number, extend: boolean): boolean => { if (review === undefined || editing) return false @@ -363,15 +300,15 @@ function Lines({ file, text, open, handlers, split, wrap, review, comments, t }: )} } - const cell = (value: SplitCell | null, side: 'L' | 'R'): ReactNode => value === null + const cell = (value: SplitCell | null, spans: SideSpans, side: 'L' | 'R'): ReactNode => value === null ? : - {number(side, value.no, true)} + {number(side, value.no, true)} const rows: ReactNode[] = [] let inserted = false - const editor = review === undefined || selection === null || !editing ? null : { const native = window.getSelection() if (root.current?.contains(native?.anchorNode ?? null) === true) native?.removeAllRanges() @@ -392,7 +329,7 @@ function Lines({ file, text, open, handlers, split, wrap, review, comments, t }: } const endsAt = (old: number | null, next: number | null): boolean => selection !== null && (selection.comment.side === 'L' ? old : next) === selection.comment.endLine - for (const [at, segment] of segments.entries()) { + for (const [at, segment] of prepared.entries()) { if (segment.kind === 'gap') { rows.push(
{expanded && (waiting ?
: )} diff --git a/packages/experimental/client-ui-forge/src/client/section-reach.ts b/packages/experimental/client-ui-forge/src/client/section-reach.ts index 1cb3997531..79a1d16561 100644 --- a/packages/experimental/client-ui-forge/src/client/section-reach.ts +++ b/packages/experimental/client-ui-forge/src/client/section-reach.ts @@ -4,14 +4,9 @@ * mounted, so long diffs draw only what the reader approaches. */ -import { createContext } from 'react' - /** Distance beyond the scrollport's edges within which a section counts as reached. */ const REACH_MARGIN = '1200px 0px' -/** Time, in milliseconds, the scrollport stays still before deferred highlighting runs. */ -const SCROLL_QUIET_MS = 200 - const noop = (): void => {} /** @@ -21,33 +16,6 @@ const noop = (): void => {} export class SectionReach { private observer: IntersectionObserver | undefined private readonly waiting = new Map void>() - private scrolledAt = Number.NEGATIVE_INFINITY - - /** Record that the scrollport scrolled, which holds back work started through {@link whenSettled}. */ - scrolled(): void { - this.scrolledAt = performance.now() - } - - /** - * Run `task` once the page is idle and the scrollport has been still for {@link SCROLL_QUIET_MS}, so work that - * can wait never interrupts a scroll in progress. - * @param task - deferrable work; it receives the idle period, or undefined without one. - * @returns the canceller, which has no effect after `task` ran. - */ - whenSettled(task: (deadline: IdleDeadline | undefined) => void): () => void { - let cancel = noop - const attempt = (deadline: IdleDeadline | undefined): void => { - const wait = this.scrolledAt + SCROLL_QUIET_MS - performance.now() - if (wait <= 0) { - task(deadline) - return - } - const id = setTimeout(() => { cancel = whenIdle(attempt) }, wait) - cancel = () => { clearTimeout(id) } - } - cancel = whenIdle(attempt) - return () => { cancel() } - } /** * Measure sections against `root`, or stop measuring when it unmounts. Sections that are still waiting move to the new root. @@ -91,26 +59,3 @@ export class SectionReach { } } } - -/** The reach of the enclosing changes view; null outside one, where every section counts as reached. */ -export const SectionReachContext = createContext(null) - -/** Whether the enclosing changes tab is visible and may show temporary menus; true outside one. */ -export const ViewVisibleContext = createContext(true) - -/** Longest wait, in milliseconds, before deferred highlighting runs although the page stays busy. */ -const IDLE_TIMEOUT_MS = 500 - -/** - * Run `task` when the page is next idle, or after other queued work where the browser lacks `requestIdleCallback`. - * @param task - work that may wait for a frame to finish; it receives the idle period, or undefined without one. - * @returns the canceller, which has no effect after `task` ran. - */ -export function whenIdle(task: (deadline: IdleDeadline | undefined) => void): () => void { - if (typeof requestIdleCallback === 'function') { - const id = requestIdleCallback(task, { timeout: IDLE_TIMEOUT_MS }) - return () => { cancelIdleCallback(id) } - } - const id = setTimeout(() => { task(undefined) }, 0) - return () => { clearTimeout(id) } -} diff --git a/packages/experimental/client-ui-forge/tests/changes-tab.client.spec.tsx b/packages/experimental/client-ui-forge/tests/changes-tab.client.spec.tsx index 230f00809e..36be08a879 100644 --- a/packages/experimental/client-ui-forge/tests/changes-tab.client.spec.tsx +++ b/packages/experimental/client-ui-forge/tests/changes-tab.client.spec.tsx @@ -1,7 +1,7 @@ // @vitest-environment jsdom /** The changes tab: reading each source and scope, the toolbar, file sections, unchanged lines, notices, and failures. */ import { useSyncExternalStore } from 'react' -import { act, cleanup, fireEvent, render, waitFor, within } from '@testing-library/react' +import { act, cleanup, fireEvent, render, within } from '@testing-library/react' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { PaneId, SidebarRightTabActions, SidebarRightTabInfo, TabId } from '@deepseek-ai/dsh-client-ui-sidebar-right/client' import { createSnapshotStore } from '@deepseek-ai/dsh-client-store' @@ -836,30 +836,6 @@ describe('ChangesTab drawing work', () => { } }) - it('highlights a file in idle periods, as many fragments as each period allows, then draws the tokens at once', async () => { - // Idle periods with this much time left, in turn; afterwards each period has none. - const remaining = [5] - const periods: number[] = [] - vi.stubGlobal('requestIdleCallback', (task: IdleRequestCallback) => setTimeout(() => { - const left = remaining.shift() ?? 0 - periods.push(left) - task({ didTimeout: false, timeRemaining: () => left }) - }, 0)) - vi.stubGlobal('cancelIdleCallback', (id: number) => { clearTimeout(id) }) - try { - const m = mount() - await settle() - const app = m.section('src/app.ts') - await waitFor(() => { expect(app.querySelectorAll('[data-diff-code]').length).toBeGreaterThan(0) }) - // The first period highlights both sides of the file's one run and still has time; the second finds nothing left. - expect(periods.slice(0, 2)).toEqual([5, 0]) - } finally { - // Unmounting cancels pending passes through the stubbed canceller. - cleanup() - vi.unstubAllGlobals() - } - }) - it('leaves file sections undrawn while the reader filters the file list', async () => { // The memoized section draws through its inner component, which the spy counts. const draw = vi.spyOn(DiffFileModule.DiffFile, 'type') @@ -883,8 +859,7 @@ describe('ChangesTab layout', () => { expect([...app.querySelectorAll('[data-diff-pair]')].map(pair => pair.getAttribute('data-diff-pair'))).toEqual([ 'context', 'change', 'change', 'context', 'context', 'change', ]) - // Highlighting waits for the page to be idle. - await waitFor(() => { expect(app.querySelectorAll('[data-diff-code]').length).toBeGreaterThan(0) }) + expect(app.querySelectorAll('[data-diff-code]').length).toBeGreaterThan(0) expect(m.section('docs/gone.txt').querySelector('[data-diff-code]')).toBeNull() expect(rowTexts(m.section('docs/gone.txt'))).toEqual(['bye']) expect(rowTexts(m.section('src/new.ts'))).toEqual(['export const added = 1', '']) diff --git a/packages/experimental/client-ui-forge/tests/diff-review.client.spec.tsx b/packages/experimental/client-ui-forge/tests/diff-review.client.spec.tsx index 71bc95a914..ed756db246 100644 --- a/packages/experimental/client-ui-forge/tests/diff-review.client.spec.tsx +++ b/packages/experimental/client-ui-forge/tests/diff-review.client.spec.tsx @@ -1,11 +1,12 @@ // @vitest-environment jsdom /** Diff gestures, browser clipboard output, and retained comments at their captured lines. */ -import { act, cleanup, fireEvent, render, waitFor, within } from '@testing-library/react' +import { act, cleanup, fireEvent, render, within } from '@testing-library/react' import { afterEach, expect, it, vi } from 'vitest' import { makeTranslate } from '@deepseek-ai/dsh-client-test-runtime' import { DiffFile, type DiffFileProps } from '../src/client/DiffFile.tsx' import { en } from '../src/client/locales.ts' import { ReviewQueue } from '../src/client/review-queue.ts' +import { SectionReach } from '../src/client/section-reach.ts' import { selectReviewLines } from '../src/client/review.ts' import type { ReviewActions } from '../src/client/ReviewComposer.tsx' import type { ForgeDiffFile } from '../src/types.ts' @@ -20,8 +21,10 @@ function mount(overrides: Partial = {}) { const addToInput = vi.fn().mockReturnValue(true) const records = new ReviewQueue('diff-review') const review: ReviewActions = { source, addToInput, records, onQuoted: (item) => { records.recordAgent(item) } } - const handlers = { toggle: vi.fn(), reveal: vi.fn(), expand: vi.fn(), view: vi.fn(), actions: () => [], section: vi.fn() } - const props: DiffFileProps = { file, expanded: true, large: false, revealed: false, + const handlers = { + toggle: vi.fn(), reveal: vi.fn(), expand: vi.fn(), view: vi.fn(), actions: () => [], section: vi.fn(), reach: new SectionReach(), + } + const props: DiffFileProps = { file, visible: true, expanded: true, large: false, revealed: false, text: undefined, open: new Set(), split: true, wrap: true, badge: '', tooltip: 'a.txt', viewed: 'unviewed', handlers, review, t, ...overrides } const view = render() @@ -250,8 +253,27 @@ it('places left and right records in unified rows and retains outdated records f expect(within(m.container).getByRole('button', { name: /changed lines/ })).toBeTruthy() }) -it('highlights a section drawn outside a changes view once the page is idle', async () => { +it('draws a file\'s highlighted lines in the same render, so a changed file never shows its previous text', () => { + const code = (text: string): ForgeDiffFile => ({ ...file, path: 'a.ts', additions: 1, deletions: 0, status: 'added', + hunks: [{ oldStart: 0, oldLines: 0, newStart: 1, newLines: 1, lines: [`+${text}`] }] }) + const m = mount({ file: code('const value = old()'), split: false }) + const line = (): HTMLElement => m.code('R', 1) + expect(line().textContent).toBe('const value = old()') + expect(line().hasAttribute('data-diff-code')).toBe(true) + + m.rerender() + expect(line().textContent).toBe('const value = renewed()') + expect(line().hasAttribute('data-diff-code')).toBe(true) +}) + +it('keeps a text selection across a render that leaves the file unchanged', () => { const m = mount({ file: { ...file, path: 'a.ts' } }) - expect(m.container.querySelector('[data-diff-code]')).toBeNull() - await waitFor(() => { expect(m.container.querySelectorAll('[data-diff-code]').length).toBeGreaterThan(0) }) + const start = m.code('R', 3) + const node = start.firstChild + act(() => { document.getSelection()!.setBaseAndExtent(start, 0, start, start.childNodes.length) }) + const selected = document.getSelection()?.toString() + expect(selected).toBe('new first') + m.rerender() + expect(m.code('R', 3).firstChild).toBe(node) + expect(document.getSelection()?.toString()).toBe(selected) }) diff --git a/packages/experimental/client-ui-forge/tests/section-reach.client.spec.ts b/packages/experimental/client-ui-forge/tests/section-reach.client.spec.ts index f33a2422ec..dbbde5d293 100644 --- a/packages/experimental/client-ui-forge/tests/section-reach.client.spec.ts +++ b/packages/experimental/client-ui-forge/tests/section-reach.client.spec.ts @@ -1,7 +1,7 @@ // @vitest-environment jsdom -/** File sections reach the scrollport once, and deferred highlighting waits for idle time. */ +/** File sections reach the scrollport once. */ import { afterEach, describe, expect, it, vi } from 'vitest' -import { SectionReach, whenIdle } from '../src/client/section-reach.ts' +import { SectionReach } from '../src/client/section-reach.ts' /** An IntersectionObserver double that records its options and the elements it watches. */ class FakeObserver { @@ -80,69 +80,3 @@ describe('SectionReach', () => { expect(FakeObserver.made).toHaveLength(2) }) }) - -describe('SectionReach.whenSettled', () => { - it('waits for the scrollport to stay still before running deferred work, and can be cancelled while it waits', () => { - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] }) - try { - vi.stubGlobal('requestIdleCallback', undefined) - const reach = new SectionReach() - const settled = vi.fn() - reach.whenSettled(settled) - vi.advanceTimersByTime(0) - expect(settled).toHaveBeenCalledOnce() - - reach.scrolled() - const held = vi.fn() - reach.whenSettled(held) - vi.advanceTimersByTime(0) - expect(held).not.toHaveBeenCalled() - vi.advanceTimersByTime(100) - reach.scrolled() - vi.advanceTimersByTime(100) - expect(held).not.toHaveBeenCalled() - vi.advanceTimersByTime(200) - expect(held).toHaveBeenCalledOnce() - - reach.scrolled() - const cancelled = vi.fn() - const stop = reach.whenSettled(cancelled) - vi.advanceTimersByTime(0) - stop() - vi.advanceTimersByTime(500) - expect(cancelled).not.toHaveBeenCalled() - } finally { - vi.useRealTimers() - } - }) -}) - -describe('whenIdle', () => { - it('waits for an idle period, at most the timeout, and can be cancelled', () => { - const request = vi.fn<(task: () => void, options: { timeout: number }) => number>(() => 7) - const cancel = vi.fn() - vi.stubGlobal('requestIdleCallback', request) - vi.stubGlobal('cancelIdleCallback', cancel) - const task = vi.fn() - const stop = whenIdle(task) - expect(request).toHaveBeenCalledWith(task, { timeout: 500 }) - stop() - expect(cancel).toHaveBeenCalledWith(7) - }) - - it('runs after queued work where the browser has no requestIdleCallback', () => { - vi.useFakeTimers() - try { - vi.stubGlobal('requestIdleCallback', undefined) - const ran = vi.fn() - whenIdle(ran) - const cancelled = vi.fn() - whenIdle(cancelled)() - vi.runAllTimers() - expect(ran).toHaveBeenCalledOnce() - expect(cancelled).not.toHaveBeenCalled() - } finally { - vi.useRealTimers() - } - }) -}) From fcc33725d9b3539a03063852db47f2478252c38c Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Mon, 9 Nov 2026 12:00:23 +0800 Subject: [PATCH 28/30] fix(worktree): keep an early streamed answer and document cleanup limits - Return the early answer from scanCommand without awaiting a successful outcome: the SSH provider rejects done when termination cancels a launch it has not yet acknowledged, which turned a retained dirty checkout into a failed cleanup. - Test that a never-initialized submodule is removed and that Git refuses a committed embedded repository, which inspection leaves to Git by design. - Record the git-status-only trade-off and its sparse-checkout rationale in the worktree Agent Note, and align the README, upgrade guide, maxOutputBytes description, and comments with what is streamed. Refs #8151 --- ...09-13-explicit-worktree-creation.i18n.yaml | 4 +- .../2026-09-13-explicit-worktree-creation.md | 2 +- ...026-09-13-explicit-worktree-creation.zh.md | 2 +- docs/config-catalog.md | 3 +- docs/config-catalog.zh.md | 3 +- .../archive-worktrees/guide.i18n.yaml | 4 +- .../v0.2.2-rc.1/archive-worktrees/guide.md | 2 +- .../v0.2.2-rc.1/archive-worktrees/guide.zh.md | 2 +- .../experimental/worktree/README.i18n.yaml | 4 +- packages/experimental/worktree/README.md | 2 +- packages/experimental/worktree/README.zh.md | 2 +- packages/experimental/worktree/src/index.ts | 3 +- packages/experimental/worktree/src/process.ts | 17 ++++-- packages/experimental/worktree/src/removal.ts | 6 +- .../worktree/tests/process.spec.ts | 60 +++++++++++++++++++ .../worktree/tests/removal.spec.ts | 28 +++++++++ 16 files changed, 122 insertions(+), 22 deletions(-) create mode 100644 packages/experimental/worktree/tests/process.spec.ts diff --git a/.agents/notes/implemented/feature/2026-09-13-explicit-worktree-creation.i18n.yaml b/.agents/notes/implemented/feature/2026-09-13-explicit-worktree-creation.i18n.yaml index b5373db47c..201593ebbe 100644 --- a/.agents/notes/implemented/feature/2026-09-13-explicit-worktree-creation.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-09-13-explicit-worktree-creation.i18n.yaml @@ -9,8 +9,8 @@ en: 12ab239ab0c58e31 zh: c32874442ce66041 /agent-note-explicit-worktree-creation/decision: - en: 92782d5964d1a021 - zh: 37b385836e9ae2b9 + en: 571a2d4c5352170b + zh: aeb12b849c5f8d31 /agent-note-explicit-worktree-creation/alternatives-considered: en: 72d42e892a37b94c zh: 92a14a51c6a89c53 diff --git a/.agents/notes/implemented/feature/2026-09-13-explicit-worktree-creation.md b/.agents/notes/implemented/feature/2026-09-13-explicit-worktree-creation.md index 33cabb9b8e..9facc68061 100644 --- a/.agents/notes/implemented/feature/2026-09-13-explicit-worktree-creation.md +++ b/.agents/notes/implemented/feature/2026-09-13-explicit-worktree-creation.md @@ -18,7 +18,7 @@ Git and directory creation use the mounted subprocess and sandbox providers with Leaving uses the shared working-directory operation and retains the checkout and branch. Failed setup can retain partial artifacts and reports their location. The service does not own a branch merge policy or a task association database. -The service records each creation as a `plugin:worktree/created` record and owns archive-triggered cleanup of those checkouts, as the job and Agent registries own their archive-admission families. A Session's inherited prefix decides whether it created a checkout or merely refers to one; cwd alone never grants cleanup ownership. Admission registers the attempt inside the Workspace write queue and returns; inspection reads cold Sessions through the projection cache or their log, without starting an Agent, and runs afterwards. Only a clean, unreferenced checkout is removed, without force and without deleting its branch; tracked changes and untracked files retain it, ignored files do not, matching Git's own non-force refusal. Processes and terminals are not consulted. A volatile `retainOnArchive` setting, read at admission, turns the cleanup off without unmounting the service; the Forge client edits it on the bundle's worktree row of the Plugins page because the service package has no Client half. Unarchiving cancels an unadmitted removal and waits for an admitted one, and directory entry into a checkout being removed waits for that removal. No startup sweep deletes previously archived worktrees. +The service records each creation as a `plugin:worktree/created` record and owns archive-triggered cleanup of those checkouts, as the job and Agent registries own their archive-admission families. A Session's inherited prefix decides whether it created a checkout or merely refers to one; cwd alone never grants cleanup ownership. Admission registers the attempt inside the Workspace write queue and returns; inspection reads cold Sessions through the projection cache or their log, without starting an Agent, and runs afterwards. Only a clean, unreferenced checkout is removed, without force and without deleting its branch; tracked changes and untracked files retain it, ignored files do not, matching Git's own non-force refusal. Inspection trusts `git status` alone, as peer agent products do: edits to skip-worktree or assume-unchanged files are discarded with the checkout, because sparse checkout sets skip-worktree on every file outside its cone and inspecting those bits would retain every sparse checkout. A checkout whose administrative directory holds submodule repositories is retained; a committed embedded repository is not inspected, and Git's refusal of that removal settles the attempt as failed. Processes and terminals are not consulted. A volatile `retainOnArchive` setting, read at admission, turns the cleanup off without unmounting the service; the Forge client edits it on the bundle's worktree row of the Plugins page because the service package has no Client half. Unarchiving cancels an unadmitted removal and waits for an admitted one, and directory entry into a checkout being removed waits for that removal. No startup sweep deletes previously archived worktrees. ## Alternatives considered diff --git a/.agents/notes/implemented/feature/2026-09-13-explicit-worktree-creation.zh.md b/.agents/notes/implemented/feature/2026-09-13-explicit-worktree-creation.zh.md index a3c7313ddd..bc9fc0670c 100644 --- a/.agents/notes/implemented/feature/2026-09-13-explicit-worktree-creation.zh.md +++ b/.agents/notes/implemented/feature/2026-09-13-explicit-worktree-creation.zh.md @@ -18,7 +18,7 @@ Git 与目录创建使用挂载的子进程及沙箱提供方，不扩大调用 离开使用共享工作目录操作，保留检出目录与分支。初始化失败可能留下部分产物，并报告其位置。该服务不拥有分支合并策略或任务关联数据库。 -服务把每次创建记录为 `plugin:worktree/created` Session 事件，并像 job 与 Agent 注册表各自拥有归档准入家族那样，拥有这些检出目录的归档触发清理。Session 的继承前缀决定它创建了某个检出目录还是仅引用它；当前工作目录本身不授予清理所有权。准入在 Workspace 写队列内登记尝试并返回；检查通过投影缓存或日志读取冷 Session，不启动 Agent，并在之后执行。只删除干净且无人引用的检出目录，不带 force 也不删除分支；跟踪文件改动与未跟踪文件保留目录，被忽略的文件不保留，与 Git 自身不带 force 时的拒绝条件一致。不咨询进程与终端。volatile 的 `retainOnArchive` 设置在准入时读取，无需卸载服务即可关闭清理；由于服务包没有 Client 半部，Forge 客户端在插件页该套件的 worktree 行上编辑它。取消归档会取消尚未获准的删除并等待已获准的删除，进入正在删除的检出目录会等待该删除结束。不在启动时扫描删除此前已归档的工作树。 +服务把每次创建记录为 `plugin:worktree/created` Session 事件，并像 job 与 Agent 注册表各自拥有归档准入家族那样，拥有这些检出目录的归档触发清理。Session 的继承前缀决定它创建了某个检出目录还是仅引用它；当前工作目录本身不授予清理所有权。准入在 Workspace 写队列内登记尝试并返回；检查通过投影缓存或日志读取冷 Session，不启动 Agent，并在之后执行。只删除干净且无人引用的检出目录，不带 force 也不删除分支；跟踪文件改动与未跟踪文件保留目录，被忽略的文件不保留，与 Git 自身不带 force 时的拒绝条件一致。检查与同类 Agent 产品一样只依据 `git status`：skip-worktree 或 assume-unchanged 文件的改动随检出目录一起丢弃，因为 sparse checkout 会给稀疏范围之外的每个文件设置 skip-worktree，检查这些标记会保留每个稀疏检出目录。管理目录中含子模块仓库的检出目录会保留；已提交的内嵌仓库不在检查范围内，Git 拒绝这类删除，本次尝试以失败结束。不咨询进程与终端。volatile 的 `retainOnArchive` 设置在准入时读取，无需卸载服务即可关闭清理；由于服务包没有 Client 半部，Forge 客户端在插件页该套件的 worktree 行上编辑它。取消归档会取消尚未获准的删除并等待已获准的删除，进入正在删除的检出目录会等待该删除结束。不在启动时扫描删除此前已归档的工作树。 ## Alternatives considered diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 678d7387d2..ff2d26774a 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1582,7 +1582,8 @@ export interface Config { graceMs?: number /** * Maximum captured bytes per subprocess output stream; archive inspection - * streams `git status` and the worktree listing, so it bounds only their stderr. + * streams `git status`, and inspection and restoration stream the worktree + * listing, so it bounds only their stderr. */ maxOutputBytes?: number } diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index b1707d7c32..2988257982 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -1584,7 +1584,8 @@ export interface Config { graceMs?: number /** * Maximum captured bytes per subprocess output stream; archive inspection - * streams `git status` and the worktree listing, so it bounds only their stderr. + * streams `git status`, and inspection and restoration stream the worktree + * listing, so it bounds only their stderr. */ maxOutputBytes?: number } diff --git a/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.i18n.yaml b/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.i18n.yaml index c17fd006c0..b748855bae 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.i18n.yaml +++ b/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.i18n.yaml @@ -9,8 +9,8 @@ en: bc309764ab7b8b98 zh: 20fdfbce76706025 /worktree-cleanup-on-archive/change: - en: c16e3db7ee76eee8 - zh: a1db1e346d5af41a + en: 990690039f918e72 + zh: 8e23fbabf88a777f /worktree-cleanup-on-archive/migration: en: 05f29b760cb99ae9 zh: 33f181ac2a582bdf diff --git a/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.md b/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.md index 9232c6ecdd..74f3986277 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.md @@ -9,7 +9,7 @@ English | [中文](guide.zh.md) ## Change -Archiving a Session previously retained its checkout. The worktree service now records each checkout it creates as a `plugin:worktree/created` record and, when the Session that created one is archived, removes it with a non-force `git worktree remove` when no unarchived Session refers to it, its creator's policy is not read-only, and Git finds no tracked changes, untracked files, submodule repositories, locks, or detached HEAD. Ignored files such as `node_modules` and edits to skip-worktree or assume-unchanged files, which `git status` hides, do not retain it; Git discards them. Branches and commits remain. The service's `retainOnArchive` setting, off by default, keeps every checkout instead. Checkouts created before this release have no such event and are never swept, nor are Sessions archived earlier. See the [worktree service](../../../../packages/experimental/worktree/README.md). +Archiving a Session previously retained its checkout. The worktree service now records each checkout it creates as a `plugin:worktree/created` record and, when the Session that created one is archived, removes it with a non-force `git worktree remove` when no unarchived Session refers to it, its creator's policy is not read-only, and Git finds no tracked changes, untracked files, initialized submodules, locks, or detached HEAD. Ignored files such as `node_modules` and edits to skip-worktree or assume-unchanged files, which `git status` hides, do not retain it; Git discards them. Git refuses to remove a checkout holding a committed embedded repository, so that checkout stays and the archive notice reports a failed cleanup. Branches and commits remain. The service's `retainOnArchive` setting, off by default, keeps every checkout instead. Checkouts created before this release have no such event and are never swept, nor are Sessions archived earlier. See the [worktree service](../../../../packages/experimental/worktree/README.md). The Forge bundle mounts one `worktree` service for both the panel and the `create_worktree` tool. The `forge-worktree` row and the `optional-forge-panel` group that isolated a second service are gone; the panel is the top-level `forge-panel` row. diff --git a/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.zh.md b/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.zh.md index 0d32dded1c..d5295cfb18 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.zh.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/archive-worktrees/guide.zh.md @@ -9,7 +9,7 @@ description: "归档 Session 可以删除其干净的工作树并保留分支， ## 变更 -归档 Session 以前会保留检出目录。工作树服务现在把它创建的每个检出目录记录为 `plugin:worktree/created` Session 事件；当创建它的 Session 被归档时，若没有未归档的 Session 引用该目录、创建者的策略不是只读，且 Git 未发现跟踪文件改动、未跟踪文件、子模块仓库、锁或分离 HEAD，就用不带 force 的 `git worktree remove` 删除它。`node_modules` 之类被忽略的文件，以及 `git status` 不报告的 skip-worktree 或 assume-unchanged 文件改动，都不会保留目录，Git 会丢弃它们。分支和提交仍会保留。服务的 `retainOnArchive` 设置默认关闭，开启后改为保留所有检出目录。本版本之前创建的检出目录没有该事件，不会被扫描；更早归档的 Session 也不会。参见[工作树服务](../../../../packages/experimental/worktree/README.zh.md)。 +归档 Session 以前会保留检出目录。工作树服务现在把它创建的每个检出目录记录为 `plugin:worktree/created` Session 事件；当创建它的 Session 被归档时，若没有未归档的 Session 引用该目录、创建者的策略不是只读，且 Git 未发现跟踪文件改动、未跟踪文件、已初始化的子模块、锁或分离 HEAD，就用不带 force 的 `git worktree remove` 删除它。`node_modules` 之类被忽略的文件，以及 `git status` 不报告的 skip-worktree 或 assume-unchanged 文件改动，都不会保留目录，Git 会丢弃它们。含已提交内嵌仓库的检出目录会被 Git 拒绝删除，因此保留，归档提示显示清理失败。分支和提交仍会保留。服务的 `retainOnArchive` 设置默认关闭，开启后改为保留所有检出目录。本版本之前创建的检出目录没有该事件，不会被扫描；更早归档的 Session 也不会。参见[工作树服务](../../../../packages/experimental/worktree/README.zh.md)。 Forge bundle 为面板和 `create_worktree` 工具挂载同一个 `worktree` 服务。隔离第二个服务的 `forge-worktree` 行与 `optional-forge-panel` 组已移除；面板是顶层的 `forge-panel` 行。 diff --git a/packages/experimental/worktree/README.i18n.yaml b/packages/experimental/worktree/README.i18n.yaml index 788a8dfda7..5e9a6604cb 100644 --- a/packages/experimental/worktree/README.i18n.yaml +++ b/packages/experimental/worktree/README.i18n.yaml @@ -15,8 +15,8 @@ en: d152484eb41ac6b4 zh: 2f8a621b991260a1 /deepseek-ai-dsh-experimental-worktree/use-this-package: - en: 56af10241dc270ee - zh: 2519395ab9641235 + en: 7fb20cca0a3154e5 + zh: 41bf0aa2bf753c07 /deepseek-ai-dsh-experimental-worktree/understand-the-implementation: en: 6971a7ca9073bef1 zh: b977c28a75c7f963 diff --git a/packages/experimental/worktree/README.md b/packages/experimental/worktree/README.md index 4c394a6d2a..e164f86600 100644 --- a/packages/experimental/worktree/README.md +++ b/packages/experimental/worktree/README.md @@ -36,7 +36,7 @@ Mount the service alongside `workingDirectory`, `fs`, `subprocess`, `sandbox`, ` `ctx.worktrees.restore(agent, { path, branch, baseCommit, repositoryRoot }, signal?, onPhase?)` checks a removed worktree out again at its recorded directory. It validates the recorded repository root, full commit object name, branch, and canonical target inside the configured pool before mutation. It removes only the missing target's unlocked registration with `git worktree remove`; other worktrees and locked registrations remain unchanged. It then checks out the branch when it still exists, or creates the branch again at `baseCommit` when it does not, and reports which happened as `branchRecreated`. An existing directory is an error. The calling Session's working directory does not change, and changes that were never committed are not recovered. Checkout disables filters as creation does. `onPhase` receives `prepare` before validation, target-registration cleanup, and branch creation, and `checkout` before the files are written; it must not throw. -Every creation appends a `plugin:worktree/created` record to the Session log after the directory change, so the `worktree` projection lists the Session's checkouts with `owned` false for one inside a fork-inherited prefix. When the Workspace archives a Session, the service registers one `scheduled` cleanup for the first checkout that Session created itself and returns; the inspection runs in the background. While `retainOnArchive` is on, archiving registers nothing and every checkout stays; the field is volatile and read at each archive, and the [Forge panel](../client-ui-forge/README.md) edits it as a switch on the bundle's worktree row of the Plugins page. The checkout is retained while any unarchived Session refers to it, by a recorded creation or by a working directory inside it (a Session beneath an archived subagent ancestor counts as archived; cold Sessions are read from the projection cache or their log), and while the creator's sandbox policy is read-only. An eligible checkout is removed with a non-force `git worktree remove` under the creator's policy after a second use check inside the path lock, which also holds `ensure` and `set` into that path through `working-directory/access`; the branch and its commits remain. Tracked changes, untracked files, submodule repositories (Git refuses to remove those), locks, detached HEAD, a mismatched branch, and targets outside the configured pool retain the checkout. Ignored files and edits to skip-worktree or assume-unchanged files do not: `git status` reports neither, and Git discards them with the checkout. Inspection reads `git status` only up to its first entry and streams the worktree listing, so neither is bounded by `maxOutputBytes`. Processes are not consulted: a process still writing there makes Git refuse, and an idle one fails its next command. Unarchiving cancels an unadmitted removal and waits for an admitted one. `cleanupOf(session, revision)` reads the attempt, or waits for a named revision to settle, as `scheduled`, `retained`, `removed`, or `failed`; a failure leaves the Session archived and logs a warning. Checkouts created before this event existed, and Sessions archived earlier, are not swept. +Every creation appends a `plugin:worktree/created` record to the Session log after the directory change, so the `worktree` projection lists the Session's checkouts with `owned` false for one inside a fork-inherited prefix. When the Workspace archives a Session, the service registers one `scheduled` cleanup for the first checkout that Session created itself and returns; the inspection runs in the background. While `retainOnArchive` is on, archiving registers nothing and every checkout stays; the field is volatile and read at each archive, and the [Forge panel](../client-ui-forge/README.md) edits it as a switch on the bundle's worktree row of the Plugins page. The checkout is retained while any unarchived Session refers to it, by a recorded creation or by a working directory inside it (a Session beneath an archived subagent ancestor counts as archived; cold Sessions are read from the projection cache or their log), and while the creator's sandbox policy is read-only. An eligible checkout is removed with a non-force `git worktree remove` under the creator's policy after a second use check inside the path lock, which also holds `ensure` and `set` into that path through `working-directory/access`; the branch and its commits remain. Tracked changes, untracked files, an initialized submodule (a repository in the checkout's administrative `modules` directory), locks, detached HEAD, a mismatched branch, and targets outside the configured pool retain the checkout. Ignored files and edits to skip-worktree or assume-unchanged files do not: `git status` reports neither, and Git discards them with the checkout. A committed embedded repository is not inspected; Git refuses that removal, so the attempt settles as `failed`. Inspection stops reading `git status` at its first entry and streams the worktree listing, keeping its entries in memory while it looks for the checkout; neither output is bounded by `maxOutputBytes`. Processes are not consulted: a process still writing there makes Git refuse, and an idle one fails its next command. Unarchiving cancels an unadmitted removal and waits for an admitted one. `cleanupOf(session, revision)` reads the attempt, or waits for a named revision to settle, as `scheduled`, `retained`, `removed`, or `failed`; a failure leaves the Session archived and logs a warning. Checkouts created before this event existed, and Sessions archived earlier, are not swept. The default checkout directory is `/.agents/worktrees/`. The returned `repositoryRoot` is the top-level directory of the caller’s current checkout; when it is a linked checkout, the pool is nested inside it. Omitted names join the `dsh-` prefix, the first eight hexadecimal digits of the UUID in the Session id, a hyphen, and four random hexadecimal digits, such as `dsh-6b1f02c4-9a3e`; a Session id without a UUID gets eight random hexadecimal digits after the prefix. `generatedWorktreeName` exports the same rule for callers that name a worktree before creating it. The service writes a `.gitignore` containing `*` and a trailing newline only when its allocation creates the pool; existing files and ignore rules are preserved. Each Git and Node command runs under its own `timeoutMs` deadline, 5 minutes by default, so checking out a large repository on a slow filesystem can finish; cancellation still stops the running command. Relative `directory`, generated `namePrefix`, executable choices, process limits, and `retainOnArchive` are configurable in the [configuration catalog](../../../docs/config-catalog.md#deepseek-aidsh-experimental-worktree). diff --git a/packages/experimental/worktree/README.zh.md b/packages/experimental/worktree/README.zh.md index 262ef39480..5c497a2606 100644 --- a/packages/experimental/worktree/README.zh.md +++ b/packages/experimental/worktree/README.zh.md @@ -36,7 +36,7 @@ kind: "package-reference" `ctx.worktrees.restore(agent, { path, branch, baseCommit, repositoryRoot }, signal?, onPhase?)` 在记录的目录重新检出被删除的 worktree。它在修改前校验记录中的仓库根目录、完整提交对象名、分支和配置池内的规范目标路径。它只通过 `git worktree remove` 移除缺失目标的未锁定登记；其他 worktree 与锁定登记保持不变。随后在分支仍存在时检出该分支，否则在 `baseCommit` 重新创建同名分支，并通过 `branchRecreated` 报告是哪种情况。目录已存在时会报错。调用方 Session 的工作目录不变，从未提交的修改无法恢复。检出与创建时一样禁用过滤器。`onPhase` 在检查目录、清理登记与创建分支之前收到 `prepare`，在写入文件之前收到 `checkout`；它不能抛出异常。 -每次创建都会在目录切换之后向 Session 日志追加一条 `plugin:worktree/created` 事件，因此 `worktree` 投影列出该 Session 的检出目录，位于 fork 继承前缀内的条目 `owned` 为 false。Workspace 归档某个 Session 时，服务为该 Session 自己创建的第一个检出目录登记一条 `scheduled` 清理并立即返回；检查在后台进行。`retainOnArchive` 开启时，归档不登记任何清理，所有检出目录都保留；该字段是 volatile 字段，每次归档时读取，[Forge 面板](../client-ui-forge/README.zh.md)在插件页该套件的 worktree 行上以开关编辑它。只要任一未归档 Session 仍引用该目录——通过记录的创建，或通过位于其中的工作目录（祖先子代理已归档的 Session 视为已归档；冷 Session 从投影缓存或其日志读取）——或创建者的沙箱策略为只读，目录就保留。符合条件的目录在路径锁内再做一次使用检查后，用创建者的策略执行不带 force 的 `git worktree remove`；路径锁同时通过 `working-directory/access` 阻塞进入该路径的 `ensure` 与 `set`；分支及其提交保留。跟踪文件改动、未跟踪文件、子模块仓库（Git 拒绝删除这类目录）、锁、分离 HEAD、分支不匹配以及位于配置池之外的目标都会保留目录。被忽略的文件以及对 skip-worktree 或 assume-unchanged 文件的改动不会：`git status` 不报告这两类内容，Git 删除目录时一并丢弃它们。检查只读取 `git status` 的第一条记录，并以流式读取工作树列表，因此两者都不受 `maxOutputBytes` 限制。不检查进程：仍在写入的进程会让 Git 拒绝删除，空闲进程的下一条命令会失败。取消归档会取消尚未获准的删除，并等待已获准的删除结束。`cleanupOf(session, revision)` 读取本次尝试，或等待指定修订号结束，状态为 `scheduled`、`retained`、`removed` 或 `failed`；失败时 Session 仍保持归档并记录警告。此事件存在之前创建的检出目录，以及更早归档的 Session，不会被扫描。 +每次创建都会在目录切换之后向 Session 日志追加一条 `plugin:worktree/created` 事件，因此 `worktree` 投影列出该 Session 的检出目录，位于 fork 继承前缀内的条目 `owned` 为 false。Workspace 归档某个 Session 时，服务为该 Session 自己创建的第一个检出目录登记一条 `scheduled` 清理并立即返回；检查在后台进行。`retainOnArchive` 开启时，归档不登记任何清理，所有检出目录都保留；该字段是 volatile 字段，每次归档时读取，[Forge 面板](../client-ui-forge/README.zh.md)在插件页该套件的 worktree 行上以开关编辑它。只要任一未归档 Session 仍引用该目录——通过记录的创建，或通过位于其中的工作目录（祖先子代理已归档的 Session 视为已归档；冷 Session 从投影缓存或其日志读取）——或创建者的沙箱策略为只读，目录就保留。符合条件的目录在路径锁内再做一次使用检查后，用创建者的策略执行不带 force 的 `git worktree remove`；路径锁同时通过 `working-directory/access` 阻塞进入该路径的 `ensure` 与 `set`；分支及其提交保留。跟踪文件改动、未跟踪文件、已初始化的子模块（仓库位于检出目录管理目录的 `modules` 中）、锁、分离 HEAD、分支不匹配以及位于配置池之外的目标都会保留目录。被忽略的文件以及对 skip-worktree 或 assume-unchanged 文件的改动不会：`git status` 不报告这两类内容，Git 删除目录时一并丢弃它们。检查不识别已提交的内嵌仓库；Git 会拒绝这类删除，本次尝试以 `failed` 结束。检查读到 `git status` 的第一条记录即停止，并以流式读取工作树列表，查找该检出目录期间在内存中保留其条目；两者的输出都不受 `maxOutputBytes` 限制。不检查进程：仍在写入的进程会让 Git 拒绝删除，空闲进程的下一条命令会失败。取消归档会取消尚未获准的删除，并等待已获准的删除结束。`cleanupOf(session, revision)` 读取本次尝试，或等待指定修订号结束，状态为 `scheduled`、`retained`、`removed` 或 `failed`；失败时 Session 仍保持归档并记录警告。此事件存在之前创建的检出目录，以及更早归档的 Session，不会被扫描。 默认检出目录是 `/.agents/worktrees/`。返回的 `repositoryRoot` 是调用方当前检出目录的顶层目录；如果它是链接检出目录，工作树父目录就嵌套在其中。省略名称时，名称由前缀 `dsh-`、Session id 中 UUID 的前 8 位十六进制数、一个连字符和 4 位随机十六进制数组成，例如 `dsh-6b1f02c4-9a3e`；Session id 不含 UUID 时，前缀后接 8 位随机十六进制数。`generatedWorktreeName` 导出同一规则，供在创建前确定名称的调用方使用。仅当服务的分配操作创建了父目录时，服务才会写入内容为 `*` 加换行符的 `.gitignore`；已有文件与忽略规则保持不变。每条 Git 和 Node 命令各自受 `timeoutMs` 时限约束，默认 5 分钟，使大仓库在较慢的文件系统上也能完成检出；取消仍会终止正在运行的命令。相对路径 `directory`、生成名称使用的 `namePrefix`、可执行程序、进程限制与 `retainOnArchive` 可在[配置目录](../../../docs/config-catalog.zh.md#deepseek-aidsh-experimental-worktree)中查阅。 diff --git a/packages/experimental/worktree/src/index.ts b/packages/experimental/worktree/src/index.ts index 683798a5d4..b445ec72b7 100644 --- a/packages/experimental/worktree/src/index.ts +++ b/packages/experimental/worktree/src/index.ts @@ -50,7 +50,8 @@ export interface Config { graceMs?: number /** * Maximum captured bytes per subprocess output stream; archive inspection - * streams `git status` and the worktree listing, so it bounds only their stderr. + * streams `git status`, and inspection and restoration stream the worktree + * listing, so it bounds only their stderr. */ maxOutputBytes?: number } diff --git a/packages/experimental/worktree/src/process.ts b/packages/experimental/worktree/src/process.ts index 94846759c1..f4bb56273a 100644 --- a/packages/experimental/worktree/src/process.ts +++ b/packages/experimental/worktree/src/process.ts @@ -1,4 +1,4 @@ -/** Bounded, cancellable commands under the calling Session's existing file policy. */ +/** Cancellable commands under the calling Session's existing file policy, whose stdout is collected within a bound or streamed. */ import type { Readable } from 'node:stream' import type { Context } from '@deepseek-ai/cordis' @@ -70,7 +70,9 @@ export async function runCommand( * field to `onField` as it arrives instead of retaining the output, so the size * of stdout is unbounded and `maxOutputBytes` limits only stderr. An * unterminated remainder is ignored. A false return from `onField` terminates - * the command, whose exit status is then not checked; otherwise only status 0 succeeds. + * the command, whose outcome is then not checked, even when the provider + * rejects it for a launch cancelled before acknowledgement; otherwise only + * status 0 succeeds. * @param ctx - subprocess and sandbox providers in the same execution world. * @param config - complete process resource limits. * @param policy - unchanged calling Session policy. @@ -105,11 +107,16 @@ export async function scanCommand( break } } - if (stopped) child.terminate() + if (stopped) { + child.terminate() + await Promise.allSettled([child.done]) + timeout.signal.throwIfAborted() + return true + } const outcome = await child.done timeout.signal.throwIfAborted() - if (!stopped) allowedExitCode(outcome, child.collected.stderr as SubprocessOutputReader, [0]) - return stopped + allowedExitCode(outcome, child.collected.stderr as SubprocessOutputReader, [0]) + return false } finally { child.terminate() await Promise.allSettled([child.done]) diff --git a/packages/experimental/worktree/src/removal.ts b/packages/experimental/worktree/src/removal.ts index bcec543e6c..4681c0c708 100644 --- a/packages/experimental/worktree/src/removal.ts +++ b/packages/experimental/worktree/src/removal.ts @@ -55,7 +55,8 @@ export async function checkoutIdentity( /** * Find the registration of `target` among the source repository's worktrees. - * The listing grows with the number of worktrees and is scanned as it arrives. + * The listing grows with the number of worktrees; it is streamed without an + * output limit, and its entries are kept in memory while `target` is resolved. * @param ctx - execution-world filesystem provider. * @param request - saved checkout identity. * @param target - resolved checkout directory. @@ -111,7 +112,8 @@ export async function removeCheckout( * are not counted, nor are edits to skip-worktree or assume-unchanged files, * which `git status` hides; `git worktree remove` discards both. A checkout * whose administrative directory holds submodule repositories is ineligible, - * as Git refuses to remove it. + * as Git refuses to remove it; a committed embedded repository is not + * inspected, so Git's refusal of that removal rejects. * @param ctx - execution-world filesystem provider. * @param directory - configured relative worktree pool. * @param request - saved checkout identity. diff --git a/packages/experimental/worktree/tests/process.spec.ts b/packages/experimental/worktree/tests/process.spec.ts new file mode 100644 index 0000000000..1ccc614dc7 --- /dev/null +++ b/packages/experimental/worktree/tests/process.spec.ts @@ -0,0 +1,60 @@ +import { rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { Context } from '@deepseek-ai/cordis' +import { scanCommand } from '../src/process.ts' +import { harness, repository } from './harness.ts' + +const disposers: (() => Promise)[] = [] +afterEach(async () => { + for (const dispose of disposers.splice(0).reverse()) await dispose() +}) + +/** + * Make `done` reject once `terminate()` is called, as the SSH provider does + * when it cancels a launch whose start it has not yet acknowledged. + */ +function rejectDoneOnTerminate(ctx: Context): void { + const spawn = ctx.subprocess.spawn.bind(ctx.subprocess) + vi.spyOn(ctx.subprocess, 'spawn').mockImplementation((spec) => { + const handle = spawn(spec) + let cancelled: Promise | undefined + return { + get stdin() { return handle.stdin }, + get stdout() { return handle.stdout }, + get stderr() { return handle.stderr }, + get control() { return handle.control }, + get collected() { return handle.collected }, + get done() { return cancelled === undefined ? handle.done : Promise.race([handle.done, cancelled]) }, + terminate() { + cancelled ??= Promise.reject(new Error('SSH process terminated before launch acknowledgement')) + handle.terminate() + }, + waitForExit: signal => handle.waitForExit(signal), + } + }) +} + +describe('streamed commands', () => { + it('returns the early answer when stopping rejects the outcome of an unacknowledged launch', async () => { + const root = await repository() + disposers.push(() => rm(root, { recursive: true, force: true })) + const ctx = await harness(root) + disposers.push(() => ctx.fiber.dispose()) + await writeFile(join(root, 'untracked.txt'), 'dirty\n') + rejectDoneOnTerminate(ctx) + const fields: string[] = [] + + const stopped = await scanCommand( + ctx, { timeoutMs: 30_000, graceMs: 1_000, maxOutputBytes: 1_024 }, ctx.sandboxPolicy.resolve(), root, + ['git', 'status', '--porcelain=v1', '-z', '--untracked-files=all'], new AbortController().signal, + (field) => { + fields.push(field) + return false + }, + ) + + expect(stopped).toBe(true) + expect(fields).toEqual(['?? untracked.txt']) + }) +}) diff --git a/packages/experimental/worktree/tests/removal.spec.ts b/packages/experimental/worktree/tests/removal.spec.ts index 27be81eca6..383b7f4c1d 100644 --- a/packages/experimental/worktree/tests/removal.spec.ts +++ b/packages/experimental/worktree/tests/removal.spec.ts @@ -94,6 +94,34 @@ describe('safe worktree removal', () => { await expect(git(root, 'worktree', 'remove', worktree.path)).rejects.toThrow('working trees containing submodules cannot be moved or removed') }) + it('removes a checkout whose submodule was never initialized there, as Git does', async () => { + const root = await repository() + disposers.push(() => rm(root, { recursive: true, force: true })) + const source = await repository() + disposers.push(() => rm(source, { recursive: true, force: true })) + await git(root, '-c', 'protocol.file.allow=always', 'submodule', 'add', source, 'nested') + await git(root, 'commit', '-m', 'submodule') + const ctx = await harness(root) + disposers.push(() => ctx.fiber.dispose()) + const agent = testAgent(ctx, root) + const worktree = await ctx.worktrees.create(agent, { name: 'archive-me' }) + expect(await git(worktree.path, 'ls-files', '--stage', 'nested')).toMatch(/^160000 /) + expect(await remove(ctx, agent, worktree)).toBe(true) + await expect(access(worktree.path)).rejects.toMatchObject({ code: 'ENOENT' }) + }) + + it('leaves a committed embedded repository to Git, which refuses the removal', async () => { + const { ctx, agent, worktree } = await fixture() + const source = await repository() + disposers.push(() => rm(source, { recursive: true, force: true })) + await git(worktree.path, 'clone', source, 'nested') + await git(worktree.path, 'add', '-A') + await git(worktree.path, 'commit', '-m', 'embedded repository') + expect(await git(worktree.path, 'status', '--porcelain')).toBe('') + await expect(remove(ctx, agent, worktree)).rejects.toThrow('working trees containing submodules cannot be moved or removed') + await access(join(worktree.path, 'nested', 'tracked.txt')) + }) + it('removes a checkout that holds only ignored files, as Git does without force', async () => { const { ctx, agent, worktree } = await fixture() await git(worktree.path, 'config', 'core.excludesFile', join(worktree.repositoryRoot, 'exclude')) From 2d44c4ecc91762de8a61867f9c09b7593193b4a1 Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Mon, 9 Nov 2026 12:07:17 +0800 Subject: [PATCH 29/30] fix(forge-panel): let scroll anchoring keep a jumped-to file in place The per-frame alignment after a jump pulled back scrolls it did not start, such as a script or find in page setting the scroll position. Measured on #7985, the browser's scroll anchoring already keeps an early, middle, or late jump target at the top while the sections above it draw. --- ...orge-changes-view-responsiveness.i18n.yaml | 8 +-- ...11-07-forge-changes-view-responsiveness.md | 6 +- ...07-forge-changes-view-responsiveness.zh.md | 6 +- apps/web/tests/forge-panel.e2e.ts | 3 +- .../client-ui-forge/src/client/ChangesTab.tsx | 46 +-------------- .../tests/changes-tab.client.spec.tsx | 56 ------------------- 6 files changed, 15 insertions(+), 110 deletions(-) diff --git a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.i18n.yaml index 170d538ff9..e1e734251c 100644 --- a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.i18n.yaml +++ b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.i18n.yaml @@ -9,11 +9,11 @@ en: 77fab27f3d4a4a59 zh: 46e0fa8b9ef978a7 /agent-note-forge-changes-view-responsiveness/decision: - en: a138be3ed592e94e - zh: 80963cd20131df3a + en: 40550cf129ab4c0c + zh: 2e9a3597d03ce185 /agent-note-forge-changes-view-responsiveness/alternatives-considered: - en: 93e305c624bfe144 - zh: e41495d9930801c9 + en: c9833999164e8b67 + zh: 9459a95f307db854 /agent-note-forge-changes-view-responsiveness/consequences: en: 2d74147e306ed174 zh: f1369020f27f92c0 diff --git a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md index 3c481c5ea3..9ceef3f98f 100644 --- a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md +++ b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.md @@ -12,13 +12,13 @@ The forge changes view of a large change request froze the browser. Opening #798 **Scoped `:has()` rules.** The navigator's covering rules name their row through `data-files-navigator-row`, which both hosts of `sidebar.files.navigator` set, so a DOM change outside such a row schedules no document-wide invalidation. -**Sections draw what changed and what the reader reaches.** `ChangesTab` passes each memoized `DiffFile` values whose identity survives its renders: one handler object, one comment-actions object, and per-file strings and booleans. Tab visibility is a prop of every section, so a visibility change redraws them all; leaving the view measured +57 ms against reading it from a React context, which the client packages forbid business components to do. The scrollport's reach tracker travels in the handler object. An expanded file holds a 22 px row per drawn line until its section first comes within 1200 px of the scrollport, then draws its lines and keeps them. A jump keeps its target aligned for one second while nearby sections draw. +**Sections draw what changed and what the reader reaches.** `ChangesTab` passes each memoized `DiffFile` values whose identity survives its renders: one handler object, one comment-actions object, and per-file strings and booleans. Tab visibility is a prop of every section, so a visibility change redraws them all; leaving the view measured +57 ms against reading it from a React context, which the client packages forbid business components to do. The scrollport's reach tracker travels in the handler object. An expanded file holds a 22 px row per drawn line until its section first comes within 1200 px of the scrollport, then draws its lines and keeps them. After a jump, the browser's scroll anchoring keeps the target in place while the sections above it draw. **Highlighting in the drawing render.** A section highlights its lines in the same render that draws them, as before, so the drawn text is always the file's current text and a native selection keeps its text nodes. **Fewer serial reads.** A panel state equal to the published one reaches no follower. `changeRequestDiff` reads the patch and `.gitattributes` together. Snapshots carry the base commit as `baseSha`; `ForgeService.diff` passes it to the provider, and forge-github starts the compare read while its GraphQL read confirms the head. -**Pixel measurement.** Two parts measure layout in pixels, which CSS cannot replace. Whether a section is near the scrollport decides whether React renders its lines at all; `content-visibility` only skips layout and paint of nodes that already exist. Keeping a jumped-to section at the top needs its offset after each later layout change, since no CSS property scrolls a container to follow one child. The client packages offer no injected measurement hook, so both read the DOM directly, as the view's file navigation already does: the reach uses one `IntersectionObserver`, which measures off the main thread, and the pin reads two rectangles per frame for at most one second. +**Pixel measurement.** Whether a section is near the scrollport decides whether React renders its lines at all, which CSS cannot express: `content-visibility` only skips layout and paint of nodes that already exist. The client packages offer no injected measurement hook, so one `IntersectionObserver` on the scrollport measures it off the main thread. ## Alternatives considered @@ -30,6 +30,8 @@ The forge changes view of a large change request froze the browser. Opening #798 **The CSS Custom Highlight API** would paint token colors over unchanged text nodes and keep deferral without either defect, but it cannot apply the themes' bold and italic, and its cost over tens of thousands of ranges is unmeasured. +**Re-aligning a jumped-to section every frame for a second** kept the last file flush with the end of the list, but it also pulled back any scroll it did not start, such as one from find in page; scroll anchoring already holds every other target at the top. + **Highlighting in a Web Worker** removes tokenization from the main thread entirely, including a heavy grammar's first use, but the shared highlighter in `ui-primitives` owns Shiki; moving it is a separate change for every consumer. **A provider-private cache of base commits** would avoid the optional `baseSha` field, but it adds retained state without an owner that knows when a change request stops being watched. diff --git a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.zh.md b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.zh.md index 6881c5b3ae..fda2dc4b11 100644 --- a/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.zh.md +++ b/.agents/notes/implemented/bug-fix/2026-11-07-forge-changes-view-responsiveness.zh.md @@ -12,13 +12,13 @@ Status: implemented **限定 `:has()` 规则。** 导航器的遮盖规则通过 `data-files-navigator-row` 指定所在行，`sidebar.files.navigator` 的两个宿主都会设置它，因此这类行之外的 DOM 变化不再触发全文档失效。 -**文件段只绘制变化的部分和读者接近的部分。** `ChangesTab` 传给每个 memo 化的 `DiffFile` 的值在多次渲染之间保持同一标识：一个处理函数对象、一个评论操作对象，以及每个文件的字符串和布尔值。标签页可见性作为每个文件段的参数传入，可见性变化时全部文件段都会重绘；离开视图实测比从 React context 读取多 57 ms，而客户端包禁止业务组件读取 context。滚动区域的接近跟踪对象放在处理函数对象中传递。展开的文件在它的段第一次进入滚动区域 1200 px 范围之前，按每个绘制行 22 px 占位，之后绘制各行并一直保留。跳转后的一秒内，目标文件在周围文件段绘制时保持对齐。 +**文件段只绘制变化的部分和读者接近的部分。** `ChangesTab` 传给每个 memo 化的 `DiffFile` 的值在多次渲染之间保持同一标识：一个处理函数对象、一个评论操作对象，以及每个文件的字符串和布尔值。标签页可见性作为每个文件段的参数传入，可见性变化时全部文件段都会重绘；离开视图实测比从 React context 读取多 57 ms，而客户端包禁止业务组件读取 context。滚动区域的接近跟踪对象放在处理函数对象中传递。展开的文件在它的段第一次进入滚动区域 1200 px 范围之前，按每个绘制行 22 px 占位，之后绘制各行并一直保留。跳转后，上方文件段绘制时，浏览器的滚动锚定让目标保持原位。 **在绘制的渲染中高亮。** 文件段与修复前一样，在绘制各行的同一次渲染中完成高亮，所以画出的始终是文件当前的文本，原生选区所在的文本节点也不会被替换。 **减少串行读取。** 与已发布状态相同的面板状态不会通知任何关注方。`changeRequestDiff` 同时读取补丁和 `.gitattributes`。快照把基准提交作为 `baseSha` 携带；`ForgeService.diff` 把它传给提供方，forge-github 在 GraphQL 读取确认 head 的同时就开始 compare 读取。 -**像素测量。** 有两处需要按像素测量布局，CSS 无法替代。文件段是否接近滚动区域，决定 React 是否渲染它的各行；`content-visibility` 只能跳过已有节点的布局与绘制。让跳转后的文件段保持在顶部，需要在之后每次布局变化后读取它的偏移，没有哪个 CSS 属性能让容器跟随某个子元素滚动。客户端包没有可注入的测量 hook，所以这两处像视图已有的文件导航一样直接读取 DOM：接近判断用一个 `IntersectionObserver`，在主线程之外测量；对齐每帧读取两个矩形，最多持续一秒。 +**像素测量。** 文件段是否接近滚动区域，决定 React 是否渲染它的各行，这一点 CSS 无法表达：`content-visibility` 只能跳过已有节点的布局与绘制。客户端包没有可注入的测量 hook，所以在滚动区域上用一个 `IntersectionObserver`，在主线程之外测量。 ## Alternatives considered @@ -30,6 +30,8 @@ Status: implemented **CSS Custom Highlight API** 可以在不变的文本节点上绘制 token 颜色，保留延迟高亮又没有上述两个问题，但它无法应用主题中的粗体和斜体，几万个范围的绘制成本也没有测过。 +**跳转后一秒内每帧重新对齐** 能让最后一个文件贴紧列表末端，但它也会把不是它发起的滚动拉回去，例如页内查找引起的滚动；而对其他目标，滚动锚定已经能让它们保持在顶部。 + **在 Web Worker 中高亮** 能把分词完全移出主线程，包括重型语法的首次使用，但 Shiki 由 `ui-primitives` 中的共享高亮器持有；迁移它是影响所有使用方的另一项改动。 **由提供方私有缓存基准提交** 可以省去可选的 `baseSha` 字段，但会引入保留状态，而且没有一个知道变更请求何时不再被 watch 的所有者。 diff --git a/apps/web/tests/forge-panel.e2e.ts b/apps/web/tests/forge-panel.e2e.ts index 51cc42479d..c42d038d56 100644 --- a/apps/web/tests/forge-panel.e2e.ts +++ b/apps/web/tests/forge-panel.e2e.ts @@ -1254,9 +1254,8 @@ it.skipIf(MODE === 'record')('uses horizontal arrows to navigate files while pre expect(await scroll.evaluate(element => document.activeElement === element)).toBe(true) await page.keyboard.press('ArrowRight') await expect.poll(() => tree.getByRole('button', { name: 'README.md', exact: true }).getAttribute('aria-current')).toBe('true') - // The last file cannot reach the top, so the jump settles at the end of the list once the file has drawn its lines. + // The last file draws its lines after the jump, which can lengthen the list; read the position once it has. await c.locator('[data-review-code]').first().waitFor() - await expect.poll(() => scroll.evaluate(element => element.scrollHeight - element.clientHeight - element.scrollTop)).toBeLessThan(1) const lastPosition = await scroll.evaluate(element => element.scrollTop) const lastBox = await c.boundingBox(), viewport = await scroll.boundingBox() if (lastBox === null || viewport === null) throw new Error('The last file and scroll viewport must be rendered') diff --git a/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx b/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx index 4cec78f68e..dd01665747 100644 --- a/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx +++ b/packages/experimental/client-ui-forge/src/client/ChangesTab.tsx @@ -149,12 +149,6 @@ const OPEN_LINES = 5000 const NO_GAPS: ReadonlySet = new Set() -/** How long, in milliseconds, a jumped-to file stays at the top while the sections around it draw. */ -const JUMP_PIN_MS = 1000 - -/** Input that scrolls the file list, which ends a jump's pin. */ -const READER_SCROLL = ['wheel', 'touchstart', 'pointerdown', 'keydown'] as const - const SCOPE_LABEL: Record = { branch: 'diff.scope.branch', uncommitted: 'diff.scope.uncommitted', @@ -600,38 +594,6 @@ export function ChangesTab({ }, [reach]) // A short final file cannot align with the viewport top; retain its jump until the reader scrolls away. const jumpPosition = useRef<{ readonly data: ChangesData; readonly top: number } | null>(null) - const releasePin = useRef<(() => void) | null>(null) - /** - * Keep a jumped-to section at the top of the scrollport for {@link JUMP_PIN_MS}: the sections around it draw their - * lines after the jump, which moves it. Scrolling input from the reader, another jump, hiding the tab, or unmounting - * ends the pin. Each frame measures the section once, which covers every cause of movement, such as a section - * drawing its lines or a review card opening, without each one reporting it. - */ - const pinJump = (port: HTMLElement, section: HTMLElement, owner: ChangesData): void => { - releasePin.current?.() - const until = performance.now() + JUMP_PIN_MS - let frame = 0 - const release = (): void => { - cancelAnimationFrame(frame) - for (const type of READER_SCROLL) port.removeEventListener(type, release) - releasePin.current = null - } - const keep = (now: number): void => { - if (now > until) { - release() - return - } - const offset = section.getBoundingClientRect().top - port.getBoundingClientRect().top - if (Math.abs(offset) > 1 && port.scrollTop < port.scrollHeight - port.clientHeight - 1) { - section.scrollIntoView({ block: 'start' }) - jumpPosition.current = { data: owner, top: port.scrollTop } - } - frame = requestAnimationFrame(keep) - } - for (const type of READER_SCROLL) port.addEventListener(type, release, { passive: true }) - releasePin.current = release - frame = requestAnimationFrame(keep) - } useLayoutEffect(() => { if (!tab.visible || wanted === null || data === null) return // A file the lens or the test switch hides is shown before the view scrolls to it. @@ -641,15 +603,11 @@ export function ChangesTab({ if (hideTests && hidden.layer === 'test') actions.toggledTests(tab.id) return } - const section = sections.current.get(wanted) - section?.scrollIntoView({ block: 'start' }) + // The browser's scroll anchoring keeps the section in place while the sections above it draw their lines. + sections.current.get(wanted)?.scrollIntoView({ block: 'start' }) if (scrollport.current !== null) jumpPosition.current = { data, top: scrollport.current.scrollTop } - if (section !== undefined && scrollport.current !== null) pinJump(scrollport.current, section, data) setWanted(null) }, [tab.visible, wanted, data, allFiles, lens, hideTests, actions, tab.id]) - // A hidden tab has nothing to keep in place, and its scrollport measures as empty. - useEffect(() => { if (!tab.visible) releasePin.current?.() }, [tab.visible]) - useEffect(() => () => { releasePin.current?.() }, []) const refresh = (): void => { setAttempt(value => value + 1) } const root = state !== null && state.checkout.state === 'ready' ? state.checkout.checkout.root : null // A chosen base branch that is gone or shares no history with HEAD offers the automatic fork point instead. diff --git a/packages/experimental/client-ui-forge/tests/changes-tab.client.spec.tsx b/packages/experimental/client-ui-forge/tests/changes-tab.client.spec.tsx index 36be08a879..2cda7a4324 100644 --- a/packages/experimental/client-ui-forge/tests/changes-tab.client.spec.tsx +++ b/packages/experimental/client-ui-forge/tests/changes-tab.client.spec.tsx @@ -946,62 +946,6 @@ describe('ChangesTab layout', () => { expect(scrolled).toHaveBeenCalledWith({ block: 'start' }) }) - it('keeps a jumped-to file at the top while sections around it draw, until the reader scrolls, the tab hides, or a second passes', async () => { - const m = mount() - await settle() - const { viewport } = fileViewport(m) - Object.defineProperty(viewport, 'scrollHeight', { configurable: true, value: 1200 }) - Object.defineProperty(viewport, 'clientHeight', { configurable: true, value: 300 }) - const target = m.section('src/new.ts') - // Sections above the target draw after the jump and push it down by this much until it scrolls again. - let shift = 0 - vi.spyOn(target, 'getBoundingClientRect').mockImplementation(() => new DOMRect(0, shift, 600, 200)) - scrolled.mockImplementation(() => { shift = 0 }) - vi.useFakeTimers({ toFake: ['requestAnimationFrame', 'cancelAnimationFrame', 'performance'] }) - try { - const jump = (): void => { - fireEvent.click(m.view.getByRole('button', { name: 'All 6 files' })) - fireEvent.click(m.view.getByRole('menuitem', { name: 'src/new.ts' })) - } - jump() - expect(scrolled).toHaveBeenCalledOnce() - shift = 80 - act(() => { vi.advanceTimersToNextFrame() }) - expect(scrolled).toHaveBeenCalledTimes(2) - act(() => { vi.advanceTimersToNextFrame() }) - expect(scrolled).toHaveBeenCalledTimes(2) - - // At the bottom of the list nothing can move the file higher. - viewport.scrollTop = 900 - shift = 40 - act(() => { vi.advanceTimersToNextFrame() }) - expect(scrolled).toHaveBeenCalledTimes(2) - viewport.scrollTop = 0 - - fireEvent.wheel(viewport) - act(() => { vi.advanceTimersToNextFrame() }) - expect(scrolled).toHaveBeenCalledTimes(2) - - jump() - expect(scrolled).toHaveBeenCalledTimes(3) - act(() => { vi.advanceTimersByTime(1100) }) - shift = 80 - act(() => { vi.advanceTimersToNextFrame() }) - expect(scrolled).toHaveBeenCalledTimes(3) - - // Hiding the tab ends the pin at once. - shift = 0 - jump() - expect(scrolled).toHaveBeenCalledTimes(4) - act(() => { m.visibility.set(false) }) - shift = 80 - act(() => { vi.advanceTimersToNextFrame() }) - expect(scrolled).toHaveBeenCalledTimes(4) - } finally { - vi.useRealTimers() - } - }) - it('scrolls once to the file each navigation names when its changes arrive', async () => { const m = mount({ params: { path: 'docs/gone.txt' } }) expect(scrolled).not.toHaveBeenCalled() From 088cd5dfba37163c1458f18bdb6e440025373d3f Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Mon, 9 Nov 2026 12:20:08 +0800 Subject: [PATCH 30/30] test(forge-panel): read the end position after scroll anchoring settles --- apps/web/tests/forge-panel.e2e.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/web/tests/forge-panel.e2e.ts b/apps/web/tests/forge-panel.e2e.ts index c42d038d56..6f6990b494 100644 --- a/apps/web/tests/forge-panel.e2e.ts +++ b/apps/web/tests/forge-panel.e2e.ts @@ -1254,8 +1254,11 @@ it.skipIf(MODE === 'record')('uses horizontal arrows to navigate files while pre expect(await scroll.evaluate(element => document.activeElement === element)).toBe(true) await page.keyboard.press('ArrowRight') await expect.poll(() => tree.getByRole('button', { name: 'README.md', exact: true }).getAttribute('aria-current')).toBe('true') - // The last file draws its lines after the jump, which can lengthen the list; read the position once it has. + // The last file draws its lines after the jump, and scroll anchoring adjusts the position at the next layout. await c.locator('[data-review-code]').first().waitFor() + await scroll.evaluate(() => new Promise((resolve) => { + requestAnimationFrame(() => { requestAnimationFrame(() => { resolve() }) }) + })) const lastPosition = await scroll.evaluate(element => element.scrollTop) const lastBox = await c.boundingBox(), viewport = await scroll.boundingBox() if (lastBox === null || viewport === null) throw new Error('The last file and scroll viewport must be rendered')