Merge remote-tracking branch 'origin/master' into perf/forge-changes-view
This commit is contained in:
Yichen Jiang committed
commit
d4a2df736b
49 files changed
+836
-146
No files matched your search
+2
-2
@@ -15,5 +15,5 @@
|
||||
en: 32c970ae153e0f6c
|
||||
zh: 8431be0a10ffed15
|
||||
/agent-note-cua-scoped-surfaces-and-separated-input-capabilities/consequences:
|
||||
en: 5c6ec60b77f1bff0
|
||||
zh: e1ae568ef3501b7b
|
||||
en: c4b12ca80865e670
|
||||
zh: 396426fad0105eaa
|
||||
@@ -34,6 +34,6 @@ Within one API instance, the object facade keeps one current accessibility snaps
|
||||
|
||||
Element IDs belong to the application's current accessibility snapshot. Reading one window replaces the facade’s current snapshot for that application, so callers observe again and confirm the IDs they use. The new read does not guarantee that an old string is rejected or that an ID changes; callers must not assume an older ID still identifies the same control. Pointer input and screenshots require an explicit window object; callers enumerate windows first.
|
||||
|
||||
Keyboard behavior follows the provider's supported scopes. A caller that uses `sendKeyEvent` owns the down/up pairing, because the API never sends the opposite event. Keyboard modifier names are provider-defined, so the same string may have different support across providers.
|
||||
Keyboard behavior follows the provider's supported scopes. A caller that uses `sendKeyEvent` owns the down/up pairing, during execution. macOS releases that run’s remaining keys when the program ends or the user pauses it. Keyboard modifier names are provider-defined, so the same string may have different support across providers.
|
||||
|
||||
Provider guidance owns the list of supported actions. Adding a method to the shared schema does not claim that every provider implements it, and the merged `getCapabilities` result remains the runtime source for permissions and session lock.
|
||||
+1
-1
@@ -34,6 +34,6 @@ Provider 的启动能力是静态的。平台、支持的方法、应用级辅
|
||||
|
||||
元素 ID 属于应用当前的辅助功能快照。读取一个窗口会替换门面对该应用的当前快照，因此调用方需要重新观察并确认要使用的 ID。新的读取不保证拒绝旧字符串，也不保证 ID 会改变；调用方不得假定旧 ID 仍指向同一控件。指针输入和截图都要求显式的窗口对象；调用方先枚举窗口。
|
||||
|
||||
键盘行为遵循 provider 支持的作用域。使用 `sendKeyEvent` 的调用方负责按下与抬起的配对，因为 API 从不发送相反事件。键盘修饰键名称由 provider 定义，同一字符串在不同 provider 上可能支持不同。
|
||||
键盘行为遵循 provider 支持的作用域。使用 `sendKeyEvent` 的调用方负责按下与抬起的配对，在 macOS 上，程序结束或用户暂停时会释放本次执行仍按着的键。键盘修饰键名称由 provider 定义，同一字符串在不同 provider 上可能支持不同。
|
||||
|
||||
Provider 指导文本负责支持动作的清单。把方法加入共享 schema 并不表示每个 provider 都已实现它；合并后的 `getCapabilities` 结果仍是权限和会话锁定状态的运行时来源。
|
||||
@@ -1848,3 +1848,51 @@ it.skipIf(MODE === 'record')('keeps local viewed marks across reloads, clears a
|
||||
await page.keyboard.press('Escape')
|
||||
expect(tripwire.pageErrors).toEqual([])
|
||||
})
|
||||
|
||||
it.skipIf(MODE === 'record')('keeps a narrow file header\'s viewed mark and menu inside its card and its name under the pointer', async () => {
|
||||
const { scaffold, page, project, tripwire } = await launchPanel('web-e2e-forge-narrow-header', 'normal', { branch: 'feat' })
|
||||
const from = 'src/original-review-module-with-a-long-name.ts'
|
||||
const to = 'src/renamed-review-module-with-an-even-longer-name.ts'
|
||||
const body = Array.from({ length: 20 }, (_, index) => `export const value${index} = ${index}\n`).join('')
|
||||
await mkdir(join(project, 'src'))
|
||||
await writeFile(join(project, from), body)
|
||||
await git(project, 'add', from)
|
||||
await git(project, 'commit', '--quiet', '-m', 'Add the module')
|
||||
// The base holds the old path, so the branch shows a rename.
|
||||
await git(project, 'update-ref', 'refs/remotes/origin/main', 'HEAD')
|
||||
await git(project, 'symbolic-ref', 'refs/remotes/origin/HEAD', 'refs/remotes/origin/main')
|
||||
await git(project, 'mv', from, to)
|
||||
await writeFile(join(project, to), `${body}export const extra = 1\n`)
|
||||
await git(project, 'commit', '--quiet', '-am', 'Rename the module')
|
||||
const { settled } = await sendPrompt(page, scaffold)
|
||||
await settled
|
||||
await page.getByRole('button', { name: '打开右侧边栏', exact: true }).click()
|
||||
await page.locator('[data-sidebar-right-guide]').getByRole('button', { name: /^代码审查和变更/ }).click()
|
||||
await page.locator('[data-forge-panel]').getByRole('button', { name: '刷新状态', exact: true }).click()
|
||||
await page.locator('[data-forge-local]').getByRole('button', { name: /^查看本会话的变更/ }).click({ timeout: 20_000 })
|
||||
const local = page.locator('[data-forge-changes="local"]')
|
||||
const name = 'renamed-review-module-with-an-even-longer-name.ts'
|
||||
const file = local.locator(`[data-forge-diff-file="${to}"]`)
|
||||
const viewed = file.getByRole('checkbox', { name: `将 ${name} 标记为已审阅`, exact: true })
|
||||
await viewed.check({ timeout: 20_000 })
|
||||
await writeFile(join(project, to), `${body}export const extra = 2\n`)
|
||||
await local.getByRole('button', { name: '刷新变更', exact: true }).click()
|
||||
await expect.poll(() => file.getAttribute('data-viewed'), { timeout: 20_000 }).toBe('changed')
|
||||
await file.getByText('由 original-review-module-with-a-long-name.ts 重命名', { exact: true }).waitFor()
|
||||
|
||||
// The changed tag, the rename badge, and the long path compete for one narrow header with the counts and actions.
|
||||
await page.setViewportSize({ width: 390, height: 780 })
|
||||
const label = file.locator('[data-forge-file-label]')
|
||||
const [card, mark, menu, labelBox] = await Promise.all([
|
||||
file.boundingBox(), viewed.boundingBox(), file.getByRole('button', { name: `${name} 的更多操作`, exact: true }).boundingBox(), label.boundingBox(),
|
||||
])
|
||||
if (card === null || mark === null || menu === null || labelBox === null) throw new Error('The header controls must be rendered')
|
||||
expect(mark.x + mark.width).toBeLessThanOrEqual(card.x + card.width)
|
||||
expect(menu.x + menu.width).toBeLessThanOrEqual(card.x + card.width)
|
||||
expect(labelBox.width).toBeGreaterThan(40)
|
||||
await label.hover()
|
||||
await page.getByRole('tooltip').filter({ hasText: `${from} → ${to}` }).waitFor()
|
||||
await viewed.check()
|
||||
await expect.poll(() => file.getAttribute('data-viewed')).toBe('viewed')
|
||||
expect(tripwire.pageErrors).toEqual([])
|
||||
})
|
||||
@@ -1156,7 +1156,7 @@ export interface Config {
|
||||
|
||||
- `inject`: `computerUse` · `tools` · `systemPrompt` · `subprocess` · `agents`
|
||||
- `refs`: [`CuaConfig`](../packages/experimental/cua/src/index.ts)
|
||||
- `source`: [`packages/experimental/cua-macos/src/index.ts:33`](../packages/experimental/cua-macos/src/index.ts)
|
||||
- `source`: [`packages/experimental/cua-macos/src/index.ts:34`](../packages/experimental/cua-macos/src/index.ts)
|
||||
|
||||
```ts config-catalog
|
||||
/** Native preparation, source-build signing, operation bounds, and profile-local application grants. */
|
||||
|
||||
@@ -1158,7 +1158,7 @@ export interface Config {
|
||||
|
||||
- `inject`: `computerUse` · `tools` · `systemPrompt` · `subprocess` · `agents`
|
||||
- `refs`: [`CuaConfig`](../packages/experimental/cua/src/index.ts)
|
||||
- `source`: [`packages/experimental/cua-macos/src/index.ts:33`](../packages/experimental/cua-macos/src/index.ts)
|
||||
- `source`: [`packages/experimental/cua-macos/src/index.ts:34`](../packages/experimental/cua-macos/src/index.ts)
|
||||
|
||||
```ts config-catalog
|
||||
/** Native preparation, source-build signing, operation bounds, and profile-local application grants. */
|
||||
|
||||
@@ -84,7 +84,7 @@ Application listeners are outside this package matrix. Desktop Host subscribes t
|
||||
| `system-prompt/assemble` | `waterfall` | [`packages/core/system-prompt/src/index.ts:31`](../packages/core/system-prompt/src/index.ts) | [`system-prompt`](../packages/core/system-prompt) (`waterfall`) | `browser-use-runtime`, [`working-directory`](../packages/session/working-directory) |
|
||||
| `system-prompt/change` | `emit` | [`packages/core/system-prompt/src/index.ts:37`](../packages/core/system-prompt/src/index.ts) | [`system-prompt`](../packages/core/system-prompt) (`emit`) | - |
|
||||
| `tools/change` | `emit` | [`packages/core/tools/src/events.ts:56`](../packages/core/tools/src/events.ts) | [`agent-preset-registry`](../packages/preset/agent-preset-registry) (`emit`), [`tools`](../packages/core/tools) (`emit`) | `browser-use-runtime` |
|
||||
| `tools/execute` | `waterfall` | [`packages/core/tools/src/events.ts:29`](../packages/core/tools/src/events.ts) | [`tools`](../packages/core/tools) (`waterfall`) | `browser-use-runtime`, `browser-use-stagehand-native`, `claude-code-mods`, `computer-use-cua-driver-native`, [`session-checkpoint-policy`](../packages/session/session-checkpoint-policy), `timeout-policy` |
|
||||
| `tools/execute` | `waterfall` | [`packages/core/tools/src/events.ts:29`](../packages/core/tools/src/events.ts) | [`tools`](../packages/core/tools) (`waterfall`) | `browser-use-runtime`, `browser-use-stagehand-native`, `claude-code-mods`, `computer-use-cua-driver-native`, `cua-macos`, [`session-checkpoint-policy`](../packages/session/session-checkpoint-policy), `timeout-policy` |
|
||||
| `tools/post-execute` | `waterfall` | [`packages/core/tools/src/events.ts:41`](../packages/core/tools/src/events.ts) | [`tools`](../packages/core/tools) (`waterfall`) | `claude-code-mods`, [`hooks-claude-code`](../packages/experimental/hooks-claude-code), [`hooks-codex`](../packages/experimental/hooks-codex), [`repeat-tool-reminder`](../packages/guard/repeat-tool-reminder), [`spill-policy`](../packages/spill/spill-policy), [`tool-fs-search`](../packages/fs/tool-fs-search) |
|
||||
| `tools/pre-execute` | `waterfall` | [`packages/core/tools/src/events.ts:18`](../packages/core/tools/src/events.ts) | [`tools`](../packages/core/tools) (`waterfall`) | `auto-review`, [`hooks-claude-code`](../packages/experimental/hooks-claude-code), [`hooks-codex`](../packages/experimental/hooks-codex), `liveshare`, [`tool-jobs`](../packages/jobs/tool-jobs), [`workspace-changes`](../packages/deliverables/workspace-changes) |
|
||||
| `tools/ptc-dispatch-log` | `waterfall` | [`packages/core/tool-executor-ptc/src/runtime-types.ts:46`](../packages/core/tool-executor-ptc/src/runtime-types.ts) | [`tool-executor-ptc`](../packages/core/tool-executor-ptc) (`waterfall`) | [`spill-policy`](../packages/spill/spill-policy) |
|
||||
|
||||
@@ -86,7 +86,7 @@
|
||||
| `system-prompt/assemble` | `waterfall` | [`packages/core/system-prompt/src/index.ts:31`](../packages/core/system-prompt/src/index.ts) | [`system-prompt`](../packages/core/system-prompt) (`waterfall`) | `browser-use-runtime`, [`working-directory`](../packages/session/working-directory) |
|
||||
| `system-prompt/change` | `emit` | [`packages/core/system-prompt/src/index.ts:37`](../packages/core/system-prompt/src/index.ts) | [`system-prompt`](../packages/core/system-prompt) (`emit`) | - |
|
||||
| `tools/change` | `emit` | [`packages/core/tools/src/events.ts:56`](../packages/core/tools/src/events.ts) | [`agent-preset-registry`](../packages/preset/agent-preset-registry) (`emit`), [`tools`](../packages/core/tools) (`emit`) | `browser-use-runtime` |
|
||||
| `tools/execute` | `waterfall` | [`packages/core/tools/src/events.ts:29`](../packages/core/tools/src/events.ts) | [`tools`](../packages/core/tools) (`waterfall`) | `browser-use-runtime`, `browser-use-stagehand-native`, `claude-code-mods`, `computer-use-cua-driver-native`, [`session-checkpoint-policy`](../packages/session/session-checkpoint-policy), `timeout-policy` |
|
||||
| `tools/execute` | `waterfall` | [`packages/core/tools/src/events.ts:29`](../packages/core/tools/src/events.ts) | [`tools`](../packages/core/tools) (`waterfall`) | `browser-use-runtime`, `browser-use-stagehand-native`, `claude-code-mods`, `computer-use-cua-driver-native`, `cua-macos`, [`session-checkpoint-policy`](../packages/session/session-checkpoint-policy), `timeout-policy` |
|
||||
| `tools/post-execute` | `waterfall` | [`packages/core/tools/src/events.ts:41`](../packages/core/tools/src/events.ts) | [`tools`](../packages/core/tools) (`waterfall`) | `claude-code-mods`, [`hooks-claude-code`](../packages/experimental/hooks-claude-code), [`hooks-codex`](../packages/experimental/hooks-codex), [`repeat-tool-reminder`](../packages/guard/repeat-tool-reminder), [`spill-policy`](../packages/spill/spill-policy), [`tool-fs-search`](../packages/fs/tool-fs-search) |
|
||||
| `tools/pre-execute` | `waterfall` | [`packages/core/tools/src/events.ts:18`](../packages/core/tools/src/events.ts) | [`tools`](../packages/core/tools) (`waterfall`) | `auto-review`, [`hooks-claude-code`](../packages/experimental/hooks-claude-code), [`hooks-codex`](../packages/experimental/hooks-codex), `liveshare`, [`tool-jobs`](../packages/jobs/tool-jobs), [`workspace-changes`](../packages/deliverables/workspace-changes) |
|
||||
| `tools/ptc-dispatch-log` | `waterfall` | [`packages/core/tool-executor-ptc/src/runtime-types.ts:46`](../packages/core/tool-executor-ptc/src/runtime-types.ts) | [`tool-executor-ptc`](../packages/core/tool-executor-ptc) (`waterfall`) | [`spill-policy`](../packages/spill/spill-policy) |
|
||||
|
||||
@@ -12,5 +12,5 @@
|
||||
en: e5d1027faa55adba
|
||||
zh: 4b88ad6a70977fb3
|
||||
/native-cua-application-and-window-api/migration:
|
||||
en: b09e3dde32391786
|
||||
zh: f702e1e25b260b29
|
||||
en: a63e88dce467d6ee
|
||||
zh: fc25bb283ca0eaff
|
||||
@@ -18,6 +18,6 @@ Providers supply startup capabilities alongside their platform guidance. The nat
|
||||
1. After `cua.getApp(selector)`, select the target from `app.getWindows()` with `app.getWindow(windowId)`. Use `window.getScreenshot()`, `window.click()`, `window.scroll()`, and `window.drag()`. Replace `hover` with `window.mouseMove()`. On macOS, `window.click(id)` sends mouse events at the element center. Switch to `invoke(id)` for the default accessibility action, or `performAction(id, "AXShowMenu")` to request an advertised menu action.
|
||||
2. Replace `getAXState()` with `getAccessibilityTree()`. Replace `getAXStateAndScreenshot()` with separate tree and screenshot reads. Selection reads the initial tree silently for a window, and for an app when `appLevelAccessibility` is true. Read the tree explicitly to display it.
|
||||
3. Pass the opaque string ID from the latest tree wherever you passed a numeric element index. In raw requests use `elementId` with `snapshotId`. Replace `performSecondaryAction` with `listActions(id)` and `performAction(id, exactName, argument?)`, or use `invoke(id)` for the default action. `setValue` accepts the provider’s supported JSON values. On macOS, `performAction(id, 'AXRaise')` performs the advertised raise action. Call `app.activate()` first when the application must become active.
|
||||
4. Replace a packed chord such as `pressKey('shift+Tab')` with `pressKey('Tab', ['shift'])`. Check `(await cua.getCapabilities()).methods` before using `sendKeyEvent`: macOS supports complete keystrokes and rejects directional events with `NOT_IMPLEMENTED`.
|
||||
4. Replace a packed chord such as `pressKey('shift+Tab')` with `pressKey('Tab', ['shift'])`. Check `(await cua.getCapabilities()).methods` before using `sendKeyEvent`. On macOS, call it inside one `run_code` to hold a key across tool calls. To request one repeat, send another down for a non-modifier key held by that execution. Returning, throwing or pausing releases any remaining keys. If your script handles unknown macOS key names as `NATIVE_ERROR`, handle `INVALID_ARGUMENT` instead. Correct the key name before retrying.
|
||||
5. Read `dispatched` from action results, then observe the application to verify the outcome. `invoke` returns `dispatched: false` when the element has no default action. Remove reads of the old `verified` field and of `pid` on selected objects or window records. Use `listApps()` or running `findApps()` matches when you need a PID.
|
||||
6. For a custom provider, replace `CuaBackend.description` with `capabilities` and pass `guidance` and `permissionParameters` to `installCua`. Rebuild any custom `helperPath` against the [operation schema](../../../../packages/experimental/cua/src/protocol.ts) and [tagged permission response](../../../../packages/experimental/cua/src/access-types.ts). Restart the provider, inspect its advertised methods and keyboard scopes, and open the Computer Use permission page to verify its platform controls.
|
||||
@@ -18,6 +18,6 @@ CUA 脚本在指定窗口上截图和操作鼠标。App 和 Window 对象共享
|
||||
1. 调用 `cua.getApp(selector)` 后，从 `app.getWindows()` 返回的列表选定目标，再调用 `app.getWindow(windowId)`。在窗口上调用 `getScreenshot()`、`click()`、`scroll()` 和 `drag()`，将原有的 `hover` 调用改为 `window.mouseMove()`。在 macOS 上，`window.click(id)` 在元素中心发送鼠标事件。需要执行默认辅助功能动作时，改用 `invoke(id)`。需要打开控件菜单且控件声明了 `AXShowMenu` 时，调用 `performAction(id, "AXShowMenu")`。
|
||||
2. 将 `getAXState()` 改为 `getAccessibilityTree()`，将 `getAXStateAndScreenshot()` 拆成读取树和截图两次调用。选择窗口时会静默读取初始树。选择应用时，只有 `appLevelAccessibility` 为真才读取。要显示树，请显式读取。
|
||||
3. 将数字元素索引改为从最新树中取得的字符串 ID。直接发送请求时使用 `elementId` 和 `snapshotId`。原先调用 `performSecondaryAction` 的地方，先用 `listActions(id)` 取得完整动作名，再调用 `performAction(id, exactName, argument?)`。要执行默认动作，可调用 `invoke(id)`。`setValue` 接受提供方支持的 JSON 值。 在 macOS 上，`performAction(id, 'AXRaise')` 只执行元素声明的抬起窗口动作。需要把应用切到前台时先调用 `app.activate()`。
|
||||
4. 将 `pressKey('shift+Tab')` 之类的组合键写法改为 `pressKey('Tab', ['shift'])`。调用 `sendKeyEvent` 前检查 `(await cua.getCapabilities()).methods`。macOS 支持完整按键，单独发送按下或松开事件会返回 `NOT_IMPLEMENTED`。
|
||||
4. 将 `pressKey('shift+Tab')` 之类的组合键写法改为 `pressKey('Tab', ['shift'])`。调用 `sendKeyEvent` 前检查 `(await cua.getCapabilities()).methods`。在 macOS 上，同一次 `run_code` 中可调用此方法跨工具调用保持按键。如果需要一次重复按下，可在这次执行中对已按住的普通键再发一次 `down`。执行返回、异常退出或用户暂停时会释放仍按着的键。如果脚本用 `NATIVE_ERROR` 识别 macOS 上的未知键名，请改为处理 `INVALID_ARGUMENT`，并在重试前修正键名。
|
||||
5. 从动作结果读取 `dispatched`，再观察应用确认效果。元素没有默认动作时，`invoke` 返回 `dispatched: false`。移除读取旧字段 `verified` 的代码，以及对已选对象和窗口记录中 `pid` 的读取。需要 PID 时，使用 `listApps()` 或 `findApps()` 返回的运行中应用记录。
|
||||
6. 自定义提供方需将 `CuaBackend.description` 改为 `capabilities`，并向 `installCua` 传入 `guidance` 和 `permissionParameters`。如配置了 `helperPath`，请按[操作协议](../../../../packages/experimental/cua/src/protocol.ts)和[带平台标记的权限响应](../../../../packages/experimental/cua/src/access-types.ts)重新构建 helper。重启提供方，检查它声明的方法和键盘输入范围，再打开 Computer Use 权限页确认平台控件。
|
||||
@@ -15,8 +15,8 @@
|
||||
en: d152484eb41ac6b4
|
||||
zh: 2f8a621b991260a1
|
||||
/deepseek-ai-dsh-experimental-client-ui-forge/use-this-package:
|
||||
en: 05ee6d9ba0b8d2c7
|
||||
zh: 98c988e8f957511f
|
||||
en: c7e1b7d87ccccd2e
|
||||
zh: 42d864b918a6175c
|
||||
/deepseek-ai-dsh-experimental-client-ui-forge/understand-the-implementation:
|
||||
en: 0afedb76112dfcb8
|
||||
zh: 52d5ddc7a4f9d31c
|
||||
|
||||
@@ -38,7 +38,7 @@ Panel text follows the application language: Chinese, English, Japanese, Spanish
|
||||
- **Session list** — after a Session row's time, a branch glyph shows while the Session works in the worktree the panel created for it, and a glyph shows the state of its most urgent change request: one needing attention first, then open, draft, not yet polled, merged, and closed. The glyph of an open change request keeps the row's subdued color, with its head branch end filled in the color of running, passed, or failed checks; merged and closed ones keep their state colors. The row's hover card names the worktree branch and what it started from, then up to three change requests with their number, title, state, and checks, and how many more there are. A restart shows the states known before it until the next poll.
|
||||
- **Tab** — the first row names the checked-out branch, or the abbreviated commit on a detached HEAD, after a folder glyph for the project directory or a branch glyph for a worktree, and the kind of checkout, which is the worktree the Session created with its base, another worktree, or the project directory; its tooltip adds the path with the Host home shortened to `~`, its button copies the full path, and for a removed worktree it offers Restore. Below it, each change request card shows a glyph for its state, its title, which opens it in the browser, and its number; the glyph tells open, draft, merged, and closed apart, and marks an open one that is not a draft with failed checks, requested changes, or conflicts as needing attention, as the header button's red dot does. The card's header also offers Open in browser, Copy change request link, and Remove from this Session, whose toast offers Undo. Cancelled checks have a separate count; a collapsed card names cancellations when no listed check failed. A collapsed card adds the checks of an open change request and the most urgent of requested changes, conflicts, or draft, or when a merged or closed one ended. An expanded card shows the head and base branches, then one row per fact: for an open change request the checks, whose row opens the list of checks with failed ones first and their durations and links, the review decision with unresolved threads, whose count ends in `+` when the provider counted only some threads, and the merge state, or when a merged or closed one ended; and the changed lines, formatted in the selected UI language, whose row opens the changes view. An open draft also offers Ready for review; a change request the host would merge now offers Merge, whose menu lists the methods the repository allows with the user's usual one chosen and pins the displayed head when opened. Confirm merge uses that head; a polled head change disables confirmation until the menu is reopened after review. A toast reports each outcome, and new commits since that head stop the merge. When exactly one bound change request is open, its card starts expanded. Refresh inspects the checkout, checks sign-in, and polls every change request again; its icon turns until the inspection and the sign-in check end. A banner shows the `gh auth login` command when sign-in is missing.
|
||||
- **Local changes card** — whenever the `branch` scope changes files, independently of bound change requests, a card above the change requests names the ref it compares with, such as `HEAD → origin/main`, which is always the default base below, and shows the added and deleted lines and the number of files, which open the changes view; its file list, which names files as the changes view does, opens the view at one file. Create change request, offered when the code host repository is recognized and no open change request (including drafts) has the checked-out branch as its head, sends the Session a user message in the UI language that asks the Agent to create a change request for these changes, and a toast reports whether it was sent.
|
||||
- **Changes view** — The Changes row of an expanded change request card and the local changes card open a right-sidebar tab that lists the changed files. The local tab is named Local changes and shows Branch by default: the working tree and untracked files against the fork point. Its header names the checked-out branch, or the abbreviated commit on a detached HEAD, and the base branch, which opens a searchable menu: first Default, the base commit of the worktree the Session created, else the merge base of HEAD with the default branch of the code host remote, then remote-tracking and then local branches, each most recently committed first, without the checked-out branch and, outside a created worktree, without the remote-tracking branch Default already names. Typing filters the listed rows at once and lists the branches whose names contain the text, ignoring case, after a short pause; Enter chooses the first row. Choosing another branch compares with the merge base of HEAD and that branch. The tab keeps the choice while other scopes show; closing the tab or reloading the application returns it to Default. When the chosen branch is gone, its name is struck through, and when it shares no history with HEAD, the view says so as well; both offer Use the default base branch instead of Retry. A narrow view hides the note that uncommitted changes count, then the checked-out branch. Uncommitted compares them with HEAD, Unstaged with the index, and Staged compares the index with HEAD. A change request view shows the host's diff at its head and says how many commits the checked-out branch has beyond that head when it is the change request's head branch. The toolbar jumps to a file, totals the changed lines, refreshes, and switches between two columns and one and between wrapped and cut long lines. Visited changes tabs stay mounted across tab and Session switches, hiding the sidebar, and docking changes, preserving file folds, revealed large files, expanded context, layout, and scroll position while the diff is unchanged. Hidden changes views release their panel subscription and defer diff reads; returning reads again only when the latest head or local summary differs. A refresh that replaces the diff discards expanded context, and changed content may move the reading position. Temporary menus close when the view is hidden. Closing the tab or reloading the application discards this progress. Files are named before their directories, which use smaller subdued type and shorten first; hovering one shows its complete path, and both paths for a renamed file. Each file shows badges for a new, deleted, or renamed file, the last naming the old file name, or the old path when only the directory changed, and, in the Branch scope, an untracked one, and a menu that copies its path and opens the whole file in the sidebar for local changes or its page at the head in the browser for a change request. Unchanged lines between hunks collapse into rows that expand from the file's new side: the working tree, the index for Staged, or the head commit through the code host; a file that changed after the diff was read stays collapsed until the view reads again. A binary file and an untracked file over `maxDiffBytes` show a note instead of their lines, a file whose hunks draw more than 1,000 lines starts as one collapsed strip that names its changed lines and draws them when opened, and a file below files that change more than 5,000 lines starts collapsed. The view reads again on Refresh and the page refresh shortcut, on each navigation, when the change request's head moves, and when the Branch scope's summary changes; a change request's view first waits for the status that names its head, unless no provider serves it or that poll failed. When the head moved past the latest status, the view keeps loading and reads again at the head of a status later than the one the read started with, at once when that status arrived during the read. While a read replaces changes already shown, the Refresh icon turns. A change request whose diff the code host or its provider refuses over a size limit, such as a GitHub diff whose output exceeds the provider's `maxOutputBytes`, says its changes are too large to show here and offers View in browser; another failed read offers Retry and, for a change request, View in browser. A change request's diff is downloaded once per head commit and base branch while it stays cached, so reopening, refreshing, and navigating at the same head read nothing from the host.
|
||||
- **Changes view** — The Changes row of an expanded change request card and the local changes card open a right-sidebar tab that lists the changed files. The local tab is named Local changes and shows Branch by default: the working tree and untracked files against the fork point. Its header names the checked-out branch, or the abbreviated commit on a detached HEAD, and the base branch, which opens a searchable menu: first Default, the base commit of the worktree the Session created, else the merge base of HEAD with the default branch of the code host remote, then remote-tracking and then local branches, each most recently committed first, without the checked-out branch and, outside a created worktree, without the remote-tracking branch Default already names. Typing filters the listed rows at once and lists the branches whose names contain the text, ignoring case, after a short pause; Enter chooses the first row. Choosing another branch compares with the merge base of HEAD and that branch. The tab keeps the choice while other scopes show; closing the tab or reloading the application returns it to Default. When the chosen branch is gone, its name is struck through, and when it shares no history with HEAD, the view says so as well; both offer Use the default base branch instead of Retry. A narrow view hides the note that uncommitted changes count, then the checked-out branch. Uncommitted compares them with HEAD, Unstaged with the index, and Staged compares the index with HEAD. A change request view shows the host's diff at its head and says how many commits the checked-out branch has beyond that head when it is the change request's head branch. The toolbar jumps to a file, totals the changed lines, refreshes, and switches between two columns and one and between wrapped and cut long lines. Visited changes tabs stay mounted across tab and Session switches, hiding the sidebar, and docking changes, preserving file folds, revealed large files, expanded context, layout, and scroll position while the diff is unchanged. Hidden changes views release their panel subscription and defer diff reads; returning reads again only when the latest head or local summary differs. A refresh that replaces the diff discards expanded context, and changed content may move the reading position. Temporary menus close when the view is hidden. Closing the tab or reloading the application discards this progress. Each file is a separate card, with a tinted header that stays at the top while its lines scroll. Files are named before their directories, which use smaller subdued type and shorten first; hovering one shows its complete path, and both paths for a renamed file. Each file shows badges for a new, deleted, or renamed file, the last naming the old file name, or the old path when only the directory changed, and, in the Branch scope, an untracked one, and a menu that copies its path and opens the whole file in the sidebar for local changes or its page at the head in the browser for a change request. When the header is narrow, the path, the badges, and Changed since viewed shorten, so the line counts, Viewed, and the menu stay whole. Unchanged lines between hunks collapse into rows that expand from the file's new side: the working tree, the index for Staged, or the head commit through the code host; a file that changed after the diff was read stays collapsed until the view reads again. A binary file and an untracked file over `maxDiffBytes` show a note instead of their lines, a file whose hunks draw more than 1,000 lines starts as one collapsed strip that names its changed lines and draws them when opened, and a file below files that change more than 5,000 lines starts collapsed. The view reads again on Refresh and the page refresh shortcut, on each navigation, when the change request's head moves, and when the Branch scope's summary changes; a change request's view first waits for the status that names its head, unless no provider serves it or that poll failed. When the head moved past the latest status, the view keeps loading and reads again at the head of a status later than the one the read started with, at once when that status arrived during the read. While a read replaces changes already shown, the Refresh icon turns. A change request whose diff the code host or its provider refuses over a size limit, such as a GitHub diff whose output exceeds the provider's `maxOutputBytes`, says its changes are too large to show here and offers View in browser; another failed read offers Retry and, for a change request, View in browser. A change request's diff is downloaded once per head commit and base branch while it stays cached, so reopening, refreshing, and navigating at the same head read nothing from the host.
|
||||
|
||||
- **Review layers, test files, viewed files, and commits** — Above the file tree, the lens keeps All files or the files of one layer, Interface, Implementation, or Tests, each with its count; where no tree shows, the toolbar offers the lens as a menu. The repository decides first: a file whose root `.gitattributes` sets `dsh-review-layer=test`, `interface`, or `implementation` is in that layer, and one it marks `linguist-generated` is in Tests. Next come the optional `testPaths` and `interfacePaths` globs, which add to the built-in rules and are empty by default. The built-in rules then follow common conventions: test directories and files of Jest, Vitest, `go test`, pytest, RSpec, XCTest, and .NET, snapshots, locale catalogs, and lockfiles are Tests; package manifests, type declarations, headers, stubs, and interface definitions such as Protocol Buffers, GraphQL, and OpenAPI are Interface. Every other file is in Implementation. For example, `*_test.go dsh-review-layer=test` or `api/**/*.yaml dsh-review-layer=interface` in `.gitattributes` records a repository's own convention. Hide test files leaves Tests out of the file tree and the diff without listing them, and disables the Tests lens. Viewed on a file header marks that change of the file viewed and collapses it; the toolbar shows how many of the shown files are viewed, and `V` marks or clears the file being read: the file holding the focus or the text cursor, else the file at the top of the view. A file whose change differs from the change it was marked at shows Changed since viewed and expands again. Marks are kept in this browser per Session, source, path, and change digest, so they survive closing the tab and reloading, and the tabs of one Session share them: a mark in one tab appears in the others and keeps their marks. A change of the file mode alone, such as `chmod +x`, also counts as a new change. All commits beside Submit review, and beside the base branch in the Branch scope, lists the commits oldest first, read once until the head, the local branch or base, or a refresh changes them; a click shows one commit and a Shift click the commits from the last one clicked, which must follow each other as first parents: a run that leaves that line, such as one across a merged branch, keeps the menu open and says why; the view then shows those commits' changes, reads unchanged lines at the last of them, and offers Add to input but no GitHub comment or review. Branch commits exclude uncommitted and untracked files. Another base or scope returns to all commits, and chosen commits that are gone or no longer one run offer Show all commits.
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ kind: "package-reference"
|
||||
- **会话列表**——Session 在面板为它创建的 worktree 中工作时，会话行的时间之后显示分支图标；另有一个图标显示它最需要关注的变更请求的状态，依次为需要处理、打开、草稿、尚未轮询、已合并、已关闭。打开中的变更请求图标保持会话行的浅色，head 分支端点填充为检查进行中、已通过或失败对应的颜色；已合并与已关闭的保留各自的状态颜色。会话行的悬浮卡片写出 worktree 分支及其起点，再列出最多三个变更请求的编号、标题、状态与检查，并说明还有多少个。重启后，下次轮询之前显示重启前已知的状态。
|
||||
- **标签页**——顶部一行显示当前检出的分支（分离 HEAD 时为缩写提交），项目目录前显示目录图标、worktree 前显示分支图标，并说明检出的类型（Session 创建的 worktree 及其基准、其他 worktree 或项目目录）；它的提示给出路径，宿主主目录缩写为 `~`，旁边的按钮复制完整路径；worktree 已被删除时提供“恢复”。其下每张变更请求卡片显示状态图标、标题（点击在浏览器中打开）与编号；图标区分打开、草稿、已合并与已关闭，打开中的变更请求（草稿除外）检查失败、被要求修改或有冲突时显示为“需要处理”，与标题栏按钮的红点一致。卡片右上角还提供“在浏览器中打开”“复制变更请求链接”与“从本会话移除”，移除后的 toast 提供“撤销”。已取消的检查单独计数；列出的检查没有失败项时，折叠卡片显示取消数量。折叠的卡片还显示打开的变更请求的检查，以及被要求修改、冲突、草稿中最需要注意的一项；已合并或已关闭的则显示结束时间。展开的卡片显示源分支与目标分支，下面每项一行：打开的变更请求依次是检查（点击该行弹出检查列表，失败的排在前面，带耗时与链接）、评审结论与未解决讨论数（提供方只统计了部分讨论时数字后带 `+`）、合并状态，已合并或已关闭的显示结束时间；最后是按所选界面语言格式化的变更行数，点击该行打开变更视图。打开中的草稿还提供“标记为可评审”；主机现在就会合并的变更请求提供“合并”，其菜单列出仓库允许的合并方式并选中用户常用的一种，打开时固定卡片显示的 head。“确认合并”使用这个 head；轮询发现 head 变化后禁用确认，需重新检查并打开菜单。每次操作的结果都以 toast 报告；该 head 之后有新提交时不会合并。恰好有一个已绑定的变更请求处于打开状态时，它的卡片默认展开。刷新会重新检查检出与登录状态，并立即再次轮询每个变更请求；检查与登录状态确认结束前，刷新图标持续旋转。缺少登录时，横幅显示 `gh auth login` 命令。
|
||||
- **本地变更卡片**——`branch` 范围有文件变更时始终显示，不受已关联变更请求影响。变更请求列表上方的卡片显示比较的基准（例如 `HEAD → origin/main`，始终是下文的默认基准）以及新增、删除行数与文件数，点击打开变更视图；展开的文件列表像变更视图一样标出各个文件，点击某个文件时，变更视图定位到该文件。识别出代码托管仓库、且没有以当前检出分支为源分支的打开中变更请求（含草稿）时，卡片提供“创建变更请求”：它以界面语言向 Session 发送一条用户消息，请 Agent 为这些变更创建变更请求，并以 toast 报告是否已发送。
|
||||
- **变更视图**——展开的变更请求卡片上的“变更”一行与本地变更卡片会打开一个右侧边栏标签页，列出所有变更的文件。本地标签页名为“本地变更”，默认显示“分支”：工作树与未跟踪文件相对于分叉点的变更。它的标题栏写出当前检出的分支（分离 HEAD 时为缩写提交）与基准分支；点击基准分支打开可搜索的菜单：最前面是“默认”，即 Session 创建的 worktree 的基准提交，否则是 HEAD 与代码托管远端默认分支的合并基准；其后依次是远程跟踪分支与本地分支，各按最近提交排序，不含当前检出的分支，不在所创建的 worktree 中时也不含“默认”已经指向的远程跟踪分支。输入文字会立即筛选已列出的行，短暂停顿后列出名称包含该文字（不区分大小写）的分支；按 Enter 选择第一行。选择其他分支后，与 HEAD 和该分支的合并基准比较。标签页切换到其他范围时仍保留这个选择；关闭标签页或重新加载应用后回到“默认”。所选分支已不存在时，它的名称显示删除线；它与 HEAD 没有共同历史时，视图同样给出说明；两种情况都提供“使用默认基准分支”而不是“重试”。视图较窄时，先隐藏“包含未提交的修改”，再隐藏当前分支。“未提交”与 HEAD 比较，“未暂存”与暂存区比较，“已暂存”比较暂存区与 HEAD。变更请求的视图显示托管平台在其 head 处的 diff；当前检出分支就是该变更请求的源分支时，还会说明本地分支比该 head 多出几个提交。工具栏可以跳到某个文件、汇总变更行数、刷新，并在左右对比与单栏对比、自动换行与截断长行之间切换。访问过的变更标签页会在切换标签页与 Session、隐藏侧边栏及调整停靠位置时保持挂载，在 diff 未改变时保留文件折叠、大文件展开、上下文展开、布局与滚动位置。隐藏的变更视图释放面板订阅并推迟 diff 读取；返回时仅在最新 head 或本地摘要不同时重新读取。刷新替换 diff 后会丢弃展开的上下文，内容变化也可能移动阅读位置。视图隐藏时关闭临时菜单。关闭标签页或重新加载应用后不保留这些进度。文件名在前，目录在后，用更小的浅色字显示，空间不够时目录先缩短；悬停时显示完整路径，重命名的文件显示新旧两条路径。每个文件带有新文件、已删除、重命名的标记（重命名标记写出原文件名，只换了目录时写出原路径），“分支”范围下的未跟踪文件还标“未提交”；文件菜单可以复制路径，本地变更可在侧边栏打开整个文件，变更请求可在浏览器中查看该文件在 head 处的页面。hunk 之间未变更的行折叠成一行，点击后从文件的新版本展开：工作树、“已暂存”时为暂存区，或经托管平台读取的 head 提交；读取 diff 之后文件又有修改时不会展开，直到视图重新读取。二进制文件与超过 `maxDiffBytes` 的未跟踪文件只显示一条说明；hunk 超过 1,000 行的文件先显示为一条写出变更行数的折叠条，点击后才绘制；前面的文件累计变更超过 5,000 行后，其后的文件默认收起。刷新与页面刷新快捷键、每次导航、变更请求的 head 变化，以及“分支”范围的摘要变化时，视图都会重新读取；变更请求的视图会先等到给出 head 的状态再读取，除非没有提供方服务它或那次轮询失败。head 已越过最新状态时，视图保持加载，等到比读取开始时更新的状态后按其 head 重新读取；该状态在读取期间已到达时立即重读。重新读取已显示的变更时，刷新图标会旋转。托管平台或其提供方因超过大小上限拒绝给出 diff 的变更请求（例如 GitHub 上输出超过提供方 `maxOutputBytes` 的 diff）显示“变更太大，无法在这里显示”并提供“在浏览器中查看”；其他读取失败提供“重试”，变更请求还提供“在浏览器中查看”。变更请求的 diff 在缓存期间对每个 head 提交与目标分支只下载一次，因此在同一 head 下重新打开、刷新与导航都不会再从托管平台读取。
|
||||
- **变更视图**——展开的变更请求卡片上的“变更”一行与本地变更卡片会打开一个右侧边栏标签页，列出所有变更的文件。本地标签页名为“本地变更”，默认显示“分支”：工作树与未跟踪文件相对于分叉点的变更。它的标题栏写出当前检出的分支（分离 HEAD 时为缩写提交）与基准分支；点击基准分支打开可搜索的菜单：最前面是“默认”，即 Session 创建的 worktree 的基准提交，否则是 HEAD 与代码托管远端默认分支的合并基准；其后依次是远程跟踪分支与本地分支，各按最近提交排序，不含当前检出的分支，不在所创建的 worktree 中时也不含“默认”已经指向的远程跟踪分支。输入文字会立即筛选已列出的行，短暂停顿后列出名称包含该文字（不区分大小写）的分支；按 Enter 选择第一行。选择其他分支后，与 HEAD 和该分支的合并基准比较。标签页切换到其他范围时仍保留这个选择；关闭标签页或重新加载应用后回到“默认”。所选分支已不存在时，它的名称显示删除线；它与 HEAD 没有共同历史时，视图同样给出说明；两种情况都提供“使用默认基准分支”而不是“重试”。视图较窄时，先隐藏“包含未提交的修改”，再隐藏当前分支。“未提交”与 HEAD 比较，“未暂存”与暂存区比较，“已暂存”比较暂存区与 HEAD。变更请求的视图显示托管平台在其 head 处的 diff；当前检出分支就是该变更请求的源分支时，还会说明本地分支比该 head 多出几个提交。工具栏可以跳到某个文件、汇总变更行数、刷新，并在左右对比与单栏对比、自动换行与截断长行之间切换。访问过的变更标签页会在切换标签页与 Session、隐藏侧边栏及调整停靠位置时保持挂载，在 diff 未改变时保留文件折叠、大文件展开、上下文展开、布局与滚动位置。隐藏的变更视图释放面板订阅并推迟 diff 读取；返回时仅在最新 head 或本地摘要不同时重新读取。刷新替换 diff 后会丢弃展开的上下文，内容变化也可能移动阅读位置。视图隐藏时关闭临时菜单。关闭标签页或重新加载应用后不保留这些进度。每个文件是一张独立的卡片，带浅色底的标题在滚动时停在顶部。文件名在前，目录在后，用更小的浅色字显示，空间不够时目录先缩短；悬停时显示完整路径，重命名的文件显示新旧两条路径。每个文件带有新文件、已删除、重命名的标记（重命名标记写出原文件名，只换了目录时写出原路径），“分支”范围下的未跟踪文件还标“未提交”；文件菜单可以复制路径，本地变更可在侧边栏打开整个文件，变更请求可在浏览器中查看该文件在 head 处的页面。标题变窄时，路径、标记和“有新改动”会缩短，行数、“已审阅”和文件菜单始终完整显示。hunk 之间未变更的行折叠成一行，点击后从文件的新版本展开：工作树、“已暂存”时为暂存区，或经托管平台读取的 head 提交；读取 diff 之后文件又有修改时不会展开，直到视图重新读取。二进制文件与超过 `maxDiffBytes` 的未跟踪文件只显示一条说明；hunk 超过 1,000 行的文件先显示为一条写出变更行数的折叠条，点击后才绘制；前面的文件累计变更超过 5,000 行后，其后的文件默认收起。刷新与页面刷新快捷键、每次导航、变更请求的 head 变化，以及“分支”范围的摘要变化时，视图都会重新读取；变更请求的视图会先等到给出 head 的状态再读取，除非没有提供方服务它或那次轮询失败。head 已越过最新状态时，视图保持加载，等到比读取开始时更新的状态后按其 head 重新读取；该状态在读取期间已到达时立即重读。重新读取已显示的变更时，刷新图标会旋转。托管平台或其提供方因超过大小上限拒绝给出 diff 的变更请求（例如 GitHub 上输出超过提供方 `maxOutputBytes` 的 diff）显示“变更太大，无法在这里显示”并提供“在浏览器中查看”；其他读取失败提供“重试”，变更请求还提供“在浏览器中查看”。变更请求的 diff 在缓存期间对每个 head 提交与目标分支只下载一次，因此在同一 head 下重新打开、刷新与导航都不会再从托管平台读取。
|
||||
|
||||
- **分层、测试文件、已审阅与提交**——文件树上方的分层透镜保留全部文件，或“接口”“实现”“测试”中一层的文件，并显示各自的数量；不显示文件树时，工具栏以菜单提供透镜。仓库的声明优先：根目录 `.gitattributes` 把文件设为 `dsh-review-layer=test`、`interface` 或 `implementation` 时，文件属于对应的层；标为 `linguist-generated` 的文件属于“测试”。其次是可选的 `testPaths` 与 `interfacePaths`，它们追加在内置规则之前，默认为空。最后是按常见约定写的内置规则：Jest、Vitest、`go test`、pytest、RSpec、XCTest 与 .NET 的测试目录和文件、快照、本地化目录与 lockfile 属于“测试”；包清单、类型声明、头文件、类型存根与 Protocol Buffers、GraphQL、OpenAPI 等接口定义属于“接口”。其余文件属于“实现”。例如在 `.gitattributes` 中写 `*_test.go dsh-review-layer=test` 或 `api/**/*.yaml dsh-review-layer=interface`，即可记录仓库自己的约定。“隐藏测试文件”让文件树与 diff 都不显示“测试”层，也不列出被隐藏的文件，同时停用“测试”透镜。文件标题上的“已审阅”把该文件的这次改动标为已审阅并折叠它；工具栏显示已显示文件中已审阅的数量，按 `V` 标记或取消当前阅读的文件：焦点或文本光标所在的文件，否则是视图顶部的文件。改动与标记时不同的文件显示“有新改动”并重新展开。标记按 Session、变更来源、路径与改动摘要保存在本浏览器中，关闭标签页或重新加载后仍在；同一 Session 的多个标签页共用这些标记：在一个标签页中标记，其他标签页随之更新，且不会覆盖它们的标记。只改文件模式（例如 `chmod +x`）也算新改动。“提交评审”左侧，以及“分支”范围中基准分支右侧的“全部提交”按从旧到新列出提交，读取一次后保留到 head、本地分支或基准变化或刷新为止；单击只看一个提交，按住 Shift 单击看从上次单击的提交到这个提交的连续多个提交，这些提交必须依次是后一个的第一父提交：离开这条链的范围（例如跨越已合并分支的范围）会让菜单保持打开并说明原因；此时视图显示这些提交的改动，在最后一个提交读取未变更的行，提供“添加到输入框”，但不提供 GitHub 评论或评审。“分支”的提交不含未提交与未跟踪的文件。换基准分支或范围后回到全部提交；所选提交已不存在或不再是一条链时提供“查看全部提交”。
|
||||
|
||||
|
||||
@@ -346,9 +346,12 @@
|
||||
}
|
||||
|
||||
.files {
|
||||
--forge-files-inset: 10px;
|
||||
flex: 1;
|
||||
box-sizing: border-box;
|
||||
min-width: 0;
|
||||
min-height: 0;
|
||||
padding: var(--forge-files-inset);
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
@@ -377,14 +380,34 @@
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
/* Each file is one card, so a file's last collapsed-context strip and the next file's header no longer read as adjacent
|
||||
rows. Diffs share R16; clip, unlike hidden, keeps the card from becoming the sticky header's scroll container. */
|
||||
.file {
|
||||
border-bottom: 0.5px solid var(--dsw-alias-border-l3);
|
||||
overflow: clip;
|
||||
border: 0.5px solid var(--dsw-alias-border-l4);
|
||||
border-radius: var(--dsw-radius-lg);
|
||||
background: var(--forge-changes-bg);
|
||||
}
|
||||
|
||||
/* Above the comment plus (z-index 1) of rows scrolled beneath it. */
|
||||
.file + .file {
|
||||
margin-top: 12px;
|
||||
}
|
||||
|
||||
@container forge-changes (max-width: 380px) {
|
||||
.files {
|
||||
--forge-files-inset: 6px;
|
||||
}
|
||||
|
||||
.file + .file {
|
||||
margin-top: 8px;
|
||||
}
|
||||
}
|
||||
|
||||
/* Above the comment plus (z-index 1) of rows scrolled beneath it. Sticky offsets start inside the list's padding,
|
||||
so the negative inset keeps a stuck header flush with the scroll area's top edge. */
|
||||
.fileHeader {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
top: calc(-1 * var(--forge-files-inset, 0px));
|
||||
z-index: 2;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
@@ -393,7 +416,8 @@
|
||||
min-height: 36px;
|
||||
padding: 0 4px 0 8px;
|
||||
border-bottom: 0.5px solid var(--dsw-alias-border-l3);
|
||||
background: var(--forge-changes-bg);
|
||||
/* The hover tint over the view background stays opaque while code scrolls beneath the sticky header. */
|
||||
background: linear-gradient(var(--dsw-alias-interactive-bg-hover), var(--dsw-alias-interactive-bg-hover)), var(--forge-changes-bg);
|
||||
}
|
||||
|
||||
/* A bare chevron: its rotation shows whether the file is expanded, so it takes no fill in any state. */
|
||||
@@ -422,10 +446,23 @@
|
||||
border-bottom: 0;
|
||||
}
|
||||
|
||||
/* The file's own information gives way inside this box, so the counts, the viewed mark, and the menu after it stay
|
||||
whole at any width. */
|
||||
.fileTitle {
|
||||
display: flex;
|
||||
flex: 1 1 0;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.fileIcon {
|
||||
flex: none;
|
||||
}
|
||||
|
||||
/* The path shares the shortage with the badge in proportion to their widths, so the longer path keeps the larger part
|
||||
and stays under the pointer for its full-path tooltip. */
|
||||
.filePath {
|
||||
display: flex;
|
||||
flex: 0 1 auto;
|
||||
@@ -445,8 +482,16 @@
|
||||
transform: rotate(-90deg);
|
||||
}
|
||||
|
||||
/* Capped rather than shrunk, so a long path never truncates the tag; a longer translation ends in an ellipsis. */
|
||||
.changedTag {
|
||||
flex: none;
|
||||
max-width: 30%;
|
||||
}
|
||||
|
||||
.changedText {
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
/* The viewed mark: a native checkbox with its label as one bordered chip, peer to the file menu button. */
|
||||
@@ -502,7 +547,8 @@
|
||||
}
|
||||
|
||||
.badge {
|
||||
flex: none;
|
||||
flex: 0 1 auto;
|
||||
min-width: 0;
|
||||
max-width: 50%;
|
||||
overflow: hidden;
|
||||
padding: 0 6px;
|
||||
|
||||
@@ -560,13 +560,17 @@ function DiffFileSection(props: DiffFileProps): ReactNode {
|
||||
>
|
||||
<IconChevronDownOutlineRegular size={14} className={css.chevron} />
|
||||
</button>
|
||||
<FileTypeIcon path={file.path} size={16} className={css.fileIcon} />
|
||||
<span className={css.filePath}>
|
||||
<FileLabel path={file.path} tooltip={tooltip} />
|
||||
<span className={css.fileTitle}>
|
||||
<FileTypeIcon path={file.path} size={16} className={css.fileIcon} />
|
||||
<span className={css.filePath}>
|
||||
<FileLabel path={file.path} tooltip={tooltip} />
|
||||
</span>
|
||||
{viewed === 'changed' && (
|
||||
<Tag tone="warning" className={css.changedTag}><span className={css.changedText}>{t('viewed.changed')}</span></Tag>
|
||||
)}
|
||||
<span className={css.fileSpacer} data-forge-path-spacer="" />
|
||||
{badge !== '' && <span className={css.badge}>{badge}</span>}
|
||||
</span>
|
||||
{viewed === 'changed' && <Tag tone="warning" className={css.changedTag}>{t('viewed.changed')}</Tag>}
|
||||
<span className={css.fileSpacer} data-forge-path-spacer="" />
|
||||
{badge !== '' && <span className={css.badge}>{badge}</span>}
|
||||
{!file.binary && <Counts file={file} t={t} />}
|
||||
<label className={css.viewed} title={t('viewed.hint')}>
|
||||
<input type="checkbox" checked={viewed === 'viewed'} aria-label={t('viewed.aria', { name })}
|
||||
|
||||
@@ -15,8 +15,8 @@
|
||||
en: 1777e7bf6785f986
|
||||
zh: e3daac1074ebb74d
|
||||
/deepseek-ai-dsh-experimental-cua-macos/use-this-package:
|
||||
en: 533ed4dd11af5126
|
||||
zh: 03036e8c949d6aca
|
||||
en: 30fefafc1915db8a
|
||||
zh: e819f1e722c0d614
|
||||
/deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment:
|
||||
en: b734a0907747580e
|
||||
zh: 8110c5951b70e100
|
||||
|
||||
@@ -79,7 +79,9 @@ Source builds default to `sourceSigning: ad-hoc`, without creating certificates
|
||||
|
||||
The [configuration catalog](../../../docs/config-catalog.md#deepseek-aidsh-experimental-cua-macos) lists all fields. Native tool calls use `{method, params}`; the complete method-specific schema is exposed to the model. `getApp` returns the `appHandle` used by every app-targeted operation, and `app.getWindow(windowId)` returns the object that owns screenshots and pointer input.
|
||||
|
||||
`pressKey(key, modifiers?)` presses and releases one key. Pass modifier names separately, for example `pressKey('Tab', ['shift'])` or `pressKey('+')`; an omitted modifier array means none. The macOS helper accepts `command`/`cmd`, `shift`, `option`/`alt`, `control`/`ctrl`, and `fn`, and maps them to their physical keys. Key names are platform-defined; keypad names (`KP_0` through `KP_9`, `KP_Enter`, and keypad operators) retain their physical key codes. `typeText` sends literal text, including its line breaks and tabs. On macOS, use `pressKey` on the App object. The shared `sendKeyEvent` operation returns `NOT_IMPLEMENTED` on this provider.
|
||||
`pressKey(key, modifiers?)` presses and releases one key. For shortcuts, pass held modifiers separately, as in `pressKey('a', ['command'])`. Pressing a modifier itself, as in `pressKey('shift')`, sets its flag on press and clears it on release. A single `+` or `plus` sends Shift+=. The macOS helper accepts `command`/`cmd`, `shift`, `option`/`alt`, `control`/`ctrl`, and `fn`, and maps them to their physical keys. Key names are platform-defined; keypad names (`KP_0` through `KP_9`, `KP_Enter`, and keypad operators) retain their physical key codes. `typeText` sends literal text, including its line breaks and tabs. Use the App object's `pressKey` for a short press.
|
||||
|
||||
For a long press, call `app.sendKeyEvent('w', 'down')` inside `run_code`. The first down holds the key across subsequent tool calls. For an ordinary key already held by this execution, each additional down sends one key-down marked as a repeat to the original process. Choose the interval between calls. A control can filter synthetic repeats, so check its response. One up releases the key. Choose when to send each up: after W down, D down and D up, W stays down. When the program returns, exits with an error, or the user pauses it, the helper releases the remaining keys at the processes that received their downs. A later execution cannot release an earlier execution's keys.
|
||||
|
||||
Agent disposal and plugin unload cancel and await the complete tool call, including image admission and attachment storage. An image write already in progress must settle before disposal returns; a canceled call cannot return a successful screenshot.
|
||||
|
||||
@@ -200,7 +202,7 @@ Setup and discovery:
|
||||
Available at app or window level depending on the platform.
|
||||
|
||||
Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the full tree text; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the tree within the provider limits; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. If the tree reports maxElements or maxDepth truncation, select a narrower window or ask the user to raise the named macOS provider setting and read again. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current.
|
||||
- listActions(elementId: string) -> string[] — list the actions an element advertises.
|
||||
- invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none.
|
||||
- performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument.
|
||||
@@ -222,6 +224,7 @@ Available at app or window level depending on the platform.
|
||||
- typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab.
|
||||
- paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry.
|
||||
- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string.
|
||||
- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.
|
||||
|
||||
Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name.
|
||||
```
|
||||
|
||||
@@ -79,7 +79,9 @@ pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis.
|
||||
|
||||
[配置目录](../../../docs/config-catalog.zh.md#deepseek-aidsh-experimental-cua-macos)列出全部字段。原生工具调用使用 `{method, params}`，模型可以看到每个方法的完整参数定义。`getApp` 返回每个面向应用的操作所需的 `appHandle`，`app.getWindow(windowId)` 返回负责截图和指针输入的对象。
|
||||
|
||||
`pressKey(key, modifiers?)` 按下并释放一个键。修饰键名称单独传入，例如 `pressKey('Tab', ['shift'])` 或 `pressKey('+')`；省略修饰键数组表示不按修饰键。macOS helper 接受 `command`/`cmd`、`shift`、`option`/`alt`、`control`/`ctrl` 和 `fn`，并映射到对应物理键。按键名由平台定义；数字键盘名称（`KP_0` 到 `KP_9`、`KP_Enter` 和数字键盘运算键）保留独立的物理键码。`typeText` 发送字面文本，包括其中的换行和制表符。在 macOS 上，通过 App 对象调用 `pressKey`。调用共享接口 `sendKeyEvent` 会收到 `NOT_IMPLEMENTED`。
|
||||
`pressKey(key, modifiers?)` 按下并松开一个键。使用快捷键时单独传入修饰键，例如 `pressKey('a', ['command'])`。单独调用 `pressKey('shift')` 时，按下事件带有 Shift 标志，释放事件清除该标志。`+` 和 `plus` 都会发送 Shift 和等号。macOS helper 接受 `command`/`cmd`、`shift`、`option`/`alt`、`control`/`ctrl` 和 `fn`，并映射到对应物理键。按键名由平台定义；数字键盘名称（`KP_0` 到 `KP_9`、`KP_Enter` 和数字键盘运算键）保留独立的物理键码。`typeText` 发送字面文本，包括其中的换行和制表符。短按使用 App 对象上的 `pressKey`。
|
||||
|
||||
长按时，在 `run_code` 中调用 `app.sendKeyEvent('w', 'down')`。首次按下后，该键在同一次执行的后续工具调用中保持按下。对自己已按住的普通键再发一次 `down`，原接收进程会收到一次带重复标志的按下事件。调用方决定是否重复和间隔多久。控件可能过滤模拟的重复事件，可通过控件的实际变化确认效果。松键只需一次 `up`。每个键可分别松开。例如先按 W，再按 D，松开 D 后 W 仍按着。程序正常返回、出错退出或用户暂停时，helper 为本次执行仍按着的键逐一发送松开事件，每个事件都发回原接收进程。后续执行不能释放前一次执行的键。
|
||||
|
||||
Agent 销毁和插件卸载会取消并等待完整工具调用，包括图像接入检查和附件存储。已经开始的图像写入必须结束后，销毁才会返回；已取消的调用不会返回成功截图。
|
||||
|
||||
@@ -200,7 +202,7 @@ Setup and discovery:
|
||||
Available at app or window level depending on the platform.
|
||||
|
||||
Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the full tree text; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the tree within the provider limits; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. If the tree reports maxElements or maxDepth truncation, select a narrower window or ask the user to raise the named macOS provider setting and read again. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current.
|
||||
- listActions(elementId: string) -> string[] — list the actions an element advertises.
|
||||
- invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none.
|
||||
- performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument.
|
||||
@@ -222,6 +224,7 @@ Available at app or window level depending on the platform.
|
||||
- typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab.
|
||||
- paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry.
|
||||
- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string.
|
||||
- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.
|
||||
|
||||
Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name.
|
||||
```
|
||||
|
||||
@@ -18,14 +18,14 @@
|
||||
en: 1435672513328c05
|
||||
zh: 11e7a1e672410c10
|
||||
/dsh-cua-native-helper/request-protocol:
|
||||
en: 88a59904ca4ebedc
|
||||
zh: 14f2b666d7382145
|
||||
en: 2a69d89b7fc95998
|
||||
zh: 084575bb31fb40dd
|
||||
/dsh-cua-native-helper/observation-and-input:
|
||||
en: 2d380e942a4057e4
|
||||
zh: fe7c9339ec63a6e5
|
||||
en: 0808c25f70ce6eff
|
||||
zh: 820864a2b18b8a24
|
||||
/dsh-cua-native-helper/verification:
|
||||
en: 49f4cce2a4cc3114
|
||||
zh: e9d65c92ef35fe24
|
||||
en: c87945aeec587bf5
|
||||
zh: e5d69e4927e2fed3
|
||||
/dsh-cua-native-helper/further-exploration:
|
||||
en: 265e1927952ce6d7
|
||||
zh: 7c35d9fd98b6e6c8
|
||||
|
||||
@@ -78,7 +78,7 @@ Successful `activate` returns `dispatched` and `foregroundChanged`. A known chan
|
||||
| `getState` | `{appHandle, windowId, focusedWindowId, snapshotId, tree, elements, windows, truncated}`. `windowId` is null for an app-wide observation. Elements include the opaque string `elementId`, role, title, value, description, identifier, bounds, actions, and enabled state. Element bounds are null or [x, y, width, height] in screen points; window input uses local points. These structured element diagnostics are retained by native checks and omitted by the model tool projection. Unavailable optional descriptive properties are null, and an unavailable action list is empty; permission, communication and window-discovery failures still reject the observation. |
|
||||
| `listActions` | Provider-defined action names for one element as a string array; an element may advertise none. |
|
||||
| `show` | `{dispatched: boolean, windowId, foregroundChanged}` records restoration dispatch. `windowId` is null when no window was selected; `foregroundChanged` is null when the sampled observation was unavailable. |
|
||||
| `invoke`, `performAction`, `mouseMove`, `click`, `setValue`, `selectText`, `typeText`, `paste`, `pressKey`, `scroll`, `drag` | `{dispatched: boolean}` records native dispatch; `invoke` reports false when no applicable default action exists. `mouseMove` also reports `hoverRoute` and `systemPointerMoved`. Read fresh state to verify the intended application change. |
|
||||
| `invoke`, `performAction`, `mouseMove`, `click`, `setValue`, `selectText`, `typeText`, `paste`, `pressKey`, `sendKeyEvent`, `scroll`, `drag` | `{dispatched: boolean}` records native dispatch; `invoke` reports false when no applicable default action exists. `mouseMove` also reports `hoverRoute` and `systemPointerMoved`. Read fresh state to verify the intended application change. |
|
||||
| `screenshot` | `{mimeType: "image/png", data, pixelWidth, pixelHeight, bounds, coordinateSpace: "window-points", windowId}`. PNG pixel dimensions can differ from window point dimensions on Retina displays. |
|
||||
|
||||
Every app-targeted operation requires `appHandle` from `getApp` in the same helper. The helper pins the PID and kernel process start time, and invalidates the handle and all of its snapshots after process exit or PID reuse. Calling `getApp` can acquire the replacement process without reactivating old handles. A random per-helper namespace prevents handles from a terminated helper matching those of its replacement. Repeated acquisition of the same live process shares its handle. A post-launch inspection failure invalidates the pending handle and preserves the original error. It does not terminate a process already started; launchApp cannot reuse the failed handle.
|
||||
@@ -103,7 +103,9 @@ Clicks send mouse-down and mouse-up events at the element center or the supplied
|
||||
|
||||
Capture and pointer delivery read CG bounds before and after an exact-window AX frame query. Movement during the query returns WINDOW_CHANGED. Valid AX/CG frames must have equal outward-rounded integral point rectangles; a known mismatch returns WINDOW_CHANGED before mouse movement, priming or button delivery. The helper preserves both coordinate sources rather than substituting AX geometry for CG geometry. Missing or unusable AX geometry remains unknown, while AX communication, permission and invalid-element errors propagate. Capture without Accessibility permission retains its CG/SCK checks. Advertised AX actions bypass this pointer preflight; an AX action error does not trigger pointer input. These reads do not lock the window against later movement, and no geometry check activates or raises it.
|
||||
|
||||
`typeText` and `pressKey` target the selected process's current focused control, not an exact window. `pressKey` presses and releases one key while holding its `modifiers` array, sends PID-targeted modifier transitions, and releases those modifiers in reverse order; these events do not use the global HID queue. Key names are platform-defined; keypad names such as `KP_0` and `KP_Enter` preserve their physical key codes. Background pointer actions may change that keyboard focus. Text injection sends Unicode characters individually and does not use the clipboard. LF, CR and CRLF send one Return; tabs send Tab. Applications apply their normal editing behavior, including indentation and focus navigation. Each `typeText` down/up event is followed by the configured text interval; `pressKey` keeps its own press and release timing, and dispatch completion does not confirm that the receiving application consumed every event. Cancellation can interrupt text or a gesture after partial input; it cannot undo received events. Very long text can exceed the host's operation deadline. Validate observed application state before deciding how to continue.
|
||||
`typeText` and `pressKey` target the selected process's current focused control, not an exact window. `pressKey` presses the listed modifiers followed by one key, then releases all pressed keys in reverse order, including on cancellation. Keyboard events go directly to the target PID and bypass the global HID queue. Key names are platform-defined; keypad names such as `KP_0` and `KP_Enter` preserve their physical key codes. Background pointer actions may change that keyboard focus. Text injection sends Unicode characters individually and does not use the clipboard. LF, CR and CRLF send one Return; tabs send Tab. Applications apply their normal editing behavior, including indentation and focus navigation. Each `typeText` down/up event is followed by the configured text interval; `pressKey` keeps its own press and release timing, and dispatch completion does not confirm that the receiving application consumed every event. Cancellation can interrupt text or a gesture after partial input; it cannot undo received events. Very long text can exceed the host's operation deadline. Validate observed application state before deciding how to continue.
|
||||
|
||||
The Host associates directional key events with one `run_code` execution. On down, the helper retains the selected application, recipient PID and process start time, physical key code, and matching up event. A repeated down from the owner posts one key-down with `keyboardEventAutorepeat` set and reuses the existing record. The event is posted to the recipient PID. An AppKit view can receive the initial down and final up while filtering the marked repeat. Repeats, explicit up and cleanup verify the original process identity. A conflicting down or up reports the key and whether another session or another run holds it. On return, error, or user pause, the Host waits for any in-flight native request and asks the helper to release only that execution’s remaining keys. Disconnect cleanup also releases keys owned by the closed connection.
|
||||
|
||||
The selected helper application needs macOS Accessibility permission for AX state and input, and Screen Recording permission for screenshots. Capability preflight and permission errors do not grant permissions. Capture refuses a window that moves or resizes while the image is being read. Unassociated system panels and accessory-only applications are not selectable.
|
||||
|
||||
@@ -157,7 +159,7 @@ For text pacing comparisons, use `native/tests/text_input_benchmark.py` with `--
|
||||
|
||||
For an independently isolated Electron/Chromium check, run `python3 packages/experimental/cua-macos/native/tests/electron_e2e.py --electron /absolute/path/to/Electron.app/Contents/MacOS/Electron`. Omitting `--electron` uses an already installed desktop workspace runtime. The check never downloads Electron. Its renderer runs with Node integration disabled, context isolation and Chromium sandboxing enabled, private userData/sessionData/log directories, and a local fixture page. Renderer click/input events write independent observations through the main process; all test actions still use the native helper. AX polling waits for actual controls after Chromium accessibility activation. Native and Electron subprocess groups are owned and stopped before temporary files are removed. Optional artifacts include a fixture-only screenshot and private request transcript.
|
||||
|
||||
Run `python3 packages/experimental/cua-macos/native/tests/capabilities_e2e.py --helper /absolute/helper --artifacts /private/evidence` to verify a stopped application's discovery and launch, Unicode selection, multiline plain/HTML paste, middle/modifier gestures and keypad identities through an isolated AppKit receiver. Add `--click-only` to check click holds, AX routing and cancellation/timeout release in a background fixture without keyboard or clipboard input. It requires existing Accessibility/Screen Recording grants and records the receiver's actual state separately from dispatch results.
|
||||
Run `python3 packages/experimental/cua-macos/native/tests/capabilities_e2e.py --helper /absolute/helper --artifacts /private/evidence` to verify a stopped application's discovery and launch, Unicode selection, multiline plain/HTML paste, middle/modifier gestures, overlapping W and D key events, and keypad identities through an isolated AppKit receiver. Add `--keyboard-only` for a background receiver that checks a two-second W+D hold, independent D release, and W cleanup. Add `--click-only` to check click holds, AX routing and cancellation/timeout release in a background fixture without keyboard or clipboard input. It requires existing Accessibility/Screen Recording grants and records the receiver's actual state separately from dispatch results.
|
||||
|
||||
Run `python3 packages/experimental/cua-macos/native/tests/paste_e2e.py --artifacts /private/evidence` to verify that an owned AppKit app accepts a background Paste action without reading its data, reports `PASTE_NOT_CONSUMED`, then consumes the same paste in the foreground. The test requires an unlocked Mac and Accessibility permission, activates only its own two applications, and records received actions and text independently of helper replies.
|
||||
|
||||
|
||||
@@ -78,7 +78,7 @@ AppKit 在主事件循环上运行。阻塞式 socket 接收与握手使用后
|
||||
| `getState` | `{appHandle, windowId, focusedWindowId, snapshotId, tree, elements, windows, truncated}`。应用级观察的 `windowId` 为 null。元素包含不透明字符串 `elementId`、role、title、value、description、identifier、bounds、actions 和 enabled 状态。元素边界为 null 或屏幕 point 单位的 [x, y, width, height]，窗口输入使用局部 point。这些结构化元素诊断供原生检查保留，模型工具投影会移除它们。不可用的可选描述属性为 null，无法取得的动作列表为空；权限、通信和窗口发现失败仍会拒绝观察。 |
|
||||
| `listActions` | 一个元素由 provider 定义的动作名数组；元素可以不声明任何动作。 |
|
||||
| `show` | `{dispatched: boolean, windowId, foregroundChanged}` 记录恢复分派。未选中窗口时 `windowId` 为 null；无法取得采样观察时 `foregroundChanged` 为 null。 |
|
||||
| `invoke`, `performAction`, `mouseMove`, `click`, `setValue`, `selectText`, `typeText`, `paste`, `pressKey`, `scroll`, `drag` | `{dispatched: boolean}` 记录原生分派；`invoke` 在没有可用的默认动作时返回 false。`mouseMove` 还报告 `hoverRoute` 和 `systemPointerMoved`。需要读取新状态以核实预期的应用变化。 |
|
||||
| `invoke`, `performAction`, `mouseMove`, `click`, `setValue`, `selectText`, `typeText`, `paste`, `pressKey`, `sendKeyEvent`, `scroll`, `drag` | `{dispatched: boolean}` 记录原生分派；`invoke` 在没有可用的默认动作时返回 false。`mouseMove` 还报告 `hoverRoute` 和 `systemPointerMoved`。需要读取新状态以核实预期的应用变化。 |
|
||||
| `screenshot` | `{mimeType: "image/png", data, pixelWidth, pixelHeight, bounds, coordinateSpace: "window-points", windowId}`。Retina 显示器上 PNG 像素尺寸可能不同于窗口 point 尺寸。 |
|
||||
|
||||
每个面向应用的操作都要求使用同一 helper 的 `getApp` 返回的 `appHandle`。helper 固定 PID 和内核进程启动时间，进程退出或 PID 被复用后使句柄及其全部快照失效。再次调用 `getApp` 可取得替代进程，不会重新激活旧句柄。随机的 helper 命名空间防止已终止 helper 的句柄与替代实例重合。重复获取同一存活进程会共享句柄。启动后检查失败会使待启动句柄失效，并保留原错误；已经启动的进程不会被终止，launchApp 也不能复用失败的句柄。
|
||||
@@ -103,7 +103,9 @@ AppKit 在主事件循环上运行。阻塞式 socket 接收与握手使用后
|
||||
|
||||
截图和鼠标输入在精确窗口的 AX 边界查询前后读取 CG 边界，查询期间移动会返回 WINDOW_CHANGED。有效的 AX/CG 边界向外取整后必须一致；已知不一致会在移动、预备事件或按键发送前返回 WINDOW_CHANGED。helper 保留两个坐标来源，不用 AX 几何替换 CG 几何。缺失或无效的 AX 几何仍表示未知，AX 通信、权限和无效元素错误继续传递。无辅助功能权限的截图保留 CG/SCK 检查。声明的 AX 动作不经过鼠标预检，AX 动作失败也不改用鼠标输入。这些读取不会锁住窗口以阻止后续移动，几何检查不激活或抬起窗口。
|
||||
|
||||
`typeText` 和 `pressKey` 面向所选进程当前获得焦点的控件，不指定窗口。`pressKey` 在按住所列 `modifiers` 时按下并释放一个键，先向 PID 发送修饰键变化，再发送主键，并逆序释放修饰键，不使用全局 HID 队列。按键名由平台定义；KP_0 和 KP_Enter 等数字键盘名称保留物理键码。后台鼠标操作可能改变键盘焦点。文本逐个 Unicode 字符输入，不使用剪贴板。LF、CR 和 CRLF 发送一次 Return，制表符发送 Tab。应用保留缩进和焦点导航等正常编辑行为。`typeText` 的每次按下和抬起后等待配置的文本事件间隔；`pressKey` 使用自己的按下与释放节奏，分派完成不证明应用已消费全部事件。取消可能中断部分文本或手势，无法撤销已接收事件。很长的文本可能超过 Host 操作期限。继续输入前应核实应用状态。
|
||||
`typeText` 和 `pressKey` 面向所选进程当前获得焦点的控件，不指定窗口。`pressKey` 向目标 PID 依次发送修饰键和一个主键的按下事件，再逆序释放。操作中途取消也会逆序释放已按下的键。键盘事件直接发给目标进程，绕过全局 HID 队列。按键名由平台定义；KP_0 和 KP_Enter 等数字键盘名称保留物理键码。后台鼠标操作可能改变键盘焦点。文本逐个 Unicode 字符输入，不使用剪贴板。LF、CR 和 CRLF 发送一次 Return，制表符发送 Tab。应用保留缩进和焦点导航等正常编辑行为。`typeText` 的每次按下和抬起后等待配置的文本事件间隔；`pressKey` 使用自己的按下与释放节奏，分派完成不证明应用已消费全部事件。取消可能中断部分文本或手势，无法撤销已接收事件。很长的文本可能超过 Host 操作期限。继续输入前应核实应用状态。
|
||||
|
||||
宿主为每次 `run_code` 的 `sendKeyEvent` 附上执行标识。按下时，helper 保存已选应用、接收进程的 PID 与启动时间、物理键码及对应的松开事件。持有方再次按下普通键时，helper 复用原记录，向接收进程发送一次带 `keyboardEventAutorepeat` 标志的按下事件。AppKit 视图可能只收到首次按下和最终松开，过滤中间的重复事件。重复、显式松键和执行结束时都会核对原进程身份。其他执行尝试按下或松开同一个键时，错误会指出键名，并区分其他会话和同一连接内的其他执行。正常返回、异常退出或用户暂停时，宿主等待在途原生请求结束，再通知 helper 释放该执行持有的其余按键。连接断开时，helper 也会清理该连接持有的按键。
|
||||
|
||||
所选 helper 应用需要 macOS 辅助功能权限以读取 AX 和输入，需要屏幕录制权限以截图。能力预检和权限错误不授予权限。截图会拒绝在读取图片期间移动或缩放的窗口。无法确认归属的系统面板及仅后台辅助类型的应用不可选择。
|
||||
|
||||
@@ -157,7 +159,7 @@ Blender 键盘检查使用 `python3 packages/experimental/cua-macos/native/tests
|
||||
|
||||
隔离的 Electron/Chromium 检查使用 `python3 packages/experimental/cua-macos/native/tests/electron_e2e.py --electron /absolute/path/to/Electron.app/Contents/MacOS/Electron`。省略 `--electron` 时使用桌面工作区已安装的运行时，检查不下载 Electron。渲染器禁用 Node 集成，启用上下文隔离和 Chromium 沙箱，使用独立 userData/sessionData/log 目录及本地 fixture 页面。渲染器的点击/输入事件通过主进程写出独立观察，测试动作仍通过原生 helper 执行。AX 轮询等待 Chromium 辅助功能启用后的真实控件。原生和 Electron 子进程组由测试持有，在删除临时文件前停止。可选产物包含仅有 fixture 的截图和私有请求记录。
|
||||
|
||||
运行 `python3 packages/experimental/cua-macos/native/tests/capabilities_e2e.py --helper /absolute/helper --artifacts /private/evidence`，通过隔离的 AppKit 接收方验证未运行应用的发现和启动、Unicode 选区、多行纯文本/HTML 粘贴、中键/修饰键手势及数字键盘身份。需要已有辅助功能/屏幕录制权限，接收方实际状态与分派结果分开记录。 添加 `--click-only` 可在后台测试窗口中验证按住时长、AX 路由和取消、超时后的松开事件，不发送键盘或剪贴板输入。
|
||||
运行 `python3 packages/experimental/cua-macos/native/tests/capabilities_e2e.py --helper /absolute/helper --artifacts /private/evidence`，通过隔离的 AppKit 接收方验证未运行应用的发现和启动、Unicode 选区、多行纯文本/HTML 粘贴、带修饰键的中键手势、W 和 D 同时按住时的事件顺序及数字键盘键码。需要已有辅助功能/屏幕录制权限，接收方实际状态与分派结果分开记录。添加 `--keyboard-only` 可用后台窗口验证 W、D 同时按住两秒、单独释放 D 和收尾时释放 W。添加 `--click-only` 可在后台测试窗口中验证按住时长、AX 路由和取消、超时后的松开事件，不发送键盘或剪贴板输入。
|
||||
|
||||
|
||||
运行 `python3 packages/experimental/cua-macos/native/tests/paste_e2e.py --artifacts /private/evidence`，验证自有 AppKit 应用在后台接收 Paste 动作但不读取数据时返回 `PASTE_NOT_CONSUMED`，以及前台读取同一次粘贴后成功。测试要求 Mac 已解锁并已授予辅助功能权限，只激活自有的两个应用，并独立记录收到的动作和文本以核对 helper 回复。
|
||||
|
||||
@@ -81,13 +81,15 @@ actor AXWorker {
|
||||
private let systemPointerHover: Bool?
|
||||
private let sessionState: @Sendable () -> Bool?
|
||||
private var access: AXAccess?
|
||||
private let heldKeys: HeldKeys
|
||||
private var keyboard: KeyboardInput?
|
||||
private var pointer: WindowPointerInput?
|
||||
private var hovers: [String: AXHover] = [:]
|
||||
private var closed = false
|
||||
|
||||
init(cancellation: CancellationLatch, axTimeout: Float, textEventInterval: Double, maxClickHoldMs: Int, systemPointerHover: Bool? = nil,
|
||||
init(cancellation: CancellationLatch, axTimeout: Float, textEventInterval: Double, maxClickHoldMs: Int, systemPointerHover: Bool? = nil, heldKeys: HeldKeys = HeldKeys(),
|
||||
sessionState: @escaping @Sendable () -> Bool? = NativePlatform.sessionLocked) {
|
||||
self.heldKeys = heldKeys
|
||||
self.cancellation = cancellation
|
||||
self.axTimeout = axTimeout
|
||||
self.textEventInterval = textEventInterval
|
||||
@@ -118,7 +120,7 @@ actor AXWorker {
|
||||
if let keyboard, let pointer { return (keyboard, pointer) }
|
||||
let poster = PIDEventPoster()
|
||||
let latch = cancellation
|
||||
let key = KeyboardInput(poster: poster, textEventInterval: textEventInterval, checkCancellation: { try latch.check() })
|
||||
let key = KeyboardInput(post: poster.post, textEventInterval: textEventInterval, held: heldKeys, checkCancellation: { try latch.check() })
|
||||
let mouse = WindowPointerInput(poster: poster, checkCancellation: { try latch.check() })
|
||||
keyboard = key; pointer = mouse
|
||||
return (key, mouse)
|
||||
@@ -208,8 +210,18 @@ actor AXWorker {
|
||||
trace: ScreenshotTrace? = nil) async throws -> DriverOutcome {
|
||||
try cancellation.check()
|
||||
guard !closed else { throw NativeFailure(code: "NATIVE_CLOSED", message: "Helper is closing") }
|
||||
if method == "sendKeyEvent" {
|
||||
throw NativeFailure(code: "NOT_IMPLEMENTED", message: "sendKeyEvent is not implemented on current platform.")
|
||||
if method == "releaseHeldKeys" {
|
||||
try keyboard?.releaseHeldKeys(run: params.string("runId"))
|
||||
return .result(Self.actionResult)
|
||||
}
|
||||
if method == "sendKeyEvent", try params.string("direction") == "up" {
|
||||
let handle = try params.string("appHandle")
|
||||
let dispatched = try inputs().0.keyUp(params.string("key"), app: handle, run: params.string("runId")) {
|
||||
try NativePlatform.requireUnlocked(observeSession())
|
||||
try NativePlatform.requireAccessibility()
|
||||
return try ax().keyboardRecipient(validateIdentity(handle), catalog: catalog)
|
||||
}
|
||||
return .result(.object(["dispatched": .bool(dispatched)]))
|
||||
}
|
||||
let locked = observeSession()
|
||||
if method == "getCapabilities" {
|
||||
@@ -331,6 +343,10 @@ actor AXWorker {
|
||||
maxBytes: try params.integer("maxClipboardBytes", in: 1024...Int.max),
|
||||
readTimeout: .milliseconds(try params.integer("pasteReadTimeoutMs", in: 1...2147483647)),
|
||||
maxElements: try params.integer("maxElements", in: 1...10000), maxDepth: try params.integer("maxDepth", in: 1...100)))
|
||||
case "sendKeyEvent":
|
||||
try inputs().0.keyDown(params.string("key"), app: handle, run: params.string("runId")) {
|
||||
try ax().keyboardRecipient(identity, catalog: catalog)
|
||||
}
|
||||
case "typeText", "pressKey":
|
||||
let recipient = try ax().keyboardRecipient(identity, catalog: catalog)
|
||||
try NativePlatform.verify(recipient)
|
||||
@@ -496,5 +512,10 @@ actor AXWorker {
|
||||
static func mouseMoveResult(route: String, pointerMoved: Bool) -> JSONValue {
|
||||
.object(["dispatched": .bool(true), "hoverRoute": .string(route), "systemPointerMoved": .bool(pointerMoved)])
|
||||
}
|
||||
func shutdown() { closed = true; snapshots.removeAll(); observedWindowBounds.removeAll(); targets.removeAll(); hovers.removeAll(); access = nil; keyboard = nil; pointer = nil }
|
||||
func shutdown() {
|
||||
do { try keyboard?.releaseHeldKeys() }
|
||||
catch { fputs("Could not release held keys on disconnect: \(error)\n", stderr) }
|
||||
closed = true; snapshots.removeAll(); observedWindowBounds.removeAll(); targets.removeAll(); hovers.removeAll()
|
||||
access = nil; keyboard = nil; pointer = nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
// All connections to the shared helper deliver to the same desktop processes.
|
||||
import CoreGraphics
|
||||
import Foundation
|
||||
|
||||
/// The lock covers both the held-key records and their corresponding PID-targeted event posts.
|
||||
/// Shutdown may release a connection's keys outside the ordinary request scheduler.
|
||||
final class HeldKeys: @unchecked Sendable {
|
||||
struct Owner: Equatable {
|
||||
let connection: UUID
|
||||
let run: String
|
||||
let app: String
|
||||
}
|
||||
|
||||
private struct Press {
|
||||
let owner: Owner
|
||||
let recipient: ProcessIdentity
|
||||
let code: CGKeyCode
|
||||
let flag: CGEventFlags
|
||||
let up: CGEvent
|
||||
}
|
||||
|
||||
private let lock = NSLock()
|
||||
private var pressed: [Press] = []
|
||||
|
||||
func snapshot(pid: Int32) -> [CGKeyCode: CGEventFlags] {
|
||||
lock.withLock { Dictionary(uniqueKeysWithValues: pressed.filter { $0.recipient.pid == pid }.map { ($0.code, $0.flag) }) }
|
||||
}
|
||||
|
||||
private func conflict(_ name: String, holder: Owner, caller: Owner) -> NativeFailure {
|
||||
let source: String
|
||||
if holder.connection != caller.connection { source = "another session" }
|
||||
else if holder.run != caller.run { source = "another run_code execution" }
|
||||
else { source = "a different application handle in this run_code" }
|
||||
return RequestParameters.invalid("Key '\(name)' is held for this target by \(source). No event was sent. Wait for the holder to release it or for its run_code to end.")
|
||||
}
|
||||
|
||||
/// Repeats use the saved recipient even when the selected application's focused panel has changed.
|
||||
func repeatIfHeld(owner: Owner, name: String, code: CGKeyCode, down: CGEvent, post: (CGEvent, Int32) -> Void) throws -> Bool {
|
||||
try lock.withLock {
|
||||
guard let key = pressed.first(where: { $0.owner == owner && $0.code == code }) else { return false }
|
||||
guard key.flag.isEmpty, down.type == .keyDown else {
|
||||
throw RequestParameters.invalid("Modifier '\(name)' is already held. Send 'up' to release it.")
|
||||
}
|
||||
try NativePlatform.verify(key.recipient)
|
||||
down.flags = flags(for: key.recipient)
|
||||
down.setIntegerValueField(.keyboardEventAutorepeat, value: 1)
|
||||
post(down, key.recipient.pid)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func down(owner: Owner, recipient: ProcessIdentity, name: String, code: CGKeyCode, flag: CGEventFlags,
|
||||
down: CGEvent, up: CGEvent, post: (CGEvent, Int32) -> Void) throws {
|
||||
try lock.withLock {
|
||||
if let key = pressed.first(where: { $0.code == code && $0.recipient.pid == recipient.pid }) {
|
||||
throw conflict(name, holder: key.owner, caller: owner)
|
||||
}
|
||||
pressed.append(Press(owner: owner, recipient: recipient, code: code, flag: flag, up: up))
|
||||
down.flags = flags(for: recipient)
|
||||
post(down, recipient.pid)
|
||||
}
|
||||
}
|
||||
|
||||
private func flags(for recipient: ProcessIdentity) -> CGEventFlags {
|
||||
pressed.reduce(into: CGEventFlags()) { if $1.recipient == recipient { $0.formUnion($1.flag) } }
|
||||
}
|
||||
|
||||
/// An up always targets the process selected at down. Never post to a recycled PID.
|
||||
private func release(at index: Int, post: (CGEvent, Int32) -> Void) throws -> Bool {
|
||||
let key = pressed[index]
|
||||
do { try NativePlatform.verify(key.recipient) }
|
||||
catch let error as NativeFailure where error.code == "STALE_APP" || error.code == "APP_UNAVAILABLE" {
|
||||
pressed.remove(at: index)
|
||||
return false
|
||||
}
|
||||
pressed.remove(at: index)
|
||||
key.up.flags = flags(for: key.recipient)
|
||||
post(key.up, key.recipient.pid)
|
||||
return true
|
||||
}
|
||||
|
||||
func up(owner: Owner, name: String, code: CGKeyCode, recipientIfUnheld: () throws -> ProcessIdentity,
|
||||
post: (CGEvent, Int32) -> Void) throws -> Bool {
|
||||
let dispatched = try lock.withLock { () throws -> Bool? in
|
||||
guard let index = pressed.firstIndex(where: { $0.owner == owner && $0.code == code }) else { return nil }
|
||||
return try release(at: index, post: post)
|
||||
}
|
||||
if let dispatched { return dispatched }
|
||||
let recipient = try recipientIfUnheld()
|
||||
throw lock.withLock {
|
||||
if let key = pressed.first(where: { $0.code == code && $0.recipient == recipient }) {
|
||||
return conflict(name, holder: key.owner, caller: owner)
|
||||
}
|
||||
return RequestParameters.invalid("Key '\(name)' is not held for this target. Send 'down' in this run_code before 'up'.")
|
||||
}
|
||||
}
|
||||
|
||||
func release(connection: UUID, run: String?, post: (CGEvent, Int32) -> Void) throws {
|
||||
try lock.withLock {
|
||||
defer {
|
||||
// A closed connection cannot retry an unverifiable recipient.
|
||||
if run == nil { pressed.removeAll { $0.owner.connection == connection } }
|
||||
}
|
||||
var firstError: Error?
|
||||
for index in pressed.indices.reversed() {
|
||||
let owner = pressed[index].owner
|
||||
guard owner.connection == connection, run == nil || owner.run == run else { continue }
|
||||
do { _ = try release(at: index, post: post) }
|
||||
catch { firstError = firstError ?? error }
|
||||
}
|
||||
if let firstError { throw firstError }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,76 +4,87 @@ import Foundation
|
||||
|
||||
/// Balanced PID-targeted chords and scalar-based Unicode input on one serial worker.
|
||||
final class KeyboardInput {
|
||||
private struct Modifier {
|
||||
private struct Key {
|
||||
let keyCode: CGKeyCode
|
||||
let flag: CGEventFlags
|
||||
}
|
||||
|
||||
private struct ModifierEvents {
|
||||
let down: CGEvent
|
||||
let up: CGEvent
|
||||
}
|
||||
|
||||
private let poster: PIDEventPoster
|
||||
private let connection = UUID()
|
||||
private let held: HeldKeys
|
||||
private let post: (CGEvent, Int32) -> Void
|
||||
private let textEventInterval: Double
|
||||
private let checkCancellation: @Sendable () throws -> Void
|
||||
|
||||
init(poster: PIDEventPoster, textEventInterval: Double, checkCancellation: @escaping @Sendable () throws -> Void = {}) {
|
||||
self.poster = poster
|
||||
init(post: @escaping (CGEvent, Int32) -> Void, textEventInterval: Double, held: HeldKeys = HeldKeys(), checkCancellation: @escaping @Sendable () throws -> Void = {}) {
|
||||
self.held = held
|
||||
self.post = post
|
||||
self.textEventInterval = textEventInterval
|
||||
self.checkCancellation = checkCancellation
|
||||
}
|
||||
|
||||
/// Construct the entire chord before posting; cleanup releases only keys pressed by this call.
|
||||
func pressKey(_ key: String, modifiers: [String], to pid: Int32) throws {
|
||||
try checkCancellation()
|
||||
var names = modifiers
|
||||
let isPlus = key == "+" || key.lowercased() == "plus"
|
||||
if isPlus { names.append("shift") }
|
||||
let code = try Self.keyCode(isPlus ? "=" : key)
|
||||
var selected: [Modifier] = []
|
||||
for name in names {
|
||||
let modifier = try Self.modifier(name)
|
||||
if !selected.contains(where: { $0.keyCode == modifier.keyCode }) {
|
||||
selected.append(modifier)
|
||||
}
|
||||
}
|
||||
func keyDown(_ name: String, app: String, run: String, recipient: () throws -> ProcessIdentity) throws {
|
||||
let owner = HeldKeys.Owner(connection: connection, run: run, app: app)
|
||||
let code = try Self.keyCode(name)
|
||||
let source = try Self.source()
|
||||
var flags: CGEventFlags = []
|
||||
var modifierEvents: [ModifierEvents] = []
|
||||
for modifier in selected {
|
||||
let down = try Self.keyEvent(source: source, code: code, down: true, flags: [])
|
||||
try checkCancellation()
|
||||
if try held.repeatIfHeld(owner: owner, name: name, code: code, down: down, post: post) { return }
|
||||
let target = try recipient()
|
||||
let up = try Self.keyEvent(source: source, code: code, down: false, flags: [])
|
||||
try checkCancellation()
|
||||
try NativePlatform.verify(target)
|
||||
try held.down(owner: owner, recipient: target, name: name, code: code, flag: Self.namedModifier(name)?.flag ?? [],
|
||||
down: down, up: up, post: post)
|
||||
}
|
||||
|
||||
func keyUp(_ name: String, app: String, run: String, recipientIfUnheld: () throws -> ProcessIdentity) throws -> Bool {
|
||||
try held.up(owner: .init(connection: connection, run: run, app: app), name: name, code: Self.keyCode(name),
|
||||
recipientIfUnheld: recipientIfUnheld, post: post)
|
||||
}
|
||||
|
||||
func releaseHeldKeys(run: String? = nil) throws {
|
||||
try held.release(connection: connection, run: run, post: post)
|
||||
}
|
||||
|
||||
/// Prepare every event before dispatch. On cancellation, release only the keys this call pressed.
|
||||
func pressKey(_ name: String, modifiers: [String], to pid: Int32) throws {
|
||||
try checkCancellation()
|
||||
var chord = try modifiers.map(Self.modifier)
|
||||
let isPlus = name == "+" || name.lowercased() == "plus"
|
||||
if isPlus { chord.append(try Self.modifier("shift")) }
|
||||
let code = try Self.keyCode(isPlus ? "=" : name)
|
||||
chord.append(Key(keyCode: code, flag: Self.namedModifier(name)?.flag ?? []))
|
||||
let alreadyHeld = held.snapshot(pid: pid)
|
||||
guard alreadyHeld[code] == nil else {
|
||||
throw RequestParameters.invalid("Key '\(name)' is already held for this target. Use pressKey after the run_code that holds it releases it.")
|
||||
}
|
||||
var selected = Set(alreadyHeld.keys)
|
||||
var flags = alreadyHeld.values.reduce(CGEventFlags()) { $0.union($1) }
|
||||
var events: [(down: CGEvent, up: CGEvent)] = []
|
||||
let source = try Self.source()
|
||||
for key in chord {
|
||||
guard selected.insert(key.keyCode).inserted else { continue }
|
||||
let releasedFlags = flags
|
||||
flags.insert(modifier.flag)
|
||||
modifierEvents.append(ModifierEvents(
|
||||
down: try Self.keyEvent(source: source, code: modifier.keyCode, down: true, flags: flags),
|
||||
up: try Self.keyEvent(source: source, code: modifier.keyCode, down: false, flags: releasedFlags)
|
||||
flags.formUnion(key.flag)
|
||||
events.append((
|
||||
down: try Self.keyEvent(source: source, code: key.keyCode, down: true, flags: flags),
|
||||
up: try Self.keyEvent(source: source, code: key.keyCode, down: false, flags: releasedFlags)
|
||||
))
|
||||
}
|
||||
let down = try Self.keyEvent(source: source, code: code, down: true, flags: flags)
|
||||
let up = try Self.keyEvent(source: source, code: code, down: false, flags: flags)
|
||||
var heldModifiers = 0
|
||||
var keyHeld = false
|
||||
var pressed = 0
|
||||
do {
|
||||
defer {
|
||||
if keyHeld { poster.post(up, to: pid) }
|
||||
for index in (0..<heldModifiers).reversed() {
|
||||
poster.post(modifierEvents[index].up, to: pid)
|
||||
Thread.sleep(forTimeInterval: 0.008)
|
||||
for index in (0..<pressed).reversed() {
|
||||
post(events[index].up, pid)
|
||||
if index > 0 { Thread.sleep(forTimeInterval: 0.008) }
|
||||
}
|
||||
}
|
||||
for events in modifierEvents {
|
||||
for event in events {
|
||||
try checkCancellation()
|
||||
poster.post(events.down, to: pid)
|
||||
heldModifiers += 1
|
||||
post(event.down, pid)
|
||||
pressed += 1
|
||||
Thread.sleep(forTimeInterval: 0.008)
|
||||
}
|
||||
try checkCancellation()
|
||||
poster.post(down, to: pid)
|
||||
keyHeld = true
|
||||
Thread.sleep(forTimeInterval: 0.008)
|
||||
try checkCancellation()
|
||||
poster.post(up, to: pid)
|
||||
keyHeld = false
|
||||
}
|
||||
try checkCancellation()
|
||||
}
|
||||
@@ -82,6 +93,8 @@ final class KeyboardInput {
|
||||
func typeText(_ text: String, to pid: Int32) throws {
|
||||
try checkCancellation()
|
||||
let source = try Self.source()
|
||||
let alreadyHeld = held.snapshot(pid: pid)
|
||||
let flags = alreadyHeld.values.reduce(CGEventFlags()) { $0.union($1) }
|
||||
for scalar in text.replacingOccurrences(of: "\r\n", with: "\n").unicodeScalars {
|
||||
try checkCancellation()
|
||||
let code: CGKeyCode
|
||||
@@ -90,8 +103,11 @@ final class KeyboardInput {
|
||||
case 9: code = try Self.keyCode("Tab")
|
||||
default: code = 0
|
||||
}
|
||||
let down = try Self.keyEvent(source: source, code: code, down: true, flags: [])
|
||||
let up = try Self.keyEvent(source: source, code: code, down: false, flags: [])
|
||||
guard alreadyHeld[code] == nil else {
|
||||
throw RequestParameters.invalid("Release the held physical key before typing text with its key code")
|
||||
}
|
||||
let down = try Self.keyEvent(source: source, code: code, down: true, flags: flags)
|
||||
let up = try Self.keyEvent(source: source, code: code, down: false, flags: flags)
|
||||
if code == 0 {
|
||||
let units = Array(String(scalar).utf16)
|
||||
units.withUnsafeBufferPointer { buffer in
|
||||
@@ -104,16 +120,16 @@ final class KeyboardInput {
|
||||
do {
|
||||
defer {
|
||||
if keyHeld {
|
||||
poster.post(up, to: pid)
|
||||
post(up, pid)
|
||||
Thread.sleep(forTimeInterval: textEventInterval)
|
||||
}
|
||||
}
|
||||
try checkCancellation()
|
||||
poster.post(down, to: pid)
|
||||
post(down, pid)
|
||||
keyHeld = true
|
||||
Thread.sleep(forTimeInterval: textEventInterval)
|
||||
try checkCancellation()
|
||||
poster.post(up, to: pid)
|
||||
post(up, pid)
|
||||
keyHeld = false
|
||||
Thread.sleep(forTimeInterval: textEventInterval)
|
||||
}
|
||||
@@ -139,22 +155,30 @@ final class KeyboardInput {
|
||||
return event
|
||||
}
|
||||
|
||||
private static func modifier(_ name: String) throws -> Modifier {
|
||||
private static func namedModifier(_ name: String) -> Key? {
|
||||
switch name.lowercased() {
|
||||
case "command", "cmd": return Modifier(keyCode: 55, flag: .maskCommand)
|
||||
case "shift": return Modifier(keyCode: 56, flag: .maskShift)
|
||||
case "option", "alt": return Modifier(keyCode: 58, flag: .maskAlternate)
|
||||
case "control", "ctrl": return Modifier(keyCode: 59, flag: .maskControl)
|
||||
case "fn": return Modifier(keyCode: 63, flag: .maskSecondaryFn)
|
||||
default: throw NativeFailure(code: "INVALID_ARGUMENT", message: "Unsupported key modifier: \(name)")
|
||||
case "command", "cmd": return Key(keyCode: 55, flag: .maskCommand)
|
||||
case "shift": return Key(keyCode: 56, flag: .maskShift)
|
||||
case "option", "alt": return Key(keyCode: 58, flag: .maskAlternate)
|
||||
case "control", "ctrl": return Key(keyCode: 59, flag: .maskControl)
|
||||
case "fn": return Key(keyCode: 63, flag: .maskSecondaryFn)
|
||||
default: return nil
|
||||
}
|
||||
}
|
||||
|
||||
private static func modifier(_ name: String) throws -> Key {
|
||||
guard let modifier = namedModifier(name) else {
|
||||
throw NativeFailure(code: "INVALID_ARGUMENT", message: "Unsupported key modifier: \(name)")
|
||||
}
|
||||
return modifier
|
||||
}
|
||||
|
||||
static func modifierFlags(_ names: [String]) throws -> CGEventFlags {
|
||||
try names.reduce(into: CGEventFlags()) { flags, name in flags.formUnion(try modifier(name).flag) }
|
||||
}
|
||||
|
||||
static func keyCode(_ key: String) throws -> CGKeyCode {
|
||||
if let modifier = namedModifier(key) { return modifier.keyCode }
|
||||
switch key.lowercased() {
|
||||
case "kp_0": return 82
|
||||
case "kp_1": return 83
|
||||
@@ -180,12 +204,7 @@ final class KeyboardInput {
|
||||
case "space": return 49
|
||||
case "delete", "backspace": return 51
|
||||
case "escape", "esc": return 53
|
||||
case "command", "cmd": return 55
|
||||
case "shift": return 56
|
||||
case "capslock": return 57
|
||||
case "option", "alt": return 58
|
||||
case "control", "ctrl": return 59
|
||||
case "fn": return 63
|
||||
case "home": return 115
|
||||
case "pageup": return 116
|
||||
case "del", "forward_delete": return 117
|
||||
@@ -254,7 +273,7 @@ final class KeyboardInput {
|
||||
case "m": return 46
|
||||
case ".": return 47
|
||||
case "`": return 50
|
||||
default: throw NativeFailure(code: "NATIVE_ERROR", message: "Unknown key name: \(key)")
|
||||
default: throw RequestParameters.invalid("Unknown key name: \(key)")
|
||||
}
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -20,11 +20,12 @@ enum NativeProtocol {
|
||||
"performAction": ["action", "appHandle", "argument", "elementId", "snapshotId", "windowId"],
|
||||
"permissionStatus": [],
|
||||
"pressKey": ["appHandle", "key", "modifiers", "windowId"],
|
||||
"releaseHeldKeys": ["runId"],
|
||||
"requestPermission": ["backLabel", "finderLabel", "instruction", "permission", "pollIntervalMs", "settingsMessage", "unavailableMessage"],
|
||||
"screenshot": ["appHandle", "windowId"],
|
||||
"scroll": ["appHandle", "deltaLinesX", "deltaLinesY", "snapshotId", "windowId", "x", "y"],
|
||||
"selectText": ["appHandle", "elementId", "prefix", "selectionType", "snapshotId", "suffix", "text", "windowId"],
|
||||
"sendKeyEvent": ["appHandle", "direction", "key", "windowId"],
|
||||
"sendKeyEvent": ["appHandle", "direction", "key", "runId", "windowId"],
|
||||
"setValue": ["appHandle", "elementId", "snapshotId", "value", "windowId"],
|
||||
"show": ["appHandle", "windowId"],
|
||||
"typeText": ["appHandle", "text", "windowId"],
|
||||
|
||||
@@ -7,7 +7,7 @@ struct RequestParameters: Sendable {
|
||||
init(_ request: NativeRequest) throws {
|
||||
guard let values = request.params.object else { throw Self.invalid("params must be an object") }
|
||||
guard let allowed = Self.fields[request.method] else { throw NativeFailure(code: "UNSUPPORTED", message: "Unsupported native operation: \(request.method)") }
|
||||
if ["typeText", "pressKey", "paste"].contains(request.method), values["windowId"] != nil {
|
||||
if ["typeText", "pressKey", "sendKeyEvent", "paste"].contains(request.method), values["windowId"] != nil {
|
||||
throw Self.invalid("macOS keyboard input follows application focus; use the App object")
|
||||
}
|
||||
guard Set(values.keys).isSubset(of: allowed) else { throw Self.invalid("Unknown operation parameter") }
|
||||
|
||||
@@ -43,7 +43,7 @@ import Darwin
|
||||
self.onFatal = onFatal
|
||||
worker = AXWorker(cancellation: cancellation, axTimeout: configuration.axTimeoutSeconds,
|
||||
textEventInterval: configuration.textEventIntervalSeconds, maxClickHoldMs: configuration.maxClickHoldMs,
|
||||
systemPointerHover: configuration.systemPointerHover)
|
||||
systemPointerHover: configuration.systemPointerHover, heldKeys: scheduler.heldKeys)
|
||||
input = JSONLInput(limit: configuration.maxRequestBytes, cancellation: cancellation, channel: channel)
|
||||
self.writer = writer
|
||||
}
|
||||
@@ -166,7 +166,7 @@ import Darwin
|
||||
let params = try RequestParameters(request)
|
||||
try cancellation.check()
|
||||
if request.method == "screenshot" { try capture.requireAvailable() }
|
||||
let requiresCatalog = ["getApp", "listApps", "findApps", "typeText", "pressKey", "click"].contains(request.method)
|
||||
let requiresCatalog = ["getApp", "listApps", "findApps", "typeText", "pressKey", "sendKeyEvent", "click"].contains(request.method)
|
||||
let catalog = requiresCatalog ? ApplicationCatalog.snapshot() : ApplicationCatalogSnapshot(applications: [], frontPID: nil)
|
||||
let result: JSONValue
|
||||
if request.method == "permissionStatus" {
|
||||
|
||||
@@ -7,6 +7,7 @@ import Dispatch
|
||||
let id: UUID
|
||||
let continuation: CheckedContinuation<Void, any Error>
|
||||
}
|
||||
let heldKeys = HeldKeys()
|
||||
private var owner: UUID?
|
||||
private var waiting: [Waiting] = []
|
||||
|
||||
|
||||
@@ -10,9 +10,12 @@ import { PermissionGuideParameters, PermissionSettingsParameters } from '../src/
|
||||
*/
|
||||
export function renderNativeProtocol(): { json: string; swift: string } {
|
||||
const { fields, enums, json } = nativeProtocol({
|
||||
sendKeyEvent: z.object({ runId: z.string().min(1) }),
|
||||
mouseMove: z.object({ allowSystemPointer: z.boolean().optional() }),
|
||||
requestPermission: PermissionGuideParameters, openPermissionSettings: PermissionSettingsParameters,
|
||||
})
|
||||
}, [z.object({
|
||||
method: z.literal('releaseHeldKeys'), params: z.object({ runId: z.string().min(1) }).strict(),
|
||||
}).strict()])
|
||||
const quoted = (values: Iterable<string>) => [...values].sort().map(value => JSON.stringify(value)).join(', ')
|
||||
const swift = [
|
||||
'// Generated by native/generate-protocol.ts from @deepseek-ai/dsh-cua/native-schema. Do not edit.',
|
||||
|
||||
@@ -1049,12 +1049,17 @@
|
||||
"down",
|
||||
"up"
|
||||
]
|
||||
},
|
||||
"runId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"appHandle",
|
||||
"key",
|
||||
"direction"
|
||||
"direction",
|
||||
"runId"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
@@ -1451,6 +1456,37 @@
|
||||
"params"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"method": {
|
||||
"type": "string",
|
||||
"const": "releaseHeldKeys"
|
||||
},
|
||||
"params": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"runId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"runId"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"method",
|
||||
"params"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -85,14 +85,20 @@ final class CapabilitiesFixture: NSObject, NSApplicationDelegate, NSWindowDelega
|
||||
guard let self else { return }
|
||||
var value: [String: Any] = ["type": event.type.rawValue, "timestamp": event.timestamp, "flags": event.modifierFlags.rawValue,
|
||||
"shift": event.modifierFlags.contains(.shift), "command": event.modifierFlags.contains(.command)]
|
||||
if [.keyDown, .keyUp].contains(event.type) { value["keyCode"] = event.keyCode; value["characters"] = event.characters ?? "" }
|
||||
else { value["button"] = event.buttonNumber }
|
||||
if [.keyDown, .keyUp].contains(event.type) {
|
||||
value["keyCode"] = event.keyCode
|
||||
value["characters"] = event.characters ?? ""
|
||||
value["repeat"] = event.isARepeat
|
||||
} else { value["button"] = event.buttonNumber }
|
||||
self.events.append(value)
|
||||
self.persist()
|
||||
}
|
||||
window.contentView!.addSubview(pad)
|
||||
if Bundle.main.object(forInfoDictionaryKey: "DSHFixtureBackground") as? Bool == true || CommandLine.arguments.contains("--background") {
|
||||
window.orderFront(nil)
|
||||
if Bundle.main.object(forInfoDictionaryKey: "DSHFixtureKeyboard") as? Bool == true {
|
||||
window.makeFirstResponder(pad)
|
||||
window.orderBack(nil)
|
||||
} else { window.orderFront(nil) }
|
||||
} else {
|
||||
window.makeKeyAndOrderFront(nil)
|
||||
window.makeFirstResponder(text)
|
||||
|
||||
@@ -123,11 +123,55 @@ def check_clicks(helper, window, point, state_file):
|
||||
return state
|
||||
|
||||
|
||||
def check_keyboard(call, owner, state_file, state):
|
||||
count = len(state["events"])
|
||||
call("pressKey", {**owner, "key": ["w", "d"]}, "INVALID_ARGUMENT")
|
||||
call("pressKey", {**owner, "key": "d", "modifiers": ["w"]}, "INVALID_ARGUMENT")
|
||||
call("pressKey", {**owner, "key": "d", "modifiers": ["shift"]})
|
||||
state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 2)
|
||||
shortcut = [(event["type"], event.get("keyCode"), event["shift"]) for event in state["events"][count:]]
|
||||
assert shortcut == [(10, 2, True), (11, 2, True)], shortcut
|
||||
count = len(state["events"])
|
||||
|
||||
def key(name, direction):
|
||||
call("sendKeyEvent", {**owner, "runId": "held-input", "key": name, "direction": direction})
|
||||
|
||||
key("w", "down")
|
||||
key("d", "down")
|
||||
state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 2)
|
||||
call("getState", {**owner, "maxElements": 2000, "maxDepth": 25})
|
||||
time.sleep(2)
|
||||
key("d", "up")
|
||||
state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 3)
|
||||
held = state["events"][count:]
|
||||
assert [(event["type"], event["keyCode"]) for event in held] == [(10, 13), (10, 2), (11, 2)], held
|
||||
assert held[2]["timestamp"] - held[1]["timestamp"] >= 2, held
|
||||
call("releaseHeldKeys", {"runId": "held-input"})
|
||||
state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 4)
|
||||
assert [(e["type"], e["keyCode"]) for e in state["events"][count:]] == [(10, 13), (10, 2), (11, 2), (11, 13)]
|
||||
|
||||
for explicit_up in (True, False):
|
||||
count = len(state["events"])
|
||||
key("w", "down")
|
||||
key("w", "down")
|
||||
if explicit_up:
|
||||
key("w", "up")
|
||||
call("releaseHeldKeys", {"runId": "held-input"})
|
||||
state = wait_for_state(state_file, lambda value: any(
|
||||
e["type"] == 11 and e.get("keyCode") == 13 for e in value["events"][count:]))
|
||||
view_events = [(e["type"], e["keyCode"], e["repeat"]) for e in state["events"][count:]]
|
||||
# AppKit can filter a synthetic repeat before NSView.keyDown receives it.
|
||||
down, repeat, up = (10, 13, False), (10, 13, True), (11, 13, False)
|
||||
assert view_events in ([down, up], [down, repeat, up]), view_events
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--helper", type=Path, required=True)
|
||||
parser.add_argument("--artifacts", type=Path)
|
||||
parser.add_argument("--click-only", action="store_true")
|
||||
only = parser.add_mutually_exclusive_group()
|
||||
only.add_argument("--click-only", action="store_true")
|
||||
only.add_argument("--keyboard-only", action="store_true")
|
||||
args = parser.parse_args()
|
||||
transcript = []
|
||||
helper = None
|
||||
@@ -141,7 +185,8 @@ def main():
|
||||
info = {"CFBundleName": "Capabilities Fixture", "CFBundleDisplayName": "Capabilities Fixture",
|
||||
"CFBundleIdentifier": "test.dsh.cua." + uuid.uuid4().hex,
|
||||
"CFBundleExecutable": "CapabilitiesFixture", "CFBundlePackageType": "APPL",
|
||||
"DSHFixtureStatePath": str(state_file), "DSHFixtureBackground": args.click_only}
|
||||
"DSHFixtureStatePath": str(state_file), "DSHFixtureBackground": args.click_only or args.keyboard_only,
|
||||
"DSHFixtureKeyboard": args.keyboard_only}
|
||||
(application / "Contents/Info.plist").write_bytes(plistlib.dumps(info))
|
||||
subprocess.run(["swiftc", str(ROOT / "tests/CapabilitiesFixture.swift"), "-o", str(executable)], check=True)
|
||||
try:
|
||||
@@ -158,6 +203,10 @@ def main():
|
||||
app = call("launchApp", owner)
|
||||
assert app["running"] is True, app
|
||||
pid = wait_for_state(state_file, lambda value: value["windowId"] > 0)["pid"]
|
||||
if args.keyboard_only:
|
||||
check_keyboard(call, owner, state_file, json.loads(state_file.read_text()))
|
||||
print("PASS: Shift+D shortcut, two-second W+D hold, independent D up, explicit W repeat and one final up")
|
||||
return
|
||||
if not args.click_only:
|
||||
call("activate", owner)
|
||||
state = wait_for_state(state_file, lambda value: value["pid"] == pid and value["active"] and value["keyWindow"])
|
||||
@@ -188,6 +237,7 @@ def main():
|
||||
if not args.click_only:
|
||||
call("click", {**window, **point, "mouseButton": "middle", "modifiers": ["shift"]})
|
||||
state = wait_for_state(state_file, lambda value: any(event["type"] == 26 and event.get("button") == 2 and event["shift"] for event in value["events"]))
|
||||
check_keyboard(call, owner, state_file, state)
|
||||
call("drag", {**window, "fromX": point["x"], "fromY": point["y"], "toX": point["x"] + 80,
|
||||
"toY": point["y"] + 20, "durationMs": 200, "steps": 10, "mouseButton": "middle", "modifiers": ["shift"]})
|
||||
state = wait_for_state(state_file, lambda value: any(event["type"] == 27 and event.get("button") == 2 and event["shift"] for event in value["events"]))
|
||||
@@ -201,7 +251,7 @@ def main():
|
||||
(args.artifacts / "capabilities.png").write_bytes(base64.b64decode(shot["data"]))
|
||||
(args.artifacts / "state.json").write_text(json.dumps(state, ensure_ascii=False, indent=2))
|
||||
print("PASS: click holds and cancellation/timeout/disconnect release" if args.click_only else
|
||||
"PASS: stopped-app launch, Unicode selection, plain/HTML paste, middle/modifier drag, keypad identity and click holds")
|
||||
"PASS: stopped-app launch, Unicode selection, plain/HTML paste, middle/modifier drag, overlapping W+D events, keypad identity and click holds")
|
||||
finally:
|
||||
if pid is None and state_file.exists():
|
||||
pid = json.loads(state_file.read_text())["pid"]
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
// Observe the actual CoreGraphics events without sending keyboard input to the desktop.
|
||||
import CoreGraphics
|
||||
import Darwin
|
||||
import Testing
|
||||
@testable import CuaNative
|
||||
|
||||
@Test func shortcutHoldsCommandAndShiftAroundOneKey() throws {
|
||||
var events: [CGEvent] = []
|
||||
let input = KeyboardInput(post: { event, pid in
|
||||
#expect(pid == 42)
|
||||
events.append(event)
|
||||
}, textEventInterval: 0)
|
||||
try input.pressKey("s", modifiers: ["cmd", "shift"], to: 42)
|
||||
#expect(events.map { $0.getIntegerValueField(.keyboardEventKeycode) } == [55, 56, 1, 1, 56, 55])
|
||||
#expect(events.map(\.type) == [.flagsChanged, .flagsChanged, .keyDown, .keyUp, .flagsChanged, .flagsChanged])
|
||||
#expect(events.map { $0.flags.contains(.maskShift) } == [false, true, true, true, false, false])
|
||||
#expect(events.map { $0.flags.contains(.maskCommand) } == [true, true, true, true, true, false])
|
||||
}
|
||||
|
||||
@Test(arguments: [1, 2])
|
||||
func cancelledShortcutReleasesOnlyPressedKeys(_ cancelAfter: Int) {
|
||||
let cancellation = CancellationLatch()
|
||||
var events: [CGEvent] = []
|
||||
let input = KeyboardInput(post: { event, _ in
|
||||
events.append(event)
|
||||
if events.count == cancelAfter { cancellation.cancel() }
|
||||
}, textEventInterval: 0, checkCancellation: cancellation.check)
|
||||
#expect(throws: NativeFailure.self) { try input.pressKey("d", modifiers: ["shift"], to: 42) }
|
||||
let held = [Int64(56), 2].prefix(cancelAfter)
|
||||
#expect(events.map { $0.getIntegerValueField(.keyboardEventKeycode) } == Array(held) + held.reversed())
|
||||
let types: [CGEventType] = cancelAfter == 1 ? [.flagsChanged, .flagsChanged] : [.flagsChanged, .keyDown, .keyUp, .flagsChanged]
|
||||
#expect(events.map(\.type) == types)
|
||||
}
|
||||
|
||||
@Test(arguments: [
|
||||
("unknown", ["shift"], "Unknown key name: unknown"),
|
||||
("w", ["d"], "Unsupported key modifier: d"),
|
||||
])
|
||||
func invalidShortcutPostsNoInput(_ key: String, _ modifiers: [String], _ message: String) throws {
|
||||
let input = KeyboardInput(post: { _, _ in Issue.record("Invalid shortcut posted input") }, textEventInterval: 0)
|
||||
do {
|
||||
try input.pressKey(key, modifiers: modifiers, to: 42)
|
||||
Issue.record("Invalid shortcut was accepted")
|
||||
} catch let error as NativeFailure {
|
||||
#expect(error.code == "INVALID_ARGUMENT")
|
||||
#expect(error.message == message)
|
||||
}
|
||||
}
|
||||
|
||||
@Test(arguments: ["+", "plus"])
|
||||
func plusAlonePressesShiftEquals(_ alias: String) throws {
|
||||
var events: [CGEvent] = []
|
||||
let input = KeyboardInput(post: { event, _ in events.append(event) }, textEventInterval: 0)
|
||||
try input.pressKey(alias, modifiers: [], to: 42)
|
||||
#expect(events.map { $0.getIntegerValueField(.keyboardEventKeycode) } == [56, 24, 24, 56])
|
||||
#expect(events.map { $0.flags.contains(.maskShift) } == [true, true, true, false])
|
||||
}
|
||||
|
||||
@Test func standaloneShiftSetsItsFlagOnlyWhileHeld() throws {
|
||||
var events: [CGEvent] = []
|
||||
let input = KeyboardInput(post: { event, _ in events.append(event) }, textEventInterval: 0)
|
||||
try input.pressKey("shift", modifiers: [], to: 42)
|
||||
#expect(events.map(\.type) == [.flagsChanged, .flagsChanged])
|
||||
#expect(events.map { $0.flags.contains(.maskShift) } == [true, false])
|
||||
}
|
||||
|
||||
@Test func directionalReleaseUsesOriginalRecipientAndPreservesOtherRuns() throws {
|
||||
let target = try NativePlatform.identity(getpid())
|
||||
let held = HeldKeys()
|
||||
var events: [CGEvent] = []
|
||||
let input = KeyboardInput(post: { event, pid in
|
||||
#expect(pid == target.pid)
|
||||
events.append(event)
|
||||
}, textEventInterval: 0, held: held)
|
||||
try input.keyDown("shift", app: "editor", run: "one", recipient: { target })
|
||||
#expect(throws: NativeFailure.self) { try input.keyDown("shift", app: "editor", run: "one", recipient: { target }) }
|
||||
try input.keyDown("w", app: "editor", run: "two", recipient: { target })
|
||||
#expect(throws: NativeFailure.self) { try input.keyDown("W", app: "editor", run: "one", recipient: { target }) }
|
||||
#expect(throws: NativeFailure.self) { try input.keyUp("w", app: "other", run: "two", recipientIfUnheld: { target }) }
|
||||
#expect(throws: NativeFailure.self) { try input.pressKey("w", modifiers: [], to: target.pid) }
|
||||
try input.pressKey("d", modifiers: ["shift"], to: target.pid)
|
||||
try input.releaseHeldKeys(run: "one")
|
||||
#expect(try input.keyUp("w", app: "editor", run: "two", recipientIfUnheld: { target }))
|
||||
try input.releaseHeldKeys(run: "two")
|
||||
#expect(events.map { $0.getIntegerValueField(.keyboardEventKeycode) } == [56, 13, 2, 2, 56, 13])
|
||||
#expect(events.map { $0.flags.contains(.maskShift) } == [true, true, true, true, false, false])
|
||||
}
|
||||
|
||||
@Test(arguments: [true, false])
|
||||
func repeatedDownUsesOriginalRecipientAndOneRelease(_ explicitUp: Bool) throws {
|
||||
let target = try NativePlatform.identity(getpid())
|
||||
var events: [CGEvent] = []
|
||||
let input = KeyboardInput(post: { event, pid in
|
||||
#expect(pid == target.pid)
|
||||
events.append(event)
|
||||
}, textEventInterval: 0)
|
||||
func changedFocus() throws -> ProcessIdentity {
|
||||
throw RequestParameters.invalid("The repeat or release looked up a new recipient")
|
||||
}
|
||||
try input.keyDown("w", app: "editor", run: "one", recipient: { target })
|
||||
try input.keyDown("W", app: "editor", run: "one", recipient: changedFocus)
|
||||
if explicitUp { #expect(try input.keyUp("w", app: "editor", run: "one", recipientIfUnheld: changedFocus)) }
|
||||
try input.releaseHeldKeys(run: "one")
|
||||
#expect(events.map(\.type) == [.keyDown, .keyDown, .keyUp])
|
||||
#expect(events.map { $0.getIntegerValueField(.keyboardEventAutorepeat) } == [0, 1, 0])
|
||||
}
|
||||
|
||||
@Test func foreignSessionCannotRepeatOrReleaseTheOwnersKey() throws {
|
||||
let target = try NativePlatform.identity(getpid())
|
||||
let held = HeldKeys()
|
||||
var events: [CGEvent] = []
|
||||
let a = KeyboardInput(post: { event, _ in events.append(event) }, textEventInterval: 0, held: held)
|
||||
let b = KeyboardInput(post: { _, _ in Issue.record("Non-owner dispatched an event") }, textEventInterval: 0, held: held)
|
||||
try a.keyDown("w", app: "a", run: "same-run-id", recipient: { target })
|
||||
for direction in ["down", "up"] {
|
||||
do {
|
||||
if direction == "down" { try b.keyDown("W", app: "b", run: "same-run-id", recipient: { target }) }
|
||||
else { _ = try b.keyUp("W", app: "b", run: "same-run-id", recipientIfUnheld: { target }) }
|
||||
Issue.record("Non-owner was allowed to send an event")
|
||||
} catch let failure as NativeFailure {
|
||||
#expect(failure.code == "INVALID_ARGUMENT")
|
||||
#expect(failure.message.contains("Key 'W' is held for this target by another session"))
|
||||
#expect(failure.message.contains("Wait for the holder to release it"))
|
||||
}
|
||||
}
|
||||
try b.releaseHeldKeys()
|
||||
try a.keyDown("W", app: "a", run: "same-run-id", recipient: { target })
|
||||
try a.releaseHeldKeys()
|
||||
#expect(events.map(\.type) == [.keyDown, .keyDown, .keyUp])
|
||||
do {
|
||||
_ = try b.keyUp("W", app: "b", run: "same-run-id", recipientIfUnheld: { target })
|
||||
Issue.record("Unheld key was released")
|
||||
} catch let failure as NativeFailure {
|
||||
#expect(failure.code == "INVALID_ARGUMENT")
|
||||
#expect(failure.message.contains("Key 'W' is not held for this target"))
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { COMMON_GUIDANCE } from '@deepseek-ai/dsh-cua/guidance'
|
||||
export const PLATFORM_GUIDANCE = 'macOS: accessibility tree operations are available at app or window level. Simulated keyboard input is available at app level. Simulated mouse input is available at window level. For pointer input on a sheet, select its window ID with app.getWindow(id). For menu items outside a window, use app.invoke(id). findApps searches /Applications, /System/Applications, and ~/Applications. Use the Computer Use permission page to grant the helper application Accessibility and Screen Recording permissions in System Settings. Native window and AX bounds use macOS points, while delivered screenshots carry their own pixel dimensions. show does not switch Spaces, and Stage Manager can present reduced window geometry; explain any required foreground step before calling app.activate() or performing an advertised AXRaise action.'
|
||||
|
||||
const methodDetails = new Map<string, string>([
|
||||
['getAccessibilityTree', 'If the tree reports maxElements or maxDepth truncation, select a narrower window or ask the user to raise the named macOS provider setting and read again. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current.'],
|
||||
['mouseMove', 'Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without an AX route it returns HOVER_UNAVAILABLE before sending input. Prefer click on a visible control or performAction with an advertised action. For web content without an AX hover action, use browser tools. Only systemPointerHover: true permits moving the visible cursor. If this returns FOREGROUND_REQUIRED, activate the app and check that the point is unobscured.'],
|
||||
['click', 'click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying.'],
|
||||
])
|
||||
@@ -13,6 +14,8 @@ const methodDetails = new Map<string, string>([
|
||||
export const GUIDANCE = `${PLATFORM_GUIDANCE}
|
||||
|
||||
${COMMON_GUIDANCE.replace(/^- ([\w.]+)\(.*$/gm, (entry, method: string) => {
|
||||
if (method === 'pressKey') return `${entry}
|
||||
- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.`
|
||||
const detail = methodDetails.get(method)
|
||||
return detail ? `${entry} ${detail}` : entry
|
||||
})}`
|
||||
@@ -0,0 +1,42 @@
|
||||
/** Directional input belongs to the enclosing run_code, including while its worker calls other tools. */
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import type { Context } from '@deepseek-ai/cordis'
|
||||
import type { CuaBackend } from '@deepseek-ai/dsh-cua'
|
||||
import { NativeError, type NativeClient } from '@deepseek-ai/dsh-cua/native-client'
|
||||
import { RUN_CODE_NAME, type ToolExecutionToken } from '@deepseek-ai/dsh-tools'
|
||||
|
||||
/**
|
||||
* Keep native key ownership alive until its program returns, throws, or is interrupted.
|
||||
* @param ctx - Provider scope owning the tool execution listener.
|
||||
* @returns Handler for authorized directional keyboard requests.
|
||||
*/
|
||||
export function installHeldKeys(ctx: Context): NonNullable<CuaBackend['sendKeyEvent']> {
|
||||
const runs = new Map<ToolExecutionToken, { id: string; client?: NativeClient }>()
|
||||
ctx.on('tools/execute', async (execution, next) => {
|
||||
if (execution.name !== RUN_CODE_NAME) return next()
|
||||
const run: { id: string; client?: NativeClient } = { id: randomUUID() }
|
||||
runs.set(execution.token, run)
|
||||
let cleanup: Promise<unknown> | undefined
|
||||
const release = () => {
|
||||
runs.delete(execution.token)
|
||||
if (run.client) cleanup ??= run.client.requestCleanup('releaseHeldKeys', { runId: run.id })
|
||||
return cleanup
|
||||
}
|
||||
// Pause may arrive while an unrelated subtool is still settling.
|
||||
const onAbort = () => { void release()?.catch(() => undefined) }
|
||||
const signal = execution.signal
|
||||
signal.addEventListener('abort', onAbort, { once: true })
|
||||
try {
|
||||
return await next()
|
||||
} finally {
|
||||
signal.removeEventListener('abort', onAbort)
|
||||
await release()
|
||||
}
|
||||
})
|
||||
return (client, params, execution) => {
|
||||
const run = execution.parent && runs.get(execution.parent)
|
||||
if (!run) throw new NativeError('INVALID_ARGUMENT', 'Use sendKeyEvent inside an active run_code execution')
|
||||
run.client = client
|
||||
return client.request('sendKeyEvent', { ...params, runId: run.id }, execution.signal, execution.callId)
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import { CuaConfigFields } from '@deepseek-ai/dsh-cua/config'
|
||||
import { createPermissionParameters } from './permission-guide.ts'
|
||||
import { nativeHostArchitecture, prepareNativeExecutable, type SourceSigning } from './native-build.ts'
|
||||
import { createSharedNativeChannel } from './shared-native.ts'
|
||||
import { installHeldKeys } from './held-keys.ts'
|
||||
import { GUIDANCE } from './guidance.ts'
|
||||
export { GUIDANCE } from './guidance.ts'
|
||||
|
||||
@@ -17,7 +18,7 @@ const CAPABILITIES: CuaStartupCapabilities = {
|
||||
methods: [
|
||||
'activate', 'click', 'drag', 'findApps', 'getApp', 'getCapabilities', 'getState', 'invoke',
|
||||
'listActions', 'listApps', 'listWindows', 'mouseMove', 'paste', 'performAction', 'pressKey', 'screenshot', 'scroll',
|
||||
'selectText', 'setValue', 'show', 'typeText',
|
||||
'selectText', 'sendKeyEvent', 'setValue', 'show', 'typeText',
|
||||
],
|
||||
appLevelAccessibility: true,
|
||||
keyboard: ['app'],
|
||||
@@ -65,7 +66,7 @@ export const Config = Schema.object({
|
||||
*/
|
||||
export async function apply(ctx: Context, config: Config): Promise<void> {
|
||||
const createChannel: NativeChannelFactory = (owner, native) => createSharedNativeChannel(owner, native, config.systemPointerHover)
|
||||
const backend = { createChannel, guidance: GUIDANCE, capabilities: CAPABILITIES,
|
||||
const backend = { createChannel, sendKeyEvent: installHeldKeys(ctx), guidance: GUIDANCE, capabilities: CAPABILITIES,
|
||||
permissionParameters: createPermissionParameters(config.permissionPollIntervalMs) }
|
||||
if (config.helperPath !== undefined) {
|
||||
if (!isAbsolute(config.helperPath)) throw new Error('dsh-cua helperPath must be absolute')
|
||||
|
||||
@@ -27,6 +27,12 @@ it('states the macOS accessibility, keyboard, and mouse levels', () => {
|
||||
expect(PLATFORM_GUIDANCE).toContain('accessibility tree operations are available at app or window level')
|
||||
expect(PLATFORM_GUIDANCE).toContain('Simulated keyboard input is available at app level')
|
||||
expect(PLATFORM_GUIDANCE).toContain('Simulated mouse input is available at window level')
|
||||
const pressKey = GUIDANCE.split('\n').find(line => line.startsWith('- pressKey('))
|
||||
expect(GUIDANCE).toContain("sendKeyEvent(key: string, direction: 'down' | 'up')")
|
||||
expect(GUIDANCE).toContain('returns, throws, or the user pauses')
|
||||
expect(COMMON_GUIDANCE).not.toContain('sendKeyEvent(')
|
||||
expect(pressKey).toBe(COMMON_GUIDANCE.split('\n').find(line => line.startsWith('- pressKey(')))
|
||||
expect(GUIDANCE).toContain('W down, D down, D up, W up')
|
||||
const hover = GUIDANCE.split('\n').find(line => line.startsWith('- mouseMove('))
|
||||
for (const term of ['AXShowAlternateUI', 'AXShowDefaultUI', 'HOVER_UNAVAILABLE', 'systemPointerHover']) expect(hover).toContain(term)
|
||||
const click = GUIDANCE.split('\n').find(line => line.startsWith('- click('))
|
||||
@@ -135,5 +141,5 @@ it('documents facade initialization, emitted observations, and separate key argu
|
||||
})
|
||||
|
||||
it('omits stale names and unavailable platform operations', () => {
|
||||
for (const stale of ['getAXState', 'disableDiffing', 'performSecondaryAction', 'hover(', 'keyChord', 'sendKeyEvent', 'window.typeText', 'window.pressKey', 'elementIndex', 'verified: false', 'browser tabs', 'not implemented']) expect(GUIDANCE).not.toContain(stale)
|
||||
for (const stale of ['getAXState', 'disableDiffing', 'performSecondaryAction', 'hover(', 'keyChord', 'window.typeText', 'window.pressKey', 'elementIndex', 'verified: false', 'browser tabs', 'not implemented']) expect(GUIDANCE).not.toContain(stale)
|
||||
})
|
||||
@@ -21,6 +21,13 @@ describe('native protocol generation', () => {
|
||||
expect(validate({ id: 'invalid', method: 'setValue', params: target })).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects a key array at both model and native pressKey admission', () => {
|
||||
const request = { method: 'pressKey', params: { appHandle: 'app', key: ['w', 'd'] } }
|
||||
expect(Request.safeParse(request).success).toBe(false)
|
||||
const validate = new Ajv2020().compile(JSON.parse(renderNativeProtocol().json))
|
||||
expect(validate({ id: 'shortcut', ...request })).toBe(false)
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ method: 'getState', params: { appHandle: 'app', maxElements: 1 } },
|
||||
{ method: 'drag', params: { appHandle: 'app', windowId: 1, fromX: 1, fromY: 1, toX: 2, toY: 2, steps: 1 } },
|
||||
@@ -28,6 +35,7 @@ describe('native protocol generation', () => {
|
||||
{ method: 'paste', params: { appHandle: 'app', text: 'text', pasteReadTimeoutMs: 1000 } },
|
||||
{ method: 'mouseMove', params: { appHandle: 'app', windowId: 1, x: 0, y: 0, allowSystemPointer: true } },
|
||||
{ method: 'findApps', params: { query: 'Editor', maxAppMatches: 1 } },
|
||||
{ method: 'releaseHeldKeys', params: { runId: 'execution' } },
|
||||
{ method: 'launchApp', params: { appHandle: 'pending' } },
|
||||
{ method: 'applicationIcons', params: { paths: ['/Applications/Fixture.app'] } },
|
||||
{ method: 'permissionStatus', params: {} },
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/** Native helper replacement is synchronized with Agent and provider disposal. */
|
||||
import { executeToolFixture } from '@deepseek-ai/dsh-tool-testkit'
|
||||
import { executeModelToolFixture, executeToolFixture } from '@deepseek-ai/dsh-tool-testkit'
|
||||
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
@@ -28,6 +28,7 @@ import SandboxPolicy from '@deepseek-ai/dsh-sandbox-policy'
|
||||
import { LlmAdapter, ToolCallId, type LlmResolvedModelInfo, type StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||
import { SessionId } from '@deepseek-ai/dsh-session'
|
||||
import Subprocess from '@deepseek-ai/dsh-subprocess'
|
||||
import SubprocessLocal from '@deepseek-ai/dsh-subprocess-local'
|
||||
import { expect, it, onTestFinished, vi } from 'vitest'
|
||||
import sharp from 'sharp'
|
||||
import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
|
||||
@@ -36,11 +37,14 @@ import type { NativeClient } from '@deepseek-ai/dsh-cua/native-client'
|
||||
import type { NativeBuildSpec } from '../src/native-build.ts'
|
||||
|
||||
const native = vi.hoisted(() => ({
|
||||
connect: vi.fn<typeof import('../src/shared-native.ts')['createSharedNativeChannel']>(),
|
||||
prepare: vi.fn<(spec: NativeBuildSpec) => Promise<string>>(),
|
||||
architecture: vi.fn(() => 'arm64' as const),
|
||||
cleanup: vi.fn<NativeClient['requestCleanup']>(),
|
||||
request: vi.fn<NativeClient['request']>(), dispose: vi.fn(), instances: [] as Array<{ disposed: boolean }>,
|
||||
reportDiagnostic: undefined as ((message: string) => void) | undefined,
|
||||
}))
|
||||
vi.mock('../src/shared-native.ts', () => ({ createSharedNativeChannel: native.connect }))
|
||||
vi.mock('../src/native-build.ts', async importOriginal => ({
|
||||
...await importOriginal<typeof import('../src/native-build.ts')>(),
|
||||
nativeHostArchitecture: native.architecture,
|
||||
@@ -51,9 +55,11 @@ vi.mock('@deepseek-ai/dsh-cua/native-client', async importOriginal => ({
|
||||
NativeClient: class {
|
||||
disposed = false
|
||||
constructor(...args: ConstructorParameters<typeof NativeClient>) {
|
||||
args[2](args[0], args[1])
|
||||
native.reportDiagnostic = args[3]
|
||||
native.instances.push(this)
|
||||
}
|
||||
requestCleanup = native.cleanup
|
||||
request = native.request
|
||||
invalidate() { this.disposed = true }
|
||||
async dispose() { this.disposed = true; await native.dispose() }
|
||||
@@ -110,19 +116,26 @@ class ImageAdapter extends LlmAdapter {
|
||||
const imageData = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC'
|
||||
const imageMetadata = { pixelWidth: 1, pixelHeight: 1, bounds: { x: 2, y: 3, width: 1, height: 1 }, coordinateSpace: 'window-points', windowId: 7 }
|
||||
|
||||
async function setup(model?: { provider: string; model: string }) {
|
||||
async function setup(model?: { provider: string; model: string }, ptc = false) {
|
||||
native.connect.mockClear()
|
||||
native.cleanup.mockReset().mockResolvedValue({ dispatched: true })
|
||||
native.request.mockReset().mockResolvedValue([])
|
||||
native.dispose.mockReset()
|
||||
native.instances.length = 0
|
||||
native.reportDiagnostic = undefined
|
||||
const ctx = new Context()
|
||||
onTestFinished(() => ctx.fiber.dispose())
|
||||
await mountAgentLoopTestDependencies(ctx)
|
||||
await mountAgentLoopTestDependencies(ctx, ptc ? { workingDirectory: true, tools: { mode: 'ptc' } } : {})
|
||||
await ctx.plugin(ComputerUseRegistry)
|
||||
await ctx.plugin(Subprocess, [])
|
||||
if (ptc) {
|
||||
await ctx.plugin(SubprocessLocal)
|
||||
await ctx.plugin(Sandbox)
|
||||
await ctx.plugin(SandboxPolicy, { mode: 'danger-full-access' })
|
||||
await ctx.plugin(NodeRuntime)
|
||||
} else await ctx.plugin(Subprocess, [])
|
||||
const provider = await ctx.plugin(NativeProvider, config)
|
||||
await mountAgentLoopTestHarness(ctx)
|
||||
const handle = await ctx.agents.create({ sessionId: SessionId('native-lifecycle'), ...model ? { agentOptions: model } : {} })
|
||||
const handle = await ctx.agents.create({ sessionId: SessionId('native-lifecycle'), meta: { cwd: process.cwd() }, ...model ? { agentOptions: model } : {} })
|
||||
let index = 0
|
||||
const execute = (method = 'listApps', params: Record<string, unknown> = {}, signal = new AbortController().signal) => executeToolFixture(ctx, {
|
||||
agent: handle.agent, signal, callId: ToolCallId(`lifecycle-${++index}`), name: 'cua', arguments: { method, params },
|
||||
@@ -138,6 +151,85 @@ async function selectApplication(f: Awaited<ReturnType<typeof setup>>) {
|
||||
expect((await f.execute('getApp', { selector: 'Fixture' })).isError).toBe(false)
|
||||
}
|
||||
|
||||
it('requires an enclosing run_code before sending a directional key', async () => {
|
||||
const f = await setup()
|
||||
await selectApplication(f)
|
||||
native.request.mockClear()
|
||||
const result = await f.execute('sendKeyEvent', { appHandle: 'app-one', key: 'w', direction: 'down' })
|
||||
expect(result.isError).toBe(true)
|
||||
expect(JSON.stringify(result.content)).toContain('active run_code')
|
||||
expect(native.request).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('finishes a shortcut-only run_code without requesting held-key cleanup', async () => {
|
||||
const f = await setup(undefined, true)
|
||||
await selectApplication(f)
|
||||
native.request.mockClear().mockResolvedValue({ dispatched: true })
|
||||
const result = await executeModelToolFixture(f.ctx, { agent: f.handle.agent, callId: ToolCallId('shortcut-run'),
|
||||
name: 'run_code', signal: new AbortController().signal, arguments: {
|
||||
code: "return await tools.cua({ method: 'pressKey', params: { appHandle: 'app-one', key: 'a', modifiers: ['command'] } });",
|
||||
description: 'Press Command+A',
|
||||
} })
|
||||
expect(result.isError).toBe(false)
|
||||
expect(native.request.mock.calls.map(([method]) => method)).toEqual(['pressKey'])
|
||||
expect(native.cleanup).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each(['return', 'throw', 'pause', 'pause with cleanup error'])('cleans up only the held W when run_code ends by %s', async (ending) => {
|
||||
const f = await setup(undefined, true)
|
||||
await selectApplication(f)
|
||||
const held = new Set<string>()
|
||||
const events: string[] = []
|
||||
const controller = new AbortController()
|
||||
const runIds = new Set<unknown>()
|
||||
native.request.mockImplementation(async (method, params) => {
|
||||
expect(method).toBe('sendKeyEvent')
|
||||
runIds.add(params.runId)
|
||||
const { key, direction } = params
|
||||
if (typeof key !== 'string' || (direction !== 'down' && direction !== 'up')) throw new Error('Invalid directional event')
|
||||
if (direction === 'down') held.add(key)
|
||||
else held.delete(key)
|
||||
events.push(`${key}:${direction}`)
|
||||
return { dispatched: true }
|
||||
})
|
||||
native.cleanup.mockImplementation(async (method, params) => {
|
||||
expect(method).toBe('releaseHeldKeys')
|
||||
expect(runIds).toEqual(new Set([params.runId]))
|
||||
if (ending === 'pause with cleanup error') throw new Error('release acknowledgement failed')
|
||||
for (const key of held) events.push(`${key}:up`)
|
||||
held.clear()
|
||||
return { dispatched: true }
|
||||
})
|
||||
f.ctx.tools.register({
|
||||
name: 'check_held_keys', description: 'Observe the keys held by the test receiver.',
|
||||
parameters: { type: 'object', properties: {} },
|
||||
output: { schema: {}, render: () => [] },
|
||||
async execute() {
|
||||
const keys = [...held]
|
||||
if (ending.startsWith('pause') && keys.length === 1) {
|
||||
controller.abort()
|
||||
await setImmediate()
|
||||
expect(held.size).toBe(ending === 'pause' ? 0 : 1)
|
||||
}
|
||||
return keys
|
||||
},
|
||||
})
|
||||
const code = [
|
||||
"const key = (key, direction) => tools.cua({ method: 'sendKeyEvent', params: { appHandle: 'app-one', key, direction } });",
|
||||
"await key('w', 'down'); await key('d', 'down');",
|
||||
"if ((await tools.check_held_keys({})).join(',') !== 'w,d') throw Error('lost held keys');",
|
||||
"await key('d', 'up');",
|
||||
"if ((await tools.check_held_keys({})).join(',') !== 'w') throw Error('lost W');",
|
||||
ending === 'throw' ? "throw Error('program failed');" : "return 'done';",
|
||||
].join('\n')
|
||||
const result = await executeModelToolFixture(f.ctx, { agent: f.handle.agent, callId: ToolCallId('keyboard-run'),
|
||||
name: 'run_code', arguments: { code, description: 'Exercise held keys' }, signal: controller.signal })
|
||||
expect(events, JSON.stringify(result.content)).toEqual(['w:down', 'd:down', 'd:up', ...ending === 'pause with cleanup error' ? [] : ['w:up']])
|
||||
expect(native.cleanup).toHaveBeenCalledTimes(1)
|
||||
expect(result.isError).toBe(ending !== 'return')
|
||||
if (ending === 'throw') expect(JSON.stringify(result.content)).toContain('program failed')
|
||||
})
|
||||
|
||||
it('rejects a relative helper path before publishing computer-use operations', async () => {
|
||||
const ctx = new Context()
|
||||
onTestFinished(() => ctx.fiber.dispose())
|
||||
@@ -200,7 +292,7 @@ it('merges startup capabilities into runtime native fields', async () => {
|
||||
coordinateSpace: 'screenshot-pixels', methods: [
|
||||
'activate', 'click', 'drag', 'findApps', 'getApp', 'getCapabilities', 'getState', 'invoke',
|
||||
'listActions', 'listApps', 'listWindows', 'mouseMove', 'paste', 'performAction', 'pressKey', 'screenshot', 'scroll',
|
||||
'selectText', 'setValue', 'show', 'typeText',
|
||||
'selectText', 'sendKeyEvent', 'setValue', 'show', 'typeText',
|
||||
] })
|
||||
expect(native.request).toHaveBeenCalledExactlyOnceWith('getCapabilities', {}, expect.any(AbortSignal), ToolCallId('lifecycle-1'))
|
||||
})
|
||||
|
||||
@@ -22,7 +22,7 @@ Setup and discovery:
|
||||
Available at app or window level depending on the platform.
|
||||
|
||||
Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the full tree text; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the tree within the provider limits; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names.
|
||||
- listActions(elementId: string) -> string[] — list the actions an element advertises.
|
||||
- invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none.
|
||||
- performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument.
|
||||
|
||||
@@ -126,6 +126,9 @@ export interface CuaBackend {
|
||||
createChannel: NativeChannelFactory
|
||||
guidance: string
|
||||
capabilities: CuaStartupCapabilities
|
||||
/** Dispatch a directional key through the provider's execution-scoped input owner, after application authorization. */
|
||||
sendKeyEvent?: (client: NativeClient, params: Record<string, JsonValue>,
|
||||
execution: Readonly<ToolExecution>) => Promise<JsonValue>
|
||||
/**
|
||||
* Build native permission parameters for the selected action and Client language.
|
||||
* @param method Native permission action.
|
||||
@@ -211,6 +214,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend)
|
||||
const signal = execution.signal
|
||||
signal.throwIfAborted()
|
||||
const client = execution.nativeClient
|
||||
if (method === 'sendKeyEvent' && backend.sendKeyEvent) return backend.sendKeyEvent(client, params, execution)
|
||||
const nativeParams = method === 'findApps'
|
||||
? { ...params, maxAppMatches: config.maxAppMatches }
|
||||
: method === 'paste'
|
||||
|
||||
@@ -81,7 +81,12 @@ export class NativeClient {
|
||||
private diagnostics: string[] = []
|
||||
private diagnosticBytes = 0
|
||||
private discardedDiagnostics = false
|
||||
private active: { id: string; resolve: (value: JsonValue) => void; reject: (error: NativeError) => void } | undefined
|
||||
private active: {
|
||||
id: string
|
||||
outcome: Promise<JsonValue>
|
||||
resolve: (value: JsonValue) => void
|
||||
reject: (error: NativeError) => void
|
||||
} | undefined
|
||||
|
||||
/**
|
||||
* @param owner - Managed subprocess provider.
|
||||
@@ -133,7 +138,10 @@ export class NativeClient {
|
||||
throw new NativeError('NATIVE_REQUEST_LIMIT', 'Request exceeded the configured byte limit before dispatch')
|
||||
}
|
||||
const outcome = Promise.withResolvers<JsonValue>()
|
||||
this.active = { id, resolve: outcome.resolve, reject: (error) => { nativeFailure = error; outcome.reject(error) } }
|
||||
this.active = {
|
||||
id, outcome: outcome.promise, resolve: outcome.resolve,
|
||||
reject: (error) => { nativeFailure = error; outcome.reject(error) },
|
||||
}
|
||||
if (capture) this.diagnostic(screenshotPrefix + JSON.stringify({ requestId: id, stage: 'host.dispatch', elapsedMs: 0 }))
|
||||
const interrupt = (error: NativeError): void => {
|
||||
if (interruption || this.active?.id !== id) return
|
||||
@@ -193,6 +201,17 @@ export class NativeClient {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch cleanup after any active request acknowledges completion or cancellation.
|
||||
* @param method - Host-only helper cleanup operation.
|
||||
* @param params - Resources owned by the enclosing execution.
|
||||
* @returns The native cleanup acknowledgement, bounded by the connection's request deadline.
|
||||
*/
|
||||
async requestCleanup(method: string, params: Record<string, JsonValue>): Promise<JsonValue> {
|
||||
while (this.active) await this.active.outcome.catch(() => undefined)
|
||||
return this.request(method, params, new AbortController().signal)
|
||||
}
|
||||
|
||||
private screenshotDiagnostics(): string {
|
||||
const records = [...this.diagnostics]
|
||||
if (this.discardedDiagnostics) records.unshift('[Earlier screenshot diagnostic records were discarded from the bounded diagnostic buffer.]')
|
||||
|
||||
@@ -14,15 +14,19 @@ function variants(schema: Schema): Schema[] { return schema.anyOf ?? schema.oneO
|
||||
/**
|
||||
* Derive native request schemas and admission fields from the shared operation definitions.
|
||||
* @param platformFields Additional native protocol fields accepted by the provider.
|
||||
* @param privateOperations Additional provider-only request schemas.
|
||||
* @returns Native field and enum tables and the complete wire schema.
|
||||
*/
|
||||
export function nativeProtocol(platformFields: Partial<Record<z.infer<typeof Request>['method'] | 'requestPermission' | 'openPermissionSettings', z.ZodObject>> = {}): {
|
||||
export function nativeProtocol(
|
||||
platformFields: Partial<Record<z.infer<typeof Request>['method'] | 'requestPermission' | 'openPermissionSettings', z.ZodObject>> = {},
|
||||
privateOperations: readonly z.ZodObject[] = [],
|
||||
): {
|
||||
fields: Map<string, Set<string>>
|
||||
enums: Map<string, Map<string, Set<string>>>
|
||||
json: string
|
||||
} {
|
||||
const resources = new Map(Object.entries({ ...NativeResourceParameters, ...platformFields }))
|
||||
const schemas = [...Request.options, ...NativeOnlyOperations]
|
||||
const schemas = [...Request.options, ...NativeOnlyOperations, ...privateOperations]
|
||||
.map(schema => z.toJSONSchema(schema))
|
||||
const fields = new Map<string, Set<string>>()
|
||||
const enums = new Map<string, Map<string, Set<string>>>()
|
||||
|
||||
@@ -63,6 +63,18 @@ function fixture(options: {
|
||||
}
|
||||
|
||||
describe('native helper transport', () => {
|
||||
it('sends run cleanup after an interrupted native request has acknowledged its outcome', async () => {
|
||||
const f = fixture()
|
||||
const active = f.client.request('getState', {}, new AbortController().signal)
|
||||
const release = f.client.requestCleanup('releaseHeldKeys', { runId: 'program' })
|
||||
expect(f.requests).toHaveLength(1)
|
||||
f.reply({ id: f.id(), error: { code: 'NATIVE_CANCELLED', message: 'paused' } })
|
||||
await expect(active).rejects.toThrow('paused')
|
||||
expect(f.requests.map(frame => frame.method)).toEqual(['getState', 'releaseHeldKeys'])
|
||||
f.reply({ id: f.id(), result: { dispatched: true } })
|
||||
await expect(release).resolves.toEqual({ dispatched: true })
|
||||
})
|
||||
|
||||
it('passes transport limits and connection deadlines to the shared channel', () => {
|
||||
const f = fixture({ maxRequestBytes: 4096, maxResponseBytes: 8192, timeoutMs: 1234 })
|
||||
expect(f.connect).toHaveBeenCalledExactlyOnceWith(f.owner, f.config)
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
{"type":"session/title","data":{"title":"Inspect CUA capabilities, select the","messageSeqs":[8],"source":{"kind":"fallback"}}}
|
||||
{"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:4}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949731596,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949731597,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}}},{"type":"chunk","time":1793949731597,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949731597,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
|
||||
{"type":"tool/call","data":{"turn":1,"step":1,"callId":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}}
|
||||
{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-1"},"toolCallId":"cua-native-1","content":[{"type":"text","text":"{\"sessionLocked\":false,\"permissions\":{\"accessibility\":true,\"screenRecording\":true},\"platform\":\"macos\",\"methods\":[\"activate\",\"click\",\"drag\",\"findApps\",\"getApp\",\"getCapabilities\",\"getState\",\"invoke\",\"listActions\",\"listApps\",\"listWindows\",\"mouseMove\",\"paste\",\"performAction\",\"pressKey\",\"screenshot\",\"scroll\",\"selectText\",\"setValue\",\"show\",\"typeText\"],\"appLevelAccessibility\":true,\"keyboard\":[\"app\"],\"browser\":false,\"coordinateSpace\":\"screenshot-pixels\"}"}],"isError":false,"id":"{{message:5}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"}
|
||||
{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-1"},"toolCallId":"cua-native-1","content":[{"type":"text","text":"{\"sessionLocked\":false,\"permissions\":{\"accessibility\":true,\"screenRecording\":true},\"platform\":\"macos\",\"methods\":[\"activate\",\"click\",\"drag\",\"findApps\",\"getApp\",\"getCapabilities\",\"getState\",\"invoke\",\"listActions\",\"listApps\",\"listWindows\",\"mouseMove\",\"paste\",\"performAction\",\"pressKey\",\"screenshot\",\"scroll\",\"selectText\",\"sendKeyEvent\",\"setValue\",\"show\",\"typeText\"],\"appLevelAccessibility\":true,\"keyboard\":[\"app\"],\"browser\":false,\"coordinateSpace\":\"screenshot-pixels\"}"}],"isError":false,"id":"{{message:5}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"}
|
||||
{"type":"step/end","data":{"turn":1,"step":1}}
|
||||
{"type":"step/start","data":{"turn":1,"step":2}}
|
||||
{"type":"assistant/message","data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-unauthorized-getState","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:6}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732188,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732188,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-unauthorized-getState","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1793949732188,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732188,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
|
||||
|
||||
@@ -57,7 +57,7 @@ Setup and discovery:
|
||||
Available at app or window level depending on the platform.
|
||||
|
||||
Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the full tree text; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the tree within the provider limits; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. If the tree reports maxElements or maxDepth truncation, select a narrower window or ask the user to raise the named macOS provider setting and read again. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current.
|
||||
- listActions(elementId: string) -> string[] — list the actions an element advertises.
|
||||
- invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none.
|
||||
- performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument.
|
||||
@@ -79,6 +79,7 @@ Available at app or window level depending on the platform.
|
||||
- typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab.
|
||||
- paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry.
|
||||
- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string.
|
||||
- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.
|
||||
|
||||
Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name.
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ Setup and discovery:
|
||||
Available at app or window level depending on the platform.
|
||||
|
||||
Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the full tree text; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the tree within the provider limits; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. If the tree reports maxElements or maxDepth truncation, select a narrower window or ask the user to raise the named macOS provider setting and read again. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current.
|
||||
- listActions(elementId: string) -> string[] — list the actions an element advertises.
|
||||
- invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none.
|
||||
- performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument.
|
||||
@@ -81,6 +81,7 @@ Available at app or window level depending on the platform.
|
||||
- typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab.
|
||||
- paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry.
|
||||
- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string.
|
||||
- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.
|
||||
|
||||
Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name.
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ Setup and discovery:
|
||||
Available at app or window level depending on the platform.
|
||||
|
||||
Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the full tree text; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the tree within the provider limits; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. If the tree reports maxElements or maxDepth truncation, select a narrower window or ask the user to raise the named macOS provider setting and read again. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current.
|
||||
- listActions(elementId: string) -> string[] — list the actions an element advertises.
|
||||
- invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none.
|
||||
- performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument.
|
||||
@@ -81,6 +81,7 @@ Available at app or window level depending on the platform.
|
||||
- typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab.
|
||||
- paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry.
|
||||
- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string.
|
||||
- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.
|
||||
|
||||
Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name.
|
||||
|
||||
|
||||
Reference in new issue
Block a user