From 636c34f5a6610541d725a3ee0ea2b683c71ce289 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 12:17:15 +0800 Subject: [PATCH 01/11] feat(cua): support overlapping macOS key presses --- .../experimental/cua-macos/README.i18n.yaml | 4 +- packages/experimental/cua-macos/README.md | 4 +- packages/experimental/cua-macos/README.zh.md | 4 +- .../cua-macos/native/README.i18n.yaml | 4 +- .../experimental/cua-macos/native/README.md | 2 +- .../cua-macos/native/README.zh.md | 2 +- .../native/Sources/CuaNative/AXWorker.swift | 4 +- .../Sources/CuaNative/KeyboardInput.swift | 109 ++++++++---------- .../Sources/CuaNative/RequestAdmission.swift | 9 ++ .../cua-macos/native/protocol.schema.json | 16 ++- .../native/tests/capabilities_e2e.py | 6 + .../tests/swift/KeyboardChordTests.swift | 36 ++++++ .../cua-macos/tests/guidance.spec.ts | 4 +- packages/experimental/cua/README.i18n.yaml | 4 +- packages/experimental/cua/README.md | 2 +- packages/experimental/cua/README.zh.md | 2 +- packages/experimental/cua/src/guidance.ts | 2 +- packages/experimental/cua/src/protocol.ts | 2 +- packages/experimental/cua/src/types.ts | 6 +- packages/experimental/cua/tests/api.spec.ts | 5 +- .../cua-native/system-prompt.expected.md | 2 +- .../cua-native/tool-schemas.expected.json | 16 ++- .../session/cua-ptc/system-prompt.expected.md | 2 +- .../system-prompt.expected.md | 2 +- 24 files changed, 158 insertions(+), 91 deletions(-) create mode 100644 packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index 0d83c6e538..bcc12b8ecc 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -15,8 +15,8 @@ en: 1777e7bf6785f986 zh: e3daac1074ebb74d /deepseek-ai-dsh-experimental-cua-macos/use-this-package: - en: 533ed4dd11af5126 - zh: 03036e8c949d6aca + en: e3643640ef764471 + zh: 5d25e833cb78bf9b /deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment: en: b734a0907747580e zh: 8110c5951b70e100 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 46b035112e..63e1c495d2 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -79,7 +79,7 @@ Source builds default to `sourceSigning: ad-hoc`, without creating certificates The [configuration catalog](../../../docs/config-catalog.md#deepseek-aidsh-experimental-cua-macos) lists all fields. Native tool calls use `{method, params}`; the complete method-specific schema is exposed to the model. `getApp` returns the `appHandle` used by every app-targeted operation, and `app.getWindow(windowId)` returns the object that owns screenshots and pointer input. -`pressKey(key, modifiers?)` presses and releases one key. Pass modifier names separately, for example `pressKey('Tab', ['shift'])` or `pressKey('+')`; an omitted modifier array means none. The macOS helper accepts `command`/`cmd`, `shift`, `option`/`alt`, `control`/`ctrl`, and `fn`, and maps them to their physical keys. Key names are platform-defined; keypad names (`KP_0` through `KP_9`, `KP_Enter`, and keypad operators) retain their physical key codes. `typeText` sends literal text, including its line breaks and tabs. On macOS, use `pressKey` on the App object. The shared `sendKeyEvent` operation returns `NOT_IMPLEMENTED` on this provider. +`pressKey(key, modifiers?)` accepts one key or a nonempty array of distinct keys. `pressKey(['w', 'd'])` presses W then D and releases D then W. Pass modifier names separately, for example `pressKey('Tab', ['shift'])` or `pressKey('+')`; an omitted modifier array means none. The macOS helper accepts `command`/`cmd`, `shift`, `option`/`alt`, `control`/`ctrl`, and `fn`, and maps them to their physical keys. Key names are platform-defined; keypad names (`KP_0` through `KP_9`, `KP_Enter`, and keypad operators) retain their physical key codes. `typeText` sends literal text, including its line breaks and tabs. On macOS, use `pressKey` on the App object. The shared `sendKeyEvent` operation returns `NOT_IMPLEMENTED` on this provider. Agent disposal and plugin unload cancel and await the complete tool call, including image admission and attachment storage. An image write already in progress must settle before disposal returns; a canceled call cannot return a successful screenshot. @@ -221,7 +221,7 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. +- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — press one key or hold distinct keys together. pressKey(['w', 'd']) presses W then D and releases D then W. Pass modifiers separately, as in pressKey('Tab', ['shift']). Key and modifier names are platform-defined strings. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. ``` diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index 341796b8c4..276f725cb3 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -79,7 +79,7 @@ pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis. [配置目录](../../../docs/config-catalog.zh.md#deepseek-aidsh-experimental-cua-macos)列出全部字段。原生工具调用使用 `{method, params}`，模型可以看到每个方法的完整参数定义。`getApp` 返回每个面向应用的操作所需的 `appHandle`，`app.getWindow(windowId)` 返回负责截图和指针输入的对象。 -`pressKey(key, modifiers?)` 按下并释放一个键。修饰键名称单独传入，例如 `pressKey('Tab', ['shift'])` 或 `pressKey('+')`；省略修饰键数组表示不按修饰键。macOS helper 接受 `command`/`cmd`、`shift`、`option`/`alt`、`control`/`ctrl` 和 `fn`，并映射到对应物理键。按键名由平台定义；数字键盘名称（`KP_0` 到 `KP_9`、`KP_Enter` 和数字键盘运算键）保留独立的物理键码。`typeText` 发送字面文本，包括其中的换行和制表符。在 macOS 上，通过 App 对象调用 `pressKey`。调用共享接口 `sendKeyEvent` 会收到 `NOT_IMPLEMENTED`。 +`pressKey(key, modifiers?)` 接受单个键名或由不同键名组成的非空数组。例如，`pressKey(['w', 'd'])` 依次按下 W、D，再依次松开 D、W。修饰键名称单独传入，例如 `pressKey('Tab', ['shift'])` 或 `pressKey('+')`；省略修饰键数组表示不按修饰键。macOS helper 接受 `command`/`cmd`、`shift`、`option`/`alt`、`control`/`ctrl` 和 `fn`，并映射到对应物理键。按键名由平台定义；数字键盘名称（`KP_0` 到 `KP_9`、`KP_Enter` 和数字键盘运算键）保留独立的物理键码。`typeText` 发送字面文本，包括其中的换行和制表符。在 macOS 上，通过 App 对象调用 `pressKey`。调用共享接口 `sendKeyEvent` 会收到 `NOT_IMPLEMENTED`。 Agent 销毁和插件卸载会取消并等待完整工具调用，包括图像接入检查和附件存储。已经开始的图像写入必须结束后，销毁才会返回；已取消的调用不会返回成功截图。 @@ -221,7 +221,7 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. +- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — press one key or hold distinct keys together. pressKey(['w', 'd']) presses W then D and releases D then W. Pass modifiers separately, as in pressKey('Tab', ['shift']). Key and modifier names are platform-defined strings. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. ``` diff --git a/packages/experimental/cua-macos/native/README.i18n.yaml b/packages/experimental/cua-macos/native/README.i18n.yaml index 8d43fbd6b8..6e9294f0ce 100644 --- a/packages/experimental/cua-macos/native/README.i18n.yaml +++ b/packages/experimental/cua-macos/native/README.i18n.yaml @@ -21,8 +21,8 @@ en: 88a59904ca4ebedc zh: 14f2b666d7382145 /dsh-cua-native-helper/observation-and-input: - en: 2d380e942a4057e4 - zh: fe7c9339ec63a6e5 + en: d19180b1a6db2e48 + zh: c6279db9957218eb /dsh-cua-native-helper/verification: en: 49f4cce2a4cc3114 zh: e9d65c92ef35fe24 diff --git a/packages/experimental/cua-macos/native/README.md b/packages/experimental/cua-macos/native/README.md index bc80110951..b94d053746 100644 --- a/packages/experimental/cua-macos/native/README.md +++ b/packages/experimental/cua-macos/native/README.md @@ -103,7 +103,7 @@ Clicks send mouse-down and mouse-up events at the element center or the supplied Capture and pointer delivery read CG bounds before and after an exact-window AX frame query. Movement during the query returns WINDOW_CHANGED. Valid AX/CG frames must have equal outward-rounded integral point rectangles; a known mismatch returns WINDOW_CHANGED before mouse movement, priming or button delivery. The helper preserves both coordinate sources rather than substituting AX geometry for CG geometry. Missing or unusable AX geometry remains unknown, while AX communication, permission and invalid-element errors propagate. Capture without Accessibility permission retains its CG/SCK checks. Advertised AX actions bypass this pointer preflight; an AX action error does not trigger pointer input. These reads do not lock the window against later movement, and no geometry check activates or raises it. -`typeText` and `pressKey` target the selected process's current focused control, not an exact window. `pressKey` presses and releases one key while holding its `modifiers` array, sends PID-targeted modifier transitions, and releases those modifiers in reverse order; these events do not use the global HID queue. Key names are platform-defined; keypad names such as `KP_0` and `KP_Enter` preserve their physical key codes. Background pointer actions may change that keyboard focus. Text injection sends Unicode characters individually and does not use the clipboard. LF, CR and CRLF send one Return; tabs send Tab. Applications apply their normal editing behavior, including indentation and focus navigation. Each `typeText` down/up event is followed by the configured text interval; `pressKey` keeps its own press and release timing, and dispatch completion does not confirm that the receiving application consumed every event. Cancellation can interrupt text or a gesture after partial input; it cannot undo received events. Very long text can exceed the host's operation deadline. Validate observed application state before deciding how to continue. +`typeText` and `pressKey` target the selected process's current focused control, not an exact window. `pressKey` presses its modifiers followed by the requested keys in order, then releases all pressed keys in reverse order, including on cancellation; these events do not use the global HID queue. Key names are platform-defined; keypad names such as `KP_0` and `KP_Enter` preserve their physical key codes. Background pointer actions may change that keyboard focus. Text injection sends Unicode characters individually and does not use the clipboard. LF, CR and CRLF send one Return; tabs send Tab. Applications apply their normal editing behavior, including indentation and focus navigation. Each `typeText` down/up event is followed by the configured text interval; `pressKey` keeps its own press and release timing, and dispatch completion does not confirm that the receiving application consumed every event. Cancellation can interrupt text or a gesture after partial input; it cannot undo received events. Very long text can exceed the host's operation deadline. Validate observed application state before deciding how to continue. The selected helper application needs macOS Accessibility permission for AX state and input, and Screen Recording permission for screenshots. Capability preflight and permission errors do not grant permissions. Capture refuses a window that moves or resizes while the image is being read. Unassociated system panels and accessory-only applications are not selectable. diff --git a/packages/experimental/cua-macos/native/README.zh.md b/packages/experimental/cua-macos/native/README.zh.md index 0927a82bce..7ddde7d4f4 100644 --- a/packages/experimental/cua-macos/native/README.zh.md +++ b/packages/experimental/cua-macos/native/README.zh.md @@ -103,7 +103,7 @@ AppKit 在主事件循环上运行。阻塞式 socket 接收与握手使用后 截图和鼠标输入在精确窗口的 AX 边界查询前后读取 CG 边界，查询期间移动会返回 WINDOW_CHANGED。有效的 AX/CG 边界向外取整后必须一致；已知不一致会在移动、预备事件或按键发送前返回 WINDOW_CHANGED。helper 保留两个坐标来源，不用 AX 几何替换 CG 几何。缺失或无效的 AX 几何仍表示未知，AX 通信、权限和无效元素错误继续传递。无辅助功能权限的截图保留 CG/SCK 检查。声明的 AX 动作不经过鼠标预检，AX 动作失败也不改用鼠标输入。这些读取不会锁住窗口以阻止后续移动，几何检查不激活或抬起窗口。 -`typeText` 和 `pressKey` 面向所选进程当前获得焦点的控件，不指定窗口。`pressKey` 在按住所列 `modifiers` 时按下并释放一个键，先向 PID 发送修饰键变化，再发送主键，并逆序释放修饰键，不使用全局 HID 队列。按键名由平台定义；KP_0 和 KP_Enter 等数字键盘名称保留物理键码。后台鼠标操作可能改变键盘焦点。文本逐个 Unicode 字符输入，不使用剪贴板。LF、CR 和 CRLF 发送一次 Return，制表符发送 Tab。应用保留缩进和焦点导航等正常编辑行为。`typeText` 的每次按下和抬起后等待配置的文本事件间隔；`pressKey` 使用自己的按下与释放节奏，分派完成不证明应用已消费全部事件。取消可能中断部分文本或手势，无法撤销已接收事件。很长的文本可能超过 Host 操作期限。继续输入前应核实应用状态。 +`typeText` 和 `pressKey` 面向所选进程当前获得焦点的控件，不指定窗口。`pressKey` 向目标 PID 依次发送修饰键和指定按键的按下事件，再逆序释放。操作中途取消也会逆序释放已按下的键。按键名由平台定义；KP_0 和 KP_Enter 等数字键盘名称保留物理键码。后台鼠标操作可能改变键盘焦点。文本逐个 Unicode 字符输入，不使用剪贴板。LF、CR 和 CRLF 发送一次 Return，制表符发送 Tab。应用保留缩进和焦点导航等正常编辑行为。`typeText` 的每次按下和抬起后等待配置的文本事件间隔；`pressKey` 使用自己的按下与释放节奏，分派完成不证明应用已消费全部事件。取消可能中断部分文本或手势，无法撤销已接收事件。很长的文本可能超过 Host 操作期限。继续输入前应核实应用状态。 所选 helper 应用需要 macOS 辅助功能权限以读取 AX 和输入，需要屏幕录制权限以截图。能力预检和权限错误不授予权限。截图会拒绝在读取图片期间移动或缩放的窗口。无法确认归属的系统面板及仅后台辅助类型的应用不可选择。 diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift index c4cc6066c8..da1afe5a3d 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift @@ -118,7 +118,7 @@ actor AXWorker { if let keyboard, let pointer { return (keyboard, pointer) } let poster = PIDEventPoster() let latch = cancellation - let key = KeyboardInput(poster: poster, textEventInterval: textEventInterval, checkCancellation: { try latch.check() }) + let key = KeyboardInput(post: poster.post, textEventInterval: textEventInterval, checkCancellation: { try latch.check() }) let mouse = WindowPointerInput(poster: poster, checkCancellation: { try latch.check() }) keyboard = key; pointer = mouse return (key, mouse) @@ -335,7 +335,7 @@ actor AXWorker { let recipient = try ax().keyboardRecipient(identity, catalog: catalog) try NativePlatform.verify(recipient) if method == "typeText" { try inputs().0.typeText(params.string("text"), to: recipient.pid) } - else { try inputs().0.pressKey(params.string("key"), modifiers: params.modifiers(), to: recipient.pid) } + else { try inputs().0.pressKey(params.pressKeys(), modifiers: params.modifiers(), to: recipient.pid) } case "setValue": try ax().setAttribute(element(handle, params), "AXValue", params.values["value"]) case "performAction": try ax().performAction(element(handle, params), action: params.string("action"), argument: params.values["argument"]) diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift index be75257f1c..2dbf648ecf 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift @@ -4,76 +4,62 @@ import Foundation /// Balanced PID-targeted chords and scalar-based Unicode input on one serial worker. final class KeyboardInput { - private struct Modifier { + private struct Key { let keyCode: CGKeyCode let flag: CGEventFlags } - private struct ModifierEvents { - let down: CGEvent - let up: CGEvent - } - - private let poster: PIDEventPoster + private let post: (CGEvent, Int32) -> Void private let textEventInterval: Double private let checkCancellation: @Sendable () throws -> Void - init(poster: PIDEventPoster, textEventInterval: Double, checkCancellation: @escaping @Sendable () throws -> Void = {}) { - self.poster = poster + init(post: @escaping (CGEvent, Int32) -> Void, textEventInterval: Double, checkCancellation: @escaping @Sendable () throws -> Void = {}) { + self.post = post self.textEventInterval = textEventInterval self.checkCancellation = checkCancellation } - /// Construct the entire chord before posting; cleanup releases only keys pressed by this call. - func pressKey(_ key: String, modifiers: [String], to pid: Int32) throws { + /// Prepare every event before dispatch. On cancellation, release only the keys this call pressed. + func pressKey(_ keys: [String], modifiers: [String], to pid: Int32) throws { try checkCancellation() - var names = modifiers - let isPlus = key == "+" || key.lowercased() == "plus" - if isPlus { names.append("shift") } - let code = try Self.keyCode(isPlus ? "=" : key) - var selected: [Modifier] = [] - for name in names { - let modifier = try Self.modifier(name) - if !selected.contains(where: { $0.keyCode == modifier.keyCode }) { - selected.append(modifier) + var modifierNames = modifiers + let mainKeys = try keys.map { name -> Key in + if name == "+" || name.lowercased() == "plus" { + modifierNames.append("shift") + return Key(keyCode: try Self.keyCode("="), flag: []) } + if let modifier = Self.namedModifier(name) { return modifier } + return Key(keyCode: try Self.keyCode(name), flag: []) } - let source = try Self.source() + guard Set(mainKeys.map(\.keyCode)).count == mainKeys.count else { + throw RequestParameters.invalid("A chord must contain distinct physical keys") + } + var selected = Set() var flags: CGEventFlags = [] - var modifierEvents: [ModifierEvents] = [] - for modifier in selected { + var events: [(down: CGEvent, up: CGEvent)] = [] + let source = try Self.source() + for key in try modifierNames.map(Self.modifier) + mainKeys where selected.insert(key.keyCode).inserted { let releasedFlags = flags - flags.insert(modifier.flag) - modifierEvents.append(ModifierEvents( - down: try Self.keyEvent(source: source, code: modifier.keyCode, down: true, flags: flags), - up: try Self.keyEvent(source: source, code: modifier.keyCode, down: false, flags: releasedFlags) + flags.formUnion(key.flag) + events.append(( + down: try Self.keyEvent(source: source, code: key.keyCode, down: true, flags: flags), + up: try Self.keyEvent(source: source, code: key.keyCode, down: false, flags: releasedFlags) )) } - let down = try Self.keyEvent(source: source, code: code, down: true, flags: flags) - let up = try Self.keyEvent(source: source, code: code, down: false, flags: flags) - var heldModifiers = 0 - var keyHeld = false + var pressed = 0 do { defer { - if keyHeld { poster.post(up, to: pid) } - for index in (0.. 0 { Thread.sleep(forTimeInterval: 0.008) } } } - for events in modifierEvents { + for event in events { try checkCancellation() - poster.post(events.down, to: pid) - heldModifiers += 1 + post(event.down, pid) + pressed += 1 Thread.sleep(forTimeInterval: 0.008) } - try checkCancellation() - poster.post(down, to: pid) - keyHeld = true - Thread.sleep(forTimeInterval: 0.008) - try checkCancellation() - poster.post(up, to: pid) - keyHeld = false } try checkCancellation() } @@ -104,16 +90,16 @@ final class KeyboardInput { do { defer { if keyHeld { - poster.post(up, to: pid) + post(up, pid) Thread.sleep(forTimeInterval: textEventInterval) } } try checkCancellation() - poster.post(down, to: pid) + post(down, pid) keyHeld = true Thread.sleep(forTimeInterval: textEventInterval) try checkCancellation() - poster.post(up, to: pid) + post(up, pid) keyHeld = false Thread.sleep(forTimeInterval: textEventInterval) } @@ -139,22 +125,30 @@ final class KeyboardInput { return event } - private static func modifier(_ name: String) throws -> Modifier { + private static func namedModifier(_ name: String) -> Key? { switch name.lowercased() { - case "command", "cmd": return Modifier(keyCode: 55, flag: .maskCommand) - case "shift": return Modifier(keyCode: 56, flag: .maskShift) - case "option", "alt": return Modifier(keyCode: 58, flag: .maskAlternate) - case "control", "ctrl": return Modifier(keyCode: 59, flag: .maskControl) - case "fn": return Modifier(keyCode: 63, flag: .maskSecondaryFn) - default: throw NativeFailure(code: "INVALID_ARGUMENT", message: "Unsupported key modifier: \(name)") + case "command", "cmd": return Key(keyCode: 55, flag: .maskCommand) + case "shift": return Key(keyCode: 56, flag: .maskShift) + case "option", "alt": return Key(keyCode: 58, flag: .maskAlternate) + case "control", "ctrl": return Key(keyCode: 59, flag: .maskControl) + case "fn": return Key(keyCode: 63, flag: .maskSecondaryFn) + default: return nil } } + private static func modifier(_ name: String) throws -> Key { + guard let modifier = namedModifier(name) else { + throw NativeFailure(code: "INVALID_ARGUMENT", message: "Unsupported key modifier: \(name)") + } + return modifier + } + static func modifierFlags(_ names: [String]) throws -> CGEventFlags { try names.reduce(into: CGEventFlags()) { flags, name in flags.formUnion(try modifier(name).flag) } } static func keyCode(_ key: String) throws -> CGKeyCode { + if let modifier = namedModifier(key) { return modifier.keyCode } switch key.lowercased() { case "kp_0": return 82 case "kp_1": return 83 @@ -180,12 +174,7 @@ final class KeyboardInput { case "space": return 49 case "delete", "backspace": return 51 case "escape", "esc": return 53 - case "command", "cmd": return 55 - case "shift": return 56 case "capslock": return 57 - case "option", "alt": return 58 - case "control", "ctrl": return 59 - case "fn": return 63 case "home": return 115 case "pageup": return 116 case "del", "forward_delete": return 117 diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift index 5eb70381e4..ef2b6dc101 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift @@ -36,6 +36,15 @@ struct RequestParameters: Sendable { return path } } + func pressKeys() throws -> [String] { + if let name = values["key"]?.string { + guard !name.isEmpty else { throw Self.invalid("key must not be empty") } + return [name] + } + let names = try strings("key") + guard !names.isEmpty else { throw Self.invalid("key must name at least one key") } + return names + } func optionalString(_ key: String) throws -> String? { contains(key) ? try string(key) : nil } func optionalBoolean(_ key: String) throws -> Bool? { guard let raw = values[key] else { return nil } diff --git a/packages/experimental/cua-macos/native/protocol.schema.json b/packages/experimental/cua-macos/native/protocol.schema.json index b0d74a84dc..aa6d201dcb 100644 --- a/packages/experimental/cua-macos/native/protocol.schema.json +++ b/packages/experimental/cua-macos/native/protocol.schema.json @@ -993,8 +993,20 @@ "maximum": 4294967295 }, "key": { - "type": "string", - "minLength": 1 + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "minItems": 1, + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + ] }, "modifiers": { "type": "array", diff --git a/packages/experimental/cua-macos/native/tests/capabilities_e2e.py b/packages/experimental/cua-macos/native/tests/capabilities_e2e.py index 650aff14e9..95a94c97b6 100644 --- a/packages/experimental/cua-macos/native/tests/capabilities_e2e.py +++ b/packages/experimental/cua-macos/native/tests/capabilities_e2e.py @@ -191,6 +191,12 @@ def main(): call("drag", {**window, "fromX": point["x"], "fromY": point["y"], "toX": point["x"] + 80, "toY": point["y"] + 20, "durationMs": 200, "steps": 10, "mouseButton": "middle", "modifiers": ["shift"]}) state = wait_for_state(state_file, lambda value: any(event["type"] == 27 and event.get("button") == 2 and event["shift"] for event in value["events"])) + call("pressKey", {**owner, "key": ["w", "d"]}) + expected_chord = [(10, 13), (10, 2), (11, 2), (11, 13)] + state = wait_for_state(state_file, lambda value: any( + event["type"] == 11 and event.get("keyCode") == 13 for event in value["events"])) + assert [(event["type"], event["keyCode"]) for event in state["events"] + if event.get("keyCode") in (13, 2)] == expected_chord, state["events"] call("pressKey", {**owner, "key": "KP_0"}) call("pressKey", {**owner, "key": "KP_Enter"}) state = wait_for_state(state_file, lambda value: {82, 76}.issubset({event.get("keyCode") for event in value["events"]})) diff --git a/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift b/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift new file mode 100644 index 0000000000..ae98c891d6 --- /dev/null +++ b/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift @@ -0,0 +1,36 @@ +// Observe the actual CoreGraphics events without sending keyboard input to the desktop. +import CoreGraphics +import Testing +@testable import CuaNative + +@Test func chordKeepsModifiersThroughBothLetterReleases() throws { + var events: [CGEvent] = [] + let input = KeyboardInput(post: { event, pid in + #expect(pid == 42) + events.append(event) + }, textEventInterval: 0) + try input.pressKey(["w", "d"], modifiers: ["shift"], to: 42) + #expect(events.map { $0.getIntegerValueField(.keyboardEventKeycode) } == [56, 13, 2, 2, 13, 56]) + #expect(events.map(\.type) == [.flagsChanged, .keyDown, .keyDown, .keyUp, .keyUp, .flagsChanged]) + #expect(events.map { $0.flags.contains(.maskShift) } == [true, true, true, true, true, false]) +} + +@Test(arguments: [1, 2]) +func cancelledChordReleasesOnlyPressedKeys(_ cancelAfter: Int) { + let cancellation = CancellationLatch() + var events: [CGEvent] = [] + let input = KeyboardInput(post: { event, _ in + events.append(event) + if events.count == cancelAfter { cancellation.cancel() } + }, textEventInterval: 0, checkCancellation: cancellation.check) + #expect(throws: NativeFailure.self) { try input.pressKey(["w", "d"], modifiers: [], to: 42) } + let held = [Int64(13), 2].prefix(cancelAfter) + #expect(events.map { $0.getIntegerValueField(.keyboardEventKeycode) } == Array(held) + held.reversed()) + #expect(events.map(\.type) == Array(repeating: .keyDown, count: cancelAfter) + Array(repeating: .keyUp, count: cancelAfter)) +} + +@Test(arguments: [["w", "unknown"], ["w", "W"], ["Enter", "Return"]]) +func invalidChordPostsNoInput(_ keys: [String]) { + let input = KeyboardInput(post: { _, _ in Issue.record("Invalid chord posted input") }, textEventInterval: 0) + #expect(throws: NativeFailure.self) { try input.pressKey(keys, modifiers: ["shift"], to: 42) } +} diff --git a/packages/experimental/cua-macos/tests/guidance.spec.ts b/packages/experimental/cua-macos/tests/guidance.spec.ts index c4a6aeec7d..f3e530b7de 100644 --- a/packages/experimental/cua-macos/tests/guidance.spec.ts +++ b/packages/experimental/cua-macos/tests/guidance.spec.ts @@ -89,7 +89,7 @@ it('preserves image formats and pointer modifier enums without restricting keybo for (const name of ['click', 'drag']) { expect(lines.find(line => line.startsWith(`- ${name}(`))).toContain("modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[]") } - expect(COMMON_GUIDANCE).toContain('pressKey(key: string, modifiers?: string[])') + expect(COMMON_GUIDANCE).toContain('pressKey(key: string | string[], modifiers?: string[])') expect(COMMON_GUIDANCE).toContain('Key and modifier names are platform-defined strings') }) @@ -130,7 +130,7 @@ it('documents facade initialization, emitted observations, and separate key argu expect(COMMON_GUIDANCE).toContain('host-provided bridge initialization once') expect(COMMON_GUIDANCE).toContain('not a preinstalled global') expect(COMMON_GUIDANCE).toContain('void await') - expect(COMMON_GUIDANCE).toContain('do not pack them into a chord string') + expect(COMMON_GUIDANCE).toContain("pressKey(['w', 'd']) presses W then D and releases D then W") expect(COMMON_GUIDANCE).toContain('perform a left-button double-click') }) diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index f4abb13c43..0d16bc4b02 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -21,8 +21,8 @@ en: d92d5948a7941e8d zh: d43491ae6521b87e /deepseek-ai-dsh-cua/understand-the-implementation: - en: 2ab74a1ff120459e - zh: 39b3ef198989f133 + en: e38d57709aa6d001 + zh: 231b2dcaf826d386 /deepseek-ai-dsh-cua/further-exploration: en: c174fb5a00ea3681 zh: 039340042676881d diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index c95e4ae8ec..e6c0f0778f 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -91,7 +91,7 @@ One object facade ([src/api.ts](src/api.ts)) builds a shared `CuaSurface` and de Provider startup capabilities are fixed at installation: platform, methods, app-level accessibility, keyboard scopes, browser support, and coordinate space. The native `getCapabilities` RPC reports runtime permissions and session lock only; `installCua` merges both layers for the model-visible result. The raw RPC remains a common operation representation rather than a capability inventory, so provider guidance states the actions its platform supports. -Check `methods` and `keyboard` in the result of `await cua.getCapabilities()` for the supported operations and input scopes. `pressKey(key, modifiers?)` presses and releases one key while holding the listed modifiers; `sendKeyEvent(key, direction)` sends exactly one key-down or key-up. `typeText` and `paste` deliver literal text at the provider's declared keyboard scope. Keyboard modifier names are plain strings that the provider interprets. +Check `methods` and `keyboard` in the result of `await cua.getCapabilities()` for the supported operations and input scopes. `pressKey(key, modifiers?)` holds the listed modifiers while pressing the requested keys in order and releasing them in reverse. For example, `pressKey(['w', 'd'])` holds W through the D keystroke; `sendKeyEvent(key, direction)` sends exactly one key-down or key-up. `typeText` and `paste` deliver literal text at the provider's declared keyboard scope. Keyboard modifier names are plain strings that the provider interprets. Platform providers import `CuaConfigFields` from `@deepseek-ai/dsh-cua/config` to extend the shared configuration. The root Host entry exports `installCua`, `CuaConfig`, and the Typert-discovered `CuaAccess` service; its `apply()` only marks the optional Client contribution. `/client` loads the permission UI through the existing module loader. Separate Host and Client compiler projects keep React, CSS, and browser registrations out of headless imports. `/api` owns the app/window API and its types; `/bridge`, `/native-client`, and `/protocol` retain runtime and platform consumers. `/types` exports only the named types required by generated Remote declarations. The Client uses package-local types. `/typert` supplies Host reflection and `/remote` supplies generated browser descriptors and codecs; permission authority remains on the Host. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index c1c5ed863c..2f8e31b77c 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -91,7 +91,7 @@ Computer Use 详情页的权限卡片、组件列表和来源信息使用 l3 边 提供方启动能力在安装时固定：平台、方法、应用级辅助功能、键盘作用域、浏览器支持和坐标空间。原生 `getCapabilities` RPC 只报告运行时权限和会话锁定状态；`installCua` 为模型可见结果合并两层。原生 RPC 仍是通用操作表示，而不是能力清单，因此 provider 指引说明其平台支持的动作。 -通过 `await cua.getCapabilities()` 返回的 `methods` 和 `keyboard` 确认支持的操作和输入范围。`pressKey(key, modifiers?)` 在按住所列修饰键时按下并释放一个键；`sendKeyEvent(key, direction)` 只发送一次按下或抬起。`typeText` 和 `paste` 在 provider 声明的键盘作用域内传送字面文本。键盘修饰键名称是 provider 解释的普通字符串。 +通过 `await cua.getCapabilities()` 返回的 `methods` 和 `keyboard` 确认支持的操作和输入范围。`pressKey(key, modifiers?)` 按住所列修饰键，依次按下指定的键，再逆序释放。例如，`pressKey(['w', 'd'])` 在按下和松开 D 期间持续按住 W。`sendKeyEvent(key, direction)` 只发送一次按下或抬起。`typeText` 和 `paste` 在 provider 声明的键盘作用域内传送字面文本。键盘修饰键名称是 provider 解释的普通字符串。 平台提供方从 `@deepseek-ai/dsh-cua/config` 导入 `CuaConfigFields` 来扩展共享配置。根 Host 入口导出 `installCua`、`CuaConfig` 和供 Typert 发现的 `CuaAccess` 服务，`apply()` 只标记可选 Client 贡献。`/client` 通过现有模块加载器加载权限界面。Host 和 Client 使用独立编译项目，headless 导入不会加载 React、CSS 或浏览器注册。`/api` 提供应用、窗口 API 及其类型；`/bridge`、`/native-client` 和 `/protocol` 保留运行时和平台消费者。`/types` 仅导出生成 Remote 声明所需的命名类型，Client 使用包内类型。`/typert` 提供 Host 反射，`/remote` 提供浏览器描述和编解码，权限判断仍由 Host 执行。 diff --git a/packages/experimental/cua/src/guidance.ts b/packages/experimental/cua/src/guidance.ts index 745ccfe9be..0677be5433 100644 --- a/packages/experimental/cua/src/guidance.ts +++ b/packages/experimental/cua/src/guidance.ts @@ -43,6 +43,6 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. +- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — press one key or hold distinct keys together. pressKey(['w', 'd']) presses W then D and releases D then W. Pass modifiers separately, as in pressKey('Tab', ['shift']). Key and modifier names are platform-defined strings. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name.` diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index 538c95cfe6..8488f32af3 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -62,7 +62,7 @@ export const Request = z.discriminatedUnion('method', [ selectionType: z.enum(['text', 'cursor_before', 'cursor_after']).optional() }), operation('typeText', { ...app, windowId: id.optional(), text: z.string() }), operation('paste', { ...app, windowId: id.optional(), text: z.string(), format: z.enum(['text', 'html']).optional() }), - operation('pressKey', { ...app, windowId: id.optional(), key: z.string().min(1), modifiers: z.array(z.string()).optional() }), + operation('pressKey', { ...app, windowId: id.optional(), key: z.union([z.string().min(1), z.array(z.string().min(1)).min(1)]), modifiers: z.array(z.string()).optional() }), operation('sendKeyEvent', { ...app, windowId: id.optional(), key: z.string().min(1), direction: z.enum(['down', 'up']) }), z.object({ method: z.literal('scroll'), params: z.union([ z.object({ ...absoluteCoordinate, diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index b84663f9e2..875418d805 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -160,13 +160,13 @@ export interface CuaSurface { */ paste(text: string, options?: CuaPasteOptions): Promise /** - * Press and release one key at the current input focus. + * Press the keys in order at the current focus, then release them in reverse order. * Rejects when the provider does not support keyboard input at this scope. - * @param key - Provider-defined key name to press and release; not a chord. + * @param key - One provider-defined key name or a nonempty array of distinct keys to hold together. * @param modifiers - Platform- or provider-defined modifier names held for this keystroke; omitted means none. * @returns Whether the provider successfully executed the requested operation. */ - pressKey(key: string, modifiers?: string[]): Promise + pressKey(key: string | string[], modifiers?: string[]): Promise /** * Send one key-down or key-up event at the current input focus. * Does not send the opposite event. diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index cbaa419951..f5b3452dc3 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -683,6 +683,8 @@ describe('input actions', () => { ['fractional scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 0.5, deltaLinesY: 0 })], ['negative drag point', ({ window }) => window.drag([0, 0], [-1, 3])], ['empty key', ({ app }) => app.pressKey('')], + ['empty chord', ({ app }) => app.pressKey([])], + ['empty chord member', ({ app }) => app.pressKey(['w', ''])], ] it.each<[string, (surfaces: Surfaces) => Promise]>([ ['getState windowId: Too small: expected number to be >0', ({ app }) => app.getWindow(0)], @@ -708,7 +710,8 @@ describe('input actions', () => { expect(f.dispatch).toHaveBeenCalledTimes(calls) }) - it.each<[string, string[] | undefined, string[]]>([ + it.each<[string | string[], string[] | undefined, string[]]>([ + [['w', 'd'], undefined, []], ['+', undefined, []], ['plus', [], []], ['+', ['command'], ['command']], diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index 0b9c1b29f6..a39b9145d4 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -78,7 +78,7 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. +- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — press one key or hold distinct keys together. pressKey(['w', 'd']) presses W then D and releases D then W. Pass modifiers separately, as in pressKey('Tab', ['shift']). Key and modifier names are platform-defined strings. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. diff --git a/snapshots/session/cua-native/tool-schemas.expected.json b/snapshots/session/cua-native/tool-schemas.expected.json index 33f3636750..8f608ec54e 100644 --- a/snapshots/session/cua-native/tool-schemas.expected.json +++ b/snapshots/session/cua-native/tool-schemas.expected.json @@ -1032,8 +1032,20 @@ "maximum": 4294967295 }, "key": { - "type": "string", - "minLength": 1 + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "minItems": 1, + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + ] }, "modifiers": { "type": "array", diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index 587436227a..017e5b103b 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -80,7 +80,7 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. +- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — press one key or hold distinct keys together. pressKey(['w', 'd']) presses W then D and releases D then W. Pass modifiers separately, as in pressKey('Tab', ['shift']). Key and modifier names are platform-defined strings. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. diff --git a/snapshots/session/ptc-script-retained/system-prompt.expected.md b/snapshots/session/ptc-script-retained/system-prompt.expected.md index dd88fa8d9e..ef58a12bd2 100644 --- a/snapshots/session/ptc-script-retained/system-prompt.expected.md +++ b/snapshots/session/ptc-script-retained/system-prompt.expected.md @@ -80,7 +80,7 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. +- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — press one key or hold distinct keys together. pressKey(['w', 'd']) presses W then D and releases D then W. Pass modifiers separately, as in pressKey('Tab', ['shift']). Key and modifier names are platform-defined strings. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. From 5a0c6c10f8163c57d0935ac791bbe5894ad59b41 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 13:08:42 +0800 Subject: [PATCH 02/11] fix(cua-macos): validate physical key chords before dispatch --- ...05-cua-scoped-surfaces-and-input.i18n.yaml | 8 ++-- ...026-11-05-cua-scoped-surfaces-and-input.md | 4 +- ...-11-05-cua-scoped-surfaces-and-input.zh.md | 4 +- .../cua-native-platform/guide.i18n.yaml | 4 +- .../v0.2.2-rc.1/cua-native-platform/guide.md | 2 +- .../cua-native-platform/guide.zh.md | 2 +- .../experimental/cua-macos/README.i18n.yaml | 4 +- packages/experimental/cua-macos/README.md | 4 +- packages/experimental/cua-macos/README.zh.md | 4 +- .../cua-macos/native/README.i18n.yaml | 8 ++-- .../experimental/cua-macos/native/README.md | 4 +- .../cua-macos/native/README.zh.md | 4 +- .../Sources/CuaNative/KeyboardInput.swift | 31 ++++++++----- .../Sources/CuaNative/RequestAdmission.swift | 10 ++-- .../cua-macos/native/protocol.schema.json | 3 +- .../native/tests/capabilities_e2e.py | 13 +++--- .../tests/swift/KeyboardChordTests.swift | 46 ++++++++++++++++--- .../native/tests/swift/ProtocolTests.swift | 12 +++++ .../experimental/cua-macos/src/guidance.ts | 1 + .../cua-macos/tests/guidance.spec.ts | 8 +++- packages/experimental/cua/README.i18n.yaml | 2 +- packages/experimental/cua/README.md | 2 +- packages/experimental/cua/src/guidance.ts | 2 +- packages/experimental/cua/src/protocol.ts | 5 +- packages/experimental/cua/src/types.ts | 2 +- .../experimental/cua/tests/protocol.spec.ts | 7 +++ .../cua-native/system-prompt.expected.md | 2 +- .../cua-native/tool-schemas.expected.json | 3 +- .../session/cua-ptc/system-prompt.expected.md | 2 +- .../system-prompt.expected.md | 2 +- 30 files changed, 137 insertions(+), 68 deletions(-) diff --git a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.i18n.yaml b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.i18n.yaml index f3a0d8f1d3..adad612610 100644 --- a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.i18n.yaml @@ -9,11 +9,11 @@ en: f72db0b79aa02fb5 zh: b91c1f177dc93ab4 /agent-note-cua-scoped-surfaces-and-separated-input-capabilities/decision: - en: ec115e11f96ffe60 - zh: c1fe9eb2866196d0 + en: 74f97ec371e61b82 + zh: a8a5cefc1e06bb20 /agent-note-cua-scoped-surfaces-and-separated-input-capabilities/alternatives-considered: - en: 32c970ae153e0f6c - zh: 8431be0a10ffed15 + en: 2113f17a49e16670 + zh: 1df04d9736a3ad4f /agent-note-cua-scoped-surfaces-and-separated-input-capabilities/consequences: en: 5c6ec60b77f1bff0 zh: e1ae568ef3501b7b diff --git a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.md b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.md index 609e5bf901..655172fb19 100644 --- a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.md +++ b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.md @@ -14,7 +14,7 @@ Application and window objects share accessibility operations, but only an expli Provider startup capabilities are static. Platform, supported methods, application-level accessibility, keyboard scopes, browser support, and coordinate space are fixed when the provider is installed. The native `getCapabilities` RPC reports runtime permissions and session lock only; `installCua` merges the two layers for the model-visible result. The neutral RPC is a common operation representation, not a capability inventory; provider guidance names the actions its platform supports. -Keyboard input keeps keys and modifiers separate: `pressKey(key, modifiers?)` presses and releases one key while holding the listed modifiers, and its keyboard modifier list is a plain string array defined by the provider. `sendKeyEvent(key, direction)` sends exactly one key-down or key-up without an implicit counterpart. A caller can therefore hold a modifier or split a gesture; neither method packs a chord string, and the shared schema does not enumerate platform key names. +Keyboard input keeps keys and modifiers separate: `pressKey(key, modifiers?)` presses one key or, on macOS, several physical keys in order and releases them in reverse while holding the listed modifiers, and its keyboard modifier list is a plain string array defined by the provider. `sendKeyEvent(key, direction)` sends exactly one key-down or key-up without an implicit counterpart. A caller can therefore hold a modifier or split a gesture; neither method packs a chord string, and the shared schema does not enumerate platform key names. Within one API instance, the object facade keeps one current accessibility snapshot per application handle, shared by every application and window object for that handle, including repeated selections of the same application. `getWindow` reads its initial tree silently; `getApp` does the same when the provider reports application-level accessibility. A later `getAccessibilityTree` read invalidates the previous snapshot before dispatch. The facade automatically attaches the latest snapshot ID to element operations; callers use element IDs from the latest tree and do not assume an older ID still identifies the same control. Element IDs are opaque provider strings. Only an explicit `CuaWindow` captures screenshots or sends pointer input. The native helper retains its own application and optional-window snapshots behind that facade; the native layer and the facade snapshot remain separate. The external runtime continues to own variable persistence and completion serialization; the facade adds no evaluator. The [native object decision](2026-09-17-experimental-cua-object-api.md) owns evaluator independence, and the [shared helper decision](2026-10-09-cua-shared-native-helper.md) owns connection lifetime. @@ -24,7 +24,7 @@ Within one API instance, the object facade keeps one current accessibility snaps **An application-specific branch in the shared base** would let window objects inherit decisions and capability checks that apply only to applications. Scope factories that inject their own requirements keep the shared operations independent of which scope uses them. -**A packed chord string** requires parsing key names, separators, and plus escaping together, and it pushes platform alias normalization into the shared layer. Keystroke and direction are separate semantics: `pressKey` describes one complete keystroke, while `sendKeyEvent` describes one explicit direction. Separate key and modifier values keep those semantics without a parser. +**A packed chord string** requires parsing key names, separators, and plus escaping together, and it pushes platform alias normalization into the shared layer. Keystroke and direction are separate semantics: `pressKey` describes a complete press and release, while `sendKeyEvent` describes one explicit direction. Separate key and modifier values keep those semantics without a parser. **Platform keyboard key enumerations in the shared schema** would couple the portable operation schema to one provider's key names and require a schema change for each platform. Provider-defined strings let the provider interpret its own names while the shared schema stays neutral. diff --git a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.zh.md b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.zh.md index 081f02986e..495ba2d399 100644 --- a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.zh.md +++ b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.zh.md @@ -14,7 +14,7 @@ Status: implemented Provider 的启动能力是静态的。平台、支持的方法、应用级辅助功能、键盘作用域、浏览器支持和坐标空间在安装 provider 时固定。原生 `getCapabilities` RPC 只报告运行时权限和会话锁定状态；`installCua` 为模型可见结果合并两层。中立的 RPC 是通用操作表示，不是能力清单；provider 指导文本说明其平台支持的动作。 -键盘输入把按键与修饰键分开：`pressKey(key, modifiers?)` 在按住所列修饰键时按下并释放一个键，其键盘修饰键列表是 provider 定义的普通字符串数组。`sendKeyEvent(key, direction)` 只发送一次按下或抬起，不隐式发送相反事件。因此调用方可以保持修饰键或拆分手势；两个方法都不接受组合键字符串，共享 schema 也不枚举平台按键名。 +键盘输入把按键与修饰键分开：`pressKey(key, modifiers?)` 在按住所列修饰键时按下并释放指定的键。macOS 也接受多个物理键，依次按下并逆序释放。修饰键名称由提供方定义，以字符串数组传入。`sendKeyEvent(key, direction)` 只发送一次按下或抬起，不隐式发送相反事件。因此调用方可以保持修饰键或拆分手势；两个方法都不接受组合键字符串，共享 schema 也不枚举平台按键名。 在单个 API 实例内，对象门面按应用句柄保存一份当前辅助功能快照，该句柄的所有应用对象和窗口对象共享这份快照，包括对同一应用的重复选择。`getWindow` 静默执行首次树读取；当 provider 报告应用级辅助功能时，`getApp` 也静默读取。之后的 `getAccessibilityTree` 读取会在分派前使上一份快照失效。门面会为元素操作自动附加最新快照 ID；调用方应使用最新树中的元素 ID，不应假定旧 ID 仍指向同一控件。元素 ID 是 provider 定义的不透明字符串。只有显式 `CuaWindow` 可以截图或发送指针输入。原生 helper 在该门面之后保留自己的应用快照和可选窗口快照；原生层与门面快照相互独立。外部运行时继续负责变量持久化和最终表达式序列化；门面不加入 evaluator。[原生对象决定](2026-09-17-experimental-cua-object-api.zh.md)负责 evaluator 独立性，[共享 helper 决定](2026-10-09-cua-shared-native-helper.zh.md)负责连接生命期。 @@ -24,7 +24,7 @@ Provider 的启动能力是静态的。平台、支持的方法、应用级辅 **在共享基类中加入应用专属分支**会让窗口对象继承只适用于应用的决定和能力检查。由各作用域工厂注入自己的要求，可以让共享操作不依赖使用它的作用域。 -**打包的组合键字符串**需要一起解析按键名、分隔符和加号转义，并把平台别名归一化推入共享层。击键与方向是两种语义：`pressKey` 描述一次完整击键，`sendKeyEvent` 描述一个显式方向。分开的按键和修饰键值让这些语义无需解析器。 +**打包的组合键字符串**需要一起解析按键名、分隔符和加号转义，并把平台别名归一化推入共享层。击键与方向是两种语义：`pressKey` 在一次调用中完成按下和释放，`sendKeyEvent` 描述一个显式方向。分开的按键和修饰键值让这些语义无需解析器。 **在共享 schema 中枚举平台键盘按键**会把可移植的操作 schema 绑定到某个 provider 的按键名，并让每个平台都修改 schema。由 provider 定义字符串，平台可以解释自己的名称，共享 schema 保持中立。 diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml index a76d4de320..c3d5791588 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml @@ -12,5 +12,5 @@ en: e5d1027faa55adba zh: 4b88ad6a70977fb3 /native-cua-application-and-window-api/migration: - en: b09e3dde32391786 - zh: f702e1e25b260b29 + en: 4cf8b18af0d2bb79 + zh: 7f490fd9785deffe diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md index 3179dae0d9..363e627e66 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md @@ -18,6 +18,6 @@ Providers supply startup capabilities alongside their platform guidance. The nat 1. After `cua.getApp(selector)`, select the target from `app.getWindows()` with `app.getWindow(windowId)`. Use `window.getScreenshot()`, `window.click()`, `window.scroll()`, and `window.drag()`. Replace `hover` with `window.mouseMove()`. On macOS, `window.click(id)` sends mouse events at the element center. Switch to `invoke(id)` for the default accessibility action, or `performAction(id, "AXShowMenu")` to request an advertised menu action. 2. Replace `getAXState()` with `getAccessibilityTree()`. Replace `getAXStateAndScreenshot()` with separate tree and screenshot reads. Selection reads the initial tree silently for a window, and for an app when `appLevelAccessibility` is true. Read the tree explicitly to display it. 3. Pass the opaque string ID from the latest tree wherever you passed a numeric element index. In raw requests use `elementId` with `snapshotId`. Replace `performSecondaryAction` with `listActions(id)` and `performAction(id, exactName, argument?)`, or use `invoke(id)` for the default action. `setValue` accepts the provider’s supported JSON values. On macOS, `performAction(id, 'AXRaise')` performs the advertised raise action. Call `app.activate()` first when the application must become active. -4. Replace a packed chord such as `pressKey('shift+Tab')` with `pressKey('Tab', ['shift'])`. Check `(await cua.getCapabilities()).methods` before using `sendKeyEvent`: macOS supports complete keystrokes and rejects directional events with `NOT_IMPLEMENTED`. +4. Replace a packed chord such as `pressKey('shift+Tab')` with `pressKey('Tab', ['shift'])`. Check `(await cua.getCapabilities()).methods` before using `sendKeyEvent`: macOS supports complete keystrokes and rejects directional events with `NOT_IMPLEMENTED`. If your script handles unknown macOS key names as `NATIVE_ERROR`, handle `INVALID_ARGUMENT` instead. Correct the key name before retrying. 5. Read `dispatched` from action results, then observe the application to verify the outcome. `invoke` returns `dispatched: false` when the element has no default action. Remove reads of the old `verified` field and of `pid` on selected objects or window records. Use `listApps()` or running `findApps()` matches when you need a PID. 6. For a custom provider, replace `CuaBackend.description` with `capabilities` and pass `guidance` and `permissionParameters` to `installCua`. Rebuild any custom `helperPath` against the [operation schema](../../../../packages/experimental/cua/src/protocol.ts) and [tagged permission response](../../../../packages/experimental/cua/src/access-types.ts). Restart the provider, inspect its advertised methods and keyboard scopes, and open the Computer Use permission page to verify its platform controls. diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md index f0a4a905e0..272e4a96d4 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md @@ -18,6 +18,6 @@ CUA 脚本在指定窗口上截图和操作鼠标。App 和 Window 对象共享 1. 调用 `cua.getApp(selector)` 后，从 `app.getWindows()` 返回的列表选定目标，再调用 `app.getWindow(windowId)`。在窗口上调用 `getScreenshot()`、`click()`、`scroll()` 和 `drag()`，将原有的 `hover` 调用改为 `window.mouseMove()`。在 macOS 上，`window.click(id)` 在元素中心发送鼠标事件。需要执行默认辅助功能动作时，改用 `invoke(id)`。需要打开控件菜单且控件声明了 `AXShowMenu` 时，调用 `performAction(id, "AXShowMenu")`。 2. 将 `getAXState()` 改为 `getAccessibilityTree()`，将 `getAXStateAndScreenshot()` 拆成读取树和截图两次调用。选择窗口时会静默读取初始树。选择应用时，只有 `appLevelAccessibility` 为真才读取。要显示树，请显式读取。 3. 将数字元素索引改为从最新树中取得的字符串 ID。直接发送请求时使用 `elementId` 和 `snapshotId`。原先调用 `performSecondaryAction` 的地方，先用 `listActions(id)` 取得完整动作名，再调用 `performAction(id, exactName, argument?)`。要执行默认动作，可调用 `invoke(id)`。`setValue` 接受提供方支持的 JSON 值。 在 macOS 上，`performAction(id, 'AXRaise')` 只执行元素声明的抬起窗口动作。需要把应用切到前台时先调用 `app.activate()`。 -4. 将 `pressKey('shift+Tab')` 之类的组合键写法改为 `pressKey('Tab', ['shift'])`。调用 `sendKeyEvent` 前检查 `(await cua.getCapabilities()).methods`。macOS 支持完整按键，单独发送按下或松开事件会返回 `NOT_IMPLEMENTED`。 +4. 将 `pressKey('shift+Tab')` 之类的组合键写法改为 `pressKey('Tab', ['shift'])`。调用 `sendKeyEvent` 前检查 `(await cua.getCapabilities()).methods`。macOS 支持完整按键，单独发送按下或松开事件会返回 `NOT_IMPLEMENTED`。如果脚本用 `NATIVE_ERROR` 识别 macOS 上的未知键名，请改为处理 `INVALID_ARGUMENT`，并在重试前修正键名。 5. 从动作结果读取 `dispatched`，再观察应用确认效果。元素没有默认动作时，`invoke` 返回 `dispatched: false`。移除读取旧字段 `verified` 的代码，以及对已选对象和窗口记录中 `pid` 的读取。需要 PID 时，使用 `listApps()` 或 `findApps()` 返回的运行中应用记录。 6. 自定义提供方需将 `CuaBackend.description` 改为 `capabilities`，并向 `installCua` 传入 `guidance` 和 `permissionParameters`。如配置了 `helperPath`，请按[操作协议](../../../../packages/experimental/cua/src/protocol.ts)和[带平台标记的权限响应](../../../../packages/experimental/cua/src/access-types.ts)重新构建 helper。重启提供方，检查它声明的方法和键盘输入范围，再打开 Computer Use 权限页确认平台控件。 diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index bcc12b8ecc..6a38002c4f 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -15,8 +15,8 @@ en: 1777e7bf6785f986 zh: e3daac1074ebb74d /deepseek-ai-dsh-experimental-cua-macos/use-this-package: - en: e3643640ef764471 - zh: 5d25e833cb78bf9b + en: 24ea893846cf0914 + zh: dca284e8e589274d /deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment: en: b734a0907747580e zh: 8110c5951b70e100 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 63e1c495d2..60d57c749d 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -79,7 +79,7 @@ Source builds default to `sourceSigning: ad-hoc`, without creating certificates The [configuration catalog](../../../docs/config-catalog.md#deepseek-aidsh-experimental-cua-macos) lists all fields. Native tool calls use `{method, params}`; the complete method-specific schema is exposed to the model. `getApp` returns the `appHandle` used by every app-targeted operation, and `app.getWindow(windowId)` returns the object that owns screenshots and pointer input. -`pressKey(key, modifiers?)` accepts one key or a nonempty array of distinct keys. `pressKey(['w', 'd'])` presses W then D and releases D then W. Pass modifier names separately, for example `pressKey('Tab', ['shift'])` or `pressKey('+')`; an omitted modifier array means none. The macOS helper accepts `command`/`cmd`, `shift`, `option`/`alt`, `control`/`ctrl`, and `fn`, and maps them to their physical keys. Key names are platform-defined; keypad names (`KP_0` through `KP_9`, `KP_Enter`, and keypad operators) retain their physical key codes. `typeText` sends literal text, including its line breaks and tabs. On macOS, use `pressKey` on the App object. The shared `sendKeyEvent` operation returns `NOT_IMPLEMENTED` on this provider. +`pressKey(key, modifiers?)` accepts one key or a nonempty array of distinct physical keys. Aliases such as `Enter` and `Return` identify the same key. `pressKey(['w', 'd'])` presses W then D and releases D then W. Pass held modifiers separately, as in `pressKey('Tab', ['shift'])`. Pressing a modifier itself, as in `pressKey('shift')`, sets its flag on press and clears it on release. A single `+` or `plus` sends Shift+=. In a multi-key chord, use `=` and pass Shift explicitly to apply it to the whole chord. The macOS helper accepts `command`/`cmd`, `shift`, `option`/`alt`, `control`/`ctrl`, and `fn`, and maps them to their physical keys. Key names are platform-defined; keypad names (`KP_0` through `KP_9`, `KP_Enter`, and keypad operators) retain their physical key codes. `typeText` sends literal text, including its line breaks and tabs. On macOS, use `pressKey` on the App object. The shared `sendKeyEvent` operation returns `NOT_IMPLEMENTED` on this provider. Agent disposal and plugin unload cancel and await the complete tool call, including image admission and attachment storage. An image write already in progress must settle before disposal returns; a canceled call cannot return a successful screenshot. @@ -221,7 +221,7 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — press one key or hold distinct keys together. pressKey(['w', 'd']) presses W then D and releases D then W. Pass modifiers separately, as in pressKey('Tab', ['shift']). Key and modifier names are platform-defined strings. +- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. ``` diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index 276f725cb3..cadefe0dd6 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -79,7 +79,7 @@ pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis. [配置目录](../../../docs/config-catalog.zh.md#deepseek-aidsh-experimental-cua-macos)列出全部字段。原生工具调用使用 `{method, params}`，模型可以看到每个方法的完整参数定义。`getApp` 返回每个面向应用的操作所需的 `appHandle`，`app.getWindow(windowId)` 返回负责截图和指针输入的对象。 -`pressKey(key, modifiers?)` 接受单个键名或由不同键名组成的非空数组。例如，`pressKey(['w', 'd'])` 依次按下 W、D，再依次松开 D、W。修饰键名称单独传入，例如 `pressKey('Tab', ['shift'])` 或 `pressKey('+')`；省略修饰键数组表示不按修饰键。macOS helper 接受 `command`/`cmd`、`shift`、`option`/`alt`、`control`/`ctrl` 和 `fn`，并映射到对应物理键。按键名由平台定义；数字键盘名称（`KP_0` 到 `KP_9`、`KP_Enter` 和数字键盘运算键）保留独立的物理键码。`typeText` 发送字面文本，包括其中的换行和制表符。在 macOS 上，通过 App 对象调用 `pressKey`。调用共享接口 `sendKeyEvent` 会收到 `NOT_IMPLEMENTED`。 +`pressKey(key, modifiers?)` 接受单个键名或由不同物理键组成的非空数组。`Enter` 和 `Return` 等别名指向同一个物理键。例如，`pressKey(['w', 'd'])` 依次按下 W、D，再依次松开 D、W。需要全程按住的修饰键单独传入，例如 `pressKey('Tab', ['shift'])`。单独调用 `pressKey('shift')` 时，按下事件带有 Shift 标志，释放事件清除该标志。单独传入 `+` 或 `plus` 会发送 Shift 和等号。在多键组合中请使用 `=`，需要全程按住 Shift 时将其列入修饰键。macOS helper 接受 `command`/`cmd`、`shift`、`option`/`alt`、`control`/`ctrl` 和 `fn`，并映射到对应物理键。按键名由平台定义；数字键盘名称（`KP_0` 到 `KP_9`、`KP_Enter` 和数字键盘运算键）保留独立的物理键码。`typeText` 发送字面文本，包括其中的换行和制表符。在 macOS 上，通过 App 对象调用 `pressKey`。调用共享接口 `sendKeyEvent` 会收到 `NOT_IMPLEMENTED`。 Agent 销毁和插件卸载会取消并等待完整工具调用，包括图像接入检查和附件存储。已经开始的图像写入必须结束后，销毁才会返回；已取消的调用不会返回成功截图。 @@ -221,7 +221,7 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — press one key or hold distinct keys together. pressKey(['w', 'd']) presses W then D and releases D then W. Pass modifiers separately, as in pressKey('Tab', ['shift']). Key and modifier names are platform-defined strings. +- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. ``` diff --git a/packages/experimental/cua-macos/native/README.i18n.yaml b/packages/experimental/cua-macos/native/README.i18n.yaml index 6e9294f0ce..7382f3006b 100644 --- a/packages/experimental/cua-macos/native/README.i18n.yaml +++ b/packages/experimental/cua-macos/native/README.i18n.yaml @@ -21,11 +21,11 @@ en: 88a59904ca4ebedc zh: 14f2b666d7382145 /dsh-cua-native-helper/observation-and-input: - en: d19180b1a6db2e48 - zh: c6279db9957218eb + en: 51a70780a0672272 + zh: e7ca17dd40b887aa /dsh-cua-native-helper/verification: - en: 49f4cce2a4cc3114 - zh: e9d65c92ef35fe24 + en: ade5a2ccd3aeb1b4 + zh: 74c2b0bcb23fc4bc /dsh-cua-native-helper/further-exploration: en: 265e1927952ce6d7 zh: 7c35d9fd98b6e6c8 diff --git a/packages/experimental/cua-macos/native/README.md b/packages/experimental/cua-macos/native/README.md index b94d053746..2437c220a4 100644 --- a/packages/experimental/cua-macos/native/README.md +++ b/packages/experimental/cua-macos/native/README.md @@ -103,7 +103,7 @@ Clicks send mouse-down and mouse-up events at the element center or the supplied Capture and pointer delivery read CG bounds before and after an exact-window AX frame query. Movement during the query returns WINDOW_CHANGED. Valid AX/CG frames must have equal outward-rounded integral point rectangles; a known mismatch returns WINDOW_CHANGED before mouse movement, priming or button delivery. The helper preserves both coordinate sources rather than substituting AX geometry for CG geometry. Missing or unusable AX geometry remains unknown, while AX communication, permission and invalid-element errors propagate. Capture without Accessibility permission retains its CG/SCK checks. Advertised AX actions bypass this pointer preflight; an AX action error does not trigger pointer input. These reads do not lock the window against later movement, and no geometry check activates or raises it. -`typeText` and `pressKey` target the selected process's current focused control, not an exact window. `pressKey` presses its modifiers followed by the requested keys in order, then releases all pressed keys in reverse order, including on cancellation; these events do not use the global HID queue. Key names are platform-defined; keypad names such as `KP_0` and `KP_Enter` preserve their physical key codes. Background pointer actions may change that keyboard focus. Text injection sends Unicode characters individually and does not use the clipboard. LF, CR and CRLF send one Return; tabs send Tab. Applications apply their normal editing behavior, including indentation and focus navigation. Each `typeText` down/up event is followed by the configured text interval; `pressKey` keeps its own press and release timing, and dispatch completion does not confirm that the receiving application consumed every event. Cancellation can interrupt text or a gesture after partial input; it cannot undo received events. Very long text can exceed the host's operation deadline. Validate observed application state before deciding how to continue. +`typeText` and `pressKey` target the selected process's current focused control, not an exact window. `pressKey` presses its modifiers followed by the requested keys in order, then releases all pressed keys in reverse order, including on cancellation. Keyboard events go directly to the target PID and bypass the global HID queue. Key names are platform-defined; keypad names such as `KP_0` and `KP_Enter` preserve their physical key codes. Background pointer actions may change that keyboard focus. Text injection sends Unicode characters individually and does not use the clipboard. LF, CR and CRLF send one Return; tabs send Tab. Applications apply their normal editing behavior, including indentation and focus navigation. Each `typeText` down/up event is followed by the configured text interval; `pressKey` keeps its own press and release timing, and dispatch completion does not confirm that the receiving application consumed every event. Cancellation can interrupt text or a gesture after partial input; it cannot undo received events. Very long text can exceed the host's operation deadline. Validate observed application state before deciding how to continue. The selected helper application needs macOS Accessibility permission for AX state and input, and Screen Recording permission for screenshots. Capability preflight and permission errors do not grant permissions. Capture refuses a window that moves or resizes while the image is being read. Unassociated system panels and accessory-only applications are not selectable. @@ -157,7 +157,7 @@ For text pacing comparisons, use `native/tests/text_input_benchmark.py` with `-- For an independently isolated Electron/Chromium check, run `python3 packages/experimental/cua-macos/native/tests/electron_e2e.py --electron /absolute/path/to/Electron.app/Contents/MacOS/Electron`. Omitting `--electron` uses an already installed desktop workspace runtime. The check never downloads Electron. Its renderer runs with Node integration disabled, context isolation and Chromium sandboxing enabled, private userData/sessionData/log directories, and a local fixture page. Renderer click/input events write independent observations through the main process; all test actions still use the native helper. AX polling waits for actual controls after Chromium accessibility activation. Native and Electron subprocess groups are owned and stopped before temporary files are removed. Optional artifacts include a fixture-only screenshot and private request transcript. -Run `python3 packages/experimental/cua-macos/native/tests/capabilities_e2e.py --helper /absolute/helper --artifacts /private/evidence` to verify a stopped application's discovery and launch, Unicode selection, multiline plain/HTML paste, middle/modifier gestures and keypad identities through an isolated AppKit receiver. Add `--click-only` to check click holds, AX routing and cancellation/timeout release in a background fixture without keyboard or clipboard input. It requires existing Accessibility/Screen Recording grants and records the receiver's actual state separately from dispatch results. +Run `python3 packages/experimental/cua-macos/native/tests/capabilities_e2e.py --helper /absolute/helper --artifacts /private/evidence` to verify a stopped application's discovery and launch, Unicode selection, multiline plain/HTML paste, middle/modifier gestures, overlapping W and D key events, and keypad identities through an isolated AppKit receiver. Add `--click-only` to check click holds, AX routing and cancellation/timeout release in a background fixture without keyboard or clipboard input. It requires existing Accessibility/Screen Recording grants and records the receiver's actual state separately from dispatch results. Run `python3 packages/experimental/cua-macos/native/tests/paste_e2e.py --artifacts /private/evidence` to verify that an owned AppKit app accepts a background Paste action without reading its data, reports `PASTE_NOT_CONSUMED`, then consumes the same paste in the foreground. The test requires an unlocked Mac and Accessibility permission, activates only its own two applications, and records received actions and text independently of helper replies. diff --git a/packages/experimental/cua-macos/native/README.zh.md b/packages/experimental/cua-macos/native/README.zh.md index 7ddde7d4f4..1224e0e6db 100644 --- a/packages/experimental/cua-macos/native/README.zh.md +++ b/packages/experimental/cua-macos/native/README.zh.md @@ -103,7 +103,7 @@ AppKit 在主事件循环上运行。阻塞式 socket 接收与握手使用后 截图和鼠标输入在精确窗口的 AX 边界查询前后读取 CG 边界，查询期间移动会返回 WINDOW_CHANGED。有效的 AX/CG 边界向外取整后必须一致；已知不一致会在移动、预备事件或按键发送前返回 WINDOW_CHANGED。helper 保留两个坐标来源，不用 AX 几何替换 CG 几何。缺失或无效的 AX 几何仍表示未知，AX 通信、权限和无效元素错误继续传递。无辅助功能权限的截图保留 CG/SCK 检查。声明的 AX 动作不经过鼠标预检，AX 动作失败也不改用鼠标输入。这些读取不会锁住窗口以阻止后续移动，几何检查不激活或抬起窗口。 -`typeText` 和 `pressKey` 面向所选进程当前获得焦点的控件，不指定窗口。`pressKey` 向目标 PID 依次发送修饰键和指定按键的按下事件，再逆序释放。操作中途取消也会逆序释放已按下的键。按键名由平台定义；KP_0 和 KP_Enter 等数字键盘名称保留物理键码。后台鼠标操作可能改变键盘焦点。文本逐个 Unicode 字符输入，不使用剪贴板。LF、CR 和 CRLF 发送一次 Return，制表符发送 Tab。应用保留缩进和焦点导航等正常编辑行为。`typeText` 的每次按下和抬起后等待配置的文本事件间隔；`pressKey` 使用自己的按下与释放节奏，分派完成不证明应用已消费全部事件。取消可能中断部分文本或手势，无法撤销已接收事件。很长的文本可能超过 Host 操作期限。继续输入前应核实应用状态。 +`typeText` 和 `pressKey` 面向所选进程当前获得焦点的控件，不指定窗口。`pressKey` 向目标 PID 依次发送修饰键和指定按键的按下事件，再逆序释放。操作中途取消也会逆序释放已按下的键。键盘事件直接发给目标进程，绕过全局 HID 队列。按键名由平台定义；KP_0 和 KP_Enter 等数字键盘名称保留物理键码。后台鼠标操作可能改变键盘焦点。文本逐个 Unicode 字符输入，不使用剪贴板。LF、CR 和 CRLF 发送一次 Return，制表符发送 Tab。应用保留缩进和焦点导航等正常编辑行为。`typeText` 的每次按下和抬起后等待配置的文本事件间隔；`pressKey` 使用自己的按下与释放节奏，分派完成不证明应用已消费全部事件。取消可能中断部分文本或手势，无法撤销已接收事件。很长的文本可能超过 Host 操作期限。继续输入前应核实应用状态。 所选 helper 应用需要 macOS 辅助功能权限以读取 AX 和输入，需要屏幕录制权限以截图。能力预检和权限错误不授予权限。截图会拒绝在读取图片期间移动或缩放的窗口。无法确认归属的系统面板及仅后台辅助类型的应用不可选择。 @@ -157,7 +157,7 @@ Blender 键盘检查使用 `python3 packages/experimental/cua-macos/native/tests 隔离的 Electron/Chromium 检查使用 `python3 packages/experimental/cua-macos/native/tests/electron_e2e.py --electron /absolute/path/to/Electron.app/Contents/MacOS/Electron`。省略 `--electron` 时使用桌面工作区已安装的运行时，检查不下载 Electron。渲染器禁用 Node 集成，启用上下文隔离和 Chromium 沙箱，使用独立 userData/sessionData/log 目录及本地 fixture 页面。渲染器的点击/输入事件通过主进程写出独立观察，测试动作仍通过原生 helper 执行。AX 轮询等待 Chromium 辅助功能启用后的真实控件。原生和 Electron 子进程组由测试持有，在删除临时文件前停止。可选产物包含仅有 fixture 的截图和私有请求记录。 -运行 `python3 packages/experimental/cua-macos/native/tests/capabilities_e2e.py --helper /absolute/helper --artifacts /private/evidence`，通过隔离的 AppKit 接收方验证未运行应用的发现和启动、Unicode 选区、多行纯文本/HTML 粘贴、中键/修饰键手势及数字键盘身份。需要已有辅助功能/屏幕录制权限，接收方实际状态与分派结果分开记录。 添加 `--click-only` 可在后台测试窗口中验证按住时长、AX 路由和取消、超时后的松开事件，不发送键盘或剪贴板输入。 +运行 `python3 packages/experimental/cua-macos/native/tests/capabilities_e2e.py --helper /absolute/helper --artifacts /private/evidence`，通过隔离的 AppKit 接收方验证未运行应用的发现和启动、Unicode 选区、多行纯文本/HTML 粘贴、带修饰键的中键手势、W 和 D 同时按住时的事件顺序及数字键盘键码。需要已有辅助功能/屏幕录制权限，接收方实际状态与分派结果分开记录。 添加 `--click-only` 可在后台测试窗口中验证按住时长、AX 路由和取消、超时后的松开事件，不发送键盘或剪贴板输入。 运行 `python3 packages/experimental/cua-macos/native/tests/paste_e2e.py --artifacts /private/evidence`，验证自有 AppKit 应用在后台接收 Paste 动作但不读取数据时返回 `PASTE_NOT_CONSUMED`，以及前台读取同一次粘贴后成功。测试要求 Mac 已解锁并已授予辅助功能权限，只激活自有的两个应用，并独立记录收到的动作和文本以核对 helper 回复。 diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift index 2dbf648ecf..3511256457 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift @@ -22,23 +22,32 @@ final class KeyboardInput { /// Prepare every event before dispatch. On cancellation, release only the keys this call pressed. func pressKey(_ keys: [String], modifiers: [String], to pid: Int32) throws { try checkCancellation() - var modifierNames = modifiers - let mainKeys = try keys.map { name -> Key in + var chord = try modifiers.map(Self.modifier) + var namesByCode: [CGKeyCode: String] = [:] + for name in keys { + let key: Key if name == "+" || name.lowercased() == "plus" { - modifierNames.append("shift") - return Key(keyCode: try Self.keyCode("="), flag: []) + guard keys.count == 1 else { + throw RequestParameters.invalid("For a multi-key chord, use = with an explicit shift modifier instead of \(name)") + } + chord.append(try Self.modifier("shift")) + key = Key(keyCode: try Self.keyCode("="), flag: []) + } else if let modifier = Self.namedModifier(name) { + key = modifier + } else { + key = Key(keyCode: try Self.keyCode(name), flag: []) } - if let modifier = Self.namedModifier(name) { return modifier } - return Key(keyCode: try Self.keyCode(name), flag: []) - } - guard Set(mainKeys.map(\.keyCode)).count == mainKeys.count else { - throw RequestParameters.invalid("A chord must contain distinct physical keys") + if let previous = namesByCode.updateValue(name, forKey: key.keyCode) { + throw RequestParameters.invalid("Keys '\(previous)' and '\(name)' name the same physical key") + } + chord.append(key) } var selected = Set() var flags: CGEventFlags = [] var events: [(down: CGEvent, up: CGEvent)] = [] let source = try Self.source() - for key in try modifierNames.map(Self.modifier) + mainKeys where selected.insert(key.keyCode).inserted { + for key in chord { + guard selected.insert(key.keyCode).inserted else { continue } let releasedFlags = flags flags.formUnion(key.flag) events.append(( @@ -243,7 +252,7 @@ final class KeyboardInput { case "m": return 46 case ".": return 47 case "`": return 50 - default: throw NativeFailure(code: "NATIVE_ERROR", message: "Unknown key name: \(key)") + default: throw RequestParameters.invalid("Unknown key name: \(key)") } } } diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift index ef2b6dc101..0041e88b2a 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift @@ -37,13 +37,11 @@ struct RequestParameters: Sendable { } } func pressKeys() throws -> [String] { - if let name = values["key"]?.string { - guard !name.isEmpty else { throw Self.invalid("key must not be empty") } - return [name] + switch values["key"] { + case .string(let name) where !name.isEmpty: return [name] + case .array(let items) where !items.isEmpty: return try strings("key") + default: throw Self.invalid("key must be a nonempty string or a nonempty array of key names") } - let names = try strings("key") - guard !names.isEmpty else { throw Self.invalid("key must name at least one key") } - return names } func optionalString(_ key: String) throws -> String? { contains(key) ? try string(key) : nil } func optionalBoolean(_ key: String) throws -> Bool? { diff --git a/packages/experimental/cua-macos/native/protocol.schema.json b/packages/experimental/cua-macos/native/protocol.schema.json index aa6d201dcb..ce960cea7e 100644 --- a/packages/experimental/cua-macos/native/protocol.schema.json +++ b/packages/experimental/cua-macos/native/protocol.schema.json @@ -1006,7 +1006,8 @@ "minLength": 1 } } - ] + ], + "description": "One key name or distinct physical keys to press in order and release in reverse." }, "modifiers": { "type": "array", diff --git a/packages/experimental/cua-macos/native/tests/capabilities_e2e.py b/packages/experimental/cua-macos/native/tests/capabilities_e2e.py index 95a94c97b6..f9c280a193 100644 --- a/packages/experimental/cua-macos/native/tests/capabilities_e2e.py +++ b/packages/experimental/cua-macos/native/tests/capabilities_e2e.py @@ -188,15 +188,14 @@ def main(): if not args.click_only: call("click", {**window, **point, "mouseButton": "middle", "modifiers": ["shift"]}) state = wait_for_state(state_file, lambda value: any(event["type"] == 26 and event.get("button") == 2 and event["shift"] for event in value["events"])) + count = len(state["events"]) + call("pressKey", {**owner, "key": ["w", "d"]}) + state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 4) + chord = [(event["type"], event.get("keyCode")) for event in state["events"][count:]] + assert chord == [(10, 13), (10, 2), (11, 2), (11, 13)], chord call("drag", {**window, "fromX": point["x"], "fromY": point["y"], "toX": point["x"] + 80, "toY": point["y"] + 20, "durationMs": 200, "steps": 10, "mouseButton": "middle", "modifiers": ["shift"]}) state = wait_for_state(state_file, lambda value: any(event["type"] == 27 and event.get("button") == 2 and event["shift"] for event in value["events"])) - call("pressKey", {**owner, "key": ["w", "d"]}) - expected_chord = [(10, 13), (10, 2), (11, 2), (11, 13)] - state = wait_for_state(state_file, lambda value: any( - event["type"] == 11 and event.get("keyCode") == 13 for event in value["events"])) - assert [(event["type"], event["keyCode"]) for event in state["events"] - if event.get("keyCode") in (13, 2)] == expected_chord, state["events"] call("pressKey", {**owner, "key": "KP_0"}) call("pressKey", {**owner, "key": "KP_Enter"}) state = wait_for_state(state_file, lambda value: {82, 76}.issubset({event.get("keyCode") for event in value["events"]})) @@ -207,7 +206,7 @@ def main(): (args.artifacts / "capabilities.png").write_bytes(base64.b64decode(shot["data"])) (args.artifacts / "state.json").write_text(json.dumps(state, ensure_ascii=False, indent=2)) print("PASS: click holds and cancellation/timeout/disconnect release" if args.click_only else - "PASS: stopped-app launch, Unicode selection, plain/HTML paste, middle/modifier drag, keypad identity and click holds") + "PASS: stopped-app launch, Unicode selection, plain/HTML paste, middle/modifier drag, overlapping W+D events, keypad identity and click holds") finally: if pid is None and state_file.exists(): pid = json.loads(state_file.read_text())["pid"] diff --git a/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift b/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift index ae98c891d6..6ca44cd59a 100644 --- a/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift +++ b/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift @@ -3,14 +3,19 @@ import CoreGraphics import Testing @testable import CuaNative -@Test func chordKeepsModifiersThroughBothLetterReleases() throws { +@Test(arguments: [ + (["w", "d"], ["shift"], [56, 13, 2, 2, 13, 56]), + (["shift", "w", "d"], [], [56, 13, 2, 2, 13, 56]), + (["a", "="], ["shift"], [56, 0, 24, 24, 0, 56]), +]) +func chordKeepsShiftThroughBothKeyReleases(_ keys: [String], _ modifiers: [String], _ codes: [Int64]) throws { var events: [CGEvent] = [] let input = KeyboardInput(post: { event, pid in #expect(pid == 42) events.append(event) }, textEventInterval: 0) - try input.pressKey(["w", "d"], modifiers: ["shift"], to: 42) - #expect(events.map { $0.getIntegerValueField(.keyboardEventKeycode) } == [56, 13, 2, 2, 13, 56]) + try input.pressKey(keys, modifiers: modifiers, to: 42) + #expect(events.map { $0.getIntegerValueField(.keyboardEventKeycode) } == codes) #expect(events.map(\.type) == [.flagsChanged, .keyDown, .keyDown, .keyUp, .keyUp, .flagsChanged]) #expect(events.map { $0.flags.contains(.maskShift) } == [true, true, true, true, true, false]) } @@ -29,8 +34,37 @@ func cancelledChordReleasesOnlyPressedKeys(_ cancelAfter: Int) { #expect(events.map(\.type) == Array(repeating: .keyDown, count: cancelAfter) + Array(repeating: .keyUp, count: cancelAfter)) } -@Test(arguments: [["w", "unknown"], ["w", "W"], ["Enter", "Return"]]) -func invalidChordPostsNoInput(_ keys: [String]) { +@Test(arguments: [ + (["w", "unknown"], "Unknown key name: unknown"), + (["w", "W"], "Keys 'w' and 'W' name the same physical key"), + (["Enter", "Return"], "Keys 'Enter' and 'Return' name the same physical key"), + (["a", "+"], "For a multi-key chord, use = with an explicit shift modifier instead of +"), + (["Plus", "w"], "For a multi-key chord, use = with an explicit shift modifier instead of Plus"), +]) +func invalidChordPostsNoInput(_ keys: [String], _ message: String) throws { let input = KeyboardInput(post: { _, _ in Issue.record("Invalid chord posted input") }, textEventInterval: 0) - #expect(throws: NativeFailure.self) { try input.pressKey(keys, modifiers: ["shift"], to: 42) } + do { + try input.pressKey(keys, modifiers: ["shift"], to: 42) + Issue.record("Invalid chord was accepted") + } catch let error as NativeFailure { + #expect(error.code == "INVALID_ARGUMENT") + #expect(error.message == message) + } +} + +@Test(arguments: ["+", "plus"]) +func plusAlonePressesShiftEquals(_ alias: String) throws { + var events: [CGEvent] = [] + let input = KeyboardInput(post: { event, _ in events.append(event) }, textEventInterval: 0) + try input.pressKey([alias], modifiers: [], to: 42) + #expect(events.map { $0.getIntegerValueField(.keyboardEventKeycode) } == [56, 24, 24, 56]) + #expect(events.map { $0.flags.contains(.maskShift) } == [true, true, true, false]) +} + +@Test func standaloneShiftSetsItsFlagOnlyWhileHeld() throws { + var events: [CGEvent] = [] + let input = KeyboardInput(post: { event, _ in events.append(event) }, textEventInterval: 0) + try input.pressKey(["shift"], modifiers: [], to: 42) + #expect(events.map(\.type) == [.flagsChanged, .flagsChanged]) + #expect(events.map { $0.flags.contains(.maskShift) } == [true, false]) } diff --git a/packages/experimental/cua-macos/native/tests/swift/ProtocolTests.swift b/packages/experimental/cua-macos/native/tests/swift/ProtocolTests.swift index 84a9fec849..85fa37aaf0 100644 --- a/packages/experimental/cua-macos/native/tests/swift/ProtocolTests.swift +++ b/packages/experimental/cua-macos/native/tests/swift/ProtocolTests.swift @@ -138,3 +138,15 @@ func rejectsMalformedApplicationIconPaths(_ value: String) throws { let request = try NativeRequest(data: Data("{\"id\":\"icons\",\"method\":\"applicationIcons\",\"params\":{\"paths\":\(value)}}".utf8)) #expect(throws: NativeFailure.self) { try RequestParameters(request).strings("paths") } } + +@Test func invalidPressKeyTypeNamesBothAcceptedForms() throws { + let request = try NativeRequest(data: Data(#"{"id":"call","method":"pressKey","params":{"key":7}}"#.utf8)) + let params = try RequestParameters(request) + do { + _ = try params.pressKeys() + Issue.record("Numeric key was admitted") + } catch let error as NativeFailure { + #expect(error.code == "INVALID_ARGUMENT") + #expect(error.message == "key must be a nonempty string or a nonempty array of key names") + } +} diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index f8779651a9..c7726f385b 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -13,6 +13,7 @@ const methodDetails = new Map([ export const GUIDANCE = `${PLATFORM_GUIDANCE} ${COMMON_GUIDANCE.replace(/^- ([\w.]+)\(.*$/gm, (entry, method: string) => { + if (method === 'pressKey') return "- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed." const detail = methodDetails.get(method) return detail ? `${entry} ${detail}` : entry })}` diff --git a/packages/experimental/cua-macos/tests/guidance.spec.ts b/packages/experimental/cua-macos/tests/guidance.spec.ts index f3e530b7de..16fb39122a 100644 --- a/packages/experimental/cua-macos/tests/guidance.spec.ts +++ b/packages/experimental/cua-macos/tests/guidance.spec.ts @@ -27,6 +27,10 @@ it('states the macOS accessibility, keyboard, and mouse levels', () => { expect(PLATFORM_GUIDANCE).toContain('accessibility tree operations are available at app or window level') expect(PLATFORM_GUIDANCE).toContain('Simulated keyboard input is available at app level') expect(PLATFORM_GUIDANCE).toContain('Simulated mouse input is available at window level') + const pressKey = GUIDANCE.split('\n').find(line => line.startsWith('- pressKey(')) + expect(pressKey).toContain('key: string | string[]') + expect(pressKey).toContain("pressKey(['w', 'd']) presses W then D and releases D then W") + expect(pressKey).toContain("Packed strings such as 'w+d'") const hover = GUIDANCE.split('\n').find(line => line.startsWith('- mouseMove(')) for (const term of ['AXShowAlternateUI', 'AXShowDefaultUI', 'HOVER_UNAVAILABLE', 'systemPointerHover']) expect(hover).toContain(term) const click = GUIDANCE.split('\n').find(line => line.startsWith('- click(')) @@ -89,7 +93,7 @@ it('preserves image formats and pointer modifier enums without restricting keybo for (const name of ['click', 'drag']) { expect(lines.find(line => line.startsWith(`- ${name}(`))).toContain("modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[]") } - expect(COMMON_GUIDANCE).toContain('pressKey(key: string | string[], modifiers?: string[])') + expect(COMMON_GUIDANCE).toContain('pressKey(key: string, modifiers?: string[])') expect(COMMON_GUIDANCE).toContain('Key and modifier names are platform-defined strings') }) @@ -130,7 +134,7 @@ it('documents facade initialization, emitted observations, and separate key argu expect(COMMON_GUIDANCE).toContain('host-provided bridge initialization once') expect(COMMON_GUIDANCE).toContain('not a preinstalled global') expect(COMMON_GUIDANCE).toContain('void await') - expect(COMMON_GUIDANCE).toContain("pressKey(['w', 'd']) presses W then D and releases D then W") + expect(COMMON_GUIDANCE).toContain('do not pack them into a chord string') expect(COMMON_GUIDANCE).toContain('perform a left-button double-click') }) diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index 0d16bc4b02..8587d8c7d5 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -21,7 +21,7 @@ en: d92d5948a7941e8d zh: d43491ae6521b87e /deepseek-ai-dsh-cua/understand-the-implementation: - en: e38d57709aa6d001 + en: 181158347d09c629 zh: 231b2dcaf826d386 /deepseek-ai-dsh-cua/further-exploration: en: c174fb5a00ea3681 diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index e6c0f0778f..9fb8f64eb7 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -91,7 +91,7 @@ One object facade ([src/api.ts](src/api.ts)) builds a shared `CuaSurface` and de Provider startup capabilities are fixed at installation: platform, methods, app-level accessibility, keyboard scopes, browser support, and coordinate space. The native `getCapabilities` RPC reports runtime permissions and session lock only; `installCua` merges both layers for the model-visible result. The raw RPC remains a common operation representation rather than a capability inventory, so provider guidance states the actions its platform supports. -Check `methods` and `keyboard` in the result of `await cua.getCapabilities()` for the supported operations and input scopes. `pressKey(key, modifiers?)` holds the listed modifiers while pressing the requested keys in order and releasing them in reverse. For example, `pressKey(['w', 'd'])` holds W through the D keystroke; `sendKeyEvent(key, direction)` sends exactly one key-down or key-up. `typeText` and `paste` deliver literal text at the provider's declared keyboard scope. Keyboard modifier names are plain strings that the provider interprets. +Check `methods` and `keyboard` in the result of `await cua.getCapabilities()` for the supported operations and input scopes. `pressKey(key, modifiers?)` holds the listed modifiers while pressing the requested keys in order and releasing them in reverse. For example, `pressKey(['w', 'd'])` holds W through the D keystroke. `sendKeyEvent(key, direction)` sends exactly one key-down or key-up. `typeText` and `paste` deliver literal text at the provider's declared keyboard scope. Keyboard modifier names are plain strings that the provider interprets. Platform providers import `CuaConfigFields` from `@deepseek-ai/dsh-cua/config` to extend the shared configuration. The root Host entry exports `installCua`, `CuaConfig`, and the Typert-discovered `CuaAccess` service; its `apply()` only marks the optional Client contribution. `/client` loads the permission UI through the existing module loader. Separate Host and Client compiler projects keep React, CSS, and browser registrations out of headless imports. `/api` owns the app/window API and its types; `/bridge`, `/native-client`, and `/protocol` retain runtime and platform consumers. `/types` exports only the named types required by generated Remote declarations. The Client uses package-local types. `/typert` supplies Host reflection and `/remote` supplies generated browser descriptors and codecs; permission authority remains on the Host. diff --git a/packages/experimental/cua/src/guidance.ts b/packages/experimental/cua/src/guidance.ts index 0677be5433..745ccfe9be 100644 --- a/packages/experimental/cua/src/guidance.ts +++ b/packages/experimental/cua/src/guidance.ts @@ -43,6 +43,6 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — press one key or hold distinct keys together. pressKey(['w', 'd']) presses W then D and releases D then W. Pass modifiers separately, as in pressKey('Tab', ['shift']). Key and modifier names are platform-defined strings. +- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name.` diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index 8488f32af3..42b545e66a 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -62,7 +62,10 @@ export const Request = z.discriminatedUnion('method', [ selectionType: z.enum(['text', 'cursor_before', 'cursor_after']).optional() }), operation('typeText', { ...app, windowId: id.optional(), text: z.string() }), operation('paste', { ...app, windowId: id.optional(), text: z.string(), format: z.enum(['text', 'html']).optional() }), - operation('pressKey', { ...app, windowId: id.optional(), key: z.union([z.string().min(1), z.array(z.string().min(1)).min(1)]), modifiers: z.array(z.string()).optional() }), + operation('pressKey', { ...app, windowId: id.optional(), + key: z.union([z.string().min(1), z.array(z.string().min(1)).min(1)]) + .describe('One key name or distinct physical keys to press in order and release in reverse.'), + modifiers: z.array(z.string()).optional() }), operation('sendKeyEvent', { ...app, windowId: id.optional(), key: z.string().min(1), direction: z.enum(['down', 'up']) }), z.object({ method: z.literal('scroll'), params: z.union([ z.object({ ...absoluteCoordinate, diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index 875418d805..da1c1eaf56 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -162,7 +162,7 @@ export interface CuaSurface { /** * Press the keys in order at the current focus, then release them in reverse order. * Rejects when the provider does not support keyboard input at this scope. - * @param key - One provider-defined key name or a nonempty array of distinct keys to hold together. + * @param key - One provider-defined key name or a nonempty array of distinct physical keys to hold together. * @param modifiers - Platform- or provider-defined modifier names held for this keystroke; omitted means none. * @returns Whether the provider successfully executed the requested operation. */ diff --git a/packages/experimental/cua/tests/protocol.spec.ts b/packages/experimental/cua/tests/protocol.spec.ts index 50fb329c07..c3d69e309c 100644 --- a/packages/experimental/cua/tests/protocol.spec.ts +++ b/packages/experimental/cua/tests/protocol.spec.ts @@ -12,6 +12,8 @@ const rejectedRequests: Array<{ reason: string; request: JsonValue }> = [ { reason: 'numeric element id', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementId: 0 } } }, { reason: 'coordinate click without a window', request: { method: 'click', params: { appHandle: 'app', x: 1, y: 2 } } }, { reason: 'mixed element and coordinate click', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementId: '0', windowId: 23, x: 1, y: 2 } } }, + { reason: 'empty pressKey chord', request: { method: 'pressKey', params: { appHandle: 'app', key: [] } } }, + { reason: 'non-string pressKey chord member', request: { method: 'pressKey', params: { appHandle: 'app', key: ['w', 7] } } }, { reason: 'non-string pressKey modifier', request: { method: 'pressKey', params: { appHandle: 'app', key: 'a', modifiers: ['command', 7] } } }, { reason: 'sendKeyEvent modifiers', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: 'a', direction: 'down', modifiers: [] } } }, { reason: 'sendKeyEvent without direction', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: 'a' } } }, @@ -23,6 +25,11 @@ it.each(rejectedRequests)('rejects $reason at operation admission', ({ request } expect(Request.safeParse(request).success).toBe(false) }) +it('admits an ordered chord in a raw pressKey request', () => { + const request = { method: 'pressKey', params: { appHandle: 'app', key: ['w', 'd'] } } + expect(Request.parse(request)).toEqual(request) +}) + it('admits arbitrary provider-defined pressKey modifier names', () => { const request = { method: 'pressKey', params: { appHandle: 'app', key: 'PlatformKey', modifiers: ['platform-modifier', 'shift'] } } expect(Request.parse(request)).toEqual(request) diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index a39b9145d4..73107a5ee3 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -78,7 +78,7 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — press one key or hold distinct keys together. pressKey(['w', 'd']) presses W then D and releases D then W. Pass modifiers separately, as in pressKey('Tab', ['shift']). Key and modifier names are platform-defined strings. +- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. diff --git a/snapshots/session/cua-native/tool-schemas.expected.json b/snapshots/session/cua-native/tool-schemas.expected.json index 8f608ec54e..3a94712d06 100644 --- a/snapshots/session/cua-native/tool-schemas.expected.json +++ b/snapshots/session/cua-native/tool-schemas.expected.json @@ -1045,7 +1045,8 @@ "minLength": 1 } } - ] + ], + "description": "One key name or distinct physical keys to press in order and release in reverse." }, "modifiers": { "type": "array", diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index 017e5b103b..14f13a5a00 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -80,7 +80,7 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — press one key or hold distinct keys together. pressKey(['w', 'd']) presses W then D and releases D then W. Pass modifiers separately, as in pressKey('Tab', ['shift']). Key and modifier names are platform-defined strings. +- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. diff --git a/snapshots/session/ptc-script-retained/system-prompt.expected.md b/snapshots/session/ptc-script-retained/system-prompt.expected.md index ef58a12bd2..6e0914ac15 100644 --- a/snapshots/session/ptc-script-retained/system-prompt.expected.md +++ b/snapshots/session/ptc-script-retained/system-prompt.expected.md @@ -80,7 +80,7 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — press one key or hold distinct keys together. pressKey(['w', 'd']) presses W then D and releases D then W. Pass modifiers separately, as in pressKey('Tab', ['shift']). Key and modifier names are platform-defined strings. +- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. From 89d374d5788c4822d86deed427d1db87c8483866 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 13:33:24 +0800 Subject: [PATCH 03/11] feat(cua-macos): hold physical keys for one run_code execution --- ...05-cua-scoped-surfaces-and-input.i18n.yaml | 4 +- ...026-11-05-cua-scoped-surfaces-and-input.md | 2 +- ...-11-05-cua-scoped-surfaces-and-input.zh.md | 2 +- .../cua-native-platform/guide.i18n.yaml | 4 +- .../v0.2.2-rc.1/cua-native-platform/guide.md | 2 +- .../cua-native-platform/guide.zh.md | 2 +- .../experimental/cua-macos/README.i18n.yaml | 4 +- packages/experimental/cua-macos/README.md | 7 +- packages/experimental/cua-macos/README.zh.md | 7 +- .../cua-macos/native/README.i18n.yaml | 8 +- .../experimental/cua-macos/native/README.md | 4 +- .../cua-macos/native/README.zh.md | 4 +- .../native/Sources/CuaNative/AXWorker.swift | 25 ++++-- .../native/Sources/CuaNative/HeldKeys.swift | 84 +++++++++++++++++++ .../Sources/CuaNative/KeyboardInput.swift | 42 ++++++++-- .../CuaNative/NativeProtocol.generated.swift | 3 +- .../Sources/CuaNative/RequestAdmission.swift | 2 +- .../CuaNative/RequestCoordinator.swift | 4 +- .../Sources/CuaNative/RequestScheduler.swift | 1 + .../cua-macos/native/generate-protocol.ts | 1 + .../cua-macos/native/protocol.schema.json | 38 ++++++++- .../native/tests/capabilities_e2e.py | 16 ++++ .../tests/swift/KeyboardChordTests.swift | 22 +++++ .../experimental/cua-macos/src/guidance.ts | 3 +- .../experimental/cua-macos/src/held-keys.ts | 42 ++++++++++ packages/experimental/cua-macos/src/index.ts | 5 +- .../cua-macos/tests/guidance.spec.ts | 5 +- .../tests/provider-lifecycle.spec.ts | 84 +++++++++++++++++-- packages/experimental/cua/src/index.ts | 4 + .../experimental/cua/src/native-client.ts | 23 ++++- packages/experimental/cua/src/protocol.ts | 2 +- packages/experimental/cua/src/types.ts | 2 +- .../cua/tests/native-client.spec.ts | 12 +++ snapshots/session/cua-native/session.v4.jsonl | 2 +- .../cua-native/system-prompt.expected.md | 3 +- .../session/cua-ptc/system-prompt.expected.md | 3 +- .../system-prompt.expected.md | 3 +- 37 files changed, 428 insertions(+), 53 deletions(-) create mode 100644 packages/experimental/cua-macos/native/Sources/CuaNative/HeldKeys.swift create mode 100644 packages/experimental/cua-macos/src/held-keys.ts diff --git a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.i18n.yaml b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.i18n.yaml index adad612610..716f07d599 100644 --- a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.i18n.yaml @@ -15,5 +15,5 @@ en: 2113f17a49e16670 zh: 1df04d9736a3ad4f /agent-note-cua-scoped-surfaces-and-separated-input-capabilities/consequences: - en: 5c6ec60b77f1bff0 - zh: e1ae568ef3501b7b + en: c4b12ca80865e670 + zh: 396426fad0105eaa diff --git a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.md b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.md index 655172fb19..6b4fe604d5 100644 --- a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.md +++ b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.md @@ -34,6 +34,6 @@ Within one API instance, the object facade keeps one current accessibility snaps Element IDs belong to the application's current accessibility snapshot. Reading one window replaces the facade’s current snapshot for that application, so callers observe again and confirm the IDs they use. The new read does not guarantee that an old string is rejected or that an ID changes; callers must not assume an older ID still identifies the same control. Pointer input and screenshots require an explicit window object; callers enumerate windows first. -Keyboard behavior follows the provider's supported scopes. A caller that uses `sendKeyEvent` owns the down/up pairing, because the API never sends the opposite event. Keyboard modifier names are provider-defined, so the same string may have different support across providers. +Keyboard behavior follows the provider's supported scopes. A caller that uses `sendKeyEvent` owns the down/up pairing, during execution. macOS releases that run’s remaining keys when the program ends or the user pauses it. Keyboard modifier names are provider-defined, so the same string may have different support across providers. Provider guidance owns the list of supported actions. Adding a method to the shared schema does not claim that every provider implements it, and the merged `getCapabilities` result remains the runtime source for permissions and session lock. diff --git a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.zh.md b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.zh.md index 495ba2d399..d03c3b4723 100644 --- a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.zh.md +++ b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.zh.md @@ -34,6 +34,6 @@ Provider 的启动能力是静态的。平台、支持的方法、应用级辅 元素 ID 属于应用当前的辅助功能快照。读取一个窗口会替换门面对该应用的当前快照，因此调用方需要重新观察并确认要使用的 ID。新的读取不保证拒绝旧字符串，也不保证 ID 会改变；调用方不得假定旧 ID 仍指向同一控件。指针输入和截图都要求显式的窗口对象；调用方先枚举窗口。 -键盘行为遵循 provider 支持的作用域。使用 `sendKeyEvent` 的调用方负责按下与抬起的配对，因为 API 从不发送相反事件。键盘修饰键名称由 provider 定义，同一字符串在不同 provider 上可能支持不同。 +键盘行为遵循 provider 支持的作用域。使用 `sendKeyEvent` 的调用方负责按下与抬起的配对，在 macOS 上，程序结束或用户暂停时会释放本次执行仍按着的键。键盘修饰键名称由 provider 定义，同一字符串在不同 provider 上可能支持不同。 Provider 指导文本负责支持动作的清单。把方法加入共享 schema 并不表示每个 provider 都已实现它；合并后的 `getCapabilities` 结果仍是权限和会话锁定状态的运行时来源。 diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml index c3d5791588..09553adc47 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml @@ -12,5 +12,5 @@ en: e5d1027faa55adba zh: 4b88ad6a70977fb3 /native-cua-application-and-window-api/migration: - en: 4cf8b18af0d2bb79 - zh: 7f490fd9785deffe + en: a0640d4cd1f2e84e + zh: be8d5b5f9d14caf8 diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md index 363e627e66..8f5b092012 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md @@ -18,6 +18,6 @@ Providers supply startup capabilities alongside their platform guidance. The nat 1. After `cua.getApp(selector)`, select the target from `app.getWindows()` with `app.getWindow(windowId)`. Use `window.getScreenshot()`, `window.click()`, `window.scroll()`, and `window.drag()`. Replace `hover` with `window.mouseMove()`. On macOS, `window.click(id)` sends mouse events at the element center. Switch to `invoke(id)` for the default accessibility action, or `performAction(id, "AXShowMenu")` to request an advertised menu action. 2. Replace `getAXState()` with `getAccessibilityTree()`. Replace `getAXStateAndScreenshot()` with separate tree and screenshot reads. Selection reads the initial tree silently for a window, and for an app when `appLevelAccessibility` is true. Read the tree explicitly to display it. 3. Pass the opaque string ID from the latest tree wherever you passed a numeric element index. In raw requests use `elementId` with `snapshotId`. Replace `performSecondaryAction` with `listActions(id)` and `performAction(id, exactName, argument?)`, or use `invoke(id)` for the default action. `setValue` accepts the provider’s supported JSON values. On macOS, `performAction(id, 'AXRaise')` performs the advertised raise action. Call `app.activate()` first when the application must become active. -4. Replace a packed chord such as `pressKey('shift+Tab')` with `pressKey('Tab', ['shift'])`. Check `(await cua.getCapabilities()).methods` before using `sendKeyEvent`: macOS supports complete keystrokes and rejects directional events with `NOT_IMPLEMENTED`. If your script handles unknown macOS key names as `NATIVE_ERROR`, handle `INVALID_ARGUMENT` instead. Correct the key name before retrying. +4. Replace a packed chord such as `pressKey('shift+Tab')` with `pressKey('Tab', ['shift'])`. Check `(await cua.getCapabilities()).methods` before using `sendKeyEvent`. On macOS, call it inside one `run_code` to hold a key across tool calls. Returning, throwing or pausing releases any remaining keys. If your script handles unknown macOS key names as `NATIVE_ERROR`, handle `INVALID_ARGUMENT` instead. Correct the key name before retrying. 5. Read `dispatched` from action results, then observe the application to verify the outcome. `invoke` returns `dispatched: false` when the element has no default action. Remove reads of the old `verified` field and of `pid` on selected objects or window records. Use `listApps()` or running `findApps()` matches when you need a PID. 6. For a custom provider, replace `CuaBackend.description` with `capabilities` and pass `guidance` and `permissionParameters` to `installCua`. Rebuild any custom `helperPath` against the [operation schema](../../../../packages/experimental/cua/src/protocol.ts) and [tagged permission response](../../../../packages/experimental/cua/src/access-types.ts). Restart the provider, inspect its advertised methods and keyboard scopes, and open the Computer Use permission page to verify its platform controls. diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md index 272e4a96d4..7580de0836 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md @@ -18,6 +18,6 @@ CUA 脚本在指定窗口上截图和操作鼠标。App 和 Window 对象共享 1. 调用 `cua.getApp(selector)` 后，从 `app.getWindows()` 返回的列表选定目标，再调用 `app.getWindow(windowId)`。在窗口上调用 `getScreenshot()`、`click()`、`scroll()` 和 `drag()`，将原有的 `hover` 调用改为 `window.mouseMove()`。在 macOS 上，`window.click(id)` 在元素中心发送鼠标事件。需要执行默认辅助功能动作时，改用 `invoke(id)`。需要打开控件菜单且控件声明了 `AXShowMenu` 时，调用 `performAction(id, "AXShowMenu")`。 2. 将 `getAXState()` 改为 `getAccessibilityTree()`，将 `getAXStateAndScreenshot()` 拆成读取树和截图两次调用。选择窗口时会静默读取初始树。选择应用时，只有 `appLevelAccessibility` 为真才读取。要显示树，请显式读取。 3. 将数字元素索引改为从最新树中取得的字符串 ID。直接发送请求时使用 `elementId` 和 `snapshotId`。原先调用 `performSecondaryAction` 的地方，先用 `listActions(id)` 取得完整动作名，再调用 `performAction(id, exactName, argument?)`。要执行默认动作，可调用 `invoke(id)`。`setValue` 接受提供方支持的 JSON 值。 在 macOS 上，`performAction(id, 'AXRaise')` 只执行元素声明的抬起窗口动作。需要把应用切到前台时先调用 `app.activate()`。 -4. 将 `pressKey('shift+Tab')` 之类的组合键写法改为 `pressKey('Tab', ['shift'])`。调用 `sendKeyEvent` 前检查 `(await cua.getCapabilities()).methods`。macOS 支持完整按键，单独发送按下或松开事件会返回 `NOT_IMPLEMENTED`。如果脚本用 `NATIVE_ERROR` 识别 macOS 上的未知键名，请改为处理 `INVALID_ARGUMENT`，并在重试前修正键名。 +4. 将 `pressKey('shift+Tab')` 之类的组合键写法改为 `pressKey('Tab', ['shift'])`。调用 `sendKeyEvent` 前检查 `(await cua.getCapabilities()).methods`。在 macOS 上，同一次 `run_code` 中可调用此方法跨工具调用保持按键。执行返回、异常退出或用户暂停时会释放仍按着的键。如果脚本用 `NATIVE_ERROR` 识别 macOS 上的未知键名，请改为处理 `INVALID_ARGUMENT`，并在重试前修正键名。 5. 从动作结果读取 `dispatched`，再观察应用确认效果。元素没有默认动作时，`invoke` 返回 `dispatched: false`。移除读取旧字段 `verified` 的代码，以及对已选对象和窗口记录中 `pid` 的读取。需要 PID 时，使用 `listApps()` 或 `findApps()` 返回的运行中应用记录。 6. 自定义提供方需将 `CuaBackend.description` 改为 `capabilities`，并向 `installCua` 传入 `guidance` 和 `permissionParameters`。如配置了 `helperPath`，请按[操作协议](../../../../packages/experimental/cua/src/protocol.ts)和[带平台标记的权限响应](../../../../packages/experimental/cua/src/access-types.ts)重新构建 helper。重启提供方，检查它声明的方法和键盘输入范围，再打开 Computer Use 权限页确认平台控件。 diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index 6a38002c4f..1abb4f6399 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -15,8 +15,8 @@ en: 1777e7bf6785f986 zh: e3daac1074ebb74d /deepseek-ai-dsh-experimental-cua-macos/use-this-package: - en: 24ea893846cf0914 - zh: dca284e8e589274d + en: f4206e2cb725c917 + zh: 37f0b04e9e74c193 /deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment: en: b734a0907747580e zh: 8110c5951b70e100 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 60d57c749d..042cc4fa94 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -79,7 +79,9 @@ Source builds default to `sourceSigning: ad-hoc`, without creating certificates The [configuration catalog](../../../docs/config-catalog.md#deepseek-aidsh-experimental-cua-macos) lists all fields. Native tool calls use `{method, params}`; the complete method-specific schema is exposed to the model. `getApp` returns the `appHandle` used by every app-targeted operation, and `app.getWindow(windowId)` returns the object that owns screenshots and pointer input. -`pressKey(key, modifiers?)` accepts one key or a nonempty array of distinct physical keys. Aliases such as `Enter` and `Return` identify the same key. `pressKey(['w', 'd'])` presses W then D and releases D then W. Pass held modifiers separately, as in `pressKey('Tab', ['shift'])`. Pressing a modifier itself, as in `pressKey('shift')`, sets its flag on press and clears it on release. A single `+` or `plus` sends Shift+=. In a multi-key chord, use `=` and pass Shift explicitly to apply it to the whole chord. The macOS helper accepts `command`/`cmd`, `shift`, `option`/`alt`, `control`/`ctrl`, and `fn`, and maps them to their physical keys. Key names are platform-defined; keypad names (`KP_0` through `KP_9`, `KP_Enter`, and keypad operators) retain their physical key codes. `typeText` sends literal text, including its line breaks and tabs. On macOS, use `pressKey` on the App object. The shared `sendKeyEvent` operation returns `NOT_IMPLEMENTED` on this provider. +`pressKey(key, modifiers?)` accepts one key or a nonempty array of distinct physical keys. Aliases such as `Enter` and `Return` identify the same key. `pressKey(['w', 'd'])` presses W then D and releases D then W. Pass held modifiers separately, as in `pressKey('Tab', ['shift'])`. Pressing a modifier itself, as in `pressKey('shift')`, sets its flag on press and clears it on release. A single `+` or `plus` sends Shift+=. In a multi-key chord, use `=` and pass Shift explicitly to apply it to the whole chord. The macOS helper accepts `command`/`cmd`, `shift`, `option`/`alt`, `control`/`ctrl`, and `fn`, and maps them to their physical keys. Key names are platform-defined; keypad names (`KP_0` through `KP_9`, `KP_Enter`, and keypad operators) retain their physical key codes. `typeText` sends literal text, including its line breaks and tabs. Use the App object's `pressKey` for a short press. + +For a long press, call `app.sendKeyEvent('w', 'down')` inside `run_code`. Each successful down remains active across subsequent tool calls in that execution. Choose when to send each up: after W down, D down and D up, W stays down. When the program returns, exits with an error, or the user pauses it, the helper releases the remaining keys at the processes that received their downs. A later execution cannot release an earlier execution's keys. Agent disposal and plugin unload cancel and await the complete tool call, including image admission and attachment storage. An image write already in progress must settle before disposal returns; a canceled call cannot return a successful screenshot. @@ -221,7 +223,8 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. +- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — hold a physical key across tool calls within one run_code. Choose the keys, how long to hold them, and which tools to call while they are down. Send 'up' for each key when needed. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. ``` diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index cadefe0dd6..0f6f4dad51 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -79,7 +79,9 @@ pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis. [配置目录](../../../docs/config-catalog.zh.md#deepseek-aidsh-experimental-cua-macos)列出全部字段。原生工具调用使用 `{method, params}`，模型可以看到每个方法的完整参数定义。`getApp` 返回每个面向应用的操作所需的 `appHandle`，`app.getWindow(windowId)` 返回负责截图和指针输入的对象。 -`pressKey(key, modifiers?)` 接受单个键名或由不同物理键组成的非空数组。`Enter` 和 `Return` 等别名指向同一个物理键。例如，`pressKey(['w', 'd'])` 依次按下 W、D，再依次松开 D、W。需要全程按住的修饰键单独传入，例如 `pressKey('Tab', ['shift'])`。单独调用 `pressKey('shift')` 时，按下事件带有 Shift 标志，释放事件清除该标志。单独传入 `+` 或 `plus` 会发送 Shift 和等号。在多键组合中请使用 `=`，需要全程按住 Shift 时将其列入修饰键。macOS helper 接受 `command`/`cmd`、`shift`、`option`/`alt`、`control`/`ctrl` 和 `fn`，并映射到对应物理键。按键名由平台定义；数字键盘名称（`KP_0` 到 `KP_9`、`KP_Enter` 和数字键盘运算键）保留独立的物理键码。`typeText` 发送字面文本，包括其中的换行和制表符。在 macOS 上，通过 App 对象调用 `pressKey`。调用共享接口 `sendKeyEvent` 会收到 `NOT_IMPLEMENTED`。 +`pressKey(key, modifiers?)` 接受单个键名或由不同物理键组成的非空数组。`Enter` 和 `Return` 等别名指向同一个物理键。例如，`pressKey(['w', 'd'])` 依次按下 W、D，再依次松开 D、W。需要全程按住的修饰键单独传入，例如 `pressKey('Tab', ['shift'])`。单独调用 `pressKey('shift')` 时，按下事件带有 Shift 标志，释放事件清除该标志。单独传入 `+` 或 `plus` 会发送 Shift 和等号。在多键组合中请使用 `=`，需要全程按住 Shift 时将其列入修饰键。macOS helper 接受 `command`/`cmd`、`shift`、`option`/`alt`、`control`/`ctrl` 和 `fn`，并映射到对应物理键。按键名由平台定义；数字键盘名称（`KP_0` 到 `KP_9`、`KP_Enter` 和数字键盘运算键）保留独立的物理键码。`typeText` 发送字面文本，包括其中的换行和制表符。短按使用 App 对象上的 `pressKey`。 + +长按时，在 `run_code` 中调用 `app.sendKeyEvent('w', 'down')`。按下成功后，在同一次执行中调用其他工具时该键仍保持按下。每个键可分别松开。例如先按 W，再按 D，松开 D 后 W 仍按着。程序正常返回、出错退出或用户暂停时，helper 向每个键原先的接收进程松开该次执行仍按着的键。后续执行不能释放前一次执行的键。 Agent 销毁和插件卸载会取消并等待完整工具调用，包括图像接入检查和附件存储。已经开始的图像写入必须结束后，销毁才会返回；已取消的调用不会返回成功截图。 @@ -221,7 +223,8 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. +- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — hold a physical key across tool calls within one run_code. Choose the keys, how long to hold them, and which tools to call while they are down. Send 'up' for each key when needed. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. ``` diff --git a/packages/experimental/cua-macos/native/README.i18n.yaml b/packages/experimental/cua-macos/native/README.i18n.yaml index 7382f3006b..2b78969668 100644 --- a/packages/experimental/cua-macos/native/README.i18n.yaml +++ b/packages/experimental/cua-macos/native/README.i18n.yaml @@ -18,11 +18,11 @@ en: 1435672513328c05 zh: 11e7a1e672410c10 /dsh-cua-native-helper/request-protocol: - en: 88a59904ca4ebedc - zh: 14f2b666d7382145 + en: 2a69d89b7fc95998 + zh: 084575bb31fb40dd /dsh-cua-native-helper/observation-and-input: - en: 51a70780a0672272 - zh: e7ca17dd40b887aa + en: 851614a21ebfeb65 + zh: ff868b14dc77adb8 /dsh-cua-native-helper/verification: en: ade5a2ccd3aeb1b4 zh: 74c2b0bcb23fc4bc diff --git a/packages/experimental/cua-macos/native/README.md b/packages/experimental/cua-macos/native/README.md index 2437c220a4..b3074d6ce9 100644 --- a/packages/experimental/cua-macos/native/README.md +++ b/packages/experimental/cua-macos/native/README.md @@ -78,7 +78,7 @@ Successful `activate` returns `dispatched` and `foregroundChanged`. A known chan | `getState` | `{appHandle, windowId, focusedWindowId, snapshotId, tree, elements, windows, truncated}`. `windowId` is null for an app-wide observation. Elements include the opaque string `elementId`, role, title, value, description, identifier, bounds, actions, and enabled state. Element bounds are null or [x, y, width, height] in screen points; window input uses local points. These structured element diagnostics are retained by native checks and omitted by the model tool projection. Unavailable optional descriptive properties are null, and an unavailable action list is empty; permission, communication and window-discovery failures still reject the observation. | | `listActions` | Provider-defined action names for one element as a string array; an element may advertise none. | | `show` | `{dispatched: boolean, windowId, foregroundChanged}` records restoration dispatch. `windowId` is null when no window was selected; `foregroundChanged` is null when the sampled observation was unavailable. | -| `invoke`, `performAction`, `mouseMove`, `click`, `setValue`, `selectText`, `typeText`, `paste`, `pressKey`, `scroll`, `drag` | `{dispatched: boolean}` records native dispatch; `invoke` reports false when no applicable default action exists. `mouseMove` also reports `hoverRoute` and `systemPointerMoved`. Read fresh state to verify the intended application change. | +| `invoke`, `performAction`, `mouseMove`, `click`, `setValue`, `selectText`, `typeText`, `paste`, `pressKey`, `sendKeyEvent`, `scroll`, `drag` | `{dispatched: boolean}` records native dispatch; `invoke` reports false when no applicable default action exists. `mouseMove` also reports `hoverRoute` and `systemPointerMoved`. Read fresh state to verify the intended application change. | | `screenshot` | `{mimeType: "image/png", data, pixelWidth, pixelHeight, bounds, coordinateSpace: "window-points", windowId}`. PNG pixel dimensions can differ from window point dimensions on Retina displays. | Every app-targeted operation requires `appHandle` from `getApp` in the same helper. The helper pins the PID and kernel process start time, and invalidates the handle and all of its snapshots after process exit or PID reuse. Calling `getApp` can acquire the replacement process without reactivating old handles. A random per-helper namespace prevents handles from a terminated helper matching those of its replacement. Repeated acquisition of the same live process shares its handle. A post-launch inspection failure invalidates the pending handle and preserves the original error. It does not terminate a process already started; launchApp cannot reuse the failed handle. @@ -105,6 +105,8 @@ Capture and pointer delivery read CG bounds before and after an exact-window AX `typeText` and `pressKey` target the selected process's current focused control, not an exact window. `pressKey` presses its modifiers followed by the requested keys in order, then releases all pressed keys in reverse order, including on cancellation. Keyboard events go directly to the target PID and bypass the global HID queue. Key names are platform-defined; keypad names such as `KP_0` and `KP_Enter` preserve their physical key codes. Background pointer actions may change that keyboard focus. Text injection sends Unicode characters individually and does not use the clipboard. LF, CR and CRLF send one Return; tabs send Tab. Applications apply their normal editing behavior, including indentation and focus navigation. Each `typeText` down/up event is followed by the configured text interval; `pressKey` keeps its own press and release timing, and dispatch completion does not confirm that the receiving application consumed every event. Cancellation can interrupt text or a gesture after partial input; it cannot undo received events. Very long text can exceed the host's operation deadline. Validate observed application state before deciding how to continue. +The Host associates directional key events with one `run_code` execution. On down, the helper retains the selected application, recipient PID and process start time, physical key code, and matching up event. Explicit up and cleanup verify the original process identity. On return, error, or user pause, the Host waits for any in-flight native request and asks the helper to release only that execution’s remaining keys. Disconnect cleanup also releases keys owned by the closed connection. + The selected helper application needs macOS Accessibility permission for AX state and input, and Screen Recording permission for screenshots. Capability preflight and permission errors do not grant permissions. Capture refuses a window that moves or resizes while the image is being read. Unassociated system panels and accessory-only applications are not selectable. Host-only permission controls are `permissionStatus {}`, `requestPermission` and `openPermissionSettings {permission}`, where permission is `accessibility` or `screenRecording`. Requests carry localized `instruction`, `backLabel`, `finderLabel`, `unavailableMessage`, `settingsMessage`, and an integer `pollIntervalMs` of 100–2147483647. The native guide substitutes its application name for `{application}`. Status reports permission observations, process diagnostics, and `launchApplication` metadata. These operations remain absent from the model tool and `getCapabilities.methods`. The helper dynamically resolves `responsibility_get_pid_responsible_for_pid` with the signature used by [Chromium](https://chromium.googlesource.com/chromium/src/+/refs/tags/134.0.6998.28/base/process/process_info_mac.mm), capturing the responsible bundle before readiness and falling back to its own bundle when unavailable. Parent paths do not determine authorization ownership. diff --git a/packages/experimental/cua-macos/native/README.zh.md b/packages/experimental/cua-macos/native/README.zh.md index 1224e0e6db..6035ffaee2 100644 --- a/packages/experimental/cua-macos/native/README.zh.md +++ b/packages/experimental/cua-macos/native/README.zh.md @@ -78,7 +78,7 @@ AppKit 在主事件循环上运行。阻塞式 socket 接收与握手使用后 | `getState` | `{appHandle, windowId, focusedWindowId, snapshotId, tree, elements, windows, truncated}`。应用级观察的 `windowId` 为 null。元素包含不透明字符串 `elementId`、role、title、value、description、identifier、bounds、actions 和 enabled 状态。元素边界为 null 或屏幕 point 单位的 [x, y, width, height]，窗口输入使用局部 point。这些结构化元素诊断供原生检查保留，模型工具投影会移除它们。不可用的可选描述属性为 null，无法取得的动作列表为空；权限、通信和窗口发现失败仍会拒绝观察。 | | `listActions` | 一个元素由 provider 定义的动作名数组；元素可以不声明任何动作。 | | `show` | `{dispatched: boolean, windowId, foregroundChanged}` 记录恢复分派。未选中窗口时 `windowId` 为 null；无法取得采样观察时 `foregroundChanged` 为 null。 | -| `invoke`, `performAction`, `mouseMove`, `click`, `setValue`, `selectText`, `typeText`, `paste`, `pressKey`, `scroll`, `drag` | `{dispatched: boolean}` 记录原生分派；`invoke` 在没有可用的默认动作时返回 false。`mouseMove` 还报告 `hoverRoute` 和 `systemPointerMoved`。需要读取新状态以核实预期的应用变化。 | +| `invoke`, `performAction`, `mouseMove`, `click`, `setValue`, `selectText`, `typeText`, `paste`, `pressKey`, `sendKeyEvent`, `scroll`, `drag` | `{dispatched: boolean}` 记录原生分派；`invoke` 在没有可用的默认动作时返回 false。`mouseMove` 还报告 `hoverRoute` 和 `systemPointerMoved`。需要读取新状态以核实预期的应用变化。 | | `screenshot` | `{mimeType: "image/png", data, pixelWidth, pixelHeight, bounds, coordinateSpace: "window-points", windowId}`。Retina 显示器上 PNG 像素尺寸可能不同于窗口 point 尺寸。 | 每个面向应用的操作都要求使用同一 helper 的 `getApp` 返回的 `appHandle`。helper 固定 PID 和内核进程启动时间，进程退出或 PID 被复用后使句柄及其全部快照失效。再次调用 `getApp` 可取得替代进程，不会重新激活旧句柄。随机的 helper 命名空间防止已终止 helper 的句柄与替代实例重合。重复获取同一存活进程会共享句柄。启动后检查失败会使待启动句柄失效，并保留原错误；已经启动的进程不会被终止，launchApp 也不能复用失败的句柄。 @@ -105,6 +105,8 @@ AppKit 在主事件循环上运行。阻塞式 socket 接收与握手使用后 `typeText` 和 `pressKey` 面向所选进程当前获得焦点的控件，不指定窗口。`pressKey` 向目标 PID 依次发送修饰键和指定按键的按下事件，再逆序释放。操作中途取消也会逆序释放已按下的键。键盘事件直接发给目标进程，绕过全局 HID 队列。按键名由平台定义；KP_0 和 KP_Enter 等数字键盘名称保留物理键码。后台鼠标操作可能改变键盘焦点。文本逐个 Unicode 字符输入，不使用剪贴板。LF、CR 和 CRLF 发送一次 Return，制表符发送 Tab。应用保留缩进和焦点导航等正常编辑行为。`typeText` 的每次按下和抬起后等待配置的文本事件间隔；`pressKey` 使用自己的按下与释放节奏，分派完成不证明应用已消费全部事件。取消可能中断部分文本或手势，无法撤销已接收事件。很长的文本可能超过 Host 操作期限。继续输入前应核实应用状态。 +宿主为每次 `run_code` 的 `sendKeyEvent` 附上执行标识。按下时，helper 保存已选应用、接收进程的 PID 与启动时间、物理键码及对应的松开事件。显式松键或执行结束时会核对原进程身份。正常返回、异常退出或用户暂停时，宿主等待在途原生请求结束，再通知 helper 释放该执行持有的其余按键。连接断开时，helper 也会清理该连接持有的按键。 + 所选 helper 应用需要 macOS 辅助功能权限以读取 AX 和输入，需要屏幕录制权限以截图。能力预检和权限错误不授予权限。截图会拒绝在读取图片期间移动或缩放的窗口。无法确认归属的系统面板及仅后台辅助类型的应用不可选择。 仅供 Host 使用的权限操作为 `permissionStatus {}`、`requestPermission` 和 `openPermissionSettings {permission}`，其中 permission 为 `accessibility` 或 `screenRecording`。请求携带本地化的 `instruction`、`backLabel`、`finderLabel`、`unavailableMessage`、`settingsMessage`，以及 100–2147483647 的整数 `pollIntervalMs`。原生引导将 `{application}` 替换为自身应用名。状态返回权限观测、进程诊断和 `launchApplication` 信息。这些操作不出现在模型工具及 `getCapabilities.methods` 中。helper 动态解析 `responsibility_get_pid_responsible_for_pid`，函数签名与 [Chromium](https://chromium.googlesource.com/chromium/src/+/refs/tags/134.0.6998.28/base/process/process_info_mac.mm) 一致，在报告就绪前记录责任 bundle，无法获取时使用自身 bundle。父进程路径不决定授权归属。 diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift index da1afe5a3d..80d820d0f1 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift @@ -81,13 +81,15 @@ actor AXWorker { private let systemPointerHover: Bool? private let sessionState: @Sendable () -> Bool? private var access: AXAccess? + private let heldKeys: HeldKeys private var keyboard: KeyboardInput? private var pointer: WindowPointerInput? private var hovers: [String: AXHover] = [:] private var closed = false - init(cancellation: CancellationLatch, axTimeout: Float, textEventInterval: Double, maxClickHoldMs: Int, systemPointerHover: Bool? = nil, + init(cancellation: CancellationLatch, axTimeout: Float, textEventInterval: Double, maxClickHoldMs: Int, systemPointerHover: Bool? = nil, heldKeys: HeldKeys = HeldKeys(), sessionState: @escaping @Sendable () -> Bool? = NativePlatform.sessionLocked) { + self.heldKeys = heldKeys self.cancellation = cancellation self.axTimeout = axTimeout self.textEventInterval = textEventInterval @@ -118,7 +120,7 @@ actor AXWorker { if let keyboard, let pointer { return (keyboard, pointer) } let poster = PIDEventPoster() let latch = cancellation - let key = KeyboardInput(post: poster.post, textEventInterval: textEventInterval, checkCancellation: { try latch.check() }) + let key = KeyboardInput(post: poster.post, textEventInterval: textEventInterval, held: heldKeys, checkCancellation: { try latch.check() }) let mouse = WindowPointerInput(poster: poster, checkCancellation: { try latch.check() }) keyboard = key; pointer = mouse return (key, mouse) @@ -208,8 +210,13 @@ actor AXWorker { trace: ScreenshotTrace? = nil) async throws -> DriverOutcome { try cancellation.check() guard !closed else { throw NativeFailure(code: "NATIVE_CLOSED", message: "Helper is closing") } - if method == "sendKeyEvent" { - throw NativeFailure(code: "NOT_IMPLEMENTED", message: "sendKeyEvent is not implemented on current platform.") + if method == "releaseHeldKeys" { + try keyboard?.releaseHeldKeys(run: params.string("runId")) + return .result(Self.actionResult) + } + if method == "sendKeyEvent", try params.string("direction") == "up" { + let dispatched = try inputs().0.keyUp(params.string("key"), app: params.string("appHandle"), run: params.string("runId")) + return .result(.object(["dispatched": .bool(dispatched)])) } let locked = observeSession() if method == "getCapabilities" { @@ -331,6 +338,9 @@ actor AXWorker { maxBytes: try params.integer("maxClipboardBytes", in: 1024...Int.max), readTimeout: .milliseconds(try params.integer("pasteReadTimeoutMs", in: 1...2147483647)), maxElements: try params.integer("maxElements", in: 1...10000), maxDepth: try params.integer("maxDepth", in: 1...100))) + case "sendKeyEvent": + let recipient = try ax().keyboardRecipient(identity, catalog: catalog) + try inputs().0.keyDown(params.string("key"), app: handle, run: params.string("runId"), recipient: recipient) case "typeText", "pressKey": let recipient = try ax().keyboardRecipient(identity, catalog: catalog) try NativePlatform.verify(recipient) @@ -496,5 +506,10 @@ actor AXWorker { static func mouseMoveResult(route: String, pointerMoved: Bool) -> JSONValue { .object(["dispatched": .bool(true), "hoverRoute": .string(route), "systemPointerMoved": .bool(pointerMoved)]) } - func shutdown() { closed = true; snapshots.removeAll(); observedWindowBounds.removeAll(); targets.removeAll(); hovers.removeAll(); access = nil; keyboard = nil; pointer = nil } + func shutdown() { + do { try keyboard?.releaseHeldKeys() } + catch { fputs("Could not release held keys on disconnect: \(error)\n", stderr) } + closed = true; snapshots.removeAll(); observedWindowBounds.removeAll(); targets.removeAll(); hovers.removeAll() + access = nil; keyboard = nil; pointer = nil + } } diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/HeldKeys.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/HeldKeys.swift new file mode 100644 index 0000000000..47c34fbe2a --- /dev/null +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/HeldKeys.swift @@ -0,0 +1,84 @@ +// All connections to the shared helper deliver to the same desktop processes. +import CoreGraphics +import Foundation + +/// The lock covers both the held-key records and their corresponding PID-targeted event posts. +/// Shutdown may release a connection's keys outside the ordinary request scheduler. +final class HeldKeys: @unchecked Sendable { + struct Owner: Equatable { + let connection: UUID + let run: String + let app: String + } + + private struct Press { + let owner: Owner + let recipient: ProcessIdentity + let code: CGKeyCode + let flag: CGEventFlags + let up: CGEvent + } + + private let lock = NSLock() + private var pressed: [Press] = [] + + func snapshot(pid: Int32) -> [CGKeyCode: CGEventFlags] { + lock.withLock { Dictionary(uniqueKeysWithValues: pressed.filter { $0.recipient.pid == pid }.map { ($0.code, $0.flag) }) } + } + + func down(owner: Owner, recipient: ProcessIdentity, code: CGKeyCode, flag: CGEventFlags, + down: CGEvent, up: CGEvent, post: (CGEvent, Int32) -> Void) throws { + try lock.withLock { + guard !pressed.contains(where: { $0.code == code && ($0.owner == owner || $0.recipient.pid == recipient.pid) }) else { + throw RequestParameters.invalid("Key is already held for this target") + } + pressed.append(Press(owner: owner, recipient: recipient, code: code, flag: flag, up: up)) + down.flags = flags(for: recipient) + post(down, recipient.pid) + } + } + + private func flags(for recipient: ProcessIdentity) -> CGEventFlags { + pressed.reduce(into: CGEventFlags()) { if $1.recipient == recipient { $0.formUnion($1.flag) } } + } + + /// An up always targets the process selected at down. Never post to a recycled PID. + private func release(at index: Int, post: (CGEvent, Int32) -> Void) throws -> Bool { + let key = pressed[index] + do { try NativePlatform.verify(key.recipient) } + catch let error as NativeFailure where error.code == "STALE_APP" || error.code == "APP_UNAVAILABLE" { + pressed.remove(at: index) + return false + } + pressed.remove(at: index) + key.up.flags = flags(for: key.recipient) + post(key.up, key.recipient.pid) + return true + } + + func up(owner: Owner, code: CGKeyCode, post: (CGEvent, Int32) -> Void) throws -> Bool { + try lock.withLock { + guard let index = pressed.firstIndex(where: { $0.owner == owner && $0.code == code }) else { + throw RequestParameters.invalid("This run has not pressed that key for this application") + } + return try release(at: index, post: post) + } + } + + func release(connection: UUID, run: String?, post: (CGEvent, Int32) -> Void) throws { + try lock.withLock { + defer { + // A closed connection cannot retry an unverifiable recipient. + if run == nil { pressed.removeAll { $0.owner.connection == connection } } + } + var firstError: Error? + for index in pressed.indices.reversed() { + let owner = pressed[index].owner + guard owner.connection == connection, run == nil || owner.run == run else { continue } + do { _ = try release(at: index, post: post) } + catch { firstError = firstError ?? error } + } + if let firstError { throw firstError } + } + } +} diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift index 3511256457..91572fbd5f 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift @@ -9,16 +9,39 @@ final class KeyboardInput { let flag: CGEventFlags } + private let connection = UUID() + private let held: HeldKeys private let post: (CGEvent, Int32) -> Void private let textEventInterval: Double private let checkCancellation: @Sendable () throws -> Void - init(post: @escaping (CGEvent, Int32) -> Void, textEventInterval: Double, checkCancellation: @escaping @Sendable () throws -> Void = {}) { + init(post: @escaping (CGEvent, Int32) -> Void, textEventInterval: Double, held: HeldKeys = HeldKeys(), checkCancellation: @escaping @Sendable () throws -> Void = {}) { + self.held = held self.post = post self.textEventInterval = textEventInterval self.checkCancellation = checkCancellation } + func keyDown(_ name: String, app: String, run: String, recipient: ProcessIdentity) throws { + let modifier = Self.namedModifier(name) + let code = try Self.keyCode(name) + let source = try Self.source() + let down = try Self.keyEvent(source: source, code: code, down: true, flags: []) + let up = try Self.keyEvent(source: source, code: code, down: false, flags: []) + try checkCancellation() + try NativePlatform.verify(recipient) + try held.down(owner: .init(connection: connection, run: run, app: app), recipient: recipient, + code: code, flag: modifier?.flag ?? [], down: down, up: up, post: post) + } + + func keyUp(_ name: String, app: String, run: String) throws -> Bool { + try held.up(owner: .init(connection: connection, run: run, app: app), code: Self.keyCode(name), post: post) + } + + func releaseHeldKeys(run: String? = nil) throws { + try held.release(connection: connection, run: run, post: post) + } + /// Prepare every event before dispatch. On cancellation, release only the keys this call pressed. func pressKey(_ keys: [String], modifiers: [String], to pid: Int32) throws { try checkCancellation() @@ -42,8 +65,12 @@ final class KeyboardInput { } chord.append(key) } - var selected = Set() - var flags: CGEventFlags = [] + let alreadyHeld = held.snapshot(pid: pid) + guard namesByCode.keys.allSatisfy({ alreadyHeld[$0] == nil }) else { + throw RequestParameters.invalid("Release the held key before using it in pressKey") + } + var selected = Set(alreadyHeld.keys) + var flags = alreadyHeld.values.reduce(CGEventFlags()) { $0.union($1) } var events: [(down: CGEvent, up: CGEvent)] = [] let source = try Self.source() for key in chord { @@ -77,6 +104,8 @@ final class KeyboardInput { func typeText(_ text: String, to pid: Int32) throws { try checkCancellation() let source = try Self.source() + let alreadyHeld = held.snapshot(pid: pid) + let flags = alreadyHeld.values.reduce(CGEventFlags()) { $0.union($1) } for scalar in text.replacingOccurrences(of: "\r\n", with: "\n").unicodeScalars { try checkCancellation() let code: CGKeyCode @@ -85,8 +114,11 @@ final class KeyboardInput { case 9: code = try Self.keyCode("Tab") default: code = 0 } - let down = try Self.keyEvent(source: source, code: code, down: true, flags: []) - let up = try Self.keyEvent(source: source, code: code, down: false, flags: []) + guard alreadyHeld[code] == nil else { + throw RequestParameters.invalid("Release the held physical key before typing text with its key code") + } + let down = try Self.keyEvent(source: source, code: code, down: true, flags: flags) + let up = try Self.keyEvent(source: source, code: code, down: false, flags: flags) if code == 0 { let units = Array(String(scalar).utf16) units.withUnsafeBufferPointer { buffer in diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift index 0da4e5cb96..57f20c732f 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift @@ -20,11 +20,12 @@ enum NativeProtocol { "performAction": ["action", "appHandle", "argument", "elementId", "snapshotId", "windowId"], "permissionStatus": [], "pressKey": ["appHandle", "key", "modifiers", "windowId"], + "releaseHeldKeys": ["runId"], "requestPermission": ["backLabel", "finderLabel", "instruction", "permission", "pollIntervalMs", "settingsMessage", "unavailableMessage"], "screenshot": ["appHandle", "windowId"], "scroll": ["appHandle", "deltaLinesX", "deltaLinesY", "snapshotId", "windowId", "x", "y"], "selectText": ["appHandle", "elementId", "prefix", "selectionType", "snapshotId", "suffix", "text", "windowId"], - "sendKeyEvent": ["appHandle", "direction", "key", "windowId"], + "sendKeyEvent": ["appHandle", "direction", "key", "runId", "windowId"], "setValue": ["appHandle", "elementId", "snapshotId", "value", "windowId"], "show": ["appHandle", "windowId"], "typeText": ["appHandle", "text", "windowId"], diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift index 0041e88b2a..6b69122d99 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift @@ -7,7 +7,7 @@ struct RequestParameters: Sendable { init(_ request: NativeRequest) throws { guard let values = request.params.object else { throw Self.invalid("params must be an object") } guard let allowed = Self.fields[request.method] else { throw NativeFailure(code: "UNSUPPORTED", message: "Unsupported native operation: \(request.method)") } - if ["typeText", "pressKey", "paste"].contains(request.method), values["windowId"] != nil { + if ["typeText", "pressKey", "sendKeyEvent", "paste"].contains(request.method), values["windowId"] != nil { throw Self.invalid("macOS keyboard input follows application focus; use the App object") } guard Set(values.keys).isSubset(of: allowed) else { throw Self.invalid("Unknown operation parameter") } diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift index 000a69e848..737d560720 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift @@ -43,7 +43,7 @@ import Darwin self.onFatal = onFatal worker = AXWorker(cancellation: cancellation, axTimeout: configuration.axTimeoutSeconds, textEventInterval: configuration.textEventIntervalSeconds, maxClickHoldMs: configuration.maxClickHoldMs, - systemPointerHover: configuration.systemPointerHover) + systemPointerHover: configuration.systemPointerHover, heldKeys: scheduler.heldKeys) input = JSONLInput(limit: configuration.maxRequestBytes, cancellation: cancellation, channel: channel) self.writer = writer } @@ -166,7 +166,7 @@ import Darwin let params = try RequestParameters(request) try cancellation.check() if request.method == "screenshot" { try capture.requireAvailable() } - let requiresCatalog = ["getApp", "listApps", "findApps", "typeText", "pressKey", "click"].contains(request.method) + let requiresCatalog = ["getApp", "listApps", "findApps", "typeText", "pressKey", "sendKeyEvent", "click"].contains(request.method) let catalog = requiresCatalog ? ApplicationCatalog.snapshot() : ApplicationCatalogSnapshot(applications: [], frontPID: nil) let result: JSONValue if request.method == "permissionStatus" { diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestScheduler.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestScheduler.swift index 54671b12d4..83807c8468 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestScheduler.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestScheduler.swift @@ -7,6 +7,7 @@ import Dispatch let id: UUID let continuation: CheckedContinuation } + let heldKeys = HeldKeys() private var owner: UUID? private var waiting: [Waiting] = [] diff --git a/packages/experimental/cua-macos/native/generate-protocol.ts b/packages/experimental/cua-macos/native/generate-protocol.ts index 398999cc2c..29ccc58344 100644 --- a/packages/experimental/cua-macos/native/generate-protocol.ts +++ b/packages/experimental/cua-macos/native/generate-protocol.ts @@ -10,6 +10,7 @@ import { PermissionGuideParameters, PermissionSettingsParameters } from '../src/ */ export function renderNativeProtocol(): { json: string; swift: string } { const { fields, enums, json } = nativeProtocol({ + sendKeyEvent: z.object({ runId: z.string().min(1) }), mouseMove: z.object({ allowSystemPointer: z.boolean().optional() }), requestPermission: PermissionGuideParameters, openPermissionSettings: PermissionSettingsParameters, }) diff --git a/packages/experimental/cua-macos/native/protocol.schema.json b/packages/experimental/cua-macos/native/protocol.schema.json index ce960cea7e..f8a9824752 100644 --- a/packages/experimental/cua-macos/native/protocol.schema.json +++ b/packages/experimental/cua-macos/native/protocol.schema.json @@ -1062,12 +1062,17 @@ "down", "up" ] + }, + "runId": { + "type": "string", + "minLength": 1 } }, "required": [ "appHandle", "key", - "direction" + "direction", + "runId" ], "additionalProperties": false } @@ -1304,6 +1309,37 @@ ], "additionalProperties": false }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "method": { + "type": "string", + "const": "releaseHeldKeys" + }, + "params": { + "type": "object", + "properties": { + "runId": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "runId" + ], + "additionalProperties": false + } + }, + "required": [ + "id", + "method", + "params" + ], + "additionalProperties": false + }, { "type": "object", "properties": { diff --git a/packages/experimental/cua-macos/native/tests/capabilities_e2e.py b/packages/experimental/cua-macos/native/tests/capabilities_e2e.py index f9c280a193..432f70e81f 100644 --- a/packages/experimental/cua-macos/native/tests/capabilities_e2e.py +++ b/packages/experimental/cua-macos/native/tests/capabilities_e2e.py @@ -193,6 +193,22 @@ def main(): state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 4) chord = [(event["type"], event.get("keyCode")) for event in state["events"][count:]] assert chord == [(10, 13), (10, 2), (11, 2), (11, 13)], chord + count = len(state["events"]) + def key(name, direction): + call("sendKeyEvent", {**owner, "runId": "held-input", "key": name, "direction": direction}) + key("w", "down") + key("d", "down") + state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 2) + call("getState", {**owner, "maxElements": 2000, "maxDepth": 25}) + time.sleep(2) + key("d", "up") + state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 3) + held = state["events"][count:] + assert [(event["type"], event["keyCode"]) for event in held] == [(10, 13), (10, 2), (11, 2)], held + assert held[2]["timestamp"] - held[1]["timestamp"] >= 2, held + call("releaseHeldKeys", {"runId": "held-input"}) + state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 4) + assert [(e["type"], e["keyCode"]) for e in state["events"][count:]] == [(10, 13), (10, 2), (11, 2), (11, 13)] call("drag", {**window, "fromX": point["x"], "fromY": point["y"], "toX": point["x"] + 80, "toY": point["y"] + 20, "durationMs": 200, "steps": 10, "mouseButton": "middle", "modifiers": ["shift"]}) state = wait_for_state(state_file, lambda value: any(event["type"] == 27 and event.get("button") == 2 and event["shift"] for event in value["events"])) diff --git a/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift b/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift index 6ca44cd59a..52cae6911e 100644 --- a/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift +++ b/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift @@ -1,5 +1,6 @@ // Observe the actual CoreGraphics events without sending keyboard input to the desktop. import CoreGraphics +import Darwin import Testing @testable import CuaNative @@ -68,3 +69,24 @@ func plusAlonePressesShiftEquals(_ alias: String) throws { #expect(events.map(\.type) == [.flagsChanged, .flagsChanged]) #expect(events.map { $0.flags.contains(.maskShift) } == [true, false]) } + +@Test func directionalReleaseUsesOriginalRecipientAndPreservesOtherRuns() throws { + let target = try NativePlatform.identity(getpid()) + let held = HeldKeys() + var events: [CGEvent] = [] + let input = KeyboardInput(post: { event, pid in + #expect(pid == target.pid) + events.append(event) + }, textEventInterval: 0, held: held) + try input.keyDown("shift", app: "editor", run: "one", recipient: target) + try input.keyDown("w", app: "editor", run: "two", recipient: target) + #expect(throws: NativeFailure.self) { try input.keyDown("W", app: "editor", run: "one", recipient: target) } + #expect(throws: NativeFailure.self) { try input.keyUp("w", app: "other", run: "two") } + #expect(throws: NativeFailure.self) { try input.pressKey(["w"], modifiers: [], to: target.pid) } + try input.pressKey(["d"], modifiers: ["shift"], to: target.pid) + try input.releaseHeldKeys(run: "one") + #expect(try input.keyUp("w", app: "editor", run: "two")) + try input.releaseHeldKeys(run: "two") + #expect(events.map { $0.getIntegerValueField(.keyboardEventKeycode) } == [56, 13, 2, 2, 56, 13]) + #expect(events.map { $0.flags.contains(.maskShift) } == [true, true, true, true, false, false]) +} diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index c7726f385b..e750c4068b 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -13,7 +13,8 @@ const methodDetails = new Map([ export const GUIDANCE = `${PLATFORM_GUIDANCE} ${COMMON_GUIDANCE.replace(/^- ([\w.]+)\(.*$/gm, (entry, method: string) => { - if (method === 'pressKey') return "- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed." + if (method === 'pressKey') return `- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — hold a physical key across tool calls within one run_code. Choose the keys, how long to hold them, and which tools to call while they are down. Send 'up' for each key when needed. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.` const detail = methodDetails.get(method) return detail ? `${entry} ${detail}` : entry })}` diff --git a/packages/experimental/cua-macos/src/held-keys.ts b/packages/experimental/cua-macos/src/held-keys.ts new file mode 100644 index 0000000000..30c1311da8 --- /dev/null +++ b/packages/experimental/cua-macos/src/held-keys.ts @@ -0,0 +1,42 @@ +/** Directional input belongs to the enclosing run_code, including while its worker calls other tools. */ +import { randomUUID } from 'node:crypto' +import type { Context } from '@deepseek-ai/cordis' +import type { CuaBackend } from '@deepseek-ai/dsh-cua' +import { NativeError, type NativeClient } from '@deepseek-ai/dsh-cua/native-client' +import { RUN_CODE_NAME, type ToolExecutionToken } from '@deepseek-ai/dsh-tools' + +/** + * Keep native key ownership alive until its program returns, throws, or is interrupted. + * @param ctx - Provider scope owning the tool execution listener. + * @returns Handler for authorized directional keyboard requests. + */ +export function installHeldKeys(ctx: Context): NonNullable { + const runs = new Map() + ctx.on('tools/execute', async (execution, next) => { + if (execution.name !== RUN_CODE_NAME) return next() + const run: { id: string; client?: NativeClient } = { id: randomUUID() } + runs.set(execution.token, run) + let cleanup: Promise | undefined + const release = () => { + runs.delete(execution.token) + if (run.client) cleanup ??= run.client.requestCleanup('releaseHeldKeys', { runId: run.id }) + return cleanup + } + // Pause may arrive while an unrelated subtool is still settling. + const onAbort = () => { void release()?.catch(() => undefined) } + execution.signal.addEventListener('abort', onAbort, { once: true }) + try { + return await next() + } finally { + execution.signal.removeEventListener('abort', onAbort) + await release() + } + }) + return (client, params, execution) => { + const run = execution.parent && runs.get(execution.parent) + if (!run) throw new NativeError('INVALID_ARGUMENT', 'Use sendKeyEvent inside an active run_code execution') + if (run.client && run.client !== client) throw new NativeError('NATIVE_CLOSED', 'The keyboard connection changed; start a new run_code') + run.client = client + return client.request('sendKeyEvent', { ...params, runId: run.id }, execution.signal, execution.callId) + } +} diff --git a/packages/experimental/cua-macos/src/index.ts b/packages/experimental/cua-macos/src/index.ts index 4b07e9c8b7..8c089c9f8e 100644 --- a/packages/experimental/cua-macos/src/index.ts +++ b/packages/experimental/cua-macos/src/index.ts @@ -9,6 +9,7 @@ import { CuaConfigFields } from '@deepseek-ai/dsh-cua/config' import { createPermissionParameters } from './permission-guide.ts' import { nativeHostArchitecture, prepareNativeExecutable, type SourceSigning } from './native-build.ts' import { createSharedNativeChannel } from './shared-native.ts' +import { installHeldKeys } from './held-keys.ts' import { GUIDANCE } from './guidance.ts' export { GUIDANCE } from './guidance.ts' @@ -17,7 +18,7 @@ const CAPABILITIES: CuaStartupCapabilities = { methods: [ 'activate', 'click', 'drag', 'findApps', 'getApp', 'getCapabilities', 'getState', 'invoke', 'listActions', 'listApps', 'listWindows', 'mouseMove', 'paste', 'performAction', 'pressKey', 'screenshot', 'scroll', - 'selectText', 'setValue', 'show', 'typeText', + 'selectText', 'sendKeyEvent', 'setValue', 'show', 'typeText', ], appLevelAccessibility: true, keyboard: ['app'], @@ -65,7 +66,7 @@ export const Config = Schema.object({ */ export async function apply(ctx: Context, config: Config): Promise { const createChannel: NativeChannelFactory = (owner, native) => createSharedNativeChannel(owner, native, config.systemPointerHover) - const backend = { createChannel, guidance: GUIDANCE, capabilities: CAPABILITIES, + const backend = { createChannel, sendKeyEvent: installHeldKeys(ctx), guidance: GUIDANCE, capabilities: CAPABILITIES, permissionParameters: createPermissionParameters(config.permissionPollIntervalMs) } if (config.helperPath !== undefined) { if (!isAbsolute(config.helperPath)) throw new Error('dsh-cua helperPath must be absolute') diff --git a/packages/experimental/cua-macos/tests/guidance.spec.ts b/packages/experimental/cua-macos/tests/guidance.spec.ts index 16fb39122a..54322e2316 100644 --- a/packages/experimental/cua-macos/tests/guidance.spec.ts +++ b/packages/experimental/cua-macos/tests/guidance.spec.ts @@ -28,6 +28,9 @@ it('states the macOS accessibility, keyboard, and mouse levels', () => { expect(PLATFORM_GUIDANCE).toContain('Simulated keyboard input is available at app level') expect(PLATFORM_GUIDANCE).toContain('Simulated mouse input is available at window level') const pressKey = GUIDANCE.split('\n').find(line => line.startsWith('- pressKey(')) + expect(GUIDANCE).toContain("sendKeyEvent(key: string, direction: 'down' | 'up')") + expect(GUIDANCE).toContain('returns, throws, or the user pauses') + expect(COMMON_GUIDANCE).not.toContain('sendKeyEvent(') expect(pressKey).toContain('key: string | string[]') expect(pressKey).toContain("pressKey(['w', 'd']) presses W then D and releases D then W") expect(pressKey).toContain("Packed strings such as 'w+d'") @@ -139,5 +142,5 @@ it('documents facade initialization, emitted observations, and separate key argu }) it('omits stale names and unavailable platform operations', () => { - for (const stale of ['getAXState', 'disableDiffing', 'performSecondaryAction', 'hover(', 'keyChord', 'sendKeyEvent', 'window.typeText', 'window.pressKey', 'elementIndex', 'verified: false', 'browser tabs', 'not implemented']) expect(GUIDANCE).not.toContain(stale) + for (const stale of ['getAXState', 'disableDiffing', 'performSecondaryAction', 'hover(', 'keyChord', 'window.typeText', 'window.pressKey', 'elementIndex', 'verified: false', 'browser tabs', 'not implemented']) expect(GUIDANCE).not.toContain(stale) }) diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index 48b9eb99e3..194f7244fd 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -1,5 +1,5 @@ /** Native helper replacement is synchronized with Agent and provider disposal. */ -import { executeToolFixture } from '@deepseek-ai/dsh-tool-testkit' +import { executeModelToolFixture, executeToolFixture } from '@deepseek-ai/dsh-tool-testkit' import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' @@ -28,6 +28,7 @@ import SandboxPolicy from '@deepseek-ai/dsh-sandbox-policy' import { LlmAdapter, ToolCallId, type LlmResolvedModelInfo, type StreamChunk } from '@deepseek-ai/dsh-llm' import { SessionId } from '@deepseek-ai/dsh-session' import Subprocess from '@deepseek-ai/dsh-subprocess' +import SubprocessLocal from '@deepseek-ai/dsh-subprocess-local' import { expect, it, onTestFinished, vi } from 'vitest' import sharp from 'sharp' import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment' @@ -38,6 +39,7 @@ import type { NativeBuildSpec } from '../src/native-build.ts' const native = vi.hoisted(() => ({ prepare: vi.fn<(spec: NativeBuildSpec) => Promise>(), architecture: vi.fn(() => 'arm64' as const), + cleanup: vi.fn(), request: vi.fn(), dispose: vi.fn(), instances: [] as Array<{ disposed: boolean }>, reportDiagnostic: undefined as ((message: string) => void) | undefined, })) @@ -54,6 +56,7 @@ vi.mock('@deepseek-ai/dsh-cua/native-client', async importOriginal => ({ native.reportDiagnostic = args[3] native.instances.push(this) } + requestCleanup = native.cleanup request = native.request invalidate() { this.disposed = true } async dispose() { this.disposed = true; await native.dispose() } @@ -110,19 +113,25 @@ class ImageAdapter extends LlmAdapter { const imageData = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC' const imageMetadata = { pixelWidth: 1, pixelHeight: 1, bounds: { x: 2, y: 3, width: 1, height: 1 }, coordinateSpace: 'window-points', windowId: 7 } -async function setup(model?: { provider: string; model: string }) { +async function setup(model?: { provider: string; model: string }, ptc = false) { + native.cleanup.mockReset().mockResolvedValue({ dispatched: true }) native.request.mockReset().mockResolvedValue([]) native.dispose.mockReset() native.instances.length = 0 native.reportDiagnostic = undefined const ctx = new Context() onTestFinished(() => ctx.fiber.dispose()) - await mountAgentLoopTestDependencies(ctx) + await mountAgentLoopTestDependencies(ctx, ptc ? { workingDirectory: true, tools: { mode: 'ptc' } } : {}) await ctx.plugin(ComputerUseRegistry) - await ctx.plugin(Subprocess, []) + if (ptc) { + await ctx.plugin(SubprocessLocal) + await ctx.plugin(Sandbox) + await ctx.plugin(SandboxPolicy, { mode: 'danger-full-access' }) + await ctx.plugin(NodeRuntime) + } else await ctx.plugin(Subprocess, []) const provider = await ctx.plugin(NativeProvider, config) await mountAgentLoopTestHarness(ctx) - const handle = await ctx.agents.create({ sessionId: SessionId('native-lifecycle'), ...model ? { agentOptions: model } : {} }) + const handle = await ctx.agents.create({ sessionId: SessionId('native-lifecycle'), meta: { cwd: process.cwd() }, ...model ? { agentOptions: model } : {} }) let index = 0 const execute = (method = 'listApps', params: Record = {}, signal = new AbortController().signal) => executeToolFixture(ctx, { agent: handle.agent, signal, callId: ToolCallId(`lifecycle-${++index}`), name: 'cua', arguments: { method, params }, @@ -138,6 +147,69 @@ async function selectApplication(f: Awaited>) { expect((await f.execute('getApp', { selector: 'Fixture' })).isError).toBe(false) } +it('requires an enclosing run_code before sending a directional key', async () => { + const f = await setup() + await selectApplication(f) + native.request.mockClear() + const result = await f.execute('sendKeyEvent', { appHandle: 'app-one', key: 'w', direction: 'down' }) + expect(result.isError).toBe(true) + expect(JSON.stringify(result.content)).toContain('active run_code') + expect(native.request).not.toHaveBeenCalled() +}) + +it.each(['return', 'throw', 'pause'])('cleans up only the held W when run_code ends by %s', async (ending) => { + const f = await setup(undefined, true) + await selectApplication(f) + const held = new Set() + const events: string[] = [] + const controller = new AbortController() + const runIds = new Set() + native.request.mockImplementation(async (method, params) => { + expect(method).toBe('sendKeyEvent') + runIds.add(params.runId) + const key = String(params.key) + if (params.direction === 'down') held.add(key) + else held.delete(key) + events.push(`${key}:${params.direction}`) + return { dispatched: true } + }) + native.cleanup.mockImplementation(async (method, params) => { + expect(method).toBe('releaseHeldKeys') + expect(runIds).toEqual(new Set([params.runId])) + for (const key of held) events.push(`${key}:up`) + held.clear() + return { dispatched: true } + }) + f.ctx.tools.register({ + name: 'check_held_keys', description: 'Observe the keys held by the test receiver.', + parameters: { type: 'object', properties: {} }, + output: { schema: {}, render: () => [] }, + async execute() { + const keys = [...held] + if (ending === 'pause' && keys.length === 1) { + controller.abort() + await setImmediate() + expect(held.size).toBe(0) + } + return keys + }, + }) + const code = [ + "const key = (key, direction) => tools.cua({ method: 'sendKeyEvent', params: { appHandle: 'app-one', key, direction } });", + "await key('w', 'down'); await key('d', 'down');", + "if ((await tools.check_held_keys({})).join(',') !== 'w,d') throw Error('lost held keys');", + "await key('d', 'up');", + "if ((await tools.check_held_keys({})).join(',') !== 'w') throw Error('lost W');", + ending === 'throw' ? "throw Error('program failed');" : "return 'done';", + ].join('\n') + const result = await executeModelToolFixture(f.ctx, { agent: f.handle.agent, callId: ToolCallId('keyboard-run'), + name: 'run_code', arguments: { code, description: 'Exercise held keys' }, signal: controller.signal }) + expect(events, JSON.stringify(result.content)).toEqual(['w:down', 'd:down', 'd:up', 'w:up']) + expect(native.cleanup).toHaveBeenCalledTimes(1) + expect(result.isError).toBe(ending !== 'return') + if (ending === 'throw') expect(JSON.stringify(result.content)).toContain('program failed') +}) + it('rejects a relative helper path before publishing computer-use operations', async () => { const ctx = new Context() onTestFinished(() => ctx.fiber.dispose()) @@ -200,7 +272,7 @@ it('merges startup capabilities into runtime native fields', async () => { coordinateSpace: 'screenshot-pixels', methods: [ 'activate', 'click', 'drag', 'findApps', 'getApp', 'getCapabilities', 'getState', 'invoke', 'listActions', 'listApps', 'listWindows', 'mouseMove', 'paste', 'performAction', 'pressKey', 'screenshot', 'scroll', - 'selectText', 'setValue', 'show', 'typeText', + 'selectText', 'sendKeyEvent', 'setValue', 'show', 'typeText', ] }) expect(native.request).toHaveBeenCalledExactlyOnceWith('getCapabilities', {}, expect.any(AbortSignal), ToolCallId('lifecycle-1')) }) diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index d6c75f46bc..e06b6989ea 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -126,6 +126,9 @@ export interface CuaBackend { createChannel: NativeChannelFactory guidance: string capabilities: CuaStartupCapabilities + /** Dispatch a directional key through the provider's execution-scoped input owner, after application authorization. */ + sendKeyEvent?: (client: NativeClient, params: Extract['params'], + execution: Readonly) => Promise /** * Build native permission parameters for the selected action and Client language. * @param method Native permission action. @@ -211,6 +214,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) const signal = execution.signal signal.throwIfAborted() const client = execution.nativeClient + if (method === 'sendKeyEvent' && backend.sendKeyEvent) return backend.sendKeyEvent(client, parsedParams, execution) const nativeParams = method === 'findApps' ? { ...params, maxAppMatches: config.maxAppMatches } : method === 'paste' diff --git a/packages/experimental/cua/src/native-client.ts b/packages/experimental/cua/src/native-client.ts index 25c93e15ce..d2e70813ec 100644 --- a/packages/experimental/cua/src/native-client.ts +++ b/packages/experimental/cua/src/native-client.ts @@ -81,7 +81,12 @@ export class NativeClient { private diagnostics: string[] = [] private diagnosticBytes = 0 private discardedDiagnostics = false - private active: { id: string; resolve: (value: JsonValue) => void; reject: (error: NativeError) => void } | undefined + private active: { + id: string + outcome: Promise + resolve: (value: JsonValue) => void + reject: (error: NativeError) => void + } | undefined /** * @param owner - Managed subprocess provider. @@ -133,7 +138,10 @@ export class NativeClient { throw new NativeError('NATIVE_REQUEST_LIMIT', 'Request exceeded the configured byte limit before dispatch') } const outcome = Promise.withResolvers() - this.active = { id, resolve: outcome.resolve, reject: (error) => { nativeFailure = error; outcome.reject(error) } } + this.active = { + id, outcome: outcome.promise, resolve: outcome.resolve, + reject: (error) => { nativeFailure = error; outcome.reject(error) }, + } if (capture) this.diagnostic(screenshotPrefix + JSON.stringify({ requestId: id, stage: 'host.dispatch', elapsedMs: 0 })) const interrupt = (error: NativeError): void => { if (interruption || this.active?.id !== id) return @@ -193,6 +201,17 @@ export class NativeClient { } } + /** + * Dispatch cleanup after any active request acknowledges completion or cancellation. + * @param method - Host-only helper cleanup operation. + * @param params - Resources owned by the enclosing execution. + * @returns The native cleanup acknowledgement, bounded by the connection's request deadline. + */ + async requestCleanup(method: string, params: Record): Promise { + while (this.active) await this.active.outcome.catch(() => undefined) + return this.request(method, params, new AbortController().signal) + } + private screenshotDiagnostics(): string { const records = [...this.diagnostics] if (this.discardedDiagnostics) records.unshift('[Earlier screenshot diagnostic records were discarded from the bounded diagnostic buffer.]') diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index 42b545e66a..e4de9f4734 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -93,7 +93,7 @@ export const NativeResourceParameters = { } /** Host-only launch continuation and user-facing system permission controls; absent from model arguments. */ export const NativeOnlyOperations = [ - operation('launchApp', app), operation('permissionStatus', {}), + operation('launchApp', app), operation('releaseHeldKeys', { runId: z.string().min(1) }), operation('permissionStatus', {}), operation('applicationIcons', { paths: z.array(z.string().min(1)) }), operation('requestPermission', { permission: z.enum(['accessibility', 'screenRecording']) }), operation('openPermissionSettings', { permission: z.enum(['accessibility', 'screenRecording']) }), diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index da1c1eaf56..ca34f16e2c 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -169,7 +169,7 @@ export interface CuaSurface { pressKey(key: string | string[], modifiers?: string[]): Promise /** * Send one key-down or key-up event at the current input focus. - * Does not send the opposite event. + * On macOS, down persists within one run_code until explicit up, program completion (including errors), or user pause. * Rejects when the provider does not support keyboard input at this scope. * @param key - Platform-defined key name; modifier keys may themselves be targets. * @param direction - Explicit event direction; the opposite event is never sent implicitly. diff --git a/packages/experimental/cua/tests/native-client.spec.ts b/packages/experimental/cua/tests/native-client.spec.ts index f7a9007f42..19475f83fe 100644 --- a/packages/experimental/cua/tests/native-client.spec.ts +++ b/packages/experimental/cua/tests/native-client.spec.ts @@ -63,6 +63,18 @@ function fixture(options: { } describe('native helper transport', () => { + it('sends run cleanup after an interrupted native request has acknowledged its outcome', async () => { + const f = fixture() + const active = f.client.request('getState', {}, new AbortController().signal) + const release = f.client.requestCleanup('releaseHeldKeys', { runId: 'program' }) + expect(f.requests).toHaveLength(1) + f.reply({ id: f.id(), error: { code: 'NATIVE_CANCELLED', message: 'paused' } }) + await expect(active).rejects.toThrow('paused') + expect(f.requests.map(frame => frame.method)).toEqual(['getState', 'releaseHeldKeys']) + f.reply({ id: f.id(), result: { dispatched: true } }) + await expect(release).resolves.toEqual({ dispatched: true }) + }) + it('passes transport limits and connection deadlines to the shared channel', () => { const f = fixture({ maxRequestBytes: 4096, maxResponseBytes: 8192, timeoutMs: 1234 }) expect(f.connect).toHaveBeenCalledExactlyOnceWith(f.owner, f.config) diff --git a/snapshots/session/cua-native/session.v4.jsonl b/snapshots/session/cua-native/session.v4.jsonl index 10a71dc794..728ff1c9c2 100644 --- a/snapshots/session/cua-native/session.v4.jsonl +++ b/snapshots/session/cua-native/session.v4.jsonl @@ -14,7 +14,7 @@ {"type":"session/title","data":{"title":"Inspect CUA capabilities, select the","messageSeqs":[8],"source":{"kind":"fallback"}}} {"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:4}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949731596,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949731597,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}}},{"type":"chunk","time":1793949731597,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949731597,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":1,"callId":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}} -{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-1"},"toolCallId":"cua-native-1","content":[{"type":"text","text":"{\"sessionLocked\":false,\"permissions\":{\"accessibility\":true,\"screenRecording\":true},\"platform\":\"macos\",\"methods\":[\"activate\",\"click\",\"drag\",\"findApps\",\"getApp\",\"getCapabilities\",\"getState\",\"invoke\",\"listActions\",\"listApps\",\"listWindows\",\"mouseMove\",\"paste\",\"performAction\",\"pressKey\",\"screenshot\",\"scroll\",\"selectText\",\"setValue\",\"show\",\"typeText\"],\"appLevelAccessibility\":true,\"keyboard\":[\"app\"],\"browser\":false,\"coordinateSpace\":\"screenshot-pixels\"}"}],"isError":false,"id":"{{message:5}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"} +{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-1"},"toolCallId":"cua-native-1","content":[{"type":"text","text":"{\"sessionLocked\":false,\"permissions\":{\"accessibility\":true,\"screenRecording\":true},\"platform\":\"macos\",\"methods\":[\"activate\",\"click\",\"drag\",\"findApps\",\"getApp\",\"getCapabilities\",\"getState\",\"invoke\",\"listActions\",\"listApps\",\"listWindows\",\"mouseMove\",\"paste\",\"performAction\",\"pressKey\",\"screenshot\",\"scroll\",\"selectText\",\"sendKeyEvent\",\"setValue\",\"show\",\"typeText\"],\"appLevelAccessibility\":true,\"keyboard\":[\"app\"],\"browser\":false,\"coordinateSpace\":\"screenshot-pixels\"}"}],"isError":false,"id":"{{message:5}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":1}} {"type":"step/start","data":{"turn":1,"step":2}} {"type":"assistant/message","data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-unauthorized-getState","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:6}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1793949732188,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1793949732188,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-unauthorized-getState","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1793949732188,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1793949732188,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index 73107a5ee3..3ed01c96c7 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -78,7 +78,8 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. +- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — hold a physical key across tool calls within one run_code. Choose the keys, how long to hold them, and which tools to call while they are down. Send 'up' for each key when needed. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index 14f13a5a00..e8c8afed86 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -80,7 +80,8 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. +- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — hold a physical key across tool calls within one run_code. Choose the keys, how long to hold them, and which tools to call while they are down. Send 'up' for each key when needed. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. diff --git a/snapshots/session/ptc-script-retained/system-prompt.expected.md b/snapshots/session/ptc-script-retained/system-prompt.expected.md index 6e0914ac15..990f35fcd0 100644 --- a/snapshots/session/ptc-script-retained/system-prompt.expected.md +++ b/snapshots/session/ptc-script-retained/system-prompt.expected.md @@ -80,7 +80,8 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. +- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — hold a physical key across tool calls within one run_code. Choose the keys, how long to hold them, and which tools to call while they are down. Send 'up' for each key when needed. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. From 7f8a6ad4d4943787be86ae9ce1d2d461ec732a8b Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 13:36:26 +0800 Subject: [PATCH 04/11] test(cua-macos): isolate desktop key holds and cover failed pause cleanup --- docs/config-catalog.md | 2 +- docs/config-catalog.zh.md | 2 +- docs/event-producer-consumer.md | 2 +- docs/event-producer-consumer.zh.md | 2 +- .../cua-macos/native/README.i18n.yaml | 4 +- .../experimental/cua-macos/native/README.md | 2 +- .../cua-macos/native/README.zh.md | 2 +- .../native/tests/CapabilitiesFixture.swift | 5 +- .../native/tests/capabilities_e2e.py | 59 +++++++++++-------- .../experimental/cua-macos/src/held-keys.ts | 1 - .../tests/provider-lifecycle.spec.ts | 13 ++-- packages/experimental/cua/src/index.ts | 4 +- 12 files changed, 59 insertions(+), 39 deletions(-) diff --git a/docs/config-catalog.md b/docs/config-catalog.md index dede6981f9..67006d1d03 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1144,7 +1144,7 @@ export interface Config { - `inject`: `computerUse` · `tools` · `systemPrompt` · `subprocess` · `agents` - `refs`: [`CuaConfig`](../packages/experimental/cua/src/index.ts) -- `source`: [`packages/experimental/cua-macos/src/index.ts:33`](../packages/experimental/cua-macos/src/index.ts) +- `source`: [`packages/experimental/cua-macos/src/index.ts:34`](../packages/experimental/cua-macos/src/index.ts) ```ts config-catalog /** Native preparation, source-build signing, operation bounds, and profile-local application grants. */ diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index 9c142b9b43..d476f746da 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -1146,7 +1146,7 @@ export interface Config { - `inject`: `computerUse` · `tools` · `systemPrompt` · `subprocess` · `agents` - `refs`: [`CuaConfig`](../packages/experimental/cua/src/index.ts) -- `source`: [`packages/experimental/cua-macos/src/index.ts:33`](../packages/experimental/cua-macos/src/index.ts) +- `source`: [`packages/experimental/cua-macos/src/index.ts:34`](../packages/experimental/cua-macos/src/index.ts) ```ts config-catalog /** Native preparation, source-build signing, operation bounds, and profile-local application grants. */ diff --git a/docs/event-producer-consumer.md b/docs/event-producer-consumer.md index b4cffaf1aa..18901156ff 100644 --- a/docs/event-producer-consumer.md +++ b/docs/event-producer-consumer.md @@ -84,7 +84,7 @@ Application listeners are outside this package matrix. Desktop Host subscribes t | `system-prompt/assemble` | `waterfall` | [`packages/core/system-prompt/src/index.ts:31`](../packages/core/system-prompt/src/index.ts) | [`system-prompt`](../packages/core/system-prompt) (`waterfall`) | `browser-use-runtime`, [`working-directory`](../packages/session/working-directory) | | `system-prompt/change` | `emit` | [`packages/core/system-prompt/src/index.ts:37`](../packages/core/system-prompt/src/index.ts) | [`system-prompt`](../packages/core/system-prompt) (`emit`) | - | | `tools/change` | `emit` | [`packages/core/tools/src/events.ts:56`](../packages/core/tools/src/events.ts) | [`agent-preset-registry`](../packages/preset/agent-preset-registry) (`emit`), [`tools`](../packages/core/tools) (`emit`) | `browser-use-runtime` | -| `tools/execute` | `waterfall` | [`packages/core/tools/src/events.ts:29`](../packages/core/tools/src/events.ts) | [`tools`](../packages/core/tools) (`waterfall`) | `browser-use-runtime`, `browser-use-stagehand-native`, `claude-code-mods`, `computer-use-cua-driver-native`, [`session-checkpoint-policy`](../packages/session/session-checkpoint-policy), `timeout-policy` | +| `tools/execute` | `waterfall` | [`packages/core/tools/src/events.ts:29`](../packages/core/tools/src/events.ts) | [`tools`](../packages/core/tools) (`waterfall`) | `browser-use-runtime`, `browser-use-stagehand-native`, `claude-code-mods`, `computer-use-cua-driver-native`, `cua-macos`, [`session-checkpoint-policy`](../packages/session/session-checkpoint-policy), `timeout-policy` | | `tools/post-execute` | `waterfall` | [`packages/core/tools/src/events.ts:41`](../packages/core/tools/src/events.ts) | [`tools`](../packages/core/tools) (`waterfall`) | `claude-code-mods`, [`hooks-claude-code`](../packages/experimental/hooks-claude-code), [`hooks-codex`](../packages/experimental/hooks-codex), [`repeat-tool-reminder`](../packages/guard/repeat-tool-reminder), [`spill-policy`](../packages/spill/spill-policy), [`tool-fs-search`](../packages/fs/tool-fs-search) | | `tools/pre-execute` | `waterfall` | [`packages/core/tools/src/events.ts:18`](../packages/core/tools/src/events.ts) | [`tools`](../packages/core/tools) (`waterfall`) | `auto-review`, [`hooks-claude-code`](../packages/experimental/hooks-claude-code), [`hooks-codex`](../packages/experimental/hooks-codex), `liveshare`, [`tool-jobs`](../packages/jobs/tool-jobs), [`workspace-changes`](../packages/deliverables/workspace-changes) | | `tools/ptc-dispatch-log` | `waterfall` | [`packages/core/tool-executor-ptc/src/runtime-types.ts:46`](../packages/core/tool-executor-ptc/src/runtime-types.ts) | [`tool-executor-ptc`](../packages/core/tool-executor-ptc) (`waterfall`) | [`spill-policy`](../packages/spill/spill-policy) | diff --git a/docs/event-producer-consumer.zh.md b/docs/event-producer-consumer.zh.md index b1b58ef08a..9ebcc28905 100644 --- a/docs/event-producer-consumer.zh.md +++ b/docs/event-producer-consumer.zh.md @@ -86,7 +86,7 @@ | `system-prompt/assemble` | `waterfall` | [`packages/core/system-prompt/src/index.ts:31`](../packages/core/system-prompt/src/index.ts) | [`system-prompt`](../packages/core/system-prompt) (`waterfall`) | `browser-use-runtime`, [`working-directory`](../packages/session/working-directory) | | `system-prompt/change` | `emit` | [`packages/core/system-prompt/src/index.ts:37`](../packages/core/system-prompt/src/index.ts) | [`system-prompt`](../packages/core/system-prompt) (`emit`) | - | | `tools/change` | `emit` | [`packages/core/tools/src/events.ts:56`](../packages/core/tools/src/events.ts) | [`agent-preset-registry`](../packages/preset/agent-preset-registry) (`emit`), [`tools`](../packages/core/tools) (`emit`) | `browser-use-runtime` | -| `tools/execute` | `waterfall` | [`packages/core/tools/src/events.ts:29`](../packages/core/tools/src/events.ts) | [`tools`](../packages/core/tools) (`waterfall`) | `browser-use-runtime`, `browser-use-stagehand-native`, `claude-code-mods`, `computer-use-cua-driver-native`, [`session-checkpoint-policy`](../packages/session/session-checkpoint-policy), `timeout-policy` | +| `tools/execute` | `waterfall` | [`packages/core/tools/src/events.ts:29`](../packages/core/tools/src/events.ts) | [`tools`](../packages/core/tools) (`waterfall`) | `browser-use-runtime`, `browser-use-stagehand-native`, `claude-code-mods`, `computer-use-cua-driver-native`, `cua-macos`, [`session-checkpoint-policy`](../packages/session/session-checkpoint-policy), `timeout-policy` | | `tools/post-execute` | `waterfall` | [`packages/core/tools/src/events.ts:41`](../packages/core/tools/src/events.ts) | [`tools`](../packages/core/tools) (`waterfall`) | `claude-code-mods`, [`hooks-claude-code`](../packages/experimental/hooks-claude-code), [`hooks-codex`](../packages/experimental/hooks-codex), [`repeat-tool-reminder`](../packages/guard/repeat-tool-reminder), [`spill-policy`](../packages/spill/spill-policy), [`tool-fs-search`](../packages/fs/tool-fs-search) | | `tools/pre-execute` | `waterfall` | [`packages/core/tools/src/events.ts:18`](../packages/core/tools/src/events.ts) | [`tools`](../packages/core/tools) (`waterfall`) | `auto-review`, [`hooks-claude-code`](../packages/experimental/hooks-claude-code), [`hooks-codex`](../packages/experimental/hooks-codex), `liveshare`, [`tool-jobs`](../packages/jobs/tool-jobs), [`workspace-changes`](../packages/deliverables/workspace-changes) | | `tools/ptc-dispatch-log` | `waterfall` | [`packages/core/tool-executor-ptc/src/runtime-types.ts:46`](../packages/core/tool-executor-ptc/src/runtime-types.ts) | [`tool-executor-ptc`](../packages/core/tool-executor-ptc) (`waterfall`) | [`spill-policy`](../packages/spill/spill-policy) | diff --git a/packages/experimental/cua-macos/native/README.i18n.yaml b/packages/experimental/cua-macos/native/README.i18n.yaml index 2b78969668..5e6adac2ce 100644 --- a/packages/experimental/cua-macos/native/README.i18n.yaml +++ b/packages/experimental/cua-macos/native/README.i18n.yaml @@ -24,8 +24,8 @@ en: 851614a21ebfeb65 zh: ff868b14dc77adb8 /dsh-cua-native-helper/verification: - en: ade5a2ccd3aeb1b4 - zh: 74c2b0bcb23fc4bc + en: c87945aeec587bf5 + zh: e5d69e4927e2fed3 /dsh-cua-native-helper/further-exploration: en: 265e1927952ce6d7 zh: 7c35d9fd98b6e6c8 diff --git a/packages/experimental/cua-macos/native/README.md b/packages/experimental/cua-macos/native/README.md index b3074d6ce9..cc1c046ca9 100644 --- a/packages/experimental/cua-macos/native/README.md +++ b/packages/experimental/cua-macos/native/README.md @@ -159,7 +159,7 @@ For text pacing comparisons, use `native/tests/text_input_benchmark.py` with `-- For an independently isolated Electron/Chromium check, run `python3 packages/experimental/cua-macos/native/tests/electron_e2e.py --electron /absolute/path/to/Electron.app/Contents/MacOS/Electron`. Omitting `--electron` uses an already installed desktop workspace runtime. The check never downloads Electron. Its renderer runs with Node integration disabled, context isolation and Chromium sandboxing enabled, private userData/sessionData/log directories, and a local fixture page. Renderer click/input events write independent observations through the main process; all test actions still use the native helper. AX polling waits for actual controls after Chromium accessibility activation. Native and Electron subprocess groups are owned and stopped before temporary files are removed. Optional artifacts include a fixture-only screenshot and private request transcript. -Run `python3 packages/experimental/cua-macos/native/tests/capabilities_e2e.py --helper /absolute/helper --artifacts /private/evidence` to verify a stopped application's discovery and launch, Unicode selection, multiline plain/HTML paste, middle/modifier gestures, overlapping W and D key events, and keypad identities through an isolated AppKit receiver. Add `--click-only` to check click holds, AX routing and cancellation/timeout release in a background fixture without keyboard or clipboard input. It requires existing Accessibility/Screen Recording grants and records the receiver's actual state separately from dispatch results. +Run `python3 packages/experimental/cua-macos/native/tests/capabilities_e2e.py --helper /absolute/helper --artifacts /private/evidence` to verify a stopped application's discovery and launch, Unicode selection, multiline plain/HTML paste, middle/modifier gestures, overlapping W and D key events, and keypad identities through an isolated AppKit receiver. Add `--keyboard-only` for a background receiver that checks a two-second W+D hold, independent D release, and W cleanup. Add `--click-only` to check click holds, AX routing and cancellation/timeout release in a background fixture without keyboard or clipboard input. It requires existing Accessibility/Screen Recording grants and records the receiver's actual state separately from dispatch results. Run `python3 packages/experimental/cua-macos/native/tests/paste_e2e.py --artifacts /private/evidence` to verify that an owned AppKit app accepts a background Paste action without reading its data, reports `PASTE_NOT_CONSUMED`, then consumes the same paste in the foreground. The test requires an unlocked Mac and Accessibility permission, activates only its own two applications, and records received actions and text independently of helper replies. diff --git a/packages/experimental/cua-macos/native/README.zh.md b/packages/experimental/cua-macos/native/README.zh.md index 6035ffaee2..73b7754b3a 100644 --- a/packages/experimental/cua-macos/native/README.zh.md +++ b/packages/experimental/cua-macos/native/README.zh.md @@ -159,7 +159,7 @@ Blender 键盘检查使用 `python3 packages/experimental/cua-macos/native/tests 隔离的 Electron/Chromium 检查使用 `python3 packages/experimental/cua-macos/native/tests/electron_e2e.py --electron /absolute/path/to/Electron.app/Contents/MacOS/Electron`。省略 `--electron` 时使用桌面工作区已安装的运行时，检查不下载 Electron。渲染器禁用 Node 集成，启用上下文隔离和 Chromium 沙箱，使用独立 userData/sessionData/log 目录及本地 fixture 页面。渲染器的点击/输入事件通过主进程写出独立观察，测试动作仍通过原生 helper 执行。AX 轮询等待 Chromium 辅助功能启用后的真实控件。原生和 Electron 子进程组由测试持有，在删除临时文件前停止。可选产物包含仅有 fixture 的截图和私有请求记录。 -运行 `python3 packages/experimental/cua-macos/native/tests/capabilities_e2e.py --helper /absolute/helper --artifacts /private/evidence`，通过隔离的 AppKit 接收方验证未运行应用的发现和启动、Unicode 选区、多行纯文本/HTML 粘贴、带修饰键的中键手势、W 和 D 同时按住时的事件顺序及数字键盘键码。需要已有辅助功能/屏幕录制权限，接收方实际状态与分派结果分开记录。 添加 `--click-only` 可在后台测试窗口中验证按住时长、AX 路由和取消、超时后的松开事件，不发送键盘或剪贴板输入。 +运行 `python3 packages/experimental/cua-macos/native/tests/capabilities_e2e.py --helper /absolute/helper --artifacts /private/evidence`，通过隔离的 AppKit 接收方验证未运行应用的发现和启动、Unicode 选区、多行纯文本/HTML 粘贴、带修饰键的中键手势、W 和 D 同时按住时的事件顺序及数字键盘键码。需要已有辅助功能/屏幕录制权限，接收方实际状态与分派结果分开记录。添加 `--keyboard-only` 可用后台窗口验证 W、D 同时按住两秒、单独释放 D 和收尾时释放 W。添加 `--click-only` 可在后台测试窗口中验证按住时长、AX 路由和取消、超时后的松开事件，不发送键盘或剪贴板输入。 运行 `python3 packages/experimental/cua-macos/native/tests/paste_e2e.py --artifacts /private/evidence`，验证自有 AppKit 应用在后台接收 Paste 动作但不读取数据时返回 `PASTE_NOT_CONSUMED`，以及前台读取同一次粘贴后成功。测试要求 Mac 已解锁并已授予辅助功能权限，只激活自有的两个应用，并独立记录收到的动作和文本以核对 helper 回复。 diff --git a/packages/experimental/cua-macos/native/tests/CapabilitiesFixture.swift b/packages/experimental/cua-macos/native/tests/CapabilitiesFixture.swift index 5382155260..1151468790 100644 --- a/packages/experimental/cua-macos/native/tests/CapabilitiesFixture.swift +++ b/packages/experimental/cua-macos/native/tests/CapabilitiesFixture.swift @@ -92,7 +92,10 @@ final class CapabilitiesFixture: NSObject, NSApplicationDelegate, NSWindowDelega } window.contentView!.addSubview(pad) if Bundle.main.object(forInfoDictionaryKey: "DSHFixtureBackground") as? Bool == true || CommandLine.arguments.contains("--background") { - window.orderFront(nil) + if Bundle.main.object(forInfoDictionaryKey: "DSHFixtureKeyboard") as? Bool == true { + window.makeFirstResponder(pad) + window.orderBack(nil) + } else { window.orderFront(nil) } } else { window.makeKeyAndOrderFront(nil) window.makeFirstResponder(text) diff --git a/packages/experimental/cua-macos/native/tests/capabilities_e2e.py b/packages/experimental/cua-macos/native/tests/capabilities_e2e.py index 432f70e81f..6649b82dd9 100644 --- a/packages/experimental/cua-macos/native/tests/capabilities_e2e.py +++ b/packages/experimental/cua-macos/native/tests/capabilities_e2e.py @@ -123,11 +123,39 @@ def check_clicks(helper, window, point, state_file): return state +def check_keyboard(call, owner, state_file, state): + count = len(state["events"]) + call("pressKey", {**owner, "key": ["w", "d"]}) + state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 4) + chord = [(event["type"], event.get("keyCode")) for event in state["events"][count:]] + assert chord == [(10, 13), (10, 2), (11, 2), (11, 13)], chord + count = len(state["events"]) + + def key(name, direction): + call("sendKeyEvent", {**owner, "runId": "held-input", "key": name, "direction": direction}) + + key("w", "down") + key("d", "down") + state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 2) + call("getState", {**owner, "maxElements": 2000, "maxDepth": 25}) + time.sleep(2) + key("d", "up") + state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 3) + held = state["events"][count:] + assert [(event["type"], event["keyCode"]) for event in held] == [(10, 13), (10, 2), (11, 2)], held + assert held[2]["timestamp"] - held[1]["timestamp"] >= 2, held + call("releaseHeldKeys", {"runId": "held-input"}) + state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 4) + assert [(e["type"], e["keyCode"]) for e in state["events"][count:]] == [(10, 13), (10, 2), (11, 2), (11, 13)] + + def main(): parser = argparse.ArgumentParser() parser.add_argument("--helper", type=Path, required=True) parser.add_argument("--artifacts", type=Path) - parser.add_argument("--click-only", action="store_true") + only = parser.add_mutually_exclusive_group() + only.add_argument("--click-only", action="store_true") + only.add_argument("--keyboard-only", action="store_true") args = parser.parse_args() transcript = [] helper = None @@ -141,7 +169,8 @@ def main(): info = {"CFBundleName": "Capabilities Fixture", "CFBundleDisplayName": "Capabilities Fixture", "CFBundleIdentifier": "test.dsh.cua." + uuid.uuid4().hex, "CFBundleExecutable": "CapabilitiesFixture", "CFBundlePackageType": "APPL", - "DSHFixtureStatePath": str(state_file), "DSHFixtureBackground": args.click_only} + "DSHFixtureStatePath": str(state_file), "DSHFixtureBackground": args.click_only or args.keyboard_only, + "DSHFixtureKeyboard": args.keyboard_only} (application / "Contents/Info.plist").write_bytes(plistlib.dumps(info)) subprocess.run(["swiftc", str(ROOT / "tests/CapabilitiesFixture.swift"), "-o", str(executable)], check=True) try: @@ -158,6 +187,10 @@ def main(): app = call("launchApp", owner) assert app["running"] is True, app pid = wait_for_state(state_file, lambda value: value["windowId"] > 0)["pid"] + if args.keyboard_only: + check_keyboard(call, owner, state_file, json.loads(state_file.read_text())) + print("PASS: W+D short press, two-second hold, independent D up and remaining W cleanup") + return if not args.click_only: call("activate", owner) state = wait_for_state(state_file, lambda value: value["pid"] == pid and value["active"] and value["keyWindow"]) @@ -188,27 +221,7 @@ def main(): if not args.click_only: call("click", {**window, **point, "mouseButton": "middle", "modifiers": ["shift"]}) state = wait_for_state(state_file, lambda value: any(event["type"] == 26 and event.get("button") == 2 and event["shift"] for event in value["events"])) - count = len(state["events"]) - call("pressKey", {**owner, "key": ["w", "d"]}) - state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 4) - chord = [(event["type"], event.get("keyCode")) for event in state["events"][count:]] - assert chord == [(10, 13), (10, 2), (11, 2), (11, 13)], chord - count = len(state["events"]) - def key(name, direction): - call("sendKeyEvent", {**owner, "runId": "held-input", "key": name, "direction": direction}) - key("w", "down") - key("d", "down") - state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 2) - call("getState", {**owner, "maxElements": 2000, "maxDepth": 25}) - time.sleep(2) - key("d", "up") - state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 3) - held = state["events"][count:] - assert [(event["type"], event["keyCode"]) for event in held] == [(10, 13), (10, 2), (11, 2)], held - assert held[2]["timestamp"] - held[1]["timestamp"] >= 2, held - call("releaseHeldKeys", {"runId": "held-input"}) - state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 4) - assert [(e["type"], e["keyCode"]) for e in state["events"][count:]] == [(10, 13), (10, 2), (11, 2), (11, 13)] + check_keyboard(call, owner, state_file, state) call("drag", {**window, "fromX": point["x"], "fromY": point["y"], "toX": point["x"] + 80, "toY": point["y"] + 20, "durationMs": 200, "steps": 10, "mouseButton": "middle", "modifiers": ["shift"]}) state = wait_for_state(state_file, lambda value: any(event["type"] == 27 and event.get("button") == 2 and event["shift"] for event in value["events"])) diff --git a/packages/experimental/cua-macos/src/held-keys.ts b/packages/experimental/cua-macos/src/held-keys.ts index 30c1311da8..0cfb58c6fa 100644 --- a/packages/experimental/cua-macos/src/held-keys.ts +++ b/packages/experimental/cua-macos/src/held-keys.ts @@ -35,7 +35,6 @@ export function installHeldKeys(ctx: Context): NonNullable { const run = execution.parent && runs.get(execution.parent) if (!run) throw new NativeError('INVALID_ARGUMENT', 'Use sendKeyEvent inside an active run_code execution') - if (run.client && run.client !== client) throw new NativeError('NATIVE_CLOSED', 'The keyboard connection changed; start a new run_code') run.client = client return client.request('sendKeyEvent', { ...params, runId: run.id }, execution.signal, execution.callId) } diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index 194f7244fd..a6efc825a9 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -37,12 +37,14 @@ import type { NativeClient } from '@deepseek-ai/dsh-cua/native-client' import type { NativeBuildSpec } from '../src/native-build.ts' const native = vi.hoisted(() => ({ + connect: vi.fn(), prepare: vi.fn<(spec: NativeBuildSpec) => Promise>(), architecture: vi.fn(() => 'arm64' as const), cleanup: vi.fn(), request: vi.fn(), dispose: vi.fn(), instances: [] as Array<{ disposed: boolean }>, reportDiagnostic: undefined as ((message: string) => void) | undefined, })) +vi.mock('../src/shared-native.ts', () => ({ createSharedNativeChannel: native.connect })) vi.mock('../src/native-build.ts', async importOriginal => ({ ...await importOriginal(), nativeHostArchitecture: native.architecture, @@ -53,6 +55,7 @@ vi.mock('@deepseek-ai/dsh-cua/native-client', async importOriginal => ({ NativeClient: class { disposed = false constructor(...args: ConstructorParameters) { + args[2](args[0], args[1]) native.reportDiagnostic = args[3] native.instances.push(this) } @@ -114,6 +117,7 @@ const imageData = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoA const imageMetadata = { pixelWidth: 1, pixelHeight: 1, bounds: { x: 2, y: 3, width: 1, height: 1 }, coordinateSpace: 'window-points', windowId: 7 } async function setup(model?: { provider: string; model: string }, ptc = false) { + native.connect.mockClear() native.cleanup.mockReset().mockResolvedValue({ dispatched: true }) native.request.mockReset().mockResolvedValue([]) native.dispose.mockReset() @@ -157,7 +161,7 @@ it('requires an enclosing run_code before sending a directional key', async () = expect(native.request).not.toHaveBeenCalled() }) -it.each(['return', 'throw', 'pause'])('cleans up only the held W when run_code ends by %s', async (ending) => { +it.each(['return', 'throw', 'pause', 'pause with cleanup error'])('cleans up only the held W when run_code ends by %s', async (ending) => { const f = await setup(undefined, true) await selectApplication(f) const held = new Set() @@ -176,6 +180,7 @@ it.each(['return', 'throw', 'pause'])('cleans up only the held W when run_code e native.cleanup.mockImplementation(async (method, params) => { expect(method).toBe('releaseHeldKeys') expect(runIds).toEqual(new Set([params.runId])) + if (ending === 'pause with cleanup error') throw new Error('release acknowledgement failed') for (const key of held) events.push(`${key}:up`) held.clear() return { dispatched: true } @@ -186,10 +191,10 @@ it.each(['return', 'throw', 'pause'])('cleans up only the held W when run_code e output: { schema: {}, render: () => [] }, async execute() { const keys = [...held] - if (ending === 'pause' && keys.length === 1) { + if (ending.startsWith('pause') && keys.length === 1) { controller.abort() await setImmediate() - expect(held.size).toBe(0) + expect(held.size).toBe(ending === 'pause' ? 0 : 1) } return keys }, @@ -204,7 +209,7 @@ it.each(['return', 'throw', 'pause'])('cleans up only the held W when run_code e ].join('\n') const result = await executeModelToolFixture(f.ctx, { agent: f.handle.agent, callId: ToolCallId('keyboard-run'), name: 'run_code', arguments: { code, description: 'Exercise held keys' }, signal: controller.signal }) - expect(events, JSON.stringify(result.content)).toEqual(['w:down', 'd:down', 'd:up', 'w:up']) + expect(events, JSON.stringify(result.content)).toEqual(['w:down', 'd:down', 'd:up', ...ending === 'pause with cleanup error' ? [] : ['w:up']]) expect(native.cleanup).toHaveBeenCalledTimes(1) expect(result.isError).toBe(ending !== 'return') if (ending === 'throw') expect(JSON.stringify(result.content)).toContain('program failed') diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index e06b6989ea..d88a958711 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -127,7 +127,7 @@ export interface CuaBackend { guidance: string capabilities: CuaStartupCapabilities /** Dispatch a directional key through the provider's execution-scoped input owner, after application authorization. */ - sendKeyEvent?: (client: NativeClient, params: Extract['params'], + sendKeyEvent?: (client: NativeClient, params: Record, execution: Readonly) => Promise /** * Build native permission parameters for the selected action and Client language. @@ -214,7 +214,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) const signal = execution.signal signal.throwIfAborted() const client = execution.nativeClient - if (method === 'sendKeyEvent' && backend.sendKeyEvent) return backend.sendKeyEvent(client, parsedParams, execution) + if (method === 'sendKeyEvent' && backend.sendKeyEvent) return backend.sendKeyEvent(client, params, execution) const nativeParams = method === 'findApps' ? { ...params, maxAppMatches: config.maxAppMatches } : method === 'paste' From de0515a771cd93ab0f4f92d6712900188046f1c6 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 13:41:15 +0800 Subject: [PATCH 05/11] fix(cua-macos): retain the run signal and validate test receiver events --- packages/experimental/cua-macos/README.i18n.yaml | 2 +- packages/experimental/cua-macos/README.zh.md | 2 +- packages/experimental/cua-macos/src/held-keys.ts | 5 +++-- .../cua-macos/tests/provider-lifecycle.spec.ts | 7 ++++--- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index 1abb4f6399..bb17939f06 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -16,7 +16,7 @@ zh: e3daac1074ebb74d /deepseek-ai-dsh-experimental-cua-macos/use-this-package: en: f4206e2cb725c917 - zh: 37f0b04e9e74c193 + zh: d10792bd8e7e242b /deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment: en: b734a0907747580e zh: 8110c5951b70e100 diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index 0f6f4dad51..51826288e4 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -81,7 +81,7 @@ pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis. `pressKey(key, modifiers?)` 接受单个键名或由不同物理键组成的非空数组。`Enter` 和 `Return` 等别名指向同一个物理键。例如，`pressKey(['w', 'd'])` 依次按下 W、D，再依次松开 D、W。需要全程按住的修饰键单独传入，例如 `pressKey('Tab', ['shift'])`。单独调用 `pressKey('shift')` 时，按下事件带有 Shift 标志，释放事件清除该标志。单独传入 `+` 或 `plus` 会发送 Shift 和等号。在多键组合中请使用 `=`，需要全程按住 Shift 时将其列入修饰键。macOS helper 接受 `command`/`cmd`、`shift`、`option`/`alt`、`control`/`ctrl` 和 `fn`，并映射到对应物理键。按键名由平台定义；数字键盘名称（`KP_0` 到 `KP_9`、`KP_Enter` 和数字键盘运算键）保留独立的物理键码。`typeText` 发送字面文本，包括其中的换行和制表符。短按使用 App 对象上的 `pressKey`。 -长按时，在 `run_code` 中调用 `app.sendKeyEvent('w', 'down')`。按下成功后，在同一次执行中调用其他工具时该键仍保持按下。每个键可分别松开。例如先按 W，再按 D，松开 D 后 W 仍按着。程序正常返回、出错退出或用户暂停时，helper 向每个键原先的接收进程松开该次执行仍按着的键。后续执行不能释放前一次执行的键。 +长按时，在 `run_code` 中调用 `app.sendKeyEvent('w', 'down')`。按下成功后，在同一次执行中调用其他工具时该键仍保持按下。每个键可分别松开。例如先按 W，再按 D，松开 D 后 W 仍按着。程序正常返回、出错退出或用户暂停时，helper 为本次执行仍按着的键逐一发送松开事件，每个事件都发回原接收进程。后续执行不能释放前一次执行的键。 Agent 销毁和插件卸载会取消并等待完整工具调用，包括图像接入检查和附件存储。已经开始的图像写入必须结束后，销毁才会返回；已取消的调用不会返回成功截图。 diff --git a/packages/experimental/cua-macos/src/held-keys.ts b/packages/experimental/cua-macos/src/held-keys.ts index 0cfb58c6fa..fee0519c53 100644 --- a/packages/experimental/cua-macos/src/held-keys.ts +++ b/packages/experimental/cua-macos/src/held-keys.ts @@ -24,11 +24,12 @@ export function installHeldKeys(ctx: Context): NonNullable { void release()?.catch(() => undefined) } - execution.signal.addEventListener('abort', onAbort, { once: true }) + const signal = execution.signal + signal.addEventListener('abort', onAbort, { once: true }) try { return await next() } finally { - execution.signal.removeEventListener('abort', onAbort) + signal.removeEventListener('abort', onAbort) await release() } }) diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index a6efc825a9..f65e1a6a61 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -171,10 +171,11 @@ it.each(['return', 'throw', 'pause', 'pause with cleanup error'])('cleans up onl native.request.mockImplementation(async (method, params) => { expect(method).toBe('sendKeyEvent') runIds.add(params.runId) - const key = String(params.key) - if (params.direction === 'down') held.add(key) + const { key, direction } = params + if (typeof key !== 'string' || (direction !== 'down' && direction !== 'up')) throw new Error('Invalid directional event') + if (direction === 'down') held.add(key) else held.delete(key) - events.push(`${key}:${params.direction}`) + events.push(`${key}:${direction}`) return { dispatched: true } }) native.cleanup.mockImplementation(async (method, params) => { From a9e0950206c575c09c84312ac410a860a230048a Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 14:10:32 +0800 Subject: [PATCH 06/11] docs(cua-macos): explain physical key repeat in sendKeyEvent guidance --- packages/experimental/cua-macos/README.md | 2 +- packages/experimental/cua-macos/README.zh.md | 2 +- packages/experimental/cua-macos/src/guidance.ts | 2 +- snapshots/session/cua-native/system-prompt.expected.md | 2 +- snapshots/session/cua-ptc/system-prompt.expected.md | 2 +- snapshots/session/ptc-script-retained/system-prompt.expected.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 042cc4fa94..02b260705a 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -224,7 +224,7 @@ Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. - pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — hold a physical key across tool calls within one run_code. Choose the keys, how long to hold them, and which tools to call while they are down. Send 'up' for each key when needed. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once to hold a key across tool calls within one run_code. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. ``` diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index 51826288e4..f67195c67b 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -224,7 +224,7 @@ Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. - pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — hold a physical key across tool calls within one run_code. Choose the keys, how long to hold them, and which tools to call while they are down. Send 'up' for each key when needed. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once to hold a key across tool calls within one run_code. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. ``` diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index e750c4068b..2fe07ef025 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -14,7 +14,7 @@ export const GUIDANCE = `${PLATFORM_GUIDANCE} ${COMMON_GUIDANCE.replace(/^- ([\w.]+)\(.*$/gm, (entry, method: string) => { if (method === 'pressKey') return `- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — hold a physical key across tool calls within one run_code. Choose the keys, how long to hold them, and which tools to call while they are down. Send 'up' for each key when needed. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.` +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once to hold a key across tool calls within one run_code. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.` const detail = methodDetails.get(method) return detail ? `${entry} ${detail}` : entry })}` diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index 3ed01c96c7..82210fa306 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -79,7 +79,7 @@ Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. - pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — hold a physical key across tool calls within one run_code. Choose the keys, how long to hold them, and which tools to call while they are down. Send 'up' for each key when needed. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once to hold a key across tool calls within one run_code. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index e8c8afed86..3acaa30b88 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -81,7 +81,7 @@ Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. - pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — hold a physical key across tool calls within one run_code. Choose the keys, how long to hold them, and which tools to call while they are down. Send 'up' for each key when needed. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once to hold a key across tool calls within one run_code. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. diff --git a/snapshots/session/ptc-script-retained/system-prompt.expected.md b/snapshots/session/ptc-script-retained/system-prompt.expected.md index 990f35fcd0..dc18435318 100644 --- a/snapshots/session/ptc-script-retained/system-prompt.expected.md +++ b/snapshots/session/ptc-script-retained/system-prompt.expected.md @@ -81,7 +81,7 @@ Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. - pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — hold a physical key across tool calls within one run_code. Choose the keys, how long to hold them, and which tools to call while they are down. Send 'up' for each key when needed. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once to hold a key across tool calls within one run_code. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. From ab9700e6ac3d8165aa3bc410198396c1e898011e Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 14:19:08 +0800 Subject: [PATCH 07/11] refactor(cua-macos): reserve overlapping ordinary keys for sendKeyEvent --- ...05-cua-scoped-surfaces-and-input.i18n.yaml | 8 +- ...026-11-05-cua-scoped-surfaces-and-input.md | 4 +- ...-11-05-cua-scoped-surfaces-and-input.zh.md | 4 +- .../experimental/cua-macos/README.i18n.yaml | 4 +- packages/experimental/cua-macos/README.md | 6 +- packages/experimental/cua-macos/README.zh.md | 6 +- .../cua-macos/native/README.i18n.yaml | 4 +- .../experimental/cua-macos/native/README.md | 2 +- .../cua-macos/native/README.zh.md | 2 +- .../native/Sources/CuaNative/AXWorker.swift | 2 +- .../Sources/CuaNative/KeyboardInput.swift | 27 ++----- .../Sources/CuaNative/RequestAdmission.swift | 7 -- .../cua-macos/native/generate-protocol.ts | 4 +- .../cua-macos/native/protocol.schema.json | 79 ++++++++----------- .../native/tests/capabilities_e2e.py | 12 +-- .../tests/swift/KeyboardChordTests.swift | 48 +++++------ .../native/tests/swift/ProtocolTests.swift | 12 --- .../experimental/cua-macos/src/guidance.ts | 4 +- .../cua-macos/tests/guidance.spec.ts | 5 +- .../tests/protocol-generation.spec.ts | 8 ++ packages/experimental/cua/README.i18n.yaml | 4 +- packages/experimental/cua/README.md | 2 +- packages/experimental/cua/README.zh.md | 2 +- .../experimental/cua/src/native-schema.ts | 8 +- packages/experimental/cua/src/protocol.ts | 7 +- packages/experimental/cua/src/types.ts | 8 +- packages/experimental/cua/tests/api.spec.ts | 5 +- .../experimental/cua/tests/protocol.spec.ts | 7 -- .../cua-native/system-prompt.expected.md | 4 +- .../cua-native/tool-schemas.expected.json | 17 +--- .../session/cua-ptc/system-prompt.expected.md | 4 +- .../system-prompt.expected.md | 4 +- 32 files changed, 128 insertions(+), 192 deletions(-) diff --git a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.i18n.yaml b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.i18n.yaml index 716f07d599..3e32b6eb92 100644 --- a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.i18n.yaml @@ -9,11 +9,11 @@ en: f72db0b79aa02fb5 zh: b91c1f177dc93ab4 /agent-note-cua-scoped-surfaces-and-separated-input-capabilities/decision: - en: 74f97ec371e61b82 - zh: a8a5cefc1e06bb20 + en: ec115e11f96ffe60 + zh: c1fe9eb2866196d0 /agent-note-cua-scoped-surfaces-and-separated-input-capabilities/alternatives-considered: - en: 2113f17a49e16670 - zh: 1df04d9736a3ad4f + en: 32c970ae153e0f6c + zh: 8431be0a10ffed15 /agent-note-cua-scoped-surfaces-and-separated-input-capabilities/consequences: en: c4b12ca80865e670 zh: 396426fad0105eaa diff --git a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.md b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.md index 6b4fe604d5..cce266e52b 100644 --- a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.md +++ b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.md @@ -14,7 +14,7 @@ Application and window objects share accessibility operations, but only an expli Provider startup capabilities are static. Platform, supported methods, application-level accessibility, keyboard scopes, browser support, and coordinate space are fixed when the provider is installed. The native `getCapabilities` RPC reports runtime permissions and session lock only; `installCua` merges the two layers for the model-visible result. The neutral RPC is a common operation representation, not a capability inventory; provider guidance names the actions its platform supports. -Keyboard input keeps keys and modifiers separate: `pressKey(key, modifiers?)` presses one key or, on macOS, several physical keys in order and releases them in reverse while holding the listed modifiers, and its keyboard modifier list is a plain string array defined by the provider. `sendKeyEvent(key, direction)` sends exactly one key-down or key-up without an implicit counterpart. A caller can therefore hold a modifier or split a gesture; neither method packs a chord string, and the shared schema does not enumerate platform key names. +Keyboard input keeps keys and modifiers separate: `pressKey(key, modifiers?)` presses and releases one key while holding the listed modifiers, and its keyboard modifier list is a plain string array defined by the provider. `sendKeyEvent(key, direction)` sends exactly one key-down or key-up without an implicit counterpart. A caller can therefore hold a modifier or split a gesture; neither method packs a chord string, and the shared schema does not enumerate platform key names. Within one API instance, the object facade keeps one current accessibility snapshot per application handle, shared by every application and window object for that handle, including repeated selections of the same application. `getWindow` reads its initial tree silently; `getApp` does the same when the provider reports application-level accessibility. A later `getAccessibilityTree` read invalidates the previous snapshot before dispatch. The facade automatically attaches the latest snapshot ID to element operations; callers use element IDs from the latest tree and do not assume an older ID still identifies the same control. Element IDs are opaque provider strings. Only an explicit `CuaWindow` captures screenshots or sends pointer input. The native helper retains its own application and optional-window snapshots behind that facade; the native layer and the facade snapshot remain separate. The external runtime continues to own variable persistence and completion serialization; the facade adds no evaluator. The [native object decision](2026-09-17-experimental-cua-object-api.md) owns evaluator independence, and the [shared helper decision](2026-10-09-cua-shared-native-helper.md) owns connection lifetime. @@ -24,7 +24,7 @@ Within one API instance, the object facade keeps one current accessibility snaps **An application-specific branch in the shared base** would let window objects inherit decisions and capability checks that apply only to applications. Scope factories that inject their own requirements keep the shared operations independent of which scope uses them. -**A packed chord string** requires parsing key names, separators, and plus escaping together, and it pushes platform alias normalization into the shared layer. Keystroke and direction are separate semantics: `pressKey` describes a complete press and release, while `sendKeyEvent` describes one explicit direction. Separate key and modifier values keep those semantics without a parser. +**A packed chord string** requires parsing key names, separators, and plus escaping together, and it pushes platform alias normalization into the shared layer. Keystroke and direction are separate semantics: `pressKey` describes one complete keystroke, while `sendKeyEvent` describes one explicit direction. Separate key and modifier values keep those semantics without a parser. **Platform keyboard key enumerations in the shared schema** would couple the portable operation schema to one provider's key names and require a schema change for each platform. Provider-defined strings let the provider interpret its own names while the shared schema stays neutral. diff --git a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.zh.md b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.zh.md index d03c3b4723..5135dc9be8 100644 --- a/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.zh.md +++ b/.agents/notes/implemented/architecture/2026-11-05-cua-scoped-surfaces-and-input.zh.md @@ -14,7 +14,7 @@ Status: implemented Provider 的启动能力是静态的。平台、支持的方法、应用级辅助功能、键盘作用域、浏览器支持和坐标空间在安装 provider 时固定。原生 `getCapabilities` RPC 只报告运行时权限和会话锁定状态；`installCua` 为模型可见结果合并两层。中立的 RPC 是通用操作表示，不是能力清单；provider 指导文本说明其平台支持的动作。 -键盘输入把按键与修饰键分开：`pressKey(key, modifiers?)` 在按住所列修饰键时按下并释放指定的键。macOS 也接受多个物理键，依次按下并逆序释放。修饰键名称由提供方定义，以字符串数组传入。`sendKeyEvent(key, direction)` 只发送一次按下或抬起，不隐式发送相反事件。因此调用方可以保持修饰键或拆分手势；两个方法都不接受组合键字符串，共享 schema 也不枚举平台按键名。 +键盘输入把按键与修饰键分开：`pressKey(key, modifiers?)` 在按住所列修饰键时按下并释放一个键，其键盘修饰键列表是 provider 定义的普通字符串数组。`sendKeyEvent(key, direction)` 只发送一次按下或抬起，不隐式发送相反事件。因此调用方可以保持修饰键或拆分手势；两个方法都不接受组合键字符串，共享 schema 也不枚举平台按键名。 在单个 API 实例内，对象门面按应用句柄保存一份当前辅助功能快照，该句柄的所有应用对象和窗口对象共享这份快照，包括对同一应用的重复选择。`getWindow` 静默执行首次树读取；当 provider 报告应用级辅助功能时，`getApp` 也静默读取。之后的 `getAccessibilityTree` 读取会在分派前使上一份快照失效。门面会为元素操作自动附加最新快照 ID；调用方应使用最新树中的元素 ID，不应假定旧 ID 仍指向同一控件。元素 ID 是 provider 定义的不透明字符串。只有显式 `CuaWindow` 可以截图或发送指针输入。原生 helper 在该门面之后保留自己的应用快照和可选窗口快照；原生层与门面快照相互独立。外部运行时继续负责变量持久化和最终表达式序列化；门面不加入 evaluator。[原生对象决定](2026-09-17-experimental-cua-object-api.zh.md)负责 evaluator 独立性，[共享 helper 决定](2026-10-09-cua-shared-native-helper.zh.md)负责连接生命期。 @@ -24,7 +24,7 @@ Provider 的启动能力是静态的。平台、支持的方法、应用级辅 **在共享基类中加入应用专属分支**会让窗口对象继承只适用于应用的决定和能力检查。由各作用域工厂注入自己的要求，可以让共享操作不依赖使用它的作用域。 -**打包的组合键字符串**需要一起解析按键名、分隔符和加号转义，并把平台别名归一化推入共享层。击键与方向是两种语义：`pressKey` 在一次调用中完成按下和释放，`sendKeyEvent` 描述一个显式方向。分开的按键和修饰键值让这些语义无需解析器。 +**打包的组合键字符串**需要一起解析按键名、分隔符和加号转义，并把平台别名归一化推入共享层。击键与方向是两种语义：`pressKey` 描述一次完整击键，`sendKeyEvent` 描述一个显式方向。分开的按键和修饰键值让这些语义无需解析器。 **在共享 schema 中枚举平台键盘按键**会把可移植的操作 schema 绑定到某个 provider 的按键名，并让每个平台都修改 schema。由 provider 定义字符串，平台可以解释自己的名称，共享 schema 保持中立。 diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index bb17939f06..00c45988de 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -15,8 +15,8 @@ en: 1777e7bf6785f986 zh: e3daac1074ebb74d /deepseek-ai-dsh-experimental-cua-macos/use-this-package: - en: f4206e2cb725c917 - zh: d10792bd8e7e242b + en: db5e8ad4d2ea832a + zh: 647cf55f4851760e /deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment: en: b734a0907747580e zh: 8110c5951b70e100 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 02b260705a..971acf4818 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -79,7 +79,7 @@ Source builds default to `sourceSigning: ad-hoc`, without creating certificates The [configuration catalog](../../../docs/config-catalog.md#deepseek-aidsh-experimental-cua-macos) lists all fields. Native tool calls use `{method, params}`; the complete method-specific schema is exposed to the model. `getApp` returns the `appHandle` used by every app-targeted operation, and `app.getWindow(windowId)` returns the object that owns screenshots and pointer input. -`pressKey(key, modifiers?)` accepts one key or a nonempty array of distinct physical keys. Aliases such as `Enter` and `Return` identify the same key. `pressKey(['w', 'd'])` presses W then D and releases D then W. Pass held modifiers separately, as in `pressKey('Tab', ['shift'])`. Pressing a modifier itself, as in `pressKey('shift')`, sets its flag on press and clears it on release. A single `+` or `plus` sends Shift+=. In a multi-key chord, use `=` and pass Shift explicitly to apply it to the whole chord. The macOS helper accepts `command`/`cmd`, `shift`, `option`/`alt`, `control`/`ctrl`, and `fn`, and maps them to their physical keys. Key names are platform-defined; keypad names (`KP_0` through `KP_9`, `KP_Enter`, and keypad operators) retain their physical key codes. `typeText` sends literal text, including its line breaks and tabs. Use the App object's `pressKey` for a short press. +`pressKey(key, modifiers?)` presses and releases one key. For shortcuts, pass held modifiers separately, as in `pressKey('a', ['command'])`. Pressing a modifier itself, as in `pressKey('shift')`, sets its flag on press and clears it on release. A single `+` or `plus` sends Shift+=. The macOS helper accepts `command`/`cmd`, `shift`, `option`/`alt`, `control`/`ctrl`, and `fn`, and maps them to their physical keys. Key names are platform-defined; keypad names (`KP_0` through `KP_9`, `KP_Enter`, and keypad operators) retain their physical key codes. `typeText` sends literal text, including its line breaks and tabs. Use the App object's `pressKey` for a short press. For a long press, call `app.sendKeyEvent('w', 'down')` inside `run_code`. Each successful down remains active across subsequent tool calls in that execution. Choose when to send each up: after W down, D down and D up, W stays down. When the program returns, exits with an error, or the user pauses it, the helper releases the remaining keys at the processes that received their downs. A later execution cannot release an earlier execution's keys. @@ -223,8 +223,8 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once to hold a key across tool calls within one run_code. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once per key to hold one or more keys across tool calls within one run_code, for example W down, D down, D up, W up. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. ``` diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index f67195c67b..57c2f3f371 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -79,7 +79,7 @@ pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis. [配置目录](../../../docs/config-catalog.zh.md#deepseek-aidsh-experimental-cua-macos)列出全部字段。原生工具调用使用 `{method, params}`，模型可以看到每个方法的完整参数定义。`getApp` 返回每个面向应用的操作所需的 `appHandle`，`app.getWindow(windowId)` 返回负责截图和指针输入的对象。 -`pressKey(key, modifiers?)` 接受单个键名或由不同物理键组成的非空数组。`Enter` 和 `Return` 等别名指向同一个物理键。例如，`pressKey(['w', 'd'])` 依次按下 W、D，再依次松开 D、W。需要全程按住的修饰键单独传入，例如 `pressKey('Tab', ['shift'])`。单独调用 `pressKey('shift')` 时，按下事件带有 Shift 标志，释放事件清除该标志。单独传入 `+` 或 `plus` 会发送 Shift 和等号。在多键组合中请使用 `=`，需要全程按住 Shift 时将其列入修饰键。macOS helper 接受 `command`/`cmd`、`shift`、`option`/`alt`、`control`/`ctrl` 和 `fn`，并映射到对应物理键。按键名由平台定义；数字键盘名称（`KP_0` 到 `KP_9`、`KP_Enter` 和数字键盘运算键）保留独立的物理键码。`typeText` 发送字面文本，包括其中的换行和制表符。短按使用 App 对象上的 `pressKey`。 +`pressKey(key, modifiers?)` 按下并松开一个键。使用快捷键时单独传入修饰键，例如 `pressKey('a', ['command'])`。单独调用 `pressKey('shift')` 时，按下事件带有 Shift 标志，释放事件清除该标志。`+` 和 `plus` 都会发送 Shift 和等号。macOS helper 接受 `command`/`cmd`、`shift`、`option`/`alt`、`control`/`ctrl` 和 `fn`，并映射到对应物理键。按键名由平台定义；数字键盘名称（`KP_0` 到 `KP_9`、`KP_Enter` 和数字键盘运算键）保留独立的物理键码。`typeText` 发送字面文本，包括其中的换行和制表符。短按使用 App 对象上的 `pressKey`。 长按时，在 `run_code` 中调用 `app.sendKeyEvent('w', 'down')`。按下成功后，在同一次执行中调用其他工具时该键仍保持按下。每个键可分别松开。例如先按 W，再按 D，松开 D 后 W 仍按着。程序正常返回、出错退出或用户暂停时，helper 为本次执行仍按着的键逐一发送松开事件，每个事件都发回原接收进程。后续执行不能释放前一次执行的键。 @@ -223,8 +223,8 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once to hold a key across tool calls within one run_code. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once per key to hold one or more keys across tool calls within one run_code, for example W down, D down, D up, W up. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. ``` diff --git a/packages/experimental/cua-macos/native/README.i18n.yaml b/packages/experimental/cua-macos/native/README.i18n.yaml index 5e6adac2ce..58af404d5d 100644 --- a/packages/experimental/cua-macos/native/README.i18n.yaml +++ b/packages/experimental/cua-macos/native/README.i18n.yaml @@ -21,8 +21,8 @@ en: 2a69d89b7fc95998 zh: 084575bb31fb40dd /dsh-cua-native-helper/observation-and-input: - en: 851614a21ebfeb65 - zh: ff868b14dc77adb8 + en: 0bbc620165d868fa + zh: abe73bd9c7b8cc5d /dsh-cua-native-helper/verification: en: c87945aeec587bf5 zh: e5d69e4927e2fed3 diff --git a/packages/experimental/cua-macos/native/README.md b/packages/experimental/cua-macos/native/README.md index cc1c046ca9..cc3a0de35f 100644 --- a/packages/experimental/cua-macos/native/README.md +++ b/packages/experimental/cua-macos/native/README.md @@ -103,7 +103,7 @@ Clicks send mouse-down and mouse-up events at the element center or the supplied Capture and pointer delivery read CG bounds before and after an exact-window AX frame query. Movement during the query returns WINDOW_CHANGED. Valid AX/CG frames must have equal outward-rounded integral point rectangles; a known mismatch returns WINDOW_CHANGED before mouse movement, priming or button delivery. The helper preserves both coordinate sources rather than substituting AX geometry for CG geometry. Missing or unusable AX geometry remains unknown, while AX communication, permission and invalid-element errors propagate. Capture without Accessibility permission retains its CG/SCK checks. Advertised AX actions bypass this pointer preflight; an AX action error does not trigger pointer input. These reads do not lock the window against later movement, and no geometry check activates or raises it. -`typeText` and `pressKey` target the selected process's current focused control, not an exact window. `pressKey` presses its modifiers followed by the requested keys in order, then releases all pressed keys in reverse order, including on cancellation. Keyboard events go directly to the target PID and bypass the global HID queue. Key names are platform-defined; keypad names such as `KP_0` and `KP_Enter` preserve their physical key codes. Background pointer actions may change that keyboard focus. Text injection sends Unicode characters individually and does not use the clipboard. LF, CR and CRLF send one Return; tabs send Tab. Applications apply their normal editing behavior, including indentation and focus navigation. Each `typeText` down/up event is followed by the configured text interval; `pressKey` keeps its own press and release timing, and dispatch completion does not confirm that the receiving application consumed every event. Cancellation can interrupt text or a gesture after partial input; it cannot undo received events. Very long text can exceed the host's operation deadline. Validate observed application state before deciding how to continue. +`typeText` and `pressKey` target the selected process's current focused control, not an exact window. `pressKey` presses the listed modifiers followed by one key, then releases all pressed keys in reverse order, including on cancellation. Keyboard events go directly to the target PID and bypass the global HID queue. Key names are platform-defined; keypad names such as `KP_0` and `KP_Enter` preserve their physical key codes. Background pointer actions may change that keyboard focus. Text injection sends Unicode characters individually and does not use the clipboard. LF, CR and CRLF send one Return; tabs send Tab. Applications apply their normal editing behavior, including indentation and focus navigation. Each `typeText` down/up event is followed by the configured text interval; `pressKey` keeps its own press and release timing, and dispatch completion does not confirm that the receiving application consumed every event. Cancellation can interrupt text or a gesture after partial input; it cannot undo received events. Very long text can exceed the host's operation deadline. Validate observed application state before deciding how to continue. The Host associates directional key events with one `run_code` execution. On down, the helper retains the selected application, recipient PID and process start time, physical key code, and matching up event. Explicit up and cleanup verify the original process identity. On return, error, or user pause, the Host waits for any in-flight native request and asks the helper to release only that execution’s remaining keys. Disconnect cleanup also releases keys owned by the closed connection. diff --git a/packages/experimental/cua-macos/native/README.zh.md b/packages/experimental/cua-macos/native/README.zh.md index 73b7754b3a..1b0c3f406f 100644 --- a/packages/experimental/cua-macos/native/README.zh.md +++ b/packages/experimental/cua-macos/native/README.zh.md @@ -103,7 +103,7 @@ AppKit 在主事件循环上运行。阻塞式 socket 接收与握手使用后 截图和鼠标输入在精确窗口的 AX 边界查询前后读取 CG 边界，查询期间移动会返回 WINDOW_CHANGED。有效的 AX/CG 边界向外取整后必须一致；已知不一致会在移动、预备事件或按键发送前返回 WINDOW_CHANGED。helper 保留两个坐标来源，不用 AX 几何替换 CG 几何。缺失或无效的 AX 几何仍表示未知，AX 通信、权限和无效元素错误继续传递。无辅助功能权限的截图保留 CG/SCK 检查。声明的 AX 动作不经过鼠标预检，AX 动作失败也不改用鼠标输入。这些读取不会锁住窗口以阻止后续移动，几何检查不激活或抬起窗口。 -`typeText` 和 `pressKey` 面向所选进程当前获得焦点的控件，不指定窗口。`pressKey` 向目标 PID 依次发送修饰键和指定按键的按下事件，再逆序释放。操作中途取消也会逆序释放已按下的键。键盘事件直接发给目标进程，绕过全局 HID 队列。按键名由平台定义；KP_0 和 KP_Enter 等数字键盘名称保留物理键码。后台鼠标操作可能改变键盘焦点。文本逐个 Unicode 字符输入，不使用剪贴板。LF、CR 和 CRLF 发送一次 Return，制表符发送 Tab。应用保留缩进和焦点导航等正常编辑行为。`typeText` 的每次按下和抬起后等待配置的文本事件间隔；`pressKey` 使用自己的按下与释放节奏，分派完成不证明应用已消费全部事件。取消可能中断部分文本或手势，无法撤销已接收事件。很长的文本可能超过 Host 操作期限。继续输入前应核实应用状态。 +`typeText` 和 `pressKey` 面向所选进程当前获得焦点的控件，不指定窗口。`pressKey` 向目标 PID 依次发送修饰键和一个主键的按下事件，再逆序释放。操作中途取消也会逆序释放已按下的键。键盘事件直接发给目标进程，绕过全局 HID 队列。按键名由平台定义；KP_0 和 KP_Enter 等数字键盘名称保留物理键码。后台鼠标操作可能改变键盘焦点。文本逐个 Unicode 字符输入，不使用剪贴板。LF、CR 和 CRLF 发送一次 Return，制表符发送 Tab。应用保留缩进和焦点导航等正常编辑行为。`typeText` 的每次按下和抬起后等待配置的文本事件间隔；`pressKey` 使用自己的按下与释放节奏，分派完成不证明应用已消费全部事件。取消可能中断部分文本或手势，无法撤销已接收事件。很长的文本可能超过 Host 操作期限。继续输入前应核实应用状态。 宿主为每次 `run_code` 的 `sendKeyEvent` 附上执行标识。按下时，helper 保存已选应用、接收进程的 PID 与启动时间、物理键码及对应的松开事件。显式松键或执行结束时会核对原进程身份。正常返回、异常退出或用户暂停时，宿主等待在途原生请求结束，再通知 helper 释放该执行持有的其余按键。连接断开时，helper 也会清理该连接持有的按键。 diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift index 80d820d0f1..1ebde9ee41 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift @@ -345,7 +345,7 @@ actor AXWorker { let recipient = try ax().keyboardRecipient(identity, catalog: catalog) try NativePlatform.verify(recipient) if method == "typeText" { try inputs().0.typeText(params.string("text"), to: recipient.pid) } - else { try inputs().0.pressKey(params.pressKeys(), modifiers: params.modifiers(), to: recipient.pid) } + else { try inputs().0.pressKey(params.string("key"), modifiers: params.modifiers(), to: recipient.pid) } case "setValue": try ax().setAttribute(element(handle, params), "AXValue", params.values["value"]) case "performAction": try ax().performAction(element(handle, params), action: params.string("action"), argument: params.values["argument"]) diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift index 91572fbd5f..1a7ca086ba 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift @@ -43,30 +43,15 @@ final class KeyboardInput { } /// Prepare every event before dispatch. On cancellation, release only the keys this call pressed. - func pressKey(_ keys: [String], modifiers: [String], to pid: Int32) throws { + func pressKey(_ name: String, modifiers: [String], to pid: Int32) throws { try checkCancellation() var chord = try modifiers.map(Self.modifier) - var namesByCode: [CGKeyCode: String] = [:] - for name in keys { - let key: Key - if name == "+" || name.lowercased() == "plus" { - guard keys.count == 1 else { - throw RequestParameters.invalid("For a multi-key chord, use = with an explicit shift modifier instead of \(name)") - } - chord.append(try Self.modifier("shift")) - key = Key(keyCode: try Self.keyCode("="), flag: []) - } else if let modifier = Self.namedModifier(name) { - key = modifier - } else { - key = Key(keyCode: try Self.keyCode(name), flag: []) - } - if let previous = namesByCode.updateValue(name, forKey: key.keyCode) { - throw RequestParameters.invalid("Keys '\(previous)' and '\(name)' name the same physical key") - } - chord.append(key) - } + let isPlus = name == "+" || name.lowercased() == "plus" + if isPlus { chord.append(try Self.modifier("shift")) } + let code = try Self.keyCode(isPlus ? "=" : name) + chord.append(Key(keyCode: code, flag: Self.namedModifier(name)?.flag ?? [])) let alreadyHeld = held.snapshot(pid: pid) - guard namesByCode.keys.allSatisfy({ alreadyHeld[$0] == nil }) else { + guard alreadyHeld[code] == nil else { throw RequestParameters.invalid("Release the held key before using it in pressKey") } var selected = Set(alreadyHeld.keys) diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift index 6b69122d99..e3d64894c1 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift @@ -36,13 +36,6 @@ struct RequestParameters: Sendable { return path } } - func pressKeys() throws -> [String] { - switch values["key"] { - case .string(let name) where !name.isEmpty: return [name] - case .array(let items) where !items.isEmpty: return try strings("key") - default: throw Self.invalid("key must be a nonempty string or a nonempty array of key names") - } - } func optionalString(_ key: String) throws -> String? { contains(key) ? try string(key) : nil } func optionalBoolean(_ key: String) throws -> Bool? { guard let raw = values[key] else { return nil } diff --git a/packages/experimental/cua-macos/native/generate-protocol.ts b/packages/experimental/cua-macos/native/generate-protocol.ts index 29ccc58344..6dafb10d01 100644 --- a/packages/experimental/cua-macos/native/generate-protocol.ts +++ b/packages/experimental/cua-macos/native/generate-protocol.ts @@ -13,7 +13,9 @@ export function renderNativeProtocol(): { json: string; swift: string } { sendKeyEvent: z.object({ runId: z.string().min(1) }), mouseMove: z.object({ allowSystemPointer: z.boolean().optional() }), requestPermission: PermissionGuideParameters, openPermissionSettings: PermissionSettingsParameters, - }) + }, [z.object({ + method: z.literal('releaseHeldKeys'), params: z.object({ runId: z.string().min(1) }).strict(), + }).strict()]) const quoted = (values: Iterable) => [...values].sort().map(value => JSON.stringify(value)).join(', ') const swift = [ '// Generated by native/generate-protocol.ts from @deepseek-ai/dsh-cua/native-schema. Do not edit.', diff --git a/packages/experimental/cua-macos/native/protocol.schema.json b/packages/experimental/cua-macos/native/protocol.schema.json index f8a9824752..5c347ae54c 100644 --- a/packages/experimental/cua-macos/native/protocol.schema.json +++ b/packages/experimental/cua-macos/native/protocol.schema.json @@ -993,21 +993,8 @@ "maximum": 4294967295 }, "key": { - "anyOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "minItems": 1, - "type": "array", - "items": { - "type": "string", - "minLength": 1 - } - } - ], - "description": "One key name or distinct physical keys to press in order and release in reverse." + "type": "string", + "minLength": 1 }, "modifiers": { "type": "array", @@ -1309,37 +1296,6 @@ ], "additionalProperties": false }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "method": { - "type": "string", - "const": "releaseHeldKeys" - }, - "params": { - "type": "object", - "properties": { - "runId": { - "type": "string", - "minLength": 1 - } - }, - "required": [ - "runId" - ], - "additionalProperties": false - } - }, - "required": [ - "id", - "method", - "params" - ], - "additionalProperties": false - }, { "type": "object", "properties": { @@ -1500,6 +1456,37 @@ "params" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "method": { + "type": "string", + "const": "releaseHeldKeys" + }, + "params": { + "type": "object", + "properties": { + "runId": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "runId" + ], + "additionalProperties": false + } + }, + "required": [ + "id", + "method", + "params" + ], + "additionalProperties": false } ] } diff --git a/packages/experimental/cua-macos/native/tests/capabilities_e2e.py b/packages/experimental/cua-macos/native/tests/capabilities_e2e.py index 6649b82dd9..86f5cdef80 100644 --- a/packages/experimental/cua-macos/native/tests/capabilities_e2e.py +++ b/packages/experimental/cua-macos/native/tests/capabilities_e2e.py @@ -125,10 +125,12 @@ def check_clicks(helper, window, point, state_file): def check_keyboard(call, owner, state_file, state): count = len(state["events"]) - call("pressKey", {**owner, "key": ["w", "d"]}) - state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 4) - chord = [(event["type"], event.get("keyCode")) for event in state["events"][count:]] - assert chord == [(10, 13), (10, 2), (11, 2), (11, 13)], chord + call("pressKey", {**owner, "key": ["w", "d"]}, "INVALID_ARGUMENT") + call("pressKey", {**owner, "key": "d", "modifiers": ["w"]}, "INVALID_ARGUMENT") + call("pressKey", {**owner, "key": "d", "modifiers": ["shift"]}) + state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 2) + shortcut = [(event["type"], event.get("keyCode"), event["shift"]) for event in state["events"][count:]] + assert shortcut == [(10, 2, True), (11, 2, True)], shortcut count = len(state["events"]) def key(name, direction): @@ -189,7 +191,7 @@ def main(): pid = wait_for_state(state_file, lambda value: value["windowId"] > 0)["pid"] if args.keyboard_only: check_keyboard(call, owner, state_file, json.loads(state_file.read_text())) - print("PASS: W+D short press, two-second hold, independent D up and remaining W cleanup") + print("PASS: Shift+D shortcut, two-second W+D hold, independent D up and remaining W cleanup") return if not args.click_only: call("activate", owner) diff --git a/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift b/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift index 52cae6911e..f81b10d1da 100644 --- a/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift +++ b/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift @@ -4,49 +4,43 @@ import Darwin import Testing @testable import CuaNative -@Test(arguments: [ - (["w", "d"], ["shift"], [56, 13, 2, 2, 13, 56]), - (["shift", "w", "d"], [], [56, 13, 2, 2, 13, 56]), - (["a", "="], ["shift"], [56, 0, 24, 24, 0, 56]), -]) -func chordKeepsShiftThroughBothKeyReleases(_ keys: [String], _ modifiers: [String], _ codes: [Int64]) throws { +@Test func shortcutHoldsCommandAndShiftAroundOneKey() throws { var events: [CGEvent] = [] let input = KeyboardInput(post: { event, pid in #expect(pid == 42) events.append(event) }, textEventInterval: 0) - try input.pressKey(keys, modifiers: modifiers, to: 42) - #expect(events.map { $0.getIntegerValueField(.keyboardEventKeycode) } == codes) - #expect(events.map(\.type) == [.flagsChanged, .keyDown, .keyDown, .keyUp, .keyUp, .flagsChanged]) - #expect(events.map { $0.flags.contains(.maskShift) } == [true, true, true, true, true, false]) + try input.pressKey("s", modifiers: ["cmd", "shift"], to: 42) + #expect(events.map { $0.getIntegerValueField(.keyboardEventKeycode) } == [55, 56, 1, 1, 56, 55]) + #expect(events.map(\.type) == [.flagsChanged, .flagsChanged, .keyDown, .keyUp, .flagsChanged, .flagsChanged]) + #expect(events.map { $0.flags.contains(.maskShift) } == [false, true, true, true, false, false]) + #expect(events.map { $0.flags.contains(.maskCommand) } == [true, true, true, true, true, false]) } @Test(arguments: [1, 2]) -func cancelledChordReleasesOnlyPressedKeys(_ cancelAfter: Int) { +func cancelledShortcutReleasesOnlyPressedKeys(_ cancelAfter: Int) { let cancellation = CancellationLatch() var events: [CGEvent] = [] let input = KeyboardInput(post: { event, _ in events.append(event) if events.count == cancelAfter { cancellation.cancel() } }, textEventInterval: 0, checkCancellation: cancellation.check) - #expect(throws: NativeFailure.self) { try input.pressKey(["w", "d"], modifiers: [], to: 42) } - let held = [Int64(13), 2].prefix(cancelAfter) + #expect(throws: NativeFailure.self) { try input.pressKey("d", modifiers: ["shift"], to: 42) } + let held = [Int64(56), 2].prefix(cancelAfter) #expect(events.map { $0.getIntegerValueField(.keyboardEventKeycode) } == Array(held) + held.reversed()) - #expect(events.map(\.type) == Array(repeating: .keyDown, count: cancelAfter) + Array(repeating: .keyUp, count: cancelAfter)) + let types: [CGEventType] = cancelAfter == 1 ? [.flagsChanged, .flagsChanged] : [.flagsChanged, .keyDown, .keyUp, .flagsChanged] + #expect(events.map(\.type) == types) } @Test(arguments: [ - (["w", "unknown"], "Unknown key name: unknown"), - (["w", "W"], "Keys 'w' and 'W' name the same physical key"), - (["Enter", "Return"], "Keys 'Enter' and 'Return' name the same physical key"), - (["a", "+"], "For a multi-key chord, use = with an explicit shift modifier instead of +"), - (["Plus", "w"], "For a multi-key chord, use = with an explicit shift modifier instead of Plus"), + ("unknown", ["shift"], "Unknown key name: unknown"), + ("w", ["d"], "Unsupported key modifier: d"), ]) -func invalidChordPostsNoInput(_ keys: [String], _ message: String) throws { - let input = KeyboardInput(post: { _, _ in Issue.record("Invalid chord posted input") }, textEventInterval: 0) +func invalidShortcutPostsNoInput(_ key: String, _ modifiers: [String], _ message: String) throws { + let input = KeyboardInput(post: { _, _ in Issue.record("Invalid shortcut posted input") }, textEventInterval: 0) do { - try input.pressKey(keys, modifiers: ["shift"], to: 42) - Issue.record("Invalid chord was accepted") + try input.pressKey(key, modifiers: modifiers, to: 42) + Issue.record("Invalid shortcut was accepted") } catch let error as NativeFailure { #expect(error.code == "INVALID_ARGUMENT") #expect(error.message == message) @@ -57,7 +51,7 @@ func invalidChordPostsNoInput(_ keys: [String], _ message: String) throws { func plusAlonePressesShiftEquals(_ alias: String) throws { var events: [CGEvent] = [] let input = KeyboardInput(post: { event, _ in events.append(event) }, textEventInterval: 0) - try input.pressKey([alias], modifiers: [], to: 42) + try input.pressKey(alias, modifiers: [], to: 42) #expect(events.map { $0.getIntegerValueField(.keyboardEventKeycode) } == [56, 24, 24, 56]) #expect(events.map { $0.flags.contains(.maskShift) } == [true, true, true, false]) } @@ -65,7 +59,7 @@ func plusAlonePressesShiftEquals(_ alias: String) throws { @Test func standaloneShiftSetsItsFlagOnlyWhileHeld() throws { var events: [CGEvent] = [] let input = KeyboardInput(post: { event, _ in events.append(event) }, textEventInterval: 0) - try input.pressKey(["shift"], modifiers: [], to: 42) + try input.pressKey("shift", modifiers: [], to: 42) #expect(events.map(\.type) == [.flagsChanged, .flagsChanged]) #expect(events.map { $0.flags.contains(.maskShift) } == [true, false]) } @@ -82,8 +76,8 @@ func plusAlonePressesShiftEquals(_ alias: String) throws { try input.keyDown("w", app: "editor", run: "two", recipient: target) #expect(throws: NativeFailure.self) { try input.keyDown("W", app: "editor", run: "one", recipient: target) } #expect(throws: NativeFailure.self) { try input.keyUp("w", app: "other", run: "two") } - #expect(throws: NativeFailure.self) { try input.pressKey(["w"], modifiers: [], to: target.pid) } - try input.pressKey(["d"], modifiers: ["shift"], to: target.pid) + #expect(throws: NativeFailure.self) { try input.pressKey("w", modifiers: [], to: target.pid) } + try input.pressKey("d", modifiers: ["shift"], to: target.pid) try input.releaseHeldKeys(run: "one") #expect(try input.keyUp("w", app: "editor", run: "two")) try input.releaseHeldKeys(run: "two") diff --git a/packages/experimental/cua-macos/native/tests/swift/ProtocolTests.swift b/packages/experimental/cua-macos/native/tests/swift/ProtocolTests.swift index 85fa37aaf0..84a9fec849 100644 --- a/packages/experimental/cua-macos/native/tests/swift/ProtocolTests.swift +++ b/packages/experimental/cua-macos/native/tests/swift/ProtocolTests.swift @@ -138,15 +138,3 @@ func rejectsMalformedApplicationIconPaths(_ value: String) throws { let request = try NativeRequest(data: Data("{\"id\":\"icons\",\"method\":\"applicationIcons\",\"params\":{\"paths\":\(value)}}".utf8)) #expect(throws: NativeFailure.self) { try RequestParameters(request).strings("paths") } } - -@Test func invalidPressKeyTypeNamesBothAcceptedForms() throws { - let request = try NativeRequest(data: Data(#"{"id":"call","method":"pressKey","params":{"key":7}}"#.utf8)) - let params = try RequestParameters(request) - do { - _ = try params.pressKeys() - Issue.record("Numeric key was admitted") - } catch let error as NativeFailure { - #expect(error.code == "INVALID_ARGUMENT") - #expect(error.message == "key must be a nonempty string or a nonempty array of key names") - } -} diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index 2fe07ef025..6b2f073924 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -13,8 +13,8 @@ const methodDetails = new Map([ export const GUIDANCE = `${PLATFORM_GUIDANCE} ${COMMON_GUIDANCE.replace(/^- ([\w.]+)\(.*$/gm, (entry, method: string) => { - if (method === 'pressKey') return `- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once to hold a key across tool calls within one run_code. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.` + if (method === 'pressKey') return `${entry} +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once per key to hold one or more keys across tool calls within one run_code, for example W down, D down, D up, W up. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.` const detail = methodDetails.get(method) return detail ? `${entry} ${detail}` : entry })}` diff --git a/packages/experimental/cua-macos/tests/guidance.spec.ts b/packages/experimental/cua-macos/tests/guidance.spec.ts index 54322e2316..542dbe74bb 100644 --- a/packages/experimental/cua-macos/tests/guidance.spec.ts +++ b/packages/experimental/cua-macos/tests/guidance.spec.ts @@ -31,9 +31,8 @@ it('states the macOS accessibility, keyboard, and mouse levels', () => { expect(GUIDANCE).toContain("sendKeyEvent(key: string, direction: 'down' | 'up')") expect(GUIDANCE).toContain('returns, throws, or the user pauses') expect(COMMON_GUIDANCE).not.toContain('sendKeyEvent(') - expect(pressKey).toContain('key: string | string[]') - expect(pressKey).toContain("pressKey(['w', 'd']) presses W then D and releases D then W") - expect(pressKey).toContain("Packed strings such as 'w+d'") + expect(pressKey).toBe(COMMON_GUIDANCE.split('\n').find(line => line.startsWith('- pressKey('))) + expect(GUIDANCE).toContain('W down, D down, D up, W up') const hover = GUIDANCE.split('\n').find(line => line.startsWith('- mouseMove(')) for (const term of ['AXShowAlternateUI', 'AXShowDefaultUI', 'HOVER_UNAVAILABLE', 'systemPointerHover']) expect(hover).toContain(term) const click = GUIDANCE.split('\n').find(line => line.startsWith('- click(')) diff --git a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts index aefc6fbee6..78135ed0c2 100644 --- a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts +++ b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts @@ -21,6 +21,13 @@ describe('native protocol generation', () => { expect(validate({ id: 'invalid', method: 'setValue', params: target })).toBe(false) }) + it('rejects a key array at both model and native pressKey admission', () => { + const request = { method: 'pressKey', params: { appHandle: 'app', key: ['w', 'd'] } } + expect(Request.safeParse(request).success).toBe(false) + const validate = new Ajv2020().compile(JSON.parse(renderNativeProtocol().json)) + expect(validate({ id: 'shortcut', ...request })).toBe(false) + }) + it.each([ { method: 'getState', params: { appHandle: 'app', maxElements: 1 } }, { method: 'drag', params: { appHandle: 'app', windowId: 1, fromX: 1, fromY: 1, toX: 2, toY: 2, steps: 1 } }, @@ -28,6 +35,7 @@ describe('native protocol generation', () => { { method: 'paste', params: { appHandle: 'app', text: 'text', pasteReadTimeoutMs: 1000 } }, { method: 'mouseMove', params: { appHandle: 'app', windowId: 1, x: 0, y: 0, allowSystemPointer: true } }, { method: 'findApps', params: { query: 'Editor', maxAppMatches: 1 } }, + { method: 'releaseHeldKeys', params: { runId: 'execution' } }, { method: 'launchApp', params: { appHandle: 'pending' } }, { method: 'applicationIcons', params: { paths: ['/Applications/Fixture.app'] } }, { method: 'permissionStatus', params: {} }, diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index 8587d8c7d5..f4abb13c43 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -21,8 +21,8 @@ en: d92d5948a7941e8d zh: d43491ae6521b87e /deepseek-ai-dsh-cua/understand-the-implementation: - en: 181158347d09c629 - zh: 231b2dcaf826d386 + en: 2ab74a1ff120459e + zh: 39b3ef198989f133 /deepseek-ai-dsh-cua/further-exploration: en: c174fb5a00ea3681 zh: 039340042676881d diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index 9fb8f64eb7..c95e4ae8ec 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -91,7 +91,7 @@ One object facade ([src/api.ts](src/api.ts)) builds a shared `CuaSurface` and de Provider startup capabilities are fixed at installation: platform, methods, app-level accessibility, keyboard scopes, browser support, and coordinate space. The native `getCapabilities` RPC reports runtime permissions and session lock only; `installCua` merges both layers for the model-visible result. The raw RPC remains a common operation representation rather than a capability inventory, so provider guidance states the actions its platform supports. -Check `methods` and `keyboard` in the result of `await cua.getCapabilities()` for the supported operations and input scopes. `pressKey(key, modifiers?)` holds the listed modifiers while pressing the requested keys in order and releasing them in reverse. For example, `pressKey(['w', 'd'])` holds W through the D keystroke. `sendKeyEvent(key, direction)` sends exactly one key-down or key-up. `typeText` and `paste` deliver literal text at the provider's declared keyboard scope. Keyboard modifier names are plain strings that the provider interprets. +Check `methods` and `keyboard` in the result of `await cua.getCapabilities()` for the supported operations and input scopes. `pressKey(key, modifiers?)` presses and releases one key while holding the listed modifiers; `sendKeyEvent(key, direction)` sends exactly one key-down or key-up. `typeText` and `paste` deliver literal text at the provider's declared keyboard scope. Keyboard modifier names are plain strings that the provider interprets. Platform providers import `CuaConfigFields` from `@deepseek-ai/dsh-cua/config` to extend the shared configuration. The root Host entry exports `installCua`, `CuaConfig`, and the Typert-discovered `CuaAccess` service; its `apply()` only marks the optional Client contribution. `/client` loads the permission UI through the existing module loader. Separate Host and Client compiler projects keep React, CSS, and browser registrations out of headless imports. `/api` owns the app/window API and its types; `/bridge`, `/native-client`, and `/protocol` retain runtime and platform consumers. `/types` exports only the named types required by generated Remote declarations. The Client uses package-local types. `/typert` supplies Host reflection and `/remote` supplies generated browser descriptors and codecs; permission authority remains on the Host. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index 2f8e31b77c..c1c5ed863c 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -91,7 +91,7 @@ Computer Use 详情页的权限卡片、组件列表和来源信息使用 l3 边 提供方启动能力在安装时固定：平台、方法、应用级辅助功能、键盘作用域、浏览器支持和坐标空间。原生 `getCapabilities` RPC 只报告运行时权限和会话锁定状态；`installCua` 为模型可见结果合并两层。原生 RPC 仍是通用操作表示，而不是能力清单，因此 provider 指引说明其平台支持的动作。 -通过 `await cua.getCapabilities()` 返回的 `methods` 和 `keyboard` 确认支持的操作和输入范围。`pressKey(key, modifiers?)` 按住所列修饰键，依次按下指定的键，再逆序释放。例如，`pressKey(['w', 'd'])` 在按下和松开 D 期间持续按住 W。`sendKeyEvent(key, direction)` 只发送一次按下或抬起。`typeText` 和 `paste` 在 provider 声明的键盘作用域内传送字面文本。键盘修饰键名称是 provider 解释的普通字符串。 +通过 `await cua.getCapabilities()` 返回的 `methods` 和 `keyboard` 确认支持的操作和输入范围。`pressKey(key, modifiers?)` 在按住所列修饰键时按下并释放一个键；`sendKeyEvent(key, direction)` 只发送一次按下或抬起。`typeText` 和 `paste` 在 provider 声明的键盘作用域内传送字面文本。键盘修饰键名称是 provider 解释的普通字符串。 平台提供方从 `@deepseek-ai/dsh-cua/config` 导入 `CuaConfigFields` 来扩展共享配置。根 Host 入口导出 `installCua`、`CuaConfig` 和供 Typert 发现的 `CuaAccess` 服务，`apply()` 只标记可选 Client 贡献。`/client` 通过现有模块加载器加载权限界面。Host 和 Client 使用独立编译项目，headless 导入不会加载 React、CSS 或浏览器注册。`/api` 提供应用、窗口 API 及其类型；`/bridge`、`/native-client` 和 `/protocol` 保留运行时和平台消费者。`/types` 仅导出生成 Remote 声明所需的命名类型，Client 使用包内类型。`/typert` 提供 Host 反射，`/remote` 提供浏览器描述和编解码，权限判断仍由 Host 执行。 diff --git a/packages/experimental/cua/src/native-schema.ts b/packages/experimental/cua/src/native-schema.ts index a6122ed71e..5bb061383f 100644 --- a/packages/experimental/cua/src/native-schema.ts +++ b/packages/experimental/cua/src/native-schema.ts @@ -14,15 +14,19 @@ function variants(schema: Schema): Schema[] { return schema.anyOf ?? schema.oneO /** * Derive native request schemas and admission fields from the shared operation definitions. * @param platformFields Additional native protocol fields accepted by the provider. + * @param privateOperations Additional provider-only request schemas. * @returns Native field and enum tables and the complete wire schema. */ -export function nativeProtocol(platformFields: Partial['method'] | 'requestPermission' | 'openPermissionSettings', z.ZodObject>> = {}): { +export function nativeProtocol( + platformFields: Partial['method'] | 'requestPermission' | 'openPermissionSettings', z.ZodObject>> = {}, + privateOperations: readonly z.ZodObject[] = [], +): { fields: Map> enums: Map>> json: string } { const resources = new Map(Object.entries({ ...NativeResourceParameters, ...platformFields })) - const schemas = [...Request.options, ...NativeOnlyOperations] + const schemas = [...Request.options, ...NativeOnlyOperations, ...privateOperations] .map(schema => z.toJSONSchema(schema)) const fields = new Map>() const enums = new Map>>() diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index e4de9f4734..538c95cfe6 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -62,10 +62,7 @@ export const Request = z.discriminatedUnion('method', [ selectionType: z.enum(['text', 'cursor_before', 'cursor_after']).optional() }), operation('typeText', { ...app, windowId: id.optional(), text: z.string() }), operation('paste', { ...app, windowId: id.optional(), text: z.string(), format: z.enum(['text', 'html']).optional() }), - operation('pressKey', { ...app, windowId: id.optional(), - key: z.union([z.string().min(1), z.array(z.string().min(1)).min(1)]) - .describe('One key name or distinct physical keys to press in order and release in reverse.'), - modifiers: z.array(z.string()).optional() }), + operation('pressKey', { ...app, windowId: id.optional(), key: z.string().min(1), modifiers: z.array(z.string()).optional() }), operation('sendKeyEvent', { ...app, windowId: id.optional(), key: z.string().min(1), direction: z.enum(['down', 'up']) }), z.object({ method: z.literal('scroll'), params: z.union([ z.object({ ...absoluteCoordinate, @@ -93,7 +90,7 @@ export const NativeResourceParameters = { } /** Host-only launch continuation and user-facing system permission controls; absent from model arguments. */ export const NativeOnlyOperations = [ - operation('launchApp', app), operation('releaseHeldKeys', { runId: z.string().min(1) }), operation('permissionStatus', {}), + operation('launchApp', app), operation('permissionStatus', {}), operation('applicationIcons', { paths: z.array(z.string().min(1)) }), operation('requestPermission', { permission: z.enum(['accessibility', 'screenRecording']) }), operation('openPermissionSettings', { permission: z.enum(['accessibility', 'screenRecording']) }), diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index ca34f16e2c..b84663f9e2 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -160,16 +160,16 @@ export interface CuaSurface { */ paste(text: string, options?: CuaPasteOptions): Promise /** - * Press the keys in order at the current focus, then release them in reverse order. + * Press and release one key at the current input focus. * Rejects when the provider does not support keyboard input at this scope. - * @param key - One provider-defined key name or a nonempty array of distinct physical keys to hold together. + * @param key - Provider-defined key name to press and release; not a chord. * @param modifiers - Platform- or provider-defined modifier names held for this keystroke; omitted means none. * @returns Whether the provider successfully executed the requested operation. */ - pressKey(key: string | string[], modifiers?: string[]): Promise + pressKey(key: string, modifiers?: string[]): Promise /** * Send one key-down or key-up event at the current input focus. - * On macOS, down persists within one run_code until explicit up, program completion (including errors), or user pause. + * Does not send the opposite event. * Rejects when the provider does not support keyboard input at this scope. * @param key - Platform-defined key name; modifier keys may themselves be targets. * @param direction - Explicit event direction; the opposite event is never sent implicitly. diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index f5b3452dc3..cbaa419951 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -683,8 +683,6 @@ describe('input actions', () => { ['fractional scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 0.5, deltaLinesY: 0 })], ['negative drag point', ({ window }) => window.drag([0, 0], [-1, 3])], ['empty key', ({ app }) => app.pressKey('')], - ['empty chord', ({ app }) => app.pressKey([])], - ['empty chord member', ({ app }) => app.pressKey(['w', ''])], ] it.each<[string, (surfaces: Surfaces) => Promise]>([ ['getState windowId: Too small: expected number to be >0', ({ app }) => app.getWindow(0)], @@ -710,8 +708,7 @@ describe('input actions', () => { expect(f.dispatch).toHaveBeenCalledTimes(calls) }) - it.each<[string | string[], string[] | undefined, string[]]>([ - [['w', 'd'], undefined, []], + it.each<[string, string[] | undefined, string[]]>([ ['+', undefined, []], ['plus', [], []], ['+', ['command'], ['command']], diff --git a/packages/experimental/cua/tests/protocol.spec.ts b/packages/experimental/cua/tests/protocol.spec.ts index c3d69e309c..50fb329c07 100644 --- a/packages/experimental/cua/tests/protocol.spec.ts +++ b/packages/experimental/cua/tests/protocol.spec.ts @@ -12,8 +12,6 @@ const rejectedRequests: Array<{ reason: string; request: JsonValue }> = [ { reason: 'numeric element id', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementId: 0 } } }, { reason: 'coordinate click without a window', request: { method: 'click', params: { appHandle: 'app', x: 1, y: 2 } } }, { reason: 'mixed element and coordinate click', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementId: '0', windowId: 23, x: 1, y: 2 } } }, - { reason: 'empty pressKey chord', request: { method: 'pressKey', params: { appHandle: 'app', key: [] } } }, - { reason: 'non-string pressKey chord member', request: { method: 'pressKey', params: { appHandle: 'app', key: ['w', 7] } } }, { reason: 'non-string pressKey modifier', request: { method: 'pressKey', params: { appHandle: 'app', key: 'a', modifiers: ['command', 7] } } }, { reason: 'sendKeyEvent modifiers', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: 'a', direction: 'down', modifiers: [] } } }, { reason: 'sendKeyEvent without direction', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: 'a' } } }, @@ -25,11 +23,6 @@ it.each(rejectedRequests)('rejects $reason at operation admission', ({ request } expect(Request.safeParse(request).success).toBe(false) }) -it('admits an ordered chord in a raw pressKey request', () => { - const request = { method: 'pressKey', params: { appHandle: 'app', key: ['w', 'd'] } } - expect(Request.parse(request)).toEqual(request) -}) - it('admits arbitrary provider-defined pressKey modifier names', () => { const request = { method: 'pressKey', params: { appHandle: 'app', key: 'PlatformKey', modifiers: ['platform-modifier', 'shift'] } } expect(Request.parse(request)).toEqual(request) diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index 82210fa306..e8deb643fb 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -78,8 +78,8 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once to hold a key across tool calls within one run_code. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once per key to hold one or more keys across tool calls within one run_code, for example W down, D down, D up, W up. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. diff --git a/snapshots/session/cua-native/tool-schemas.expected.json b/snapshots/session/cua-native/tool-schemas.expected.json index 3a94712d06..33f3636750 100644 --- a/snapshots/session/cua-native/tool-schemas.expected.json +++ b/snapshots/session/cua-native/tool-schemas.expected.json @@ -1032,21 +1032,8 @@ "maximum": 4294967295 }, "key": { - "anyOf": [ - { - "type": "string", - "minLength": 1 - }, - { - "minItems": 1, - "type": "array", - "items": { - "type": "string", - "minLength": 1 - } - } - ], - "description": "One key name or distinct physical keys to press in order and release in reverse." + "type": "string", + "minLength": 1 }, "modifiers": { "type": "array", diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index 3acaa30b88..5322aa44d7 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -80,8 +80,8 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once to hold a key across tool calls within one run_code. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once per key to hold one or more keys across tool calls within one run_code, for example W down, D down, D up, W up. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. diff --git a/snapshots/session/ptc-script-retained/system-prompt.expected.md b/snapshots/session/ptc-script-retained/system-prompt.expected.md index dc18435318..4c5b91329b 100644 --- a/snapshots/session/ptc-script-retained/system-prompt.expected.md +++ b/snapshots/session/ptc-script-retained/system-prompt.expected.md @@ -80,8 +80,8 @@ A string target is an element id; coordinate targets are [x, y] and are pixels i Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. -- pressKey(key: string | string[], modifiers?: string[]) -> ActionResult — use for a short press. pressKey(['w', 'd']) presses W then D and releases D then W. Each name identifies one physical key. Packed strings such as 'w+d', unknown names and duplicate physical keys are rejected before input. Pass held modifiers separately, as in pressKey('Tab', ['shift']). '+' and 'plus' mean Shift+= when used alone. In a multi-key chord use '=' and pass 'shift' explicitly if needed. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once to hold a key across tool calls within one run_code. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once per key to hold one or more keys across tool calls within one run_code, for example W down, D down, D up, W up. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. From dd5cda398ceb95fd50e1c3310a7fd429e9c8cf66 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 14:47:29 +0800 Subject: [PATCH 08/11] feat(cua-macos): send owner key repeats and explain conflicting holds --- .../experimental/cua-macos/README.i18n.yaml | 4 +- packages/experimental/cua-macos/README.md | 4 +- packages/experimental/cua-macos/README.zh.md | 4 +- .../cua-macos/native/README.i18n.yaml | 4 +- .../experimental/cua-macos/native/README.md | 2 +- .../cua-macos/native/README.zh.md | 2 +- .../native/Sources/CuaNative/AXWorker.swift | 12 +++- .../native/Sources/CuaNative/HeldKeys.swift | 46 +++++++++++--- .../Sources/CuaNative/KeyboardInput.swift | 20 +++--- .../native/tests/CapabilitiesFixture.swift | 7 ++- .../native/tests/capabilities_e2e.py | 13 +++- .../tests/swift/KeyboardChordTests.swift | 61 +++++++++++++++++-- .../experimental/cua-macos/src/guidance.ts | 2 +- .../cua-native/system-prompt.expected.md | 2 +- .../session/cua-ptc/system-prompt.expected.md | 2 +- .../system-prompt.expected.md | 2 +- 16 files changed, 146 insertions(+), 41 deletions(-) diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index 00c45988de..1be5a7abd9 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -15,8 +15,8 @@ en: 1777e7bf6785f986 zh: e3daac1074ebb74d /deepseek-ai-dsh-experimental-cua-macos/use-this-package: - en: db5e8ad4d2ea832a - zh: 647cf55f4851760e + en: 83d4eb62d0d0db2f + zh: 3afde1c3b8c25211 /deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment: en: b734a0907747580e zh: 8110c5951b70e100 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 971acf4818..26a218c40d 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -81,7 +81,7 @@ The [configuration catalog](../../../docs/config-catalog.md#deepseek-aidsh-exper `pressKey(key, modifiers?)` presses and releases one key. For shortcuts, pass held modifiers separately, as in `pressKey('a', ['command'])`. Pressing a modifier itself, as in `pressKey('shift')`, sets its flag on press and clears it on release. A single `+` or `plus` sends Shift+=. The macOS helper accepts `command`/`cmd`, `shift`, `option`/`alt`, `control`/`ctrl`, and `fn`, and maps them to their physical keys. Key names are platform-defined; keypad names (`KP_0` through `KP_9`, `KP_Enter`, and keypad operators) retain their physical key codes. `typeText` sends literal text, including its line breaks and tabs. Use the App object's `pressKey` for a short press. -For a long press, call `app.sendKeyEvent('w', 'down')` inside `run_code`. Each successful down remains active across subsequent tool calls in that execution. Choose when to send each up: after W down, D down and D up, W stays down. When the program returns, exits with an error, or the user pauses it, the helper releases the remaining keys at the processes that received their downs. A later execution cannot release an earlier execution's keys. +For a long press, call `app.sendKeyEvent('w', 'down')` inside `run_code`. The first down holds the key across subsequent tool calls. For an ordinary key already held by this execution, each additional down sends one key-down marked as a repeat to the original process. Choose the interval between calls. One up releases the key. Choose when to send each up: after W down, D down and D up, W stays down. When the program returns, exits with an error, or the user pauses it, the helper releases the remaining keys at the processes that received their downs. A later execution cannot release an earlier execution's keys. Agent disposal and plugin unload cancel and await the complete tool call, including image admission and attachment storage. An image write already in progress must settle before disposal returns; a canceled call cannot return a successful screenshot. @@ -224,7 +224,7 @@ Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. - pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once per key to hold one or more keys across tool calls within one run_code, for example W down, D down, D up, W up. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call sends one event. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. ``` diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index 57c2f3f371..a40be5399b 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -81,7 +81,7 @@ pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis. `pressKey(key, modifiers?)` 按下并松开一个键。使用快捷键时单独传入修饰键，例如 `pressKey('a', ['command'])`。单独调用 `pressKey('shift')` 时，按下事件带有 Shift 标志，释放事件清除该标志。`+` 和 `plus` 都会发送 Shift 和等号。macOS helper 接受 `command`/`cmd`、`shift`、`option`/`alt`、`control`/`ctrl` 和 `fn`，并映射到对应物理键。按键名由平台定义；数字键盘名称（`KP_0` 到 `KP_9`、`KP_Enter` 和数字键盘运算键）保留独立的物理键码。`typeText` 发送字面文本，包括其中的换行和制表符。短按使用 App 对象上的 `pressKey`。 -长按时，在 `run_code` 中调用 `app.sendKeyEvent('w', 'down')`。按下成功后，在同一次执行中调用其他工具时该键仍保持按下。每个键可分别松开。例如先按 W，再按 D，松开 D 后 W 仍按着。程序正常返回、出错退出或用户暂停时，helper 为本次执行仍按着的键逐一发送松开事件，每个事件都发回原接收进程。后续执行不能释放前一次执行的键。 +长按时，在 `run_code` 中调用 `app.sendKeyEvent('w', 'down')`。首次按下后，该键在同一次执行的后续工具调用中保持按下。对自己已按住的普通键再发一次 `down`，原接收进程会收到一次带重复标志的按下事件。调用方决定是否重复和间隔多久，松键只需一次 `up`。每个键可分别松开。例如先按 W，再按 D，松开 D 后 W 仍按着。程序正常返回、出错退出或用户暂停时，helper 为本次执行仍按着的键逐一发送松开事件，每个事件都发回原接收进程。后续执行不能释放前一次执行的键。 Agent 销毁和插件卸载会取消并等待完整工具调用，包括图像接入检查和附件存储。已经开始的图像写入必须结束后，销毁才会返回；已取消的调用不会返回成功截图。 @@ -224,7 +224,7 @@ Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. - pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once per key to hold one or more keys across tool calls within one run_code, for example W down, D down, D up, W up. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call sends one event. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. ``` diff --git a/packages/experimental/cua-macos/native/README.i18n.yaml b/packages/experimental/cua-macos/native/README.i18n.yaml index 58af404d5d..57420b0348 100644 --- a/packages/experimental/cua-macos/native/README.i18n.yaml +++ b/packages/experimental/cua-macos/native/README.i18n.yaml @@ -21,8 +21,8 @@ en: 2a69d89b7fc95998 zh: 084575bb31fb40dd /dsh-cua-native-helper/observation-and-input: - en: 0bbc620165d868fa - zh: abe73bd9c7b8cc5d + en: c6787eda94ba4833 + zh: 855e3c6432918775 /dsh-cua-native-helper/verification: en: c87945aeec587bf5 zh: e5d69e4927e2fed3 diff --git a/packages/experimental/cua-macos/native/README.md b/packages/experimental/cua-macos/native/README.md index cc3a0de35f..51d0a7b131 100644 --- a/packages/experimental/cua-macos/native/README.md +++ b/packages/experimental/cua-macos/native/README.md @@ -105,7 +105,7 @@ Capture and pointer delivery read CG bounds before and after an exact-window AX `typeText` and `pressKey` target the selected process's current focused control, not an exact window. `pressKey` presses the listed modifiers followed by one key, then releases all pressed keys in reverse order, including on cancellation. Keyboard events go directly to the target PID and bypass the global HID queue. Key names are platform-defined; keypad names such as `KP_0` and `KP_Enter` preserve their physical key codes. Background pointer actions may change that keyboard focus. Text injection sends Unicode characters individually and does not use the clipboard. LF, CR and CRLF send one Return; tabs send Tab. Applications apply their normal editing behavior, including indentation and focus navigation. Each `typeText` down/up event is followed by the configured text interval; `pressKey` keeps its own press and release timing, and dispatch completion does not confirm that the receiving application consumed every event. Cancellation can interrupt text or a gesture after partial input; it cannot undo received events. Very long text can exceed the host's operation deadline. Validate observed application state before deciding how to continue. -The Host associates directional key events with one `run_code` execution. On down, the helper retains the selected application, recipient PID and process start time, physical key code, and matching up event. Explicit up and cleanup verify the original process identity. On return, error, or user pause, the Host waits for any in-flight native request and asks the helper to release only that execution’s remaining keys. Disconnect cleanup also releases keys owned by the closed connection. +The Host associates directional key events with one `run_code` execution. On down, the helper retains the selected application, recipient PID and process start time, physical key code, and matching up event. A repeated down from the owner posts one key-down with `keyboardEventAutorepeat` set and reuses the existing record. Repeats, explicit up and cleanup verify the original process identity. A conflicting down or up reports the key and whether another session or another run holds it. On return, error, or user pause, the Host waits for any in-flight native request and asks the helper to release only that execution’s remaining keys. Disconnect cleanup also releases keys owned by the closed connection. The selected helper application needs macOS Accessibility permission for AX state and input, and Screen Recording permission for screenshots. Capability preflight and permission errors do not grant permissions. Capture refuses a window that moves or resizes while the image is being read. Unassociated system panels and accessory-only applications are not selectable. diff --git a/packages/experimental/cua-macos/native/README.zh.md b/packages/experimental/cua-macos/native/README.zh.md index 1b0c3f406f..eef79a770f 100644 --- a/packages/experimental/cua-macos/native/README.zh.md +++ b/packages/experimental/cua-macos/native/README.zh.md @@ -105,7 +105,7 @@ AppKit 在主事件循环上运行。阻塞式 socket 接收与握手使用后 `typeText` 和 `pressKey` 面向所选进程当前获得焦点的控件，不指定窗口。`pressKey` 向目标 PID 依次发送修饰键和一个主键的按下事件，再逆序释放。操作中途取消也会逆序释放已按下的键。键盘事件直接发给目标进程，绕过全局 HID 队列。按键名由平台定义；KP_0 和 KP_Enter 等数字键盘名称保留物理键码。后台鼠标操作可能改变键盘焦点。文本逐个 Unicode 字符输入，不使用剪贴板。LF、CR 和 CRLF 发送一次 Return，制表符发送 Tab。应用保留缩进和焦点导航等正常编辑行为。`typeText` 的每次按下和抬起后等待配置的文本事件间隔；`pressKey` 使用自己的按下与释放节奏，分派完成不证明应用已消费全部事件。取消可能中断部分文本或手势，无法撤销已接收事件。很长的文本可能超过 Host 操作期限。继续输入前应核实应用状态。 -宿主为每次 `run_code` 的 `sendKeyEvent` 附上执行标识。按下时，helper 保存已选应用、接收进程的 PID 与启动时间、物理键码及对应的松开事件。显式松键或执行结束时会核对原进程身份。正常返回、异常退出或用户暂停时，宿主等待在途原生请求结束，再通知 helper 释放该执行持有的其余按键。连接断开时，helper 也会清理该连接持有的按键。 +宿主为每次 `run_code` 的 `sendKeyEvent` 附上执行标识。按下时，helper 保存已选应用、接收进程的 PID 与启动时间、物理键码及对应的松开事件。持有方再次按下普通键时，helper 复用原记录，发送一次带 `keyboardEventAutorepeat` 标志的按下事件。重复、显式松键和执行结束时都会核对原进程身份。其他执行尝试按下或松开同一个键时，错误会指出键名，并区分其他会话和同一连接内的其他执行。正常返回、异常退出或用户暂停时，宿主等待在途原生请求结束，再通知 helper 释放该执行持有的其余按键。连接断开时，helper 也会清理该连接持有的按键。 所选 helper 应用需要 macOS 辅助功能权限以读取 AX 和输入，需要屏幕录制权限以截图。能力预检和权限错误不授予权限。截图会拒绝在读取图片期间移动或缩放的窗口。无法确认归属的系统面板及仅后台辅助类型的应用不可选择。 diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift index 1ebde9ee41..b8121fee03 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift @@ -215,7 +215,12 @@ actor AXWorker { return .result(Self.actionResult) } if method == "sendKeyEvent", try params.string("direction") == "up" { - let dispatched = try inputs().0.keyUp(params.string("key"), app: params.string("appHandle"), run: params.string("runId")) + let handle = try params.string("appHandle") + let dispatched = try inputs().0.keyUp(params.string("key"), app: handle, run: params.string("runId")) { + try NativePlatform.requireUnlocked(observeSession()) + try NativePlatform.requireAccessibility() + return try ax().keyboardRecipient(validateIdentity(handle), catalog: catalog) + } return .result(.object(["dispatched": .bool(dispatched)])) } let locked = observeSession() @@ -339,8 +344,9 @@ actor AXWorker { readTimeout: .milliseconds(try params.integer("pasteReadTimeoutMs", in: 1...2147483647)), maxElements: try params.integer("maxElements", in: 1...10000), maxDepth: try params.integer("maxDepth", in: 1...100))) case "sendKeyEvent": - let recipient = try ax().keyboardRecipient(identity, catalog: catalog) - try inputs().0.keyDown(params.string("key"), app: handle, run: params.string("runId"), recipient: recipient) + try inputs().0.keyDown(params.string("key"), app: handle, run: params.string("runId")) { + try ax().keyboardRecipient(identity, catalog: catalog) + } case "typeText", "pressKey": let recipient = try ax().keyboardRecipient(identity, catalog: catalog) try NativePlatform.verify(recipient) diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/HeldKeys.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/HeldKeys.swift index 47c34fbe2a..06f47981d2 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/HeldKeys.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/HeldKeys.swift @@ -26,11 +26,34 @@ final class HeldKeys: @unchecked Sendable { lock.withLock { Dictionary(uniqueKeysWithValues: pressed.filter { $0.recipient.pid == pid }.map { ($0.code, $0.flag) }) } } - func down(owner: Owner, recipient: ProcessIdentity, code: CGKeyCode, flag: CGEventFlags, + private func conflict(_ name: String, holder: Owner, caller: Owner) -> NativeFailure { + let source: String + if holder.connection != caller.connection { source = "another session" } + else if holder.run != caller.run { source = "another run_code execution" } + else { source = "a different application handle in this run_code" } + return RequestParameters.invalid("Key '\(name)' is held for this target by \(source). No event was sent. Wait for the holder to release it or for its run_code to end.") + } + + /// Repeats use the saved recipient even when the selected application's focused panel has changed. + func repeatIfHeld(owner: Owner, name: String, code: CGKeyCode, down: CGEvent, post: (CGEvent, Int32) -> Void) throws -> Bool { + try lock.withLock { + guard let key = pressed.first(where: { $0.owner == owner && $0.code == code }) else { return false } + guard down.type == .keyDown else { + throw RequestParameters.invalid("Modifier '\(name)' is already held. Send 'up' to release it.") + } + try NativePlatform.verify(key.recipient) + down.flags = flags(for: key.recipient) + down.setIntegerValueField(.keyboardEventAutorepeat, value: 1) + post(down, key.recipient.pid) + return true + } + } + + func down(owner: Owner, recipient: ProcessIdentity, name: String, code: CGKeyCode, flag: CGEventFlags, down: CGEvent, up: CGEvent, post: (CGEvent, Int32) -> Void) throws { try lock.withLock { - guard !pressed.contains(where: { $0.code == code && ($0.owner == owner || $0.recipient.pid == recipient.pid) }) else { - throw RequestParameters.invalid("Key is already held for this target") + if let key = pressed.first(where: { $0.code == code && $0.recipient.pid == recipient.pid }) { + throw conflict(name, holder: key.owner, caller: owner) } pressed.append(Press(owner: owner, recipient: recipient, code: code, flag: flag, up: up)) down.flags = flags(for: recipient) @@ -56,13 +79,20 @@ final class HeldKeys: @unchecked Sendable { return true } - func up(owner: Owner, code: CGKeyCode, post: (CGEvent, Int32) -> Void) throws -> Bool { - try lock.withLock { - guard let index = pressed.firstIndex(where: { $0.owner == owner && $0.code == code }) else { - throw RequestParameters.invalid("This run has not pressed that key for this application") - } + func up(owner: Owner, name: String, code: CGKeyCode, recipientIfUnheld: () throws -> ProcessIdentity, + post: (CGEvent, Int32) -> Void) throws -> Bool { + let dispatched = try lock.withLock { () throws -> Bool? in + guard let index = pressed.firstIndex(where: { $0.owner == owner && $0.code == code }) else { return nil } return try release(at: index, post: post) } + if let dispatched { return dispatched } + let recipient = try recipientIfUnheld() + throw lock.withLock { + if let key = pressed.first(where: { $0.code == code && $0.recipient == recipient }) { + return conflict(name, holder: key.owner, caller: owner) + } + return RequestParameters.invalid("Key '\(name)' is not held for this target. Send 'down' in this run_code before 'up'.") + } } func release(connection: UUID, run: String?, post: (CGEvent, Int32) -> Void) throws { diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift index 1a7ca086ba..b435d4e46b 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/KeyboardInput.swift @@ -22,20 +22,24 @@ final class KeyboardInput { self.checkCancellation = checkCancellation } - func keyDown(_ name: String, app: String, run: String, recipient: ProcessIdentity) throws { - let modifier = Self.namedModifier(name) + func keyDown(_ name: String, app: String, run: String, recipient: () throws -> ProcessIdentity) throws { + let owner = HeldKeys.Owner(connection: connection, run: run, app: app) let code = try Self.keyCode(name) let source = try Self.source() let down = try Self.keyEvent(source: source, code: code, down: true, flags: []) + try checkCancellation() + if try held.repeatIfHeld(owner: owner, name: name, code: code, down: down, post: post) { return } + let target = try recipient() let up = try Self.keyEvent(source: source, code: code, down: false, flags: []) try checkCancellation() - try NativePlatform.verify(recipient) - try held.down(owner: .init(connection: connection, run: run, app: app), recipient: recipient, - code: code, flag: modifier?.flag ?? [], down: down, up: up, post: post) + try NativePlatform.verify(target) + try held.down(owner: owner, recipient: target, name: name, code: code, flag: Self.namedModifier(name)?.flag ?? [], + down: down, up: up, post: post) } - func keyUp(_ name: String, app: String, run: String) throws -> Bool { - try held.up(owner: .init(connection: connection, run: run, app: app), code: Self.keyCode(name), post: post) + func keyUp(_ name: String, app: String, run: String, recipientIfUnheld: () throws -> ProcessIdentity) throws -> Bool { + try held.up(owner: .init(connection: connection, run: run, app: app), name: name, code: Self.keyCode(name), + recipientIfUnheld: recipientIfUnheld, post: post) } func releaseHeldKeys(run: String? = nil) throws { @@ -52,7 +56,7 @@ final class KeyboardInput { chord.append(Key(keyCode: code, flag: Self.namedModifier(name)?.flag ?? [])) let alreadyHeld = held.snapshot(pid: pid) guard alreadyHeld[code] == nil else { - throw RequestParameters.invalid("Release the held key before using it in pressKey") + throw RequestParameters.invalid("Key '\(name)' is already held for this target. Use pressKey after the run_code that holds it releases it.") } var selected = Set(alreadyHeld.keys) var flags = alreadyHeld.values.reduce(CGEventFlags()) { $0.union($1) } diff --git a/packages/experimental/cua-macos/native/tests/CapabilitiesFixture.swift b/packages/experimental/cua-macos/native/tests/CapabilitiesFixture.swift index 1151468790..d937c35ac2 100644 --- a/packages/experimental/cua-macos/native/tests/CapabilitiesFixture.swift +++ b/packages/experimental/cua-macos/native/tests/CapabilitiesFixture.swift @@ -85,8 +85,11 @@ final class CapabilitiesFixture: NSObject, NSApplicationDelegate, NSWindowDelega guard let self else { return } var value: [String: Any] = ["type": event.type.rawValue, "timestamp": event.timestamp, "flags": event.modifierFlags.rawValue, "shift": event.modifierFlags.contains(.shift), "command": event.modifierFlags.contains(.command)] - if [.keyDown, .keyUp].contains(event.type) { value["keyCode"] = event.keyCode; value["characters"] = event.characters ?? "" } - else { value["button"] = event.buttonNumber } + if [.keyDown, .keyUp].contains(event.type) { + value["keyCode"] = event.keyCode + value["characters"] = event.characters ?? "" + value["repeat"] = event.isARepeat + } else { value["button"] = event.buttonNumber } self.events.append(value) self.persist() } diff --git a/packages/experimental/cua-macos/native/tests/capabilities_e2e.py b/packages/experimental/cua-macos/native/tests/capabilities_e2e.py index 86f5cdef80..66dd004700 100644 --- a/packages/experimental/cua-macos/native/tests/capabilities_e2e.py +++ b/packages/experimental/cua-macos/native/tests/capabilities_e2e.py @@ -150,6 +150,17 @@ def check_keyboard(call, owner, state_file, state): state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 4) assert [(e["type"], e["keyCode"]) for e in state["events"][count:]] == [(10, 13), (10, 2), (11, 2), (11, 13)] + for explicit_up in (True, False): + count = len(state["events"]) + key("w", "down") + key("w", "down") + if explicit_up: + key("w", "up") + call("releaseHeldKeys", {"runId": "held-input"}) + state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 3) + repeat = [(e["type"], e["keyCode"], e["repeat"]) for e in state["events"][count:]] + assert repeat == [(10, 13, False), (10, 13, True), (11, 13, False)], repeat + def main(): parser = argparse.ArgumentParser() @@ -191,7 +202,7 @@ def main(): pid = wait_for_state(state_file, lambda value: value["windowId"] > 0)["pid"] if args.keyboard_only: check_keyboard(call, owner, state_file, json.loads(state_file.read_text())) - print("PASS: Shift+D shortcut, two-second W+D hold, independent D up and remaining W cleanup") + print("PASS: Shift+D shortcut, two-second W+D hold, independent D up, explicit W repeat and one final up") return if not args.click_only: call("activate", owner) diff --git a/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift b/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift index f81b10d1da..a971328ce3 100644 --- a/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift +++ b/packages/experimental/cua-macos/native/tests/swift/KeyboardChordTests.swift @@ -72,15 +72,66 @@ func plusAlonePressesShiftEquals(_ alias: String) throws { #expect(pid == target.pid) events.append(event) }, textEventInterval: 0, held: held) - try input.keyDown("shift", app: "editor", run: "one", recipient: target) - try input.keyDown("w", app: "editor", run: "two", recipient: target) - #expect(throws: NativeFailure.self) { try input.keyDown("W", app: "editor", run: "one", recipient: target) } - #expect(throws: NativeFailure.self) { try input.keyUp("w", app: "other", run: "two") } + try input.keyDown("shift", app: "editor", run: "one", recipient: { target }) + #expect(throws: NativeFailure.self) { try input.keyDown("shift", app: "editor", run: "one", recipient: { target }) } + try input.keyDown("w", app: "editor", run: "two", recipient: { target }) + #expect(throws: NativeFailure.self) { try input.keyDown("W", app: "editor", run: "one", recipient: { target }) } + #expect(throws: NativeFailure.self) { try input.keyUp("w", app: "other", run: "two", recipientIfUnheld: { target }) } #expect(throws: NativeFailure.self) { try input.pressKey("w", modifiers: [], to: target.pid) } try input.pressKey("d", modifiers: ["shift"], to: target.pid) try input.releaseHeldKeys(run: "one") - #expect(try input.keyUp("w", app: "editor", run: "two")) + #expect(try input.keyUp("w", app: "editor", run: "two", recipientIfUnheld: { target })) try input.releaseHeldKeys(run: "two") #expect(events.map { $0.getIntegerValueField(.keyboardEventKeycode) } == [56, 13, 2, 2, 56, 13]) #expect(events.map { $0.flags.contains(.maskShift) } == [true, true, true, true, false, false]) } + +@Test(arguments: [true, false]) +func repeatedDownUsesOriginalRecipientAndOneRelease(_ explicitUp: Bool) throws { + let target = try NativePlatform.identity(getpid()) + var events: [CGEvent] = [] + let input = KeyboardInput(post: { event, pid in + #expect(pid == target.pid) + events.append(event) + }, textEventInterval: 0) + func changedFocus() throws -> ProcessIdentity { + throw RequestParameters.invalid("The repeat or release looked up a new recipient") + } + try input.keyDown("w", app: "editor", run: "one", recipient: { target }) + try input.keyDown("W", app: "editor", run: "one", recipient: changedFocus) + if explicitUp { #expect(try input.keyUp("w", app: "editor", run: "one", recipientIfUnheld: changedFocus)) } + try input.releaseHeldKeys(run: "one") + #expect(events.map(\.type) == [.keyDown, .keyDown, .keyUp]) + #expect(events.map { $0.getIntegerValueField(.keyboardEventAutorepeat) } == [0, 1, 0]) +} + +@Test func foreignSessionCannotRepeatOrReleaseTheOwnersKey() throws { + let target = try NativePlatform.identity(getpid()) + let held = HeldKeys() + var events: [CGEvent] = [] + let a = KeyboardInput(post: { event, _ in events.append(event) }, textEventInterval: 0, held: held) + let b = KeyboardInput(post: { _, _ in Issue.record("Non-owner dispatched an event") }, textEventInterval: 0, held: held) + try a.keyDown("w", app: "a", run: "same-run-id", recipient: { target }) + for direction in ["down", "up"] { + do { + if direction == "down" { try b.keyDown("W", app: "b", run: "same-run-id", recipient: { target }) } + else { _ = try b.keyUp("W", app: "b", run: "same-run-id", recipientIfUnheld: { target }) } + Issue.record("Non-owner was allowed to send an event") + } catch let failure as NativeFailure { + #expect(failure.code == "INVALID_ARGUMENT") + #expect(failure.message.contains("Key 'W' is held for this target by another session")) + #expect(failure.message.contains("Wait for the holder to release it")) + } + } + try b.releaseHeldKeys() + try a.keyDown("W", app: "a", run: "same-run-id", recipient: { target }) + try a.releaseHeldKeys() + #expect(events.map(\.type) == [.keyDown, .keyDown, .keyUp]) + do { + _ = try b.keyUp("W", app: "b", run: "same-run-id", recipientIfUnheld: { target }) + Issue.record("Unheld key was released") + } catch let failure as NativeFailure { + #expect(failure.code == "INVALID_ARGUMENT") + #expect(failure.message.contains("Key 'W' is not held for this target")) + } +} diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index 6b2f073924..839eb56a3c 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -14,7 +14,7 @@ export const GUIDANCE = `${PLATFORM_GUIDANCE} ${COMMON_GUIDANCE.replace(/^- ([\w.]+)\(.*$/gm, (entry, method: string) => { if (method === 'pressKey') return `${entry} -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once per key to hold one or more keys across tool calls within one run_code, for example W down, D down, D up, W up. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.` +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call sends one event. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.` const detail = methodDetails.get(method) return detail ? `${entry} ${detail}` : entry })}` diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index e8deb643fb..a01916dbe2 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -79,7 +79,7 @@ Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. - pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once per key to hold one or more keys across tool calls within one run_code, for example W down, D down, D up, W up. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call sends one event. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index 5322aa44d7..ef5ce31b6d 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -81,7 +81,7 @@ Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. - pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once per key to hold one or more keys across tool calls within one run_code, for example W down, D down, D up, W up. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call sends one event. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. diff --git a/snapshots/session/ptc-script-retained/system-prompt.expected.md b/snapshots/session/ptc-script-retained/system-prompt.expected.md index 4c5b91329b..9c072b94ea 100644 --- a/snapshots/session/ptc-script-retained/system-prompt.expected.md +++ b/snapshots/session/ptc-script-retained/system-prompt.expected.md @@ -81,7 +81,7 @@ Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. - pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' once per key to hold one or more keys across tool calls within one run_code, for example W down, D down, D up, W up. Holding a physical key on a keyboard can produce repeated key-down events after a delay. This method posts one event per call and rejects a second 'down' for a held key. Use the app's response to decide what to do while holding the key and when to send 'up'. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call sends one event. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. From e426855e0359491b42915d07e0f98105e5cf387a Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 15:01:16 +0800 Subject: [PATCH 09/11] test(cua-macos): distinguish process delivery from AppKit repeat handling --- packages/experimental/cua-macos/README.i18n.yaml | 4 ++-- packages/experimental/cua-macos/README.md | 4 ++-- packages/experimental/cua-macos/README.zh.md | 4 ++-- packages/experimental/cua-macos/native/README.i18n.yaml | 4 ++-- packages/experimental/cua-macos/native/README.md | 2 +- packages/experimental/cua-macos/native/README.zh.md | 2 +- .../cua-macos/native/Sources/CuaNative/HeldKeys.swift | 2 +- .../cua-macos/native/tests/capabilities_e2e.py | 9 ++++++--- packages/experimental/cua-macos/src/guidance.ts | 2 +- snapshots/session/cua-native/system-prompt.expected.md | 2 +- snapshots/session/cua-ptc/system-prompt.expected.md | 2 +- .../ptc-script-retained/system-prompt.expected.md | 2 +- 12 files changed, 21 insertions(+), 18 deletions(-) diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index 1be5a7abd9..7220e21ad2 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -15,8 +15,8 @@ en: 1777e7bf6785f986 zh: e3daac1074ebb74d /deepseek-ai-dsh-experimental-cua-macos/use-this-package: - en: 83d4eb62d0d0db2f - zh: 3afde1c3b8c25211 + en: 30fefafc1915db8a + zh: e819f1e722c0d614 /deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment: en: b734a0907747580e zh: 8110c5951b70e100 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 26a218c40d..34975631af 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -81,7 +81,7 @@ The [configuration catalog](../../../docs/config-catalog.md#deepseek-aidsh-exper `pressKey(key, modifiers?)` presses and releases one key. For shortcuts, pass held modifiers separately, as in `pressKey('a', ['command'])`. Pressing a modifier itself, as in `pressKey('shift')`, sets its flag on press and clears it on release. A single `+` or `plus` sends Shift+=. The macOS helper accepts `command`/`cmd`, `shift`, `option`/`alt`, `control`/`ctrl`, and `fn`, and maps them to their physical keys. Key names are platform-defined; keypad names (`KP_0` through `KP_9`, `KP_Enter`, and keypad operators) retain their physical key codes. `typeText` sends literal text, including its line breaks and tabs. Use the App object's `pressKey` for a short press. -For a long press, call `app.sendKeyEvent('w', 'down')` inside `run_code`. The first down holds the key across subsequent tool calls. For an ordinary key already held by this execution, each additional down sends one key-down marked as a repeat to the original process. Choose the interval between calls. One up releases the key. Choose when to send each up: after W down, D down and D up, W stays down. When the program returns, exits with an error, or the user pauses it, the helper releases the remaining keys at the processes that received their downs. A later execution cannot release an earlier execution's keys. +For a long press, call `app.sendKeyEvent('w', 'down')` inside `run_code`. The first down holds the key across subsequent tool calls. For an ordinary key already held by this execution, each additional down sends one key-down marked as a repeat to the original process. Choose the interval between calls. A control can filter synthetic repeats, so check its response. One up releases the key. Choose when to send each up: after W down, D down and D up, W stays down. When the program returns, exits with an error, or the user pauses it, the helper releases the remaining keys at the processes that received their downs. A later execution cannot release an earlier execution's keys. Agent disposal and plugin unload cancel and await the complete tool call, including image admission and attachment storage. An image write already in progress must settle before disposal returns; a canceled call cannot return a successful screenshot. @@ -224,7 +224,7 @@ Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. - pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call sends one event. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. ``` diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index a40be5399b..e6a0ff313f 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -81,7 +81,7 @@ pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis. `pressKey(key, modifiers?)` 按下并松开一个键。使用快捷键时单独传入修饰键，例如 `pressKey('a', ['command'])`。单独调用 `pressKey('shift')` 时，按下事件带有 Shift 标志，释放事件清除该标志。`+` 和 `plus` 都会发送 Shift 和等号。macOS helper 接受 `command`/`cmd`、`shift`、`option`/`alt`、`control`/`ctrl` 和 `fn`，并映射到对应物理键。按键名由平台定义；数字键盘名称（`KP_0` 到 `KP_9`、`KP_Enter` 和数字键盘运算键）保留独立的物理键码。`typeText` 发送字面文本，包括其中的换行和制表符。短按使用 App 对象上的 `pressKey`。 -长按时，在 `run_code` 中调用 `app.sendKeyEvent('w', 'down')`。首次按下后，该键在同一次执行的后续工具调用中保持按下。对自己已按住的普通键再发一次 `down`，原接收进程会收到一次带重复标志的按下事件。调用方决定是否重复和间隔多久，松键只需一次 `up`。每个键可分别松开。例如先按 W，再按 D，松开 D 后 W 仍按着。程序正常返回、出错退出或用户暂停时，helper 为本次执行仍按着的键逐一发送松开事件，每个事件都发回原接收进程。后续执行不能释放前一次执行的键。 +长按时，在 `run_code` 中调用 `app.sendKeyEvent('w', 'down')`。首次按下后，该键在同一次执行的后续工具调用中保持按下。对自己已按住的普通键再发一次 `down`，原接收进程会收到一次带重复标志的按下事件。调用方决定是否重复和间隔多久。控件可能过滤模拟的重复事件，可通过控件的实际变化确认效果。松键只需一次 `up`。每个键可分别松开。例如先按 W，再按 D，松开 D 后 W 仍按着。程序正常返回、出错退出或用户暂停时，helper 为本次执行仍按着的键逐一发送松开事件，每个事件都发回原接收进程。后续执行不能释放前一次执行的键。 Agent 销毁和插件卸载会取消并等待完整工具调用，包括图像接入检查和附件存储。已经开始的图像写入必须结束后，销毁才会返回；已取消的调用不会返回成功截图。 @@ -224,7 +224,7 @@ Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. - pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call sends one event. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. ``` diff --git a/packages/experimental/cua-macos/native/README.i18n.yaml b/packages/experimental/cua-macos/native/README.i18n.yaml index 57420b0348..d4cfe23155 100644 --- a/packages/experimental/cua-macos/native/README.i18n.yaml +++ b/packages/experimental/cua-macos/native/README.i18n.yaml @@ -21,8 +21,8 @@ en: 2a69d89b7fc95998 zh: 084575bb31fb40dd /dsh-cua-native-helper/observation-and-input: - en: c6787eda94ba4833 - zh: 855e3c6432918775 + en: 0808c25f70ce6eff + zh: 820864a2b18b8a24 /dsh-cua-native-helper/verification: en: c87945aeec587bf5 zh: e5d69e4927e2fed3 diff --git a/packages/experimental/cua-macos/native/README.md b/packages/experimental/cua-macos/native/README.md index 51d0a7b131..4f2283e654 100644 --- a/packages/experimental/cua-macos/native/README.md +++ b/packages/experimental/cua-macos/native/README.md @@ -105,7 +105,7 @@ Capture and pointer delivery read CG bounds before and after an exact-window AX `typeText` and `pressKey` target the selected process's current focused control, not an exact window. `pressKey` presses the listed modifiers followed by one key, then releases all pressed keys in reverse order, including on cancellation. Keyboard events go directly to the target PID and bypass the global HID queue. Key names are platform-defined; keypad names such as `KP_0` and `KP_Enter` preserve their physical key codes. Background pointer actions may change that keyboard focus. Text injection sends Unicode characters individually and does not use the clipboard. LF, CR and CRLF send one Return; tabs send Tab. Applications apply their normal editing behavior, including indentation and focus navigation. Each `typeText` down/up event is followed by the configured text interval; `pressKey` keeps its own press and release timing, and dispatch completion does not confirm that the receiving application consumed every event. Cancellation can interrupt text or a gesture after partial input; it cannot undo received events. Very long text can exceed the host's operation deadline. Validate observed application state before deciding how to continue. -The Host associates directional key events with one `run_code` execution. On down, the helper retains the selected application, recipient PID and process start time, physical key code, and matching up event. A repeated down from the owner posts one key-down with `keyboardEventAutorepeat` set and reuses the existing record. Repeats, explicit up and cleanup verify the original process identity. A conflicting down or up reports the key and whether another session or another run holds it. On return, error, or user pause, the Host waits for any in-flight native request and asks the helper to release only that execution’s remaining keys. Disconnect cleanup also releases keys owned by the closed connection. +The Host associates directional key events with one `run_code` execution. On down, the helper retains the selected application, recipient PID and process start time, physical key code, and matching up event. A repeated down from the owner posts one key-down with `keyboardEventAutorepeat` set and reuses the existing record. The event is posted to the recipient PID. An AppKit view can receive the initial down and final up while filtering the marked repeat. Repeats, explicit up and cleanup verify the original process identity. A conflicting down or up reports the key and whether another session or another run holds it. On return, error, or user pause, the Host waits for any in-flight native request and asks the helper to release only that execution’s remaining keys. Disconnect cleanup also releases keys owned by the closed connection. The selected helper application needs macOS Accessibility permission for AX state and input, and Screen Recording permission for screenshots. Capability preflight and permission errors do not grant permissions. Capture refuses a window that moves or resizes while the image is being read. Unassociated system panels and accessory-only applications are not selectable. diff --git a/packages/experimental/cua-macos/native/README.zh.md b/packages/experimental/cua-macos/native/README.zh.md index eef79a770f..e010295168 100644 --- a/packages/experimental/cua-macos/native/README.zh.md +++ b/packages/experimental/cua-macos/native/README.zh.md @@ -105,7 +105,7 @@ AppKit 在主事件循环上运行。阻塞式 socket 接收与握手使用后 `typeText` 和 `pressKey` 面向所选进程当前获得焦点的控件，不指定窗口。`pressKey` 向目标 PID 依次发送修饰键和一个主键的按下事件，再逆序释放。操作中途取消也会逆序释放已按下的键。键盘事件直接发给目标进程，绕过全局 HID 队列。按键名由平台定义；KP_0 和 KP_Enter 等数字键盘名称保留物理键码。后台鼠标操作可能改变键盘焦点。文本逐个 Unicode 字符输入，不使用剪贴板。LF、CR 和 CRLF 发送一次 Return，制表符发送 Tab。应用保留缩进和焦点导航等正常编辑行为。`typeText` 的每次按下和抬起后等待配置的文本事件间隔；`pressKey` 使用自己的按下与释放节奏，分派完成不证明应用已消费全部事件。取消可能中断部分文本或手势，无法撤销已接收事件。很长的文本可能超过 Host 操作期限。继续输入前应核实应用状态。 -宿主为每次 `run_code` 的 `sendKeyEvent` 附上执行标识。按下时，helper 保存已选应用、接收进程的 PID 与启动时间、物理键码及对应的松开事件。持有方再次按下普通键时，helper 复用原记录，发送一次带 `keyboardEventAutorepeat` 标志的按下事件。重复、显式松键和执行结束时都会核对原进程身份。其他执行尝试按下或松开同一个键时，错误会指出键名，并区分其他会话和同一连接内的其他执行。正常返回、异常退出或用户暂停时，宿主等待在途原生请求结束，再通知 helper 释放该执行持有的其余按键。连接断开时，helper 也会清理该连接持有的按键。 +宿主为每次 `run_code` 的 `sendKeyEvent` 附上执行标识。按下时，helper 保存已选应用、接收进程的 PID 与启动时间、物理键码及对应的松开事件。持有方再次按下普通键时，helper 复用原记录，向接收进程发送一次带 `keyboardEventAutorepeat` 标志的按下事件。AppKit 视图可能只收到首次按下和最终松开，过滤中间的重复事件。重复、显式松键和执行结束时都会核对原进程身份。其他执行尝试按下或松开同一个键时，错误会指出键名，并区分其他会话和同一连接内的其他执行。正常返回、异常退出或用户暂停时，宿主等待在途原生请求结束，再通知 helper 释放该执行持有的其余按键。连接断开时，helper 也会清理该连接持有的按键。 所选 helper 应用需要 macOS 辅助功能权限以读取 AX 和输入，需要屏幕录制权限以截图。能力预检和权限错误不授予权限。截图会拒绝在读取图片期间移动或缩放的窗口。无法确认归属的系统面板及仅后台辅助类型的应用不可选择。 diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/HeldKeys.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/HeldKeys.swift index 06f47981d2..3d29795127 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/HeldKeys.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/HeldKeys.swift @@ -38,7 +38,7 @@ final class HeldKeys: @unchecked Sendable { func repeatIfHeld(owner: Owner, name: String, code: CGKeyCode, down: CGEvent, post: (CGEvent, Int32) -> Void) throws -> Bool { try lock.withLock { guard let key = pressed.first(where: { $0.owner == owner && $0.code == code }) else { return false } - guard down.type == .keyDown else { + guard key.flag.isEmpty, down.type == .keyDown else { throw RequestParameters.invalid("Modifier '\(name)' is already held. Send 'up' to release it.") } try NativePlatform.verify(key.recipient) diff --git a/packages/experimental/cua-macos/native/tests/capabilities_e2e.py b/packages/experimental/cua-macos/native/tests/capabilities_e2e.py index 66dd004700..fb7a022a83 100644 --- a/packages/experimental/cua-macos/native/tests/capabilities_e2e.py +++ b/packages/experimental/cua-macos/native/tests/capabilities_e2e.py @@ -157,9 +157,12 @@ def check_keyboard(call, owner, state_file, state): if explicit_up: key("w", "up") call("releaseHeldKeys", {"runId": "held-input"}) - state = wait_for_state(state_file, lambda value: len(value["events"]) >= count + 3) - repeat = [(e["type"], e["keyCode"], e["repeat"]) for e in state["events"][count:]] - assert repeat == [(10, 13, False), (10, 13, True), (11, 13, False)], repeat + state = wait_for_state(state_file, lambda value: any( + e["type"] == 11 and e.get("keyCode") == 13 for e in value["events"][count:])) + view_events = [(e["type"], e["keyCode"], e["repeat"]) for e in state["events"][count:]] + # AppKit can filter a synthetic repeat before NSView.keyDown receives it. + down, repeat, up = (10, 13, False), (10, 13, True), (11, 13, False) + assert view_events in ([down, up], [down, repeat, up]), view_events def main(): diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index 839eb56a3c..47d2eb8f5c 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -14,7 +14,7 @@ export const GUIDANCE = `${PLATFORM_GUIDANCE} ${COMMON_GUIDANCE.replace(/^- ([\w.]+)\(.*$/gm, (entry, method: string) => { if (method === 'pressKey') return `${entry} -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call sends one event. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.` +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients.` const detail = methodDetails.get(method) return detail ? `${entry} ${detail}` : entry })}` diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index a01916dbe2..8a8938c6ab 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -79,7 +79,7 @@ Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. - pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call sends one event. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index ef5ce31b6d..4276485cd5 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -81,7 +81,7 @@ Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. - pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call sends one event. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. diff --git a/snapshots/session/ptc-script-retained/system-prompt.expected.md b/snapshots/session/ptc-script-retained/system-prompt.expected.md index 9c072b94ea..b83ccdc8f3 100644 --- a/snapshots/session/ptc-script-retained/system-prompt.expected.md +++ b/snapshots/session/ptc-script-retained/system-prompt.expected.md @@ -81,7 +81,7 @@ Available at app or window level depending on the platform. - typeText(text: string) -> ActionResult — type literal text at the app's current focus; line endings send Return and tabs send Tab. - paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry. - pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string. -- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call sends one event. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. +- sendKeyEvent(key: string, direction: 'down' | 'up') -> ActionResult — send 'down' to hold a key across tool calls in one run_code, for example W down, D down, D up, W up. Holding a key on a physical keyboard can produce repeated key-down events after a delay. To send one such repeat for a non-modifier key this run already holds, send 'down' again. Each call posts one event. A control can filter repeats. Use the app's response to decide whether and when to repeat or release the key. When this run_code returns, throws, or the user pauses it, macOS releases only its remaining keys at their original recipients. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. From 29ceb385e69fbd1720f1280a773b7e13574a39b2 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 15:02:33 +0800 Subject: [PATCH 10/11] docs(cua-macos): document caller-requested key repeats in the upgrade guide --- .../v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml | 4 ++-- docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md | 2 +- .../upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml index 09553adc47..a017e7f82b 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml @@ -12,5 +12,5 @@ en: e5d1027faa55adba zh: 4b88ad6a70977fb3 /native-cua-application-and-window-api/migration: - en: a0640d4cd1f2e84e - zh: be8d5b5f9d14caf8 + en: a63e88dce467d6ee + zh: fc25bb283ca0eaff diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md index 8f5b092012..2675699a71 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md @@ -18,6 +18,6 @@ Providers supply startup capabilities alongside their platform guidance. The nat 1. After `cua.getApp(selector)`, select the target from `app.getWindows()` with `app.getWindow(windowId)`. Use `window.getScreenshot()`, `window.click()`, `window.scroll()`, and `window.drag()`. Replace `hover` with `window.mouseMove()`. On macOS, `window.click(id)` sends mouse events at the element center. Switch to `invoke(id)` for the default accessibility action, or `performAction(id, "AXShowMenu")` to request an advertised menu action. 2. Replace `getAXState()` with `getAccessibilityTree()`. Replace `getAXStateAndScreenshot()` with separate tree and screenshot reads. Selection reads the initial tree silently for a window, and for an app when `appLevelAccessibility` is true. Read the tree explicitly to display it. 3. Pass the opaque string ID from the latest tree wherever you passed a numeric element index. In raw requests use `elementId` with `snapshotId`. Replace `performSecondaryAction` with `listActions(id)` and `performAction(id, exactName, argument?)`, or use `invoke(id)` for the default action. `setValue` accepts the provider’s supported JSON values. On macOS, `performAction(id, 'AXRaise')` performs the advertised raise action. Call `app.activate()` first when the application must become active. -4. Replace a packed chord such as `pressKey('shift+Tab')` with `pressKey('Tab', ['shift'])`. Check `(await cua.getCapabilities()).methods` before using `sendKeyEvent`. On macOS, call it inside one `run_code` to hold a key across tool calls. Returning, throwing or pausing releases any remaining keys. If your script handles unknown macOS key names as `NATIVE_ERROR`, handle `INVALID_ARGUMENT` instead. Correct the key name before retrying. +4. Replace a packed chord such as `pressKey('shift+Tab')` with `pressKey('Tab', ['shift'])`. Check `(await cua.getCapabilities()).methods` before using `sendKeyEvent`. On macOS, call it inside one `run_code` to hold a key across tool calls. To request one repeat, send another down for a non-modifier key held by that execution. Returning, throwing or pausing releases any remaining keys. If your script handles unknown macOS key names as `NATIVE_ERROR`, handle `INVALID_ARGUMENT` instead. Correct the key name before retrying. 5. Read `dispatched` from action results, then observe the application to verify the outcome. `invoke` returns `dispatched: false` when the element has no default action. Remove reads of the old `verified` field and of `pid` on selected objects or window records. Use `listApps()` or running `findApps()` matches when you need a PID. 6. For a custom provider, replace `CuaBackend.description` with `capabilities` and pass `guidance` and `permissionParameters` to `installCua`. Rebuild any custom `helperPath` against the [operation schema](../../../../packages/experimental/cua/src/protocol.ts) and [tagged permission response](../../../../packages/experimental/cua/src/access-types.ts). Restart the provider, inspect its advertised methods and keyboard scopes, and open the Computer Use permission page to verify its platform controls. diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md index 7580de0836..b4551f624b 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md @@ -18,6 +18,6 @@ CUA 脚本在指定窗口上截图和操作鼠标。App 和 Window 对象共享 1. 调用 `cua.getApp(selector)` 后，从 `app.getWindows()` 返回的列表选定目标，再调用 `app.getWindow(windowId)`。在窗口上调用 `getScreenshot()`、`click()`、`scroll()` 和 `drag()`，将原有的 `hover` 调用改为 `window.mouseMove()`。在 macOS 上，`window.click(id)` 在元素中心发送鼠标事件。需要执行默认辅助功能动作时，改用 `invoke(id)`。需要打开控件菜单且控件声明了 `AXShowMenu` 时，调用 `performAction(id, "AXShowMenu")`。 2. 将 `getAXState()` 改为 `getAccessibilityTree()`，将 `getAXStateAndScreenshot()` 拆成读取树和截图两次调用。选择窗口时会静默读取初始树。选择应用时，只有 `appLevelAccessibility` 为真才读取。要显示树，请显式读取。 3. 将数字元素索引改为从最新树中取得的字符串 ID。直接发送请求时使用 `elementId` 和 `snapshotId`。原先调用 `performSecondaryAction` 的地方，先用 `listActions(id)` 取得完整动作名，再调用 `performAction(id, exactName, argument?)`。要执行默认动作，可调用 `invoke(id)`。`setValue` 接受提供方支持的 JSON 值。 在 macOS 上，`performAction(id, 'AXRaise')` 只执行元素声明的抬起窗口动作。需要把应用切到前台时先调用 `app.activate()`。 -4. 将 `pressKey('shift+Tab')` 之类的组合键写法改为 `pressKey('Tab', ['shift'])`。调用 `sendKeyEvent` 前检查 `(await cua.getCapabilities()).methods`。在 macOS 上，同一次 `run_code` 中可调用此方法跨工具调用保持按键。执行返回、异常退出或用户暂停时会释放仍按着的键。如果脚本用 `NATIVE_ERROR` 识别 macOS 上的未知键名，请改为处理 `INVALID_ARGUMENT`，并在重试前修正键名。 +4. 将 `pressKey('shift+Tab')` 之类的组合键写法改为 `pressKey('Tab', ['shift'])`。调用 `sendKeyEvent` 前检查 `(await cua.getCapabilities()).methods`。在 macOS 上，同一次 `run_code` 中可调用此方法跨工具调用保持按键。如果需要一次重复按下，可在这次执行中对已按住的普通键再发一次 `down`。执行返回、异常退出或用户暂停时会释放仍按着的键。如果脚本用 `NATIVE_ERROR` 识别 macOS 上的未知键名，请改为处理 `INVALID_ARGUMENT`，并在重试前修正键名。 5. 从动作结果读取 `dispatched`，再观察应用确认效果。元素没有默认动作时，`invoke` 返回 `dispatched: false`。移除读取旧字段 `verified` 的代码，以及对已选对象和窗口记录中 `pid` 的读取。需要 PID 时，使用 `listApps()` 或 `findApps()` 返回的运行中应用记录。 6. 自定义提供方需将 `CuaBackend.description` 改为 `capabilities`，并向 `installCua` 传入 `guidance` 和 `permissionParameters`。如配置了 `helperPath`，请按[操作协议](../../../../packages/experimental/cua/src/protocol.ts)和[带平台标记的权限响应](../../../../packages/experimental/cua/src/access-types.ts)重新构建 helper。重启提供方，检查它声明的方法和键盘输入范围，再打开 Computer Use 权限页确认平台控件。 From fa993abc5ae3789339e160dcb9417812dc970631 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Sat, 7 Nov 2026 15:14:59 +0800 Subject: [PATCH 11/11] test(cua-macos): cover runs that only use keyboard shortcuts --- .../cua-macos/tests/provider-lifecycle.spec.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index f65e1a6a61..86bf7b9ba5 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -161,6 +161,20 @@ it('requires an enclosing run_code before sending a directional key', async () = expect(native.request).not.toHaveBeenCalled() }) +it('finishes a shortcut-only run_code without requesting held-key cleanup', async () => { + const f = await setup(undefined, true) + await selectApplication(f) + native.request.mockClear().mockResolvedValue({ dispatched: true }) + const result = await executeModelToolFixture(f.ctx, { agent: f.handle.agent, callId: ToolCallId('shortcut-run'), + name: 'run_code', signal: new AbortController().signal, arguments: { + code: "return await tools.cua({ method: 'pressKey', params: { appHandle: 'app-one', key: 'a', modifiers: ['command'] } });", + description: 'Press Command+A', + } }) + expect(result.isError).toBe(false) + expect(native.request.mock.calls.map(([method]) => method)).toEqual(['pressKey']) + expect(native.cleanup).not.toHaveBeenCalled() +}) + it.each(['return', 'throw', 'pause', 'pause with cleanup error'])('cleans up only the held W when run_code ends by %s', async (ending) => { const f = await setup(undefined, true) await selectApplication(f)