From feba1df810efd279fff5926b39de0bc3fe5a6bed Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 24 Sep 2026 16:52:28 +0800 Subject: [PATCH] fix(cua): avoid redundant foreground requests and correlate their observations --- packages/experimental/cua/README.i18n.yaml | 4 +- packages/experimental/cua/README.md | 2 + packages/experimental/cua/README.zh.md | 2 + packages/experimental/cua/native/README.txt | 2 + packages/experimental/cua/native/src/apps.rs | 161 ++++++++++++++++- .../cua/native/src/element_input.rs | 162 +++++++++++++++++- packages/experimental/cua/native/src/main.rs | 74 +++++++- .../experimental/cua/native/upstream.json | 2 +- packages/experimental/cua/src/index.ts | 2 +- .../experimental/cua/src/native-client.ts | 6 +- .../cua/tests/loader-composition.spec.ts | 23 ++- .../cua/tests/native-client.spec.ts | 9 + .../cua/tests/provider-lifecycle.spec.ts | 6 + .../session/cua-native/replay.override.json | 66 +++++++ .../cua-native/workspace/native-helper.mjs | 23 ++- 15 files changed, 517 insertions(+), 27 deletions(-) diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index 6744bac92a..f93eef9753 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/experimental/cua/README.md -README.md: b1913f4fa37ab1315e2e3eddf40e7228f27986f5 -README.zh.md: 543dea78a8c08d6f96fbb5b106ceecd541150e28 +README.md: bb1607da3931918d62226acb2a2eeb71997cbdd1 +README.zh.md: c2b148c88e879d310ae3eba5c8e51acd96f13f2c diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index b1913f4fa3..bb1607da39 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -79,6 +79,8 @@ Indexed clicks support a single right click or a single/double left click. The h The object facade translates app/window methods into JSON operations. The bridge routes them through the normal tool pipeline, preserving authorization, nested logs and image admission. A separate serial helper belongs to each Agent. Native handles bind a process identity to its kernel start time, and snapshots retain native AX references only inside that helper. The same operation protocol can serve a future platform provider without adding platform logic to the object API. +Native request IDs match the logged tool call IDs. Successful activation and window-raise results retain foreground request details and immediate before/after focus observations in the raw tool result. An already-frontmost app skips redundant activation; a window raise is skipped only when the exact window is also focused, main, unminimized and first among visible normal windows. Unknown observations retain the requested operation. These observations do not establish who caused an unrelated focus change or verify completion. + AX traversal visits each native identity once, including proxy references that compare equal, so cyclic accessibility children cannot repeat whole subtrees. The [source manifest](native/upstream.json) identifies unchanged copied files and local integration code. No invariant companion is published: there is no independently projected registry or durable evaluator state to reconcile. Protocol validation, target identity and resource cleanup are checked by their owning implementations. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index 543dea78a8..c2b148c88e 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -79,6 +79,8 @@ AppKit sheet 可以通过其窗口 ID 单独选择，也会出现在所属文档 对象接口将应用和窗口方法转换为 JSON 操作。适配模块通过标准工具链保留授权、嵌套日志和图片接入。每个 Agent 拥有一个串行处理请求的辅助进程。原生句柄绑定进程身份及内核启动时间，快照中的 AX 引用只保存在辅助进程内。未来平台可以实现相同操作协议，无需在对象接口中加入平台逻辑。 +原生请求 ID 与工具日志中的调用 ID 一致。成功的激活和窗口抬起操作会在原始工具结果中保留前台操作信息，以及操作前后的即时焦点观察。应用已在前台时跳过重复激活；窗口抬起还要求该窗口正是焦点主窗口、未最小化且位于可见普通窗口的最前方，才会跳过。无法确认时保留原请求。这些观察不能确认其他焦点切换的发起者，也不代表操作已经完成。 + AX 遍历对每个原生身份只访问一次，包括表示同一元素的不同代理引用，避免循环子节点重复展开整个子树。[源码清单](native/upstream.json)区分未修改的上游文件和本地接入代码。本包不发布 invariant 入口，因为没有需要对照的独立注册表投影或持久执行器状态。协议验证、目标身份和资源清理由对应实现负责。 diff --git a/packages/experimental/cua/native/README.txt b/packages/experimental/cua/native/README.txt index f82c066afd..683d62bee2 100644 --- a/packages/experimental/cua/native/README.txt +++ b/packages/experimental/cua/native/README.txt @@ -6,6 +6,8 @@ Build with `cargo build --release --locked --manifest-path packages/experimental Each stdin line is `{ "id": "opaque request id", "method": "method name", "params": {} }`. Each stdout line contains the same ID and exactly one of `result` or `error: {code, message}`. Malformed envelopes return `INVALID_REQUEST` with a null ID. Requests execute serially. Diagnostics must not be written to stdout. `protocol.schema.json` defines request fields. +Successful `activate` and `performSecondaryAction` AXRaise results include `foreground` alongside `{dispatched: true, verified: false}`. It records `requestId`, `operation`, `targetPid`, nullable `targetWindowId`, `activationRequested`, `raiseRequested`, and immediate `before`/`after` observations. `frontPid` comes from WindowServer; `focusedWindowId` is the target app's AX-focused window; `frontWindowId` and `frontWindowPid` identify the first visible normal window. Unknown values are null. Observations are sequential, not atomic or proof of causality. Tool-owned requests use the logged tool call ID. Failed operations retain the existing error response and may have partial effects. + | Method | Result | | --- | --- | | `getCapabilities` | `{platform: "macos", permissions: {accessibility, screenRecording}, methods, coordinateSpace: "window-points", browser: false}`. Permission values are boolean preflight observations; no system prompt opens. | diff --git a/packages/experimental/cua/native/src/apps.rs b/packages/experimental/cua/native/src/apps.rs index 451f687872..8c4500d3a3 100644 --- a/packages/experimental/cua/native/src/apps.rs +++ b/packages/experimental/cua/native/src/apps.rs @@ -1,6 +1,103 @@ //! Running-application discovery adapted from Cua Driver's apps/mod.rs. use serde::Serialize; +/// Immediate focus observations; missing values do not establish focus or stacking. +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ForegroundObservation { + pub front_pid: Option, + /// The target application's AX focus, which can persist while the app is inactive. + pub focused_window_id: Option, + /// The first on-screen layer-0 window across all applications. + pub front_window_id: Option, + pub front_window_pid: Option, +} + +/// Requests made by one operation; sequential before/after observations do not verify causality. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ForegroundAction { + pub operation: &'static str, + pub target_pid: i32, + pub target_window_id: Option, + pub before: ForegroundObservation, + pub after: ForegroundObservation, + pub activation_requested: bool, + pub raise_requested: bool, +} + +#[derive(Default)] +#[repr(C)] +struct ProcessSerialNumber { + high: u32, + low: u32, +} + +type GetFrontProcess = unsafe extern "C" fn(*mut ProcessSerialNumber) -> i32; + +#[link(name = "ApplicationServices", kind = "framework")] +extern "C" { + // HIServices/Processes.h: OSStatus GetProcessPID(const ProcessSerialNumber *, pid_t *). + fn GetProcessPID(psn: *const ProcessSerialNumber, pid: *mut libc::pid_t) -> i32; +} + +fn read_front_pid( + get_front: impl FnOnce(&mut ProcessSerialNumber) -> i32, + get_pid: impl FnOnce(&ProcessSerialNumber, &mut i32) -> i32, +) -> Option { + let mut psn = ProcessSerialNumber::default(); + if get_front(&mut psn) != 0 { + return None; + } + let mut pid = 0; + (get_pid(&psn, &mut pid) == 0 && pid > 0).then_some(pid) +} + +/// Read WindowServer's front process directly; unavailable SPI leaves the observation unknown. +pub fn front_pid() -> Option { + use std::sync::OnceLock; + static GET_FRONT: OnceLock> = OnceLock::new(); + let get_front = (*GET_FRONT.get_or_init(|| unsafe { + // Retain the library for the cached function pointer's process lifetime. + libc::dlopen( + c"/System/Library/PrivateFrameworks/SkyLight.framework/SkyLight".as_ptr(), + libc::RTLD_LAZY | libc::RTLD_GLOBAL, + ); + let symbol = libc::dlsym(libc::RTLD_DEFAULT, c"_SLPSGetFrontProcess".as_ptr()); + (!symbol.is_null()) + .then(|| std::mem::transmute::<*mut libc::c_void, GetFrontProcess>(symbol)) + }))?; + read_front_pid( + |psn| unsafe { get_front(psn) }, + |psn, pid| unsafe { GetProcessPID(psn, pid) }, + ) +} + +/// Observe the target's AX focus and the first on-screen normal window alongside WindowServer focus. +pub fn observe_foreground(pid: i32) -> ForegroundObservation { + let focused_window_id = crate::ax::bindings::focused_window_id_of_pid(pid); + let front_window = crate::windows::visible_windows().into_iter().next(); + let front_pid = front_pid(); + ForegroundObservation { + front_pid, + focused_window_id, + front_window_id: front_window.as_ref().map(|window| window.window_id), + front_window_pid: front_window.map(|window| window.pid), + } +} + +fn activate_if_needed( + pid: i32, + front_pid: Option, + request: impl FnOnce() -> anyhow::Result<()>, +) -> anyhow::Result { + if front_pid == Some(pid) { + return Ok(false); + } + request()?; + Ok(true) +} + #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct AppInfo { @@ -52,9 +149,11 @@ pub fn path_for_pid(pid: i32) -> Option { } } -pub fn activate(pid: i32) -> anyhow::Result<()> { +/// Request activation unless WindowServer already reports this process frontmost. +pub fn activate(pid: i32) -> anyhow::Result { use objc2_app_kit::{NSApplicationActivationOptions, NSRunningApplication}; - unsafe { + let before = observe_foreground(pid); + let activation_requested = activate_if_needed(pid, before.front_pid, || unsafe { let app = NSRunningApplication::runningApplicationWithProcessIdentifier(pid) .ok_or_else(|| anyhow::anyhow!("Application exited before activation"))?; app.unhide(); @@ -62,6 +161,62 @@ pub fn activate(pid: i32) -> anyhow::Result<()> { app.activateWithOptions(NSApplicationActivationOptions::empty()), "macOS refused application activation" ); + Ok(()) + })?; + Ok(ForegroundAction { + operation: "activate", + target_pid: pid, + target_window_id: None, + before, + after: observe_foreground(pid), + activation_requested, + raise_requested: false, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + #[test] + fn front_process_read_requires_both_queries_and_a_positive_pid() { + assert_eq!( + read_front_pid(|_| -1, |_, _| panic!("no PSN was read")), + None + ); + assert_eq!(read_front_pid(|_| 0, |_, _| -1), None); + assert_eq!(read_front_pid(|_| 0, |_, _| 0), None); + assert_eq!( + read_front_pid( + |_| 0, + |_, pid| { + *pid = 42; + 0 + } + ), + Some(42) + ); } - Ok(()) + + #[test] + fn already_frontmost_process_does_not_request_activation() { + assert!(!activate_if_needed(42, Some(42), || panic!("activation repeated")).unwrap()); + } + + #[test] + fn other_or_unknown_front_process_preserves_activation_and_its_errors() { + let calls = Cell::new(0); + for front in [Some(7), None] { + assert!(activate_if_needed(42, front, || { + calls.set(calls.get() + 1); + Ok(()) + }) + .unwrap()); + } + assert_eq!(calls.get(), 2); + let error = + activate_if_needed(42, None, || anyhow::bail!("activation refused")).unwrap_err(); + assert_eq!(error.to_string(), "activation refused"); + } } diff --git a/packages/experimental/cua/native/src/element_input.rs b/packages/experimental/cua/native/src/element_input.rs index 70ecf85610..0c2ff4dfee 100644 --- a/packages/experimental/cua/native/src/element_input.rs +++ b/packages/experimental/cua/native/src/element_input.rs @@ -1,5 +1,5 @@ //! Indexed clicks use advertised AX actions or the observed element's live window bounds. -use crate::{ax::bindings::*, input, panels}; +use crate::{apps, ax::bindings::*, input, panels}; use anyhow::{ensure, Context, Result}; fn action_key(value: &str) -> String { @@ -12,7 +12,12 @@ fn action_key(value: &str) -> String { .collect() } -pub fn secondary_action(pid: i32, pointer: usize, requested: &str) -> Result<()> { +/// Run an advertised action, reporting foreground requests only for AXRaise. +pub fn secondary_action( + pid: i32, + pointer: usize, + requested: &str, +) -> Result> { let element = pointer as AXUIElementRef; let actions = unsafe { copy_action_names(element) }; let action = actions @@ -21,14 +26,73 @@ pub fn secondary_action(pid: i32, pointer: usize, requested: &str) -> Result<()> .context("Element does not advertise this action; refresh getAXState")?; input::ax_actions::ensure_ax_action_enabled(pointer, action)?; if action == "AXRaise" { - crate::apps::activate(pid)?; + return raise_window(pid, element, requested).map(Some); } let error = unsafe { perform_action(element, action) }; ensure!( error == kAXErrorSuccess, "Accessibility action {requested} failed with error {error}" ); - Ok(()) + Ok(None) +} + +fn can_skip_raise( + pid: i32, + window_id: Option, + before: &apps::ForegroundObservation, + role: Option<&str>, + main: Option, + minimized: Option, +) -> bool { + window_id.is_some() + && role == Some("AXWindow") + && main == Some(true) + && minimized == Some(false) + && before.front_pid == Some(pid) + && before.focused_window_id == window_id + && before.front_window_id == window_id + && before.front_window_pid == Some(pid) +} + +fn raise_window( + pid: i32, + element: AXUIElementRef, + requested: &str, +) -> Result { + let target_window_id = unsafe { ax_get_window_id(element) }; + let role = unsafe { copy_string_attr(element, "AXRole") }; + let main = unsafe { copy_bool_attr(element, "AXMain") }; + let minimized = unsafe { copy_bool_attr(element, "AXMinimized") }; + let before = apps::observe_foreground(pid); + // App-level focus alone does not establish that this particular window is raised. + let raise_requested = !can_skip_raise( + pid, + target_window_id, + &before, + role.as_deref(), + main, + minimized, + ); + let activation_requested = if raise_requested { + let activation = apps::activate(pid)?; + let error = unsafe { perform_action(element, "AXRaise") }; + ensure!( + error == kAXErrorSuccess, + "Accessibility action {requested} failed with error {error}" + ); + activation.activation_requested + } else { + false + }; + Ok(apps::ForegroundAction { + operation: "AXRaise", + target_pid: pid, + target_window_id, + before, + after: apps::observe_foreground(pid), + activation_requested, + raise_requested, + }) } pub fn click(pid: i32, pointer: usize, button: &str, count: usize) -> Result<()> { @@ -90,7 +154,7 @@ pub fn click(pid: i32, pointer: usize, button: &str, count: usize) -> Result<()> #[cfg(test)] mod tests { - use super::action_key; + use super::*; #[test] fn observed_native_and_display_action_names_match() { @@ -98,4 +162,92 @@ mod tests { assert_eq!(action_key("AXZoomWindow"), action_key("zoom window")); assert_ne!(action_key("AXRaise"), action_key("AXPress")); } + + #[test] + fn raise_requires_the_exact_focused_front_window_to_be_skipped() { + let focused = apps::ForegroundObservation { + front_pid: Some(42), + focused_window_id: Some(7), + front_window_id: Some(7), + front_window_pid: Some(42), + }; + assert!(can_skip_raise( + 42, + Some(7), + &focused, + Some("AXWindow"), + Some(true), + Some(false) + )); + for window_id in [None, Some(8)] { + assert!(!can_skip_raise( + 42, + window_id, + &focused, + Some("AXWindow"), + Some(true), + Some(false) + )); + } + for before in [ + apps::ForegroundObservation { + front_pid: None, + ..focused + }, + apps::ForegroundObservation { + front_pid: Some(99), + ..focused + }, + apps::ForegroundObservation { + focused_window_id: None, + ..focused + }, + apps::ForegroundObservation { + focused_window_id: Some(8), + ..focused + }, + apps::ForegroundObservation { + front_window_id: None, + ..focused + }, + apps::ForegroundObservation { + front_window_id: Some(8), + ..focused + }, + apps::ForegroundObservation { + front_window_pid: None, + ..focused + }, + apps::ForegroundObservation { + front_window_pid: Some(99), + ..focused + }, + ] { + assert!(!can_skip_raise( + 42, + Some(7), + &before, + Some("AXWindow"), + Some(true), + Some(false) + )); + } + for (role, main, minimized) in [ + (None, Some(true), Some(false)), + (Some("AXSheet"), Some(true), Some(false)), + (Some("AXWindow"), None, Some(false)), + (Some("AXWindow"), Some(false), Some(false)), + (Some("AXWindow"), Some(true), None), + (Some("AXWindow"), Some(true), Some(true)), + ] { + assert!(!can_skip_raise( + 42, + Some(7), + &focused, + role, + main, + minimized + )); + } + } } diff --git a/packages/experimental/cua/native/src/main.rs b/packages/experimental/cua/native/src/main.rs index 8776b959ee..65edb14cad 100644 --- a/packages/experimental/cua/native/src/main.rs +++ b/packages/experimental/cua/native/src/main.rs @@ -74,6 +74,15 @@ fn error_reply(error: anyhow::Error) -> Value { json!({"code":error.downcast_ref::().map_or("NATIVE_ERROR", |error| error.code),"message":error.to_string()}) } +fn action_reply(request_id: &str, foreground: Option) -> Value { + let mut result = json!({"dispatched":true,"verified":false}); + if let Some(foreground) = foreground { + result["foreground"] = json!(foreground); + result["foreground"]["requestId"] = json!(request_id); + } + result +} + #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct Request { @@ -405,7 +414,7 @@ impl Driver { Ok(result) } - fn call(&mut self, method: &str, params: &Value) -> Result { + fn call(&mut self, request_id: &str, method: &str, params: &Value) -> Result { let object = params .as_object() .ok_or_else(|| invalid("params must be an object"))?; @@ -532,8 +541,8 @@ impl Driver { } match method { "activate" => { - apps::activate(pid)?; - Ok(json!({"dispatched":true,"verified":false})) + let foreground = apps::activate(pid)?; + Ok(action_reply(request_id, Some(foreground))) } "getState" => { let window = if params.get("windowId").is_some() { @@ -674,12 +683,12 @@ impl Driver { Ok(json!({"dispatched":true,"verified":false})) } "performSecondaryAction" => { - element_input::secondary_action( + let foreground = element_input::secondary_action( pid, self.element(&handle, params)?, text(params, "action")?, )?; - Ok(json!({"dispatched":true,"verified":false})) + Ok(action_reply(request_id, foreground)) } "typeText" => { input::keyboard::type_text(panels::keyboard_pid(pid)?, text(params, "text")?)?; @@ -800,7 +809,7 @@ fn main() -> Result<()> { for line in stdin.lock().lines() { let line = line?; let response = match serde_json::from_str::(&line) { - Ok(request) => match driver.call(&request.method, &request.params) { + Ok(request) => match driver.call(&request.id, &request.method, &request.params) { Ok(result) => json!({"id":request.id,"result":result}), Err(error) => json!({"id":request.id,"error":error_reply(error)}), }, @@ -818,6 +827,51 @@ fn main() -> Result<()> { #[cfg(test)] mod tests { use super::*; + + #[test] + fn foreground_action_reply_correlates_requests_without_claiming_verified_effects() { + let before = apps::ForegroundObservation { + front_pid: None, + focused_window_id: Some(7), + front_window_id: Some(8), + front_window_pid: Some(99), + }; + let after = apps::ForegroundObservation { + front_pid: Some(42), + focused_window_id: Some(7), + front_window_id: Some(7), + front_window_pid: Some(42), + }; + let result = action_reply( + "tool-call-7", + Some(apps::ForegroundAction { + operation: "AXRaise", + target_pid: 42, + target_window_id: Some(7), + before, + after, + activation_requested: true, + raise_requested: true, + }), + ); + assert_eq!( + result, + json!({ + "dispatched":true,"verified":false, + "foreground":{ + "requestId":"tool-call-7","operation":"AXRaise","targetPid":42,"targetWindowId":7, + "before":{"frontPid":null,"focusedWindowId":7,"frontWindowId":8,"frontWindowPid":99}, + "after":{"frontPid":42,"focusedWindowId":7,"frontWindowId":7,"frontWindowPid":42}, + "activationRequested":true,"raiseRequested":true, + }, + }) + ); + assert_eq!( + action_reply("ordinary-action", None), + json!({"dispatched":true,"verified":false}) + ); + } + fn snapshot(id: &str, window_id: Option) -> Snapshot { Snapshot { id: id.to_owned(), @@ -1133,13 +1187,17 @@ mod tests { assert_eq!( error_reply( driver - .call("listApps", &json!({"launch":true})) + .call("invalid-params", "listApps", &json!({"launch":true})) .unwrap_err() )["code"], "INVALID_ARGUMENT" ); assert_eq!( - error_reply(driver.call("absent", &json!({})).unwrap_err())["code"], + error_reply( + driver + .call("unknown-method", "absent", &json!({})) + .unwrap_err() + )["code"], "UNSUPPORTED" ); assert!(window_id(&json!({"windowId":-1})).is_err()); diff --git a/packages/experimental/cua/native/upstream.json b/packages/experimental/cua/native/upstream.json index 2dfe881620..a010aad7f5 100644 --- a/packages/experimental/cua/native/upstream.json +++ b/packages/experimental/cua/native/upstream.json @@ -34,7 +34,7 @@ { "local": "src/apps.rs", "upstream": "libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs", - "changes": "Keep regular running applications, bundle ID and application-path lookup; add activation, camelCase output and omit absent bundle identifiers." + "changes": "Keep regular running applications, bundle ID and application-path lookup; add activation, camelCase output and omit absent bundle identifiers. Skip activation when WindowServer already reports the target frontmost, and record immediate foreground observations with activation results." }, { "local": "src/ax/tree.rs", diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index daac884cd0..96f31821cf 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -154,7 +154,7 @@ export function apply(ctx: Context, config: Config): void { return client.request(method, method === 'getState' ? { ...params, maxElements: config.maxElements, maxDepth: config.maxDepth } : method === 'drag' ? { ...params, durationMs: config.dragDurationMs, steps: config.dragSteps } - : params, signal) + : params, signal, execution.callId) })() pending.set(operation, agent) try { diff --git a/packages/experimental/cua/src/native-client.ts b/packages/experimental/cua/src/native-client.ts index 5e3849f6dc..7010aa27fc 100644 --- a/packages/experimental/cua/src/native-client.ts +++ b/packages/experimental/cua/src/native-client.ts @@ -1,6 +1,7 @@ /** Bounded JSON-line transport to one owned native helper process. */ import { randomUUID } from 'node:crypto' import type { SubprocessHandle, SubprocessRuntime } from '@deepseek-ai/dsh-subprocess' +import type { ToolExecution } from '@deepseek-ai/dsh-tools' import type { JsonValue } from '@deepseek-ai/dsh-util-values' import { z } from 'zod' @@ -63,16 +64,17 @@ export class NativeClient { * @param method - Helper operation name. * @param params - JSON operation parameters. * @param signal - Cancellation kills this helper and invalidates its retained handles. + * @param callId - Logged tool invocation ID, when this request belongs to a tool execution. * @returns The validated helper result, after failure cleanup when needed. */ - async request(method: string, params: Record, signal: AbortSignal): Promise { + async request(method: string, params: Record, signal: AbortSignal, callId?: ToolExecution['callId']): Promise { if (signal.aborted) throw new NativeError('NATIVE_CANCELLED', 'Operation was cancelled before dispatch') if (this.disposed) { await this.dispose() throw new NativeError('NATIVE_CLOSED', 'Helper is closed; acquire fresh application handles', { cause: this.failure }) } if (this.active) throw new NativeError('NATIVE_BUSY', 'Await native operations sequentially') - const id = randomUUID() + const id = callId ?? randomUUID() const frame = `${JSON.stringify({ id, method, params })}\n` if (Buffer.byteLength(frame) > this.config.maxRequestBytes) { throw new NativeError('NATIVE_REQUEST_LIMIT', 'Request exceeded the configured byte limit before dispatch') diff --git a/packages/experimental/cua/tests/loader-composition.spec.ts b/packages/experimental/cua/tests/loader-composition.spec.ts index 4093e20af5..af849b22d5 100644 --- a/packages/experimental/cua/tests/loader-composition.spec.ts +++ b/packages/experimental/cua/tests/loader-composition.spec.ts @@ -56,7 +56,7 @@ class VisualModel extends LlmAdapter { const AuditEvent = z.object({ event: z.string(), pid: z.number(), method: z.string().optional(), count: z.number().optional(), - params: z.record(z.string(), z.json()).optional(), + requestId: z.string().optional(), params: z.record(z.string(), z.json()).optional(), }) // The fixture uses a POSIX executable shebang; stream/protocol unit tests run on every host. @@ -118,6 +118,8 @@ it.skipIf(process.platform === 'win32')('loads the native provider, records veri const events = await readEvents() expect(events.filter(event => event.event === 'start')).toHaveLength(1) expect(events.filter(event => event.event === 'call').map(event => event.method)).toEqual(calls.map(call => call.method)) + const toolCalls = agent.session.snapshotEvents().filter(event => event.type === 'tool/call') + expect(events.filter(event => event.event === 'call').map(event => event.requestId)).toEqual(toolCalls.map(event => event.data.callId)) expect(events.find(event => event.event === 'state')?.count).toBe(1) expect(events.find(event => event.method === 'getState')?.params).toMatchObject({ maxElements: 2000, maxDepth: 25 }) @@ -127,6 +129,25 @@ it.skipIf(process.platform === 'win32')('loads the native provider, records veri expect(direct.isError).toBe(false) expect(direct.value).toMatchObject({ structuredContent: { mimeType: 'image/png', data: png } }) expect(context.computerUse.providerName).toBe('dsh-cua') + for (const [callId, activationRequested, beforePid, beforeWindowId] of [ + ['activate-first', true, 42, 99], ['activate-repeat', false, 9, 7], + ] as const) { + const activated = await context.tools.execute({ + agent, signal: new AbortController().signal, callId: ToolCallId(callId), name: 'cua', + arguments: { method: 'activate', params: { appHandle: 'fixture-app' } }, + }) + expect(activated.isError).toBe(false) + expect(activated.value).toMatchObject({ structuredContent: { + dispatched: true, verified: false, foreground: { + requestId: callId, operation: 'activate', targetPid: 9, targetWindowId: null, activationRequested, raiseRequested: false, + before: { frontPid: beforePid, focusedWindowId: 7, frontWindowId: beforeWindowId, frontWindowPid: beforePid }, + after: { frontPid: 9, focusedWindowId: 7, frontWindowId: 7, frontWindowPid: 9 }, + }, + } }) + expect(JSON.stringify(activated.content)).toContain(callId) + } + const directCalls = (await readEvents()).filter(event => event.event === 'call').slice(-3) + expect(directCalls.map(event => event.requestId)).toEqual(['programmatic-window', 'activate-first', 'activate-repeat']) await handle.dispose() expect((await readEvents()).filter(event => event.event === 'exit')).toHaveLength(1) diff --git a/packages/experimental/cua/tests/native-client.spec.ts b/packages/experimental/cua/tests/native-client.spec.ts index b1e0d7eb13..171d9e66a2 100644 --- a/packages/experimental/cua/tests/native-client.spec.ts +++ b/packages/experimental/cua/tests/native-client.spec.ts @@ -1,5 +1,6 @@ /** Controlled process streams exercise the platform-independent native protocol client. */ import { PassThrough, Writable } from 'node:stream' +import { ToolCallId } from '@deepseek-ai/dsh-llm' import type { SubprocessHandle, SubprocessOutcome } from '@deepseek-ai/dsh-subprocess' import { describe, expect, it, onTestFinished, vi } from 'vitest' import { z } from 'zod' @@ -55,6 +56,14 @@ function fixture(options: { } describe('native helper transport', () => { + it('uses the logged tool invocation ID for the native request and its matching reply', async () => { + const f = fixture() + const pending = f.client.request('activate', { appHandle: 'app-one' }, new AbortController().signal, ToolCallId('foreground-call')) + expect(f.id()).toBe('foreground-call') + f.reply({ id: 'foreground-call', result: { dispatched: true, verified: false } }) + await expect(pending).resolves.toEqual({ dispatched: true, verified: false }) + }) + it('accepts split UTF-8 replies and preserves native refusal codes without replacing the helper', async () => { const f = fixture() const first = f.client.request('listApps', {}, new AbortController().signal) diff --git a/packages/experimental/cua/tests/provider-lifecycle.spec.ts b/packages/experimental/cua/tests/provider-lifecycle.spec.ts index 006205d16a..220b39c281 100644 --- a/packages/experimental/cua/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua/tests/provider-lifecycle.spec.ts @@ -56,6 +56,12 @@ it('rejects a relative helper path before publishing computer-use operations', ( expect(() => { NativeProvider.apply(ctx, { ...config, helperPath: 'relative-helper' }) }).toThrow('helperPath must be absolute') }) +it('correlates a native request with its logged tool invocation', async () => { + const f = await setup() + expect((await f.execute()).isError).toBe(false) + expect(native.request).toHaveBeenCalledExactlyOnceWith('listApps', {}, expect.any(AbortSignal), ToolCallId('lifecycle-1')) +}) + it('rejects helper acquisition when Agent disposal begins during replacement cleanup', async () => { const f = await setup() await f.execute() diff --git a/snapshots/session/cua-native/replay.override.json b/snapshots/session/cua-native/replay.override.json index bb7a506383..52c32b85ae 100644 --- a/snapshots/session/cua-native/replay.override.json +++ b/snapshots/session/cua-native/replay.override.json @@ -362,6 +362,72 @@ } ] }, + { + "kind": "chunks", + "chunks": [ + { + "type": "block-start", + "index": 0, + "blockType": "tool-call" + }, + { + "type": "block-end", + "index": 0, + "block": { + "type": "tool-call", + "id": "cua-native-activate-first", + "name": "cua", + "arguments": "{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}" + } + }, + { + "type": "usage", + "usage": { + "inputTokens": 3, + "outputTokens": 3 + } + }, + { + "type": "finish", + "reason": { + "kind": "tool-calls" + } + } + ] + }, + { + "kind": "chunks", + "chunks": [ + { + "type": "block-start", + "index": 0, + "blockType": "tool-call" + }, + { + "type": "block-end", + "index": 0, + "block": { + "type": "tool-call", + "id": "cua-native-activate-repeat", + "name": "cua", + "arguments": "{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}" + } + }, + { + "type": "usage", + "usage": { + "inputTokens": 3, + "outputTokens": 3 + } + }, + { + "type": "finish", + "reason": { + "kind": "tool-calls" + } + } + ] + }, { "kind": "chunks", "chunks": [ diff --git a/snapshots/session/cua-native/workspace/native-helper.mjs b/snapshots/session/cua-native/workspace/native-helper.mjs index 75b9f688f4..a0751fa667 100755 --- a/snapshots/session/cua-native/workspace/native-helper.mjs +++ b/snapshots/session/cua-native/workspace/native-helper.mjs @@ -16,15 +16,17 @@ const bounds = { x: 40, y: 60, width: 320, height: 200 } const window = { windowId: 7, pid: 9, appName: 'Fixture', title: 'CUA fixture', bounds, isOnScreen: true, onCurrentSpace: true } const axlessWindow = { ...window, windowId: 8, title: 'AXless canvas' } const snapshots = new Map() -const methods = ['getCapabilities', 'listApps', 'getApp', 'listWindows', 'getState', 'click', 'setValue', 'typeText', 'pressKey', 'scroll', 'drag', 'screenshot'] +const methods = ['getCapabilities', 'listApps', 'getApp', 'activate', 'listWindows', 'getState', 'click', 'setValue', 'typeText', 'pressKey', 'scroll', 'drag', 'screenshot'] let acquired = false let generation = 0 let count = 0 let value = '' let visualCount = 0 +let frontPid = 42 +let frontWindowId = 99 function refuse(code, message) { throw Object.assign(new Error(message), { code }) } -function call(method, params) { - record('call', { method, params }) +function call(requestId, method, params) { + record('call', { requestId, method, params }) if (method === 'getCapabilities') return { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: true }, methods, coordinateSpace: 'window-points', browser: false } if (method === 'listApps') return [{ pid: app.pid, name: app.name, bundleId: app.bundleId }] if (method === 'getApp') { @@ -33,6 +35,19 @@ function call(method, params) { return app } if (!acquired || params.appHandle !== app.appHandle) refuse('APP_NOT_FOUND', 'Acquire an application in this helper first') + if (method === 'activate') { + const before = { frontPid, focusedWindowId: window.windowId, frontWindowId, frontWindowPid: frontPid } + const activationRequested = frontPid !== app.pid + if (activationRequested) { + frontPid = app.pid + frontWindowId = window.windowId + } + const after = { frontPid, focusedWindowId: window.windowId, frontWindowId, frontWindowPid: frontPid } + return { dispatched: true, verified: false, foreground: { + requestId, operation: 'activate', targetPid: app.pid, targetWindowId: null, + before, after, activationRequested, raiseRequested: false, + } } + } if (method === 'listWindows') return [window, axlessWindow] if (method === 'getState') { generation++ @@ -66,7 +81,7 @@ input.once('close', () => process.exit(0)) input.on('line', (line) => { const request = JSON.parse(line) let response - try { response = { id: request.id, result: call(request.method, request.params) } } + try { response = { id: request.id, result: call(request.id, request.method, request.params) } } catch (error) { response = { id: request.id, error: { code: error.code ?? 'FIXTURE_ERROR', message: error.message } } } process.stdout.write(JSON.stringify(response) + '\n') })