fix(cua): activate selected apps before observing their windows

This commit is contained in:
creatixchu committed 2026-09-23 20:20:33 +08:00
1 parent 9107ae0d21
commit f5c4110b5e
7 files changed
+23 -6

No files matched your search

+2 -2
View File
@@ -2,5 +2,5 @@
# side as of the last confirmed-consistent state. Both languages carry equal authority;
# after editing either side, bring the other along and re-record with:
# pnpm run verify-translation-pairing --write packages/experimental/cua/README.md
README.md: 04209c613df7024f696b19325ee1c3144bab4e0b
README.zh.md: e7c4531ed925e7bc287d1af89eedf32d68a1f3e8
README.md: e21ff31fc5f69743094efeb05653857e63ac5560
README.zh.md: c73d39298b808dafae9771d2cf69669c086695be
+1 -1
View File
@@ -34,7 +34,7 @@ export DSH_CUA_HELPER="$cua_native_root/target/release/dsh-cua-native"
pnpm dsh headless --patch packages/experimental/cua/examples/macos.cordis.yml --dump-config
```
The dump command checks profile composition without a model request. For a real task, replace `--dump-config` with a prompt and configure the profile’s model credentials. The overlay mounts the computer-use registry and this provider; remove an existing competing provider before using it. The responsible host/helper needs Accessibility permission for AX/input and Screen Recording permission for capture. `getCapabilities` reports permission state without opening a system dialog. Start the target application yourself. Select it by name, bundle ID, absolute application path, or PID.
The dump command checks profile composition without a model request. For a real task, replace `--dump-config` with a prompt and configure the profile’s model credentials. The overlay mounts the computer-use registry and this provider; remove an existing competing provider before using it. The responsible host/helper needs Accessibility permission for AX/input and Screen Recording permission for capture. `getCapabilities` reports permission state without opening a system dialog. Start the target application yourself. Select it by name, bundle ID, absolute application path, or PID. `getApp` activates the selected application before reading its initial state.
| Field | Default | Meaning |
| --- | --- | --- |
+1 -1
View File
@@ -34,7 +34,7 @@ export DSH_CUA_HELPER="$cua_native_root/target/release/dsh-cua-native"
pnpm dsh headless --patch packages/experimental/cua/examples/macos.cordis.yml --dump-config
```
配置输出命令不发起模型请求。执行真实任务时,将 `--dump-config` 替换为任务提示,并配置该 profile 的模型凭据。配置挂载 computer-use 注册表和本 provider;使用前需移除已有的竞争 provider。负责执行的宿主或辅助进程需要辅助功能权限来读取 AX 和输入,需要屏幕录制权限来截图。`getCapabilities` 只查询权限,不弹出系统授权窗口。目标应用需提前启动,可按名称、bundle ID、应用绝对路径或 PID 选择。
配置输出命令不发起模型请求。执行真实任务时,将 `--dump-config` 替换为任务提示,并配置该 profile 的模型凭据。配置挂载 computer-use 注册表和本 provider;使用前需移除已有的竞争 provider。负责执行的宿主或辅助进程需要辅助功能权限来读取 AX 和输入,需要屏幕录制权限来截图。`getCapabilities` 只查询权限,不弹出系统授权窗口。目标应用需提前启动,可按名称、bundle ID、应用绝对路径或 PID 选择。`getApp` 激活选中的应用,再读取初始状态。
| 字段 | 默认值 | 含义 |
| --- | --- | --- |
@@ -51,3 +51,19 @@ pub fn path_for_pid(pid: i32) -> Option<String> {
.map(|value| value.to_string())
}
}
pub fn activate(pid: i32) -> anyhow::Result<()> {
use objc2_app_kit::{NSApplicationActivationOptions, NSRunningApplication};
unsafe {
let app = NSRunningApplication::runningApplicationWithProcessIdentifier(pid)
.ok_or_else(|| anyhow::anyhow!("Application exited before activation"))?;
app.unhide();
anyhow::ensure!(
app.activateWithOptions(
NSApplicationActivationOptions::NSApplicationActivateIgnoringOtherApps
),
"macOS refused application activation"
);
}
Ok(())
}
@@ -397,6 +397,7 @@ impl Driver {
"Selector matches multiple running apps; select an exact PID from listApps",
)?;
let app = &matches[0];
apps::activate(app.pid)?;
let handle = self.remember_app(app.pid, process_started(app.pid)?);
let mut result = json!({"pid":app.pid,"appHandle":handle,"name":app.name});
if let Some(bundle_id) = &app.bundle_id {
@@ -54,7 +54,7 @@
{
"local": "src/apps.rs",
"upstream": "libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs",
"changes": "Keep regular running applications, bundle ID and application-path lookup; camelCase output and omit absent bundle identifiers."
"changes": "Keep regular running applications, bundle ID and application-path lookup; add activation, camelCase output and omit absent bundle identifiers."
}
],
"localFiles": [
+1 -1
View File
@@ -163,7 +163,7 @@ export interface CuaApi {
*/
getState(options?: CuaObservationOptions): Promise<{ apps: CuaAppInfo[]; browsers: [] }>
/**
* Select an existing app and emit initial AX state.
* Activate an existing app and emit initial AX state.
* @param selector - exact name, bundle ID, absolute application path or PID string.
* @returns a handle tied to this process lifetime.
*/