fix(cua): activate selected apps before observing their windows
This commit is contained in:
1 parent
9107ae0d21
commit
f5c4110b5e
7 files changed
+23
-6
No files matched your search
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write packages/experimental/cua/README.md
|
||||
README.md: 04209c613df7024f696b19325ee1c3144bab4e0b
|
||||
README.zh.md: e7c4531ed925e7bc287d1af89eedf32d68a1f3e8
|
||||
README.md: e21ff31fc5f69743094efeb05653857e63ac5560
|
||||
README.zh.md: c73d39298b808dafae9771d2cf69669c086695be
|
||||
@@ -34,7 +34,7 @@ export DSH_CUA_HELPER="$cua_native_root/target/release/dsh-cua-native"
|
||||
pnpm dsh headless --patch packages/experimental/cua/examples/macos.cordis.yml --dump-config
|
||||
```
|
||||
|
||||
The dump command checks profile composition without a model request. For a real task, replace `--dump-config` with a prompt and configure the profile’s model credentials. The overlay mounts the computer-use registry and this provider; remove an existing competing provider before using it. The responsible host/helper needs Accessibility permission for AX/input and Screen Recording permission for capture. `getCapabilities` reports permission state without opening a system dialog. Start the target application yourself. Select it by name, bundle ID, absolute application path, or PID.
|
||||
The dump command checks profile composition without a model request. For a real task, replace `--dump-config` with a prompt and configure the profile’s model credentials. The overlay mounts the computer-use registry and this provider; remove an existing competing provider before using it. The responsible host/helper needs Accessibility permission for AX/input and Screen Recording permission for capture. `getCapabilities` reports permission state without opening a system dialog. Start the target application yourself. Select it by name, bundle ID, absolute application path, or PID. `getApp` activates the selected application before reading its initial state.
|
||||
|
||||
| Field | Default | Meaning |
|
||||
| --- | --- | --- |
|
||||
|
||||
@@ -34,7 +34,7 @@ export DSH_CUA_HELPER="$cua_native_root/target/release/dsh-cua-native"
|
||||
pnpm dsh headless --patch packages/experimental/cua/examples/macos.cordis.yml --dump-config
|
||||
```
|
||||
|
||||
配置输出命令不发起模型请求。执行真实任务时,将 `--dump-config` 替换为任务提示,并配置该 profile 的模型凭据。配置挂载 computer-use 注册表和本 provider;使用前需移除已有的竞争 provider。负责执行的宿主或辅助进程需要辅助功能权限来读取 AX 和输入,需要屏幕录制权限来截图。`getCapabilities` 只查询权限,不弹出系统授权窗口。目标应用需提前启动,可按名称、bundle ID、应用绝对路径或 PID 选择。
|
||||
配置输出命令不发起模型请求。执行真实任务时,将 `--dump-config` 替换为任务提示,并配置该 profile 的模型凭据。配置挂载 computer-use 注册表和本 provider;使用前需移除已有的竞争 provider。负责执行的宿主或辅助进程需要辅助功能权限来读取 AX 和输入,需要屏幕录制权限来截图。`getCapabilities` 只查询权限,不弹出系统授权窗口。目标应用需提前启动,可按名称、bundle ID、应用绝对路径或 PID 选择。`getApp` 激活选中的应用,再读取初始状态。
|
||||
|
||||
| 字段 | 默认值 | 含义 |
|
||||
| --- | --- | --- |
|
||||
|
||||
@@ -51,3 +51,19 @@ pub fn path_for_pid(pid: i32) -> Option<String> {
|
||||
.map(|value| value.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn activate(pid: i32) -> anyhow::Result<()> {
|
||||
use objc2_app_kit::{NSApplicationActivationOptions, NSRunningApplication};
|
||||
unsafe {
|
||||
let app = NSRunningApplication::runningApplicationWithProcessIdentifier(pid)
|
||||
.ok_or_else(|| anyhow::anyhow!("Application exited before activation"))?;
|
||||
app.unhide();
|
||||
anyhow::ensure!(
|
||||
app.activateWithOptions(
|
||||
NSApplicationActivationOptions::NSApplicationActivateIgnoringOtherApps
|
||||
),
|
||||
"macOS refused application activation"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -397,6 +397,7 @@ impl Driver {
|
||||
"Selector matches multiple running apps; select an exact PID from listApps",
|
||||
)?;
|
||||
let app = &matches[0];
|
||||
apps::activate(app.pid)?;
|
||||
let handle = self.remember_app(app.pid, process_started(app.pid)?);
|
||||
let mut result = json!({"pid":app.pid,"appHandle":handle,"name":app.name});
|
||||
if let Some(bundle_id) = &app.bundle_id {
|
||||
|
||||
@@ -54,7 +54,7 @@
|
||||
{
|
||||
"local": "src/apps.rs",
|
||||
"upstream": "libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs",
|
||||
"changes": "Keep regular running applications, bundle ID and application-path lookup; camelCase output and omit absent bundle identifiers."
|
||||
"changes": "Keep regular running applications, bundle ID and application-path lookup; add activation, camelCase output and omit absent bundle identifiers."
|
||||
}
|
||||
],
|
||||
"localFiles": [
|
||||
|
||||
@@ -163,7 +163,7 @@ export interface CuaApi {
|
||||
*/
|
||||
getState(options?: CuaObservationOptions): Promise<{ apps: CuaAppInfo[]; browsers: [] }>
|
||||
/**
|
||||
* Select an existing app and emit initial AX state.
|
||||
* Activate an existing app and emit initial AX state.
|
||||
* @param selector - exact name, bundle ID, absolute application path or PID string.
|
||||
* @returns a handle tied to this process lifetime.
|
||||
*/
|
||||
|
||||
Reference in new issue
Block a user