refactor(cua): move native provider into experimental
This commit is contained in:
1 parent
17bfa6fd3c
commit
c36a40656b
117 files changed
+200
-200
No files matched your search
@@ -797,7 +797,7 @@ export interface Config {
|
||||
|
||||
- `inject`: `computerUse` · `tools` · `systemPrompt` · `subprocess` · `agents`
|
||||
- `refs`: [`Branded`](../packages/util/brand/src/index.ts) · `Volatile` (`@deepseek-ai/cordis`)
|
||||
- `source`: [`packages/computer-use/cua/src/index.ts:32`](../packages/computer-use/cua/src/index.ts)
|
||||
- `source`: [`packages/experimental/cua/src/index.ts:32`](../packages/experimental/cua/src/index.ts)
|
||||
|
||||
```ts config-catalog
|
||||
/** Native executable and per-operation resource bounds. */
|
||||
|
||||
@@ -799,7 +799,7 @@ export interface Config {
|
||||
|
||||
- `inject`: `computerUse` · `tools` · `systemPrompt` · `subprocess` · `agents`
|
||||
- `refs`: [`Branded`](../packages/util/brand/src/index.ts) · `Volatile` (`@deepseek-ai/cordis`)
|
||||
- `source`: [`packages/computer-use/cua/src/index.ts:32`](../packages/computer-use/cua/src/index.ts)
|
||||
- `source`: [`packages/experimental/cua/src/index.ts:32`](../packages/experimental/cua/src/index.ts)
|
||||
|
||||
```ts config-catalog
|
||||
/** Native executable and per-operation resource bounds. */
|
||||
|
||||
@@ -102,7 +102,7 @@ This matrix shows which packages dispatch each harness-owned event and which pac
|
||||
| `internal/service` | - | `gateway` |
|
||||
| `internal/status` | - | [`agent`](../packages/core/agent), `inspector`, `translator`, [`web`](../packages/web/web) |
|
||||
| `internal/update` | - | [`app-boot`](../packages/boot/app-boot) |
|
||||
| `loader/volatile-update` | - | [`cua`](../packages/computer-use/cua), [`llm-deepseek`](../packages/llm/llm-deepseek), [`llm-pi-ai`](../packages/llm/llm-pi-ai), `product-analytics`, `speech-to-text` |
|
||||
| `loader/volatile-update` | - | [`cua`](../packages/experimental/cua), [`llm-deepseek`](../packages/llm/llm-deepseek), [`llm-pi-ai`](../packages/llm/llm-pi-ai), `product-analytics`, `speech-to-text` |
|
||||
| `slots/changed` | `ui-renderer` (`emit`) | - |
|
||||
<!-- END GENERATED event-producer-consumer:undeclared -->
|
||||
|
||||
|
||||
@@ -104,7 +104,7 @@
|
||||
| `internal/service` | - | `gateway` |
|
||||
| `internal/status` | - | [`agent`](../packages/core/agent), `inspector`, `translator`, [`web`](../packages/web/web) |
|
||||
| `internal/update` | - | [`app-boot`](../packages/boot/app-boot) |
|
||||
| `loader/volatile-update` | - | [`cua`](../packages/computer-use/cua), [`llm-deepseek`](../packages/llm/llm-deepseek), [`llm-pi-ai`](../packages/llm/llm-pi-ai), `product-analytics`, `speech-to-text` |
|
||||
| `loader/volatile-update` | - | [`cua`](../packages/experimental/cua), [`llm-deepseek`](../packages/llm/llm-deepseek), [`llm-pi-ai`](../packages/llm/llm-pi-ai), `product-analytics`, `speech-to-text` |
|
||||
| `slots/changed` | `ui-renderer` (`emit`) | - |
|
||||
<!-- END GENERATED event-producer-consumer:undeclared -->
|
||||
|
||||
|
||||
@@ -214,7 +214,6 @@ flowchart TD
|
||||
end
|
||||
subgraph group_computer_use["packages/computer-use"]
|
||||
pkg_computer_use["computer-use"]
|
||||
pkg_cua["cua"]
|
||||
end
|
||||
subgraph group_context["packages/context"]
|
||||
pkg_agent_instructions["agent-instructions"]
|
||||
@@ -239,6 +238,7 @@ flowchart TD
|
||||
pkg_office_to_pdf["office-to-pdf"]
|
||||
end
|
||||
subgraph group_experimental["packages/experimental"]
|
||||
pkg_cua["cua"]
|
||||
pkg_experimental_agent_team["experimental-agent-team"]
|
||||
pkg_experimental_agent_team_profile["experimental-agent-team-profile"]
|
||||
pkg_experimental_api_speech_to_text["experimental-api-speech-to-text"]
|
||||
@@ -864,12 +864,6 @@ flowchart TD
|
||||
pkg_compaction_image_offload --> pkg_compaction
|
||||
pkg_compaction_image_offload --> pkg_llm
|
||||
pkg_compaction_image_offload --> pkg_session
|
||||
pkg_cua --> pkg_agent
|
||||
pkg_cua --> pkg_computer_use
|
||||
pkg_cua --> pkg_subprocess
|
||||
pkg_cua --> pkg_system_prompt
|
||||
pkg_cua --> pkg_tools
|
||||
pkg_cua --> pkg_user_questions
|
||||
pkg_agent_instructions --> pkg_agent
|
||||
pkg_agent_instructions --> pkg_fs
|
||||
pkg_agent_instructions --> pkg_home_paths
|
||||
@@ -895,6 +889,12 @@ flowchart TD
|
||||
pkg_workspace_changes --> pkg_session
|
||||
pkg_workspace_changes --> pkg_subprocess
|
||||
pkg_workspace_changes --> pkg_tools
|
||||
pkg_cua --> pkg_agent
|
||||
pkg_cua --> pkg_computer_use
|
||||
pkg_cua --> pkg_subprocess
|
||||
pkg_cua --> pkg_system_prompt
|
||||
pkg_cua --> pkg_tools
|
||||
pkg_cua --> pkg_user_questions
|
||||
pkg_experimental_browser_use_chrome_devtools_mcp --> pkg_agent
|
||||
pkg_experimental_browser_use_chrome_devtools_mcp --> pkg_browser_use
|
||||
pkg_experimental_browser_use_chrome_devtools_mcp --> pkg_system_prompt
|
||||
@@ -1687,11 +1687,11 @@ flowchart TD
|
||||
| [`plan-mode`](../packages/plan/plan-mode) | `plan` | [`agent`](../packages/core/agent), [`commands`](../packages/interaction/commands), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-projection`](../packages/session/session-projection), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-questions`](../packages/interaction/user-questions) |
|
||||
| [`command-compact`](../packages/compaction/command-compact) | `compaction` | [`commands`](../packages/interaction/commands), [`compaction`](../packages/compaction/compaction) |
|
||||
| [`compaction-image-offload`](../packages/compaction/compaction-image-offload) | `compaction` | [`agent`](../packages/core/agent), [`compaction`](../packages/compaction/compaction), [`llm`](../packages/llm/llm), [`session`](../packages/core/session) |
|
||||
| [`cua`](../packages/computer-use/cua) | `computer-use` | [`agent`](../packages/core/agent), [`computer-use`](../packages/computer-use/computer-use), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-questions`](../packages/interaction/user-questions) |
|
||||
| [`agent-instructions`](../packages/context/agent-instructions) | `context` | [`agent`](../packages/core/agent), [`fs`](../packages/fs/fs), [`home-paths`](../packages/util/home-paths), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-projection`](../packages/session/session-projection), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) |
|
||||
| [`file-reference-local`](../packages/context/file-reference-local) | `context` | [`agent`](../packages/core/agent), [`file-reference`](../packages/context/file-reference), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) |
|
||||
| [`tool-present`](../packages/deliverables/tool-present) | `deliverables` | [`agent`](../packages/core/agent), [`fs`](../packages/fs/fs), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-projection`](../packages/session/session-projection), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) |
|
||||
| [`workspace-changes`](../packages/deliverables/workspace-changes) | `deliverables` | [`agent`](../packages/core/agent), [`fs`](../packages/fs/fs), [`session`](../packages/core/session), [`subprocess`](../packages/subprocess/subprocess), [`tools`](../packages/core/tools) |
|
||||
| [`cua`](../packages/experimental/cua) | `experimental` | [`agent`](../packages/core/agent), [`computer-use`](../packages/computer-use/computer-use), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-questions`](../packages/interaction/user-questions) |
|
||||
| [`experimental-browser-use-chrome-devtools-mcp`](../packages/experimental/browser-use-chrome-devtools-mcp) | `experimental` | [`agent`](../packages/core/agent), [`browser-use`](../packages/browser-use/browser-use), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) |
|
||||
| [`experimental-browser-use-playwright-mcp`](../packages/experimental/browser-use-playwright-mcp) | `experimental` | [`agent`](../packages/core/agent), [`browser-use`](../packages/browser-use/browser-use), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) |
|
||||
| [`experimental-browser-use-stagehand-native`](../packages/experimental/browser-use-stagehand-native) | `experimental` | [`agent`](../packages/core/agent), [`browser-use`](../packages/browser-use/browser-use), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) |
|
||||
|
||||
@@ -216,7 +216,6 @@ flowchart TD
|
||||
end
|
||||
subgraph group_computer_use["packages/computer-use"]
|
||||
pkg_computer_use["computer-use"]
|
||||
pkg_cua["cua"]
|
||||
end
|
||||
subgraph group_context["packages/context"]
|
||||
pkg_agent_instructions["agent-instructions"]
|
||||
@@ -241,6 +240,7 @@ flowchart TD
|
||||
pkg_office_to_pdf["office-to-pdf"]
|
||||
end
|
||||
subgraph group_experimental["packages/experimental"]
|
||||
pkg_cua["cua"]
|
||||
pkg_experimental_agent_team["experimental-agent-team"]
|
||||
pkg_experimental_agent_team_profile["experimental-agent-team-profile"]
|
||||
pkg_experimental_api_speech_to_text["experimental-api-speech-to-text"]
|
||||
@@ -866,12 +866,6 @@ flowchart TD
|
||||
pkg_compaction_image_offload --> pkg_compaction
|
||||
pkg_compaction_image_offload --> pkg_llm
|
||||
pkg_compaction_image_offload --> pkg_session
|
||||
pkg_cua --> pkg_agent
|
||||
pkg_cua --> pkg_computer_use
|
||||
pkg_cua --> pkg_subprocess
|
||||
pkg_cua --> pkg_system_prompt
|
||||
pkg_cua --> pkg_tools
|
||||
pkg_cua --> pkg_user_questions
|
||||
pkg_agent_instructions --> pkg_agent
|
||||
pkg_agent_instructions --> pkg_fs
|
||||
pkg_agent_instructions --> pkg_home_paths
|
||||
@@ -897,6 +891,12 @@ flowchart TD
|
||||
pkg_workspace_changes --> pkg_session
|
||||
pkg_workspace_changes --> pkg_subprocess
|
||||
pkg_workspace_changes --> pkg_tools
|
||||
pkg_cua --> pkg_agent
|
||||
pkg_cua --> pkg_computer_use
|
||||
pkg_cua --> pkg_subprocess
|
||||
pkg_cua --> pkg_system_prompt
|
||||
pkg_cua --> pkg_tools
|
||||
pkg_cua --> pkg_user_questions
|
||||
pkg_experimental_browser_use_chrome_devtools_mcp --> pkg_agent
|
||||
pkg_experimental_browser_use_chrome_devtools_mcp --> pkg_browser_use
|
||||
pkg_experimental_browser_use_chrome_devtools_mcp --> pkg_system_prompt
|
||||
@@ -1689,11 +1689,11 @@ flowchart TD
|
||||
| [`plan-mode`](../packages/plan/plan-mode) | `plan` | [`agent`](../packages/core/agent), [`commands`](../packages/interaction/commands), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-projection`](../packages/session/session-projection), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-questions`](../packages/interaction/user-questions) |
|
||||
| [`command-compact`](../packages/compaction/command-compact) | `compaction` | [`commands`](../packages/interaction/commands), [`compaction`](../packages/compaction/compaction) |
|
||||
| [`compaction-image-offload`](../packages/compaction/compaction-image-offload) | `compaction` | [`agent`](../packages/core/agent), [`compaction`](../packages/compaction/compaction), [`llm`](../packages/llm/llm), [`session`](../packages/core/session) |
|
||||
| [`cua`](../packages/computer-use/cua) | `computer-use` | [`agent`](../packages/core/agent), [`computer-use`](../packages/computer-use/computer-use), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-questions`](../packages/interaction/user-questions) |
|
||||
| [`agent-instructions`](../packages/context/agent-instructions) | `context` | [`agent`](../packages/core/agent), [`fs`](../packages/fs/fs), [`home-paths`](../packages/util/home-paths), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-projection`](../packages/session/session-projection), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) |
|
||||
| [`file-reference-local`](../packages/context/file-reference-local) | `context` | [`agent`](../packages/core/agent), [`file-reference`](../packages/context/file-reference), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) |
|
||||
| [`tool-present`](../packages/deliverables/tool-present) | `deliverables` | [`agent`](../packages/core/agent), [`fs`](../packages/fs/fs), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-projection`](../packages/session/session-projection), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) |
|
||||
| [`workspace-changes`](../packages/deliverables/workspace-changes) | `deliverables` | [`agent`](../packages/core/agent), [`fs`](../packages/fs/fs), [`session`](../packages/core/session), [`subprocess`](../packages/subprocess/subprocess), [`tools`](../packages/core/tools) |
|
||||
| [`cua`](../packages/experimental/cua) | `experimental` | [`agent`](../packages/core/agent), [`computer-use`](../packages/computer-use/computer-use), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-questions`](../packages/interaction/user-questions) |
|
||||
| [`experimental-browser-use-chrome-devtools-mcp`](../packages/experimental/browser-use-chrome-devtools-mcp) | `experimental` | [`agent`](../packages/core/agent), [`browser-use`](../packages/browser-use/browser-use), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) |
|
||||
| [`experimental-browser-use-playwright-mcp`](../packages/experimental/browser-use-playwright-mcp) | `experimental` | [`agent`](../packages/core/agent), [`browser-use`](../packages/browser-use/browser-use), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) |
|
||||
| [`experimental-browser-use-stagehand-native`](../packages/experimental/browser-use-stagehand-native) | `experimental` | [`agent`](../packages/core/agent), [`browser-use`](../packages/browser-use/browser-use), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) |
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
# After editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write docs/subsystems/computer-use.md
|
||||
/computer-use:
|
||||
en: d6bcbf1280703efa
|
||||
zh: f09fc8a6fdc88201
|
||||
en: f7410eb6db7e1636
|
||||
zh: 5adc97516d1d6c6f
|
||||
/computer-use/choose-a-provider:
|
||||
en: fa2e0d7b41c0450b
|
||||
zh: af9a799711f901e6
|
||||
|
||||
@@ -5,7 +5,7 @@ English | [中文](computer-use.zh.md)
|
||||
Computer use lets a model observe and operate the local desktop through a configured provider. The shared DSH capability is called **computer use**; **Cua Driver** names the upstream implementation.
|
||||
|
||||
|
||||
The native [`dsh-cua`](../../packages/computer-use/cua/README.md) provider supports explicit source composition and enforces application consent through User Questions and profile configuration.
|
||||
The native [`dsh-cua`](../../packages/experimental/cua/README.md) provider supports explicit source composition and enforces application consent through User Questions and profile configuration.
|
||||
|
||||
## Choose a provider
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
计算机操作让模型通过配置的提供方观察并操作本地桌面。DSH 的共享能力称为 **computer use(计算机操作)**;**Cua Driver** 是上游实现的名称。
|
||||
|
||||
|
||||
原生 [`dsh-cua`](../../packages/computer-use/cua/README.zh.md) 提供方支持显式源码组合,通过 User Questions 和配置文件执行应用授权规则。
|
||||
原生 [`dsh-cua`](../../packages/experimental/cua/README.zh.md) 提供方支持显式源码组合,通过 User Questions 和配置文件执行应用授权规则。
|
||||
|
||||
## 选择提供方
|
||||
|
||||
|
||||
@@ -9,17 +9,17 @@
|
||||
en: 77ea90624a004773
|
||||
zh: fe6dbf8414a5aff0
|
||||
/packages-computer-use/summary:
|
||||
en: 3820da56a601d3f1
|
||||
zh: 358e65f8a7cf7446
|
||||
en: c4ca74960d331ef6
|
||||
zh: b682b6337c89a280
|
||||
/packages-computer-use/table-of-contents:
|
||||
en: 1eb35b015a6c423f
|
||||
zh: a8a3dceee2c9e943
|
||||
/packages-computer-use/packages:
|
||||
en: 80be7a1677eeaf09
|
||||
zh: 8517044dbc07802e
|
||||
en: 9aa86e6451f60c3d
|
||||
zh: c5e6816e90f2d74b
|
||||
/packages-computer-use/related-documentation:
|
||||
en: ad08d3417a0d4630
|
||||
zh: cfe8e03033c2fede
|
||||
en: 3a94b377247283ab
|
||||
zh: c97d88230ddc7c37
|
||||
/packages-computer-use/dev-note:
|
||||
en: b75c8458928b72b9
|
||||
zh: 003ba4e17b4b5269
|
||||
@@ -9,7 +9,7 @@ English | [中文](README.zh.md)
|
||||
|
||||
## Summary
|
||||
|
||||
Computer-use providers let models observe and operate a desktop. This group owns exclusive provider registration. Each provider owns its operations, tools, and platform requirements; the experimental Cua Driver providers live in the experimental group.
|
||||
Computer-use providers let models observe and operate a desktop. This group owns exclusive provider registration. Each provider owns its operations, tools, and platform requirements; the native CUA and Cua Driver providers live in the experimental group.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
@@ -27,13 +27,12 @@ Choose one provider and mount the shared registration service.
|
||||
| Package | Role | ctx key |
|
||||
|---|---|---|
|
||||
| [`computer-use`](computer-use/README.md) | Exclusive named provider registration | `ctx.computerUse` |
|
||||
| [`cua`](cua/README.md) | Native macOS application and window operations | `ctx.computerUse` |
|
||||
|
||||
<a id="related-documentation"></a>
|
||||
## Related documentation
|
||||
|
||||
- [Computer use](../../docs/subsystems/computer-use.md) — capability ownership and provider choices.
|
||||
- [Experimental packages](../experimental/README.md) — Cua Driver MCP and native providers.
|
||||
- [Experimental packages](../experimental/README.md) — native CUA and Cua Driver providers.
|
||||
|
||||
<a id="dev-note"></a>
|
||||
## Dev Note
|
||||
|
||||
@@ -9,7 +9,7 @@ kind: "package-group"
|
||||
|
||||
## 概述
|
||||
|
||||
计算机操作提供方让模型观察并操作桌面。本组负责提供方的独占注册。各提供方拥有自己的操作、工具和平台要求;实验性 Cua Driver 提供方位于 experimental 组。
|
||||
计算机操作提供方让模型观察并操作桌面。本组负责提供方的独占注册。各提供方拥有自己的操作、工具和平台要求;原生 CUA 和 Cua Driver 提供方位于 experimental 组。
|
||||
|
||||
## 目录
|
||||
|
||||
@@ -27,13 +27,12 @@ kind: "package-group"
|
||||
| 包 | 职责 | ctx 键 |
|
||||
|---|---|---|
|
||||
| [`computer-use`](computer-use/README.zh.md) | 按名称独占注册提供方 | `ctx.computerUse` |
|
||||
| [`cua`](cua/README.zh.md) | 原生 macOS 应用与窗口操作 | `ctx.computerUse` |
|
||||
|
||||
<a id="related-documentation"></a>
|
||||
## 相关文档
|
||||
|
||||
- [计算机操作](../../docs/subsystems/computer-use.zh.md) — 能力归属和提供方选择。
|
||||
- [实验性包](../experimental/README.zh.md) — Cua Driver MCP 和原生提供方。
|
||||
- [实验性包](../experimental/README.zh.md) — 原生 CUA 和 Cua Driver 提供方。
|
||||
|
||||
<a id="dev-note"></a>
|
||||
## 开发备注
|
||||
|
||||
@@ -18,8 +18,8 @@
|
||||
en: b55aeb927737b90d
|
||||
zh: 0b215fcb5c2c7908
|
||||
/packages-experimental/packages:
|
||||
en: 8ea21461f84d30ad
|
||||
zh: b521c2d38a2b8eec
|
||||
en: b404c79bddc92273
|
||||
zh: d6b9afb8777a8126
|
||||
/packages-experimental/related-documentation:
|
||||
en: 1cb9d3448d44e48d
|
||||
zh: cd326d70a746b896
|
||||
|
||||
@@ -66,6 +66,7 @@ Promotion requires a named product role, documented public behavior and limitati
|
||||
| [`claude-code-mods`](claude-code-mods/README.md) | Run Claude Code mods as plugins: their hook chains on harness extension points and a band above the prompt | `ctx.claudeCodeMods` |
|
||||
| [`client-ui-claude-code-mods`](client-ui-claude-code-mods/README.md) | The Web band that draws a mod's tree above the prompt and sends button clicks back | — |
|
||||
| [`ptc-runtime-python`](ptc-runtime-python/README.md) | CPython subprocess backend for the PTC execution seam | `ctx.ptcRuntime` |
|
||||
| [`cua`](cua/README.md) | Native macOS application and window operations | `ctx.computerUse` |
|
||||
| [`computer-use-cua-driver-mcp`](computer-use-cua-driver-mcp/README.md) | Use an installed Cua Driver through MCP | `ctx.computerUse` |
|
||||
| [`computer-use-cua-driver-native`](computer-use-cua-driver-native/README.md) | Embed the Cua Driver native npm runtime | `ctx.computerUse` |
|
||||
| [`browser-use-playwright-mcp`](browser-use-playwright-mcp/README.md) | Playwright browser tools over MCP | `ctx.browserUse` |
|
||||
|
||||
@@ -66,6 +66,7 @@ kind: "package-group"
|
||||
| [`claude-code-mods`](claude-code-mods/README.zh.md) | 把 Claude Code 模组作为插件运行:钩子链落在 harness 扩展点上,并在提示框上方绘制横幅 | `ctx.claudeCodeMods` |
|
||||
| [`client-ui-claude-code-mods`](client-ui-claude-code-mods/README.zh.md) | 在提示框上方绘制模组树并把按钮点击发回的 Web 横幅 | — |
|
||||
| [`ptc-runtime-python`](ptc-runtime-python/README.zh.md) | PTC 执行 seam 的 CPython 子进程后端 | `ctx.ptcRuntime` |
|
||||
| [`cua`](cua/README.zh.md) | 原生 macOS 应用与窗口操作 | `ctx.computerUse` |
|
||||
| [`computer-use-cua-driver-mcp`](computer-use-cua-driver-mcp/README.zh.md) | 通过 MCP 使用已安装的 Cua Driver | `ctx.computerUse` |
|
||||
| [`computer-use-cua-driver-native`](computer-use-cua-driver-native/README.zh.md) | 嵌入 Cua Driver 原生 npm 运行时 | `ctx.computerUse` |
|
||||
| [`browser-use-playwright-mcp`](browser-use-playwright-mcp/README.zh.md) | 通过 MCP 提供 Playwright 浏览器工具 | `ctx.browserUse` |
|
||||
|
||||
+11
-11
@@ -1,13 +1,13 @@
|
||||
# Bilingual-pair consistency record for README.md (docs/i18n/README.md): per heading
|
||||
# section, a hash of its English and Chinese blocks outside code blocks and generated regions.
|
||||
# After editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write packages/computer-use/cua/README.md
|
||||
# pnpm run verify-translation-pairing --write packages/experimental/cua/README.md
|
||||
/:
|
||||
en: a1503331d3f35516
|
||||
zh: fcb8e5220ff732e1
|
||||
/deepseek-ai-dsh-cua:
|
||||
en: 10b9bc038b8de707
|
||||
zh: 60d3eaf0b4349754
|
||||
en: 0ef7ee882840cf1b
|
||||
zh: 01b9e768eff2d302
|
||||
/deepseek-ai-dsh-cua/summary:
|
||||
en: 4549f02fc53854a5
|
||||
zh: 6f74f1130b6623a8
|
||||
@@ -15,14 +15,14 @@
|
||||
en: d152484eb41ac6b4
|
||||
zh: 7b30b973a849247d
|
||||
/deepseek-ai-dsh-cua/use-this-package:
|
||||
en: 851d4fc9ce8988cd
|
||||
zh: 581a3c45f3c0ffea
|
||||
en: 760c77bce6472262
|
||||
zh: a8806a04335dd6e5
|
||||
/deepseek-ai-dsh-cua/use-this-package/connect-an-external-execution-environment:
|
||||
en: 8c3ee662123e7bb8
|
||||
zh: f92124b776260c1b
|
||||
en: 98a598ea10daad8c
|
||||
zh: 62ea7d5a1d14d0c4
|
||||
/deepseek-ai-dsh-cua/understand-the-implementation:
|
||||
en: 40978d9a4963a5b6
|
||||
zh: b4641bd2a739a987
|
||||
en: 66c97ed581583ba3
|
||||
zh: 35bb9ee599415a9b
|
||||
/deepseek-ai-dsh-cua/further-exploration:
|
||||
en: b1047318879cd6f4
|
||||
zh: e4ce4a5ad2a9c4dc
|
||||
@@ -45,8 +45,8 @@
|
||||
en: a2c9acb5a4320dda
|
||||
zh: e8d1586af60215c4
|
||||
/deepseek-ai-dsh-cua/known-limitations-and-deferred-work:
|
||||
en: c4f43ec8daf7537a
|
||||
zh: 246280e9142fca01
|
||||
en: 26dd94b47d269b44
|
||||
zh: d29a51deb3e96756
|
||||
/deepseek-ai-dsh-cua/known-limitations-and-deferred-work/dev-note:
|
||||
en: b836f01bd0bceb4a
|
||||
zh: 8122fa08b2aa95fa
|
||||
@@ -25,13 +25,13 @@ Inspect macOS permissions, select running applications and windows, read accessi
|
||||
<a id="use-this-package"></a>
|
||||
## Use this package
|
||||
|
||||
Enable this provider through explicit source composition on macOS 14 or later. Shipped profiles do not mount it. Prepare a native helper and load the registry/provider overlay:
|
||||
Enable this experimental provider through explicit source composition on macOS 14 or later. Windows support is in development. Shipped profiles do not mount it. Prepare a native helper and load the registry/provider overlay:
|
||||
|
||||
```sh
|
||||
swift build --package-path packages/computer-use/cua/native --configuration release
|
||||
cua_native_root="$PWD/packages/computer-use/cua/native"
|
||||
swift build --package-path packages/experimental/cua/native --configuration release
|
||||
cua_native_root="$PWD/packages/experimental/cua/native"
|
||||
export DSH_CUA_HELPER="$cua_native_root/.build/release/dsh-cua-native"
|
||||
pnpm dsh headless --patch packages/computer-use/cua/examples/macos.cordis.yml --dump-config
|
||||
pnpm dsh headless --patch packages/experimental/cua/examples/macos.cordis.yml --dump-config
|
||||
```
|
||||
|
||||
The dump command checks profile composition without a model request. For a real task, replace `--dump-config` with a prompt and configure the profile’s model credentials. The overlay mounts the computer-use registry and this provider; remove an existing competing provider before using it. The responsible host/helper needs Accessibility permission for AX/input and Screen Recording permission for capture. `getCapabilities` reports permissions and session lock state without opening a system dialog. Locked or unavailable desktop sessions reject UI observations and input before dispatch; the user must unlock macOS before resuming. `listApps` and `cua.getState` list running applications. `findApps(query)` searches standard macOS application directories by name, bundle ID or absolute path. `getApp` selects a unique name, bundle ID, path or running PID, obtains application consent, and launches a stopped app in the background when needed. A stopped executable is identified before approval and checked again before and after launch; unsigned code only receives task access. The returned object reads initial AX without activating the app. Launch cancellation may leave the application running. Model guidance starts with background observation and input, without activation during initialization or before each action. For hidden or minimized windows, try `app.show()` or `window.show()` and read fresh state. This requests unhide and unminimize without activation, raising or switching Spaces; app-level restoration selects the focused, main or sole AX window. The result names the selected `windowId`, or null when no window was selected, and reports `foregroundChanged` by comparing the front app and top window before and after the request. Null means foreground evidence was unavailable; later focus changes remain possible. If restoration fails or the window remains unusable, explain the needed foreground step, call `app.activate()` or an observed window’s `raise` action, then observe again. Necessary foreground access takes priority over preserving focus. Applications may change focus after input; background delivery does not guarantee an undisturbed user session.
|
||||
@@ -74,7 +74,7 @@ Native preparation serializes compiler checks and publication with the shared fi
|
||||
|
||||
### Connect an external execution environment
|
||||
|
||||
To run through persistent PTC, add `--patch packages/computer-use/cua/examples/persistent-ptc.cordis.yml` after the macOS overlay. This provider keeps variables for the live Agent and applies the selected file sandbox policy. The caller imports the bridge and supplies the current authorized `tools.cua` binding. Persistent PTC receives an installation-owned bridge file URL in its CUA instructions, so the project does not need to install this package. The example below is for an external host that resolves the package from its own installation and supplies `tools`; the package does not install a global object or keep variables alive:
|
||||
To run through persistent PTC, add `--patch packages/experimental/cua/examples/persistent-ptc.cordis.yml` after the macOS overlay. This provider keeps variables for the live Agent and applies the selected file sandbox policy. The caller imports the bridge and supplies the current authorized `tools.cua` binding. Persistent PTC receives an installation-owned bridge file URL in its CUA instructions, so the project does not need to install this package. The example below is for an external host that resolves the package from its own installation and supplies `tools`; the package does not install a global object or keep variables alive:
|
||||
|
||||
```javascript
|
||||
import { connectCua } from '@deepseek-ai/dsh-cua/bridge'
|
||||
@@ -111,7 +111,7 @@ Indexed clicks support a single right/middle click or a single/double left click
|
||||
|
||||
The object facade translates app/window methods into JSON operations. The bridge routes them through the normal tool pipeline, preserving authorization, nested logs and image admission. Agents and Hosts belonging to one user share a helper for the same native build. Each connection retains independent handles, snapshots, configuration and cancellation state; desktop operations use a global queue. The managed launcher transfers service lifetime to these connections, and the helper exits after the last connection finishes cleanup. Process locks cover both build identity and the real executable path, preventing duplicate startup and path collisions after a running executable is replaced. Native handles bind a process identity to its kernel start time, and snapshots retain native AX references only inside that helper. The same operation protocol can serve a future platform provider without adding platform logic to the object API.
|
||||
|
||||
[Zod operation definitions](src/protocol.ts) own model parameters and separately declare Host resource fields and private continuations. Public data types derive from those definitions. `pnpm exec tsx packages/computer-use/cua/native/generate-protocol.ts` generates the native JSON Schema and Swift field/enum admission tables; the package test lane checks freshness and rejects Host-only fields at model admission. Native target validation, resource limits and operation behavior remain enforced at execution.
|
||||
[Zod operation definitions](src/protocol.ts) own model parameters and separately declare Host resource fields and private continuations. Public data types derive from those definitions. `pnpm exec tsx packages/experimental/cua/native/generate-protocol.ts` generates the native JSON Schema and Swift field/enum admission tables; the package test lane checks freshness and rejects Host-only fields at model admission. Native target validation, resource limits and operation behavior remain enforced at execution.
|
||||
|
||||
[AX text formatting](native/Sources/CuaNative/AXTextFormatter.swift) consumes already-read node data. The AX walker alone assigns indices and retains native references, so formatting owns no second snapshot or target registry.
|
||||
|
||||
@@ -126,7 +126,7 @@ Node roles and metadata share one AX batch per observation. Group and scroll-are
|
||||
<a id="further-exploration"></a>
|
||||
## Further Exploration
|
||||
|
||||
- [Computer-use registration](../computer-use/README.md)
|
||||
- [Computer-use registration](../../computer-use/computer-use/README.md)
|
||||
- [Native protocol](native/protocol.schema.json)
|
||||
- [Object interfaces](src/types.ts)
|
||||
|
||||
@@ -197,7 +197,7 @@ The queue orders individual native requests. Holding an app object does not rese
|
||||
|
||||
<a id="known-limitations-and-deferred-work"></a>
|
||||
|
||||
- Only the macOS native implementation is provided. Browser tabs, AX diffs, unassociated system panels and accessory-only apps are unsupported.
|
||||
- Only the macOS native implementation is provided. Windows support is in development. Browser tabs, AX diffs, unassociated system panels and accessory-only apps are unsupported.
|
||||
- Coordinates use window-local points including the title bar. PNG pixel dimensions can differ. Window movement, application changes and other desktop users can race observations and input.
|
||||
- macOS Stage Manager can reduce background window bounds and screenshots to thumbnails. Full-window observation and coordinate input in this state are not guaranteed; model guidance requires explaining foreground needs and avoiding repeated focus reclamation.
|
||||
- Background input depends on private SkyLight symbols; macOS updates can make them unavailable. Validate each target OS version.
|
||||
@@ -25,13 +25,13 @@ kind: "package-reference"
|
||||
<a id="use-this-package"></a>
|
||||
## 使用本包
|
||||
|
||||
该提供方需要在 macOS 14 及以上通过源码配置显式启用,默认配置不加载它。先准备原生组件,再加载注册表和提供方覆盖层:
|
||||
该实验性提供方需要在 macOS 14 及以上通过源码配置显式启用。Windows 支持正在开发中。默认配置不加载它。先准备原生组件,再加载注册表和提供方覆盖层:
|
||||
|
||||
```sh
|
||||
swift build --package-path packages/computer-use/cua/native --configuration release
|
||||
cua_native_root="$PWD/packages/computer-use/cua/native"
|
||||
swift build --package-path packages/experimental/cua/native --configuration release
|
||||
cua_native_root="$PWD/packages/experimental/cua/native"
|
||||
export DSH_CUA_HELPER="$cua_native_root/.build/release/dsh-cua-native"
|
||||
pnpm dsh headless --patch packages/computer-use/cua/examples/macos.cordis.yml --dump-config
|
||||
pnpm dsh headless --patch packages/experimental/cua/examples/macos.cordis.yml --dump-config
|
||||
```
|
||||
|
||||
配置输出命令不发起模型请求。执行真实任务时,将 `--dump-config` 替换为任务提示,并配置该 profile 的模型凭据。配置挂载 computer-use 注册表和本 provider;使用前需移除已有的竞争 provider。负责执行的宿主或辅助进程需要辅助功能权限来读取 AX 和输入,需要屏幕录制权限来截图。`getCapabilities` 查询权限和锁屏状态,不弹出系统授权窗口。桌面会话锁定或不可用时,界面观察与输入会在分派前拒绝,需由用户解锁 macOS 后继续。`listApps` 和 `cua.getState` 列出运行中的应用。`findApps(query)` 按名称、bundle ID 或绝对路径搜索 macOS 标准应用目录。`getApp` 按唯一名称、bundle ID、路径或运行中的 PID 选择应用,取得应用许可后,在需要时后台启动尚未运行的应用。启动前先确认可执行文件身份,授权后在启动前后再次核对;未签名代码只取得任务许可。返回的对象读取初始 AX,不自动激活应用。启动被取消时,应用可能已经运行。模型指引要求先在后台观察和输入,不在初始化或每次动作前激活应用。窗口隐藏或最小化时,先尝试 `app.show()` 或 `window.show()`,再重新观察。它请求取消隐藏和恢复最小化,不主动激活、抬起窗口或切换桌面空间;应用级恢复选择焦点窗口、主窗口或唯一 AX 窗口。结果通过 `windowId` 指明选中的窗口,未选中时为 null;`foregroundChanged` 比较操作前后的前台应用和最上层窗口,无法取得完整观察时为 null,返回后仍可能发生焦点变化。恢复失败或窗口仍不可用时,说明需要前台的步骤,再调用 `app.activate()` 或窗口的 `raise` 动作并重新观察。必须使用前台时,以完成任务为先。应用收到输入后仍可能改变焦点,后台投递不保证用户操作始终不受干扰。
|
||||
@@ -74,7 +74,7 @@ AX 观察默认只展示菜单栏入口。使用 `app.getAXState({ includeMenus:
|
||||
|
||||
### 接入外部执行环境
|
||||
|
||||
使用 persistent PTC 时,在 macOS 配置后追加 `--patch packages/computer-use/cua/examples/persistent-ptc.cordis.yml`。该 provider 为存活的 Agent 保留变量,并遵守当前选择的文件沙箱策略。调用方导入适配模块,并提供当前执行中已授权的 `tools.cua` 绑定。persistent PTC 会在 CUA 指引中收到安装目录内 bridge 的文件 URL,项目无须安装本包。下面的例子用于能从自身安装目录解析本包、且已提供 `tools` 的外部宿主;本包不安装全局对象,也不负责保留变量:
|
||||
使用 persistent PTC 时,在 macOS 配置后追加 `--patch packages/experimental/cua/examples/persistent-ptc.cordis.yml`。该 provider 为存活的 Agent 保留变量,并遵守当前选择的文件沙箱策略。调用方导入适配模块,并提供当前执行中已授权的 `tools.cua` 绑定。persistent PTC 会在 CUA 指引中收到安装目录内 bridge 的文件 URL,项目无须安装本包。下面的例子用于能从自身安装目录解析本包、且已提供 `tools` 的外部宿主;本包不安装全局对象,也不负责保留变量:
|
||||
|
||||
```javascript
|
||||
import { connectCua } from '@deepseek-ai/dsh-cua/bridge'
|
||||
@@ -111,7 +111,7 @@ AppKit sheet 可以通过其窗口 ID 单独选择,也会出现在所属文档
|
||||
|
||||
对象接口将应用和窗口方法转换为 JSON 操作。适配模块通过标准工具链保留授权、嵌套日志和图片接入。同一用户的相同 helper 构建由多个 Agent 和 Host 共享。每条连接独立保存句柄、快照、配置和取消状态,桌面操作通过全局队列执行。受管启动器将服务生命期交给这些连接,最后一条连接清理完成后服务退出。进程锁覆盖构建身份及真实可执行路径,防止重复启动或覆盖运行版本后产生路径冲突。原生句柄绑定进程身份及内核启动时间,快照中的 AX 引用只保存在辅助进程内。未来平台可以实现相同操作协议,无需在对象接口中加入平台逻辑。
|
||||
|
||||
[Zod 操作定义](src/protocol.ts)负责模型参数,并单独声明 Host 资源字段和内部续接操作。公共数据类型从这些定义推导。`pnpm exec tsx packages/computer-use/cua/native/generate-protocol.ts` 生成原生 JSON Schema 和 Swift 字段、枚举校验表;包测试检查产物是否同步,并验证模型不能提交 Host 专属字段。原生目标验证、资源上限和操作行为仍在执行处检查。
|
||||
[Zod 操作定义](src/protocol.ts)负责模型参数,并单独声明 Host 资源字段和内部续接操作。公共数据类型从这些定义推导。`pnpm exec tsx packages/experimental/cua/native/generate-protocol.ts` 生成原生 JSON Schema 和 Swift 字段、枚举校验表;包测试检查产物是否同步,并验证模型不能提交 Host 专属字段。原生目标验证、资源上限和操作行为仍在执行处检查。
|
||||
|
||||
[AX 文本格式化](native/Sources/CuaNative/AXTextFormatter.swift)只接收已读取的节点数据。AX 遍历器独自分配索引并保存原生引用,格式化不维护另一套快照或目标注册表。
|
||||
|
||||
@@ -126,7 +126,7 @@ AppKit sheet 可以通过其窗口 ID 单独选择,也会出现在所属文档
|
||||
<a id="further-exploration"></a>
|
||||
## 进一步阅读
|
||||
|
||||
- [Computer-use 注册](../computer-use/README.zh.md)
|
||||
- [Computer-use 注册](../../computer-use/computer-use/README.zh.md)
|
||||
- [原生协议](native/protocol.schema.json)
|
||||
- [对象接口](src/types.ts)
|
||||
|
||||
@@ -197,7 +197,7 @@ Use selectText only on controls advertising selectionWritable. It selects a uniq
|
||||
|
||||
<a id="known-limitations-and-deferred-work"></a>
|
||||
|
||||
- 只提供 macOS 原生实现。不支持浏览器标签、启动应用、AX 差分、无法确认应用归属的系统面板,以及仅后台辅助类型的应用。
|
||||
- 只提供 macOS 原生实现,Windows 支持正在开发中。不支持浏览器标签、启动应用、AX 差分、无法确认应用归属的系统面板,以及仅后台辅助类型的应用。
|
||||
- 坐标使用包含标题栏的窗口内部 point,可能不同于 PNG 像素尺寸。窗口移动、应用变化及其他桌面使用者的操作可能与观察和输入交错。
|
||||
- macOS 台前调度可能将后台窗口的尺寸与截图变为缩略图,当前无法保证这种状态下的完整画面与坐标输入;模型指引要求说明前台需求并避免反复抢回焦点。
|
||||
- 后台输入依赖私有 SkyLight 符号,macOS 升级可能使其不可用,需在目标系统版本验证。
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
+11
-11
@@ -1,8 +1,8 @@
|
||||
# dsh-cua native helper
|
||||
|
||||
This macOS 14+ JSON-lines helper uses Swift 6, AppKit, Accessibility, CoreGraphics and ScreenCaptureKit. It adapts selected MIT-licensed Cua Driver behavior without loading an external driver. `upstream.json` records the fixed upstream version, adapted Swift files and local modules; `LICENSE.cua-driver` preserves the MIT notice.
|
||||
This macOS 14+ JSON-lines helper uses Swift 6, AppKit, Accessibility, CoreGraphics and ScreenCaptureKit. Windows support is in development. It adapts selected MIT-licensed Cua Driver behavior without loading an external driver. `upstream.json` records the fixed upstream version, adapted Swift files and local modules; `LICENSE.cua-driver` preserves the MIT notice.
|
||||
|
||||
Build with `swift build --package-path packages/computer-use/cua/native --configuration release`. The binary is `.build/release/dsh-cua-native` beneath this directory. The Host runs `--ensure-shared` as a managed startup command; the serving process then belongs to its connected clients and exits after the last connection finishes cleanup. The `.build/` directory is not committed.
|
||||
Build with `swift build --package-path packages/experimental/cua/native --configuration release`. The binary is `.build/release/dsh-cua-native` beneath this directory. The Host runs `--ensure-shared` as a managed startup command; the serving process then belongs to its connected clients and exits after the last connection finishes cleanup. The `.build/` directory is not committed.
|
||||
|
||||
The shared service uses an owner-only Unix socket under `/tmp/dsh-cua-<uid>`, keyed by the executable SHA-256. A version lock elects one process; an additional real-executable-path lock excludes a different build at the same path. The first frame is protocol-1 `hello` with the runtime key and this connection’s byte, AX, operation, screenshot, event-spacing and cleanup limits. `ready` reports the verified runtime key, PID and instance ID. Every connection owns an independent AXWorker. The stdin/stdout entry is a diagnostic proxy to this service.
|
||||
|
||||
@@ -38,27 +38,27 @@ Capture and pointer delivery read CG bounds before and after an exact-window AX
|
||||
|
||||
The responsible host/helper needs macOS Accessibility permission for AX state and input, and Screen Recording permission for screenshots. Capability preflight and permission errors do not grant permissions. Capture refuses a window that moves or resizes while the image is being read. Unassociated system panels and accessory-only applications are not selectable.
|
||||
|
||||
Use `python3 packages/computer-use/cua/native/tests/test_swift.py` for offline unit checks with the selected Apple toolchain’s Swift Testing framework/runtime paths and `python3 packages/computer-use/cua/native/tests/protocol_e2e.py` for JSONL admission, ordered replies and EOF shutdown without desktop operations. All Python native checks accept `--helper` to select a separately built comparison binary.
|
||||
Use `python3 packages/experimental/cua/native/tests/test_swift.py` for offline unit checks with the selected Apple toolchain’s Swift Testing framework/runtime paths and `python3 packages/experimental/cua/native/tests/protocol_e2e.py` for JSONL admission, ordered replies and EOF shutdown without desktop operations. All Python native checks accept `--helper` to select a separately built comparison binary.
|
||||
|
||||
Run `python3 packages/computer-use/cua/native/tests/shared_helper_e2e.py --helper /absolute/helper` to verify concurrent startup, a shared service PID, independent handles, same-application handoff, request-local screenshot deadlines, connection cleanup and signal shutdown with an owned AppKit fixture. Existing Accessibility and Screen Recording grants are required. `--without-desktop` limits the check to protocol and process lifetime.
|
||||
Run `python3 packages/experimental/cua/native/tests/shared_helper_e2e.py --helper /absolute/helper` to verify concurrent startup, a shared service PID, independent handles, same-application handoff, request-local screenshot deadlines, connection cleanup and signal shutdown with an owned AppKit fixture. Existing Accessibility and Screen Recording grants are required. `--without-desktop` limits the check to protocol and process lifetime.
|
||||
|
||||
Run `python3 packages/computer-use/cua/native/tests/axless_e2e.py` after building and unlocking the Mac to verify an owned window whose application removes and later restores its AX roots. The check requires existing Accessibility/Screen Recording grants, rejects stale element indices, captures the AXless window, verifies a coordinate click in the fixture state file, and checks restoration. It never changes system permission settings.
|
||||
Run `python3 packages/experimental/cua/native/tests/axless_e2e.py` after building and unlocking the Mac to verify an owned window whose application removes and later restores its AX roots. The check requires existing Accessibility/Screen Recording grants, rejects stale element indices, captures the AXless window, verifies a coordinate click in the fixture state file, and checks restoration. It never changes system permission settings.
|
||||
|
||||
Stable error codes are `INVALID_REQUEST` (malformed envelope), `INVALID_ARGUMENT` (invalid parameters), `UNSUPPORTED` (unknown operation), `APP_UNAVAILABLE`, `AMBIGUOUS_APP`, `STALE_APP`, `STALE_SNAPSHOT`, `WINDOW_UNAVAILABLE`, `WINDOW_CHANGED`, `PERMISSION_DENIED`, `PANEL_NOT_READY` (system panel recipient unavailable before input), and `NATIVE_ERROR` (remaining operating-system failures). An error never promises that a partially dispatched input was rolled back. The host must not retry input automatically.
|
||||
|
||||
Observation reads each non-layout node’s descriptive properties and children through `AXUIElementCopyMultipleAttributeValues`. Per-attribute errors retain the individual-read rules; unsupported batch APIs use individual reads. Values are retained only during that observation, and input reads live attributes. Application state can change between nodes; a batch does not make the whole tree atomic.
|
||||
|
||||
Run `python3 packages/computer-use/cua/native/tests/observation_benchmark.py --reference-helper /absolute/reference --candidate-helper /absolute/candidate --artifacts /private/evidence` to compare complete observations of the same owned AppKit window with 35 and 400 synthetic rows. Each helper receives one warmup and three alternating measured requests. All result fields must match except helper-generated app and snapshot IDs. The report keeps raw timings, byte counts, hashes and replies; it sets no CI speed threshold and makes no memory claim. Fixture startup can activate the application. Run without other desktop automation or compilation jobs.
|
||||
Run `python3 packages/experimental/cua/native/tests/observation_benchmark.py --reference-helper /absolute/reference --candidate-helper /absolute/candidate --artifacts /private/evidence` to compare complete observations of the same owned AppKit window with 35 and 400 synthetic rows. Each helper receives one warmup and three alternating measured requests. All result fields must match except helper-generated app and snapshot IDs. The report keeps raw timings, byte counts, hashes and replies; it sets no CI speed threshold and makes no memory claim. Fixture startup can activate the application. Run without other desktop automation or compilation jobs.
|
||||
|
||||
Use `--panels-only` to isolate the system file-panel checks. Fixture windows can join application sets so Stage Manager does not shrink them during background input.
|
||||
|
||||
Run `python3 packages/computer-use/cua/native/tests/e2e.py` after building to exercise an isolated two-window AppKit fixture. It independently records button counts, Unicode text, actual scroll position, right-click callbacks, and drag release position. Scroll, right-button down/up, and drag down/up counters must each increase exactly once per requested gesture. The check also verifies a real NSOpenPanel sheet, path-entry keyboard routing and the selected file URL. It covers scoped snapshot replacement, screenshot dimensions, stale app cleanup and EOF shutdown. `--artifacts DIRECTORY` writes fixture-only screenshots and a private protocol transcript; the transcript contains the running-app inventory and is not a publication artifact.
|
||||
Run `python3 packages/experimental/cua/native/tests/e2e.py` after building to exercise an isolated two-window AppKit fixture. It independently records button counts, Unicode text, actual scroll position, right-click callbacks, and drag release position. Scroll, right-button down/up, and drag down/up counters must each increase exactly once per requested gesture. The check also verifies a real NSOpenPanel sheet, path-entry keyboard routing and the selected file URL. It covers scoped snapshot replacement, screenshot dimensions, stale app cleanup and EOF shutdown. `--artifacts DIRECTORY` writes fixture-only screenshots and a private protocol transcript; the transcript contains the running-app inventory and is not a publication artifact.
|
||||
|
||||
For a Blender keyboard check, run `python3 packages/computer-use/cua/native/tests/blender_keyboard_e2e.py --blender /absolute/path/to/Blender.app/Contents/MacOS/Blender`. It launches one owned factory-startup GUI with private user directories, observes actual Text Editor text, selection and received modifier flags, and checks Command+A replacement, Shift+Left selection and ordinary input after modifier release. It requires an unlocked macOS session and Accessibility permission. The test activates its own instance; run it without concurrent desktop automation. `--helper` selects a previous compatible binary for a negative control; `--artifacts` selects the parent of a new private evidence directory. No Blender runtime is downloaded.
|
||||
For a Blender keyboard check, run `python3 packages/experimental/cua/native/tests/blender_keyboard_e2e.py --blender /absolute/path/to/Blender.app/Contents/MacOS/Blender`. It launches one owned factory-startup GUI with private user directories, observes actual Text Editor text, selection and received modifier flags, and checks Command+A replacement, Shift+Left selection and ordinary input after modifier release. It requires an unlocked macOS session and Accessibility permission. The test activates its own instance; run it without concurrent desktop automation. `--helper` selects a previous compatible binary for a negative control; `--artifacts` selects the parent of a new private evidence directory. No Blender runtime is downloaded.
|
||||
|
||||
For text pacing comparisons, use `native/tests/text_input_benchmark.py` with `--reference-helper`, `--candidate-helper`, `--blender` and `--artifacts`. It types identical synthetic 200/2000-scalar strings into fresh Blender Text Editors and requires full text equality, including Chinese, emoji and newlines. Repeated `--reference-arg` / `--candidate-arg` options select helper startup flags. Raw request and verified-effect timings include observer publication latency and exclude startup; these manual measurements have no CI speed threshold.
|
||||
|
||||
For an independently isolated Electron/Chromium check, run `python3 packages/computer-use/cua/native/tests/electron_e2e.py --electron /absolute/path/to/Electron.app/Contents/MacOS/Electron`. Omitting `--electron` uses an already installed desktop workspace runtime. The check never downloads Electron. Its renderer runs with Node integration disabled, context isolation and Chromium sandboxing enabled, private userData/sessionData/log directories, and a local fixture page. Renderer click/input events write independent observations through the main process; all test actions still use the native helper. AX polling waits for actual controls after Chromium accessibility activation. Native and Electron subprocess groups are owned and stopped before temporary files are removed. Optional artifacts include a fixture-only screenshot and private request transcript.
|
||||
For an independently isolated Electron/Chromium check, run `python3 packages/experimental/cua/native/tests/electron_e2e.py --electron /absolute/path/to/Electron.app/Contents/MacOS/Electron`. Omitting `--electron` uses an already installed desktop workspace runtime. The check never downloads Electron. Its renderer runs with Node integration disabled, context isolation and Chromium sandboxing enabled, private userData/sessionData/log directories, and a local fixture page. Renderer click/input events write independent observations through the main process; all test actions still use the native helper. AX polling waits for actual controls after Chromium accessibility activation. Native and Electron subprocess groups are owned and stopped before temporary files are removed. Optional artifacts include a fixture-only screenshot and private request transcript.
|
||||
|
||||
System file-panel keyboard routing resolves the remote recipient through the selected app’s focused AX sheet. Only Apple’s Open and Save Panel Service is accepted, and its focused ancestry must still contain the identified remote window. Element pointer fallbacks resolve an app-owned window or sheet through AX parents when AXWindow is unavailable. AX actions retain priority over pointer input; visual click markers need no system-cursor movement.
|
||||
|
||||
@@ -66,10 +66,10 @@ Request modifier arrays accept command/control/option/shift/fn. The object API n
|
||||
|
||||
Application selection verifies the executable path and code-signing requirement before returning an authorization identity to the Host. Unsigned running applications receive a process-lifetime identity; stopped unsigned applications use an executable digest and cannot receive persistent consent. The Host obtains human consent before launch, observation or input.
|
||||
|
||||
Protocol declarations are generated from src/protocol.ts. Run `pnpm exec tsx packages/computer-use/cua/native/generate-protocol.ts` from the repository root; package tests verify generated JSON and Swift field/enum tables. Model-submittable parameters exclude Host resource bounds and the private launch continuation.
|
||||
Protocol declarations are generated from src/protocol.ts. Run `pnpm exec tsx packages/experimental/cua/native/generate-protocol.ts` from the repository root; package tests verify generated JSON and Swift field/enum tables. Model-submittable parameters exclude Host resource bounds and the private launch continuation.
|
||||
|
||||
`selectText` requires a fresh element snapshot and writable AXSelectedTextRange. It matches one literal occurrence, narrowed by immediately adjacent prefix/suffix, and selects text or places the cursor before/after it. Matching and reported ranges use UTF-16 offsets. Ambiguous or missing text fails; the helper rechecks the value and selected range without rewriting content. AX text includes available writable, disabled, selected and selection-range information; AXAccess owns indices and native references while AXTextFormatter renders the observed values.
|
||||
|
||||
`paste` publishes one plain-text or HTML clipboard format, dispatches the application's enabled standard Paste menu command, and waits for a clipboard data request. It then restores the preserved formats only while its ownership marker and change count still match. The Host serializes its pastes across helpers with a file lock. maxClipboardBytes bounds both input and preserved contents; unsupported menus and oversized data fail before dispatch. A later copy is preserved. Cancellation restores owned clipboard data when possible; forced termination cannot guarantee restoration, and a data request does not prove the target completed editing.
|
||||
|
||||
Run `python3 packages/computer-use/cua/native/tests/capabilities_e2e.py --helper /absolute/helper --artifacts /private/evidence` to verify a stopped application's discovery and launch, Unicode selection, multiline plain/HTML paste, middle/modifier gestures and keypad identities through an isolated AppKit receiver. It requires existing Accessibility/Screen Recording grants and records the receiver's actual state separately from dispatch results.
|
||||
Run `python3 packages/experimental/cua/native/tests/capabilities_e2e.py --helper /absolute/helper --artifacts /private/evidence` to verify a stopped application's discovery and launch, Unicode selection, multiline plain/HTML paste, middle/modifier gestures and keypad identities through an isolated AppKit receiver. It requires existing Accessibility/Screen Recording grants and records the receiver's actual state separately from dispatch results.
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -8,7 +8,7 @@
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/deepseek-ai/deepseek-harness.git",
|
||||
"directory": "packages/computer-use/cua"
|
||||
"directory": "packages/experimental/cua"
|
||||
},
|
||||
"type": "module",
|
||||
"main": "lib/index.js",
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
+2
-2
@@ -17,7 +17,7 @@ export type NativeArchitecture = 'arm64' | 'x64'
|
||||
* @throws When the current host has no native implementation.
|
||||
*/
|
||||
export function nativeHostArchitecture(platform: string, architecture: string): NativeArchitecture {
|
||||
if (platform !== 'darwin' || (architecture !== 'arm64' && architecture !== 'x64')) throw new Error('Computer Use currently requires macOS on Apple silicon or Intel')
|
||||
if (platform !== 'darwin' || (architecture !== 'arm64' && architecture !== 'x64')) throw new Error('Computer Use currently requires macOS on Apple silicon or Intel. Windows support is in development')
|
||||
return architecture
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ export function nativeExecutablePath(packageRoot: string, architecture: NativeAr
|
||||
*/
|
||||
export async function isSourcePackage(packageRoot: string): Promise<boolean> {
|
||||
const root = resolve(packageRoot, '../../..')
|
||||
if (join(root, 'packages/computer-use/cua') !== packageRoot) return false
|
||||
if (join(root, 'packages/experimental/cua') !== packageRoot) return false
|
||||
try {
|
||||
const manifest: unknown = JSON.parse(await readFile(join(root, 'package.json'), 'utf8'))
|
||||
return typeof manifest === 'object' && manifest !== null && 'name' in manifest && manifest.name === '@deepseek-ai/dsh-root'
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Loaded 100 of 117 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user