fix(cua): resolve the bridge outside projects and retain setup diagnostics
This commit is contained in:
1 parent
74d1f1e55d
commit
19b2cfa0fc
31 files changed
+195
-24
No files matched your search
@@ -7,7 +7,7 @@
|
||||
- list:
|
||||
- listitem:
|
||||
- button "查看 Computer Use": Computer Use
|
||||
- text: 实验性 仅支持 macOS。启用后当前配置档使用 persistent PTC,保留变量且无文件沙箱。开启和后续项目构建会准备本机组件;关闭清空变量。
|
||||
- text: 实验性 仅支持 macOS。请在 ptc 模式下使用。启用后当前配置档使用 persistent PTC,保留变量且无文件沙箱。开启和后续项目构建会准备本机组件;关闭清空变量。
|
||||
- switch "启用 Computer Use"
|
||||
- listitem:
|
||||
- button "查看 智能体团队": 智能体团队
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
- list:
|
||||
- listitem:
|
||||
- button "查看 Computer Use": Computer Use
|
||||
- text: 实验性 仅支持 macOS。启用后当前配置档使用 persistent PTC,保留变量且无文件沙箱。开启和后续项目构建会准备本机组件;关闭清空变量。
|
||||
- text: 实验性 仅支持 macOS。请在 ptc 模式下使用。启用后当前配置档使用 persistent PTC,保留变量且无文件沙箱。开启和后续项目构建会准备本机组件;关闭清空变量。
|
||||
- switch "启用 Computer Use"
|
||||
- listitem:
|
||||
- button "查看 智能体团队": 智能体团队
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
- list:
|
||||
- listitem:
|
||||
- button "查看 Computer Use": Computer Use
|
||||
- text: 实验性 仅支持 macOS。启用后当前配置档使用 persistent PTC,保留变量且无文件沙箱。开启和后续项目构建会准备本机组件;关闭清空变量。
|
||||
- text: 实验性 仅支持 macOS。请在 ptc 模式下使用。启用后当前配置档使用 persistent PTC,保留变量且无文件沙箱。开启和后续项目构建会准备本机组件;关闭清空变量。
|
||||
- switch "启用 Computer Use"
|
||||
- listitem:
|
||||
- button "查看 智能体团队": 智能体团队
|
||||
|
||||
@@ -30,8 +30,8 @@
|
||||
en: c32019330fd9586d
|
||||
zh: dd22110abd10159c
|
||||
/deepseek-ai-dsh-client-ui-plugin-manager/use-this-package/detail-page-extension-points:
|
||||
en: 87f6d96467dc2383
|
||||
zh: e0fb91aa4c4d5a12
|
||||
en: 391f863e6d9b0df0
|
||||
zh: 336de7da8b2cfe44
|
||||
/deepseek-ai-dsh-client-ui-plugin-manager/understand-the-implementation:
|
||||
en: 8ca48e1365f18adf
|
||||
zh: 91eadd683d2d8b5b
|
||||
|
||||
@@ -79,6 +79,8 @@ ctx.slots.inject('plugins.detail.section', () => ctx.slots.register({
|
||||
|
||||
A row's page exists only while a `plugins.row.config` entry names the row, so a contribution meant for a row renders on the page that configuration opens.
|
||||
|
||||
Component lists and detail pages show current startup diagnostics from the Host inventory. The diagnostic remains visible after reopening the page and disappears when the component recovers.
|
||||
|
||||
-----
|
||||
|
||||
<a id="understand-the-implementation"></a>
|
||||
|
||||
@@ -79,6 +79,8 @@ ctx.slots.inject('plugins.detail.section', () => ctx.slots.register({
|
||||
|
||||
行页只在某个 `plugins.row.config` 条目点名这一行时存在,因此给行的贡献渲染在该配置打开的页面上。
|
||||
|
||||
组件列表和详情页显示 Host 清单中的当前启动诊断。重新打开页面后仍能看到原因,组件恢复后诊断消失。
|
||||
|
||||
-----
|
||||
|
||||
<a id="understand-the-implementation"></a>
|
||||
|
||||
@@ -185,6 +185,11 @@ function MetadataError({ error, t }: { readonly error: string | undefined; reado
|
||||
return error === undefined ? null : <p className={css.reason} role="status" data-package-meta-error>{t('metadataError', { error })}</p>
|
||||
}
|
||||
|
||||
/** A failed component keeps its current Host diagnostic beside its controls. */
|
||||
function RuntimeError({ error, t }: { readonly error: string | undefined; readonly t: Translate }): ReactNode {
|
||||
return error === undefined ? null : <p className={css.reason} role="status" data-plugin-runtime-error>{t('reasonLabel')}: {error}</p>
|
||||
}
|
||||
|
||||
/**
|
||||
* A pack's rows as a list in the order the pack declares them: a state dot,
|
||||
* the row id, one line saying its state, a configure control for a row that
|
||||
@@ -261,6 +266,7 @@ function RowsSection({ rows, t, resolveText, toggle, configure }: {
|
||||
/>}
|
||||
</div>
|
||||
<MetadataError error={row.meta?.error} t={t} />
|
||||
<RuntimeError error={row.diagnostic} t={t} />
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
@@ -491,6 +497,7 @@ function RowDetail({ pkg, row, t, resolveText, onBack, renderSlot, form }: {
|
||||
<p className={css.detailDesc}>{description ?? renderSlot('plugins.row.config', { view: 'summary' }, { entryKey: key })}</p>
|
||||
</div>
|
||||
<MetadataError error={row.meta?.error} t={t} />
|
||||
<RuntimeError error={row.diagnostic} t={t} />
|
||||
<div className={css.detailSections} data-plugin-config>
|
||||
{renderSlot('plugins.row.config', { view: 'page', form }, { entryKey: key })}
|
||||
{renderSlot('plugins.detail.section', { subject })}
|
||||
|
||||
@@ -70,6 +70,8 @@ export interface PackageRow {
|
||||
readonly enabled: boolean
|
||||
/** The entry's fiber phase, null without a live fiber. */
|
||||
readonly phase: PluginInfo['fiberPhase']
|
||||
/** Current startup failure projected by the Host inventory. */
|
||||
readonly diagnostic?: string
|
||||
/** Why the Host refuses to switch the row, when it does. */
|
||||
readonly readOnlyReason?: ReadOnlyReason
|
||||
}
|
||||
@@ -403,6 +405,7 @@ export function packageView(bundle: BundleInfo, plugins: readonly PluginInfo[]):
|
||||
moduleName: row.moduleName,
|
||||
enabled: live?.enabled ?? false,
|
||||
phase: live?.fiberPhase ?? null,
|
||||
...live?.diagnostic === undefined ? {} : { diagnostic: live.diagnostic },
|
||||
...row.meta === undefined ? {} : { meta: row.meta },
|
||||
...row.entryId === undefined ? {} : { entryId: row.entryId },
|
||||
...live?.readOnlyReason === undefined ? {} : { readOnlyReason: live.readOnlyReason },
|
||||
|
||||
@@ -315,9 +315,11 @@ describe('PluginManagerPage', () => {
|
||||
})
|
||||
|
||||
it('keeps the persisted selection while marking a failed component as a problem', () => {
|
||||
renderTab({ packages: [pkg({ name: 'cua', enabled: true, rows: [row({ phase: 'failed' })] })] })
|
||||
renderTab({ packages: [pkg({ name: 'cua', enabled: true, rows: [row({ phase: 'failed', diagnostic: 'native preparation unavailable' })] })] })
|
||||
expect(document.querySelector('[data-plugin-package="cua"]')?.getAttribute('data-plugin-status')).toBe('problem')
|
||||
expect(screen.getByRole('switch', { name: en.enableToggle.replace('{name}', 'cua') }).getAttribute('aria-checked')).toBe('true')
|
||||
fireEvent.click(screen.getByRole('button', { name: en.openDetail.replace('{name}', 'cua') }))
|
||||
expect(screen.getByText(`${en.reasonLabel}: native preparation unavailable`)).toBeTruthy()
|
||||
})
|
||||
|
||||
it('marks a normally named Computer Use bundle experimental and explains persistent PTC before enabling', () => {
|
||||
|
||||
@@ -109,6 +109,13 @@ it('hands a custom page the shared configuration form of its entry', () => {
|
||||
})
|
||||
|
||||
describe('packageView', () => {
|
||||
it('carries the current inventory failure into a component row', () => {
|
||||
const plugins = PLUGINS.map(plugin => plugin.entryId === ROW_ENTRY
|
||||
? { ...plugin, fiberPhase: 'failed' as const, diagnostic: 'native preparation unavailable' } : plugin)
|
||||
expect(packageView(BUNDLE, plugins).rows[0]?.diagnostic).toBe('native preparation unavailable')
|
||||
expect(packageView(BUNDLE, PLUGINS).rows[0]?.diagnostic).toBeUndefined()
|
||||
})
|
||||
|
||||
it('joins a bundle with the entries its rows run as', () => {
|
||||
expect(packageView(BUNDLE, PLUGINS)).toEqual({
|
||||
name: 'dsh-better-sidebar', version: '0.16.0', description: 'A sidebar.',
|
||||
|
||||
@@ -15,8 +15,8 @@
|
||||
en: b563e4ec4727d0de
|
||||
zh: d0228da978238e05
|
||||
/deepseek-ai-dsh-cua-bundle/use-this-package:
|
||||
en: 3efb9862b6071b01
|
||||
zh: dbbbdc33477d27ec
|
||||
en: 9179de60df18fd0c
|
||||
zh: 3d14d7331af0bb0f
|
||||
/deepseek-ai-dsh-cua-bundle/understand-the-implementation:
|
||||
en: 86e0dd2fb9d6c485
|
||||
zh: 221055b0c0659290
|
||||
|
||||
@@ -35,6 +35,8 @@ The switch reuses `dsh.profile.bundles` in the profile's `package.json`. Enablin
|
||||
|
||||
Disabling unloads CUA and its persistent runtime, cancels and awaits owned operations, clears retained variables, and restores the preceding tool/runtime configuration. After a project build updates the executable, re-enable or restart to replace running helpers.
|
||||
|
||||
If preparation fails, the selected bundle and its persistent PTC configuration remain in effect, while CUA operations stay unavailable. The failed component shows its current diagnostic after refreshing or reopening the plugin page. Disable Computer Use to restore the previous runtime, or correct the reported cause and re-enable it.
|
||||
|
||||
-----
|
||||
|
||||
<a id="understand-the-implementation"></a>
|
||||
|
||||
@@ -35,6 +35,8 @@ pnpm dsh web
|
||||
|
||||
关闭会卸载 CUA 和持久执行器,取消并等待自有操作结束;保留的代码变量随之清除,原有工具与执行器配置重新生效。项目构建更新可执行文件后,重新启用或重启,运行中的 helper 才会换用新版。
|
||||
|
||||
准备失败时,组合包选择和 persistent PTC 配置仍生效,CUA 操作暂不可用。刷新或重新打开插件页后,失败组件仍显示当前诊断。关闭 Computer Use 可恢复原有运行时;修复报告的问题后可重新启用。
|
||||
|
||||
-----
|
||||
|
||||
<a id="understand-the-implementation"></a>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"meta": {
|
||||
"title": "Computer Use",
|
||||
"description": "macOS only. Enables persistent PTC for this profile (retained variables, no file sandbox). Activation and project builds prepare local components; disabling clears variables."
|
||||
"description": "macOS only. Use in PTC mode. Enables persistent PTC for this profile (retained variables, no file sandbox). Activation and project builds prepare local components; disabling clears variables."
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"meta": {
|
||||
"title": "Computer Use",
|
||||
"description": "仅支持 macOS。启用后当前配置档使用 persistent PTC,保留变量且无文件沙箱。开启和后续项目构建会准备本机组件;关闭清空变量。"
|
||||
"description": "仅支持 macOS。请在 ptc 模式下使用。启用后当前配置档使用 persistent PTC,保留变量且无文件沙箱。开启和后续项目构建会准备本机组件;关闭清空变量。"
|
||||
}
|
||||
}
|
||||
@@ -18,8 +18,8 @@
|
||||
en: 9843d79e506382f1
|
||||
zh: 091cc642cc9a2f03
|
||||
/deepseek-ai-dsh-cua/use-this-package/connect-an-external-execution-environment:
|
||||
en: 59b551986b1da1df
|
||||
zh: 0dd69fc4633492df
|
||||
en: e2d25d61e40dc88a
|
||||
zh: 1b25e636684e14ca
|
||||
/deepseek-ai-dsh-cua/understand-the-implementation:
|
||||
en: ee9fd0ddf93d496d
|
||||
zh: 38d8d5f8b58e6f50
|
||||
|
||||
@@ -59,7 +59,7 @@ Native preparation serializes compiler checks and executable publication with th
|
||||
|
||||
### Connect an external execution environment
|
||||
|
||||
To run through persistent PTC, add `--patch packages/computer-use/cua/examples/persistent-ptc.cordis.yml` after the macOS overlay. This provider keeps variables for the live Agent and runs without a file sandbox. The caller imports the bridge and supplies the current authorized `tools.cua` binding. This integration snippet assumes an execution host has supplied `tools`; the package does not install a global object or keep variables alive:
|
||||
To run through persistent PTC, add `--patch packages/computer-use/cua/examples/persistent-ptc.cordis.yml` after the macOS overlay. This provider keeps variables for the live Agent and runs without a file sandbox. The caller imports the bridge and supplies the current authorized `tools.cua` binding. Persistent PTC receives an installation-owned bridge file URL in its CUA instructions, so the project does not need to install this package. The example below is for an external host that resolves the package from its own installation and supplies `tools`; the package does not install a global object or keep variables alive:
|
||||
|
||||
```javascript
|
||||
import { connectCua } from '@deepseek-ai/dsh-cua/bridge'
|
||||
@@ -151,7 +151,7 @@ Raw operation tools use { method, params }, including params: {} for parameterle
|
||||
|
||||
Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording permission.
|
||||
|
||||
Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste inserts plain text using native Unicode input without changing the clipboard; rich-text formats are unavailable. Browser applications can be controlled with getApp like other native apps; browser tabs and app launching are unavailable. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, initialize once with var { connectCua } = await import("@deepseek-ai/dsh-cua/bridge"); var cua = connectCua(() => tools.cua, text => console.log(text)). Retain app objects between calls. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.
|
||||
Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste inserts plain text using native Unicode input without changing the clipboard; rich-text formats are unavailable. Browser applications can be controlled with getApp like other native apps; browser tabs and app launching are unavailable. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.
|
||||
```
|
||||
|
||||
#### Token effect
|
||||
|
||||
@@ -59,7 +59,7 @@ Agent 销毁和插件卸载会取消并等待完整工具调用,包括图像
|
||||
|
||||
### 接入外部执行环境
|
||||
|
||||
使用 persistent PTC 时,在 macOS 配置后追加 `--patch packages/computer-use/cua/examples/persistent-ptc.cordis.yml`。该 provider 为存活的 Agent 保留变量,不启用文件沙箱。调用方导入适配模块,并提供当前执行中已授权的 `tools.cua` 绑定。下面的接入代码假设执行宿主已提供 `tools`;本包不安装全局对象,也不负责保留变量:
|
||||
使用 persistent PTC 时,在 macOS 配置后追加 `--patch packages/computer-use/cua/examples/persistent-ptc.cordis.yml`。该 provider 为存活的 Agent 保留变量,不启用文件沙箱。调用方导入适配模块,并提供当前执行中已授权的 `tools.cua` 绑定。persistent PTC 会在 CUA 指引中收到安装目录内 bridge 的文件 URL,项目无须安装本包。下面的例子用于能从自身安装目录解析本包、且已提供 `tools` 的外部宿主;本包不安装全局对象,也不负责保留变量:
|
||||
|
||||
```javascript
|
||||
import { connectCua } from '@deepseek-ai/dsh-cua/bridge'
|
||||
@@ -151,7 +151,7 @@ Raw operation tools use { method, params }, including params: {} for parameterle
|
||||
|
||||
Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording permission.
|
||||
|
||||
Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste inserts plain text using native Unicode input without changing the clipboard; rich-text formats are unavailable. Browser applications can be controlled with getApp like other native apps; browser tabs and app launching are unavailable. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, initialize once with var { connectCua } = await import("@deepseek-ai/dsh-cua/bridge"); var cua = connectCua(() => tools.cua, text => console.log(text)). Retain app objects between calls. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.
|
||||
Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste inserts plain text using native Unicode input without changing the clipboard; rich-text formats are unavailable. Browser applications can be controlled with getApp like other native apps; browser tabs and app launching are unavailable. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.
|
||||
```
|
||||
|
||||
#### Token 影响
|
||||
|
||||
@@ -83,7 +83,8 @@
|
||||
"@deepseek-ai/dsh-ptc-runtime-node": "workspace:*",
|
||||
"@deepseek-ai/dsh-sandbox-local": "workspace:*",
|
||||
"@deepseek-ai/dsh-sandbox-policy": "workspace:*",
|
||||
"sharp": "^0.35.3"
|
||||
"sharp": "^0.35.3",
|
||||
"@deepseek-ai/dsh-ptc-runtime": "workspace:*"
|
||||
},
|
||||
"scripts": {
|
||||
"build:native": "swift build --package-path native --configuration release"
|
||||
|
||||
@@ -14,6 +14,7 @@ import type {} from '@deepseek-ai/dsh-system-prompt'
|
||||
import type { ToolExecution } from '@deepseek-ai/dsh-tools'
|
||||
import type {} from '@deepseek-ai/dsh-subprocess'
|
||||
import type {} from '@deepseek-ai/dsh-computer-use'
|
||||
import type {} from '@deepseek-ai/dsh-ptc-runtime'
|
||||
|
||||
/** Loader plugin identity. */
|
||||
export const name = 'cua'
|
||||
@@ -95,7 +96,7 @@ Raw operation tools use { method, params }, including params: {} for parameterle
|
||||
|
||||
Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording permission.
|
||||
|
||||
Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste inserts plain text using native Unicode input without changing the clipboard; rich-text formats are unavailable. Browser applications can be controlled with getApp like other native apps; browser tabs and app launching are unavailable. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, initialize once with var { connectCua } = await import("@deepseek-ai/dsh-cua/bridge"); var cua = connectCua(() => tools.cua, text => console.log(text)). Retain app objects between calls. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.`
|
||||
Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste inserts plain text using native Unicode input without changing the clipboard; rich-text formats are unavailable. Browser applications can be controlled with getApp like other native apps; browser tabs and app launching are unavailable. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.`
|
||||
|
||||
function executionAgent(execution: Readonly<ToolExecution>): Agent {
|
||||
if (!execution.agent) throw new Error('dsh-cua requires an agent-owned execution')
|
||||
@@ -241,6 +242,13 @@ function mount(ctx: Context, config: Config, helperPath: string): void {
|
||||
return scoped === undefined ? undefined : definition.projectContent?.(scoped, result)
|
||||
},
|
||||
})
|
||||
yield ctx.systemPrompt.section({ name: 'computer-use:dsh-cua', order: ctx.systemPrompt.getSectionOrder('TOOL_COMPUTER_USE'), text: GUIDANCE })
|
||||
yield ctx.systemPrompt.section({
|
||||
name: 'computer-use:dsh-cua', order: ctx.systemPrompt.getSectionOrder('TOOL_COMPUTER_USE'),
|
||||
text: () => ctx.get('ptcRuntime')?.programFormat !== 'repl' ? GUIDANCE : `${GUIDANCE}
|
||||
|
||||
Persistent PTC initialization:
|
||||
var { connectCua } = await import(${JSON.stringify(new URL('../lib/bridge.js', import.meta.url).href)});
|
||||
var cua = connectCua(() => tools.cua, text => console.log(text));`,
|
||||
})
|
||||
}, 'dsh-cua.runtime')
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
/** Built bridge imports and retained objects from a project outside the Harness installation. */
|
||||
import { existsSync } from 'node:fs'
|
||||
import { chmod, copyFile, mkdtemp, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { Context } from '@deepseek-ai/cordis'
|
||||
import { mountAgentLoopTestDependencies, mountAgentLoopTestHarness } from '@deepseek-ai/dsh-agent-loop-testkit'
|
||||
import LocalAttachmentStore from '@deepseek-ai/dsh-attachment-local'
|
||||
import ComputerUseRegistry from '@deepseek-ai/dsh-computer-use'
|
||||
import FileSystem from '@deepseek-ai/dsh-fs-local'
|
||||
import { LlmAdapter, ToolCallId, type LlmResolvedModelInfo, type StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||
import PersistentRuntime from '@deepseek-ai/dsh-ptc-runtime-node/persistent'
|
||||
import { SessionId } from '@deepseek-ai/dsh-session'
|
||||
import Subprocess from '@deepseek-ai/dsh-subprocess-local'
|
||||
import { expect, it } from 'vitest'
|
||||
import * as Cua from '../src/index.ts'
|
||||
import { Request } from '../src/protocol.ts'
|
||||
|
||||
const bridge = fileURLToPath(new URL('../lib/bridge.js', import.meta.url))
|
||||
const bootstrap = join(dirname(fileURLToPath(import.meta.resolve('@deepseek-ai/dsh-ptc-runtime-node/package.json'))), 'lib/persistent-process.js')
|
||||
|
||||
class VisualAdapter extends LlmAdapter {
|
||||
override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
|
||||
return Promise.resolve({ provider, id: model, name: model, inputModalities: ['text', 'image'] })
|
||||
}
|
||||
stream(): AsyncIterable<StreamChunk> { throw new Error('This bridge integration does not request model generation') }
|
||||
}
|
||||
|
||||
// This lane runs after build; the fixture helper uses a POSIX shebang and never accesses the desktop.
|
||||
it.skipIf(process.platform === 'win32' || !existsSync(bridge) || !existsSync(bootstrap))(
|
||||
'imports the installation-owned bridge outside the repository and retains objects with current bindings', async () => {
|
||||
const cwd = await mkdtemp(join(tmpdir(), 'dsh-cua-external-project-'))
|
||||
const ctx = new Context()
|
||||
try {
|
||||
const helperPath = join(cwd, 'native-helper.mjs')
|
||||
await copyFile(new URL('../../../../snapshots/session/cua-native/workspace/native-helper.mjs', import.meta.url), helperPath)
|
||||
await chmod(helperPath, 0o700)
|
||||
await mountAgentLoopTestDependencies(ctx)
|
||||
await ctx.plugin(FileSystem)
|
||||
await ctx.plugin(Subprocess)
|
||||
await ctx.plugin(LocalAttachmentStore, { dshHome: join(cwd, 'home') })
|
||||
await ctx.plugin(ComputerUseRegistry)
|
||||
await ctx.plugin(PersistentRuntime, { bootstrapPath: bootstrap })
|
||||
await ctx.plugin(Cua, { helperPath })
|
||||
await mountAgentLoopTestHarness(ctx)
|
||||
ctx.llm.registerAdapter(['bridge-fixture'], new VisualAdapter())
|
||||
const handle = await ctx.agents.create({ sessionId: SessionId('external-cua-project'), agentOptions: { provider: 'bridge-fixture', model: 'vision' } })
|
||||
let count = 0
|
||||
let images = 0
|
||||
const calls: Array<{ phase: string; method: string }> = []
|
||||
const run = (program: string, phase: string) => ctx.ptcRuntime.run(ctx.ptcRuntime.resolve({
|
||||
cwd, program, stateOwner: handle.agent,
|
||||
bindings: [{ global: 'tools', functions: { cua: async (args) => {
|
||||
const operation = Request.parse(args)
|
||||
calls.push({ phase, method: operation.method })
|
||||
const result = await ctx.tools.execute({ agent: handle.agent, signal: new AbortController().signal,
|
||||
callId: ToolCallId(`external-cua-${++count}`), name: 'cua', arguments: operation })
|
||||
images += result.content.filter(block => block.type === 'image').length
|
||||
return result.value
|
||||
} } }],
|
||||
}))
|
||||
expect((await run('await import("@deepseek-ai/dsh-cua/bridge")', 'unresolved')).error?.message).toContain('Cannot find package')
|
||||
const section = (await ctx.systemPrompt.assemble()).sections.find(value => value.name === 'computer-use:dsh-cua')
|
||||
const initialization = section?.text.split('Persistent PTC initialization:\n')[1]
|
||||
expect(initialization).toContain('/lib/bridge.js')
|
||||
const first = await run(`${initialization}\nvar app = await cua.getApp("test.fixture"); var win = await app.getWindow(7); await win.click(0); void await win.getScreenshot();`, 'first')
|
||||
expect(first.error).toBeUndefined()
|
||||
expect(images).toBe(1)
|
||||
expect(JSON.stringify(first)).not.toContain('iVBORw0KGgo')
|
||||
const second = await run('await win.getAXState({emit:false}); await win.click(0); await win.getAXState({emit:false});', 'second')
|
||||
expect(second.error).toBeUndefined()
|
||||
expect(second.value).toContain('Count: 2')
|
||||
expect(calls.filter(call => call.phase === 'second').map(call => call.method)).toEqual(['getState', 'click', 'getState'])
|
||||
} finally {
|
||||
try { await ctx.fiber.dispose() } finally { await rm(cwd, { recursive: true, force: true }) }
|
||||
}
|
||||
},
|
||||
)
|
||||
@@ -4,9 +4,12 @@ import { tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { setImmediate } from 'node:timers/promises'
|
||||
import { Context, FiberState } from '@deepseek-ai/cordis'
|
||||
import Loader from '@deepseek-ai/cordis-plugin-loader'
|
||||
import { mountAgentLoopTestDependencies, mountAgentLoopTestHarness } from '@deepseek-ai/dsh-agent-loop-testkit'
|
||||
import ComputerUseRegistry from '@deepseek-ai/dsh-computer-use'
|
||||
import LocalAttachmentStore from '@deepseek-ai/dsh-attachment-local'
|
||||
import FileSystem from '@deepseek-ai/dsh-fs-local'
|
||||
import PersistentRuntime from '@deepseek-ai/dsh-ptc-runtime-node/persistent'
|
||||
import { LlmAdapter, ToolCallId, type LlmResolvedModelInfo, type StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||
import { SessionId } from '@deepseek-ai/dsh-session'
|
||||
import Subprocess from '@deepseek-ai/dsh-subprocess'
|
||||
@@ -357,6 +360,15 @@ it('prepares the selected native component before registering CUA and creates no
|
||||
expect(signal.aborted).toBe(true)
|
||||
})
|
||||
|
||||
it('adds an installation-owned bridge URL when persistent PTC is available', async () => {
|
||||
const f = await setup()
|
||||
const text = async () => (await f.ctx.systemPrompt.assemble()).sections.find(section => section.name === 'computer-use:dsh-cua')?.text
|
||||
expect(await text()).not.toContain('Persistent PTC initialization:')
|
||||
await f.ctx.plugin(FileSystem)
|
||||
await f.ctx.plugin(PersistentRuntime)
|
||||
expect(await text()).toContain(new URL('../lib/bridge.js', import.meta.url).href)
|
||||
})
|
||||
|
||||
it('keeps Cordis loading until preparation succeeds and reports preparation failure through the fiber', async () => {
|
||||
const f = await setup()
|
||||
await f.provider.dispose()
|
||||
@@ -364,7 +376,10 @@ it('keeps Cordis loading until preparation succeeds and reports preparation fail
|
||||
const entered: PromiseWithResolvers<void> = Promise.withResolvers()
|
||||
onTestFinished(() => { ready.resolve(resolve('prepared-helper')) })
|
||||
native.prepare.mockReset().mockImplementation(() => { entered.resolve(); return ready.promise })
|
||||
const provider = f.ctx.plugin(NativeProvider, automaticConfig)
|
||||
await f.ctx.plugin(Loader)
|
||||
f.ctx.loader.builtins.cua = NativeProvider
|
||||
const entry = await f.ctx.loader.create({ name: 'cordis:cua', config: automaticConfig })
|
||||
const provider = entry.fiber!
|
||||
await entered.promise
|
||||
expect(provider.state).toBe(FiberState.LOADING)
|
||||
expect(f.ctx.tools.schemas().some(tool => tool.name === 'cua')).toBe(false)
|
||||
|
||||
@@ -40,6 +40,9 @@
|
||||
},
|
||||
{
|
||||
"path": "../../util/atomic-write"
|
||||
},
|
||||
{
|
||||
"path": "../../ptc-runtime/ptc-runtime"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -24,8 +24,8 @@
|
||||
en: dfc89b14d4c0d21e
|
||||
zh: 062b233951ed157b
|
||||
/deepseek-ai-dsh-host-plugin-inventory/use-this-package/what-you-can-and-cannot-do-with-it:
|
||||
en: 387f87777eef7ec5
|
||||
zh: badcd90c8840f870
|
||||
en: 0243819e8404c74a
|
||||
zh: 05dab926c0ecff34
|
||||
/deepseek-ai-dsh-host-plugin-inventory/understand-the-implementation:
|
||||
en: 42b15ac6321706aa
|
||||
zh: 4ed0861911abaf5d
|
||||
|
||||
@@ -41,6 +41,8 @@ With a roster composed, `agentPresets` carries one group per preset in roster or
|
||||
|
||||
The inventory is a snapshot for display and diagnostics: a client can render the roster, flag failed entries, and detect changes by comparing snapshots. It cannot enable, disable, add, or remove plugins, and it carries no history — a fiber that already failed and was removed is absent. Because the service reads the Loader on every call, the answer always reflects the current composition rather than a cached view.
|
||||
|
||||
Failed entries include the current startup diagnostic read from their Cordis fiber. Recovery removes it; inventory reads do not wait for loading entries or retain a separate error cache.
|
||||
|
||||
-----
|
||||
|
||||
<a id="understand-the-implementation"></a>
|
||||
|
||||
@@ -41,6 +41,8 @@ Loader 行与预设行可以携带可选的 `meta`,其中包含标题、描述
|
||||
|
||||
该清单是供展示与诊断的快照:客户端可以渲染名单、标出失败条目,并通过比较快照检测变化。它不能启用、停用、添加或移除插件,也不携带历史——已经失败并被移除的 fiber 缺席。由于服务每次调用都读取 Loader,答案总是反映当前组合,而不是缓存视图。
|
||||
|
||||
失败条目包含从 Cordis fiber 读取的当前启动诊断。恢复后移除该诊断;清单查询不等待仍在加载的条目,也不另存错误缓存。
|
||||
|
||||
-----
|
||||
|
||||
<a id="understand-the-implementation"></a>
|
||||
|
||||
@@ -86,11 +86,18 @@ export async function readPluginInventory(ctx: Context): Promise<PluginInventory
|
||||
if (entry.options.group) continue
|
||||
const base = entry.parent.tree.ctx.baseUrl
|
||||
const meta = base === undefined ? undefined : packages?.metaOf(entry.options.name, base)
|
||||
const fiber = entry.fiber
|
||||
const fiberPhase = fiber === undefined ? null : FIBER_PHASE[fiber.state]
|
||||
let diagnostic: string | undefined
|
||||
if (fiber !== undefined && fiberPhase === 'failed') {
|
||||
try { await fiber.await() } catch (error) { diagnostic = error instanceof Error ? error.message : String(error) }
|
||||
}
|
||||
entries.push({
|
||||
entryId: pluginEntryId(entry.id),
|
||||
moduleName: entry.options.name,
|
||||
enabled: !entry.disabled,
|
||||
fiberPhase: entry.fiber === undefined ? null : FIBER_PHASE[entry.fiber.state],
|
||||
fiberPhase,
|
||||
...diagnostic === undefined ? {} : { diagnostic },
|
||||
...meta === undefined ? {} : { meta },
|
||||
})
|
||||
}
|
||||
|
||||
@@ -23,6 +23,8 @@ export interface PluginInventoryEntry {
|
||||
/** Effective Loader enablement, including disabled ancestor groups. */
|
||||
readonly enabled: boolean
|
||||
readonly fiberPhase: PluginFiberPhase
|
||||
/** Current startup failure message; absent when this entry has no failed fiber. */
|
||||
readonly diagnostic?: string
|
||||
}
|
||||
|
||||
/** Effective enablement of one preset composition row. */
|
||||
|
||||
@@ -149,6 +149,28 @@ describe('PluginInventoryGateway', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('reads startup diagnostics from failed fibers and clears them after recovery', async () => {
|
||||
const { ctx, inventory } = await harness()
|
||||
ctx.loader.builtins.failing = {
|
||||
async apply(_ctx: Context, config: { fail: boolean }) {
|
||||
if (config.fail) throw new Error('native preparation unavailable')
|
||||
},
|
||||
}
|
||||
const entry = await ctx.loader.create({ name: 'cordis:failing', config: { fail: true } })
|
||||
await expect(entry.fiber!.await()).rejects.toThrow('native preparation unavailable')
|
||||
for (let index = 0; index < 2; index++) {
|
||||
expect((await inventory.list()).entries).toContainEqual({
|
||||
entryId: entry.id, moduleName: 'cordis:failing', enabled: true,
|
||||
fiberPhase: 'failed', diagnostic: 'native preparation unavailable',
|
||||
})
|
||||
}
|
||||
await entry.update({ config: { fail: false } })
|
||||
await entry.fiber!.await()
|
||||
expect((await inventory.list()).entries).toContainEqual({
|
||||
entryId: entry.id, moduleName: 'cordis:failing', enabled: true, fiberPhase: 'active',
|
||||
})
|
||||
})
|
||||
|
||||
it('projects current non-group Loader entries without a second cache', async () => {
|
||||
const { ctx, inventory } = await harness()
|
||||
const activeId = await ctx.loader.create({ name: 'cordis:active' })
|
||||
|
||||
Generated
+3
@@ -6420,6 +6420,9 @@ importers:
|
||||
'@deepseek-ai/dsh-llm-deepseek':
|
||||
specifier: workspace:*
|
||||
version: link:../../llm/llm-deepseek
|
||||
'@deepseek-ai/dsh-ptc-runtime':
|
||||
specifier: workspace:*
|
||||
version: link:../../ptc-runtime/ptc-runtime
|
||||
'@deepseek-ai/dsh-ptc-runtime-node':
|
||||
specifier: workspace:*
|
||||
version: link:../../ptc-runtime/ptc-runtime-node
|
||||
|
||||
@@ -59,4 +59,4 @@ Raw operation tools use { method, params }, including params: {} for parameterle
|
||||
|
||||
Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording permission.
|
||||
|
||||
Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste inserts plain text using native Unicode input without changing the clipboard; rich-text formats are unavailable. Browser applications can be controlled with getApp like other native apps; browser tabs and app launching are unavailable. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, initialize once with var { connectCua } = await import("@deepseek-ai/dsh-cua/bridge"); var cua = connectCua(() => tools.cua, text => console.log(text)). Retain app objects between calls. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.
|
||||
Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste inserts plain text using native Unicode input without changing the clipboard; rich-text formats are unavailable. Browser applications can be controlled with getApp like other native apps; browser tabs and app launching are unavailable. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.
|
||||
Reference in new issue
Block a user