From bde18e994e8d5a606363a6fd36b1da4f4344c9db Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 19 Oct 2026 03:27:12 +0800 Subject: [PATCH 01/55] test(web): align fork snapshot clocks --- apps/web/tests/message-actions.e2e.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/apps/web/tests/message-actions.e2e.ts b/apps/web/tests/message-actions.e2e.ts index d70558dab4..466393228c 100644 --- a/apps/web/tests/message-actions.e2e.ts +++ b/apps/web/tests/message-actions.e2e.ts @@ -15,7 +15,7 @@ import { acknowledgeReloadConnectionLoss, assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts, launchWebScaffold, parseSeedFixture, realizeSeedFixture, renderSeedFixture, seedSession, watchConsole, webSnapshotMode, type WebScaffold, } from './scaffold.ts' -import { openSettings, newEnglishPage, pinBrowserClock, saveFailureShot, WEB_FIXTURE_TIME } from './support.ts' +import { openSettings, newEnglishPage, pinBrowserClock, pinHostClock, saveFailureShot, WEB_FIXTURE_TIME } from './support.ts' const SNAPSHOT_DIR = fileURLToPath(new URL('../../../snapshots/web/message-actions', import.meta.url)) // Borrowed read-only: this scenario needs any settled user+assistant pair, not @@ -191,6 +191,7 @@ describe('web e2e: message IconActions and clocks on settled history', () => { let browser: Browser let page: Page let tripwire: ReturnType + let hostClock: ReturnType | undefined beforeAll(async () => { scaffold = await launchWebScaffold() @@ -211,6 +212,8 @@ describe('web e2e: message IconActions and clocks on settled history', () => { await seedSession(scaffold, raw, SEED_ID, undefined, { createdAt: WEB_FIXTURE_TIME - 90_000 }) browser = await chromium.launch() page = await newEnglishPage(browser) + // Fork summaries and browser placeholders must rank against the same clock. + hostClock = pinHostClock() await pinBrowserClock(page) tripwire = watchConsole(page) await page.goto(scaffold.authenticatedUrl, { waitUntil: 'load' }) @@ -218,8 +221,12 @@ describe('web e2e: message IconActions and clocks on settled history', () => { }, 120_000) afterAll(async () => { - await browser?.close() - await scaffold?.close() + try { + await browser?.close() + await scaffold?.close() + } finally { + hostClock?.mockRestore() + } }) it.skipIf(MODE === 'record')('enables branch only on the completed transcript tail', async () => { From 36a80c41b336a671ee9fe7611010cd365d00e782 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 19 Oct 2026 03:35:05 +0800 Subject: [PATCH 02/55] test(web): explain the adjacent fork clock pins --- apps/web/tests/message-actions.e2e.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/web/tests/message-actions.e2e.ts b/apps/web/tests/message-actions.e2e.ts index 466393228c..2f96305714 100644 --- a/apps/web/tests/message-actions.e2e.ts +++ b/apps/web/tests/message-actions.e2e.ts @@ -212,7 +212,9 @@ describe('web e2e: message IconActions and clocks on settled history', () => { await seedSession(scaffold, raw, SEED_ID, undefined, { createdAt: WEB_FIXTURE_TIME - 90_000 }) browser = await chromium.launch() page = await newEnglishPage(browser) - // Fork summaries and browser placeholders must rank against the same clock. + // Fork summaries use Host time; placeholders use browser time. Each pin + // starts at the fixture date, so keep these calls adjacent: setup between + // them can create enough clock skew to reverse the fork order. hostClock = pinHostClock() await pinBrowserClock(page) tripwire = watchConsole(page) From ce7414da1497943bf3bd5d0ded3c64f8af5c61be Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 19 Oct 2026 03:58:22 +0800 Subject: [PATCH 03/55] test(office): report render routing failure evidence --- .../skill/skill-office/tests/render-tools.e2e.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/skill/skill-office/tests/render-tools.e2e.ts b/packages/skill/skill-office/tests/render-tools.e2e.ts index f2dc0cfbf2..d795b86aec 100644 --- a/packages/skill/skill-office/tests/render-tools.e2e.ts +++ b/packages/skill/skill-office/tests/render-tools.e2e.ts @@ -63,17 +63,22 @@ it.skipIf(!process.env.DEEPSEEK_API_KEY || !runtime).each([{ format: 'docx', mod const pixels = await execa(python!, ['-c', "from pathlib import Path\nfrom PIL import Image\nfiles=list(Path('preview').glob('*.png'))\nassert files\nfor f in files:\n im=Image.open(f).convert('RGBA')\n visible=Image.alpha_composite(Image.new('RGBA',im.size,'white'),im).convert('RGB')\n print('content' if any(a!=b for a,b in visible.getextrema()) else 'blank')"], { cwd }) expect(pixels.exitCode).toBe(0) if (format === 'pdf') expect(pixels.stdout.trim()).toBe('content') + const final = events.filter(e => e.type === 'assistant/message').at(-1)?.data.message.content + .filter(b => b.type === 'text').map(b => b.text).join(' ') + const diagnostic = JSON.stringify({ + format, mode, callOrder: calls.map(call => call.name), skills, commands, pixels: pixels.stdout, final, + }, null, 2) const authoring = calls.findIndex(call => call.name === 'skill' && /office-(docx|pptx|xlsx)/u.test(call.arguments)) if (pixels.stdout.includes('blank')) { if (authoring !== -1) { const rendered = calls.findIndex(call => (call.name === 'bash' || call.name === 'pwsh') && /\srender\b/u.test(call.arguments)) expect(authoring, 'content checks after a blank preview may need the format skill').toBeGreaterThan(rendered) } - const final = events.filter(e => e.type === 'assistant/message').at(-1)?.data.message.content - .filter(b => b.type === 'text').map(b => b.text).join(' ') - expect(final).toMatch(/visual (?:inspection|check)[^.!?]{0,100}(?:not completed|incomplete|could not be completed|unverified)/iu) + expect(final, diagnostic).toMatch( + /visual (?:inspection|check)[^.!?]{0,100}(?:not completed|incomplete|could not be completed|unverified)/iu, + ) } else { - expect(authoring, JSON.stringify(skills)).toBe(-1) + expect(authoring, diagnostic).toBe(-1) } expect(calls.some(e => e.name === 'read_image')).toBe(true) }, From b416e328471b8eff85af05fa9e3c224e9bc7c504 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 19 Oct 2026 04:11:05 +0800 Subject: [PATCH 04/55] test(office): scope render checks to the exported image --- packages/skill/skill-office/tests/render-tools.e2e.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/skill/skill-office/tests/render-tools.e2e.ts b/packages/skill/skill-office/tests/render-tools.e2e.ts index d795b86aec..126290b3c2 100644 --- a/packages/skill/skill-office/tests/render-tools.e2e.ts +++ b/packages/skill/skill-office/tests/render-tools.e2e.ts @@ -37,17 +37,22 @@ async function sessionEvents(cwd: string) { } it.skipIf(!process.env.DEEPSEEK_API_KEY || !runtime).each([{ format: 'docx', mode: 'native' }, { format: 'pdf', mode: 'native' }, { format: 'docx', mode: 'ptc' }])('loads the operation skill and renders $format through the shell in $mode mode', async ({ format, mode }) => { + let original: Buffer | undefined await runLoaderSmoke({ label: 'Office CLI rendering', tempDirPrefix: 'dsh-office-render-', ...launch, env: { ...launch.env, DSH_TOOLS_MODE: mode }, binArgs: ['--profile', 'headless', '--patch', 'office.patch.json', - `Export page 1 of input.${format} as PNG into preview and check that the exported image is readable. Report briefly in English.`], + // Source-content verification can require an authoring skill; this case covers image-only inspection. + `Export page 1 of input.${format} as PNG into preview. Check only that the exported image is readable; ` + + 'do not inspect the source document content or metadata, and keep the source file unchanged. Report briefly in English.'], async prepare(cwd) { await execa(python!, ['-c', "from docx import Document\nd=Document()\nd.add_heading('Quarterly revenue',0)\nd.add_paragraph('Q1: 12; Q2: 18; Total: 30')\nd.save('input.docx')"], { cwd }) if (format === 'pdf') await execa(python!, ['-c', "from PIL import Image, ImageDraw\nim=Image.new('RGB',(300,200),'white')\nImageDraw.Draw(im).rectangle((30,30,150,100),fill='black')\nim.save('input.pdf','PDF')"], { cwd }) await prepareOffice(cwd) + original = await readFile(join(cwd, `input.${format}`)) }, async inspect(cwd) { + expect(await readFile(join(cwd, `input.${format}`))).toEqual(original) const events = await sessionEvents(cwd) const calls = events.flatMap((event) => { if (event.type === 'tool/call') return [event.data] From 1954e948cbe4e2cf87c7e97ffe948da87ad45f6c Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 11:10:31 +0800 Subject: [PATCH 05/55] feat(cua): add optional macOS setup with existing plugin interfaces --- ...capabilities-as-optional-bundles.i18n.yaml | 8 +- ...mental-capabilities-as-optional-bundles.md | 6 +- ...tal-capabilities-as-optional-bundles.zh.md | 6 +- ...26-09-26-cua-application-consent.i18n.yaml | 12 +- .../2026-09-26-cua-application-consent.md | 23 +- .../2026-09-26-cua-application-consent.zh.md | 23 +- apps/cli/package.json | 1 + apps/web/tests/cua-optin.e2e.ts | 98 +++++ apps/web/tests/cua-permissions.e2e.ts | 139 ++++++ .../cua-permissions/missing.expected.md | 26 ++ .../plugin-config/official.expected.md | 4 + .../plugin-manager/live-enabled.expected.md | 4 + .../plugin-manager/manager.expected.md | 4 + .../plugin-manager/refresh.expected.md | 4 +- apps/web/tests/preview-boot.e2e.ts | 14 +- apps/web/tsconfig.json | 3 + docs/capability-seams.i18n.yaml | 4 +- docs/capability-seams.md | 6 + docs/capability-seams.zh.md | 6 + docs/config-catalog.md | 4 +- docs/config-catalog.zh.md | 4 +- docs/event-producer-consumer.md | 4 +- docs/event-producer-consumer.zh.md | 4 +- docs/module-graph.md | 24 +- docs/module-graph.zh.md | 24 +- docs/subsystems/computer-use.i18n.yaml | 4 +- docs/subsystems/computer-use.md | 66 +++ docs/subsystems/computer-use.zh.md | 66 +++ .../cua-preset-scope/guide.i18n.yaml | 16 + .../v0.2.1-alpha.2/cua-preset-scope/guide.md | 18 + .../cua-preset-scope/guide.zh.md | 18 + package.json | 1 + packages/boot/app-boot/src/profile.ts | 1 + packages/computer-use/README.i18n.yaml | 8 +- packages/computer-use/README.md | 5 +- packages/computer-use/README.zh.md | 5 +- .../computer-use/README.i18n.yaml | 8 +- packages/computer-use/computer-use/README.md | 5 +- .../computer-use/computer-use/README.zh.md | 5 +- packages/experimental/README.i18n.yaml | 4 +- packages/experimental/README.md | 2 + packages/experimental/README.zh.md | 2 + .../client-ui-cua/README.i18n.yaml | 37 ++ packages/experimental/client-ui-cua/README.md | 92 ++++ .../experimental/client-ui-cua/README.zh.md | 92 ++++ .../experimental/client-ui-cua/package.json | 72 ++++ .../src/client/PermissionPanel.module.css | 48 +++ .../src/client/PermissionPanel.tsx | 183 ++++++++ .../client-ui-cua/src/client/controller.ts | 136 ++++++ .../client-ui-cua/src/client/index.ts | 15 + .../client-ui-cua/src/client/locales.ts | 64 +++ .../client-ui-cua/src/client/mount.ts | 82 ++++ .../client-ui-cua/src/css-modules.d.ts | 4 + .../experimental/client-ui-cua/src/index.ts | 2 + .../tests/controller.client.spec.ts | 253 +++++++++++ .../client-ui-cua/tests/mount.client.spec.ts | 181 ++++++++ .../client-ui-cua/tests/panel.client.spec.tsx | 332 +++++++++++++++ .../experimental/client-ui-cua/tsconfig.json | 51 +++ .../client-ui-cua/tsdown.config.ts | 3 + .../experimental/cua-bundle/README.i18n.yaml | 46 ++ packages/experimental/cua-bundle/README.md | 94 +++++ packages/experimental/cua-bundle/README.zh.md | 94 +++++ .../experimental/cua-bundle/cordis.patch.yml | 9 + packages/experimental/cua-bundle/icon.svg | 12 + .../experimental/cua-bundle/locale/en.json | 6 + .../experimental/cua-bundle/locale/zh.json | 6 + packages/experimental/cua-bundle/package.json | 54 +++ packages/experimental/cua-bundle/src/index.ts | 2 + .../experimental/cua-bundle/tsconfig.json | 15 + .../experimental/cua-macos/README.i18n.yaml | 12 +- packages/experimental/cua-macos/README.md | 16 +- packages/experimental/cua-macos/README.zh.md | 16 +- .../cua-macos/examples/macos.cordis.yml | 1 + .../cua-macos/native/README.i18n.yaml | 16 +- .../experimental/cua-macos/native/README.md | 8 +- .../cua-macos/native/README.zh.md | 8 +- .../Sources/CuaNative/NativePlatform.swift | 27 +- .../CuaNative/NativeProtocol.generated.swift | 6 + .../Sources/CuaNative/Permissions.swift | 69 +++ .../Sources/CuaNative/RequestAdmission.swift | 7 + .../CuaNative/RequestCoordinator.swift | 27 +- .../CuaNative/SharedHelperServer.swift | 97 +++-- .../cua-macos/native/protocol.schema.json | 125 ++++++ .../native/tests/LauncherFixture.swift | 29 ++ .../native/tests/launcher_identity_e2e.py | 76 ++++ .../native/tests/shared_helper_e2e.py | 57 ++- .../tests/swift/PermissionIdentityTests.swift | 36 ++ .../native/tests/swift/ProtocolTests.swift | 6 + .../experimental/cua-macos/src/guidance.ts | 4 +- .../cua-macos/tests/large-image-ptc.spec.ts | 2 +- .../tests/loader-composition.spec.ts | 2 +- .../tests/persistent-bridge.built.e2e.ts | 2 +- .../tests/protocol-generation.spec.ts | 9 + .../tests/provider-lifecycle.spec.ts | 118 ++---- packages/experimental/cua/README.i18n.yaml | 12 +- packages/experimental/cua/README.md | 6 +- packages/experimental/cua/README.zh.md | 6 +- packages/experimental/cua/package.json | 32 +- packages/experimental/cua/src/access-types.ts | 30 ++ packages/experimental/cua/src/access.ts | 142 +++++++ .../cua/src/application-access.ts | 124 ++++-- packages/experimental/cua/src/index.ts | 66 ++- packages/experimental/cua/src/protocol.ts | 12 +- packages/experimental/cua/src/types.ts | 1 + .../experimental/cua/tests/access.spec.ts | 398 ++++++++++++++++++ .../cua/tests/agent-scopes.spec.ts | 95 +++++ .../cua/tests/application-access.spec.ts | 77 +++- packages/experimental/cua/tsconfig.json | 9 + packages/experimental/cua/tsdown.config.ts | 2 +- .../src/client/slot-catalog.ts | 7 +- .../extensions/tool-cordis/src/api-catalog.ts | 73 ++++ .../references/packages.md | 1 + .../tests/retired-content-admission.spec.ts | 13 +- pnpm-lock.yaml | 94 +++++ scripts/check-workspace-constraints.ts | 2 +- scripts/experimental-package-policy.ts | 2 + scripts/gen-cordis-catalog.ts | 7 + scripts/gen-doc-graphs.ts | 5 + scripts/release/families.spec.ts | 2 + .../verify-default-product-isolation.spec.ts | 11 +- scripts/verify-default-product-isolation.ts | 3 +- .../verify-package-readme-model-experience.ts | 1 + .../session/cua-native/consent-fixture.mjs | 14 - .../session/cua-native/cordis.snapshot.yml | 6 +- snapshots/session/cua-native/cordis.yml | 6 +- .../session/cua-native/permission-fixture.mjs | 23 + .../session/cua-native/replay.override.json | 198 +++++++++ snapshots/session/cua-native/session.v4.jsonl | 326 +++++++------- .../cua-native/system-prompt.expected.md | 4 +- .../cua-native/workspace/native-helper.mjs | 8 +- snapshots/session/cua-ptc/consent-fixture.mjs | 2 +- snapshots/session/cua-ptc/cordis.snapshot.yml | 1 + snapshots/session/cua-ptc/cordis.yml | 1 + .../session/cua-ptc/system-prompt.expected.md | 4 +- tsconfig.base.json | 4 + tsconfig.client.json | 1 + tsconfig.host.json | 4 + vitest.config.ts | 3 + 138 files changed, 4796 insertions(+), 492 deletions(-) create mode 100644 apps/web/tests/cua-optin.e2e.ts create mode 100644 apps/web/tests/cua-permissions.e2e.ts create mode 100644 apps/web/tests/expected/cua-permissions/missing.expected.md create mode 100644 docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.i18n.yaml create mode 100644 docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.md create mode 100644 docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.zh.md create mode 100644 packages/experimental/client-ui-cua/README.i18n.yaml create mode 100644 packages/experimental/client-ui-cua/README.md create mode 100644 packages/experimental/client-ui-cua/README.zh.md create mode 100644 packages/experimental/client-ui-cua/package.json create mode 100644 packages/experimental/client-ui-cua/src/client/PermissionPanel.module.css create mode 100644 packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx create mode 100644 packages/experimental/client-ui-cua/src/client/controller.ts create mode 100644 packages/experimental/client-ui-cua/src/client/index.ts create mode 100644 packages/experimental/client-ui-cua/src/client/locales.ts create mode 100644 packages/experimental/client-ui-cua/src/client/mount.ts create mode 100644 packages/experimental/client-ui-cua/src/css-modules.d.ts create mode 100644 packages/experimental/client-ui-cua/src/index.ts create mode 100644 packages/experimental/client-ui-cua/tests/controller.client.spec.ts create mode 100644 packages/experimental/client-ui-cua/tests/mount.client.spec.ts create mode 100644 packages/experimental/client-ui-cua/tests/panel.client.spec.tsx create mode 100644 packages/experimental/client-ui-cua/tsconfig.json create mode 100644 packages/experimental/client-ui-cua/tsdown.config.ts create mode 100644 packages/experimental/cua-bundle/README.i18n.yaml create mode 100644 packages/experimental/cua-bundle/README.md create mode 100644 packages/experimental/cua-bundle/README.zh.md create mode 100644 packages/experimental/cua-bundle/cordis.patch.yml create mode 100644 packages/experimental/cua-bundle/icon.svg create mode 100644 packages/experimental/cua-bundle/locale/en.json create mode 100644 packages/experimental/cua-bundle/locale/zh.json create mode 100644 packages/experimental/cua-bundle/package.json create mode 100644 packages/experimental/cua-bundle/src/index.ts create mode 100644 packages/experimental/cua-bundle/tsconfig.json create mode 100644 packages/experimental/cua-macos/native/Sources/CuaNative/Permissions.swift create mode 100644 packages/experimental/cua-macos/native/tests/LauncherFixture.swift create mode 100644 packages/experimental/cua-macos/native/tests/launcher_identity_e2e.py create mode 100644 packages/experimental/cua/src/access-types.ts create mode 100644 packages/experimental/cua/src/access.ts create mode 100644 packages/experimental/cua/tests/access.spec.ts create mode 100644 packages/experimental/cua/tests/agent-scopes.spec.ts delete mode 100644 snapshots/session/cua-native/consent-fixture.mjs create mode 100644 snapshots/session/cua-native/permission-fixture.mjs diff --git a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.i18n.yaml b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.i18n.yaml index 448629299d..a2748e8846 100644 --- a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.i18n.yaml @@ -9,11 +9,11 @@ en: 2ba6c43b20a3d347 zh: 5647179ddaa38ad9 /agent-note-ship-experimental-capabilities-as-optional-bundles/decision: - en: 6c8583bdefc5e36c - zh: 0928b4f2876bbfe4 + en: 69df6602f92a3e09 + zh: 2e2ed04760086f4c /agent-note-ship-experimental-capabilities-as-optional-bundles/alternatives-considered: en: d6f40bf02df37ab7 zh: e5b85ce9d41e8754 /agent-note-ship-experimental-capabilities-as-optional-bundles/consequences: - en: 38d27583533cd028 - zh: 9bf11f5489c1aa43 + en: 3064da835e892958 + zh: bfcf967b6acc6f85 diff --git a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md index dbdd104da0..f0e6ce7337 100644 --- a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md +++ b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md @@ -12,12 +12,14 @@ The Web plugin page offered two optional bundles, Agent Teams and voice input. A `OPTIONAL_BUNDLES` owns the installation's default-off selections, including maintained optional capabilities and experimental capabilities. Every bundle declares `icon` and exports `./locale/*.json` with `meta.title` and `meta.description`; the Official group displays that metadata. A bundle's presentation does not determine its [experimental status](../../../../packages/experimental/README.md#status). The isolation check validates admission and still rejects experimental dependencies of nonexperimental bundles. -An optional bundle's dependencies are downloaded with every `dsh` installation. [Reasoning translation](2026-10-05-anonymous-reasoning-translation.md) uses three experimental packages without adding third-party runtime dependencies. Admission therefore considers installation cost and whether a switch supplies a usable composition without an additional configuration form. Browser and computer providers remain explicit compositions because they add substantial runtimes or require an external executable. Stagehand requires model credentials; Python PTC replaces a core provider and conflicts with TypeScript workflow consumers. Demonstrative hooks, webhooks, and mods stay outside the list. [Session Inspector](../feature/2026-09-24-session-inspector.md) owns the Inspector's distinct activation behavior. +An optional bundle's dependencies are downloaded with every `dsh` installation. [Reasoning translation](2026-10-05-anonymous-reasoning-translation.md) uses three experimental packages without adding third-party runtime dependencies. Admission therefore considers installation cost and whether a switch supplies a usable composition without an additional configuration form. External browser and Cua Driver providers remain explicit compositions because they add substantial runtimes or require an external executable. Stagehand requires model credentials; Python PTC replaces a core provider and conflicts with TypeScript workflow consumers. Demonstrative hooks, webhooks, and mods stay outside the list. [Session Inspector](../feature/2026-09-24-session-inspector.md) owns the Inspector's distinct activation behavior. Bundles insert ordinary Host rows; optional model tools register in the global tool layer, so every preset, including minimal, receives them. Plugin management, profile composition, and reload need no bundle-specific behavior. Search, Ralph, and terminals supply their own isolated services in a Host group. Search opens one in-memory index on demand; the Host search policy stays unchanged. Badge skills and title providers act on the Host. Switching a bundle registers or removes its tools for live Agents, and later user patch layers retain precedence. A replacement title provider may register once the previous provider's disposal starts; the previous provider's late results never commit. Claude Code and Codex install on demand through the separate `ON_DEMAND_BUNDLES` catalog; the [on-demand native-bundle decision](2026-10-05-official-on-demand-bundles.md) owns the cost, version, and offline-discovery trade-offs. +Computer Use is a shipped optional bundle in the experimental group. Its repository-owned native component is prepared when the provider loads, without an external Cua Driver runtime. + ## Alternatives considered **Ship every provider as an optional bundle.** Composes and displays correctly, but grows every install by provider runtimes for a capability most installations never switch on; voice input's `sherpa-onnx-node` is the one accepted precedent. @@ -31,3 +33,5 @@ Claude Code and Codex install on demand through the separate `ON_DEMAND_BUNDLES` ## Consequences Official discovery does not imply experimental status or a different plugin lifecycle. Switching a bundle off keeps its installed files; failed activation retains the existing saved-selection behavior. Bundle rows are ordinary Host rows with their usual switches. Individual and combined composition tests cover the Host rows and every preset's tool catalog; live switching and recorded Sessions cover activation and model-visible behavior. + +Computer Use keeps its permission service and native provider on the Host and registers tools in Agent scopes whose bound preset is `ptc`. Existing Agent events and `createScope()` maintain current, new and resumed Agents. Preset declarations remain unchanged. Only the explicit detail-page action changes the new-session default through the existing settings form; the activation switch does not configure PTC execution. diff --git a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.zh.md b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.zh.md index 6c9df974e8..c90290b20d 100644 --- a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.zh.md +++ b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.zh.md @@ -12,12 +12,14 @@ Web 插件页只提供两个可选 bundle：Agent Teams 与语音输入。Auto r `OPTIONAL_BUNDLES` 管理安装随附且默认关闭的选项，包括维护中的可选能力与实验性能力。每个 bundle 声明 `icon`，并导出带 `meta.title` 与 `meta.description` 的 `./locale/*.json`；官方分组显示这些元数据。Bundle 的展示方式不决定其[实验性状态](../../../../packages/experimental/README.zh.md#status)。隔离检查校验准入，并继续拒绝非实验 bundle 的实验性依赖。 -可选 bundle 的依赖随每次 `dsh` 安装下载。[思考过程翻译](2026-10-05-anonymous-reasoning-translation.zh.md)使用三个实验包，不增加第三方运行时依赖。因此准入需要考虑安装成本，以及开关能否在不增加配置表单的情况下提供可用组合。浏览器与电脑提供方仍使用显式组合，因为它们会增加较大的运行时或要求外部可执行文件。Stagehand 需要模型凭据；Python PTC 替换核心提供方，并与 TypeScript workflow 消费方冲突。演示性钩子、webhook 与模组不进入名单。[Session Inspector](../feature/2026-09-24-session-inspector.zh.md) 管理 Inspector 特有的激活行为。 +可选 bundle 的依赖随每次 `dsh` 安装下载。[思考过程翻译](2026-10-05-anonymous-reasoning-translation.zh.md)使用三个实验包，不增加第三方运行时依赖。因此准入需要考虑安装成本，以及开关能否在不增加配置表单的情况下提供可用组合。外部浏览器与 Cua Driver 提供方仍使用显式组合，因为它们会增加较大的运行时或要求外部可执行文件。Stagehand 需要模型凭据；Python PTC 替换核心提供方，并与 TypeScript workflow 消费方冲突。演示性钩子、webhook 与模组不进入名单。[Session Inspector](../feature/2026-09-24-session-inspector.zh.md) 管理 Inspector 特有的激活行为。 Bundle 插入普通 Host 行；可选模型工具注册到全局工具层，因此包括 minimal 在内的所有预设都会获得它们。插件管理、profile 组合与重载无需针对 bundle 的特殊行为。搜索、Ralph 与终端在 Host 组内提供自己的隔离服务。搜索按需打开一个内存索引；Host 搜索策略保持不变。徽章技能与标题提供方作用于 Host。切换 bundle 会为运行中的 Agent 注册或移除其工具，后续用户 patch 层继续优先。前一个标题提供方开始清理后，替代提供方即可注册；前一个提供方的迟到结果不会提交。 Claude Code 与 Codex 通过独立的 `ON_DEMAND_BUNDLES` 目录按需安装，其成本、版本与离线发现取舍由[按需原生 bundle 决策](2026-10-05-official-on-demand-bundles.zh.md)管理。 +Computer Use 是 experimental 包组内随安装提供的可选组合包。提供方加载时准备仓库自有的本机组件，无需外部 Cua Driver 运行时。 + ## 考虑过的替代方案 **把所有提供方都做成可选 bundle。** 组合与展示都正确，但会为多数安装从不启用的能力把提供方运行时塞进每次安装；语音输入的 `sherpa-onnx-node` 是唯一被接受的先例。 @@ -31,3 +33,5 @@ Claude Code 与 Codex 通过独立的 `ON_DEMAND_BUNDLES` 目录按需安装， ## 后果 官方发现不表示实验性状态，也不引入不同的插件生命周期。关闭 bundle 保留已安装文件；激活失败保留现有的已保存选择行为。Bundle 行是普通 Host 行，使用常规开关。独立与组合测试覆盖 Host 行及每个预设的工具目录；实时切换与已记录 Session 覆盖激活和模型可见行为。 + +Computer Use 将权限服务和原生提供方保留在 Host，在已绑定 `ptc` preset 的 Agent 作用域中注册工具。现有 Agent 事件和 `createScope()` 维护当前、新建和恢复的 Agent，preset 声明保持不变。只有详情页的显式操作通过现有设置表单修改新会话默认模式，启用开关不配置 PTC 执行方式。 diff --git a/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.i18n.yaml b/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.i18n.yaml index a5e3035bd0..2cac3e4e63 100644 --- a/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.i18n.yaml @@ -9,11 +9,11 @@ en: b32a7ce6483b9b48 zh: ec0938ad032b99d8 /agent-note-application-consent-for-native-computer-use/decision: - en: 92247ff599c1103f - zh: 26845420a9711c83 + en: 22b3e4b770d2517f + zh: 2250b31a5a50758a /agent-note-application-consent-for-native-computer-use/alternatives-considered: - en: 4693dcd217c4fedb - zh: 1dc9bf9067314006 + en: 80cd5fd93a42d937 + zh: 2a0e241b1b8c2e1d /agent-note-application-consent-for-native-computer-use/consequences: - en: 0146a69ab2086edc - zh: 4257bc1ab4432c3f + en: 4bda3366af65250a + zh: 7c83561b4db07284 diff --git a/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.md b/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.md index 3fb91170e5..d69ba1218c 100644 --- a/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.md +++ b/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.md @@ -10,29 +10,38 @@ macOS privileges authorize an execution host, not a user's choice of target appl ## Decision -The CUA executor owns application consent for every object-API and native-tool dispatch. It uses the existing User Questions service for human decisions and the provider's existing profile configuration for saved grants. General tool approval and operating-system privileges remain separate requirements. Arbitrary persistent PTC code and separately authorized shell tools retain their own execution authority. +The CUA executor owns application consent for every object-API and native-tool dispatch. It uses the existing User Questions service for human decisions and the provider's existing profile configuration for saved grants. The existing question UI presents the offered labels; the executor rejects custom text or answers that do not select exactly one offered label. Permission-page icons and names are presentation metadata, excluded from saved identities and model results. General tool approval and operating-system privileges remain separate requirements. Arbitrary persistent PTC code and separately authorized shell tools retain their own execution authority. -A saved identity combines the canonical executable path with a verified designated signing requirement, represented by a SHA-256 digest. The path prevents authorization from following a copied application, and the signing requirement distinguishes replacements while allowing matching signed updates. Unverifiable and ad-hoc applications receive task access only, tied to a process instance and Agent connection lifetime. Labels do not establish identity. +System-permission buttons call the native helper on the DSH host and remain available to remote browsers. The [usage documentation](../../../../packages/experimental/client-ui-cua/README.md#use-this-package) explains where to complete authorization; the page has no persistent remote-access notice. + +A saved identity combines the canonical executable path with a verified designated signing requirement, represented by a SHA-256 digest. The path prevents authorization from following a copied application, and the signing requirement distinguishes replacements while allowing matching signed updates. Unverifiable and ad-hoc applications receive session access only, tied to a process instance. Labels do not establish identity. Cold application selection prepares an executable identity and handle without launching. The same consent decision guards the private launch continuation, and the helper checks the executable again before launch and binds the resulting process afterward. Signed selections retain the normal designated-requirement identity; an unverifiable executable uses a content identity for this pending selection and cannot create a saved grant. Preparing metadata before asking avoids starting an application the user refuses. -Selections of the same application within one task are serialized, so a pending question does not produce competing decisions. Saved-grant additions read the current configuration inside one mutation queue. Refusals prevent another prompt within the task; an explicitly restored saved grant takes precedence at dispatch. +Session decisions are kept in memory by Session ID and application identity. Agent disposal and helper replacement release native handles without discarding these decisions; provider disposal or DSH restart clears them. This scope covers follow-up turns without relying on the model to decide when a user intent has ended. -Removing a saved grant invalidates Agent connections retaining the application's handles and cancels active operations; Agent/provider disposal awaits native connection cleanup. Reauthorization uses a fresh connection, so old app and window snapshots cannot become valid again. The [native object decision](2026-09-17-experimental-cua-object-api.md) continues to own process identity, coordinate semantics, and separation from the evaluator. +Selections of the same application within one session are serialized, so a pending question does not produce competing decisions. Saved-grant additions and revocations share one mutation queue and read the current grant list inside it. Provider shutdown cancels writes still waiting for HMR admission; admitted writes settle before disposal completes. Refusals prevent another prompt until the user permits asking again; a restored saved grant takes precedence in both dispatch and the permission-page projection. + +Revocation invalidates the Agent connections that retain the application's handles, then awaits native cleanup. Reauthorization uses a fresh connection, so old app and window snapshots cannot become valid again. The [native object decision](2026-09-17-experimental-cua-object-api.md) continues to own process identity, coordinate semantics, and separation from the evaluator. ## Alternatives considered **Trust model instructions or hide controls.** Retained objects and direct tool calls can still reach native operations. The executor must enforce the decision at dispatch. -**Persist application names or unsigned paths.** A replacement can reuse both. Durable grants require a verifiable signing identity; task-only access supports unsigned development applications without that promise. +**Persist application names or unsigned paths.** A replacement can reuse both. Durable grants require a verifiable signing identity; session access supports unsigned development applications without that promise. -**Add an approval database or Session control stream.** Profile settings already own user configuration, and User Questions already owns interactive answers. +**Add an approval database or Session control stream.** Profile settings already own user configuration, and User Questions already owns interactive answers. The permission UI observes the provider and its owning settings namespace without another authorization store or event channel. + +**Disable permission buttons based on the browser address.** A user may operate the DSH host through screen sharing and complete authorization there. Port forwarding and proxies make the address an unreliable indication of whether the browser and DSH run on the same computer. + +**Use the nearest GUI ancestor as the permission target.** A source-built Desktop app can sit between Terminal and the helper while macOS still attributes Accessibility and Screen Recording to Terminal. Process responsibility supplies the system’s attribution; ancestry alone does not. The optional SPI adds compatibility risk, so lookup failure omits the application instead of guessing. **Keep the native context after revocation.** Its application lookup can return an earlier handle and retain window snapshots. Closing the Agent connection discards its native context through existing transport cleanup; preserving unrelated handles would require a native per-application retirement protocol. ## Consequences -Revoking one application also invalidates other application handles on the same Agent connection. Those applications require fresh selection; saved grants remain reusable, while task grants expire. Other tasks with independent connections remain unaffected unless they also hold access to the revoked application. +Refusing or revoking one application also invalidates other application handles on the same Agent connection. Those applications require fresh selection; session and saved grants remain reusable. Other sessions with independent connections remain unaffected unless they also hold access to the revoked application. +Explicit enablement may start the shared helper for a read-only permission probe. The helper captures the application identified by macOS process responsibility before readiness releases the launcher, retaining its name and icon after reparenting. The optional native query is dynamically resolved; unavailable attribution uses generic system-prompt guidance. The macOS prompt remains authoritative. Enabling does not request macOS authorization or inspect application content. Application decisions and failure results are exercised through policy regressions and the recorded CUA scenario. Desktop binary distribution, signing, and packaged-host permissions remain outside the source-launch workflow. diff --git a/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.zh.md b/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.zh.md index 73d4640db7..b758b796c9 100644 --- a/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.zh.md +++ b/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.zh.md @@ -10,29 +10,38 @@ macOS 权限授权执行宿主，不能表达用户允许操作哪些目标应 ## 决策 -CUA 执行器在每次对象 API 和原生工具调用时检查应用授权。人工决定复用 User Questions 服务，长期许可保存在 provider 的现有 profile 配置中。通用工具审批和操作系统权限仍是独立要求。任意 persistent PTC 代码和另行授权的 shell 工具保留各自的执行权限。 +CUA 执行器在每次对象 API 和原生工具调用时检查应用授权。人工决定复用 User Questions 服务，长期许可保存在 provider 的现有 profile 配置中。通用工具审批和操作系统权限仍是独立要求。任意 persistent PTC 代码和另行授权的 shell 工具保留各自的执行权限。现有问答界面展示授权选项；执行器拒绝自由文本或未准确选中一个所给选项的回答。权限页图标和名称仅用于展示，不进入保存的身份信息或模型结果。 -长期身份由规范化可执行文件路径和经过验证的签名指定要求组成，以 SHA-256 摘要表示。路径阻止许可跟随应用副本，签名要求区分替换程序，并允许符合相同签名要求的更新。签名不可验证或使用 ad-hoc 签名的应用只能获得任务授权，许可绑定进程实例和 Agent 连接生命周期。显示名称不用于确定身份。 +系统权限按钮调用运行 DSH 的电脑上的原生 helper，远程访问时仍可使用。授权地点写在[使用说明](../../../../packages/experimental/client-ui-cua/README.zh.md#use-this-package)中，页面不常驻远程访问提示。 + +长期身份由规范化可执行文件路径和经过验证的签名指定要求组成，以 SHA-256 摘要表示。路径阻止许可跟随应用副本，签名要求区分替换程序，并允许符合相同签名要求的更新。签名不可验证或使用 ad-hoc 签名的应用只能获得会话授权，许可绑定进程实例。显示名称不用于确定身份。 选择尚未运行的应用时，先准备可执行文件身份和句柄，不立即启动。同一授权决定保护内部启动续接，helper 在启动前再次核对可执行文件，并在启动后绑定实际进程。已签名应用沿用签名指定要求身份；签名不可验证的程序使用本次待启动选择的内容身份，不能生成长期许可。先准备元数据再询问，可以避免启动用户拒绝的应用。 -同一任务对同一应用的选择串行执行，避免未完成的问题产生相互冲突的决定。长期许可的新增在同一写入队列内读取当前配置。拒绝会阻止任务内的再次询问；用户显式恢复的长期许可在执行检查时优先于旧拒绝记录。 +会话决定按 Session ID 和应用身份保存在内存中。Agent 销毁或 helper 重建会释放原生句柄，但保留授权决定；提供方卸载或 DSH 重启后清除。会话范围可以覆盖用户的后续指令，无须由模型判断一次用户意图何时结束。 -删除长期许可会使保留目标应用句柄的 Agent 连接失效，并取消活动操作；Agent 或提供方销毁时等待对应的原生连接清理。重新授权使用新连接，因此旧应用对象和窗口快照不能重新生效。[原生对象决策](2026-09-17-experimental-cua-object-api.zh.md)继续负责进程身份、坐标语义和与执行器分离的设计。 +同一会话对同一应用的选择串行执行，避免未完成的问题产生相互冲突的决定。长期许可的新增和撤销共用写入队列，并在队列内读取当前许可列表。提供方关闭时取消尚在等待 HMR 调度的写入，已开始的写入会在销毁完成前结束。拒绝会阻止再次询问，直到用户允许重询；重新添加的长期许可在执行检查和权限页中均优先于旧拒绝记录。 + +撤销使保留目标应用句柄的 Agent 连接失效，并等待原生清理完成。重新授权使用新连接，因此旧应用对象和窗口快照不能重新生效。[原生对象决策](2026-09-17-experimental-cua-object-api.zh.md)继续负责进程身份、坐标语义和与执行器分离的设计。 ## 考虑过的替代方案 **依赖模型指引或隐藏控件。** 保留的对象和直接工具调用仍能到达原生操作。执行器必须在派发时检查用户决定。 -**保存应用名称或未签名路径。** 替换程序可以复用二者。长期许可需要可验证的签名身份；任务授权支持未签名的开发应用，但不承诺跨进程复用。 +**保存应用名称或未签名路径。** 替换程序可以复用二者。长期许可需要可验证的签名身份；会话授权支持未签名的开发应用，但不承诺跨进程复用。 -**新增审批数据库或 Session 控制流。** Profile settings 已负责用户配置，User Questions 已负责交互回答。 +**新增审批数据库或 Session 控制流。** Profile settings 已负责用户配置，User Questions 已负责交互回答。权限界面查询 provider，并监听其配置 namespace，无须另一份授权存储或事件通道。 + +**根据浏览器访问地址禁用权限按钮。** 用户可能通过屏幕共享操作运行 DSH 的电脑，并在该电脑上完成授权。端口转发和代理使访问地址无法可靠判断浏览器与 DSH 是否运行在同一台电脑上。 + +**把最近的 GUI 祖先当作授权对象。** 源码构建的 Desktop 可以位于 Terminal 和 helper 之间，但 macOS 仍将辅助功能和屏幕录制归属于 Terminal。责任进程记录提供系统认定的归属，仅凭父进程链无法确定。可选内部接口有兼容性风险，因此查询失败时省略应用信息，不再猜测。 **撤销后保留原生状态。** 原生应用查找可能返回旧句柄并保留窗口快照。关闭 Agent 连接可以通过现有传输清理释放其原生状态；保留其他应用的句柄则需要新增按应用销毁状态的原生协议。 ## 影响 -撤销一个应用也会使同一 Agent 连接中其他应用的句柄失效。这些应用需要重新选择；长期许可仍可复用，任务许可随之失效。其他任务使用独立连接，除非也持有被撤销应用的许可，否则不受影响。 +拒绝或撤销一个应用也会使同一 Agent 连接中其他应用的句柄失效。这些应用需要重新选择；会话许可和长期许可仍可复用。其他会话使用独立连接，除非也持有被撤销应用的许可，否则不受影响。 +显式启用可以启动共享 helper 进行只读权限检查。helper 在发布就绪状态、让启动器退出之前，记录 macOS 责任进程对应的应用，因此父进程改变后仍能提供应用名称和图标。可选的本机查询通过动态加载解析，无法取得归属时显示通用系统弹窗指引。授权归属仍以 macOS 提示为准。启用不会申请 macOS 授权或检查应用内容。 授权决定和失败结果由策略回归测试及 CUA 录制会话验证。桌面安装包的二进制分发、签名和打包宿主权限不属于源码启动流程。 diff --git a/apps/cli/package.json b/apps/cli/package.json index 6f4f927b48..d7c3e78f72 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -105,6 +105,7 @@ "@deepseek-ai/dsh-experimental-terminal-bundle": "workspace:*", "@deepseek-ai/dsh-experimental-session-search": "workspace:*", "@deepseek-ai/dsh-experimental-tool-worktree": "workspace:*", + "@deepseek-ai/dsh-cua-bundle": "workspace:*", "@deepseek-ai/dsh-experimental-plugin-market-bundle": "workspace:*" }, "devDependencies": { diff --git a/apps/web/tests/cua-optin.e2e.ts b/apps/web/tests/cua-optin.e2e.ts new file mode 100644 index 0000000000..4c954f1cbe --- /dev/null +++ b/apps/web/tests/cua-optin.e2e.ts @@ -0,0 +1,98 @@ +/** Optional CUA uses existing plugin activation without changing the shared PTC runtime. */ +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { chromium, type Browser } from 'playwright' +import { readProfileManifest } from '@deepseek-ai/dsh-app-boot' +import { SessionId } from '@deepseek-ai/dsh-session' +import type {} from '@deepseek-ai/dsh-ptc-runtime' +import type {} from '@deepseek-ai/dsh-agent-preset-registry' +import { expect, it } from 'vitest' +import { launchWebScaffold, type WebScaffold } from './scaffold.ts' +import { connectFreshWorkspaceZh, ZH_BROWSER_LOCALE } from './support.ts' + +it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserving default %s and runtime configuration', async (selectedDefault) => { + const directory = await mkdtemp(join(tmpdir(), 'cua-optin-web-')) + let ownedScaffold: WebScaffold | undefined + let browser: Browser | undefined + try { + const patch = join(directory, 'native-fixture.yml') + // The executable cannot answer native permission queries, so setup must offer retry in the detail page. + await writeFile(patch, JSON.stringify([{ id: 'cua', config: { helperPath: process.execPath } }])) + const scaffold = ownedScaffold = await launchWebScaffold({ harnessHome: join(directory, 'home'), extraOverlayPath: [ + fileURLToPath(new URL('./pin-browse-picker.overlay.yml', import.meta.url)), patch, + ] }) + const initialDefault = selectedDefault ?? 'standard' + if (selectedDefault !== undefined) await scaffold.ctx.settings.update('agent-preset-registry', { selectedDefault }) + browser = await chromium.launch() + const page = await browser.newPage({ viewport: { width: 1680, height: 1000 }, locale: ZH_BROWSER_LOCALE }) + await page.goto(scaffold.authenticatedUrl) + await connectFreshWorkspaceZh(page, scaffold.workspaceCwd) + const originalAgent = scaffold.ctx.agents.list()[0]! + const ptcAgent = (await scaffold.ctx.agents.create({ sessionId: SessionId('cua-existing-ptc'), + setup: async (ctx) => { await scaffold.ctx.agentPresets.mount(ctx, 'ptc') }, + })).agent + const originalRuntime = scaffold.ctx.ptcRuntime + expect(originalRuntime.programFormat).toBe('function-body') + const settingsPath = join(scaffold.harnessHome, 'profiles/scaffold/cordis.patch.yml') + const before = await readFile(settingsPath, 'utf8') + const selected = () => readProfileManifest('scaffold', join(scaffold.harnessHome, 'profiles/scaffold')).dsh?.profile?.bundles + await page.getByRole('navigation', { name: '全局面板' }).getByRole('button', { name: '插件', exact: true }).click() + const card = page.locator('[data-plugin-package="@deepseek-ai/dsh-cua-bundle"]') + const toggle = card.getByRole('switch', { name: '启用 Computer Use', exact: true }) + expect(await toggle.getAttribute('aria-checked')).toBe('false') + expect(await card.textContent()).toContain('请切换到 PTC 模式') + await toggle.click() + await expect.poll(() => toggle.getAttribute('aria-checked')).toBe('true') + const setup = page.getByRole('dialog', { name: '使用 Computer Use 前需要设置', exact: true }) + await setup.waitFor() + expect(await setup.getByText('Computer Use 插件已开启，但暂时无法检测系统权限。请前往插件详情页查看原因并重试。', { exact: true }).count()).toBe(1) + await setup.getByRole('button', { name: '稍后', exact: true }).click() + expect(selected()).toContain('@deepseek-ai/dsh-cua-bundle') + expect(await readFile(settingsPath, 'utf8')).toBe(before) + expect(scaffold.ctx.ptcRuntime).toBe(originalRuntime) + expect(scaffold.ctx.agentPresets.defaultId).toBe(initialDefault) + expect(scaffold.ctx.agents.get(originalAgent.id)).toBe(originalAgent) + expect(scaffold.ctx.tools.schemas(originalAgent).some(tool => tool.name === 'cua')).toBe(false) + expect(scaffold.ctx.tools.schemas(ptcAgent).some(tool => tool.name === 'cua')).toBe(true) + expect((await scaffold.ctx.systemPrompt.assemble({ scope: ptcAgent })).tools.map(tool => tool.name)).toEqual(['run_code']) + const fresh = await scaffold.ctx.agents.create({ sessionId: SessionId('cua-new-ptc'), + setup: async (ctx) => { await scaffold.ctx.agentPresets.mount(ctx, 'ptc') }, + }) + expect(scaffold.ctx.tools.schemas(fresh.agent).some(tool => tool.name === 'cua')).toBe(true) + await toggle.click() + await expect.poll(() => toggle.getAttribute('aria-checked')).toBe('false') + expect(scaffold.ctx.get('computerUse')).toBeUndefined() + for (const agent of [ptcAgent, fresh.agent]) expect(scaffold.ctx.tools.schemas(agent).some(tool => tool.name === 'cua')).toBe(false) + expect(scaffold.ctx.ptcRuntime).toBe(originalRuntime) + expect(await readFile(settingsPath, 'utf8')).toBe(before) + await card.getByRole('button', { name: '查看 Computer Use', exact: true }).click() + const detailToggle = page.locator('[data-plugin-panel]').getByRole('switch', { name: '启用 Computer Use', exact: true }) + await detailToggle.click() + await expect.poll(() => detailToggle.getAttribute('aria-checked')).toBe('true') + const panel = page.locator('[data-cua-settings]') + await panel.waitFor() + const defaultSetting = panel.getByRole('group', { name: '新会话默认模式', exact: true }) + await expect.poll(() => defaultSetting.getByRole('button', { name: '设为 PTC', exact: true }).isEnabled()).toBe(true) + await defaultSetting.getByRole('button', { name: '设为 PTC', exact: true }).click() + await expect.poll(() => scaffold.ctx.agentPresets.defaultId).toBe('ptc') + expect(scaffold.ctx.agentPresets.composedPreset(originalAgent.ctx)).toBe(initialDefault) + expect(scaffold.ctx.ptcRuntime).toBe(originalRuntime) + const confirmed = await scaffold.ctx.agents.create({ sessionId: SessionId('cua-confirmed-default'), + setup: async (ctx) => { await scaffold.ctx.agentPresets.mount(ctx) }, + }) + expect(scaffold.ctx.tools.schemas(confirmed.agent).some(tool => tool.name === 'cua')).toBe(true) + const saved = await readFile(settingsPath, 'utf8') + expect(saved).toContain('selectedDefault: ptc') + expect(saved).not.toContain('executionMode:') + await detailToggle.click() + await expect.poll(() => detailToggle.getAttribute('aria-checked')).toBe('false') + expect(await readFile(settingsPath, 'utf8')).toBe(saved) + expect(scaffold.ctx.ptcRuntime).toBe(originalRuntime) + } finally { + try { await browser?.close() } finally { + try { await ownedScaffold?.close() } finally { await rm(directory, { recursive: true, force: true }) } + } + } +}) diff --git a/apps/web/tests/cua-permissions.e2e.ts b/apps/web/tests/cua-permissions.e2e.ts new file mode 100644 index 0000000000..3490749230 --- /dev/null +++ b/apps/web/tests/cua-permissions.e2e.ts @@ -0,0 +1,139 @@ +/** Built permission UI talks to the real provider; the isolated helper substitutes only macOS privilege observations. */ +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { ToolCallId } from '@deepseek-ai/dsh-llm' +import type {} from '@deepseek-ai/dsh-user-approval' +import type {} from '@deepseek-ai/dsh-agent-preset-registry' +import { chromium, type Browser } from 'playwright' +import { expect, it, onTestFailed, onTestFinished } from 'vitest' +import { captureStableAria, compareOrRefreshGolden, launchWebScaffold, webSnapshotMode, watchConsole, type WebScaffold } from './scaffold.ts' +import { connectFreshWorkspace, newEnglishPage, saveFailureShot } from './support.ts' +import { installNativeFixture, startFixtureHost } from '../../../packages/experimental/cua-macos/tests/native-fixture.ts' + +const bundle = fileURLToPath(new URL('../../../packages/experimental/cua-bundle', import.meta.url)) +const expected = fileURLToPath(new URL('./expected/cua-permissions/missing.expected.md', import.meta.url)) + +// System-permission onboarding belongs to the macOS-only bundle; rendering and policy units remain portable. +it.skipIf(process.platform !== 'darwin')('keeps system requests pending until fresh observation and persists application revocation', async () => { + const root = await mkdtemp(join(tmpdir(), 'dsh-cua-permissions-web-')) + const resources: { scaffold?: WebScaffold; browser?: Browser; closeFixture?: () => Promise } = {} + onTestFinished(async () => { + try { await resources.browser?.close() } finally { + try { await resources.scaffold?.close() } finally { + try { await resources.closeFixture?.() } finally { await rm(root, { recursive: true, force: true }) } + } + } + }) + const helper = await installNativeFixture(root) + resources.closeFixture = await startFixtureHost(helper) + const state = join(root, 'permissions.json'), audit = join(root, 'helper-events.jsonl') + const launchApplication = { name: 'Visual Studio Code.app', path: '/Applications/Visual Studio Code.app', + icon: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC' } + await writeFile(state, JSON.stringify({ accessibility: false, screenRecording: false, launchApplication })) + await writeFile(join(root, 'observe-helper'), '') + const fixtureBundle = join(root, 'fixture-bundle') + await mkdir(fixtureBundle) + await writeFile(join(fixtureBundle, 'package.json'), JSON.stringify({ name: 'cua-permission-fixture', version: '1.0.0', + dsh: { bundle: { patch: 'cordis.patch.yml' } } })) + await writeFile(join(fixtureBundle, 'cordis.patch.yml'), `- id: cua\n config: ${JSON.stringify({ helperPath: helper })}\n`) + const scaffold = await launchWebScaffold({ + profile: { packages: [{ dir: bundle, enabled: true }, { dir: fixtureBundle, enabled: true }] }, + }) + resources.scaffold = scaffold + await scaffold.ctx.settings.mutate('cua', [{ op: 'set', path: ['allowedApps'], + value: [{ id: 'a'.repeat(64), name: 'Fixture', executablePath: '/Applications/Fixture.app' }] }]) + const browser = await chromium.launch() + resources.browser = browser + const page = await newEnglishPage(browser), tripwire = watchConsole(page) + onTestFailed(() => saveFailureShot(page, 'web-e2e-cua-permissions')) + await page.goto(scaffold.authenticatedUrl) + await connectFreshWorkspace(page, scaffold.workspaceCwd) + const agent = scaffold.ctx.agents.list()[0]! + await scaffold.ctx.agentPresets.select(agent, 'ptc') + scaffold.ctx.approval.setPolicy(agent, 'never') + await scaffold.ctx.settings.mutate('cua', [{ op: 'set', path: ['allowedApps'], value: [] }]) + // Application consent remains independent of the session's tool-approval policy. + const grant = scaffold.ctx.tools.execute({ agent, signal: new AbortController().signal, + callId: ToolCallId('cua-app-approval'), name: 'run_code', + arguments: { code: 'return await tools.cua({ method: "getApp", params: { selector: "Fixture" } })', + description: 'Request application access' } }) + const approval = page.locator('[data-question-key]') + await Promise.race([approval.waitFor(), grant.then((result) => { + throw new Error(`Application selection completed before consent: ${JSON.stringify(result)}`) + })]) + expect(await approval.textContent()).toContain('Fixture') + await approval.getByRole('radio', { name: 'Always allow', exact: true }).click() + await approval.getByRole('button', { name: 'Submit', exact: true }).click() + expect((await grant).isError).toBe(false) + await approval.waitFor({ state: 'hidden' }) + await page.getByRole('button', { name: 'Plugins', exact: true }).click() + const card = page.locator('[data-plugin-package="@deepseek-ai/dsh-cua-bundle"]') + expect(await card.getByRole('button', { name: /Accessibility permission|Screen Recording permission/ }).count()).toBe(0) + await page.setViewportSize({ width: 1680, height: 600 }) + const plugins = page.locator('[data-plugin-panel]') + const openDetails = page.getByRole('button', { name: 'View Computer Use', exact: true }) + await openDetails.scrollIntoViewIfNeeded() + expect(await plugins.evaluate(element => element.scrollTop)).toBeGreaterThan(0) + await openDetails.click() + const panel = page.locator('[data-cua-settings]') + const accessibility = panel.getByRole('group', { name: 'Accessibility', exact: true }) + await accessibility.getByText('Not allowed', { exact: true }).waitFor() + expect(await panel.getByRole('region', { name: 'Authorize Computer Use', exact: true }).getByRole('group').count()).toBe(3) + expect(await panel.getByRole('region', { name: 'Manage applications Computer Use can access', exact: true }).count()).toBe(1) + await expect.poll(() => panel.locator('li img').evaluate((image: HTMLImageElement) => image.naturalWidth)).toBe(1) + expect(await panel.getByText('/fixture/app', { exact: true }).count()).toBe(0) + expect(await accessibility.getByText('Visual Studio Code.app', { exact: true }).count()).toBe(1) + expect(await accessibility.textContent()).toContain('In System Settings, allow') + expect(await accessibility.textContent()).toContain('to read interface controls, click, and type.') + expect(await panel.getByRole('group', { name: 'Screen Recording', exact: true }).getByText('Visual Studio Code.app', { exact: true }).count()).toBe(1) + await expect.poll(() => accessibility.locator('img').evaluate((image: HTMLImageElement) => image.naturalWidth)).toBe(1) + await expect.poll(() => panel.getAttribute('aria-busy')).toBe('false') + await compareOrRefreshGolden(expected, await captureStableAria(page, '[data-cua-settings]', scaffold.workspaceCwd), webSnapshotMode()) + try { + for (const width of [1680, 390]) { + await page.setViewportSize({ width, height: 1000 }) + for (const colorScheme of ['light', 'dark'] as const) { + await page.emulateMedia({ colorScheme }) + await expect.poll(() => page.evaluate(() => document.body.hasAttribute('data-ds-dark-theme'))).toBe(colorScheme === 'dark') + expect(await panel.evaluate(element => element.scrollWidth <= element.clientWidth)).toBe(true) + if (webSnapshotMode() === 'refresh') { + const directory = fileURLToPath(new URL('../../../.artifacts', import.meta.url)) + await mkdir(directory, { recursive: true }) + const path = join(directory, `cua-permission-design-${process.pid}-${width}-${colorScheme}.png`) + await panel.screenshot({ path, animations: 'disabled' }) + console.log(`CUA permission design: ${path}`) + } + } + } + } finally { + await page.emulateMedia({ colorScheme: null }) + await page.setViewportSize({ width: 1680, height: 1000 }) + } + await accessibility.getByRole('button', { name: 'Request permission', exact: true }).click() + await expect.poll(() => panel.getAttribute('aria-busy')).toBe('false') + expect(await accessibility.getByText('Not allowed', { exact: true }).count()).toBe(1) + await writeFile(state, JSON.stringify({ accessibility: true, screenRecording: false, launchApplication })) + // Headless pages retain document focus; model returning from System Settings at the window-event boundary. + await page.evaluate(() => { window.dispatchEvent(new Event('focus')) }) + await accessibility.getByText('Allowed', { exact: true }).waitFor() + const screen = panel.getByRole('group', { name: 'Screen Recording', exact: true }) + await screen.getByRole('button', { name: 'Open System Settings', exact: true }).click() + await expect.poll(() => panel.getAttribute('aria-busy')).toBe('false') + await panel.getByRole('button', { name: 'Revoke', exact: true }).click() + await panel.getByRole('heading', { name: 'Always-allowed applications', exact: true }).waitFor({ state: 'hidden' }) + await panel.getByRole('heading', { name: 'Applications not allowed', exact: true }).waitFor() + await panel.getByRole('button', { name: 'Allow asking again', exact: true }).click() + await panel.getByText('Fixture', { exact: true }).waitFor({ state: 'hidden' }) + const saved = await readFile(join(scaffold.harnessHome, 'profiles/scaffold/cordis.patch.yml'), 'utf8') + expect(saved).toContain('allowedApps: []') + const events = (await readFile(audit, 'utf8')).trim().split('\n').map(line => JSON.parse(line) as { event: string; method: string; params: object }) + .filter(record => record.event === 'call').map(({ method, params }) => ({ method, params })) + expect(events).toContainEqual({ method: 'requestPermission', params: { permission: 'accessibility' } }) + expect(events.filter(event => event.method === 'openPermissionSettings')).toEqual([ + { method: 'openPermissionSettings', params: { permission: 'screenRecording' } }, + ]) + expect(events.every(event => ['permissionStatus', 'requestPermission', 'openPermissionSettings', 'applicationIcons', 'getApp'].includes(event.method))).toBe(true) + expect(tripwire.pageErrors).toEqual([]) +}, 60_000) diff --git a/apps/web/tests/expected/cua-permissions/missing.expected.md b/apps/web/tests/expected/cua-permissions/missing.expected.md new file mode 100644 index 0000000000..d8fafe9400 --- /dev/null +++ b/apps/web/tests/expected/cua-permissions/missing.expected.md @@ -0,0 +1,26 @@ +- region "Authorize Computer Use": + - heading "Authorize Computer Use" [level=3] + - group "Default mode for new sessions": + - strong: Default mode for new sessions + - paragraph: Computer Use is available only in PTC mode. + - button "Set to PTC" + - group "Accessibility": + - strong: Accessibility + - paragraph: In System Settings, allow Visual Studio Code.app to read interface controls, click, and type. + - text: Not allowed + - button "Request permission" + - button "Open System Settings" + - group "Screen Recording": + - strong: Screen Recording + - paragraph: In System Settings, allow Visual Studio Code.app to capture application windows. + - text: Not allowed + - button "Request permission" + - button "Open System Settings" + - paragraph: Enable the application in System Settings. Follow the macOS prompt if it names another application. +- region "Manage applications Computer Use can access": + - heading "Manage applications Computer Use can access" [level=3] + - heading "Always-allowed applications" [level=4] + - list: + - listitem: + - strong: Fixture + - button "Revoke" diff --git a/apps/web/tests/expected/plugin-config/official.expected.md b/apps/web/tests/expected/plugin-config/official.expected.md index 37d3a232b4..71c7f9218e 100644 --- a/apps/web/tests/expected/plugin-config/official.expected.md +++ b/apps/web/tests/expected/plugin-config/official.expected.md @@ -38,6 +38,10 @@ - button "查看 开发者工具": 开发者工具 - text: 实验性 查看调试会话原始数据、聊天消息分组数据，以及调试 NodeJS 后端 - switch "启用 开发者工具" + - listitem: + - button "查看 Computer Use": Computer Use + - text: 实验性 目前仅支持 macOS，Windows 支持正在开发中。为已有及新建 PTC 会话加载 Computer Use。请切换到 PTC 模式来使用，并授予辅助功能和屏幕录制权限。 + - switch "启用 Computer Use" - heading "已安装插件" [level=3] - list: - listitem: diff --git a/apps/web/tests/expected/plugin-manager/live-enabled.expected.md b/apps/web/tests/expected/plugin-manager/live-enabled.expected.md index eab9dd48e1..7298240404 100644 --- a/apps/web/tests/expected/plugin-manager/live-enabled.expected.md +++ b/apps/web/tests/expected/plugin-manager/live-enabled.expected.md @@ -38,6 +38,10 @@ - button "查看 开发者工具": 开发者工具 - text: 实验性 查看调试会话原始数据、聊天消息分组数据，以及调试 NodeJS 后端 - switch "启用 开发者工具" + - listitem: + - button "查看 Computer Use": Computer Use + - text: 实验性 目前仅支持 macOS，Windows 支持正在开发中。为已有及新建 PTC 会话加载 Computer Use。请切换到 PTC 模式来使用，并授予辅助功能和屏幕录制权限。 + - switch "启用 Computer Use" - heading "已安装插件" [level=3] - list: - listitem: diff --git a/apps/web/tests/expected/plugin-manager/manager.expected.md b/apps/web/tests/expected/plugin-manager/manager.expected.md index f3dd4a7bc6..cb09a9e29e 100644 --- a/apps/web/tests/expected/plugin-manager/manager.expected.md +++ b/apps/web/tests/expected/plugin-manager/manager.expected.md @@ -38,6 +38,10 @@ - button "查看 开发者工具": 开发者工具 - text: 实验性 查看调试会话原始数据、聊天消息分组数据，以及调试 NodeJS 后端 - switch "启用 开发者工具" + - listitem: + - button "查看 Computer Use": Computer Use + - text: 实验性 目前仅支持 macOS，Windows 支持正在开发中。为已有及新建 PTC 会话加载 Computer Use。请切换到 PTC 模式来使用，并授予辅助功能和屏幕录制权限。 + - switch "启用 Computer Use" - heading "已安装插件" [level=3] - list: - listitem: diff --git a/apps/web/tests/expected/plugin-manager/refresh.expected.md b/apps/web/tests/expected/plugin-manager/refresh.expected.md index 75dda78845..85d666fd3e 100644 --- a/apps/web/tests/expected/plugin-manager/refresh.expected.md +++ b/apps/web/tests/expected/plugin-manager/refresh.expected.md @@ -6,7 +6,7 @@ Focus at 500ms: - tooltip "刷新" failure: pending - button "刷新" [disabled] -{"busy":"true","spinners":1,"cards":11,"skeletons":0,"alerts":[],"retry":0,"toasts":[]} +{"busy":"true","spinners":1,"cards":12,"skeletons":0,"alerts":[],"retry":0,"toasts":[]} failure: settled - button "刷新" -{"busy":"false","spinners":0,"cards":11,"skeletons":0,"alerts":[],"retry":0,"toasts":["刷新失败，请重试"]} +{"busy":"false","spinners":0,"cards":12,"skeletons":0,"alerts":[],"retry":0,"toasts":["刷新失败，请重试"]} diff --git a/apps/web/tests/preview-boot.e2e.ts b/apps/web/tests/preview-boot.e2e.ts index 0e32a5a7d6..a6d1faaadb 100644 --- a/apps/web/tests/preview-boot.e2e.ts +++ b/apps/web/tests/preview-boot.e2e.ts @@ -681,15 +681,23 @@ async function toggleOptionalBundles(page: Page): Promise { } }) expect(result.optional.map(entry => entry.name).sort()).toEqual([...OPTIONAL_BUNDLES].sort()) - expect(result.optional.filter(entry => entry.enabled || entry.availability !== 'installation' || entry.error !== null)).toEqual([]) + expect(result.optional.filter(entry => entry.enabled || entry.availability !== 'installation')).toEqual([]) + expect(result.optional.filter(entry => entry.error !== null)).toEqual([ + expect.objectContaining({ name: '@deepseek-ai/dsh-cua-bundle', error: { code: 'unsupported-platform' } }), + ]) // Metadata resolves through the Worker's Node internal modules: every optional bundle shows its title and icon. expect(result.optional.filter(entry => entry.metaError !== null || entry.title !== 'object' || entry.icon !== 'string')).toEqual([]) expect(result.inventoryErrors).toEqual([]) console.info(`preview boot: optional bundle toggles ${JSON.stringify(result.toggles.map(({ name, enable, disable, error }) => ({ name, enable, disable, error })))}`) - expect(result.toggles.filter(toggle => toggle.enable !== 'applied' || toggle.disable !== 'applied') + expect(result.toggles.find(toggle => toggle.name === '@deepseek-ai/dsh-cua-bundle')).toMatchObject({ + enable: 'failed', disable: 'applied', error: { code: 'unsupported-platform' }, + changed: 0, inactive: [], restored: true, + }) + const supported = result.toggles.filter(toggle => toggle.name !== '@deepseek-ai/dsh-cua-bundle') + expect(supported.filter(toggle => toggle.enable !== 'applied' || toggle.disable !== 'applied') .map(({ name, enable, disable }) => ({ name, enable, disable }))).toEqual([]) // Each bundle changes Loader rows, every enabled row it changes runs, and switching it off restores the prior rows. - expect(result.toggles.filter(toggle => toggle.changed === 0 || toggle.inactive.length > 0 || !toggle.restored) + expect(supported.filter(toggle => toggle.changed === 0 || toggle.inactive.length > 0 || !toggle.restored) .map(({ name, changed, inactive, restored }) => ({ name, changed, inactive, restored }))).toEqual([]) const pluginsWith = (bundle: string): Array<{ moduleName: string; enabled: boolean; fiberPhase: string | null }> => result.toggles.find(toggle => toggle.name === bundle)?.plugins ?? [] diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json index d1da30db67..666db22e82 100644 --- a/apps/web/tsconfig.json +++ b/apps/web/tsconfig.json @@ -82,6 +82,9 @@ "tests/plugin-config.e2e.ts", "tests/input-plugin-extensions.e2e.ts", "tests/plugin-manager.e2e.ts", + "tests/cua-optin.e2e.ts", + "tests/bundle-platform.e2e.ts", + "tests/cua-permissions.e2e.ts", "tests/schedule-header.e2e.ts", "tests/worktree-opt-in.e2e.ts", "tests/session-inspector.e2e.ts", diff --git a/docs/capability-seams.i18n.yaml b/docs/capability-seams.i18n.yaml index 573a76f8b7..3443e528f0 100644 --- a/docs/capability-seams.i18n.yaml +++ b/docs/capability-seams.i18n.yaml @@ -6,5 +6,5 @@ en: 41d772192075e133 zh: 5ef3da69b4de1d8a /capability-seams-and-core-services: - en: f209321ab0da4f90 - zh: 4af5c135eec51ce1 + en: b360360db9a3f6a4 + zh: d14808d61000bdff diff --git a/docs/capability-seams.md b/docs/capability-seams.md index 865162e7df..93b22913dc 100644 --- a/docs/capability-seams.md +++ b/docs/capability-seams.md @@ -52,6 +52,9 @@ flowchart LR svc_computerUse["ctx.computerUse
Computer-use provider registration"] pkg_experimental_computer_use_cua_driver_mcp["experimental-computer-use-cua-driver-mcp"] pkg_experimental_computer_use_cua_driver_native["experimental-computer-use-cua-driver-native"] + pkg_cua["cua"] + svc_cuaAccess["ctx.cuaAccess
Computer Use permission controls"] + pkg_client_ui_cua["client-ui-cua"] pkg_office_to_pdf["office-to-pdf"] svc_officeToPdf["ctx.officeToPdf
Office to PDF conversion"] pkg_client_ui_sidebar_documentpreview["client-ui-sidebar-documentpreview"] @@ -334,6 +337,7 @@ flowchart LR pkg_cordis_host_runner --> svc_dynamicCordisRunner pkg_credentials --> svc_credentials pkg_credentials_local --> svc_credentials + pkg_cua --> svc_cuaAccess pkg_deepseek_account --> svc_deepseekAccount pkg_deepseek_account_platform --> svc_deepseekAccount pkg_deepseek_llm_api_extensions --> svc_deepseekLlmApiExtensions @@ -484,6 +488,7 @@ flowchart LR svc_credentials --> pkg_api_settings_controller svc_credentials --> pkg_llm_deepseek svc_credentials --> pkg_llm_pi_ai + svc_cuaAccess --> pkg_client_ui_cua svc_deepseekAccount --> pkg_api_account_controller svc_deepseekAccount --> pkg_llm_deepseek svc_deepseekLlmApiExtensions --> pkg_llm_deepseek @@ -628,6 +633,7 @@ flowchart LR | `ctx.mcpResources` | `seam` | [`mcp-resources`](../packages/mcp/mcp-resources) | [`mcp-client`](../packages/mcp/mcp-client) | [`mcp-resources`](../packages/mcp/mcp-resources) | - | Connection-owned providers serve shared resource tools in the calling agent scope. | | `ctx.browserUse` | `seam` | [`browser-use`](../packages/browser-use/browser-use) | [`experimental-browser-use-playwright-mcp`](../packages/experimental/browser-use-playwright-mcp), [`experimental-browser-use-chrome-devtools-mcp`](../packages/experimental/browser-use-chrome-devtools-mcp), [`experimental-browser-use-stagehand-native`](../packages/experimental/browser-use-stagehand-native) | [`experimental-browser-use-playwright-mcp`](../packages/experimental/browser-use-playwright-mcp), [`experimental-browser-use-chrome-devtools-mcp`](../packages/experimental/browser-use-chrome-devtools-mcp), [`experimental-browser-use-stagehand-native`](../packages/experimental/browser-use-stagehand-native) | - | One provider-owned name per service instance. Providers own their tools and browser resources per live Session; the shared service has no browser operation API. | | `ctx.computerUse` | `seam` | [`computer-use`](../packages/computer-use/computer-use) | [`experimental-computer-use-cua-driver-mcp`](../packages/experimental/computer-use-cua-driver-mcp), [`experimental-computer-use-cua-driver-native`](../packages/experimental/computer-use-cua-driver-native) | [`experimental-computer-use-cua-driver-mcp`](../packages/experimental/computer-use-cua-driver-mcp), [`experimental-computer-use-cua-driver-native`](../packages/experimental/computer-use-cua-driver-native) | - | One provider-owned name per service instance. Each provider also owns its model tools; the service has no common action API, runtime selection, or Session workflow lock. | +| `ctx.cuaAccess` | `core` | [`cua`](../packages/experimental/cua) | - | [`client-ui-cua`](../packages/experimental/client-ui-cua) | - | The native provider owns application consent and cancellation; the optional Client page queries and changes those decisions through its Remote. | | `ctx.officeToPdf` | `core` | [`office-to-pdf`](../packages/document/office-to-pdf) | - | [`client-ui-sidebar-documentpreview`](../packages/client/ui-sidebar-documentpreview) | - | Authorized Office bytes are converted on the Host using the declared native target engine, or Node WASM when no native target is declared. | | `ctx.attachments` | `seam` | [`attachment`](../packages/attachment/attachment) | [`attachment-local`](../packages/attachment/attachment-local) | [`api-session-controller`](../packages/api/session-controller), [`tool-fs`](../packages/fs/tool-fs), [`llm-pi-ai`](../packages/llm/llm-pi-ai), [`llm-deepseek`](../packages/llm/llm-deepseek) | - | The host commits accepted images before session events; provider adapters resolve authorized durable references into provider-native content. | | `ctx.fileUploads` | `core` | [`client-file-upload`](../packages/client/file-upload) | - | [`api-session-controller`](../packages/api/session-controller) | - | Owns streaming intake, durable storage, and staged receipt lifetime; the Session controller binds receipts to accepted submissions. | diff --git a/docs/capability-seams.zh.md b/docs/capability-seams.zh.md index 395986ab96..fb70a985bf 100644 --- a/docs/capability-seams.zh.md +++ b/docs/capability-seams.zh.md @@ -54,6 +54,9 @@ flowchart LR svc_computerUse["ctx.computerUse
Computer-use provider registration"] pkg_experimental_computer_use_cua_driver_mcp["experimental-computer-use-cua-driver-mcp"] pkg_experimental_computer_use_cua_driver_native["experimental-computer-use-cua-driver-native"] + pkg_cua["cua"] + svc_cuaAccess["ctx.cuaAccess
Computer Use permission controls"] + pkg_client_ui_cua["client-ui-cua"] pkg_office_to_pdf["office-to-pdf"] svc_officeToPdf["ctx.officeToPdf
Office to PDF conversion"] pkg_client_ui_sidebar_documentpreview["client-ui-sidebar-documentpreview"] @@ -336,6 +339,7 @@ flowchart LR pkg_cordis_host_runner --> svc_dynamicCordisRunner pkg_credentials --> svc_credentials pkg_credentials_local --> svc_credentials + pkg_cua --> svc_cuaAccess pkg_deepseek_account --> svc_deepseekAccount pkg_deepseek_account_platform --> svc_deepseekAccount pkg_deepseek_llm_api_extensions --> svc_deepseekLlmApiExtensions @@ -486,6 +490,7 @@ flowchart LR svc_credentials --> pkg_api_settings_controller svc_credentials --> pkg_llm_deepseek svc_credentials --> pkg_llm_pi_ai + svc_cuaAccess --> pkg_client_ui_cua svc_deepseekAccount --> pkg_api_account_controller svc_deepseekAccount --> pkg_llm_deepseek svc_deepseekLlmApiExtensions --> pkg_llm_deepseek @@ -630,6 +635,7 @@ flowchart LR | `ctx.mcpResources` | `seam` | [`mcp-resources`](../packages/mcp/mcp-resources) | [`mcp-client`](../packages/mcp/mcp-client) | [`mcp-resources`](../packages/mcp/mcp-resources) | - | 连接所有者提供的操作在调用 agent 的作用域内服务于共享资源工具。 | | `ctx.browserUse` | `seam` | [`browser-use`](../packages/browser-use/browser-use) | [`experimental-browser-use-playwright-mcp`](../packages/experimental/browser-use-playwright-mcp), [`experimental-browser-use-chrome-devtools-mcp`](../packages/experimental/browser-use-chrome-devtools-mcp), [`experimental-browser-use-stagehand-native`](../packages/experimental/browser-use-stagehand-native) | [`experimental-browser-use-playwright-mcp`](../packages/experimental/browser-use-playwright-mcp), [`experimental-browser-use-chrome-devtools-mcp`](../packages/experimental/browser-use-chrome-devtools-mcp), [`experimental-browser-use-stagehand-native`](../packages/experimental/browser-use-stagehand-native) | - | 每个服务实例注册一个提供方拥有的名称。提供方按实时 Session 拥有自己的工具与浏览器资源；共享服务不提供浏览器操作 API。 | | `ctx.computerUse` | `seam` | [`computer-use`](../packages/computer-use/computer-use) | [`experimental-computer-use-cua-driver-mcp`](../packages/experimental/computer-use-cua-driver-mcp), [`experimental-computer-use-cua-driver-native`](../packages/experimental/computer-use-cua-driver-native) | [`experimental-computer-use-cua-driver-mcp`](../packages/experimental/computer-use-cua-driver-mcp), [`experimental-computer-use-cua-driver-native`](../packages/experimental/computer-use-cua-driver-native) | - | 每个服务实例只注册一个提供方自定的名称。各提供方也拥有自己的模型工具；服务不提供通用操作 API、运行时选择或 Session 流程锁。 | +| `ctx.cuaAccess` | `core` | [`cua`](../packages/experimental/cua) | - | [`client-ui-cua`](../packages/experimental/client-ui-cua) | - | 原生 provider 负责应用授权和取消操作；可选的 Client 权限页面通过 Remote 查询和修改授权。 | | `ctx.officeToPdf` | `core` | [`office-to-pdf`](../packages/document/office-to-pdf) | - | [`client-ui-sidebar-documentpreview`](../packages/client/ui-sidebar-documentpreview) | - | 已授权的 Office 字节在宿主上使用已声明的原生目标引擎转换；未声明原生目标时使用 Node WASM。 | | `ctx.attachments` | `seam` | [`attachment`](../packages/attachment/attachment) | [`attachment-local`](../packages/attachment/attachment-local) | [`api-session-controller`](../packages/api/session-controller), [`tool-fs`](../packages/fs/tool-fs), [`llm-pi-ai`](../packages/llm/llm-pi-ai), [`llm-deepseek`](../packages/llm/llm-deepseek) | - | 宿主会在会话事件之前提交已接受的图片；提供方适配器将已授权的持久引用解析为提供方原生内容。 | | `ctx.fileUploads` | `core` | [`client-file-upload`](../packages/client/file-upload) | - | [`api-session-controller`](../packages/api/session-controller) | - | 负责流式接收、持久存储和暂存回执生命周期；Session Controller 将回执绑定到已接受的提交。 | diff --git a/docs/config-catalog.md b/docs/config-catalog.md index be5681c61e..da39766988 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -2482,7 +2482,7 @@ export interface PlanModeConfig { ## `@deepseek-ai/dsh-plugin-manager` - `inject`: `loader` · `profileContext` -- `source`: [`packages/boot/plugin-manager/src/index.ts:42`](../packages/boot/plugin-manager/src/index.ts) +- `source`: [`packages/boot/plugin-manager/src/index.ts:43`](../packages/boot/plugin-manager/src/index.ts) ```ts config-catalog /** The pnpm executable, registries, and limits for diagnostics, lookups and connection checks. */ @@ -4684,6 +4684,7 @@ These load from a `cordis.yml` entry with no `config:` block; they declare no co | `@deepseek-ai/dsh-client-ui-commands` | — | [`packages/client/ui-commands/src/index.ts`](../packages/client/ui-commands/src/index.ts) | | `@deepseek-ai/dsh-client-ui-conversation` | — | [`packages/client/ui-conversation/src/index.ts`](../packages/client/ui-conversation/src/index.ts) | | `@deepseek-ai/dsh-client-ui-cordis` | — | [`packages/extensions/ui-cordis/src/index.ts`](../packages/extensions/ui-cordis/src/index.ts) | +| `@deepseek-ai/dsh-client-ui-cua` | — | [`packages/experimental/client-ui-cua/src/index.ts`](../packages/experimental/client-ui-cua/src/index.ts) | | `@deepseek-ai/dsh-client-ui-deepseek-model-label-1017` | — | [`packages/client/ui-deepseek-model-label-1017/src/index.ts`](../packages/client/ui-deepseek-model-label-1017/src/index.ts) | | `@deepseek-ai/dsh-client-ui-deliverables` | `systemPrompt` · `connection` · `sessionQuery` · `sessionController` · `workspaceFiles` · `fs` · `sandboxPolicy` · `workspaceChanges` | [`packages/client/ui-deliverables/src/index.ts`](../packages/client/ui-deliverables/src/index.ts) | | `@deepseek-ai/dsh-client-ui-directory-picker-browse` | — | [`packages/client/ui-directory-picker-browse/src/index.ts`](../packages/client/ui-directory-picker-browse/src/index.ts) | @@ -4821,6 +4822,7 @@ Imported as libraries by other packages; a `cordis.yml` cannot load them. | `@deepseek-ai/dsh-client-web` | — | [`packages/client/web/src/index.ts`](../packages/client/web/src/index.ts) | | `@deepseek-ai/dsh-cmdline` | — | [`packages/boot/cmdline/src/index.ts`](../packages/boot/cmdline/src/index.ts) | | `@deepseek-ai/dsh-cua` | — | [`packages/experimental/cua/src/index.ts`](../packages/experimental/cua/src/index.ts) | +| `@deepseek-ai/dsh-cua-bundle` | — | [`packages/experimental/cua-bundle/src/index.ts`](../packages/experimental/cua-bundle/src/index.ts) | | `@deepseek-ai/dsh-deque` | — | [`packages/util/deque/src/index.ts`](../packages/util/deque/src/index.ts) | | `@deepseek-ai/dsh-experimental-agent-team-profile` | — | [`packages/experimental/agent-team-profile/src/index.ts`](../packages/experimental/agent-team-profile/src/index.ts) | | `@deepseek-ai/dsh-experimental-badge-skill-bundle` | — | [`packages/experimental/badge-skill-bundle/src/index.ts`](../packages/experimental/badge-skill-bundle/src/index.ts) | diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index 61b506ec56..df608d7977 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -2484,7 +2484,7 @@ export interface PlanModeConfig { ## `@deepseek-ai/dsh-plugin-manager` - `inject`: `loader` · `profileContext` -- `source`: [`packages/boot/plugin-manager/src/index.ts:42`](../packages/boot/plugin-manager/src/index.ts) +- `source`: [`packages/boot/plugin-manager/src/index.ts:43`](../packages/boot/plugin-manager/src/index.ts) ```ts config-catalog /** The pnpm executable, registries, and limits for diagnostics, lookups and connection checks. */ @@ -4686,6 +4686,7 @@ export interface Config { | `@deepseek-ai/dsh-client-ui-commands` | — | [`packages/client/ui-commands/src/index.ts`](../packages/client/ui-commands/src/index.ts) | | `@deepseek-ai/dsh-client-ui-conversation` | — | [`packages/client/ui-conversation/src/index.ts`](../packages/client/ui-conversation/src/index.ts) | | `@deepseek-ai/dsh-client-ui-cordis` | — | [`packages/extensions/ui-cordis/src/index.ts`](../packages/extensions/ui-cordis/src/index.ts) | +| `@deepseek-ai/dsh-client-ui-cua` | — | [`packages/experimental/client-ui-cua/src/index.ts`](../packages/experimental/client-ui-cua/src/index.ts) | | `@deepseek-ai/dsh-client-ui-deepseek-model-label-1017` | — | [`packages/client/ui-deepseek-model-label-1017/src/index.ts`](../packages/client/ui-deepseek-model-label-1017/src/index.ts) | | `@deepseek-ai/dsh-client-ui-deliverables` | `systemPrompt` · `connection` · `sessionQuery` · `sessionController` · `workspaceFiles` · `fs` · `sandboxPolicy` · `workspaceChanges` | [`packages/client/ui-deliverables/src/index.ts`](../packages/client/ui-deliverables/src/index.ts) | | `@deepseek-ai/dsh-client-ui-directory-picker-browse` | — | [`packages/client/ui-directory-picker-browse/src/index.ts`](../packages/client/ui-directory-picker-browse/src/index.ts) | @@ -4823,6 +4824,7 @@ export interface Config { | `@deepseek-ai/dsh-client-web` | — | [`packages/client/web/src/index.ts`](../packages/client/web/src/index.ts) | | `@deepseek-ai/dsh-cmdline` | — | [`packages/boot/cmdline/src/index.ts`](../packages/boot/cmdline/src/index.ts) | | `@deepseek-ai/dsh-cua` | — | [`packages/experimental/cua/src/index.ts`](../packages/experimental/cua/src/index.ts) | +| `@deepseek-ai/dsh-cua-bundle` | — | [`packages/experimental/cua-bundle/src/index.ts`](../packages/experimental/cua-bundle/src/index.ts) | | `@deepseek-ai/dsh-deque` | — | [`packages/util/deque/src/index.ts`](../packages/util/deque/src/index.ts) | | `@deepseek-ai/dsh-experimental-agent-team-profile` | — | [`packages/experimental/agent-team-profile/src/index.ts`](../packages/experimental/agent-team-profile/src/index.ts) | | `@deepseek-ai/dsh-experimental-badge-skill-bundle` | — | [`packages/experimental/badge-skill-bundle/src/index.ts`](../packages/experimental/badge-skill-bundle/src/index.ts) | diff --git a/docs/event-producer-consumer.md b/docs/event-producer-consumer.md index 277786cc82..fe621249d8 100644 --- a/docs/event-producer-consumer.md +++ b/docs/event-producer-consumer.md @@ -9,9 +9,9 @@ This matrix shows which packages dispatch each harness-owned event and which pac | Event | Mode | Declared in | Dispatchers | Listeners | | --- | --- | --- | --- | --- | | `agent-loop/config-start-failed` | `emit` | [`packages/core/agent-loop/src/index.ts:249`](../packages/core/agent-loop/src/index.ts) | [`agent-loop`](../packages/core/agent-loop) (`events.dispatch`) | - | -| `agent-preset/selected` | `emit` | [`packages/preset/agent-preset-registry/src/types.ts:69`](../packages/preset/agent-preset-registry/src/types.ts) | [`agent-preset-registry`](../packages/preset/agent-preset-registry) (`emit`) | `remotes` | +| `agent-preset/selected` | `emit` | [`packages/preset/agent-preset-registry/src/types.ts:69`](../packages/preset/agent-preset-registry/src/types.ts) | [`agent-preset-registry`](../packages/preset/agent-preset-registry) (`emit`) | [`cua`](../packages/experimental/cua), `remotes` | | `agent/assistant-stream` | `emit` | [`packages/core/agent/src/runtime-types.ts:363`](../packages/core/agent/src/runtime-types.ts) | [`agent-loop`](../packages/core/agent-loop) (`emit`) | [`headless`](../packages/bundle/headless), `session-controller` | -| `agent/created` | `serial` | [`packages/core/agent/src/runtime-types.ts:261`](../packages/core/agent/src/runtime-types.ts) | [`agent`](../packages/core/agent) (`serial`) | `agent-team`, `browser-use-runtime`, `claude-code-mods`, [`file-reference-local`](../packages/context/file-reference-local), [`goal`](../packages/goal/goal), [`goal-round-driver`](../packages/goal/goal-round-driver), [`hooks-claude-code`](../packages/experimental/hooks-claude-code), [`hooks-codex`](../packages/experimental/hooks-codex), [`loader-smoke`](../packages/test-support/loader-smoke), `session-controller`, `tool-agent-team`, [`tool-subagent`](../packages/subagent/tool-subagent) | +| `agent/created` | `serial` | [`packages/core/agent/src/runtime-types.ts:261`](../packages/core/agent/src/runtime-types.ts) | [`agent`](../packages/core/agent) (`serial`) | `agent-team`, `browser-use-runtime`, `claude-code-mods`, [`cua`](../packages/experimental/cua), [`file-reference-local`](../packages/context/file-reference-local), [`goal`](../packages/goal/goal), [`goal-round-driver`](../packages/goal/goal-round-driver), [`hooks-claude-code`](../packages/experimental/hooks-claude-code), [`hooks-codex`](../packages/experimental/hooks-codex), [`loader-smoke`](../packages/test-support/loader-smoke), `session-controller`, `tool-agent-team`, [`tool-subagent`](../packages/subagent/tool-subagent) | | `agent/disposed` | `emit` | [`packages/core/agent/src/runtime-types.ts:270`](../packages/core/agent/src/runtime-types.ts) | [`agent`](../packages/core/agent) (`events.dispatch`) | [`agent-loop`](../packages/core/agent-loop), `claude-code-mods`, [`file-reference-local`](../packages/context/file-reference-local), [`goal-round-driver`](../packages/goal/goal-round-driver), `session-controller`, [`subagent`](../packages/subagent/subagent), `tool-agent-team`, [`tool-subagent`](../packages/subagent/tool-subagent) | | `agent/error` | `emit` | [`packages/core/agent/src/runtime-types.ts:393`](../packages/core/agent/src/runtime-types.ts) | [`agent-loop`](../packages/core/agent-loop) (`emit`) | [`acp`](../packages/acp/acp), [`deepseek-account`](../packages/credentials/deepseek-account), [`goal-round-driver`](../packages/goal/goal-round-driver), [`headless`](../packages/bundle/headless), `server`, `session-controller`, [`session-telemetry`](../packages/session/session-telemetry), [`subagent`](../packages/subagent/subagent) | | `agent/inbox/claimed` | `emit` | [`packages/core/agent/src/runtime-types.ts:299`](../packages/core/agent/src/runtime-types.ts) | [`agent-loop`](../packages/core/agent-loop) (`emit`) | [`acp`](../packages/acp/acp), [`goal-round-driver`](../packages/goal/goal-round-driver), [`subagent`](../packages/subagent/subagent), [`tool-jobs`](../packages/jobs/tool-jobs), [`user-questions`](../packages/interaction/user-questions) | diff --git a/docs/event-producer-consumer.zh.md b/docs/event-producer-consumer.zh.md index 79a63c476c..ba3d8b06f8 100644 --- a/docs/event-producer-consumer.zh.md +++ b/docs/event-producer-consumer.zh.md @@ -11,9 +11,9 @@ | Event | Mode | Declared in | Dispatchers | Listeners | | --- | --- | --- | --- | --- | | `agent-loop/config-start-failed` | `emit` | [`packages/core/agent-loop/src/index.ts:249`](../packages/core/agent-loop/src/index.ts) | [`agent-loop`](../packages/core/agent-loop) (`events.dispatch`) | - | -| `agent-preset/selected` | `emit` | [`packages/preset/agent-preset-registry/src/types.ts:69`](../packages/preset/agent-preset-registry/src/types.ts) | [`agent-preset-registry`](../packages/preset/agent-preset-registry) (`emit`) | `remotes` | +| `agent-preset/selected` | `emit` | [`packages/preset/agent-preset-registry/src/types.ts:69`](../packages/preset/agent-preset-registry/src/types.ts) | [`agent-preset-registry`](../packages/preset/agent-preset-registry) (`emit`) | [`cua`](../packages/experimental/cua), `remotes` | | `agent/assistant-stream` | `emit` | [`packages/core/agent/src/runtime-types.ts:363`](../packages/core/agent/src/runtime-types.ts) | [`agent-loop`](../packages/core/agent-loop) (`emit`) | [`headless`](../packages/bundle/headless), `session-controller` | -| `agent/created` | `serial` | [`packages/core/agent/src/runtime-types.ts:261`](../packages/core/agent/src/runtime-types.ts) | [`agent`](../packages/core/agent) (`serial`) | `agent-team`, `browser-use-runtime`, `claude-code-mods`, [`file-reference-local`](../packages/context/file-reference-local), [`goal`](../packages/goal/goal), [`goal-round-driver`](../packages/goal/goal-round-driver), [`hooks-claude-code`](../packages/experimental/hooks-claude-code), [`hooks-codex`](../packages/experimental/hooks-codex), [`loader-smoke`](../packages/test-support/loader-smoke), `session-controller`, `tool-agent-team`, [`tool-subagent`](../packages/subagent/tool-subagent) | +| `agent/created` | `serial` | [`packages/core/agent/src/runtime-types.ts:261`](../packages/core/agent/src/runtime-types.ts) | [`agent`](../packages/core/agent) (`serial`) | `agent-team`, `browser-use-runtime`, `claude-code-mods`, [`cua`](../packages/experimental/cua), [`file-reference-local`](../packages/context/file-reference-local), [`goal`](../packages/goal/goal), [`goal-round-driver`](../packages/goal/goal-round-driver), [`hooks-claude-code`](../packages/experimental/hooks-claude-code), [`hooks-codex`](../packages/experimental/hooks-codex), [`loader-smoke`](../packages/test-support/loader-smoke), `session-controller`, `tool-agent-team`, [`tool-subagent`](../packages/subagent/tool-subagent) | | `agent/disposed` | `emit` | [`packages/core/agent/src/runtime-types.ts:270`](../packages/core/agent/src/runtime-types.ts) | [`agent`](../packages/core/agent) (`events.dispatch`) | [`agent-loop`](../packages/core/agent-loop), `claude-code-mods`, [`file-reference-local`](../packages/context/file-reference-local), [`goal-round-driver`](../packages/goal/goal-round-driver), `session-controller`, [`subagent`](../packages/subagent/subagent), `tool-agent-team`, [`tool-subagent`](../packages/subagent/tool-subagent) | | `agent/error` | `emit` | [`packages/core/agent/src/runtime-types.ts:393`](../packages/core/agent/src/runtime-types.ts) | [`agent-loop`](../packages/core/agent-loop) (`emit`) | [`acp`](../packages/acp/acp), [`deepseek-account`](../packages/credentials/deepseek-account), [`goal-round-driver`](../packages/goal/goal-round-driver), [`headless`](../packages/bundle/headless), `server`, `session-controller`, [`session-telemetry`](../packages/session/session-telemetry), [`subagent`](../packages/subagent/subagent) | | `agent/inbox/claimed` | `emit` | [`packages/core/agent/src/runtime-types.ts:299`](../packages/core/agent/src/runtime-types.ts) | [`agent-loop`](../packages/core/agent-loop) (`emit`) | [`acp`](../packages/acp/acp), [`goal-round-driver`](../packages/goal/goal-round-driver), [`subagent`](../packages/subagent/subagent), [`tool-jobs`](../packages/jobs/tool-jobs), [`user-questions`](../packages/interaction/user-questions) | diff --git a/docs/module-graph.md b/docs/module-graph.md index eb1afc4e0f..39941991c8 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -246,7 +246,9 @@ flowchart TD pkg_office_to_pdf["office-to-pdf"] end subgraph group_experimental["packages/experimental"] + pkg_client_ui_cua["client-ui-cua"] pkg_cua["cua"] + pkg_cua_bundle["cua-bundle"] pkg_experimental_agent_team["experimental-agent-team"] pkg_experimental_agent_team_profile["experimental-agent-team-profile"] pkg_experimental_api_speech_to_text["experimental-api-speech-to-text"] @@ -905,14 +907,6 @@ flowchart TD pkg_workspace_changes --> pkg_session pkg_workspace_changes --> pkg_subprocess pkg_workspace_changes --> pkg_tools - pkg_cua --> pkg_agent - pkg_cua --> pkg_attachment - pkg_cua --> pkg_computer_use - pkg_cua --> pkg_llm - pkg_cua --> pkg_subprocess - pkg_cua --> pkg_system_prompt - pkg_cua --> pkg_tools - pkg_cua --> pkg_user_questions pkg_experimental_browser_use_chrome_devtools_mcp --> pkg_agent pkg_experimental_browser_use_chrome_devtools_mcp --> pkg_browser_use pkg_experimental_browser_use_chrome_devtools_mcp --> pkg_system_prompt @@ -1076,6 +1070,16 @@ flowchart TD pkg_api_settings_controller --> pkg_settings pkg_api_settings_controller --> pkg_typert_protocol pkg_web_app --> pkg_shell_env + pkg_cua --> pkg_agent + pkg_cua --> pkg_agent_preset_registry + pkg_cua --> pkg_attachment + pkg_cua --> pkg_computer_use + pkg_cua --> pkg_llm + pkg_cua --> pkg_scope + pkg_cua --> pkg_subprocess + pkg_cua --> pkg_system_prompt + pkg_cua --> pkg_tools + pkg_cua --> pkg_user_questions pkg_experimental_auto_review --> pkg_agent pkg_experimental_auto_review --> pkg_agent_instructions pkg_experimental_auto_review --> pkg_llm @@ -1540,6 +1544,8 @@ flowchart TD | [`client-web`](../packages/client/web) | `client` | — | | [`credentials`](../packages/credentials/credentials) | `credentials` | — | | [`office-to-pdf`](../packages/document/office-to-pdf) | `document` | — | +| [`client-ui-cua`](../packages/experimental/client-ui-cua) | `experimental` | — | +| [`cua-bundle`](../packages/experimental/cua-bundle) | `experimental` | — | | [`experimental-agent-team-profile`](../packages/experimental/agent-team-profile) | `experimental` | — | | [`experimental-badge-skill-bundle`](../packages/experimental/badge-skill-bundle) | `experimental` | — | | [`experimental-client-ui-agent-team`](../packages/experimental/client-ui-agent-team) | `experimental` | — | @@ -1704,7 +1710,6 @@ flowchart TD | [`file-reference-local`](../packages/context/file-reference-local) | `context` | [`agent`](../packages/core/agent), [`file-reference`](../packages/context/file-reference), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) | | [`tool-present`](../packages/deliverables/tool-present) | `deliverables` | [`agent`](../packages/core/agent), [`fs`](../packages/fs/fs), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-projection`](../packages/session/session-projection), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) | | [`workspace-changes`](../packages/deliverables/workspace-changes) | `deliverables` | [`agent`](../packages/core/agent), [`fs`](../packages/fs/fs), [`session`](../packages/core/session), [`subprocess`](../packages/subprocess/subprocess), [`tools`](../packages/core/tools) | -| [`cua`](../packages/experimental/cua) | `experimental` | [`agent`](../packages/core/agent), [`attachment`](../packages/attachment/attachment), [`computer-use`](../packages/computer-use/computer-use), [`llm`](../packages/llm/llm), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-questions`](../packages/interaction/user-questions) | | [`experimental-browser-use-chrome-devtools-mcp`](../packages/experimental/browser-use-chrome-devtools-mcp) | `experimental` | [`agent`](../packages/core/agent), [`browser-use`](../packages/browser-use/browser-use), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) | | [`experimental-browser-use-playwright-mcp`](../packages/experimental/browser-use-playwright-mcp) | `experimental` | [`agent`](../packages/core/agent), [`browser-use`](../packages/browser-use/browser-use), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) | | [`experimental-browser-use-stagehand-native`](../packages/experimental/browser-use-stagehand-native) | `experimental` | [`agent`](../packages/core/agent), [`browser-use`](../packages/browser-use/browser-use), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) | @@ -1737,6 +1742,7 @@ flowchart TD | [`session-query`](../packages/session-query/session-query) | `session-query` | [`brand`](../packages/util/brand), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-persistence`](../packages/session/session-persistence), [`session-projection`](../packages/session/session-projection), [`session-projection-cache`](../packages/session/session-projection-cache), [`session-title`](../packages/session/session-title), [`tool-todo`](../packages/todo/tool-todo) | | [`api-settings-controller`](../packages/api/settings-controller) | `api` | [`agent-preset-registry`](../packages/preset/agent-preset-registry), [`credentials`](../packages/credentials/credentials), [`native-command`](../packages/util/native-command), [`session`](../packages/core/session), [`settings`](../packages/settings/settings), [`typert-protocol`](../packages/typert/protocol) | | [`web-app`](../packages/bundle/web-app) | `bundle` | [`shell-env`](../packages/shell/shell-env) | +| [`cua`](../packages/experimental/cua) | `experimental` | [`agent`](../packages/core/agent), [`agent-preset-registry`](../packages/preset/agent-preset-registry), [`attachment`](../packages/attachment/attachment), [`computer-use`](../packages/computer-use/computer-use), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-questions`](../packages/interaction/user-questions) | | [`experimental-auto-review`](../packages/experimental/auto-review) | `experimental` | [`agent`](../packages/core/agent), [`agent-instructions`](../packages/context/agent-instructions), [`llm`](../packages/llm/llm), [`permission-presets`](../packages/interaction/permission-presets), [`session`](../packages/core/session), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) | | [`webhook`](../packages/experimental/webhook) | `experimental` | [`agent`](../packages/core/agent), [`agent-default-model`](../packages/core/agent-default-model), [`agent-preset-registry`](../packages/preset/agent-preset-registry), [`llm`](../packages/llm/llm), [`permission-presets`](../packages/interaction/permission-presets), [`session`](../packages/core/session), [`session-title`](../packages/session/session-title), [`workspace`](../packages/workspace/workspace) | | [`tool-cordis`](../packages/extensions/tool-cordis) | `extensions` | [`agent`](../packages/core/agent), [`app-boot`](../packages/boot/app-boot), [`cordis-host-runner`](../packages/extensions/cordis-host-runner), [`tools`](../packages/core/tools) | diff --git a/docs/module-graph.zh.md b/docs/module-graph.zh.md index 04677f718a..97e302bb3d 100644 --- a/docs/module-graph.zh.md +++ b/docs/module-graph.zh.md @@ -248,7 +248,9 @@ flowchart TD pkg_office_to_pdf["office-to-pdf"] end subgraph group_experimental["packages/experimental"] + pkg_client_ui_cua["client-ui-cua"] pkg_cua["cua"] + pkg_cua_bundle["cua-bundle"] pkg_experimental_agent_team["experimental-agent-team"] pkg_experimental_agent_team_profile["experimental-agent-team-profile"] pkg_experimental_api_speech_to_text["experimental-api-speech-to-text"] @@ -907,14 +909,6 @@ flowchart TD pkg_workspace_changes --> pkg_session pkg_workspace_changes --> pkg_subprocess pkg_workspace_changes --> pkg_tools - pkg_cua --> pkg_agent - pkg_cua --> pkg_attachment - pkg_cua --> pkg_computer_use - pkg_cua --> pkg_llm - pkg_cua --> pkg_subprocess - pkg_cua --> pkg_system_prompt - pkg_cua --> pkg_tools - pkg_cua --> pkg_user_questions pkg_experimental_browser_use_chrome_devtools_mcp --> pkg_agent pkg_experimental_browser_use_chrome_devtools_mcp --> pkg_browser_use pkg_experimental_browser_use_chrome_devtools_mcp --> pkg_system_prompt @@ -1078,6 +1072,16 @@ flowchart TD pkg_api_settings_controller --> pkg_settings pkg_api_settings_controller --> pkg_typert_protocol pkg_web_app --> pkg_shell_env + pkg_cua --> pkg_agent + pkg_cua --> pkg_agent_preset_registry + pkg_cua --> pkg_attachment + pkg_cua --> pkg_computer_use + pkg_cua --> pkg_llm + pkg_cua --> pkg_scope + pkg_cua --> pkg_subprocess + pkg_cua --> pkg_system_prompt + pkg_cua --> pkg_tools + pkg_cua --> pkg_user_questions pkg_experimental_auto_review --> pkg_agent pkg_experimental_auto_review --> pkg_agent_instructions pkg_experimental_auto_review --> pkg_llm @@ -1542,6 +1546,8 @@ flowchart TD | [`client-web`](../packages/client/web) | `client` | — | | [`credentials`](../packages/credentials/credentials) | `credentials` | — | | [`office-to-pdf`](../packages/document/office-to-pdf) | `document` | — | +| [`client-ui-cua`](../packages/experimental/client-ui-cua) | `experimental` | — | +| [`cua-bundle`](../packages/experimental/cua-bundle) | `experimental` | — | | [`experimental-agent-team-profile`](../packages/experimental/agent-team-profile) | `experimental` | — | | [`experimental-badge-skill-bundle`](../packages/experimental/badge-skill-bundle) | `experimental` | — | | [`experimental-client-ui-agent-team`](../packages/experimental/client-ui-agent-team) | `experimental` | — | @@ -1706,7 +1712,6 @@ flowchart TD | [`file-reference-local`](../packages/context/file-reference-local) | `context` | [`agent`](../packages/core/agent), [`file-reference`](../packages/context/file-reference), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) | | [`tool-present`](../packages/deliverables/tool-present) | `deliverables` | [`agent`](../packages/core/agent), [`fs`](../packages/fs/fs), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-projection`](../packages/session/session-projection), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) | | [`workspace-changes`](../packages/deliverables/workspace-changes) | `deliverables` | [`agent`](../packages/core/agent), [`fs`](../packages/fs/fs), [`session`](../packages/core/session), [`subprocess`](../packages/subprocess/subprocess), [`tools`](../packages/core/tools) | -| [`cua`](../packages/experimental/cua) | `experimental` | [`agent`](../packages/core/agent), [`attachment`](../packages/attachment/attachment), [`computer-use`](../packages/computer-use/computer-use), [`llm`](../packages/llm/llm), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-questions`](../packages/interaction/user-questions) | | [`experimental-browser-use-chrome-devtools-mcp`](../packages/experimental/browser-use-chrome-devtools-mcp) | `experimental` | [`agent`](../packages/core/agent), [`browser-use`](../packages/browser-use/browser-use), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) | | [`experimental-browser-use-playwright-mcp`](../packages/experimental/browser-use-playwright-mcp) | `experimental` | [`agent`](../packages/core/agent), [`browser-use`](../packages/browser-use/browser-use), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) | | [`experimental-browser-use-stagehand-native`](../packages/experimental/browser-use-stagehand-native) | `experimental` | [`agent`](../packages/core/agent), [`browser-use`](../packages/browser-use/browser-use), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) | @@ -1739,6 +1744,7 @@ flowchart TD | [`session-query`](../packages/session-query/session-query) | `session-query` | [`brand`](../packages/util/brand), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-persistence`](../packages/session/session-persistence), [`session-projection`](../packages/session/session-projection), [`session-projection-cache`](../packages/session/session-projection-cache), [`session-title`](../packages/session/session-title), [`tool-todo`](../packages/todo/tool-todo) | | [`api-settings-controller`](../packages/api/settings-controller) | `api` | [`agent-preset-registry`](../packages/preset/agent-preset-registry), [`credentials`](../packages/credentials/credentials), [`native-command`](../packages/util/native-command), [`session`](../packages/core/session), [`settings`](../packages/settings/settings), [`typert-protocol`](../packages/typert/protocol) | | [`web-app`](../packages/bundle/web-app) | `bundle` | [`shell-env`](../packages/shell/shell-env) | +| [`cua`](../packages/experimental/cua) | `experimental` | [`agent`](../packages/core/agent), [`agent-preset-registry`](../packages/preset/agent-preset-registry), [`attachment`](../packages/attachment/attachment), [`computer-use`](../packages/computer-use/computer-use), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-questions`](../packages/interaction/user-questions) | | [`experimental-auto-review`](../packages/experimental/auto-review) | `experimental` | [`agent`](../packages/core/agent), [`agent-instructions`](../packages/context/agent-instructions), [`llm`](../packages/llm/llm), [`permission-presets`](../packages/interaction/permission-presets), [`session`](../packages/core/session), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) | | [`webhook`](../packages/experimental/webhook) | `experimental` | [`agent`](../packages/core/agent), [`agent-default-model`](../packages/core/agent-default-model), [`agent-preset-registry`](../packages/preset/agent-preset-registry), [`llm`](../packages/llm/llm), [`permission-presets`](../packages/interaction/permission-presets), [`session`](../packages/core/session), [`session-title`](../packages/session/session-title), [`workspace`](../packages/workspace/workspace) | | [`tool-cordis`](../packages/extensions/tool-cordis) | `extensions` | [`agent`](../packages/core/agent), [`app-boot`](../packages/boot/app-boot), [`cordis-host-runner`](../packages/extensions/cordis-host-runner), [`tools`](../packages/core/tools) | diff --git a/docs/subsystems/computer-use.i18n.yaml b/docs/subsystems/computer-use.i18n.yaml index a0cf7675d0..77ba44e680 100644 --- a/docs/subsystems/computer-use.i18n.yaml +++ b/docs/subsystems/computer-use.i18n.yaml @@ -3,8 +3,8 @@ # After editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/subsystems/computer-use.md /computer-use: - en: a82a6d1ec3fa9724 - zh: 17483872a54d6899 + en: 6f40d55bef2d7753 + zh: 0fc4eb79afb3cd73 /computer-use/choose-a-provider: en: fa2e0d7b41c0450b zh: af9a799711f901e6 diff --git a/docs/subsystems/computer-use.md b/docs/subsystems/computer-use.md index 6331c0f492..efff79fb54 100644 --- a/docs/subsystems/computer-use.md +++ b/docs/subsystems/computer-use.md @@ -4,9 +4,19 @@ English | [中文](computer-use.zh.md) Computer use lets a model observe and operate the local desktop through a configured provider. The shared DSH capability is called **computer use**; **Cua Driver** names the upstream implementation. +[Computer Use](../../packages/experimental/cua-bundle/README.md) provides an optional experimental switch for macOS CUA permission setup. The [`macOS CUA provider`](../../packages/experimental/cua-macos/README.md) uses the [shared CUA objects and executor](../../packages/experimental/cua/README.md), which enforce application consent through User Questions and profile configuration. +The CUA provider owns `ctx.cuaAccess` for system-permission status, explicit system requests, and application revocation. Its [permission page](../../packages/experimental/client-ui-cua/README.md) projects these controls; app/window operations enforce the provider's consent policy before dispatch. + +| Permission type | Meaning | +|---|---| +| `CuaApplicationId`, `CuaApplicationGrant` | An executable/signature identity and its displayed saved decision. | +| `CuaSystemPermission` | Accessibility or Screen Recording, selected by the human. | +| `CuaPermissionProcess`, `CuaSystemPermissionState` | Native privilege checks, observed helper/parent processes, and the application identified by macOS process responsibility at helper startup. Application metadata is absent if the optional system query or application lookup is unavailable. | +| `CuaApplicationState` | Saved grants, refused applications, and whether persistence is available. Session-only grants remain internal to the provider. System observations and application icons are queried separately so the page can display each result as it arrives. | + ## Choose a provider Mount [`dsh-computer-use`](../../packages/computer-use/computer-use/README.md) and one provider in the same composition. Both Cua Driver providers are experimental public npm packages and require explicit activation. @@ -56,4 +66,60 @@ register(name: ComputerUseProviderName): () => Promise ``` Source: [`packages/computer-use/computer-use/src/index.ts`](../../packages/computer-use/computer-use/src/index.ts) + + + +### `ctx.cuaAccess` — `CuaAccess` + +Browser controls for provider-owned native permission and application decisions. + +```ts cordis-catalog +/** + * Read application decisions without waiting for the native helper. + * @returns Saved grants and refused applications with their persistence owner. + */ +@Remote applications(): CuaApplicationState + +/** + * Inspect system privileges independently of application decisions and icons. + * @returns Fresh native preflight and launch metadata. + */ +@Remote async systemStatus(): Promise + +/** + * Load presentation icons separately from application decisions. + * @returns Optional native PNG icons keyed by executable path for saved and blocked applications. + */ +@Remote async applicationIcons(): Promise> + +/** + * Request a system privilege explicitly selected by the user. + * @param permission macOS privilege to request. + * @returns Preflight state after requesting; a false value never implies consent. + */ +@Remote async requestPermission(permission: CuaSystemPermission): Promise + +/** + * Open the macOS privacy pane selected by the user. + * @param permission Corresponding system privilege. + * @returns Completion after macOS accepted the open request. + */ +@Remote async openSettings(permission: CuaSystemPermission): Promise + +/** + * Remove saved and session access and join affected operations. + * @param id Application identity displayed on the permission page. + * @returns Completion after persistence and cancellation; dispatched input cannot be undone. + */ +@Remote revoke(id: CuaApplicationId): Promise + +/** + * Clear this application's refusals across sessions so each session may ask for access again. + * @param id Refused or revoked application displayed on the permission page. + * @returns Completion after clearing refusals; no permission is granted and no action is replayed. + */ +@Remote async retryApproval(id: CuaApplicationId): Promise +``` + +Source: [`packages/experimental/cua/src/access.ts`](../../packages/experimental/cua/src/access.ts) diff --git a/docs/subsystems/computer-use.zh.md b/docs/subsystems/computer-use.zh.md index cd4901bd1a..997bf7523c 100644 --- a/docs/subsystems/computer-use.zh.md +++ b/docs/subsystems/computer-use.zh.md @@ -4,9 +4,19 @@ 计算机操作让模型通过配置的提供方观察并操作本地桌面。DSH 的共享能力称为 **computer use（计算机操作）**；**Cua Driver** 是上游实现的名称。 +[Computer Use](../../packages/experimental/cua-bundle/README.zh.md) 提供 macOS CUA 及权限设置 的可选实验性开关。 [macOS CUA 提供方](../../packages/experimental/cua-macos/README.zh.md)使用[共享 CUA 对象与执行器](../../packages/experimental/cua/README.zh.md)，通过 User Questions 和配置档设置执行应用授权。 +CUA provider 通过 `ctx.cuaAccess` 提供系统权限状态、用户主动申请和应用授权撤销。[权限页面](../../packages/experimental/client-ui-cua/README.zh.md)投影这些控件；app/window 操作在分派前执行 provider 的授权检查。 + +| 权限类型 | 含义 | +|---|---| +| `CuaApplicationId`、`CuaApplicationGrant` | 可执行文件和签名身份，以及显示给用户的已保存决定。 | +| `CuaSystemPermission` | 用户选择申请的辅助功能或屏幕录制权限。 | +| `CuaPermissionProcess`、`CuaSystemPermissionState` | 观测到的 helper、父进程和原生权限状态，以及 helper 启动时由 macOS 责任进程记录识别出的应用。可选系统查询或应用查找不可用时，省略应用信息。 | +| `CuaApplicationState` | 持久授权、拒绝记录，以及能否持久保存。会话授权只保留在 provider 内部。系统权限和应用图标分别查询，页面在各项结果返回后分别显示。 | + ## 选择提供方 在同一组合中挂载 [`dsh-computer-use`](../../packages/computer-use/computer-use/README.zh.md) 和一个提供方。两个 Cua Driver 提供方都是公开发布到 npm 的实验性包，均需显式启用。 @@ -56,4 +66,60 @@ register(name: ComputerUseProviderName): () => Promise ``` Source: [`packages/computer-use/computer-use/src/index.ts`](../../packages/computer-use/computer-use/src/index.ts) + + + +### `ctx.cuaAccess` — `CuaAccess` + +Browser controls for provider-owned native permission and application decisions. + +```ts cordis-catalog +/** + * Read application decisions without waiting for the native helper. + * @returns Saved grants and refused applications with their persistence owner. + */ +@Remote applications(): CuaApplicationState + +/** + * Inspect system privileges independently of application decisions and icons. + * @returns Fresh native preflight and launch metadata. + */ +@Remote async systemStatus(): Promise + +/** + * Load presentation icons separately from application decisions. + * @returns Optional native PNG icons keyed by executable path for saved and blocked applications. + */ +@Remote async applicationIcons(): Promise> + +/** + * Request a system privilege explicitly selected by the user. + * @param permission macOS privilege to request. + * @returns Preflight state after requesting; a false value never implies consent. + */ +@Remote async requestPermission(permission: CuaSystemPermission): Promise + +/** + * Open the macOS privacy pane selected by the user. + * @param permission Corresponding system privilege. + * @returns Completion after macOS accepted the open request. + */ +@Remote async openSettings(permission: CuaSystemPermission): Promise + +/** + * Remove saved and session access and join affected operations. + * @param id Application identity displayed on the permission page. + * @returns Completion after persistence and cancellation; dispatched input cannot be undone. + */ +@Remote revoke(id: CuaApplicationId): Promise + +/** + * Clear this application's refusals across sessions so each session may ask for access again. + * @param id Refused or revoked application displayed on the permission page. + * @returns Completion after clearing refusals; no permission is granted and no action is replayed. + */ +@Remote async retryApproval(id: CuaApplicationId): Promise +``` + +Source: [`packages/experimental/cua/src/access.ts`](../../packages/experimental/cua/src/access.ts) diff --git a/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.i18n.yaml b/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.i18n.yaml new file mode 100644 index 0000000000..bb91afebfd --- /dev/null +++ b/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.i18n.yaml @@ -0,0 +1,16 @@ +# Bilingual-pair consistency record for guide.md (docs/i18n/README.md): per heading +# section, a hash of its English and Chinese blocks outside code blocks and generated regions. +# After editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.md +/: + en: 36d6b79b48c08dd5 + zh: 814caece159e9b0e +/cua-defaults-to-ptc-session-scopes: + en: 1d6eee8020ebed3b + zh: d14279f527983c3c +/cua-defaults-to-ptc-session-scopes/change: + en: dac96a29e3751249 + zh: d48cb030f43c07e4 +/cua-defaults-to-ptc-session-scopes/migration: + en: f3d82fe45e41c712 + zh: e55e424f5a2c096a diff --git a/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.md b/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.md new file mode 100644 index 0000000000..7db56c3afd --- /dev/null +++ b/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.md @@ -0,0 +1,18 @@ +--- +kind: upgrade-guide +description: "Standalone native CUA compositions must explicitly select global tool registration." +--- + +# CUA defaults to PTC session scopes + +English | [中文](guide.zh.md) + +## Change + +The native CUA provider defaults `agentPreset` to `ptc`. It registers tools for existing and new PTC Agents, including when a saved application-grants configuration omits the new field. Custom compositions without an Agent preset registry no longer receive global CUA tools by default. + +## Migration + +1. For the standard Computer Use switch, keep the default; existing application grants remain usable. +2. For a standalone composition that needs global tools, add `agentPreset: false` under the native CUA entry’s `config` in its profile patch or overlay. Preserve its other fields, including `allowedApps` and `helperPath`. +3. Restart or let HMR apply the edit, then confirm that the intended Agent exposes `cua` in native mode or through the PTC SDK. diff --git a/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.zh.md b/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.zh.md new file mode 100644 index 0000000000..ff6f6ccbc0 --- /dev/null +++ b/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.zh.md @@ -0,0 +1,18 @@ +--- +kind: upgrade-guide +description: "独立原生 CUA 组合需要显式选择全局工具注册。" +--- + +# CUA 默认使用 PTC 会话作用域 + +[English](guide.md) | 中文 + +## 变更 + +原生 CUA 提供方的 `agentPreset` 默认为 `ptc`。工具注册到已有和新建 PTC Agent，即使保存的应用授权配置没有这个新字段也会生效。不包含 Agent preset 注册表的自定义组合不再默认获得全局 CUA 工具。 + +## 迁移 + +1. 使用标准 Computer Use 开关时保持默认值，已有应用授权仍可使用。 +2. 独立组合若需要全局工具，在 profile patch 或 overlay 的原生 CUA 条目 `config` 下添加 `agentPreset: false`，保留 `allowedApps`、`helperPath` 等其他字段。 +3. 重启或等待 HMR 应用后，确认目标 Agent 能在原生模式或 PTC SDK 中使用 `cua`。 diff --git a/package.json b/package.json index 39c7f94f95..b3837d941d 100644 --- a/package.json +++ b/package.json @@ -290,6 +290,7 @@ "@deepseek-ai/dsh-working-directory": "workspace:*", "@deepseek-ai/dsh-experimental-worktree": "workspace:*", "@deepseek-ai/dsh-experimental-tool-worktree": "workspace:*", + "@deepseek-ai/dsh-cua-bundle": "workspace:*", "@deepseek-ai/dsh-experimental-subagent-oar": "workspace:*", "@deepseek-ai/dsh-experimental-tool-subagent-oar": "workspace:*", "@deepseek-ai/dsh-experimental-cua-macos": "workspace:*" diff --git a/packages/boot/app-boot/src/profile.ts b/packages/boot/app-boot/src/profile.ts index 3c7aa74762..e5111b82dc 100644 --- a/packages/boot/app-boot/src/profile.ts +++ b/packages/boot/app-boot/src/profile.ts @@ -221,6 +221,7 @@ export const DEFAULT_PROFILE_BUNDLES: readonly string[] = ['@deepseek-ai/dsh-bas * [admission](../../../../.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md)). */ export const OPTIONAL_BUNDLES: readonly string[] = [ + '@deepseek-ai/dsh-cua-bundle', '@deepseek-ai/dsh-experimental-session-search', '@deepseek-ai/dsh-experimental-ralph-bundle', '@deepseek-ai/dsh-experimental-terminal-bundle', diff --git a/packages/computer-use/README.i18n.yaml b/packages/computer-use/README.i18n.yaml index c6e98fc6e5..1b2c63a951 100644 --- a/packages/computer-use/README.i18n.yaml +++ b/packages/computer-use/README.i18n.yaml @@ -9,8 +9,8 @@ en: 77ea90624a004773 zh: fe6dbf8414a5aff0 /packages-computer-use/summary: - en: 3820da56a601d3f1 - zh: 358e65f8a7cf7446 + en: 7e5fcf9bb48c9677 + zh: abee2be288115045 /packages-computer-use/table-of-contents: en: 1eb35b015a6c423f zh: a8a3dceee2c9e943 @@ -18,8 +18,8 @@ en: 9aa86e6451f60c3d zh: c5e6816e90f2d74b /packages-computer-use/related-documentation: - en: ad08d3417a0d4630 - zh: cfe8e03033c2fede + en: 58198f9f17e6120f + zh: 188fa0ec5c4678f3 /packages-computer-use/dev-note: en: b75c8458928b72b9 zh: 003ba4e17b4b5269 diff --git a/packages/computer-use/README.md b/packages/computer-use/README.md index 1b07c668ad..d19b0a3b07 100644 --- a/packages/computer-use/README.md +++ b/packages/computer-use/README.md @@ -9,7 +9,7 @@ English | [中文](README.zh.md) ## Summary -Computer-use providers let models observe and operate a desktop. This group owns exclusive provider registration. Each provider owns its operations, tools, and platform requirements; the experimental Cua Driver providers live in the experimental group. +Computer-use providers let models observe and operate a desktop. This group contains the shared registry that prevents multiple providers from being enabled in one composition. DSH's [CUA implementation](../experimental/cua/README.md), platform providers, and opt-in bundle currently live in `experimental`, alongside the Cua Driver adapters. The CUA package documents the platform requirements for promotion to this group. ## Table of Contents @@ -32,7 +32,8 @@ Choose one provider and mount the shared registration service. ## Related documentation - [Computer use](../../docs/subsystems/computer-use.md) — capability ownership and provider choices. -- [Experimental packages](../experimental/README.md) — Cua Driver MCP and native providers. +- [Computer Use bundle](../experimental/cua-bundle/README.md) — enable native CUA and access its permission controls. +- [Experimental packages](../experimental/README.md) — native CUA packages and Cua Driver adapters. ## Dev Note diff --git a/packages/computer-use/README.zh.md b/packages/computer-use/README.zh.md index d1aa763117..34204bc7a0 100644 --- a/packages/computer-use/README.zh.md +++ b/packages/computer-use/README.zh.md @@ -9,7 +9,7 @@ kind: "package-group" ## 概述 -计算机操作提供方让模型观察并操作桌面。本组负责提供方的独占注册。各提供方拥有自己的操作、工具和平台要求；实验性 Cua Driver 提供方位于 experimental 组。 +计算机操作提供方让模型观察并操作桌面。本组包含共享注册器，防止同一组合同时启用多个提供方。DSH 的 [CUA 实现](../experimental/cua/README.zh.md)、平台提供方和启用组合包目前位于 `experimental`，Cua Driver 适配包也位于该组。CUA 包文档说明了迁入本组所需的平台支持条件。 ## 目录 @@ -32,7 +32,8 @@ kind: "package-group" ## 相关文档 - [计算机操作](../../docs/subsystems/computer-use.zh.md) — 能力归属和提供方选择。 -- [实验性包](../experimental/README.zh.md) — Cua Driver MCP 和原生提供方。 +- [Computer Use 组合包](../experimental/cua-bundle/README.zh.md)：启用原生 CUA 并使用权限管理功能。 +- [实验性包](../experimental/README.zh.md)：原生 CUA 包和 Cua Driver 适配包。 ## 开发备注 diff --git a/packages/computer-use/computer-use/README.i18n.yaml b/packages/computer-use/computer-use/README.i18n.yaml index f35f34159f..94dc062523 100644 --- a/packages/computer-use/computer-use/README.i18n.yaml +++ b/packages/computer-use/computer-use/README.i18n.yaml @@ -9,8 +9,8 @@ en: 2efee0c5f0d577bc zh: fef4b81bfc121052 /deepseek-ai-dsh-computer-use/summary: - en: 4de047849e68701f - zh: 90a32c0e279b4b05 + en: 4cb331c550852b23 + zh: d638995ab250397b /deepseek-ai-dsh-computer-use/table-of-contents: en: d152484eb41ac6b4 zh: 405b30feaf48c087 @@ -21,8 +21,8 @@ en: 181679887837591a zh: cc2e96766ce0aa63 /deepseek-ai-dsh-computer-use/further-exploration: - en: b1bdbcbc150036b3 - zh: 1d356e0522587abc + en: 987295d677466c04 + zh: 9d7b40a04786490e /deepseek-ai-dsh-computer-use/model-experience: en: 8780c250dfaefaaa zh: 1c4b3598a131fb65 diff --git a/packages/computer-use/computer-use/README.md b/packages/computer-use/computer-use/README.md index 32fccb6d6f..fb39d02482 100644 --- a/packages/computer-use/computer-use/README.md +++ b/packages/computer-use/computer-use/README.md @@ -9,7 +9,7 @@ English | [中文](README.zh.md) ## Summary -A deployment can enable one computer-use provider at a time. Loading another provider fails with the registered provider name. Each provider supplies its own tools and desktop operations. This package adds no model-visible tools and does not coordinate concurrent Sessions. +A deployment can enable one computer-use provider at a time. Loading another provider fails with the registered provider name. Both DSH's native CUA and the Cua Driver adapters use this registry. Each provider supplies its own tools and desktop operations. This package adds no model-visible tools and does not coordinate concurrent Sessions. ## Table of Contents @@ -53,6 +53,9 @@ One private name owns the slot. Cordis effects remove contributions when their p ## Further Exploration - [Computer use](../../../docs/subsystems/computer-use.md) — provider selection and shared-desktop limits. +- [Computer Use bundle](../../experimental/cua-bundle/README.md) — enable DSH's native CUA and permission controls. +- [Shared CUA library](../../experimental/cua/README.md) — application/window objects, tool execution, and application consent. +- [macOS CUA provider](../../experimental/cua-macos/README.md) — DSH's Swift implementation and native helper lifecycle. - [Cua Driver MCP provider](../../experimental/computer-use-cua-driver-mcp/README.md) — use an installed driver. - [Cua Driver native provider](../../experimental/computer-use-cua-driver-native/README.md) — use the npm runtime. diff --git a/packages/computer-use/computer-use/README.zh.md b/packages/computer-use/computer-use/README.zh.md index 1b3dfb4e77..81e6b7e264 100644 --- a/packages/computer-use/computer-use/README.zh.md +++ b/packages/computer-use/computer-use/README.zh.md @@ -9,7 +9,7 @@ kind: "package-reference" ## 概述 -部署每次可启用一个计算机操作提供方。加载另一个提供方时会失败，并报告已注册的提供方名称。各提供方提供自己的工具和桌面操作。本包不添加模型可见工具，也不协调并发 Session。 +部署每次可启用一个计算机操作提供方。加载另一个提供方时会失败，并报告已注册的提供方名称。DSH 原生 CUA 和 Cua Driver 适配包都使用此注册器。各提供方提供自己的工具和桌面操作。本包不添加模型可见工具，也不协调并发 Session。 ## 目录 @@ -53,6 +53,9 @@ kind: "package-reference" ## 延伸阅读 - [计算机操作](../../../docs/subsystems/computer-use.zh.md) — 提供方选择和共享桌面限制。 +- [Computer Use 组合包](../../experimental/cua-bundle/README.zh.md)：启用 DSH 原生 CUA 和权限管理功能。 +- [共享 CUA 库](../../experimental/cua/README.zh.md)：应用与窗口对象、工具执行和应用授权。 +- [macOS CUA 提供方](../../experimental/cua-macos/README.zh.md)：DSH 的 Swift 实现和原生辅助进程生命周期。 - [Cua Driver MCP 提供方](../../experimental/computer-use-cua-driver-mcp/README.zh.md) — 使用已安装的驱动。 - [Cua Driver 原生提供方](../../experimental/computer-use-cua-driver-native/README.zh.md) — 使用 npm 运行时。 diff --git a/packages/experimental/README.i18n.yaml b/packages/experimental/README.i18n.yaml index 116f3a9ec4..58388301cb 100644 --- a/packages/experimental/README.i18n.yaml +++ b/packages/experimental/README.i18n.yaml @@ -18,8 +18,8 @@ en: b55aeb927737b90d zh: 0b215fcb5c2c7908 /packages-experimental/packages: - en: 372b182dbf0f662b - zh: 7a8031c506394785 + en: e12eeae4540a53e0 + zh: 44f65caca730252d /packages-experimental/related-documentation: en: 1cb9d3448d44e48d zh: cd326d70a746b896 diff --git a/packages/experimental/README.md b/packages/experimental/README.md index d8ccf0dfaa..ed093206a6 100644 --- a/packages/experimental/README.md +++ b/packages/experimental/README.md @@ -71,6 +71,8 @@ Promotion requires a named product role, documented public behavior and limitati | [`ptc-runtime-python`](ptc-runtime-python/README.md) | CPython subprocess backend for the PTC execution seam | `ctx.ptcRuntime` | | [CUA](cua/README.md) | Shared application/window objects, consent, and logged execution. | Library | | [macOS CUA](cua-macos/README.md) | Swift native execution and platform preparation. | `ctx.computerUse` | +| [`cua-bundle`](cua-bundle/README.md) | Optional Computer Use switch with persistent PTC | — | +| [`client-ui-cua`](client-ui-cua/README.md) | Computer Use permission setup and application access controls | — | | [`computer-use-cua-driver-mcp`](computer-use-cua-driver-mcp/README.md) | Use an installed Cua Driver through MCP | `ctx.computerUse` | | [`computer-use-cua-driver-native`](computer-use-cua-driver-native/README.md) | Embed the Cua Driver native npm runtime | `ctx.computerUse` | | [`browser-use-playwright-mcp`](browser-use-playwright-mcp/README.md) | Playwright browser tools over MCP | `ctx.browserUse` | diff --git a/packages/experimental/README.zh.md b/packages/experimental/README.zh.md index 35b805ef06..edc4efb886 100644 --- a/packages/experimental/README.zh.md +++ b/packages/experimental/README.zh.md @@ -71,6 +71,8 @@ kind: "package-group" | [`ptc-runtime-python`](ptc-runtime-python/README.zh.md) | PTC 执行 seam 的 CPython 子进程后端 | `ctx.ptcRuntime` | | [CUA](cua/README.zh.md) | 共享应用与窗口对象、授权和已记录的执行。 | 库 | | [macOS CUA](cua-macos/README.zh.md) | Swift 原生执行与平台组件准备。 | `ctx.computerUse` | +| [`cua-bundle`](cua-bundle/README.zh.md) | 默认关闭的 Computer Use 开关与 persistent PTC 组合 | — | +| [`client-ui-cua`](client-ui-cua/README.zh.md) | Computer Use 权限设置与应用授权管理 | — | | [`computer-use-cua-driver-mcp`](computer-use-cua-driver-mcp/README.zh.md) | 通过 MCP 使用已安装的 Cua Driver | `ctx.computerUse` | | [`computer-use-cua-driver-native`](computer-use-cua-driver-native/README.zh.md) | 嵌入 Cua Driver 原生 npm 运行时 | `ctx.computerUse` | | [`browser-use-playwright-mcp`](browser-use-playwright-mcp/README.zh.md) | 通过 MCP 提供 Playwright 浏览器工具 | `ctx.browserUse` | diff --git a/packages/experimental/client-ui-cua/README.i18n.yaml b/packages/experimental/client-ui-cua/README.i18n.yaml new file mode 100644 index 0000000000..957097ba2c --- /dev/null +++ b/packages/experimental/client-ui-cua/README.i18n.yaml @@ -0,0 +1,37 @@ +# Bilingual-pair consistency record for README.md (docs/i18n/README.md): per heading +# section, a hash of its English and Chinese blocks outside code blocks and generated regions. +# After editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write packages/experimental/client-ui-cua/README.md +/: + en: e5ff656e81db5fc8 + zh: 43a8e818159176b3 +/deepseek-ai-dsh-client-ui-cua: + en: 43859860b79a3757 + zh: e8e0829c36613320 +/deepseek-ai-dsh-client-ui-cua/summary: + en: 4e62e511b5d14986 + zh: 1fce04dbad7515f7 +/deepseek-ai-dsh-client-ui-cua/table-of-contents: + en: 6658ab477751154b + zh: 9b371809a6fe38dc +/deepseek-ai-dsh-client-ui-cua/use-this-package: + en: d366fc19d6806a9f + zh: 0f6b02f19139c3f6 +/deepseek-ai-dsh-client-ui-cua/understand-the-implementation: + en: 7e661a9d00f98068 + zh: a0508c9b1d48cbb7 +/deepseek-ai-dsh-client-ui-cua/further-exploration: + en: f0fde9f736ade61a + zh: 43a534a21bcc1c13 +/deepseek-ai-dsh-client-ui-cua/model-experience: + en: 7cfbaa62f70c1916 + zh: 384ec816a50fc7c5 +/deepseek-ai-dsh-client-ui-cua/model-experience/kv-cache-effect: + en: f04fc6e6f503960f + zh: f080408ae6729fee +/deepseek-ai-dsh-client-ui-cua/known-limitations-and-deferred-work: + en: c032612d923e1367 + zh: eb77d474e21ca5d9 +/deepseek-ai-dsh-client-ui-cua/known-limitations-and-deferred-work/dev-note: + en: b836f01bd0bceb4a + zh: a6ea1361c4a48df0 diff --git a/packages/experimental/client-ui-cua/README.md b/packages/experimental/client-ui-cua/README.md new file mode 100644 index 0000000000..912d387180 --- /dev/null +++ b/packages/experimental/client-ui-cua/README.md @@ -0,0 +1,92 @@ +--- +description: "Check macOS Computer Use permissions and revoke application access." +kind: "package-reference" +--- + +# @deepseek-ai/dsh-client-ui-cua + +English | [中文](README.zh.md) + +## Summary + +Open Computer Use in Plugins to check Accessibility and Screen Recording, request missing system permissions, and revoke application access. Explicit activation probes permissions and may start a dedicated native helper. Missing privileges show setup guidance; a failed probe directs the user to diagnostics and retry in the details page. System prompts still require the user's decision. + +## Table of Contents + +- [Use this package](#use-this-package) +- [Understand the implementation](#understand-the-implementation) +- [Further Exploration](#further-exploration) +- [Model Experience](#model-experience) +- [Known Limitations and Deferred Work](#known-limitations-and-deferred-work) +- [Dev Note](#dev-note) + +The permission-page lists show native icons and application names. Paths remain available on hover. Icons are presentation metadata and are never saved with grants. Application access uses the existing User Questions interface, with session access, refusal and, for verifiable applications with writable settings, persistent access. + +----- + + +## Use this package + +The [Computer Use bundle](../cua-bundle/README.md) mounts this page with its native provider. **Authorize Computer Use** groups the new-session default, Accessibility, and Screen Recording inside one border, with dividers between the rows. Each system-permission row shows the system-reported responsible application’s `.app` bundle filename and icon together in a bordered label. **Manage applications Computer Use can access** places the application lists inside a separate bordered group. Choose **Request permission**, find that application in the macOS permission list, and enable its switch; if macOS names another application, follow that prompt. With no detected source, the page directs the user to the application named by macOS. **Open System Settings** opens the corresponding privacy pane. The page refreshes on entry, when the window regains focus, and after permission actions. A failed query offers **Retry**. Follow any macOS quit-and-reopen prompt and restart DSH as needed. + +System-permission operations affect the computer running DSH. When accessing DSH remotely, complete authorization on that computer. + +Enabling from the plugin list shows setup guidance when system permissions need attention, with **Later** and **Open settings** actions. Enabling from the detail page displays the permission controls directly. The detail page provides **Set to PTC** under **Default mode for new sessions**; only this action saves the default. Existing sessions keep their preset. + +The page lists always-allowed applications and refusals from any session. A refusal in one session does not cancel another session’s grant or prevent a new session from asking. Session-only grants remain effective within their sessions and are omitted from this page. **Revoke** removes the selected application's access and cancels affected work. Input already delivered to an application cannot be undone; observe the application before any further action. **Allow asking again** clears that application’s refusal records across all sessions; the next application selection still requires a new decision. + +This plugin has no configurable fields. + +----- + + +## Understand the implementation + +
+Implementation internals + +The optional browser contribution mounts the native provider's permission Remote and reuses the Plugins activation and configuration slots. The three setup rows remain visible while unknown fields use placeholders. The default-mode form, system checks, application decisions, and application icons fill in as each query completes. Application names and actions do not wait for icons; icon positions retain their space. User actions run in order without waiting for pending observation queries. Disposal waits for both actions and queries to settle before releasing the separately owned Remote mount. Query errors stay with the affected section, and activation guidance depends only on system checks. Late results after disposal are discarded. One controller projects queries into the page and setup dialog; reconnects and updates to the Host-reported settings namespace refresh it. Operation notices live in the shell overlay, so closing the page does not remove them. The provider owns permission checks, persisted decisions, and cancellation; the browser owns no authorization state. + +The detail page reads the effective default (`selectedDefault ?? default`) from the existing `configForms` service. Its button writes only `selectedDefault` through the form's revision-checked settings path. Loading, read-only and unavailable settings cannot submit a change; a failed write remains beside the setting for retry. + +
+ +----- + + +## Further Exploration + +- [Native provider](../cua/README.md): executable identity and operation checks. +- [Plugin manager](../../client/ui-plugin-manager/README.md): activation and page slots. +- [UI primitives](../../client/ui-primitives/README.md): shared modal, buttons, and toast. + +----- + + +## Model Experience + +None, as this package registers browser controls and no model-facing tools or prompt sections. + +#### KV Cache effect + +None; the native provider owns model-visible guidance and operation results. + +## Known Limitations and Deferred Work + + + +This CUA-specific UI remains experimental while Windows support is in development; promotion returns it to the `client` group as `ui-cua`. + +- macOS may require restarting the launching application after a system permission change. A pending request is not an authorization grant. +- Application attribution depends on an optional native system query and available application metadata. If either is unavailable, the page uses generic system-prompt guidance. Packaged Desktop distribution and permission attribution remain deferred. +- No invariant companion is published: this UI projects the provider's query and stores no independent authorization decisions. + + +### Dev Note + +
+Working context for maintainers + +None. + +
diff --git a/packages/experimental/client-ui-cua/README.zh.md b/packages/experimental/client-ui-cua/README.zh.md new file mode 100644 index 0000000000..3f75fcf364 --- /dev/null +++ b/packages/experimental/client-ui-cua/README.zh.md @@ -0,0 +1,92 @@ +--- +description: "检查 macOS Computer Use 权限并撤销应用访问权限。" +kind: "package-reference" +--- + +# @deepseek-ai/dsh-client-ui-cua + +[English](README.md) | 中文 + +## 概述 + +在插件页打开 Computer Use，可以检查辅助功能和屏幕录制权限、申请缺失的系统权限，并撤销应用授权。用户显式启用时会检查权限，可能启动专用原生 helper。缺少权限时显示设置引导，查询失败时引导用户到详情页查看原因并重试。系统授权仍由用户决定。 + +## 目录 + +- [使用本包](#use-this-package) +- [理解实现](#understand-the-implementation) +- [延伸阅读](#further-exploration) +- [模型体验](#model-experience) +- [已知限制与后续工作](#known-limitations-and-deferred-work) +- [开发备注](#dev-note) + +权限页列表显示本机图标与应用名称，悬停可查看路径。图标仅用于展示，不随授权保存。应用授权沿用现有 User Questions 界面，提供本会话允许和拒绝；应用签名可验证且配置可写时，还提供始终允许。 + +----- + + +## 使用本包 + +[Computer Use 组合](../cua-bundle/README.zh.md)将本页与原生 provider 一起挂载。“给 Computer Use 授权”把新会话默认模式、辅助功能和屏幕录制放在同一个边框内，三排之间使用分隔线。两条系统权限要求在能识别责任应用时，把该应用的 `.app` 包名称和图标放在同一个标签中。“管理 Computer Use 可访问的应用”使用独立边框包住应用列表。点击“申请权限”，在 macOS 权限列表中找到该应用并开启对应开关；若 macOS 显示其他名称，以系统提示为准。来源无法识别时，页面引导用户查找系统提示的应用。“打开系统设置”会打开对应的隐私设置页。进入详情页、返回窗口或完成授权操作后会自动刷新；查询失败时可点击“重试”。若系统要求退出并重新打开应用，请按提示操作并重新启动 DSH。 + +系统权限操作作用于运行 DSH 的电脑。远程访问时，请在该电脑上完成授权。 + +从插件列表启用后，系统权限尚未就绪时会显示设置引导，提供“稍后”和“前往设置”。从详情页启用后直接显示权限控件。详情页的“新会话默认模式”提供“设为 PTC”按钮，仅点击此按钮才保存默认值。已有会话保留原 preset。 + +页面展示“始终允许的应用”和“不允许访问的应用”。后者汇总任一会话的拒绝记录，不取消其他会话已有的授权，也不阻止新会话询问授权。会话授权仍在对应会话中生效，详情页不展示这类应用。“撤销”会移除对应应用的访问许可并取消相关操作。已发送到应用的输入无法撤回，继续操作前需重新观察应用。“允许再次询问”清除该应用在所有会话中的拒绝记录；下次选择应用时仍需用户重新决定。 + +本插件没有配置字段。 + +----- + + +## 理解实现 + +
+实现细节 + +可选浏览器插件挂载原生 provider 的权限 Remote，复用插件页的启用引导和配置插槽。三排使用条件始终可见，未加载的字段保留占位。默认模式、系统权限、应用列表和应用图标分别查询，结果返回后各自显示。应用名称和操作按钮不等待图标，图标位置保留空位。用户操作按顺序执行，不等待尚未完成的查询；卸载时等待操作和查询全部结束，再释放独立持有的 Remote 挂载。查询错误只显示在对应部分，启用引导只等待系统权限结果。卸载后的迟到结果会被丢弃。一个控制器把查询结果投影到页面和设置弹窗；重新连接或 Host 返回的配置 namespace 更新时刷新状态。操作提示挂载到 shell overlay，关闭页面后仍可显示。provider 负责权限检查、持久授权和取消；浏览器不保存授权决定。 + +详情页通过现有 `configForms` 服务读取实际默认值（`selectedDefault ?? default`）。按钮仅写入 `selectedDefault`，沿用表单带版本检查的设置写入流程。设置仍在加载、只读或不可用时不能提交；写入失败时在设置旁显示错误，供用户重试。 + +
+ +----- + + +## 延伸阅读 + +- [原生 provider](../cua/README.zh.md)：可执行文件身份和操作检查。 +- [插件管理](../../client/ui-plugin-manager/README.zh.md)：启用引导和页面插槽。 +- [界面基础组件](../../client/ui-primitives/README.zh.md)：共用弹窗、按钮和提示。 + +----- + + +## 模型体验 + +无。本包注册浏览器控件，不注册模型工具或提示词段落。 + +#### KV 缓存影响 + +无。模型可见的指引和操作结果由原生 provider 负责。 + +## 已知限制与后续工作 + + + +这套 CUA 专用界面随 Windows 支持一起开发，暂放在 `experimental`；转为正式功能后迁回 `client` 组的 `ui-cua`。 + +- 系统权限变更后，macOS 可能要求重启启动程序。申请尚未完成时，不能视为已授权。 +- 应用归属依赖可选的本机系统查询和可读取的应用信息；不可用时，页面使用通用指引，让用户按系统弹窗完成授权。桌面安装包的分发和权限归属留待后续处理。 +- 不提供 invariant companion：本界面投影 provider 查询，不独立保存授权决定。 + + +### 开发备注 + +
+维护者工作记录 + +无。 + +
diff --git a/packages/experimental/client-ui-cua/package.json b/packages/experimental/client-ui-cua/package.json new file mode 100644 index 0000000000..6554c14387 --- /dev/null +++ b/packages/experimental/client-ui-cua/package.json @@ -0,0 +1,72 @@ +{ + "name": "@deepseek-ai/dsh-client-ui-cua", + "description": "Computer Use permission setup and application access controls", + "version": "0.2.1-rc.2", + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/deepseek-ai/deepseek-harness.git", + "directory": "packages/experimental/client-ui-cua" + }, + "type": "module", + "main": "lib/index.js", + "types": "lib/types/index.d.ts", + "exports": { + ".": { + "types": "./lib/types/index.d.ts", + "default": "./lib/index.js" + }, + "./client": { + "types": "./lib/types/client/index.d.ts", + "default": "./lib/client.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "dsh": { + "client": { + "inject": [ + "@deepseek-ai/dsh-api-remotes", + "@deepseek-ai/dsh-client-locale", + "@deepseek-ai/dsh-client-ui-plugin-manager", + "@deepseek-ai/dsh-client-ui-settings", + "@deepseek-ai/dsh-client-ui-layout" + ], + "platform": "web" + } + }, + "scripts": { + "bundle": "tsdown", + "watch": "tsdown --watch" + }, + "license": "MIT", + "peerDependencies": { + "@deepseek-ai/cordis": "workspace:~" + }, + "devDependencies": { + "@deepseek-ai/cordis": "workspace:~", + "@deepseek-ai/dsh-api-remotes": "workspace:*", + "@deepseek-ai/dsh-api-gateway": "workspace:*", + "@deepseek-ai/dsh-client-locale": "workspace:*", + "@deepseek-ai/dsh-client-store": "workspace:*", + "@deepseek-ai/dsh-client-ui-primitives": "workspace:*", + "@deepseek-ai/dsh-client-ui-renderer": "workspace:*", + "@deepseek-ai/dsh-client-ui-slots": "workspace:*", + "@deepseek-ai/dsh-client-ui-plugin-manager": "workspace:*", + "@deepseek-ai/dsh-client-ui-settings": "workspace:*", + "@deepseek-ai/dsh-client-ui-layout": "workspace:*", + "@deepseek-ai/dsh-cua": "workspace:*", + "@deepseek-ai/dsh-client-test-runtime": "workspace:*", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "@testing-library/react": "^16.1.0", + "@deepseek-ai/dsh-typert-protocol": "workspace:*" + }, + "files": [ + "lib/index.js", + "lib/client.js", + "lib/types/**/*.d.ts" + ] +} diff --git a/packages/experimental/client-ui-cua/src/client/PermissionPanel.module.css b/packages/experimental/client-ui-cua/src/client/PermissionPanel.module.css new file mode 100644 index 0000000000..f4fb0334d0 --- /dev/null +++ b/packages/experimental/client-ui-cua/src/client/PermissionPanel.module.css @@ -0,0 +1,48 @@ +.panels { display: grid; gap: 24px; margin: 18px 0; font-size: 13px; } +.panel { display: grid; gap: 12px; min-width: 0; } +.card { min-width: 0; border: 0.5px solid var(--dsw-alias-border-l1); border-radius: var(--dsw-radius-md); } +.card > * + * { border-top: 0.5px solid var(--dsw-alias-border-l1); } +.panel p { margin: 0; color: var(--dsw-alias-label-secondary); line-height: 1.6; } +.panel h3 { margin: 0; font-size: 14px; font-weight: 500; } +.panel h4 { margin: 0; font-size: 13px; font-weight: 500; color: var(--dsw-alias-label-secondary); } +.permission, .apps li { display: flex; flex-wrap: wrap; align-items: center; gap: 12px; } +.permission, .group, .emptyState, .notice { padding: 16px; } +.permission { min-height: 80px; box-sizing: border-box; } +.defaultSetting > p { margin: 0 16px 16px; } +.permission > div:first-child, .apps li > div { flex: 1; min-width: 140px; } +.actions, .notice { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; } +.group { display: grid; gap: 4px; } +.apps { display: grid; padding: 0; margin: 0; list-style: none; } +.apps li { padding: 12px 0; } +.apps li:last-child { padding-bottom: 0; } +.panel strong { font-weight: 500; } +.panel .error { color: var(--dsw-alias-state-error-primary); overflow-wrap: anywhere; } +.application { display: flex; align-items: center; gap: 10px; } +.application strong { min-width: 0; overflow-wrap: anywhere; } +.application img { flex-shrink: 0; object-fit: contain; } +.applicationBadge { + display: inline-flex; + align-items: center; + gap: 6px; + max-width: 100%; + padding: 2px 6px; + border: 0.5px solid var(--dsw-alias-border-l1); + border-radius: var(--dsw-radius-sm); + color: var(--dsw-alias-label-primary); + vertical-align: middle; +} +.applicationBadge img { flex-shrink: 0; object-fit: contain; } +.applicationBadge > span { min-width: 0; overflow-wrap: anywhere; } +.permissionStatus { min-width: 72px; } +.buttonPlaceholder, .statusPlaceholder, .appPlaceholder, .appsPlaceholder { + display: inline-block; + max-width: 100%; + border-radius: var(--dsw-radius-xs); + background: var(--dsw-alias-bg-skeleton); +} +.buttonPlaceholder { width: 120px; height: 28px; } +.statusPlaceholder { width: 64px; height: 12px; } +.appPlaceholder { width: 160px; height: 20px; } +.appsPlaceholder { width: 240px; height: 40px; } + +.iconPlaceholder { display: inline-block; width: 32px; height: 32px; flex-shrink: 0; } diff --git a/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx b/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx new file mode 100644 index 0000000000..4c9ae041fd --- /dev/null +++ b/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx @@ -0,0 +1,183 @@ +/** Native permission status and revocable application consent on the existing plugin page. */ +import { useEffect, useState } from 'react' +import { Button, Modal, Toast } from '@deepseek-ai/dsh-client-ui-primitives' +import type { InjectFace, PropsLocale, PropsRuntime } from '@deepseek-ai/dsh-client-ui-slots' +import type {} from '@deepseek-ai/dsh-client-ui-plugin-manager/client' +import type { ConfigForm } from '@deepseek-ai/dsh-client-ui-settings/client' +import type { CuaApplicationGrant, CuaSystemPermissionState } from '@deepseek-ai/dsh-cua/types' +import type { PermissionFace } from './controller.ts' +import type { NS } from './locales.ts' +import css from './PermissionPanel.module.css' + +/** Renderer-bound permission observation and localized controls. */ +export type PermissionProps = InjectFace & PropsLocale + +/** Permission guidance only navigates to the existing bundle details. */ +export type PermissionSetupProps = PermissionProps + & Pick, 'onDismiss' | 'onOpenDetails'> + +type DefaultPresetProps = PropsLocale & InjectFace<{ + hooks: { cuaDefaultPreset: ConfigForm<{ default: string; selectedDefault?: string }> } + setPtcDefault: () => Promise +}> + +/** The detail page owns the optional new-session default action. */ +export type PermissionPanelProps = PermissionProps & DefaultPresetProps + +function DefaultPresetSetting(props: DefaultPresetProps) { + const { t } = props + const preset = props.useCuaDefaultPreset(value => value) + const isPtc = (preset.value?.selectedDefault ?? preset.value?.default) === 'ptc' + const loading = preset.value === undefined && preset.status === 'loading' + const [saving, setSaving] = useState(false) + const [error, setError] = useState(null) + const setDefault = async (): Promise => { + setSaving(true) + setError(null) + try { + if (!await props.setPtcDefault()) setError(t('defaultNotSaved')) + } catch (error) { + setError(t('defaultFailed', { message: error instanceof Error ? error.message : String(error) })) + } finally { setSaving(false) } + } + return
+
+
{t('defaultTitle')}

{t('defaultBody')}

+ {loading ? + : } +
+ {!isPtc && (preset.status === 'unavailable' ?

{t('defaultUnavailable')}

+ : preset.status === 'ready' && !preset.writable &&

{t('defaultReadOnly')}

)} + {error !== null &&

{error}

} +
+} + +function Applications({ title, apps, icons, onAction, label, disabled }: { + title: string + apps: readonly CuaApplicationGrant[] + icons: Record | null + disabled: boolean + label: string + onAction: PermissionProps['revoke'] +}) { + return
+

{title}

+
    {apps.map(app =>
  • +
    + {icons?.[app.executablePath] !== undefined ? + : } + {app.name} +
    + +
    • )}
+
+} + +function PermissionDescription({ application, permission, loading, t }: Pick & { + application: CuaSystemPermissionState['launchApplication'] + permission: 'accessibility' | 'screenRecording' + loading: boolean +}) { + if (!loading && application === undefined) return

{t(permission === 'accessibility' ? 'accessibilityUse' : 'screenRecordingUse')}

+ return

+ {t('permissionPrefix')}{' '} + + {application === undefined ? : <> + {application.icon !== undefined && } + {application.name} + } + {' '} + {t(permission === 'accessibility' ? 'accessibilityAction' : 'screenRecordingAction')} +

+} + +/** @param props Host observation and explicit user actions. @returns Permission controls without automatic system prompts. */ +export function PermissionPanel(props: PermissionPanelProps) { + const { t, refresh, setActive } = props + const state = props.useCuaPermissions(value => value) + const observed = state.system.value + const apps = state.applications.value + const busy = state.system.busy + const appsBusy = state.applications.busy + const loading = observed === null && (busy || state.system.error === null) + const application = !busy && state.system.error !== null ? undefined : observed?.launchApplication + useEffect(() => { + setActive(true) + window.addEventListener('focus', refresh) + return () => { window.removeEventListener('focus', refresh) } + }, [refresh, setActive]) + return
+
+

{t('title')}

+
+ + {!busy && state.system.error !== null &&
+ {t('queryFailed', { message: state.system.error })} + +
} + {(['accessibility', 'screenRecording'] as const).map(permission =>
+
+ {t(permission)} + +
+ {loading ? + : t(observed === null || state.system.error !== null ? 'unknown' : observed[permission] ? 'granted' : 'missing')} +
+ {loading ? + : observed?.[permission] === false && } + +
+
)} +
+ {observed !== null && (!observed.accessibility || !observed.screenRecording) &&

+ {t(application === undefined ? 'permissionHintUnknown' : 'permissionHint')} +

} +
+
+

{t('applicationsTitle')}

+
+ {(['applications', 'icons'] as const).map(key => !state[key].busy && state[key].error !== null &&
+ {t(key === 'applications' ? 'applicationsFailed' : 'iconsFailed', { message: state[key].error })} + +
)} + {apps === null ?
: <> + {apps.allowedApps.length === 0 && apps.blockedApps.length === 0 &&

{t('noApps')}

} + {apps.allowedApps.length > 0 && } + {apps.blockedApps.length > 0 && } + } +
+
+
+} + +/** + * @param props Permission observation and activation navigation callbacks. + * @returns Setup guidance when system permissions need attention. + */ +export function PermissionSetup({ t, setActive, useCuaPermissions, onDismiss, onOpenDetails }: PermissionSetupProps) { + const state = useCuaPermissions(value => value.system) + useEffect(() => { setActive(true) }, [setActive]) + const ready = state.value !== null && state.error === null + && state.value.accessibility && state.value.screenRecording + useEffect(() => { if (ready && !state.busy) onDismiss() }, [ready, state.busy, onDismiss]) + return + + + }> +

{t(state.error === null ? 'setupBody' : 'setupQueryFailed')}

+ +} + +/** @param props Shared operation outcome, independent of page lifetime. @returns The existing global toast presentation. */ +export function PermissionToast(props: PermissionProps) { + const toast = props.useCuaPermissions(value => value.toast) + if (!toast) return null + return +} diff --git a/packages/experimental/client-ui-cua/src/client/controller.ts b/packages/experimental/client-ui-cua/src/client/controller.ts new file mode 100644 index 0000000000..552dcefbba --- /dev/null +++ b/packages/experimental/client-ui-cua/src/client/controller.ts @@ -0,0 +1,136 @@ +/** One query projection for the permission page and activation guidance. */ +import { createSnapshotStore, type SnapshotStore } from '@deepseek-ai/dsh-client-store' +import type { CuaApplicationId, CuaApplicationState, CuaSystemPermission, CuaSystemPermissionState } from '@deepseek-ai/dsh-cua/types' + +/** Host operations available to the permission UI. */ +export interface PermissionActions { + readSystem: () => Promise + readApplications: () => Promise + readIcons: () => Promise> + request: (permission: CuaSystemPermission) => Promise + openSettings: (permission: CuaSystemPermission) => Promise + revoke: (id: CuaApplicationId) => Promise + retryApproval: (id: CuaApplicationId) => Promise +} +interface PermissionData { + system: CuaSystemPermissionState + applications: CuaApplicationState + icons: Record +} +/** Query failures retain the last observation without presenting it as current. */ +interface Query { + value: T | null + busy: boolean + error: string | null +} +/** Independently settled observations; the overall busy flag follows the latest refresh. */ +export type PermissionView = { [K in keyof PermissionData]: Query } & { + busy: boolean + toast: { id: number; kind: 'requested' | 'opened' | 'revoked' | 'retryAllowed' } | { id: number; kind: 'failed'; message: string } | null +} +/** Slot actions and the renderer-bound permission projection. */ +export interface PermissionFace { + hooks: { cuaPermissions: SnapshotStore } + /** Pause observation during bundle changes, or resume with a fresh read. */ + setActive: (active: boolean) => void + refresh: () => void + request: (permission: CuaSystemPermission) => void + openSettings: (permission: CuaSystemPermission) => void + revoke: (id: CuaApplicationId) => void + retryApproval: (id: CuaApplicationId) => void + dismissToast: () => void +} + +/** Serializes user actions and drops superseded refresh results or results arriving after disposal. */ +export class PermissionController { + /** Last observed permissions and the lifetime of the current refresh/action. */ + readonly state = createSnapshotStore({ + system: { value: null, busy: false, error: null }, + applications: { value: null, busy: false, error: null }, + icons: { value: null, busy: false, error: null }, + busy: false, toast: null, + }) + private tail: Promise = Promise.resolve() + private reads: Promise = Promise.resolve() + private closed = false + private active = true + private epoch = 0 + private sequence = 0 + /** @param actions Browser-owned Remote calls that reject on failed responses. */ + constructor(private readonly actions: PermissionActions) {} + /** + * Bind the permission projection to its user actions. + * @returns Reactive state and controls shared by the page, prompt, and toast host. + */ + face(): PermissionFace { + return { + hooks: { cuaPermissions: this.state }, + setActive: (active) => { + if (this.closed) return + if (active !== this.active) { this.active = active; this.epoch++ } + if (active) this.enqueue() + else this.state.update((state) => { + state.busy = true + for (const key of ['system', 'applications', 'icons'] as const) { + state[key].busy = true; state[key].error = null + } + }) + }, + refresh: () => { this.enqueue() }, + request: (permission) => { this.enqueue(() => this.actions.request(permission), 'requested') }, + openSettings: (permission) => { this.enqueue(() => this.actions.openSettings(permission), 'opened') }, + revoke: (id) => { this.enqueue(() => this.actions.revoke(id), 'revoked') }, + retryApproval: (id) => { this.enqueue(() => this.actions.retryApproval(id), 'retryAllowed') }, + dismissToast: () => { if (!this.closed) this.state.update((state) => { state.toast = null }) }, + } + } + /** Prevent publication after UI withdrawal and join the accepted Remote work. */ + async dispose(): Promise { this.closed = true; await this.tail; await this.reads } + private enqueue(action?: () => Promise, kind?: 'requested' | 'opened' | 'revoked' | 'retryAllowed'): void { + if (this.closed || !this.active) return + const epoch = this.epoch + const current = () => !this.closed && this.active && epoch === this.epoch + const sequence = ++this.sequence + this.state.update((state) => { + state.busy = true + for (const key of ['system', 'applications', 'icons'] as const) state[key].busy = true + }) + this.tail = this.tail.then(async () => { + if (!current()) return + try { + await action?.() + if (!current()) return + if (kind) this.state.update((state) => { state.toast = { id: sequence, kind } }) + } catch (error) { + if (!current()) return + const message = error instanceof Error ? error.message : String(error) + this.state.update((state) => { state.toast = { id: sequence, kind: 'failed', message } }) + } + const latest = () => current() && this.sequence === sequence + const refresh = Promise.all([ + this.read('system', this.actions.readSystem, latest), + this.read('applications', this.actions.readApplications, latest), + this.read('icons', this.actions.readIcons, latest), + ]).then(() => { + if (latest()) this.state.update((state) => { state.busy = false }) + }) + this.reads = Promise.all([this.reads, refresh]).then(() => {}) + }) + } + + private async read( + key: K, read: () => Promise, current: () => boolean, + ): Promise { + if (!current()) return + try { + const value = await read() + if (!current()) return + this.state.update((state) => { state[key].value = value; state[key].error = null; state[key].busy = false }) + } catch (error) { + if (!current()) return + this.state.update((state) => { + state[key].error = error instanceof Error ? error.message : String(error); state[key].busy = false + }) + } + } +} diff --git a/packages/experimental/client-ui-cua/src/client/index.ts b/packages/experimental/client-ui-cua/src/client/index.ts new file mode 100644 index 0000000000..a4b192ce58 --- /dev/null +++ b/packages/experimental/client-ui-cua/src/client/index.ts @@ -0,0 +1,15 @@ +/** Bind the source-owned permission UI to its generated Remote contribution. */ +import type { Context } from '@deepseek-ai/cordis' +import cuaRemote from '@deepseek-ai/dsh-cua/remote' +import { mountCuaPermissions } from './mount.ts' + +export { inject } from './mount.ts' + +/** + * Activate the optional Computer Use permission controls. + * @param ctx Browser plugin context. + * @returns Cleanup of the UI and Remote contribution. + */ +export async function apply(ctx: Context): Promise<() => Promise> { + return mountCuaPermissions(ctx, cuaRemote) +} diff --git a/packages/experimental/client-ui-cua/src/client/locales.ts b/packages/experimental/client-ui-cua/src/client/locales.ts new file mode 100644 index 0000000000..03e86f33d0 --- /dev/null +++ b/packages/experimental/client-ui-cua/src/client/locales.ts @@ -0,0 +1,64 @@ +/** Product copy for native permission onboarding and application revocation. */ +export const NS = 'cuaPermissions' +/** English permission-page dictionary. */ +export const en = { + title: 'Authorize Computer Use', + applicationsTitle: 'Manage applications Computer Use can access', + setupTitle: 'Set up Computer Use before use', + setupBody: 'Computer Use is enabled. Before using it, switch to PTC mode and grant Accessibility and Screen Recording permissions. Open the plugin details to complete setup.', + setupQueryFailed: 'Computer Use is enabled, but system permissions could not be checked. Open the plugin details to see the error and retry.', + defaultTitle: 'Default mode for new sessions', + defaultBody: 'Computer Use is available only in PTC mode.', + defaultConfirm: 'Set to PTC', defaultSet: 'PTC is the default', + defaultUnavailable: 'Could not read the default mode. Check it in Settings.', + defaultReadOnly: 'Default-mode settings are read-only.', + defaultNotSaved: 'The default mode was not saved. Try again.', + defaultFailed: 'Could not change the default mode: {message}', + openSetup: 'Open settings', later: 'Later', close: 'Close', + accessibility: 'Accessibility', screenRecording: 'Screen Recording', + accessibilityUse: 'Read interface controls, click, and type.', screenRecordingUse: 'Capture the selected application window.', + permissionPrefix: 'In System Settings, allow', + accessibilityAction: 'to read interface controls, click, and type.', + screenRecordingAction: 'to capture application windows.', + permissionHint: 'Enable the application in System Settings. Follow the macOS prompt if it names another application.', + permissionHintUnknown: 'Enable these permissions for the application named in the macOS prompt.', + granted: 'Allowed', missing: 'Not allowed', unknown: 'Unknown', request: 'Request permission', settings: 'Open System Settings', retry: 'Retry', + allowedApps: 'Always-allowed applications', + blockedApps: 'Applications not allowed', retryApproval: 'Allow asking again', + retryAllowed: 'The next application selection will ask again; access has not been granted', + noApps: 'No always-allowed or blocked applications.', + revoke: 'Revoke', revoked: 'Application access revoked', requested: 'Review the macOS prompt, or open System Settings if no prompt appears', opened: 'System Settings opened', + applicationsFailed: 'Could not load application access: {message}', iconsFailed: 'Could not load application icons: {message}', + failed: 'Could not update permissions: {message}', queryFailed: 'Could not check permissions: {message}', +} +/** Keys shared by all supported locale dictionaries. */ +export type CuaPermissionLocaleKey = keyof typeof en +/** Chinese permission-page dictionary. */ +export const zh: Record = { + title: '给 Computer Use 授权', applicationsTitle: '管理 Computer Use 可访问的应用', setupTitle: '使用 Computer Use 前需要设置', + setupBody: 'Computer Use 插件已开启。使用前请切换到 PTC 模式，并授予辅助功能和屏幕录制权限。请前往插件详情页完成设置。', + setupQueryFailed: 'Computer Use 插件已开启，但暂时无法检测系统权限。请前往插件详情页查看原因并重试。', + defaultTitle: '新会话默认模式', + defaultBody: 'Computer Use 仅在 PTC 模式中可用。', + defaultConfirm: '设为 PTC', defaultSet: '已设为 PTC', + defaultUnavailable: '无法读取默认模式，请在设置中检查', + defaultReadOnly: '默认模式设置为只读', + defaultNotSaved: '默认模式未保存，请重试', + defaultFailed: '无法切换默认模式：{message}', + openSetup: '前往设置', later: '稍后', close: '关闭', + accessibility: '辅助功能', screenRecording: '屏幕录制', + accessibilityUse: '读取界面控件、点击和输入文字', screenRecordingUse: '截取所选应用的窗口', + permissionPrefix: '在系统设置中允许', + accessibilityAction: '读取界面、点击和输入', + screenRecordingAction: '截取应用窗口', + permissionHint: '在系统设置中开启对应应用；若 macOS 提示其他应用，以系统提示为准', + permissionHintUnknown: '在系统设置中，为 macOS 提示的应用开启对应权限', + granted: '已允许', missing: '未允许', unknown: '未确认', request: '申请权限', settings: '打开系统设置', retry: '重试', + allowedApps: '始终允许的应用', + blockedApps: '不允许访问的应用', retryApproval: '允许再次询问', + retryAllowed: '下次选择此应用时将重新询问，当前尚未授权', + noApps: '暂无始终允许或不允许访问的应用。', + revoke: '撤销', revoked: '已撤销应用访问权限', requested: '请查看 macOS 提示；若未出现，请打开系统设置', opened: '已打开系统设置', + applicationsFailed: '无法读取应用授权：{message}', iconsFailed: '无法读取应用图标：{message}', + failed: '无法更新权限：{message}', queryFailed: '无法检测权限：{message}', +} diff --git a/packages/experimental/client-ui-cua/src/client/mount.ts b/packages/experimental/client-ui-cua/src/client/mount.ts new file mode 100644 index 0000000000..a475c43d8e --- /dev/null +++ b/packages/experimental/client-ui-cua/src/client/mount.ts @@ -0,0 +1,82 @@ +/** Optional Computer Use controls mounted through existing plugin-page and overlay slots. */ +import type { Context } from '@deepseek-ai/cordis' +import type {} from '@deepseek-ai/dsh-cua/remote' +import type { TypertRemoteContribution } from '@deepseek-ai/dsh-typert-protocol' +import type {} from '@deepseek-ai/dsh-api-remotes/client' +import type {} from '@deepseek-ai/dsh-client-locale/client' +import type {} from '@deepseek-ai/dsh-client-ui-renderer/client' +import type {} from '@deepseek-ai/dsh-client-ui-layout/client' +import type {} from '@deepseek-ai/dsh-client-ui-plugin-manager/client' +import type {} from '@deepseek-ai/dsh-client-ui-settings/client' +import { PermissionController } from './controller.ts' +import { PermissionPanel, PermissionSetup, PermissionToast } from './PermissionPanel.tsx' +import { en, NS, zh, type CuaPermissionLocaleKey } from './locales.ts' + +declare module '@deepseek-ai/dsh-client-ui-slots' { + interface LocaleNamespaceMap { + /** Native permission controls and application consent. */ + cuaPermissions: CuaPermissionLocaleKey + } +} + +/** Existing browser registries needed by the optional permission UI. */ +export const inject = ['remote', 'slots', 'locale', 'configForms'] + +/** + * Mount the optional permission Remote and its browser controls. + * @param ctx Browser plugin context. + * @param contribution Generated permission Remote definitions. + * @returns Cleanup for the optional Remote and UI registrations. + */ +export async function mountCuaPermissions(ctx: Context, contribution: TypertRemoteContribution): Promise<() => Promise> { + const transport = ctx.plugin(async (owner) => { await owner.remote.$mount(contribution) }) + const ui = ctx.inject(['remote.cuaAccess', 'slots', 'locale'], (ctx) => { + ctx.effect(() => ctx.locale.register(NS, { en, zh })) + const controller = new PermissionController({ + readSystem: async () => { + const result = await ctx.remote.cuaAccess.systemStatus(); if (!result.ok) throw result.error; return result.value + }, + readApplications: async () => { + const result = await ctx.remote.cuaAccess.applications(); if (!result.ok) throw result.error; return result.value + }, + readIcons: async () => { + const result = await ctx.remote.cuaAccess.applicationIcons(); if (!result.ok) throw result.error; return result.value + }, + request: async (permission) => { + const result = await ctx.remote.cuaAccess.requestPermission(permission); if (!result.ok) throw result.error + }, + openSettings: async (permission) => { + const result = await ctx.remote.cuaAccess.openSettings(permission); if (!result.ok) throw result.error + }, + revoke: async (id) => { const result = await ctx.remote.cuaAccess.revoke(id); if (!result.ok) throw result.error }, + retryApproval: async (id) => { const result = await ctx.remote.cuaAccess.retryApproval(id); if (!result.ok) throw result.error }, + }) + const face = controller.face() + const preset = ctx.configForms.get<{ default: string; selectedDefault?: string }>('agent-preset-registry') + const settings = { + ...face, + hooks: { ...face.hooks, cuaDefaultPreset: preset }, + setPtcDefault: () => preset.set('selectedDefault', 'ptc'), + } + ctx.effect(() => () => controller.dispose()) + ctx.on('connection/reset', face.refresh) + ctx.effect(() => ctx.remote.$on('settings/document-updated', (namespace) => { + if (namespace === controller.state.getSnapshot().applications.value?.settingsNamespace) face.refresh() + })) + ctx.slots.inject('plugins.bundle.config', () => ctx.slots.register({ + name: 'plugins.bundle.config', key: '@deepseek-ai/dsh-cua-bundle', locale: NS, inject: () => settings, + }, PermissionPanel)) + ctx.slots.inject('plugins.bundle.activation', () => ctx.slots.register({ + name: 'plugins.bundle.activation', key: '@deepseek-ai/dsh-cua-bundle', locale: NS, inject: () => face, + }, PermissionSetup)) + ctx.slots.inject('shell.overlay', () => ctx.slots.register({ + name: 'shell.overlay', id: 'cua-permission-outcome', locale: NS, inject: () => face, + }, PermissionToast)) + }) + const dispose = ctx.effect(function* () { + yield transport.dispose + yield ui.dispose + }, 'cua-permissions.mount') + try { await Promise.all([transport, ui]) } catch (error) { await dispose(); throw error } + return dispose +} diff --git a/packages/experimental/client-ui-cua/src/css-modules.d.ts b/packages/experimental/client-ui-cua/src/css-modules.d.ts new file mode 100644 index 0000000000..8811db1264 --- /dev/null +++ b/packages/experimental/client-ui-cua/src/css-modules.d.ts @@ -0,0 +1,4 @@ +declare module '*.module.css' { + const classes: Record + export default classes +} diff --git a/packages/experimental/client-ui-cua/src/index.ts b/packages/experimental/client-ui-cua/src/index.ts new file mode 100644 index 0000000000..ceee879f3f --- /dev/null +++ b/packages/experimental/client-ui-cua/src/index.ts @@ -0,0 +1,2 @@ +/** Host marker; the client export owns the optional permission UI, while dsh-cua owns native policy. */ +export function apply(): void {} diff --git a/packages/experimental/client-ui-cua/tests/controller.client.spec.ts b/packages/experimental/client-ui-cua/tests/controller.client.spec.ts new file mode 100644 index 0000000000..6c12499c16 --- /dev/null +++ b/packages/experimental/client-ui-cua/tests/controller.client.spec.ts @@ -0,0 +1,253 @@ +/** Permission refreshes preserve observations and join user actions before withdrawal. */ +import { setImmediate } from 'node:timers/promises' +import { expect, it, onTestFinished, vi } from 'vitest' +import type { CuaApplicationId, CuaApplicationState, CuaSystemPermissionState } from '@deepseek-ai/dsh-cua/types' +import { PermissionController } from '../src/client/controller.ts' + +const value: CuaSystemPermissionState = { accessibility: true, screenRecording: false, helper: { pid: 9 }, parent: { pid: 8 } } +const applications: CuaApplicationState = { + allowedApps: [], blockedApps: [], settingsNamespace: 'cua', persistentAvailable: true } +function fixture() { + const actions = { readSystem: vi.fn(async () => value), readApplications: vi.fn(async () => applications), + readIcons: vi.fn(async (): Promise> => ({})), request: vi.fn(async () => {}), + openSettings: vi.fn(async () => {}), revoke: vi.fn(async () => {}), retryApproval: vi.fn(async () => {}) } + const controller = new PermissionController(actions) + return { actions, controller, face: controller.face() } +} + +it('refreshes after a user request without treating the request itself as a grant', async () => { + const f = fixture() + f.face.request('screenRecording') + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) + expect(f.actions.request).toHaveBeenCalledWith('screenRecording') + expect(f.controller.state.getSnapshot().system.value?.screenRecording).toBe(false) + expect(f.controller.state.getSnapshot().toast?.kind).toBe('requested') + f.face.dismissToast() + expect(f.controller.state.getSnapshot().toast).toBeNull() + await f.controller.dispose() +}) + +it('retains the last observation while marking a failed refresh', async () => { + const f = fixture() + f.face.refresh() + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().system.value).toEqual(value) }) + f.actions.readSystem.mockRejectedValueOnce(new Error('connection closed')) + f.face.refresh() + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().system.error).toBe('connection closed') }) + expect(f.controller.state.getSnapshot().system.value).toEqual(value) + await f.controller.dispose() +}) + +it.each([new Error('write refused'), 'write refused'])('reports settings and revocation outcomes through the shared toast state', async (error) => { + const f = fixture() + f.face.openSettings('accessibility') + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().toast?.kind).toBe('opened') }) + f.face.revoke('a'.repeat(64) as CuaApplicationId) + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().toast?.kind).toBe('revoked') }) + f.face.retryApproval('a'.repeat(64) as CuaApplicationId) + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().toast?.kind).toBe('retryAllowed') }) + f.actions.revoke.mockRejectedValueOnce(error) + f.face.revoke('a'.repeat(64) as CuaApplicationId) + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().toast).toMatchObject({ kind: 'failed', message: 'write refused' }) }) + await f.controller.dispose() +}) + +it('joins a pending read without publishing after disposal or starting queued actions', async () => { + const f = fixture() + const pending = Promise.withResolvers() + const entered = Promise.withResolvers() + f.actions.readSystem.mockImplementationOnce(() => { entered.resolve(undefined); return pending.promise }) + f.face.refresh() + await entered.promise + f.face.request('accessibility') + const before = f.controller.state.getSnapshot() + const closed = f.controller.dispose() + pending.resolve(value) + await closed + expect(f.controller.state.getSnapshot()).toBe(before) + expect(f.actions.request).not.toHaveBeenCalled() + f.face.refresh() + expect(f.actions.readSystem).toHaveBeenCalledOnce() +}) + +it('revokes access while icons are pending, serializes writes, and joins reads on disposal', async () => { + const f = fixture(), icons = Promise.withResolvers>() + const action = Promise.withResolvers(), entered = Promise.withResolvers() + onTestFinished(async () => { icons.resolve({}); action.resolve(undefined); await f.controller.dispose() }) + f.actions.readIcons.mockReturnValueOnce(icons.promise) + f.actions.revoke.mockImplementationOnce(() => { entered.resolve(undefined); return action.promise }) + f.face.refresh() + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().applications.busy).toBe(false) }) + expect(f.controller.state.getSnapshot().icons.busy).toBe(true) + f.face.revoke('a'.repeat(64) as CuaApplicationId) + await entered.promise + f.face.retryApproval('a'.repeat(64) as CuaApplicationId) + await setImmediate() + expect(f.actions.retryApproval).not.toHaveBeenCalled() + action.resolve(undefined) + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) + expect(f.actions.retryApproval).toHaveBeenCalledOnce() + expect(f.controller.state.getSnapshot().toast?.kind).toBe('retryAllowed') + const before = f.controller.state.getSnapshot() + let disposed = false + const closed = f.controller.dispose().then(() => { disposed = true }) + await setImmediate() + expect(disposed).toBe(false) + icons.resolve({ '/old/app': 'old-icon' }) + await closed + expect(f.controller.state.getSnapshot()).toBe(before) +}) + +it('ignores stale toast completion after the permission UI is disposed', async () => { + const f = fixture() + f.face.request('accessibility') + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().toast?.kind).toBe('requested') }) + await f.controller.dispose() + const before = f.controller.state.getSnapshot() + f.face.dismissToast() + f.face.setActive(true) + expect(f.controller.state.getSnapshot()).toBe(before) +}) + +it('contains a late read rejection and supports errors without an Error instance', async () => { + const f = fixture() + f.actions.readSystem.mockRejectedValueOnce('offline') + f.face.refresh() + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().system.error).toBe('offline') }) + const pending = Promise.withResolvers(), entered = Promise.withResolvers() + f.actions.readSystem.mockImplementationOnce(() => { entered.resolve(undefined); return pending.promise }) + f.face.refresh() + await entered.promise + const before = f.controller.state.getSnapshot(), closed = f.controller.dispose() + pending.reject(new Error('late disconnect')) + await closed + expect(f.controller.state.getSnapshot()).toBe(before) +}) + + +it.each(['success', 'failure'])('does not publish an older refresh %s while the activation refresh is queued', async (outcome) => { + const f = fixture() + const first = Promise.withResolvers(), firstEntered = Promise.withResolvers() + const latest = Promise.withResolvers(), latestEntered = Promise.withResolvers() + onTestFinished(async () => { first.resolve(value); latest.resolve(value); await f.controller.dispose() }) + f.actions.readSystem.mockImplementationOnce(() => { firstEntered.resolve(undefined); return first.promise }) + f.actions.readSystem.mockImplementationOnce(() => { latestEntered.resolve(undefined); return latest.promise }) + const observations: Array> = [] + const unsubscribe = f.controller.state.subscribe(() => { observations.push(f.controller.state.getSnapshot()) }) + onTestFinished(unsubscribe) + f.face.refresh() + await firstEntered.promise + f.face.refresh() + if (outcome === 'failure') first.reject(new Error('provider is reloading')) + else first.resolve({ ...value, screenRecording: true }) + await latestEntered.promise + expect(observations.every(state => state.system.error === null && state.system.value === null && state.busy)).toBe(true) + latest.resolve(value) + await vi.waitFor(() => { expect(f.controller.state.getSnapshot()).toMatchObject({ busy: false, system: { error: null, value } }) }) + expect(observations.every(state => state.system.error === null)).toBe(true) +}) + +it('keeps a user-action failure visible when a refresh is queued behind it', async () => { + const f = fixture(), action = Promise.withResolvers(), entered = Promise.withResolvers() + onTestFinished(async () => { action.resolve(undefined); await f.controller.dispose() }) + f.actions.request.mockImplementationOnce(() => { entered.resolve(undefined); return action.promise }) + f.face.request('accessibility') + await entered.promise + f.face.refresh() + action.reject(new Error('permission request failed')) + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) + expect(f.controller.state.getSnapshot()).toMatchObject({ system: { value, error: null }, + toast: { kind: 'failed', message: 'permission request failed' } }) +}) + + +it.each(['success', 'failure'])('drops a permission read %s from the previous bundle activation', async (outcome) => { + const f = fixture() + f.face.setActive(true) + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) + const pending = Promise.withResolvers(), entered = Promise.withResolvers() + onTestFinished(async () => { pending.resolve(value); await f.controller.dispose() }) + f.actions.readSystem.mockImplementationOnce(() => { entered.resolve(undefined); return pending.promise }) + f.face.refresh() + await entered.promise + f.face.setActive(false) + f.face.setActive(false) + expect(f.controller.state.getSnapshot()).toMatchObject({ busy: true, system: { error: null, value } }) + f.face.request('accessibility') + expect(f.actions.request).not.toHaveBeenCalled() + if (outcome === 'failure') pending.reject(new Error('old helper stopped')) + else pending.resolve({ ...value, screenRecording: true }) + await Promise.allSettled([pending.promise]) + expect(f.controller.state.getSnapshot()).toMatchObject({ busy: true, system: { error: null, value } }) + f.face.setActive(true) + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) + expect(f.controller.state.getSnapshot()).toMatchObject({ system: { error: null, value } }) + expect(f.actions.readSystem).toHaveBeenCalledTimes(3) +}) + +it.each(['success', 'failure'])('does not publish a user-action %s after its bundle is replaced', async (outcome) => { + const f = fixture(), pending = Promise.withResolvers(), entered = Promise.withResolvers() + onTestFinished(async () => { pending.resolve(undefined); await f.controller.dispose() }) + f.actions.request.mockImplementationOnce(() => { entered.resolve(undefined); return pending.promise }) + f.face.request('accessibility') + await entered.promise + f.face.setActive(false) + if (outcome === 'failure') pending.reject(new Error('old request cancelled')) + else pending.resolve(undefined) + f.face.setActive(true) + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) + expect(f.controller.state.getSnapshot()).toMatchObject({ toast: null, system: { error: null, value } }) + expect(f.actions.readSystem).toHaveBeenCalledOnce() +}) + +it('publishes each observation before the other queries finish', async () => { + const f = fixture() + const system = Promise.withResolvers() + const apps = Promise.withResolvers() + const icons = Promise.withResolvers>() + onTestFinished(async () => { system.resolve(value); apps.resolve(applications); icons.resolve({}); await f.controller.dispose() }) + f.actions.readSystem.mockReturnValueOnce(system.promise) + f.actions.readApplications.mockReturnValueOnce(apps.promise) + f.actions.readIcons.mockReturnValueOnce(icons.promise) + f.face.refresh() + apps.resolve(applications) + await vi.waitFor(() => { + expect(f.controller.state.getSnapshot().applications).toEqual({ value: applications, error: null, busy: false }) + }) + expect(f.controller.state.getSnapshot().system).toMatchObject({ value: null, busy: true }) + system.resolve(value) + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().system).toEqual({ value, error: null, busy: false }) }) + expect(f.controller.state.getSnapshot().icons).toMatchObject({ value: null, busy: true }) + icons.reject(new Error('icon unavailable')) + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) + expect(f.controller.state.getSnapshot()).toMatchObject({ + system: { value, error: null }, applications: { value: applications, error: null }, + icons: { value: null, error: 'icon unavailable', busy: false }, + }) +}) + +it('keeps application decisions available when system inspection fails', async () => { + const f = fixture() + f.actions.readSystem.mockRejectedValueOnce(new Error('helper offline')) + f.face.refresh() + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) + expect(f.controller.state.getSnapshot()).toMatchObject({ + system: { value: null, error: 'helper offline' }, + applications: { value: applications, error: null, busy: false }, + }) + await f.controller.dispose() +}) + +it.each(['success', 'failure'])('drops old icon %s after disabling and reenabling the bundle', async (outcome) => { + const f = fixture(), pending = Promise.withResolvers>() + onTestFinished(async () => { pending.resolve({}); await f.controller.dispose() }) + f.actions.readIcons.mockReturnValueOnce(pending.promise) + f.face.refresh() + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().system.busy).toBe(false) }) + f.face.setActive(false) + f.face.setActive(true) + if (outcome === 'success') pending.resolve({ '/old/app': 'old-icon' }) + else pending.reject(new Error('old helper stopped')) + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) + expect(f.controller.state.getSnapshot().icons).toEqual({ value: {}, error: null, busy: false }) +}) diff --git a/packages/experimental/client-ui-cua/tests/mount.client.spec.ts b/packages/experimental/client-ui-cua/tests/mount.client.spec.ts new file mode 100644 index 0000000000..c79bfed54b --- /dev/null +++ b/packages/experimental/client-ui-cua/tests/mount.client.spec.ts @@ -0,0 +1,181 @@ +/** The optional permission namespace and existing page slots share one disposal lifetime. */ +import assert from 'node:assert/strict' +import { setImmediate } from 'node:timers/promises' +import { Context, Service } from '@deepseek-ai/cordis' +import { LocaleRuntime } from '@deepseek-ai/dsh-client-locale/client' +import { SlotRegistry } from '@deepseek-ai/dsh-client-ui-renderer/client' +import { createSnapshotStore } from '@deepseek-ai/dsh-client-store' +import { RemoteError, type RemoteResult, type TypertRemoteContribution } from '@deepseek-ai/dsh-typert-protocol' +import type { CuaApplicationState, CuaSystemPermissionState } from '@deepseek-ai/dsh-cua/types' +import { expect, it, vi } from 'vitest' +import type { PermissionFace } from '../src/client/controller.ts' +import { mountCuaPermissions, inject } from '../src/client/mount.ts' +import { apply as hostApply } from '../src/index.ts' + +const contribution: TypertRemoteContribution = { package: '@deepseek-ai/dsh-cua', descriptors: [] } +const value: CuaApplicationState & { system: CuaSystemPermissionState } = { + system: { accessibility: false, screenRecording: false, helper: { pid: 2 }, parent: { pid: 1 } }, + allowedApps: [], blockedApps: [], settingsNamespace: 'cua', persistentAvailable: true } + +function actionsOf(value: Record): asserts value is Record & PermissionFace { + assert(typeof value.setActive === 'function') + assert(typeof value.refresh === 'function' && typeof value.request === 'function' && typeof value.openSettings === 'function') + assert(typeof value.revoke === 'function' && typeof value.dismissToast === 'function') + assert(typeof value.retryApproval === 'function') + assert(typeof value.hooks === 'object' && value.hooks !== null) +} + +async function fixture(fail = false) { + const ctx = new Context(), unmount = vi.fn(async () => {}), unsubscribe = vi.fn() + let settingsListener: ((namespace: string, revision: number) => void) | undefined + class Remote extends Service { + constructor() { super(ctx, 'remote') } + async $mount(received: TypertRemoteContribution) { + expect(received).toBe(contribution) + const owned = this.ctx.effect(() => unmount) + return () => owned() + } + $on(_event: string, listener: (namespace: string, revision: number) => void) { settingsListener = listener; return unsubscribe } + } + new Remote() + const setDefault = vi.fn(async () => true) + const preset = createSnapshotStore({ status: 'ready' as const, value: { default: 'standard' }, + base: {}, user: {}, revision: 1, writable: true }) + ctx.provide('configForms', { get: (namespace: string) => { + expect(namespace).toBe('agent-preset-registry') + return { ...preset, set: setDefault } + } }) + const status = vi.fn<() => Promise>>().mockResolvedValue({ ok: true, value }) + const systemStatus = vi.fn<() => Promise>>().mockResolvedValue({ ok: true, value: value.system }) + const applicationIcons = vi.fn<() => Promise>>>().mockResolvedValue({ ok: true, value: {} }) + const requestPermission = vi.fn<() => Promise>>() + .mockResolvedValue({ ok: true, value: value.system }) + const openSettings = vi.fn<() => Promise>>().mockResolvedValue({ ok: true, value: undefined }) + const revoke = vi.fn<() => Promise>>().mockResolvedValue({ ok: true, value: undefined }) + const retryApproval = vi.fn<() => Promise>>().mockResolvedValue({ ok: true, value: undefined }) + ctx.provide('remote.cuaAccess', { applications: status, systemStatus, applicationIcons, requestPermission, openSettings, revoke, retryApproval }) + ctx.provide('locale', new LocaleRuntime(ctx)) + await ctx.plugin(SlotRegistry) + ctx.slots.register({ name: 'root', children: { + 'plugins.bundle.activation': { kind: 'keyed', scope: 'root' }, + 'plugins.bundle.config': { kind: 'keyed', scope: 'root' }, 'shell.overlay': { kind: 'list', scope: 'root' }, + } } as never, () => null) + if (fail) vi.spyOn(ctx.slots, 'inject').mockImplementationOnce(() => { throw new Error('slot unavailable') }) + return { ctx, unmount, unsubscribe, status, systemStatus, applicationIcons, + requestPermission, openSettings, revoke, retryApproval, setDefault, + updateSettings(namespace: string) { settingsListener!(namespace, 1) }, + } +} + +it('withdraws the permission Remote and all controls with the optional plugin', async () => { + hostApply() + const f = await fixture() + try { + const fiber = f.ctx.plugin({ inject: [...inject], apply: ctx => mountCuaPermissions(ctx, contribution) }) + await fiber + const entry = f.ctx.slots.entries('plugins.bundle.config')[0]! + const actions = entry.inject!() + actionsOf(actions) + actions.refresh() + await vi.waitFor(() => { expect(actions.hooks.cuaPermissions.getSnapshot().applications.value).toEqual(value) }) + actions.request('accessibility') + await vi.waitFor(() => { expect(actions.hooks.cuaPermissions.getSnapshot().toast?.kind).toBe('requested') }) + actions.openSettings('screenRecording') + await vi.waitFor(() => { expect(actions.hooks.cuaPermissions.getSnapshot().toast?.kind).toBe('opened') }) + actions.revoke('a'.repeat(64) as import('@deepseek-ai/dsh-cua/types').CuaApplicationId) + await vi.waitFor(() => { expect(actions.hooks.cuaPermissions.getSnapshot().toast?.kind).toBe('revoked') }) + actions.retryApproval('a'.repeat(64) as import('@deepseek-ai/dsh-cua/types').CuaApplicationId) + await vi.waitFor(() => { expect(actions.hooks.cuaPermissions.getSnapshot().toast?.kind).toBe('retryAllowed') }) + const failed = { ok: false as const, error: new RemoteError('gateway/internal', 'offline', {}) } + for (const [target, invoke] of [ + [f.status, actions.refresh], [f.systemStatus, actions.refresh], [f.applicationIcons, actions.refresh], [f.requestPermission, () => { actions.request('accessibility') }], + [f.openSettings, () => { actions.openSettings('accessibility') }], + [f.revoke, () => { actions.revoke('a'.repeat(64) as import('@deepseek-ai/dsh-cua/types').CuaApplicationId) }], + [f.retryApproval, () => { actions.retryApproval('a'.repeat(64) as import('@deepseek-ai/dsh-cua/types').CuaApplicationId) }], + ] as const) { + actions.refresh() + await vi.waitFor(() => { expect(actions.hooks.cuaPermissions.getSnapshot().busy).toBe(false) }) + target.mockResolvedValueOnce(failed) + invoke() + await vi.waitFor(() => { + const state = actions.hooks.cuaPermissions.getSnapshot() + const error = target === f.status ? state.applications.error : target === f.systemStatus ? state.system.error + : target === f.applicationIcons ? state.icons.error : state.toast?.kind === 'failed' ? state.toast.message : null + expect(error).toContain('offline') + }) + } + for (const name of ['plugins.bundle.activation', 'shell.overlay'] as const) { + const other = f.ctx.slots.entries(name)[0]! + const otherActions = other.inject!() + actionsOf(otherActions) + expect(otherActions.hooks.cuaPermissions).toBe(actions.hooks.cuaPermissions) + } + const settings = f.ctx.slots.entries('plugins.bundle.config')[0]!.inject!() + assert(typeof settings.setPtcDefault === 'function') + expect(f.ctx.slots.entries('plugins.bundle.activation')[0]!.inject!()).not.toHaveProperty('setPtcDefault') + expect(f.setDefault).not.toHaveBeenCalled() + await (settings.setPtcDefault as () => Promise)() + expect(f.setDefault).toHaveBeenCalledExactlyOnceWith('selectedDefault', 'ptc') + await fiber.dispose() + expect(f.ctx.slots.entries('conversation.composer')).toHaveLength(0) + expect(f.ctx.slots.entries('plugins.bundle.config')).toHaveLength(0) + expect(f.ctx.slots.entries('shell.overlay')).toHaveLength(0) + expect(f.unsubscribe).toHaveBeenCalledOnce() + expect(f.unmount).toHaveBeenCalledOnce() + } finally { await f.ctx.fiber.dispose() } +}) + +it('withdraws the Remote contribution when UI registration fails', async () => { + const f = await fixture(true) + try { + await expect(mountCuaPermissions(f.ctx, contribution)).rejects.toThrow('slot unavailable') + expect(f.unmount).toHaveBeenCalledOnce() + } finally { await f.ctx.fiber.dispose() } +}) + +it.each(['plugin', 'root'])('keeps the Remote mounted until an accepted revoke completes during %s disposal', async (owner) => { + const f = await fixture() + const response = Promise.withResolvers>(), entered = Promise.withResolvers() + f.revoke.mockImplementationOnce(() => { entered.resolve(undefined); return response.promise }) + f.unmount.mockImplementation(async () => { response.resolve({ ok: false, error: new RemoteError('gateway/internal', 'unmounted', {}) }) }) + let closing: Promise | undefined + try { + const fiber = await f.ctx.plugin({ inject: [...inject], apply: ctx => mountCuaPermissions(ctx, contribution) }) + const actions = f.ctx.slots.entries('plugins.bundle.config')[0]!.inject!() + actionsOf(actions) + actions.revoke('a'.repeat(64) as import('@deepseek-ai/dsh-cua/types').CuaApplicationId) + await entered.promise + let closed = false + closing = Promise.resolve(owner === 'plugin' ? fiber.dispose() : f.ctx.fiber.dispose()).then(() => { closed = true }) + await setImmediate() + expect(f.unmount).not.toHaveBeenCalled() + expect(closed).toBe(false) + response.resolve({ ok: true, value: undefined }) + await closing + expect(f.unmount).toHaveBeenCalledOnce() + } finally { + response.resolve({ ok: true, value: undefined }) + await closing + await f.ctx.fiber.dispose() + } +}) + +it('refreshes only the settings namespace reported by the permission owner', async () => { + const f = await fixture() + f.status.mockResolvedValue({ ok: true, value: { ...value, settingsNamespace: 'custom-cua-entry' } }) + try { + await f.ctx.plugin({ inject: [...inject], apply: ctx => mountCuaPermissions(ctx, contribution) }) + const actions = f.ctx.slots.entries('plugins.bundle.config')[0]!.inject!() + actionsOf(actions) + f.updateSettings('locale') + expect(f.status).not.toHaveBeenCalled() + actions.refresh() + await vi.waitFor(() => { expect(actions.hooks.cuaPermissions.getSnapshot().busy).toBe(false) }) + expect(f.status).toHaveBeenCalledOnce() + f.updateSettings('cua') + f.updateSettings('locale') + expect(f.status).toHaveBeenCalledOnce() + f.updateSettings('custom-cua-entry') + await vi.waitFor(() => { expect(f.status).toHaveBeenCalledTimes(2) }) + } finally { await f.ctx.fiber.dispose() } +}) diff --git a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx b/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx new file mode 100644 index 0000000000..905321e79c --- /dev/null +++ b/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx @@ -0,0 +1,332 @@ +// @vitest-environment jsdom +/** Permission UI distinguishes missing privileges, uncertainty, and application consent. */ +import { act, cleanup, fireEvent, render, screen, waitFor, within } from '@testing-library/react' +import { bindSnapshotSelector, makeTranslate } from '@deepseek-ai/dsh-client-test-runtime' +import { createSnapshotStore } from '@deepseek-ai/dsh-client-store' +import type { CuaApplicationId, CuaApplicationState, CuaSystemPermissionState } from '@deepseek-ai/dsh-cua/types' +import type { ConfigFormSnapshot } from '@deepseek-ai/dsh-client-ui-settings/client' +import { afterEach, expect, it, vi } from 'vitest' +import { PermissionPanel, PermissionSetup, PermissionToast, type PermissionPanelProps } from '../src/client/PermissionPanel.tsx' +import type { PermissionView } from '../src/client/controller.ts' +import { en, zh } from '../src/client/locales.ts' + +afterEach(() => { cleanup(); vi.useRealTimers() }) +const value: CuaApplicationState & { system: CuaSystemPermissionState } = { system: { accessibility: false, screenRecording: true, + helper: { pid: 9, path: '/native/helper' }, parent: { pid: 8, path: '/host/node' } }, +allowedApps: [], blockedApps: [], settingsNamespace: 'cua', persistentAvailable: true } + +function observe(value: CuaApplicationState & { system: CuaSystemPermissionState }) { + return { system: { value: value.system, busy: false, error: null }, + applications: { value, busy: false, error: null }, icons: { value: {}, busy: false, error: null }, busy: false } +} + +function fixture() { + const state = createSnapshotStore({ + system: { value: null, busy: true, error: null }, applications: { value: null, busy: true, error: null }, + icons: { value: null, busy: true, error: null }, busy: true, toast: null, + }) + const preset = createSnapshotStore>({ + status: 'ready', value: { default: 'standard', selectedDefault: 'ptc' }, + base: {}, user: {}, revision: 1, writable: true, + }) + const refresh = vi.fn() + const props = { useCuaPermissions: bindSnapshotSelector(state), t: makeTranslate(en), + useCuaDefaultPreset: bindSnapshotSelector(preset), setPtcDefault: vi.fn(async () => true), + refresh, setActive: vi.fn((active: boolean) => { if (active) refresh() }), request: vi.fn(), openSettings: vi.fn(), revoke: vi.fn(), + retryApproval: vi.fn(), dismissToast: vi.fn(), + } satisfies PermissionPanelProps + return { state, preset, props, + publish(patch: Partial) { act(() => { state.set({ ...state.getSnapshot(), ...patch }) }) }, + } +} + +it('requests only a missing privilege and leaves OS attribution to the system prompt', () => { + const f = fixture() + const shown = render() + expect(f.props.refresh).toHaveBeenCalledOnce() + f.publish({ ...observe(value) }) + fireEvent.click(screen.getByRole('button', { name: en.request })) + expect(f.props.request).toHaveBeenCalledWith('accessibility') + const controls = screen.getAllByRole('button', { name: en.settings }) + fireEvent.click(controls[0]!) + fireEvent.click(controls[1]!) + expect(f.props.openSettings.mock.calls).toEqual([['accessibility'], ['screenRecording']]) + fireEvent.focus(window) + expect(f.props.refresh).toHaveBeenCalledTimes(2) + shown.unmount() + fireEvent.focus(window) + expect(f.props.refresh).toHaveBeenCalledTimes(2) +}) + +it.each([en, zh])('changes the default from the detail page and retains a refused write for retry', async (dictionary) => { + const f = fixture() + f.preset.set({ ...f.preset.getSnapshot(), value: { default: 'standard' } }) + f.publish({ ...observe(value) }) + render() + const group = screen.getByRole('group', { name: dictionary.defaultTitle }) + expect(within(group).getByText(dictionary.defaultBody)).toBeTruthy() + expect(f.props.setPtcDefault).not.toHaveBeenCalled() + const write = Promise.withResolvers() + f.props.setPtcDefault.mockReturnValueOnce(write.promise) + fireEvent.click(within(group).getByRole('button', { name: dictionary.defaultConfirm })) + expect(within(group).getByRole('button', { name: dictionary.defaultConfirm }).hasAttribute('disabled')).toBe(true) + await act(async () => { write.resolve(false); await write.promise }) + expect(screen.getByRole('alert').textContent).toBe(dictionary.defaultNotSaved) + fireEvent.click(within(group).getByRole('button', { name: dictionary.defaultConfirm })) + await waitFor(() => { expect(screen.queryByRole('alert')).toBeNull() }) + expect(f.props.setPtcDefault).toHaveBeenCalledTimes(2) + act(() => { f.preset.set({ ...f.preset.getSnapshot(), value: { default: 'standard', selectedDefault: 'ptc' } }) }) + expect(within(group).getByRole('button', { name: dictionary.defaultSet }).hasAttribute('disabled')).toBe(true) +}) + +it('dismisses activation when permissions are ready without changing the default', () => { + const f = fixture(), onDismiss = vi.fn() + f.preset.set({ ...f.preset.getSnapshot(), value: { default: 'standard', selectedDefault: 'minimal' } }) + f.publish({ ...observe({ ...value, system: { ...value.system, accessibility: true } }) }) + render() + expect(onDismiss).toHaveBeenCalledOnce() + expect(screen.queryByRole('dialog')).toBeNull() + expect(f.props.setPtcDefault).not.toHaveBeenCalled() +}) + +it.each([new Error('offline'), 'offline'])('reports a failed default-mode request on the detail page', async (error) => { + const f = fixture() + f.preset.set({ ...f.preset.getSnapshot(), value: { default: 'standard' } }) + f.props.setPtcDefault.mockRejectedValueOnce(error) + render() + fireEvent.click(screen.getByRole('button', { name: en.defaultConfirm })) + await waitFor(() => { expect(screen.getByRole('alert').textContent).toContain('offline') }) +}) + +it('waits for the settings document and recognizes PTC as the deployment default', () => { + const f = fixture() + f.preset.set({ ...f.preset.getSnapshot(), status: 'loading', value: undefined }) + render() + expect(screen.queryByRole('button', { name: en.defaultConfirm })).toBeNull() + act(() => { f.preset.set({ ...f.preset.getSnapshot(), status: 'ready', value: { default: 'ptc' } }) }) + expect(screen.getByRole('button', { name: en.defaultSet }).hasAttribute('disabled')).toBe(true) + expect(f.props.setPtcDefault).not.toHaveBeenCalled() +}) + +it.each(['unavailable', 'ready'] as const)('explains why the %s settings document cannot change the default', (status) => { + const f = fixture() + f.preset.set({ ...f.preset.getSnapshot(), status, writable: false, + value: status === 'ready' ? { default: 'standard' } : undefined }) + render() + expect(screen.getByRole('alert').textContent).toBe(status === 'ready' ? en.defaultReadOnly : en.defaultUnavailable) + expect(screen.getByRole('button', { name: en.defaultConfirm }).hasAttribute('disabled')).toBe(true) + expect(f.props.setPtcDefault).not.toHaveBeenCalled() +}) + +it('shows query failures as unknown while retaining previously displayed apps', () => { + const f = fixture(), app = { id: 'a'.repeat(64) as CuaApplicationId, name: 'Fixture', executablePath: '/fixture/app' } + render() + f.publish({ ...observe({ ...value, allowedApps: [app] }), system: { value: value.system, busy: false, error: 'offline' }, busy: false }) + expect(screen.getAllByRole('button', { name: en.revoke })).toHaveLength(1) + expect(screen.getAllByText(en.unknown)).toHaveLength(2) + expect(screen.queryByText(en.granted)).toBeNull() + expect(screen.getByRole('alert').textContent).toContain('offline') + fireEvent.click(screen.getAllByRole('button', { name: en.revoke })[0]!) + expect(f.props.revoke).toHaveBeenCalledWith(app.id) + fireEvent.click(screen.getByRole('button', { name: en.retry })) + expect(f.props.refresh).toHaveBeenCalledTimes(2) + f.publish({ system: { ...f.state.getSnapshot().system, busy: true }, busy: true }) + expect(screen.queryByRole('button', { name: en.retry })).toBeNull() + f.publish({ system: { ...f.state.getSnapshot().system, busy: false, error: null }, busy: false }) + expect(screen.queryByRole('button', { name: en.retry })).toBeNull() + f.publish(observe(value)) + expect(screen.getByText(en.noApps)).toBeTruthy() + expect(screen.queryByRole('button', { name: en.revoke })).toBeNull() +}) + +it('keeps unknown initial status distinct from a confirmed missing privilege in Chinese', () => { + const f = fixture() + render() + f.publish({ system: { ...f.state.getSnapshot().system, busy: false, error: 'connection lost' }, busy: false }) + expect(screen.queryByText(zh.granted)).toBeNull() + expect(screen.queryByRole('button', { name: zh.request })).toBeNull() + f.publish({ ...observe({ ...value, system: { ...value.system, accessibility: true, screenRecording: false } }) }) + fireEvent.click(screen.getByRole('button', { name: zh.request })) + expect(f.props.request).toHaveBeenCalledWith('screenRecording') +}) + +it('offers setup after inspection, permits dismissal, and dismisses once both privileges are observed', () => { + const f = fixture(), onDismiss = vi.fn(), onOpenDetails = vi.fn() + render() + expect(screen.queryByRole('dialog')).toBeNull() + f.publish({ ...observe(value) }) + const dialog = screen.getByRole('dialog', { name: en.setupTitle }) + fireEvent.click(within(dialog).getByRole('button', { name: en.openSetup })) + expect(onOpenDetails).toHaveBeenCalledOnce() + fireEvent.click(within(dialog).getByRole('button', { name: en.later })) + expect(onDismiss).toHaveBeenCalledOnce() + f.publish({ ...observe({ ...value, system: { ...value.system, accessibility: true } }) }) + expect(screen.queryByRole('dialog')).toBeNull() + expect(onDismiss).toHaveBeenCalledTimes(2) +}) + +it('offers the details page when permission inspection fails', () => { + const f = fixture() + render() + f.publish({ system: { ...f.state.getSnapshot().system, busy: false, error: 'offline' }, busy: false }) + expect(screen.getByRole('dialog')).toBeTruthy() + expect(screen.queryByRole('alert')).toBeNull() + expect(screen.getByText(en.setupQueryFailed)).toBeTruthy() + expect(screen.queryByText(en.setupBody)).toBeNull() +}) + +it('exposes a new-request action for a blocked application without calling grant or revoke', () => { + const f = fixture(), app = { id: 'a'.repeat(64) as CuaApplicationId, name: 'Fixture', executablePath: '/fixture/app' } + render() + f.publish({ ...observe({ ...value, blockedApps: [app] }) }) + expect(screen.queryByText(en.noApps)).toBeNull() + fireEvent.click(screen.getByRole('button', { name: en.retryApproval })) + expect(f.props.retryApproval).toHaveBeenCalledWith(app.id) + expect(f.props.revoke).not.toHaveBeenCalled() + expect(f.props.request).not.toHaveBeenCalled() +}) + +it('renders action outcomes outside the permission page and clears finished toasts', async () => { + vi.useFakeTimers() + const f = fixture() + render() + expect(screen.queryByRole('alert')).toBeNull() + f.publish({ toast: { id: 1, kind: 'revoked' } }) + expect(screen.getByRole('alert').textContent).toBe(en.revoked) + await act(async () => { await vi.advanceTimersByTimeAsync(4000) }) + expect(f.props.dismissToast).toHaveBeenCalledOnce() + f.publish({ toast: { id: 2, kind: 'failed', message: 'write refused' } }) + expect(screen.getByRole('alert').textContent).toContain('write refused') +}) + +it('keeps permission guidance open while a fresh observation is pending', () => { + const f = fixture() + render() + f.publish({ ...observe(value) }) + const dialog = screen.getByRole('dialog') + f.publish({ system: { ...f.state.getSnapshot().system, busy: true }, busy: true }) + expect(screen.getByRole('dialog')).toBe(dialog) +}) + +it('shows application icons and names while keeping the executable path in the tooltip', () => { + const f = fixture() + render() + f.publish({ ...observe({ ...value, allowedApps: [{ id: 'a'.repeat(64) as CuaApplicationId, + name: 'Calculator', executablePath: '/Applications/Calculator.app/Contents/MacOS/Calculator' }] }), + icons: { value: { '/Applications/Calculator.app/Contents/MacOS/Calculator': 'data:image/png;base64,aWNvbg==' }, busy: false, error: null }, busy: false }) + expect(screen.getByText('Calculator')).toBeTruthy() + expect(screen.queryByText('/Applications/Calculator.app/Contents/MacOS/Calculator')).toBeNull() + const identity = screen.getByTitle('/Applications/Calculator.app/Contents/MacOS/Calculator') + expect(identity.querySelector('img')?.getAttribute('src')).toBe('data:image/png;base64,aWNvbg==') +}) + +it.each([en, zh])('shows three requirements with the target application beside each system permission', (dictionary) => { + const f = fixture() + render() + const application = { name: 'Visual Studio Code.app', path: '/Applications/Visual Studio Code.app', icon: 'data:image/png;base64,aWNvbg==' } + f.publish({ ...observe({ ...value, system: { ...value.system, parent: { pid: 1, path: '/sbin/launchd' }, launchApplication: application } }) }) + const requirements = screen.getByRole('region', { name: dictionary.title }) + expect(within(requirements).getAllByRole('group')).toHaveLength(3) + expect(screen.getByRole('region', { name: dictionary.applicationsTitle })).toBeTruthy() + const accessibility = within(requirements).getByRole('group', { name: dictionary.accessibility }) + expect(accessibility.textContent).toContain(dictionary.permissionPrefix) + expect(within(accessibility).getByText(application.name)).toBeTruthy() + expect(accessibility.textContent).toContain(dictionary.accessibilityAction) + expect(accessibility.querySelector('img')?.getAttribute('src')).toBe(application.icon) + const recording = within(requirements).getByRole('group', { name: dictionary.screenRecording }) + expect(within(recording).getByText(application.name)).toBeTruthy() + expect(recording.textContent).toContain(dictionary.screenRecordingAction) + expect(within(requirements).getByText(dictionary.permissionHint)).toBeTruthy() + f.publish({ system: { ...f.state.getSnapshot().system, error: 'offline' } }) + expect(screen.queryByText(application.name)).toBeNull() + expect(within(requirements).getByText(dictionary.permissionHintUnknown)).toBeTruthy() +}) + +it('names the authorization target even when its icon is unavailable', () => { + const f = fixture() + render() + f.publish({ ...observe({ ...value, system: { ...value.system, + launchApplication: { name: 'iTerm.app', path: '/Applications/iTerm.app' } } }) }) + const accessibility = screen.getByRole('group', { name: en.accessibility }) + expect(within(accessibility).getByText('iTerm.app')).toBeTruthy() + expect(accessibility.querySelector('img')).toBeNull() +}) + +it('waits for the first current observation instead of opening setup with a stale error', () => { + const f = fixture() + render() + f.publish({ system: { ...f.state.getSnapshot().system, busy: true, error: 'provider is reloading' }, busy: true }) + expect(screen.queryByRole('dialog')).toBeNull() + f.publish({ ...observe(value) }) + expect(screen.getByRole('dialog', { name: en.setupTitle })).toBeTruthy() + expect(screen.queryByRole('alert')).toBeNull() +}) + + +it('omits setup instructions once both system permissions are granted', () => { + const f = fixture() + render() + f.publish({ ...observe({ ...value, system: { ...value.system, accessibility: true, screenRecording: true } }) }) + expect(screen.getAllByText(en.granted)).toHaveLength(2) + expect(screen.queryByText(en.permissionHintUnknown)).toBeNull() +}) + + +it('keeps all three requirements visible before the first permission response', () => { + const f = fixture() + render() + const requirements = screen.getByRole('region', { name: en.title }) + expect(within(requirements).getAllByRole('group')).toHaveLength(3) + expect(screen.queryByRole('alert')).toBeNull() + expect(screen.queryByRole('button', { name: en.request })).toBeNull() + for (const button of screen.getAllByRole('button', { name: en.settings })) expect(button.hasAttribute('disabled')).toBe(true) +}) + +it('fills application names, system permissions, and icons as their observations arrive', () => { + const f = fixture() + render() + const requirements = screen.getByRole('region', { name: en.title }) + const access = screen.getByRole('region', { name: en.applicationsTitle }) + const app = { id: 'a'.repeat(64) as CuaApplicationId, name: 'Calculator', executablePath: '/Applications/Calculator.app' } + f.publish({ applications: { value: { ...value, allowedApps: [app] }, busy: false, error: null } }) + expect(within(access).getByText(app.name)).toBeTruthy() + expect(within(access).getByRole('button', { name: en.revoke }).hasAttribute('disabled')).toBe(false) + expect(screen.queryByText(en.granted)).toBeNull() + const identity = within(access).getByTitle(app.executablePath) + expect(identity.querySelector('img')).toBeNull() + const placeholder = identity.querySelector('[aria-hidden]')! + expect(placeholder).toBeTruthy() + f.publish({ system: { value: value.system, busy: false, error: null } }) + expect(within(requirements).getByText(en.granted)).toBeTruthy() + expect(within(requirements).getByRole('button', { name: en.request }).hasAttribute('disabled')).toBe(false) + expect(within(access).getByTitle(app.executablePath)).toBe(identity) + expect(identity.querySelector('[aria-hidden]')).toBe(placeholder) + f.publish({ icons: { value: { [app.executablePath]: 'data:image/png;base64,aWNvbg==' }, busy: false, error: null } }) + expect(identity.querySelector('img')?.getAttribute('src')).toBe('data:image/png;base64,aWNvbg==') + expect(identity.querySelector('[aria-hidden]')).toBeNull() + expect(screen.getByRole('region', { name: en.title })).toBe(requirements) +}) + +it.each(['applications', 'icons'] as const)('contains a failed %s query in the application section', (key) => { + const f = fixture() + f.publish({ ...observe(value), [key]: { value: null, busy: false, error: 'offline' } }) + const view = render() + const requirements = screen.getByRole('region', { name: en.title }) + const access = screen.getByRole('region', { name: en.applicationsTitle }) + expect(within(requirements).getByText(en.granted)).toBeTruthy() + expect(within(requirements).queryByRole('alert')).toBeNull() + expect(within(access).getByRole('alert').textContent).toContain('offline') + fireEvent.click(within(access).getByRole('button', { name: en.retry })) + expect(f.props.refresh).toHaveBeenCalledTimes(2) + view.rerender() + expect(screen.queryByRole('alert')).toBeNull() +}) + +it('finishes activation guidance without waiting for application icons', () => { + const f = fixture(), onDismiss = vi.fn() + render() + f.publish({ system: { value: { ...value.system, accessibility: true }, busy: false, error: null } }) + expect(onDismiss).toHaveBeenCalledOnce() + expect(screen.queryByRole('dialog')).toBeNull() + expect(f.state.getSnapshot().icons.busy).toBe(true) +}) diff --git a/packages/experimental/client-ui-cua/tsconfig.json b/packages/experimental/client-ui-cua/tsconfig.json new file mode 100644 index 0000000000..66cf034b76 --- /dev/null +++ b/packages/experimental/client-ui-cua/tsconfig.json @@ -0,0 +1,51 @@ +{ + "extends": "../../../tsconfig.base.client.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib/types" + }, + "include": [ + "src" + ], + "references": [ + { + "path": "../../../vendor/cordis" + }, + { + "path": "../../client/locale/tsconfig.client.json" + }, + { + "path": "../../client/store" + }, + { + "path": "../../client/ui-primitives" + }, + { + "path": "../../client/ui-renderer" + }, + { + "path": "../../client/ui-slots" + }, + { + "path": "../../client/ui-plugin-manager/tsconfig.client.json" + }, + { + "path": "../../client/ui-settings/tsconfig.client.json" + }, + { + "path": "../../client/ui-layout" + }, + { + "path": "../../api/remotes/tsconfig.client.json" + }, + { + "path": "../../api/gateway/tsconfig.client.json" + }, + { + "path": "../cua" + }, + { + "path": "../../typert/protocol" + } + ] +} diff --git a/packages/experimental/client-ui-cua/tsdown.config.ts b/packages/experimental/client-ui-cua/tsdown.config.ts new file mode 100644 index 0000000000..71d5bd77f8 --- /dev/null +++ b/packages/experimental/client-ui-cua/tsdown.config.ts @@ -0,0 +1,3 @@ +import { clientBundle } from '../../client/tsdown.client.ts' + +export default clientBundle(import.meta.dirname, ['lib/types/index.js']) diff --git a/packages/experimental/cua-bundle/README.i18n.yaml b/packages/experimental/cua-bundle/README.i18n.yaml new file mode 100644 index 0000000000..e379b108fa --- /dev/null +++ b/packages/experimental/cua-bundle/README.i18n.yaml @@ -0,0 +1,46 @@ +# Bilingual-pair consistency record for README.md (docs/i18n/README.md): per heading +# section, a hash of its English and Chinese blocks outside code blocks and generated regions. +# After editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write packages/experimental/cua-bundle/README.md +/: + en: cc518dcd97629a96 + zh: 465e999011282dc3 +/deepseek-ai-dsh-cua-bundle: + en: 9084fb48d0f6489c + zh: ce692972d91cf663 +/deepseek-ai-dsh-cua-bundle/summary: + en: 06d8bf4ee1ef6f3b + zh: 8568f649fb846615 +/deepseek-ai-dsh-cua-bundle/table-of-contents: + en: b563e4ec4727d0de + zh: d0228da978238e05 +/deepseek-ai-dsh-cua-bundle/use-this-package: + en: 7d378196959b1629 + zh: e735e217d1e35216 +/deepseek-ai-dsh-cua-bundle/understand-the-implementation: + en: 057771fe13aaa337 + zh: 4fdb2b98ff3e7a78 +/deepseek-ai-dsh-cua-bundle/further-exploration: + en: 9f15d56b5b16e1ca + zh: 75eb08e8828c8877 +/deepseek-ai-dsh-cua-bundle/model-experience: + en: 215e7ba838619b7b + zh: a311da3843709f90 +/deepseek-ai-dsh-cua-bundle/model-experience/computer-use-in-ptc-sessions: + en: e8f11f45b375771e + zh: e8f11f45b375771e +/deepseek-ai-dsh-cua-bundle/model-experience/computer-use-in-ptc-sessions/what-the-model-sees: + en: 591af735d1eb92f6 + zh: 3632bffa33c77b28 +/deepseek-ai-dsh-cua-bundle/model-experience/computer-use-in-ptc-sessions/token-effect: + en: e8de232e96220fe7 + zh: 5c791307e2123667 +/deepseek-ai-dsh-cua-bundle/model-experience/computer-use-in-ptc-sessions/kv-cache-effect: + en: 417c40acd93df532 + zh: 69cc5cf7140c75a9 +/deepseek-ai-dsh-cua-bundle/known-limitations-and-deferred-work: + en: 41f8c54d76d92c95 + zh: 44c9662e437f4cbe +/deepseek-ai-dsh-cua-bundle/known-limitations-and-deferred-work/dev-note: + en: 0a1253c6f7b5e8b9 + zh: ce96645b3e75874e diff --git a/packages/experimental/cua-bundle/README.md b/packages/experimental/cua-bundle/README.md new file mode 100644 index 0000000000..d87df12f07 --- /dev/null +++ b/packages/experimental/cua-bundle/README.md @@ -0,0 +1,94 @@ +--- +description: "Enable optional macOS Computer Use and its permission page." +kind: "package-bundle" +--- + +# @deepseek-ai/dsh-cua-bundle + +English | [中文](README.zh.md) + +## Summary + +Enable macOS Computer Use from Plugins for existing and new PTC sessions. Windows support is in development. This bundle remains in `experimental` until that support is complete, then moves to the `computer-use` group. It is off by default; disabled profiles prepare no native components and register neither CUA tools nor their model guidance. + +## Table of Contents + +- [Use this package](#use-this-package) +- [Understand the implementation](#understand-the-implementation) +- [Further Exploration](#further-exploration) +- [Model Experience](#model-experience) +- [Known Limitations and Deferred Work](#known-limitations-and-deferred-work) + +----- + + +## Use this package + +This workflow targets developers running macOS Web or unpackaged Desktop from source. Open Plugins → Computer Use and enable the switch. The permission page identifies the responsible application reported by macOS when the optional system query is available. Request access and enable that application in System Settings; follow the application named by macOS if it differs from the detected source. For Web, run these commands from the repository root: + +```sh +pnpm run build +pnpm dsh web +``` + +For the source Desktop launcher and its separate development profile, follow [Desktop development](../../../apps/desktop/README.md). Enable Computer Use in each profile that needs it. + +The switch saves bundle selection in the profile’s `package.json`. Computer Use is installed only in PTC sessions. Enabling from the plugin list offers setup guidance when system permissions need attention; enabling from its detail page displays the permission controls directly. The detail page offers **Set to PTC** for the new-session default; existing sessions keep their preset. The switch leaves PTC execution mode and other settings unchanged. Native preparation runs when the provider loads and reuses the source-hashed executable when available. + +Disabling unloads the provider and withdraws CUA tools from its Agent scopes. Runtime configuration, the new-session default and saved grants remain. Session-only application decisions end with provider disposal. + +Computer Use requires a macOS DSH Host. Other operating systems can display the entry, but native provider startup fails. A browser on another operating system can connect to a supported Mac Host. + +If preparation fails, the saved selection remains and CUA operations stay unavailable. Read the Host startup diagnostic, correct its cause, then restart DSH. + +Explicit activation runs a read-only permission check, which may start a helper without opening a system dialog. Open the [permission page](../client-ui-cua/README.md) for system requests, settings navigation and automatic refresh. Tasks ask separately before accessing an application. Saved application decisions use the existing profile configuration; session decisions remain in memory across turns and connections until DSH restarts or Computer Use is disabled. Revoke access on the same page. + +----- + + +## Understand the implementation + +
+Maintainer details + +The bundle patch mounts the native provider and permission UI with `agentPreset: ptc`. Existing Agent scopes register the tools without changing preset documents. The detail-page action uses the existing settings form to save `agent-preset-registry.selectedDefault: ptc` only when clicked. [CUA](../cua/README.md) owns registration and disposal. + +
+ +----- + + +## Further Exploration + +[Computer use](../../../docs/subsystems/computer-use.md) + +----- + + +## Model Experience + +### Computer Use in PTC sessions + +#### What the model sees + +Existing and new PTC sessions receive `cua` through the generated SDK and `run_code`, plus CUA guidance. Disabling withdraws these contributions and keeps PTC presentation. Other presets receive no CUA contributions. [CUA](../cua/README.md) and [PTC](../../ptc-runtime/ptc-runtime-node/README.md#persistent-repl) own the interfaces and output semantics. + +#### Token effect + +CUA activation adds its guidance. Disabling removes that guidance; the existing PTC execution instructions remain. + +#### KV Cache effect + +CUA activation and removal change the PTC SDK and system guidance; subsequent requests use the updated prefix. The top-level `run_code` transport remains present. + +## Known Limitations and Deferred Work + + + +- Source Web and Desktop share permission controls on macOS. Packaged Desktop helper distribution, signing, and permission attribution are deferred. +- PTC execution is shared by the profile and is configured independently. A Host without HMR applies saved changes at restart. Saving or revoking application grants while another HMR transaction unloads the provider can block; queued-write cancellation is a separate pending lifecycle fix. + + +### Dev Note + +None. diff --git a/packages/experimental/cua-bundle/README.zh.md b/packages/experimental/cua-bundle/README.zh.md new file mode 100644 index 0000000000..b6446f65f8 --- /dev/null +++ b/packages/experimental/cua-bundle/README.zh.md @@ -0,0 +1,94 @@ +--- +description: "开启可选 macOS Computer Use 及权限页面。" +kind: "package-bundle" +--- + +# @deepseek-ai/dsh-cua-bundle + +[English](README.md) | 中文 + +## 概述 + +从插件页为已有和新建 PTC 会话开启 macOS Computer Use。Windows 支持正在开发中。本组合包暂放在 `experimental`，待 Windows 支持完成后迁回 `computer-use` 组。默认关闭，关闭时不准备本机组件、不注册 CUA 工具或添加其模型指引。 + +## 目录 + +- [使用本包](#use-this-package) +- [理解实现](#understand-the-implementation) +- [进一步探索](#further-exploration) +- [模型体验](#model-experience) +- [已知限制与延后工作](#known-limitations-and-deferred-work) + +----- + + +## 使用本包 + +本流程面向从源码运行 macOS Web 或未打包 Desktop 的开发者。打开“插件 → Computer Use”并启用。权限页通过可选系统查询显示 macOS 报告的责任应用，指引用户申请权限并在系统设置中开启对应开关。若 macOS 显示其他应用名称，以系统提示为准。Web 版在仓库根目录运行以下命令： + +```sh +pnpm run build +pnpm dsh web +``` + +源码 Desktop 的启动方式和独立开发配置档见[桌面开发说明](../../../apps/desktop/README.zh.md)。需要使用 Computer Use 的配置档分别开启该功能。 + +开关将组合包选择保存到 profile 的 `package.json`。Computer Use 仅安装在 PTC 会话中。从插件列表启用时，系统权限未就绪会显示设置引导；从详情页启用后直接显示权限控件。详情页提供“设为 PTC”按钮来修改新会话默认模式，已有会话保留原 preset。开关保持 PTC 执行方式和其他设置不变。提供方加载时准备本机组件，已有与源码摘要匹配的可执行文件时直接复用。 + +关闭会卸载提供方，并从其 Agent 作用域中撤回 CUA 工具。执行器配置、新会话默认值和已保存的授权保留。会话级应用授权随提供方卸载而失效。 + +Computer Use 要求 DSH Host 运行在 macOS 上。其他系统可以显示入口，但原生提供方启动会失败。其他操作系统上的浏览器可以连接受支持的 Mac Host。 + +准备失败时，已保存的选择保留，CUA 操作不可用。查看 Host 的启动诊断，修复原因后重启 DSH。 + +显式启用会进行只读权限检查，可能启动 helper，但不会打开系统授权弹窗。[权限页面](../client-ui-cua/README.zh.md) 提供系统申请入口、设置跳转和自动检测。任务访问应用前会单独询问用户。长期应用授权保存在现有 profile 配置中，会话授权保存在内存中，跨 turn 和连接复用，重启 DSH 或关闭 Computer Use 后失效。可以在同一页面撤销授权。 + +----- + + +## 理解实现 + +
+维护者说明 + +组合包 patch 通过 `agentPreset: ptc` 挂载原生提供方和权限界面。工具通过现有 Agent 作用域注册，preset 文档保持不变。详情页按钮复用现有设置表单，仅在点击后保存 `agent-preset-registry.selectedDefault: ptc`。[CUA](../cua/README.zh.md)负责注册和销毁。 + +
+ +----- + + +## 进一步探索 + +[Computer use](../../../docs/subsystems/computer-use.zh.md) + +----- + + +## 模型体验 + +### Computer Use in PTC sessions + +#### What the model sees + +已有和新建 PTC 会话通过生成的 SDK 与 `run_code` 使用 `cua`，并获得 CUA 指引。关闭撤回这些贡献，保留 PTC 展示方式。其他 preset 不获得 CUA 贡献。完整接口与输出约定由 [CUA](../cua/README.zh.md) 和 [PTC](../../ptc-runtime/ptc-runtime-node/README.zh.md#persistent-repl) 说明。 + +#### Token effect + +CUA 生效时添加其指引。关闭撤回这些指引，现有 PTC 执行说明保留。 + +#### KV Cache effect + +CUA 生效和卸载会改变 PTC SDK 与系统指引，后续请求使用更新后的前缀。顶层 `run_code` 工具保持存在。 + +## 已知限制与延后工作 + + + +- 源码 Web 与 Desktop 共用 macOS 权限控件。桌面安装包的 helper 分发、签名和权限归属留待后续处理。 +- PTC 执行方式由 profile 共享，需单独配置。未启用 HMR 的 Host 在重启时应用已保存的变更。保存或撤销应用授权与另一次卸载提供方的 HMR 同时发生时可能阻塞；排队写入取消另行修复。 + + +### 开发备注 + +None. diff --git a/packages/experimental/cua-bundle/cordis.patch.yml b/packages/experimental/cua-bundle/cordis.patch.yml new file mode 100644 index 0000000000..e468011f06 --- /dev/null +++ b/packages/experimental/cua-bundle/cordis.patch.yml @@ -0,0 +1,9 @@ +- insert: + - id: cua-provider-registry + name: '@deepseek-ai/dsh-computer-use' + - id: cua + name: '@deepseek-ai/dsh-experimental-cua-macos' + config: + agentPreset: ptc + - id: ui-cua + name: '@deepseek-ai/dsh-client-ui-cua' diff --git a/packages/experimental/cua-bundle/icon.svg b/packages/experimental/cua-bundle/icon.svg new file mode 100644 index 0000000000..5d50e0fa5a --- /dev/null +++ b/packages/experimental/cua-bundle/icon.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/packages/experimental/cua-bundle/locale/en.json b/packages/experimental/cua-bundle/locale/en.json new file mode 100644 index 0000000000..eaab4261ac --- /dev/null +++ b/packages/experimental/cua-bundle/locale/en.json @@ -0,0 +1,6 @@ +{ + "meta": { + "title": "Computer Use", + "description": "Currently supports macOS; Windows support is in development. Loads Computer Use for existing and new PTC sessions. Switch to PTC mode and grant Accessibility and Screen Recording permissions to use it." + } +} diff --git a/packages/experimental/cua-bundle/locale/zh.json b/packages/experimental/cua-bundle/locale/zh.json new file mode 100644 index 0000000000..3daf33eab3 --- /dev/null +++ b/packages/experimental/cua-bundle/locale/zh.json @@ -0,0 +1,6 @@ +{ + "meta": { + "title": "Computer Use", + "description": "目前仅支持 macOS，Windows 支持正在开发中。为已有及新建 PTC 会话加载 Computer Use。请切换到 PTC 模式来使用，并授予辅助功能和屏幕录制权限。" + } +} diff --git a/packages/experimental/cua-bundle/package.json b/packages/experimental/cua-bundle/package.json new file mode 100644 index 0000000000..96935054cc --- /dev/null +++ b/packages/experimental/cua-bundle/package.json @@ -0,0 +1,54 @@ +{ + "name": "@deepseek-ai/dsh-cua-bundle", + "description": "Optional macOS Computer Use and permission setup", + "version": "0.2.1-rc.2", + "icon": "./icon.svg", + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/deepseek-ai/deepseek-harness.git", + "directory": "packages/experimental/cua-bundle" + }, + "type": "module", + "main": "lib/index.js", + "types": "lib/types/index.d.ts", + "exports": { + ".": { + "types": "./lib/types/index.d.ts", + "default": "./lib/index.js" + }, + "./cordis.patch.yml": "./cordis.patch.yml", + "./locale/*.json": "./locale/*.json", + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "icon.svg", + "locale/*.json", + "lib/index.js", + "cordis.patch.yml", + "lib/types/**/*.d.ts" + ], + "license": "MIT", + "dsh": { + "bundle": { + "patch": "./cordis.patch.yml" + } + }, + "dependencies": { + "@deepseek-ai/dsh-experimental-cua-macos": "workspace:*", + "@deepseek-ai/dsh-computer-use": "workspace:*", + "@deepseek-ai/dsh-ptc-runtime-node": "workspace:*", + "@deepseek-ai/dsh-client-ui-cua": "workspace:*" + }, + "peerDependencies": { + "@deepseek-ai/cordis": "workspace:~" + }, + "devDependencies": { + "@deepseek-ai/cordis": "workspace:~", + "@deepseek-ai/cordis-plugin-include": "workspace:~", + "js-yaml": "^4.2.0" + } +} diff --git a/packages/experimental/cua-bundle/src/index.ts b/packages/experimental/cua-bundle/src/index.ts new file mode 100644 index 0000000000..f04783f7c0 --- /dev/null +++ b/packages/experimental/cua-bundle/src/index.ts @@ -0,0 +1,2 @@ +/** Optional Computer Use composition; configuration rows live in cordis.patch.yml. */ +export {} diff --git a/packages/experimental/cua-bundle/tsconfig.json b/packages/experimental/cua-bundle/tsconfig.json new file mode 100644 index 0000000000..f1a449634c --- /dev/null +++ b/packages/experimental/cua-bundle/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib/types" + }, + "include": [ + "src" + ], + "references": [ + { + "path": "../../../vendor/cordis" + } + ] +} diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index 5ed55c261a..477e2b7958 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -12,11 +12,11 @@ en: 4549f02fc53854a5 zh: 6f74f1130b6623a8 /deepseek-ai-dsh-experimental-cua-macos/table-of-contents: - en: d152484eb41ac6b4 - zh: 7b30b973a849247d + en: 867d9ec544719acf + zh: 92348599582dc33a /deepseek-ai-dsh-experimental-cua-macos/use-this-package: - en: e2e1797ad6b73f25 - zh: 35b7a06e471e433c + en: f2ceaaa0c834a931 + zh: 0edd75ea5323d07f /deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment: en: e7909c9b43e44fbc zh: 98949877d4ea3a86 @@ -45,8 +45,8 @@ en: a2c9acb5a4320dda zh: e8d1586af60215c4 /deepseek-ai-dsh-experimental-cua-macos/known-limitations-and-deferred-work: - en: 9d8af4c39fe407a3 - zh: afec49eedfd702cb + en: 1c42b446c186b01b + zh: 0f1ecea4fd0a1a13 /deepseek-ai-dsh-experimental-cua-macos/known-limitations-and-deferred-work/dev-note: en: b836f01bd0bceb4a zh: 8122fa08b2aa95fa diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 74887ae936..ed56375c26 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -20,12 +20,16 @@ Inspect macOS permissions, select running applications and windows, read accessi - [Known Limitations and Deferred Work](#known-limitations-and-deferred-work) - [Dev Note](#dev-note) +The helper supplies bounded 64-pixel PNG application icons for permission prompts and saved-application lists. Icon reads do not launch an application or inspect its windows; executable and signing checks still determine authorization. + +The shared helper queries the process macOS considers responsible for it and captures that application’s name and icon before announcing readiness. A Desktop app launched from Terminal can therefore identify Terminal even when another GUI application sits between it and the helper. Permission queries retain this metadata after the responsible application exits and across connections sharing the helper. If the system query or application metadata is unavailable, the field is omitted and the UI uses the system prompt for guidance. + ----- ## Use this package -Enable this experimental provider through explicit source composition on macOS 14 or later. Windows support is in development. Shipped profiles do not mount it. Prepare a native helper and load the registry/provider overlay: +For repository Web use, enable [Computer Use](../cua-bundle/README.md) in Plugins. This prepares the local executable on macOS 14 or later; Windows support is in development. Subsequent builds check it while the existing Web profile keeps the bundle selected. Disabled profiles do not prepare native components. Explicit source composition is also available: ```sh swift build --package-path packages/experimental/cua-macos/native --configuration release @@ -34,7 +38,7 @@ export DSH_CUA_HELPER="$cua_native_root/.build/release/dsh-cua-native" pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis.yml --dump-config ``` -The dump command checks profile composition without a model request. For a real task, replace `--dump-config` with a prompt and configure the profile’s model credentials. The overlay mounts the computer-use registry and this provider; remove an existing competing provider before using it. The responsible host/helper needs Accessibility permission for AX/input and Screen Recording permission for capture. `getCapabilities` reports permissions and session lock state without opening a system dialog. Locked or unavailable desktop sessions reject UI observations and input before dispatch; the user must unlock macOS before resuming. `listApps` and `cua.getState` list running applications. `findApps(query)` searches standard macOS application directories by name, bundle ID or absolute path. `getApp` selects a unique name, bundle ID, path or running PID, obtains application consent, and launches a stopped app in the background when needed. A stopped executable is identified before approval and checked again before and after launch; unsigned code only receives task access. The returned object reads initial AX without activating the app. Launch cancellation may leave the application running. Model guidance starts with background observation and input, without activation during initialization or before each action. For hidden or minimized windows, try `app.show()` or `window.show()` and read fresh state. This requests unhide and unminimize without activation, raising or switching Spaces; app-level restoration selects the focused, main or sole AX window. The result names the selected `windowId`, or null when no window was selected, and reports `foregroundChanged` by comparing the front app and top window before and after the request. Null means foreground evidence was unavailable; later focus changes remain possible. If restoration fails or the window remains unusable, explain the needed foreground step, call `app.activate()` or an observed window’s `raise` action, then observe again. Necessary foreground access takes priority over preserving focus. Applications may change focus after input; background delivery does not guarantee an undisturbed user session. +The dump command checks profile composition without a model request. For a real task, replace `--dump-config` with a prompt and configure the profile’s model credentials. The overlay mounts the computer-use registry and this provider; remove an existing competing provider before using it. The responsible host/helper needs Accessibility permission for AX/input and Screen Recording permission for capture. `getCapabilities` reports permissions and session lock state without opening a system dialog. Locked or unavailable desktop sessions reject UI observations and input before dispatch; the user must unlock macOS before resuming. `listApps` and `cua.getState` list running applications. `findApps(query)` searches standard macOS application directories by name, bundle ID or absolute path. `getApp` selects a unique name, bundle ID, path or running PID, obtains application consent, and launches a stopped app in the background when needed. A stopped executable is identified before approval and checked again before and after launch; unsigned code only receives session access for the selected instance. The returned object reads initial AX without activating the app. Launch cancellation may leave the application running. Model guidance starts with background observation and input, without activation during initialization or before each action. For hidden or minimized windows, try `app.show()` or `window.show()` and read fresh state. This requests unhide and unminimize without activation, raising or switching Spaces; app-level restoration selects the focused, main or sole AX window. The result names the selected `windowId`, or null when no window was selected, and reports `foregroundChanged` by comparing the front app and top window before and after the request. Null means foreground evidence was unavailable; later focus changes remain possible. If restoration fails or the window remains unusable, explain the needed foreground step, call `app.activate()` or an observed window’s `raise` action, then observe again. Necessary foreground access takes priority over preserving focus. Applications may change focus after input; background delivery does not guarantee an undisturbed user session. | Field | Default | Meaning | | --- | --- | --- | @@ -59,7 +63,7 @@ Keypad names (`KP_0` through `KP_9`, `KP_Enter`, and keypad operators) retain th Agent disposal and plugin unload cancel and await the complete tool call, including image admission and attachment storage. An image write already in progress must settle before disposal returns; a canceled call cannot return a successful screenshot. -Application consent is enforced before AX reads, window-title reads, screenshots, activation, and input through the CUA tool. User Questions offers task access or denial, plus saved access when the application has a verifiable signature and the composition provides settings persistence. Task access is bound to the selected process handle; saved access is bound to its canonical executable path and designated code-signing requirement. Denial prevents repeated prompts in that task. Removing a saved `allowedApps` entry cancels affected operations and invalidates their helper handles; reauthorization requires fresh selection. Other handles on the same Agent connection also expire, while saved grants remain reusable. Agent/provider disposal joins cleanup of the corresponding native connection. See the [application consent decision](../../../.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.md). These checks govern CUA calls; persistent PTC and separately authorized shell tools retain their own execution authority. +Application consent is enforced before AX reads, window-title reads, screenshots, activation, and input through the CUA tool. The first selection asks the human for session access, denial, or saved access when the application has a verifiable signature. Session access is kept in memory by session and application identity, survives agent and helper replacement, and expires when DSH restarts or Computer Use is disabled. Signed application identities combine the canonical executable path and designated code-signing requirement; unverifiable applications remain bound to their process instance. Denial prevents repeated prompts in that session. Revocation reads and removes saved access in the same queue as grant creation, invalidates each affected Agent connection, and awaits its native cleanup without replaying input. Every app handle on those Agent connections becomes unusable; other applications need fresh selection, but their session and saved grants remain reusable. See the [application consent decision](../../../.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.md). These checks govern CUA calls; persistent PTC and separately authorized shell tools are not an operating-system sandbox. Raw `getState` exposes validated state fields and the complete text for the requested disclosure options. The native helper also returns structured element diagnostics for native checks; the tool projection omits that duplicate array from model text and structured tool values. @@ -147,7 +151,7 @@ The plugin contributes one `cua` operation tool with method-specific parameters. ##### Operation guidance (verbatim) ```markdown -Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. +Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user allows another request on the permission page. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again. @@ -184,7 +188,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to complete the Computer Use permission setup for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. @@ -210,7 +214,7 @@ The queue orders individual native requests. Holding an app object does not rese - macOS Stage Manager can reduce background window bounds and screenshots to thumbnails. Full-window observation and coordinate input in this state are not guaranteed; model guidance requires explaining foreground needs and avoiding repeated focus reclamation. - Background input depends on private SkyLight symbols; macOS updates can make them unavailable. Validate each target OS version. - Observed failures include dispatched Unicode text not appearing in Blender and background Command+A not selecting all text in AppKit. Inspect application state after input; a successful tool result confirms dispatch only. -- Native binaries require a local build. Signing, distribution, and system-permission onboarding remain deferred. +- Native binaries require a local build. Signing and Desktop distribution remain future work. The [permission page](../client-ui-cua/README.md) offers macOS prompts and app revocation; packaged Desktop permission attribution remains deferred. - Persistent PTC is an external dependency assumption, not an implementation in this package. Real-model task success requires separate evaluation beyond native and protocol fixtures. ----- diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index 43c37296bb..ffbd21b79c 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -20,12 +20,16 @@ kind: "package-reference" - [已知限制与后续工作](#known-limitations-and-deferred-work) - [开发备注](#dev-note) +helper 为授权提示和已授权应用列表提供 64 像素 PNG 图标。读取图标不会启动应用或检查窗口，授权仍由可执行文件和签名校验决定。 + +共享 helper 查询 macOS 为它记录的责任进程，在发布就绪状态前保存对应应用的名称和图标。因此，从 Terminal 启动 Desktop 时，即使中间还有另一个 GUI 应用，也能识别到 Terminal。责任应用退出后，权限查询仍保留这些信息，后续复用该 helper 的连接读取同一份记录。系统查询或应用信息不可用时省略该字段，由界面引导用户按系统弹窗选择授权对象。 + ----- ## 使用本包 -该实验性提供方需要在 macOS 14 及以上通过源码配置显式启用。Windows 支持正在开发中。默认配置不加载它。先准备原生组件，再加载注册表和提供方覆盖层： +从仓库运行 Web 时，在插件页开启 [Computer Use](../cua-bundle/README.zh.md)。首次启用会在 macOS 14 及以上准备本机组件，Windows 支持正在开发中。现有 Web profile 保持选中时，后续构建继续检查更新。未开启的 profile 不准备原生组件。也可以显式加载源码覆盖层： ```sh swift build --package-path packages/experimental/cua-macos/native --configuration release @@ -34,7 +38,7 @@ export DSH_CUA_HELPER="$cua_native_root/.build/release/dsh-cua-native" pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis.yml --dump-config ``` -配置输出命令不发起模型请求。执行真实任务时，将 `--dump-config` 替换为任务提示，并配置该 profile 的模型凭据。配置挂载 computer-use 注册表和本 provider；使用前需移除已有的竞争 provider。负责执行的宿主或辅助进程需要辅助功能权限来读取 AX 和输入，需要屏幕录制权限来截图。`getCapabilities` 查询权限和锁屏状态，不弹出系统授权窗口。桌面会话锁定或不可用时，界面观察与输入会在分派前拒绝，需由用户解锁 macOS 后继续。`listApps` 和 `cua.getState` 列出运行中的应用。`findApps(query)` 按名称、bundle ID 或绝对路径搜索 macOS 标准应用目录。`getApp` 按唯一名称、bundle ID、路径或运行中的 PID 选择应用，取得应用许可后，在需要时后台启动尚未运行的应用。启动前先确认可执行文件身份，授权后在启动前后再次核对；未签名代码只取得任务许可。返回的对象读取初始 AX，不自动激活应用。启动被取消时，应用可能已经运行。模型指引要求先在后台观察和输入，不在初始化或每次动作前激活应用。窗口隐藏或最小化时，先尝试 `app.show()` 或 `window.show()`，再重新观察。它请求取消隐藏和恢复最小化，不主动激活、抬起窗口或切换桌面空间；应用级恢复选择焦点窗口、主窗口或唯一 AX 窗口。结果通过 `windowId` 指明选中的窗口，未选中时为 null；`foregroundChanged` 比较操作前后的前台应用和最上层窗口，无法取得完整观察时为 null，返回后仍可能发生焦点变化。恢复失败或窗口仍不可用时，说明需要前台的步骤，再调用 `app.activate()` 或窗口的 `raise` 动作并重新观察。必须使用前台时，以完成任务为先。应用收到输入后仍可能改变焦点，后台投递不保证用户操作始终不受干扰。 +配置输出命令不发起模型请求。执行真实任务时，将 `--dump-config` 替换为任务提示，并配置该 profile 的模型凭据。配置挂载 computer-use 注册表和本 provider；使用前需移除已有的竞争 provider。负责执行的宿主或辅助进程需要辅助功能权限来读取 AX 和输入，需要屏幕录制权限来截图。`getCapabilities` 查询权限和锁屏状态，不弹出系统授权窗口。桌面会话锁定或不可用时，界面观察与输入会在分派前拒绝，需由用户解锁 macOS 后继续。`listApps` 和 `cua.getState` 列出运行中的应用。`findApps(query)` 按名称、bundle ID 或绝对路径搜索 macOS 标准应用目录。`getApp` 按唯一名称、bundle ID、路径或运行中的 PID 选择应用，取得应用许可后，在需要时后台启动尚未运行的应用。启动前先确认可执行文件身份，授权后在启动前后再次核对；未签名代码只能取得所选实例的会话许可。返回的对象读取初始 AX，不自动激活应用。启动被取消时，应用可能已经运行。模型指引要求先在后台观察和输入，不在初始化或每次动作前激活应用。窗口隐藏或最小化时，先尝试 `app.show()` 或 `window.show()`，再重新观察。它请求取消隐藏和恢复最小化，不主动激活、抬起窗口或切换桌面空间；应用级恢复选择焦点窗口、主窗口或唯一 AX 窗口。结果通过 `windowId` 指明选中的窗口，未选中时为 null；`foregroundChanged` 比较操作前后的前台应用和最上层窗口，无法取得完整观察时为 null，返回后仍可能发生焦点变化。恢复失败或窗口仍不可用时，说明需要前台的步骤，再调用 `app.activate()` 或窗口的 `raise` 动作并重新观察。必须使用前台时，以完成任务为先。应用收到输入后仍可能改变焦点，后台投递不保证用户操作始终不受干扰。 | 字段 | 默认值 | 含义 | | --- | --- | --- | @@ -59,7 +63,7 @@ pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis. Agent 销毁和插件卸载会取消并等待完整工具调用，包括图像接入检查和附件存储。已经开始的图像写入必须结束后，销毁才会返回；已取消的调用不会返回成功截图。 -CUA 工具在读取 AX、窗口标题、截图、激活和输入之前检查应用授权。User Questions 提供任务级允许和拒绝选项；应用签名可验证且组合支持配置持久化时，也可保存授权。任务授权绑定所选进程句柄，长期授权绑定真实可执行路径和指定代码签名要求。拒绝后，同一任务不再反复询问。删除保存的 `allowedApps` 条目会取消相关操作并使原生组件中的句柄失效；重新授权需要重新选择应用。同一 Agent 连接中的其他句柄也会失效，已保存的授权仍可复用。Agent 或提供方销毁时等待对应连接的原生状态清理。设计理由见[应用授权决策](../../../.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.zh.md)。这些检查约束 CUA 调用；持久 PTC 和另行授权的 shell 工具保留各自的执行权限。 +CUA 工具在读取 AX、窗口标题、截图、激活和输入前检查应用授权。首次选择应用时询问用户，可选择本会话允许、拒绝；签名可验证的应用还可保存长期授权。会话许可按会话和应用身份保存在内存中，Agent 或 helper 重建后仍可复用，重启 DSH 或关闭 Computer Use 后失效。已签名应用的身份由规范化可执行文件路径和代码签名指定要求组成；签名不可验证的应用仍绑定进程实例。拒绝后，同一会话不会反复询问。撤销与长期许可的保存共用队列，在队列内读取并移除许可，然后使相关 Agent 连接失效并等待原生清理完成，不自动重放输入。同一 Agent 连接中的其他应用也需要重新选择，其他应用的会话许可和长期许可仍可复用。设计理由见[应用授权决策](../../../.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.zh.md)。这些检查的范围是 CUA 调用。persistent PTC 与已授权的 shell 仍按各自执行权限运行。 原始 `getState` 操作返回经过校验的状态字段，以及按本次披露选项生成的完整文本。原生 helper 还返回供原生检查使用的结构化元素诊断数组；工具投影会从模型文本和结构化工具值中移除这份重复数组。 @@ -147,7 +151,7 @@ AppKit sheet 可以通过其窗口 ID 单独选择，也会出现在所属文档 ##### 操作指导文本（原文） ```markdown -Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. +Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user allows another request on the permission page. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again. @@ -184,7 +188,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to complete the Computer Use permission setup for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. @@ -210,7 +214,7 @@ Use selectText only on controls advertising selectionWritable. It selects a uniq - macOS 台前调度可能将后台窗口的尺寸与截图变为缩略图，当前无法保证这种状态下的完整画面与坐标输入；模型指引要求说明前台需求并避免反复抢回焦点。 - 后台输入依赖私有 SkyLight 符号，macOS 升级可能使其不可用，需在目标系统版本验证。 - 实测 Blender Unicode 输入曾出现分派成功但界面未显示文本，后台 AppKit 的 Command+A 曾未选中全文。输入后必须检查应用状态；工具返回成功只表示已分派。 -- 原生二进制需要本地构建。签名、分发和系统权限引导留待后续。 +- 原生二进制需要本地构建，签名和桌面分发留待后续。[权限页面](../client-ui-cua/README.zh.md)提供 macOS 申请入口和应用授权撤销；桌面安装包的权限归属留待后续处理。 - 持久 PTC 是对外部运行环境的假设，本包不实现它。真实模型任务成功率需要在原生测试和协议测试之外单独评估。 ----- diff --git a/packages/experimental/cua-macos/examples/macos.cordis.yml b/packages/experimental/cua-macos/examples/macos.cordis.yml index 94942836e0..f1b471523d 100644 --- a/packages/experimental/cua-macos/examples/macos.cordis.yml +++ b/packages/experimental/cua-macos/examples/macos.cordis.yml @@ -4,4 +4,5 @@ - id: cua name: '../src/index.ts' config: + agentPreset: false helperPath: !!js process.env.DSH_CUA_HELPER diff --git a/packages/experimental/cua-macos/native/README.i18n.yaml b/packages/experimental/cua-macos/native/README.i18n.yaml index 626dc16c0d..cdaaf8ba43 100644 --- a/packages/experimental/cua-macos/native/README.i18n.yaml +++ b/packages/experimental/cua-macos/native/README.i18n.yaml @@ -15,17 +15,17 @@ en: e8cbc43d38786c40 zh: 52b3ba85d98011b9 /dsh-cua-native-helper/build-and-service: - en: 824e4a2b9036d9bf - zh: 37c34a454abc6965 + en: 02b25ac1126b6094 + zh: 3995a1f46354531b /dsh-cua-native-helper/request-protocol: - en: 3202c567a6e1a654 - zh: 143eebd0096a0487 + en: c8ad13843989b314 + zh: 2a42acb2faee8803 /dsh-cua-native-helper/observation-and-input: - en: f2db99ae46a21c61 - zh: 48c6752bd71dbf27 + en: ce99636fae06e66c + zh: d3c983e8a47916b2 /dsh-cua-native-helper/verification: - en: 2afe5ceb0a3c4965 - zh: 86bf7187ddae489f + en: ba3277817f4da729 + zh: 4642abf830234a3b /dsh-cua-native-helper/further-exploration: en: 265e1927952ce6d7 zh: 7c35d9fd98b6e6c8 diff --git a/packages/experimental/cua-macos/native/README.md b/packages/experimental/cua-macos/native/README.md index c5ce8798f4..4299301dfc 100644 --- a/packages/experimental/cua-macos/native/README.md +++ b/packages/experimental/cua-macos/native/README.md @@ -27,7 +27,7 @@ Build with `swift build --package-path packages/experimental/cua-macos/native -- The shared service uses an owner-only Unix socket under `/tmp/dsh-cua-`, keyed by the executable SHA-256. A version lock elects one process; an additional real-executable-path lock excludes a different build at the same path. The first frame is protocol-1 `hello` with the runtime key and this connection’s byte, AX, operation, screenshot, event-spacing and cleanup limits. `ready` reports the verified runtime key, PID and instance ID. Every connection owns an independent AXWorker. The stdin/stdout entry is a diagnostic proxy to this service. -AppKit runs on the main event loop. Blocking AX and input use each connection’s serial actor; all connections share a FIFO for complete desktop operations. Socket writes do not hold that operation slot. A request cancellation waits for native work to stop; it does not cancel the connection’s lifetime latch. Screenshot timeout ends only the request. An unfinished system capture retains the single capture slot until its real callback arrives, rejects new captures with CAPTURE_BUSY and leaves AX/input available. Only an operation or cleanup that cannot stop can invalidate the shared process. The proxy accepts `--max-request-bytes`, `--max-response-bytes`, `--ax-timeout-ms`, `--timeout-ms`, `--screenshot-timeout-ms`, `--shutdown-grace-ms` and `--text-event-interval-ms`; the Host sets the same limits in hello. An AX deadline bounds an individual system call. Cancellation can follow partial input, and uncertain input is never replayed automatically. +AppKit runs on the main event loop. Blocking socket accepts and handshakes use background queues; server callbacks explicitly carry `@MainActor @Sendable` isolation and weak ownership. Shutdown releases accepted connections even when their handshake is incomplete. Blocking AX and input use each connection’s serial actor; all connections share a FIFO for complete desktop operations. Socket writes do not hold that operation slot. A request cancellation waits for native work to stop; it does not cancel the connection’s lifetime latch. Screenshot timeout ends only the request. An unfinished system capture retains the single capture slot until its real callback arrives, rejects new captures with CAPTURE_BUSY and leaves AX/input available. Only an operation or cleanup that cannot stop can invalidate the shared process. The proxy accepts `--max-request-bytes`, `--max-response-bytes`, `--ax-timeout-ms`, `--timeout-ms`, `--screenshot-timeout-ms`, `--shutdown-grace-ms` and `--text-event-interval-ms`; the Host sets the same limits in hello. An AX deadline bounds an individual system call. Cancellation can follow partial input, and uncertain input is never replayed automatically. ----- @@ -57,7 +57,7 @@ Every app-targeted operation requires `appHandle` from `getApp` in the same help Stable error codes are `INVALID_REQUEST` (malformed envelope), `INVALID_ARGUMENT` (invalid parameters), `UNSUPPORTED` (unknown operation), `APP_UNAVAILABLE`, `AMBIGUOUS_APP`, `STALE_APP`, `STALE_SNAPSHOT`, `WINDOW_UNAVAILABLE`, `WINDOW_CHANGED`, `PERMISSION_DENIED`, `FOREGROUND_REQUIRED` (hover target is not the unobscured foreground window), `PANEL_NOT_READY` (system panel recipient unavailable before input), and `NATIVE_ERROR` (remaining operating-system failures). An error never promises that a partially dispatched input was rolled back. The host must not retry input automatically. -Application selection verifies the executable path and code-signing requirement before returning an authorization identity to the Host. Unsigned running applications receive a process-lifetime identity; stopped unsigned applications use an executable digest and cannot receive persistent consent. The Host obtains human consent before launch, observation or input. +Application selection returns an authorization record to the Host: a digest of canonical executable path and verified designated signing requirement, display name, executable path, and persistable flag. Unsigned or ad-hoc-signed applications cannot receive saved consent; running instances use process-lifetime identity, while stopped applications use an executable digest. The Host strips this record from model results and obtains human consent before launch, observation or input. This pipe protocol is trusted Host transport, not a sandbox against a separately authorized shell process. Protocol declarations are generated from [the shared protocol](../../cua/src/protocol.ts). Run `pnpm exec tsx packages/experimental/cua-macos/native/generate-protocol.ts` from the repository root; package tests verify generated JSON and Swift field/enum tables. Model-submittable parameters exclude Host resource bounds and the private launch continuation. @@ -77,6 +77,8 @@ Capture and pointer delivery read CG bounds before and after an exact-window AX The responsible host/helper needs macOS Accessibility permission for AX state and input, and Screen Recording permission for screenshots. Capability preflight and permission errors do not grant permissions. Capture refuses a window that moves or resizes while the image is being read. Unassociated system panels and accessory-only applications are not selectable. +Host-only permission controls are `permissionStatus {}`, `requestPermission {permission}`, and `openPermissionSettings {permission}`, where permission is `accessibility` or `screenRecording`. They are excluded from model tools and from `getCapabilities.methods`. Permission status includes observed helper/parent process paths; these paths do not identify the TCC responsible application. The operating-system prompt names the application and asks for the user’s decision. The helper identifies the responsible process with the optional libSystem SPI `responsibility_get_pid_responsible_for_pid`, then captures its application name and icon before publishing readiness. The function is resolved dynamically, following the same signature used by [Chromium](https://chromium.googlesource.com/chromium/src/+/refs/tags/134.0.6998.28/base/process/process_info_mac.mm). If the symbol, process, or application metadata is unavailable, attribution stays unknown; it is not inferred from the nearest GUI ancestor. Permission checks remain independent of this metadata. + Observation reads each non-layout node’s descriptive properties and children through `AXUIElementCopyMultipleAttributeValues`. Per-attribute errors retain the individual-read rules; unsupported batch APIs use individual reads. Values are retained only during that observation, and input reads live attributes. Application state can change between nodes; a batch does not make the whole tree atomic. System file-panel keyboard routing resolves the remote recipient through the selected app’s focused AX sheet. Only Apple’s Open and Save Panel Service is accepted, and its focused ancestry must still contain the identified remote window. Element pointer fallbacks resolve an app-owned window or sheet through AX parents when AXWindow is unavailable. AX actions retain priority over pointer input; visual click markers need no system-cursor movement. @@ -101,6 +103,8 @@ Use `python3 packages/experimental/cua-macos/native/tests/test_swift.py` for off Run `python3 packages/experimental/cua-macos/native/tests/shared_helper_e2e.py --helper /absolute/helper` to verify concurrent startup, a shared service PID, independent handles, same-application handoff, request-local screenshot deadlines, connection cleanup and signal shutdown with an owned AppKit fixture. Existing Accessibility and Screen Recording grants are required. `--without-desktop` limits the check to protocol and process lifetime. +Run `python3 packages/experimental/cua-macos/native/tests/launcher_identity_e2e.py --helper /absolute/helper` in a GUI login session to verify a Launch Services application that starts a nested GUI launcher and then the helper. The helper identifies the outer responsible application and retains its name and icon after the launchers exit. This check owns both applications and the helper, cleans up its Launch Services registrations, and does not request Accessibility or Screen Recording access. + Run `python3 packages/experimental/cua-macos/native/tests/axless_e2e.py` after building and unlocking the Mac to verify an owned window whose application removes and later restores its AX roots. The check requires existing Accessibility/Screen Recording grants, rejects stale element indices, captures the AXless window, verifies a coordinate click in the fixture state file, and checks restoration. It never changes system permission settings. Run `python3 packages/experimental/cua-macos/native/tests/observation_benchmark.py --reference-helper /absolute/reference --candidate-helper /absolute/candidate --artifacts /private/evidence` to compare complete observations of the same owned AppKit window with 35 and 400 synthetic rows. Each helper receives one warmup and three alternating measured requests. All result fields must match except helper-generated app and snapshot IDs. The report keeps raw timings, byte counts, hashes and replies; it sets no CI speed threshold and makes no memory claim. Fixture startup can activate the application. Run without other desktop automation or compilation jobs. diff --git a/packages/experimental/cua-macos/native/README.zh.md b/packages/experimental/cua-macos/native/README.zh.md index ed5a5c8687..6c5e4b5f8a 100644 --- a/packages/experimental/cua-macos/native/README.zh.md +++ b/packages/experimental/cua-macos/native/README.zh.md @@ -27,7 +27,7 @@ kind: "package-reference" 共享服务在 `/tmp/dsh-cua-` 下使用仅所有者可访问的 Unix socket，以可执行文件的 SHA-256 区分实例。版本锁选出一个进程，另一个真实可执行路径锁排除同一路径的不同构建。首帧为协议版本 1 的 `hello`，包含运行时标识以及该连接的字节、AX、操作、截图、事件间隔和清理限制。`ready` 返回已核实的运行时标识、PID 和实例 ID。每条连接拥有独立的 AXWorker。stdin/stdout 入口是该服务的诊断代理。 -AppKit 在主事件循环上运行。阻塞式 AX 和输入使用每条连接的串行 actor，所有连接共享用于完整桌面操作的 FIFO。socket 写出不占用操作槽位。请求取消会等待原生工作停止，不取消连接的生命期控制器。截图超时只结束该请求。未完成的系统捕获保留唯一捕获槽位，直至真实回调到达；期间新截图返回 CAPTURE_BUSY，AX 和输入仍可用。只有无法停止的操作或清理才可能使共享进程失效。代理接受 `--max-request-bytes`、`--max-response-bytes`、`--ax-timeout-ms`、`--timeout-ms`、`--screenshot-timeout-ms`、`--shutdown-grace-ms` 和 `--text-event-interval-ms`；Host 在 hello 中设置相同限制。AX 超时约束单次系统调用。取消可能发生在部分输入之后，结果不确定的输入不自动重放。 +AppKit 在主事件循环上运行。阻塞式 socket 接收与握手使用后台队列，服务回调明确声明 `@MainActor @Sendable` 隔离，并弱引用服务对象。关闭时会释放已接收的连接，包括尚未完成握手的连接。阻塞式 AX 和输入使用每条连接的串行 actor，所有连接共享用于完整桌面操作的 FIFO。socket 写出不占用操作槽位。请求取消会等待原生工作停止，不取消连接的生命期控制器。截图超时只结束该请求。未完成的系统捕获保留唯一捕获槽位，直至真实回调到达；期间新截图返回 CAPTURE_BUSY，AX 和输入仍可用。只有无法停止的操作或清理才可能使共享进程失效。代理接受 `--max-request-bytes`、`--max-response-bytes`、`--ax-timeout-ms`、`--timeout-ms`、`--screenshot-timeout-ms`、`--shutdown-grace-ms` 和 `--text-event-interval-ms`；Host 在 hello 中设置相同限制。AX 超时约束单次系统调用。取消可能发生在部分输入之后，结果不确定的输入不自动重放。 ----- @@ -57,7 +57,7 @@ AppKit 在主事件循环上运行。阻塞式 AX 和输入使用每条连接的 稳定错误码包括 `INVALID_REQUEST`（信封无效）、`INVALID_ARGUMENT`（参数无效）、`UNSUPPORTED`（操作未知）、`APP_UNAVAILABLE`、`AMBIGUOUS_APP`、`STALE_APP`、`STALE_SNAPSHOT`、`WINDOW_UNAVAILABLE`、`WINDOW_CHANGED`、`PERMISSION_DENIED`、`FOREGROUND_REQUIRED`（悬停目标不是未被遮挡的前台窗口）、`PANEL_NOT_READY`（输入前系统面板接收方尚不可用）及 `NATIVE_ERROR`（其余操作系统失败）。错误不保证已分派的部分输入被回滚，Host 不得自动重试输入。 -应用选择先核实可执行路径和代码签名要求，再向 Host 返回授权身份。未签名的运行中应用使用进程生命期身份，未启动的未签名应用使用可执行文件摘要，不能获得持久授权。Host 在启动、观察和输入前取得用户许可。 +应用选择向 Host 返回授权记录：规范化可执行路径与经验证的代码签名指定要求的摘要、显示名称、可执行路径和可持久化标记。未签名或临时签名的应用不能获得长期授权；运行中实例使用进程生命期身份，未启动应用使用可执行文件摘要。Host 从模型结果中移除此记录，并在启动、观察和输入前取得用户许可。此管道协议用于可信 Host 传输，无法隔离另一个已获授权的 shell 进程。 协议声明从[共享协议](../../cua/src/protocol.ts)生成。在仓库根目录运行 `pnpm exec tsx packages/experimental/cua-macos/native/generate-protocol.ts`，包测试检查生成的 JSON 和 Swift 字段/枚举表。模型参数不包含 Host 资源上限和私有启动续接操作。 @@ -77,6 +77,8 @@ AppKit 在主事件循环上运行。阻塞式 AX 和输入使用每条连接的 负责执行的 host/helper 需要 macOS 辅助功能权限以读取 AX 和输入，需要屏幕录制权限以截图。能力预检和权限错误不授予权限。截图会拒绝在读取图片期间移动或缩放的窗口。无法确认归属的系统面板及仅后台辅助类型的应用不可选择。 +仅供 Host 使用的权限操作为 `permissionStatus {}`、`requestPermission {permission}` 和 `openPermissionSettings {permission}`，其中 permission 为 `accessibility` 或 `screenRecording`。这些操作不出现在模型工具及 `getCapabilities.methods` 中。权限状态包含观察到的 helper 和父进程路径，这些路径不能确定 TCC 归属的应用。系统授权提示负责指出应用并询问用户决定。helper 使用可选的 libSystem 内部接口 `responsibility_get_pid_responsible_for_pid` 查询责任进程，在报告就绪前记录对应应用的名称和图标。接口通过动态加载解析，函数签名与 [Chromium](https://chromium.googlesource.com/chromium/src/+/refs/tags/134.0.6998.28/base/process/process_info_mac.mm) 使用的一致。符号、进程或应用信息不可用时，归属保持未知，不根据最近的 GUI 祖先推断。权限检查独立于这些信息。 + 观察通过 `AXUIElementCopyMultipleAttributeValues` 读取每个非布局节点的描述属性和子节点。逐属性错误保留单独读取的规则；不支持批量 API 时使用单独读取。值只在本次观察期间保留，输入读取即时属性。应用状态可能在节点之间变化，批量读取不使整棵树具备原子性。 系统文件面板的键盘路由通过所选应用的 AX 焦点 sheet 解析远端接收方。只接受 Apple 的 Open and Save Panel Service，且其焦点祖先链仍须包含已识别的远端窗口。AXWindow 不可用时，元素鼠标后备路径沿 AX 父节点解析应用所属窗口或 sheet。AX 动作保持优先，可视点击标记不需要系统鼠标移动。 @@ -101,6 +103,8 @@ AppKit 在主事件循环上运行。阻塞式 AX 和输入使用每条连接的 运行 `python3 packages/experimental/cua-macos/native/tests/shared_helper_e2e.py --helper /absolute/helper`，通过测试自建的 AppKit fixture 核实并发启动、共享服务 PID、独立句柄、同一应用交接、请求级截图期限、连接清理和信号退出。需要已有辅助功能和屏幕录制权限。`--without-desktop` 将检查限定为协议与进程生命期。 +在 GUI 登录会话中运行 `python3 packages/experimental/cua-macos/native/tests/launcher_identity_e2e.py --helper /absolute/helper`，验证由 Launch Services 启动的应用先启动另一个 GUI 启动器，再启动 helper。helper 识别外层责任应用，并在启动器退出后保留其名称和图标。检查只使用测试自建的两个应用和 helper，结束后清理对应的 Launch Services 注册，不申请辅助功能或屏幕录制权限。 + 构建并解锁 Mac 后运行 `python3 packages/experimental/cua-macos/native/tests/axless_e2e.py`，验证应用移除并恢复 AX 根节点的窗口。检查需要已有辅助功能/屏幕录制权限，会拒绝过期元素索引，捕获无 AX 窗口，通过 fixture 状态文件核实坐标点击，并检查恢复。不会修改系统权限设置。 运行 `python3 packages/experimental/cua-macos/native/tests/observation_benchmark.py --reference-helper /absolute/reference --candidate-helper /absolute/candidate --artifacts /private/evidence`，对比同一自建 AppKit 窗口中 35 行和 400 行合成内容的完整观察。每个 helper 先预热一次，再交替测量三次。除 helper 生成的应用和快照 ID 外，结果字段必须一致。报告保留原始耗时、字节数、哈希和回复，不设置 CI 性能阈值，也不声称内存表现。fixture 启动可能激活应用。运行时避免其他桌面自动化或编译任务。 diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/NativePlatform.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/NativePlatform.swift index 8dd6b4f61f..e4dcfe354e 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/NativePlatform.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/NativePlatform.swift @@ -36,6 +36,22 @@ struct ApplicationCatalogSnapshot: Sendable { } return ApplicationCatalogSnapshot(applications: apps, frontPID: NSWorkspace.shared.frontmostApplication?.processIdentifier) } + /// A bounded PNG for human recognition; authorization continues to use executable/signing identity. + static func icon(at path: String) -> String? { + guard let bitmap = NSBitmapImageRep(bitmapDataPlanes: nil, pixelsWide: 64, pixelsHigh: 64, + bitsPerSample: 8, samplesPerPixel: 4, hasAlpha: true, isPlanar: false, + colorSpaceName: .deviceRGB, bytesPerRow: 0, bitsPerPixel: 0), + let context = NSGraphicsContext(bitmapImageRep: bitmap) else { return nil } + NSGraphicsContext.saveGraphicsState() + defer { NSGraphicsContext.restoreGraphicsState() } + NSGraphicsContext.current = context + var url = URL(fileURLWithPath: path) + while url.path != "/" && url.pathExtension != "app" { url.deleteLastPathComponent() } + let iconPath = url.pathExtension == "app" ? url.path : path + NSWorkspace.shared.icon(forFile: iconPath).draw(in: NSRect(x: 0, y: 0, width: 64, height: 64), + from: .zero, operation: .copy, fraction: 1) + return bitmap.representation(using: .png, properties: [:]).map { "data:image/png;base64," + $0.base64EncodedString() } + } static func activate(_ identity: ProcessIdentity) throws { try NativePlatform.verify(identity) guard let app = NSRunningApplication(processIdentifier: identity.pid), !app.isTerminated else { @@ -47,9 +63,18 @@ struct ApplicationCatalogSnapshot: Sendable { } enum NativePlatform { + /// Optional libSystem SPI: unavailable attribution stays unknown rather than guessing from parentage. + static func responsiblePID(_ pid: Int32) -> Int32? { + typealias Resolve = @convention(c) (pid_t) -> pid_t + guard let symbol = dlsym(UnsafeMutableRawPointer(bitPattern: -2), "responsibility_get_pid_responsible_for_pid") else { return nil } + let resolve = unsafeBitCast(symbol, to: Resolve.self) + let responsible = resolve(pid) + return responsible > 0 ? responsible : nil + } static func executablePath(_ pid: Int32) -> String? { var buffer = [CChar](repeating: 0, count: 4 * Int(MAXPATHLEN)) - return proc_pidpath(pid, &buffer, UInt32(buffer.count)) > 0 ? String(cString: buffer) : nil + guard proc_pidpath(pid, &buffer, UInt32(buffer.count)) > 0 else { return nil } + return String(decoding: buffer.prefix { $0 != 0 }.map { UInt8(bitPattern: $0) }, as: UTF8.self) } static func identity(_ pid: Int32) throws -> ProcessIdentity { var info = proc_bsdinfo() diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift index a37cd0d421..62d52a787e 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift @@ -3,6 +3,7 @@ enum NativeProtocol { static let modelMethods: [String] = ["activate", "click", "drag", "findApps", "getApp", "getCapabilities", "getState", "hover", "listApps", "listWindows", "paste", "performSecondaryAction", "pressKey", "screenshot", "scroll", "selectText", "setValue", "show", "typeText"] static let fields: [String: Set] = [ "activate": ["appHandle"], + "applicationIcons": ["paths"], "click": ["appHandle", "clickCount", "elementIndex", "modifiers", "mouseButton", "snapshotId", "windowId", "x", "y"], "drag": ["appHandle", "durationMs", "fromX", "fromY", "modifiers", "mouseButton", "snapshotId", "steps", "toX", "toY", "windowId"], "findApps": ["maxAppMatches", "query"], @@ -13,9 +14,12 @@ enum NativeProtocol { "launchApp": ["appHandle"], "listApps": [], "listWindows": ["appHandle"], + "openPermissionSettings": ["permission"], "paste": ["appHandle", "format", "maxClipboardBytes", "maxDepth", "maxElements", "pasteReadTimeoutMs", "text"], "performSecondaryAction": ["action", "appHandle", "elementIndex", "snapshotId"], + "permissionStatus": [], "pressKey": ["appHandle", "key", "modifiers"], + "requestPermission": ["permission"], "screenshot": ["appHandle", "windowId"], "scroll": ["appHandle", "deltaLinesX", "deltaLinesY", "snapshotId", "windowId", "x", "y"], "selectText": ["appHandle", "elementIndex", "prefix", "selectionType", "snapshotId", "suffix", "text"], @@ -26,8 +30,10 @@ enum NativeProtocol { static let stringEnums: [String: [String: Set]] = [ "click": ["modifiers": ["command", "control", "fn", "option", "shift"], "mouseButton": ["left", "middle", "right"]], "drag": ["modifiers": ["command", "control", "fn", "option", "shift"], "mouseButton": ["left", "middle", "right"]], + "openPermissionSettings": ["permission": ["accessibility", "screenRecording"]], "paste": ["format": ["html", "text"]], "pressKey": ["modifiers": ["command", "control", "fn", "option", "shift"]], + "requestPermission": ["permission": ["accessibility", "screenRecording"]], "selectText": ["selectionType": ["cursor_after", "cursor_before", "text"]], ] } diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/Permissions.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/Permissions.swift new file mode 100644 index 0000000000..b4a9dcc587 --- /dev/null +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/Permissions.swift @@ -0,0 +1,69 @@ +// System consent is requested only by the Host's user-facing permission controls. +import AppKit +@preconcurrency import ApplicationServices +import Darwin +import Foundation + +@MainActor struct SystemPermissions { + private let launchApplication: JSONValue? + + // Capture the system-reported responsible app before startup completes and the launcher can exit. + init() { launchApplication = Self.responsibleApplication(for: getpid()) } + + func status() -> JSONValue { + var result: [String: JSONValue] = [ + "accessibility": .bool(AXIsProcessTrusted()), + "screenRecording": .bool(CGPreflightScreenCaptureAccess()), + "helper": Self.process(getpid()), + "parent": Self.process(getppid()), + ] + if let launchApplication { result["launchApplication"] = launchApplication } + return .object(result) + } + + func request(_ permission: String) throws -> JSONValue { + switch permission { + case "accessibility": + if !AXIsProcessTrusted() { + let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary + _ = AXIsProcessTrustedWithOptions(options) + } + case "screenRecording": + if !CGPreflightScreenCaptureAccess() { _ = CGRequestScreenCaptureAccess() } + default: throw RequestParameters.invalid("permission must be accessibility or screenRecording") + } + return status() + } + + func openSettings(_ permission: String) throws -> JSONValue { + let pane: String + switch permission { + case "accessibility": pane = "Privacy_Accessibility" + case "screenRecording": pane = "Privacy_ScreenCapture" + default: throw RequestParameters.invalid("permission must be accessibility or screenRecording") + } + guard let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?\(pane)"), + NSWorkspace.shared.open(url) else { + throw NativeFailure(code: "SETTINGS_UNAVAILABLE", message: "Open System Settings > Privacy & Security manually") + } + return .object(["opened": .bool(true)]) + } + + /// Resolve macOS process responsibility, including launchers outside the nearest GUI application. + static func responsibleApplication(for pid: Int32) -> JSONValue? { + guard let responsible = NativePlatform.responsiblePID(pid), + let app = NSRunningApplication(processIdentifier: responsible), !app.isTerminated, + let bundle = app.bundleURL else { return nil } + var result: [String: JSONValue] = ["name": .string(bundle.lastPathComponent), "path": .string(bundle.path)] + if let icon = ApplicationCatalog.icon(at: bundle.path) { result["icon"] = .string(icon) } + return .object(result) + } + + private static func process(_ pid: Int32) -> JSONValue { + let path = NativePlatform.executablePath(pid) + var result: [String: JSONValue] = ["pid": .number(Double(pid))] + if let path { result["path"] = .string(path) } + if let name = NSRunningApplication(processIdentifier: pid)?.localizedName { result["name"] = .string(name) } + return .object(result) + } +} diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift index 08bd3b7f6d..9cfb5a1a04 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestAdmission.swift @@ -26,6 +26,13 @@ struct RequestParameters: Sendable { guard let value = values[key]?.string else { throw Self.invalid("\(key) must be a string") } return value } + func strings(_ key: String) throws -> [String] { + guard let raw = values[key], case .array(let items) = raw else { throw Self.invalid("\(key) must be a string array") } + return try items.map { value in + guard let path = value.string, !path.isEmpty else { throw Self.invalid("\(key) must contain nonempty strings") } + return path + } + } func optionalString(_ key: String) throws -> String? { contains(key) ? try string(key) : nil } func optionalBoolean(_ key: String) throws -> Bool? { guard let raw = values[key] else { return nil } diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift index ae9969cb1d..8931d3bfff 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift @@ -9,6 +9,7 @@ import Darwin private let cancellation: CancellationLatch private let worker: AXWorker private let capture: WindowCapture + private let permissions: SystemPermissions private let clipboard = ClipboardPaste() private let input: JSONLInput private let writer: JSONLWriter @@ -27,11 +28,12 @@ import Darwin private var nextCancelled = false init(configuration: HelperConfiguration, capture: WindowCapture, channel: SharedSocket, - cancellation: CancellationLatch, scheduler: RequestScheduler, writer: JSONLWriter, + cancellation: CancellationLatch, scheduler: RequestScheduler, writer: JSONLWriter, permissions: SystemPermissions, onClose: @escaping @MainActor @Sendable () -> Void, onFatal: @escaping @Sendable (NativeFailure, Double) -> Void) { self.configuration = configuration self.capture = capture + self.permissions = permissions self.cancellation = cancellation self.channel = channel self.scheduler = scheduler @@ -159,11 +161,25 @@ import Darwin } private func execute(_ request: NativeRequest, trace: ScreenshotTrace?) async throws -> JSONValue { let params = try RequestParameters(request) + try cancellation.check() if request.method == "screenshot" { try capture.requireAvailable() } let requiresCatalog = ["getApp", "listApps", "findApps", "typeText", "pressKey"].contains(request.method) let catalog = requiresCatalog ? ApplicationCatalog.snapshot() : ApplicationCatalogSnapshot(applications: [], frontPID: nil) let result: JSONValue - if request.method == "findApps" { + if request.method == "permissionStatus" { + result = permissions.status() + } else if request.method == "applicationIcons" { + var icons: [String: JSONValue] = [:] + for path in try params.strings("paths") { + try cancellation.check() + if let icon = ApplicationCatalog.icon(at: path) { icons[path] = .string(icon) } + } + result = .object(icons) + } else if request.method == "requestPermission" { + result = try permissions.request(params.string("permission")) + } else if request.method == "openPermissionSettings" { + result = try permissions.openSettings(params.string("permission")) + } else if request.method == "findApps" { result = .array(try ApplicationDiscovery.applications(matching: params.string("query"), exact: false, limit: params.integer("maxAppMatches", in: 1...1000), cancellation: cancellation).map { $0.json(catalog: catalog) }) } else { @@ -179,7 +195,12 @@ import Darwin let outcome = try await worker.execute(request.method, params: params, catalog: catalog, installed: installed, trace: trace) trace?.mark("worker.end") switch outcome { - case .result(let value): result = value + case .result(let value): + if request.method == "getApp", var fields = value.object { + let path = installed?.path ?? catalog.applications.first { Double($0.pid) == value["pid"].number }?.path + if let path, let icon = ApplicationCatalog.icon(at: path) { fields["applicationIcon"] = .string(icon) } + result = .object(fields) + } else { result = value } case .capture(let target): let image = try await capture.capture(target, cancellation: cancellation, trace: trace) result = try await worker.finishCapture(target, image: image, trace: trace) diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/SharedHelperServer.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/SharedHelperServer.swift index 2de82bce2c..253dbfcaa4 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/SharedHelperServer.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/SharedHelperServer.swift @@ -67,6 +67,29 @@ final class NativeSignals { deinit { sources.forEach { $0.cancel() } } } +// Blocking accepts own only Sendable state; server access belongs to the MainActor callback. +private func acceptConnections(listener: SharedSocket, lifetime: SharedHelperLifetime, + onAccept: @escaping @MainActor @Sendable (UUID, SharedSocket, CancellationLatch) -> Void) { + while !lifetime.isStopping { + let descriptor = accept(listener.descriptor, nil, nil) + if descriptor < 0 && errno == EINTR { continue } + guard descriptor >= 0 else { + if !lifetime.isStopping { lifetime.abort(grace: lifetime.shutdownGrace) } + return + } + let channel = SharedSocket(descriptor: descriptor) + var uid: uid_t = 0 + var gid: gid_t = 0 + guard getpeereid(descriptor, &uid, &gid) == 0, uid == getuid() else { channel.shutdown(); continue } + let id = UUID() + let cancellation = CancellationLatch() + guard lifetime.add(id, channel: channel, cancellation: cancellation) else { channel.shutdown(); return } + Task { @MainActor [onAccept, id, channel, cancellation] in + onAccept(id, channel, cancellation) + } + } +} + @MainActor final class SharedHelperServer { private let options: SharedHelperOptions private let versionLock: SharedHelperLock @@ -75,6 +98,7 @@ final class NativeSignals { private let lifetime: SharedHelperLifetime private let scheduler = RequestScheduler() private let capture = WindowCapture() + private let permissions = SystemPermissions() private let instanceID = UUID().uuidString private var connections: [UUID: RequestCoordinator] = [:] private var startupDeadline: NativeDeadline? @@ -99,12 +123,18 @@ final class NativeSignals { func start() throws { let lifetime = self.lifetime - signals = NativeSignals { [weak self] number in - lifetime.abort(grace: lifetime.shutdownGrace, status: 128 + number) - Task { @MainActor in self?.closeConnections(); self?.exitIfEmpty() } + let onStop: @MainActor @Sendable () -> Void = { [weak self] in + guard let self else { return } + self.closeConnections() + self.exitIfEmpty() } - startupDeadline = NativeDeadline(seconds: options.startupTimeout) { [weak self] in - Task { @MainActor in self?.exitIfEmpty() } + signals = NativeSignals { [lifetime, onStop] number in + lifetime.abort(grace: lifetime.shutdownGrace, status: 128 + number) + Task { @MainActor [onStop] in onStop() } + } + let onIdle: @MainActor @Sendable () -> Void = { [weak self] in self?.exitIfEmpty() } + startupDeadline = NativeDeadline(seconds: options.startupTimeout) { [onIdle] in + Task { @MainActor [onIdle] in onIdle() } } let ready = options.ready(pid: getpid(), instanceID: instanceID) let data = try JSONEncoder().encode(ready) @@ -113,38 +143,45 @@ final class NativeSignals { try FileHandle.standardOutput.write(contentsOf: data + Data([10])) try FileHandle.standardOutput.close() let listener = self.listener - DispatchQueue.global(qos: .utility).async { [weak self] in - while !lifetime.isStopping { - let descriptor = accept(listener.descriptor, nil, nil) - if descriptor < 0 && errno == EINTR { continue } - guard descriptor >= 0 else { - if !lifetime.isStopping { lifetime.abort(grace: lifetime.shutdownGrace) } - return - } - let channel = SharedSocket(descriptor: descriptor) - var uid: uid_t = 0 - var gid: gid_t = 0 - guard getpeereid(descriptor, &uid, &gid) == 0, uid == getuid() else { channel.shutdown(); continue } - let id = UUID() - let cancellation = CancellationLatch() - guard lifetime.add(id, channel: channel, cancellation: cancellation) else { channel.shutdown(); return } - Task { @MainActor in self?.admit(id, channel: channel, cancellation: cancellation) } + let onAccept: @MainActor @Sendable (UUID, SharedSocket, CancellationLatch) -> Void = { [weak self] id, channel, cancellation in + guard let self else { + channel.shutdown() + lifetime.remove(id) + return } + self.admit(id, channel: channel, cancellation: cancellation) + } + DispatchQueue.global(qos: .utility).async { [listener, lifetime, onAccept] in + acceptConnections(listener: listener, lifetime: lifetime, onAccept: onAccept) } } private func admit(_ id: UUID, channel: SharedSocket, cancellation: CancellationLatch) { + guard !lifetime.isStopping else { channel.shutdown(); disconnected(id); return } let options = self.options - DispatchQueue.global(qos: .utility).async { [weak self] in + let lifetime = self.lifetime + let onHello: @MainActor @Sendable (HelperConfiguration) async -> Void = { [weak self] configuration in + guard let self else { + channel.shutdown() + lifetime.remove(id) + return + } + await self.connected(id, channel: channel, cancellation: cancellation, configuration: configuration) + } + let onFailure: @MainActor @Sendable () -> Void = { [weak self] in + lifetime.remove(id) + self?.exitIfEmpty() + } + DispatchQueue.global(qos: .utility).async { [options, channel, onHello, onFailure] in do { let configuration = try HelperConfiguration(hello: channel.readHello(timeout: options.startupTimeout), runtimeKey: options.runtimeKey) - Task { @MainActor in await self?.connected(id, channel: channel, cancellation: cancellation, configuration: configuration) } + Task { @MainActor [onHello, configuration] in await onHello(configuration) } } catch { let failure = error as? NativeFailure ?? NativeFailure(code: "NATIVE_PROTOCOL", message: "Invalid shared helper hello") let response = JSONValue.object(["type": .string("error"), "code": .string(failure.code), "message": .string(failure.message)]) if let data = try? JSONEncoder().encode(response) { try? channel.write(data + Data([10]), timeout: options.startupTimeout) } channel.shutdown() - Task { @MainActor in self?.disconnected(id) } + Task { @MainActor [onFailure] in onFailure() } } } } @@ -155,14 +192,18 @@ final class NativeSignals { let writer = JSONLWriter(limit: configuration.maxResponseBytes, channel: channel, writeTimeout: configuration.shutdownGraceSeconds) do { try await writer.write(options.ready(pid: getpid(), instanceID: instanceID)) } catch { channel.shutdown(); disconnected(id); return } + // Signal shutdown can run while the ready frame is being written. + guard !lifetime.isStopping else { channel.shutdown(); disconnected(id); return } let lifetime = self.lifetime + let onFatal: @MainActor @Sendable () -> Void = { [weak self] in self?.closeConnections() } let coordinator = RequestCoordinator(configuration: configuration, capture: capture, channel: channel, cancellation: cancellation, - scheduler: scheduler, writer: writer, onClose: { [weak self] in + scheduler: scheduler, writer: writer, permissions: permissions, onClose: { [weak self] in channel.shutdown() - self?.disconnected(id) - }, onFatal: { [weak self] _, grace in + guard let self else { lifetime.remove(id); return } + self.disconnected(id) + }, onFatal: { [lifetime, onFatal] _, grace in lifetime.abort(grace: grace) - Task { @MainActor in self?.closeConnections() } + Task { @MainActor [onFatal] in onFatal() } }) connections[id] = coordinator startupDeadline?.cancel() diff --git a/packages/experimental/cua-macos/native/protocol.schema.json b/packages/experimental/cua-macos/native/protocol.schema.json index 8282bd5750..e66efb5989 100644 --- a/packages/experimental/cua-macos/native/protocol.schema.json +++ b/packages/experimental/cua-macos/native/protocol.schema.json @@ -1025,6 +1025,131 @@ "params" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "method": { + "type": "string", + "const": "permissionStatus" + }, + "params": { + "type": "object", + "properties": {}, + "additionalProperties": false + } + }, + "required": [ + "id", + "method", + "params" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "method": { + "type": "string", + "const": "applicationIcons" + }, + "params": { + "type": "object", + "properties": { + "paths": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": [ + "paths" + ], + "additionalProperties": false + } + }, + "required": [ + "id", + "method", + "params" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "method": { + "type": "string", + "const": "requestPermission" + }, + "params": { + "type": "object", + "properties": { + "permission": { + "type": "string", + "enum": [ + "accessibility", + "screenRecording" + ] + } + }, + "required": [ + "permission" + ], + "additionalProperties": false + } + }, + "required": [ + "id", + "method", + "params" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "method": { + "type": "string", + "const": "openPermissionSettings" + }, + "params": { + "type": "object", + "properties": { + "permission": { + "type": "string", + "enum": [ + "accessibility", + "screenRecording" + ] + } + }, + "required": [ + "permission" + ], + "additionalProperties": false + } + }, + "required": [ + "id", + "method", + "params" + ], + "additionalProperties": false } ] } diff --git a/packages/experimental/cua-macos/native/tests/LauncherFixture.swift b/packages/experimental/cua-macos/native/tests/LauncherFixture.swift new file mode 100644 index 0000000000..ac297f92e5 --- /dev/null +++ b/packages/experimental/cua-macos/native/tests/LauncherFixture.swift @@ -0,0 +1,29 @@ +// An owned GUI launcher with no windows; its child outlives the launch application. +import AppKit +import Foundation +import Darwin + +@MainActor final class Launcher: NSObject, NSApplicationDelegate { + private let child = Process() + func applicationDidFinishLaunching(_ notification: Notification) { + let arguments = Array(CommandLine.arguments.dropFirst()) + guard let executable = arguments.first else { exit(64) } + child.executableURL = URL(fileURLWithPath: executable) + child.arguments = Array(arguments.dropFirst()) + child.standardOutput = FileHandle.standardOutput + child.standardError = FileHandle.standardError + child.terminationHandler = { @Sendable process in exit(process.terminationStatus) } + do { try child.run() } + catch { fputs("Launching fixture child failed: \(error)\n", stderr); exit(1) } + } +} + +@main enum LauncherMain { + @MainActor static func main() { + let app = NSApplication.shared + app.setActivationPolicy(.accessory) + let launcher = Launcher() + app.delegate = launcher + withExtendedLifetime(launcher) { app.run() } + } +} diff --git a/packages/experimental/cua-macos/native/tests/launcher_identity_e2e.py b/packages/experimental/cua-macos/native/tests/launcher_identity_e2e.py new file mode 100644 index 0000000000..e8204274d1 --- /dev/null +++ b/packages/experimental/cua-macos/native/tests/launcher_identity_e2e.py @@ -0,0 +1,76 @@ +"""Verify system responsibility across nested GUI launchers and helper reuse; no TCC grants are required.""" +import argparse +import base64 +from contextlib import ExitStack +import hashlib +import json +from pathlib import Path +import plistlib +import shutil +import struct +import subprocess +import tempfile +import uuid + +from native_test_support import BINARY, ROOT +from shared_helper_e2e import Connection, wait_released + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--helper", type=Path, default=BINARY) + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix="cua-", dir="/tmp") as temporary, ExitStack() as cleanup: + root = Path(temporary) + helper = root / "helper" + shutil.copy2(args.helper.resolve(strict=True), helper) + key = hashlib.sha256(helper.read_bytes()).hexdigest() + runtime = root / "r" + bundle = root / "Visual Studio Code.app" + binaries = bundle / "Contents" / "MacOS" + binaries.mkdir(parents=True) + launcher = binaries / "launcher" + subprocess.run(["xcrun", "swiftc", "-parse-as-library", str(ROOT / "tests/LauncherFixture.swift"), "-o", str(launcher)], + check=True, timeout=60) + nested = root / "Harness Dev.app" + nested_launcher = nested / "Contents" / "MacOS" / "launcher" + nested_launcher.parent.mkdir(parents=True) + shutil.copy2(launcher, nested_launcher) + identity = uuid.uuid4().hex + register = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister" + for role, application, name in [("owner", bundle, "Code"), ("nested", nested, "Harness Dev")]: + with (application / "Contents" / "Info.plist").open("wb") as output: + plistlib.dump({"CFBundleExecutable": "launcher", + "CFBundleIdentifier": f"dev.dsh.cua.launcher-fixture.{role}-{identity}", + "CFBundleName": name, "CFBundleDisplayName": name, "CFBundlePackageType": "APPL", + "LSUIElement": True}, output) + cleanup.callback(subprocess.run, [register, "-u", str(application)], check=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10) + ready_file, stderr_file = root / "ready.json", root / "stderr.log" + # Launch Services establishes the outer app's responsibility; its GUI child inherits it. + subprocess.run(["/usr/bin/open", "-n", "-g", "-W", "--stdout", str(ready_file), "--stderr", str(stderr_file), + str(bundle), "--args", str(nested_launcher), str(helper), "--ensure-shared", "--runtime-key", key, + "--runtime-directory", str(runtime), "--startup-timeout-ms", "10000"], + check=True, capture_output=True, text=True, timeout=25) + ready = json.loads(ready_file.read_text()) + owner = ready["pid"] + cleanup.callback(wait_released, runtime / (key + ".lock"), owner) + first = Connection(runtime / (key + ".sock"), key) + cleanup.callback(first.close) + state = first.call("permissionStatus", {}) + assert state["parent"]["pid"] == 1, state["parent"] + source = state["launchApplication"] + assert source["name"] == bundle.name, source["name"] + assert Path(source["path"]).resolve() == bundle.resolve(), source["path"] + assert source["icon"].startswith("data:image/png;base64,") + image = base64.b64decode(source["icon"].split(",", 1)[1], validate=True) + assert image[:8] == b"\x89PNG\r\n\x1a\n" + assert struct.unpack(">II", image[16:24]) == (64, 64) + second = Connection(runtime / (key + ".sock"), key) + cleanup.callback(second.close) + assert second.call("permissionStatus", {})["launchApplication"] == source + print("PASS: system-responsible app wins over the nearest GUI ancestor and survives launcher exit and helper reuse") + + +if __name__ == "__main__": + main() diff --git a/packages/experimental/cua-macos/native/tests/shared_helper_e2e.py b/packages/experimental/cua-macos/native/tests/shared_helper_e2e.py index 85a74e7e64..c624fc5b2f 100644 --- a/packages/experimental/cua-macos/native/tests/shared_helper_e2e.py +++ b/packages/experimental/cua-macos/native/tests/shared_helper_e2e.py @@ -5,6 +5,7 @@ Accessibility/Screen Recording grants are required unless --without-desktop is s """ import argparse import concurrent.futures +from contextlib import ExitStack import fcntl import hashlib import json @@ -128,6 +129,15 @@ def main(): assert a.ready == b.ready == ready[0] for client in clients: client.call("protocol-unsupported", {}, error="UNSUPPORTED") + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as invalid: + invalid.settimeout(15) + invalid.connect(str(socket_path)) + invalid.sendall(b'{"type":"hello","protocol":2}\n') + with invalid.makefile("rb") as reader: + failure = json.loads(reader.readline()) + assert failure["type"] == "error" and failure["code"] == "NATIVE_PROTOCOL", failure + assert reader.readline() == b"", "Rejected handshake retained its connection" + b.call("protocol-after-invalid-hello", {}, error="UNSUPPORTED") # Coalesced frames retain socket order, including cancellation before execution starts. a.socket.sendall(b'{"id":"coalesced","method":"listApps","params":{}}\n' b'{"type":"cancel","id":"coalesced"}\n') @@ -218,20 +228,39 @@ def main(): wait_released(lock_path, owner_pid) owner_pid = None - daemon = subprocess.Popen([base[0], "--serve", *base[1:]], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - try: - assert select.select([daemon.stdout], [], [], 15)[0], "Daemon did not report ready" - signalled_ready = json.loads(daemon.stdout.readline()) - assert signalled_ready["type"] == "ready", signalled_ready - daemon.send_signal(signal.SIGTERM) - status = daemon.wait(timeout=15) - assert status == 128 + signal.SIGTERM, (status, daemon.stderr.read()) - wait_released(lock_path, daemon.pid) - finally: - stop(daemon) - daemon.stdout.close() - daemon.stderr.close() - print("PASS: concurrent launch, shared PID, client isolation, cleanup/lock release and SIGTERM") + for shutdown in ["signal", "incomplete-hello", "idle"]: + serve_options = base[1:] if shutdown != "idle" else [*base[1:-1], "100"] + daemon = subprocess.Popen([base[0], "--serve", *serve_options], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + with ExitStack() as connections: + assert select.select([daemon.stdout], [], [], 15)[0], "Daemon did not report ready" + signalled_ready = json.loads(daemon.stdout.readline()) + assert signalled_ready["type"] == "ready", signalled_ready + if shutdown == "incomplete-hello": + pending = connections.enter_context(socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)) + pending.settimeout(15) + pending.connect(str(socket_path)) + pending.sendall(b'{"type":"hello"') + # A later completed handshake observes the accept loop progressing past this socket. + active = Connection(socket_path, key) + connections.callback(active.socket.close) + connections.callback(active.reader.close) + if shutdown != "idle": + daemon.send_signal(signal.SIGTERM) + status = daemon.wait(timeout=15) + expected_status = 0 if shutdown == "idle" else 128 + signal.SIGTERM + assert status == expected_status, (status, daemon.stderr.read()) + wait_released(lock_path, daemon.pid) + # The forced-exit deadline does not unlink these files; require normal cleanup. + assert not socket_path.exists(), "Shutdown left its listening socket" + assert not (runtime / (key + ".ready")).exists(), "Shutdown left its ready record" + if shutdown == "incomplete-hello": + assert pending.recv(1) == b"", "Signal shutdown retained an incomplete handshake" + finally: + stop(daemon) + daemon.stdout.close() + daemon.stderr.close() + print("PASS: concurrent launch, shared PID, client isolation, rejected/incomplete handshakes, cleanup/lock release, SIGTERM and idle exit") finally: errors = [] for client in reversed(clients): diff --git a/packages/experimental/cua-macos/native/tests/swift/PermissionIdentityTests.swift b/packages/experimental/cua-macos/native/tests/swift/PermissionIdentityTests.swift index 6c29e6afa6..58755b15b1 100644 --- a/packages/experimental/cua-macos/native/tests/swift/PermissionIdentityTests.swift +++ b/packages/experimental/cua-macos/native/tests/swift/PermissionIdentityTests.swift @@ -1,5 +1,7 @@ // The selected catalog entry must still identify the executable inspected for consent. +import AppKit import Darwin +import Foundation import Testing @testable import CuaNative @@ -22,3 +24,37 @@ import Testing #expect(error.code == "APP_UNAVAILABLE") } } + +@MainActor @Test func applicationIconsUseBoundedNativePNGs() throws { + let encoded = try #require(ApplicationCatalog.icon(at: "/System/Library/CoreServices/Finder.app/Contents/MacOS/Finder")) + #expect(encoded.hasPrefix("data:image/png;base64,")) + let data = try #require(Data(base64Encoded: String(encoded.dropFirst("data:image/png;base64,".count)))) + let bitmap = try #require(NSBitmapImageRep(data: data)) + #expect(bitmap.pixelsWide == 64) + #expect(bitmap.pixelsHigh == 64) + #expect(encoded.utf8.count <= 65536) +} + +@Test func resolvesTheCurrentProcessResponsibility() throws { + let responsible = try #require(NativePlatform.responsiblePID(getpid())) + #expect(responsible > 0) + #expect(NativePlatform.responsiblePID(responsible) == responsible) +} + +@MainActor @Test func unknownResponsibilityDoesNotInventAnApplication() { + #expect(SystemPermissions.responsibleApplication(for: 1) == nil) + #expect(SystemPermissions.responsibleApplication(for: Int32.max) == nil) + #expect(NativePlatform.responsiblePID(Int32.max) == nil) +} + +@MainActor @Test func permissionStatusKeepsProcessDiagnosticsSeparateFromLaunchMetadata() { + let permissions = SystemPermissions() + let state = permissions.status() + #expect(state["helper"]["pid"] == .number(Double(getpid()))) + #expect(state["parent"]["pid"] == .number(Double(getppid()))) + if let source = state["launchApplication"].object { + #expect(source["name"]?.string?.isEmpty == false) + #expect(source["path"]?.string?.hasSuffix(".app") == true) + } + #expect(permissions.status()["launchApplication"] == state["launchApplication"]) +} diff --git a/packages/experimental/cua-macos/native/tests/swift/ProtocolTests.swift b/packages/experimental/cua-macos/native/tests/swift/ProtocolTests.swift index 6ebed6fb7c..611fa6786c 100644 --- a/packages/experimental/cua-macos/native/tests/swift/ProtocolTests.swift +++ b/packages/experimental/cua-macos/native/tests/swift/ProtocolTests.swift @@ -116,3 +116,9 @@ func rejectsInvalidWindowIDs(_ value: String) throws { #expect(throws: NativeFailure.self) { try params.optionalBoolean("includeMenus") } } } + +@Test(arguments: ["null", "true", "[1]", "[\"\"]"]) +func rejectsMalformedApplicationIconPaths(_ value: String) throws { + let request = try NativeRequest(data: Data("{\"id\":\"icons\",\"method\":\"applicationIcons\",\"params\":{\"paths\":\(value)}}".utf8)) + #expect(throws: NativeFailure.self) { try RequestParameters(request).strings("paths") } +} diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index 829706b8ab..742783330a 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -1,6 +1,6 @@ /** macOS operation guidance supplied with the shared CUA object API. */ /** Instructions for raw operation calls and optional object-API bindings. */ -export const GUIDANCE = `Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. +export const GUIDANCE = `Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user allows another request on the permission page. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again. @@ -37,7 +37,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to complete the Computer Use permission setup for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. diff --git a/packages/experimental/cua-macos/tests/large-image-ptc.spec.ts b/packages/experimental/cua-macos/tests/large-image-ptc.spec.ts index a8a93b1857..9f8124ff99 100644 --- a/packages/experimental/cua-macos/tests/large-image-ptc.spec.ts +++ b/packages/experimental/cua-macos/tests/large-image-ptc.spec.ts @@ -73,7 +73,7 @@ it.skipIf(process.platform === 'win32')('delivers a valid PNG larger than 4 MiB const configPath = join(root, 'cordis.yml') const configs: Record = { tools: { mode: 'ptc' }, attachments: { dshHome: join(root, 'home') }, workingDirectory: { defaultDirectory: root }, - fs: { cwd: root }, sandboxPolicy: { mode: 'danger-full-access', workspaceRoot: root }, cua: { helperPath, allowedApps: [{ 'id': 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'name': 'Fixture', 'executablePath': '/fixture/app' }] }, + fs: { cwd: root }, sandboxPolicy: { mode: 'danger-full-access', workspaceRoot: root }, cua: { agentPreset: false, helperPath, allowedApps: [{ 'id': 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'name': 'Fixture', 'executablePath': '/fixture/app' }] }, } await writeFile(configPath, Object.keys(modules).flatMap(name => [ `- name: 'cordis:${name}'`, ...configs[name] === undefined ? [] : [` config: ${JSON.stringify(configs[name])}`], diff --git a/packages/experimental/cua-macos/tests/loader-composition.spec.ts b/packages/experimental/cua-macos/tests/loader-composition.spec.ts index 9b65b66398..289a47a658 100644 --- a/packages/experimental/cua-macos/tests/loader-composition.spec.ts +++ b/packages/experimental/cua-macos/tests/loader-composition.spec.ts @@ -85,7 +85,7 @@ it.skipIf(process.platform === 'win32')('shares one application between retained await writeFile(configPath, Object.keys(modules).flatMap(name => [ `- name: 'cordis:${name}'`, ...name === 'attachments' ? [' config:', ` dshHome: ${JSON.stringify(root)}`] : [], - ...name === 'cua' ? [' config:', ` helperPath: ${JSON.stringify(helperPath)}`, ' allowedApps: [{"id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "name": "Fixture", "executablePath": "/fixture/app"}]'] : [], + ...name === 'cua' ? [' config:', ' agentPreset: false', ` helperPath: ${JSON.stringify(helperPath)}`, ' allowedApps: [{"id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "name": "Fixture", "executablePath": "/fixture/app"}]'] : [], ]).join('\n') + '\n') context.baseUrl = pathToFileURL(root).href + '/' await context.plugin(Loader) diff --git a/packages/experimental/cua-macos/tests/persistent-bridge.built.e2e.ts b/packages/experimental/cua-macos/tests/persistent-bridge.built.e2e.ts index 9dd6452bde..b9573c9eca 100644 --- a/packages/experimental/cua-macos/tests/persistent-bridge.built.e2e.ts +++ b/packages/experimental/cua-macos/tests/persistent-bridge.built.e2e.ts @@ -50,7 +50,7 @@ it.skipIf(process.platform === 'win32' || !existsSync(bridge) || !existsSync(boo await ctx.plugin(NodeRuntime, { executionMode: 'persistent-in-memory', launch: { kind: 'node-script', executable: process.execPath, bootstrapPath: bootstrap } }) await ctx.plugin(Loader) ctx.loader.builtins.cua = Cua - const entry = await ctx.loader.create({ name: 'cordis:cua', config: { helperPath, allowedApps: [{ 'id': 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'name': 'Fixture', 'executablePath': '/fixture/app' }] } }) + const entry = await ctx.loader.create({ name: 'cordis:cua', config: { agentPreset: false, helperPath, allowedApps: [{ 'id': 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'name': 'Fixture', 'executablePath': '/fixture/app' }] } }) await ctx.loader.resolve(entry).fiber!.await() await mountAgentLoopTestHarness(ctx) ctx.llm.registerAdapter(['bridge-fixture'], new VisualAdapter()) diff --git a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts index 879657018f..54b4bf3a19 100644 --- a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts +++ b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts @@ -18,6 +18,15 @@ describe('native protocol generation', () => { { method: 'paste', params: { appHandle: 'app', text: 'text', pasteReadTimeoutMs: 1000 } }, { method: 'findApps', params: { query: 'Editor', maxAppMatches: 1 } }, { method: 'launchApp', params: { appHandle: 'pending' } }, + { method: 'permissionStatus', params: {} }, + { method: 'requestPermission', params: { permission: 'accessibility' } }, + { method: 'openPermissionSettings', params: { permission: 'screenRecording' } }, + { method: 'permissionStatus', params: {} }, + { method: 'requestPermission', params: { permission: 'accessibility' } }, + { method: 'openPermissionSettings', params: { permission: 'screenRecording' } }, + { method: 'permissionStatus', params: {} }, + { method: 'requestPermission', params: { permission: 'accessibility' } }, + { method: 'openPermissionSettings', params: { permission: 'screenRecording' } }, ])('rejects Host-only request data at model admission: $method', (request) => { expect(Request.safeParse(request).success).toBe(false) }) diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index b91166bad4..06d3f8a846 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -1,16 +1,9 @@ /** Native helper replacement is synchronized with Agent and provider disposal. */ -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' -import { pathToFileURL } from 'node:url' import { setImmediate } from 'node:timers/promises' import { Context, FiberState } from '@deepseek-ai/cordis' -import Loader from '@deepseek-ai/cordis-plugin-loader' -import Timer from '@deepseek-ai/cordis-plugin-timer' -import { initProfile, mountRootInclude, readProfilePatches, type ProfileContext } from '@deepseek-ai/dsh-app-boot' -import ConfigEditor from '@deepseek-ai/dsh-config-editor' -import Settings from '@deepseek-ai/dsh-settings' -import Hmr from '@deepseek-ai/dsh-hmr' import { mountAgentLoopTestDependencies, mountAgentLoopTestHarness } from '@deepseek-ai/dsh-agent-loop-testkit' import ComputerUseRegistry from '@deepseek-ai/dsh-computer-use' import LocalAttachmentStore from '@deepseek-ai/dsh-attachment-local' @@ -55,6 +48,7 @@ vi.mock('@deepseek-ai/dsh-cua/native-client', async importOriginal => ({ })) const config = { + agentPreset: false, helperPath: resolve('fixture-helper'), buildTimeoutMs: 300_000, timeoutMs: 10_000, connectTimeoutMs: 5000, screenshotTimeoutMs: 5000, axTimeoutMs: 3000, maxRequestBytes: 1024, maxResponseBytes: 4096, maxClipboardBytes: 8192, pasteReadTimeoutMs: 1000, maxAppMatches: 20, graceMs: 10, textEventIntervalMs: 8, maxElements: 64, maxDepth: 8, dragDurationMs: 200, dragSteps: 10, } satisfies Omit @@ -93,9 +87,9 @@ async function setup(model?: { provider: string; model: string }) { async function selectApplication(f: Awaited>) { await f.ctx.plugin(UserQuestions) - f.ctx.on('user-questions/request', async () => ({ answers: [{ id: 'cua-app-access', selected: ['Allow for this task'] }] })) + f.ctx.on('user-questions/request', async () => ({ answers: [{ id: 'cua-app-access', selected: ['Allow for this session'] }] })) native.request.mockResolvedValueOnce({ appHandle: 'app-one', pid: 17, name: 'Fixture', - authorization: { id: 'a'.repeat(64), name: 'Fixture', executablePath: '/fixture/app', persistable: true } }) + authorization: { id: 'a'.repeat(64), name: 'Fixture', executablePath: '/fixture/app', persistable: false } }) expect((await f.execute('getApp', { selector: 'Fixture' })).isError).toBe(false) } @@ -105,7 +99,7 @@ it('rejects a relative helper path before publishing computer-use operations', a await expect(NativeProvider.apply(ctx, { ...config, allowedApps: { get: () => [] }, helperPath: 'relative-helper' })).rejects.toThrow('helperPath must be absolute') }) -it.each(['Allow for this task', 'Deny'])('obtains %s before a stopped application can launch', async (choice) => { +it.each(['Allow for this session', 'Deny'])('obtains %s before a stopped application can launch', async (choice) => { const f = await setup() await f.ctx.plugin(UserQuestions) const asked: PromiseWithResolvers = Promise.withResolvers() @@ -130,7 +124,7 @@ it.each(['Allow for this task', 'Deny'])('obtains %s before a stopped applicatio it('rejects a launch result that changes the approved handle', async () => { const f = await setup() await f.ctx.plugin(UserQuestions) - f.ctx.on('user-questions/request', async () => ({ answers: [{ id: 'cua-app-access', selected: ['Allow for this task'] }] })) + f.ctx.on('user-questions/request', async () => ({ answers: [{ id: 'cua-app-access', selected: ['Allow for this session'] }] })) native.request.mockResolvedValueOnce({ appHandle: 'pending-app', pid: null, name: 'Fixture', authorization: { id: 'a'.repeat(64), name: 'Fixture', executablePath: '/fixture/app', persistable: false, } }).mockResolvedValueOnce({ appHandle: 'another-app', pid: 17, name: 'Fixture' }) @@ -257,6 +251,24 @@ it('keeps Agent disposal pending until its helper cleanup completes', async () = } finally { release.resolve(); await disposing } }) +it('retains the computer-use registration until permission helper cleanup settles', async () => { + const f = await setup() + native.request.mockResolvedValue({ accessibility: false, screenRecording: false, helper: { pid: 9 }, parent: { pid: 8 } }) + await f.ctx.cuaAccess.systemStatus() + const entered = Promise.withResolvers(), release = Promise.withResolvers() + native.dispose.mockImplementation(async () => { entered.resolve(undefined); await release.promise }) + onTestFinished(() => { release.resolve(undefined) }) + const registry = f.ctx.computerUse + const closing = f.provider.dispose() + try { + await entered.promise + expect(registry.providerName).toBe('dsh-cua') + release.resolve(undefined) + await closing + expect(registry.providerName).toBeUndefined() + } finally { release.resolve(undefined); await closing } +}) + it('joins helper cleanup already started by Agent disposal when the provider unloads', async () => { const f = await setup() await f.execute() @@ -497,6 +509,8 @@ it.each([ { owner: 'provider', stage: 'storage' }, { owner: 'agent', stage: 'admission' }, { owner: 'agent', stage: 'storage' }, + { owner: 'revocation', stage: 'admission' }, + { owner: 'revocation', stage: 'storage' }, ])('waits for screenshot $stage and cancels its result during $owner disposal', async ({ owner, stage }) => { const f = await setup({ provider: 'image-fixture', model: 'vision' }) await selectApplication(f) @@ -526,7 +540,9 @@ it.each([ const reached = await Promise.race([entered.promise.then(() => true), pending.then(() => false)]) expect(reached).toBe(true) let closed = false - closing = (owner === 'provider' ? f.provider.dispose() : f.handle.dispose()).then(() => { closed = true }) + closing = (owner === 'provider' ? f.provider.dispose() + : owner === 'agent' ? f.handle.dispose() + : f.ctx.cuaAccess.revoke('a'.repeat(64) as import('@deepseek-ai/dsh-cua/types').CuaApplicationId)).then(() => { closed = true }) await setImmediate() expect(admissionSignal?.aborted).toBe(true) expect(closed).toBe(false) @@ -622,79 +638,3 @@ it('rejects an unsupported native host before starting preparation', async () => await expect(NativeProvider.apply(f.ctx, { ...automaticConfig, allowedApps: { get: () => [] } })).rejects.toThrow('unsupported host') expect(native.prepare).not.toHaveBeenCalled() }) - - -it('stores signed application consent in the current profile and refuses unavailable settings', async () => { - const f = await setup() - await f.provider.dispose() - const home = await mkdtemp(join(tmpdir(), 'cua-grant-profile-')), dir = join(home, 'profile') - onTestFinished(async () => { await f.ctx.fiber.dispose(); await rm(home, { recursive: true, force: true }) }) - initProfile(dir, ['permission-fixture-bundle']) - const bundle = join(dir, 'node_modules', 'permission-fixture-bundle') - await mkdir(bundle, { recursive: true }) - await writeFile(join(home, 'package.json'), '{"name":"test-installation"}\n') - await writeFile(join(bundle, 'package.json'), JSON.stringify({ name: 'permission-fixture-bundle', version: '1.0.0', - dsh: { bundle: { patch: 'cordis.patch.yml' } } })) - await writeFile(join(bundle, 'cordis.patch.yml'), JSON.stringify([{ insert: [{ id: 'cua', name: 'cordis:cua', config }] }])) - await writeFile(join(dir, 'cordis.yml'), '[]\n') - const profile: ProfileContext = { name: 'test', startedBundles: ['permission-fixture-bundle'], dir, - patchPath: join(dir, 'cordis.patch.yml'), installAnchor: join(home, 'package.json'), cwd: home, home, - overlays: [], telemetryDisabledEnv: undefined } - f.ctx.baseUrl = pathToFileURL(home).href + '/' - f.ctx.provide('profileContext', profile) - f.ctx.provide('appReady', { onReady: (listener: () => void) => { listener(); return () => {} } }) - await f.ctx.plugin(Loader) - f.ctx.loader.builtins.cua = NativeProvider - await f.ctx.plugin(ConfigEditor) - await mountRootInclude(f.ctx, join(dir, 'cordis.yml'), readProfilePatches('test', profile)) - await f.ctx.loader.await() - await f.ctx.plugin(UserQuestions) - const answer = vi.fn(async () => ({ answers: [{ id: 'cua-app-access', selected: ['Allow for this task'] }] })) - f.ctx.on('user-questions/request', answer) - native.request.mockResolvedValue({ appHandle: 'task-app', pid: 17, name: 'Fixture', - authorization: { id: 'a'.repeat(64), name: 'Fixture', executablePath: '/fixture/app', persistable: true } }) - expect((await f.execute('getApp', { selector: 'Fixture' })).isError).toBe(false) - await f.ctx.plugin(Settings) - await f.ctx.plugin(Timer) - await f.ctx.plugin(Hmr, { root: [], ignored: [], debounce: 0 }) - await f.ctx.hmr.runExclusive(async () => {}) - expect(f.ctx.settings.describe().find(row => row.ns === 'cua')?.autoGenerate).toBe(false) - answer.mockResolvedValue({ answers: [{ id: 'cua-app-access', selected: ['Always allow'] }] }) - const authorization = { id: 'a'.repeat(64), name: 'Fixture', executablePath: '/fixture/app', persistable: true } - native.request.mockResolvedValue({ appHandle: 'signed-app', pid: 17, name: 'Fixture', authorization }) - expect((await f.execute('getApp', { selector: 'Fixture' })).isError).toBe(false) - expect(await readFile(profile.patchPath, 'utf8')).toContain(authorization.id) - const settings = f.ctx.settings.describe().find(row => row.ns === 'cua')! - await f.ctx.settings.mutate('cua', [{ op: 'set', path: ['allowedApps'], value: [] }], settings.revision) - expect(await readFile(profile.patchPath, 'utf8')).toContain('allowedApps: []') - const revoked = await f.execute('getState', { appHandle: 'signed-app' }) - expect(revoked.isError).toBe(true) - expect(JSON.stringify(revoked.content)).toContain('APP_ACCESS_REVOKED') - vi.spyOn(f.ctx.settings, 'describe').mockReturnValue([]) - native.request.mockResolvedValue({ appHandle: 'different-app', pid: 18, name: 'Other fixture', authorization: { ...authorization, id: 'b'.repeat(64) } }) - const unavailable = await f.execute('getApp', { selector: 'Fixture' }) - expect(unavailable.isError).toBe(true) - expect(JSON.stringify(unavailable.content)).toContain('APP_ACCESS_DENIED') -}) - -it('rejects an application handle before any helper or consent has been acquired', async () => { - const f = await setup() - const result = await f.execute('getState', { appHandle: 'unselected-app' }) - expect(result.isError).toBe(true) - expect(JSON.stringify(result.content)).toContain('APP_ACCESS_REQUIRED') - expect(native.instances).toHaveLength(0) - expect(native.request).not.toHaveBeenCalled() -}) - -it('uses configured signed application grants without requesting a human answer', async () => { - const f = await setup() - await f.provider.dispose() - const grant = { id: 'a'.repeat(64), name: 'Fixture', executablePath: '/fixture/app' } - await f.ctx.plugin(NativeProvider, { ...config, allowedApps: [grant] }) - native.request.mockResolvedValueOnce({ appHandle: 'saved-app', pid: 17, name: 'Fixture', authorization: { ...grant, persistable: true } }) - expect((await f.execute('getApp', { selector: 'Fixture' })).isError).toBe(false) - native.request.mockResolvedValue({ dispatched: true, verified: false }) - expect((await f.execute('activate', { appHandle: 'saved-app' })).isError).toBe(false) - expect(native.instances).toHaveLength(1) - expect(native.request).toHaveBeenCalledTimes(2) -}) diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index a99f1eaf24..48e9b2c525 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -12,14 +12,14 @@ en: f4d9cc48587d62a2 zh: d1bac6c1fe7eb29e /deepseek-ai-dsh-cua/table-of-contents: - en: d152484eb41ac6b4 - zh: 1db86285abf6c499 + en: 4081cf444a77bf20 + zh: def91d58969ec281 /deepseek-ai-dsh-cua/use-this-package: - en: 52cacbe2d2aeb2bc - zh: b3c65814d9a17052 + en: e230429b9e15efd8 + zh: 10dcb9ba5194dc76 /deepseek-ai-dsh-cua/understand-the-implementation: - en: effcee57d8854c32 - zh: d02c6b76e7249051 + en: 751fe580f79dfaae + zh: 3811db65f634ef5a /deepseek-ai-dsh-cua/further-exploration: en: b8b0aece886fadaa zh: ecd8c88d601f2179 diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index dcd8dc186d..866aa27baa 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -20,6 +20,8 @@ Inspect applications, retain window objects across PTC calls, and send actions t - [Known Limitations and Deferred Work](#known-limitations-and-deferred-work) - [Dev Note](#dev-note) +Application consent uses the existing User Questions options and remains independent of the session tool-approval policy. The provider accepts only one exact offered label without custom text. Native icons accompany permission-page observations and remain outside saved identities and model results. System checks, saved grants and refusals, and icons use separate queries. + ----- @@ -33,7 +35,7 @@ Observation methods emit and return their result. Use `void await app.getAXState `tools.cua` returns operation values directly. Screenshot values retain the encoded image Base64 and add a durable `attachment` reference for the tool renderer; model content contains coordinate metadata and one image, without the encoded bytes. Before capture, the tool requires an attachment store and a resolved model route declaring image input. Admission and storage failures reject the call. The object bridge preserves the screenshot bytes and validated metadata without uploading or emitting the same image again. -Application consent precedes launch, observation, capture, and input. Task access belongs to the selected instance; saved grants require platform-verified executable identity. Revocation closes affected Agent connections, so their old objects require fresh selection. No input is replayed after cancellation, timeout, or reconnect. +Application consent precedes launch, observation, capture, and input. Session access is kept in memory by session and application identity, survives agent and helper replacement, and expires when DSH restarts or Computer Use is disabled. Saved grants require platform-verified executable identity. Refusal and revocation close affected Agent connections, so their old objects require fresh selection. No input is replayed after cancellation, timeout, or reconnect. Actions return `dispatched:true, verified:false`; read fresh state to verify the result. Coordinate actions default to pixels in the latest returned screenshot; `relative: true` accepts integers from 0 to 1000 after a successful window observation. The Host resolves attachment normalization and the selected model route’s image projection before returning the screenshot, then converts input to native window points. Resizing invalidates the screenshot mapping; `SCREENSHOT_REQUIRED` requires a new screenshot for absolute coordinates. Resizing rejects old coordinates with `WINDOW_CHANGED`; movement alone preserves window-local coordinates. @@ -47,6 +49,8 @@ Actions return `dispatched:true, verified:false`; read fresh state to verify the The provider calls [installCua](src/index.ts) with its prepared helper, connection factory, and guidance. [NativeClient](src/native-client.ts) owns request correlation, byte limits, cancellation acknowledgement, and failure settlement. The platform connector owns startup and connection cleanup. The shared executor retains one connection per Agent and joins in-flight image writes during disposal. +`agentPreset` defaults to `ptc`, restricting tools and guidance to Agents bound to that preset. The provider uses the existing `agent/created` and `agent-preset/selected` events and Agent-owned scopes, including Agents already present when it loads. Provider unload and Agent disposal release those registrations. Set `agentPreset: false` for global registration in standalone compositions without presets. The Host continues to own permissions, native connections and cancellation; preset definitions remain unchanged. + [The operation schema](src/protocol.ts) is the authority for model parameters and separately declared Host-only fields. Native implementations enforce target identity, window geometry, resource limits, and operating-system access at execution. No invariant companion is published because this library has no separately projected registry or durable state. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index a526735ec5..be9d9dc314 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -20,6 +20,8 @@ kind: "package-library" - [已知限制与后续工作](#known-limitations-and-deferred-work) - [开发备注](#dev-note) +应用授权使用现有 User Questions 选项，仍独立于会话的工具审批策略。提供方只接受一个准确匹配所给选项的标签，附带自由文本的回答不会授权。本机图标用于权限页展示，不进入保存的身份信息或模型结果。系统权限、持久授权与拒绝记录、图标分别查询。 + ----- @@ -33,7 +35,7 @@ kind: "package-library" `tools.cua` 直接返回操作结果。截图结果保留图片 Base64，并增加供工具渲染使用的持久 `attachment` 引用；模型内容只包含坐标元数据和一张图片，不包含编码字节。截图前要求附件服务可用，并确认当前模型支持图片输入。图片接入或保存失败会拒绝调用。对象桥接保留截图字节和已校验的元数据，不重复上传或输出同一张图片。 -应用授权发生在启动、观察、截图和输入之前。任务许可绑定已选择的应用实例；长期许可要求平台验证可执行文件身份。撤销会关闭受影响的 Agent 连接，因此旧对象必须重新选择目标。取消、超时或重新连接后均不重放输入。 +应用授权发生在启动、观察、截图和输入之前。会话许可按会话和应用身份保存在内存中，Agent 或 helper 重建后仍可复用，重启 DSH 或关闭 Computer Use 后失效。长期许可要求平台验证可执行文件身份。拒绝或撤销会关闭受影响的 Agent 连接，因此旧对象必须重新选择目标。取消、超时或重新连接后均不重放输入。 动作返回 `dispatched:true, verified:false`；请读取新状态以验证结果。坐标操作默认使用最近返回截图中的像素；成功观察窗口后，传入 `relative: true` 可使用 0～1000 的整数比例。Host 在返回截图前解析附件归一化和当前模型的图片缩放规则，再将输入换算为原生窗口 point。窗口尺寸变化会使截图映射失效，绝对坐标遇到 `SCREENSHOT_REQUIRED` 时需要重新截图。窗口缩放后以 `WINDOW_CHANGED` 拒绝旧坐标；单纯移动窗口不改变窗口内坐标。 @@ -47,6 +49,8 @@ kind: "package-library" 提供方将准备好的 helper、连接工厂和指引传给 [installCua](src/index.ts)。[NativeClient](src/native-client.ts) 负责请求关联、字节上限、取消确认和失败处理。平台连接器负责启动和连接清理。共享执行器为每个 Agent 保留一个连接，并在销毁时等待进行中的图像写入。 +`agentPreset` 默认为 `ptc`，将工具和指引限制到绑定该 preset 的 Agent。提供方使用现有 `agent/created`、`agent-preset/selected` 事件及 Agent 拥有的作用域，也处理启用前已存在的 Agent。提供方卸载和 Agent 销毁都会释放这些注册。不使用 preset 的独立组合可设置 `agentPreset: false` 来全局注册。Host 仍拥有权限、本机连接和取消操作，preset 定义保持不变。 + [操作 schema](src/protocol.ts) 定义模型参数，并单独声明 Host 专用字段。原生实现在执行时检查目标身份、窗口几何、资源上限和操作系统访问条件。本库没有独立投影的注册表或持久状态，因此不发布 invariant 模块。 diff --git a/packages/experimental/cua/package.json b/packages/experimental/cua/package.json index a1fb346e0a..a28b368b46 100644 --- a/packages/experimental/cua/package.json +++ b/packages/experimental/cua/package.json @@ -28,6 +28,18 @@ "types": "./lib/types/bridge.d.ts", "default": "./lib/bridge.js" }, + "./typert": { + "types": "./lib/typert.host.d.ts", + "default": "./lib/typert.host.js" + }, + "./remote": { + "types": "./lib/typert.remote-client.d.ts", + "default": "./lib/typert.remote-client.js" + }, + "./types": { + "types": "./lib/types/types.d.ts", + "default": "./lib/types.js" + }, "./native-client": { "types": "./lib/types/native-client.d.ts", "default": "./lib/native-client.js" @@ -39,11 +51,16 @@ }, "files": [ "lib/index.js", + "lib/types.js", "lib/api.js", "lib/bridge.js", "lib/native-client.js", "lib/protocol.js", - "lib/types/**/*.d.ts" + "lib/types/**/*.d.ts", + "lib/typert.host.js", + "lib/typert.host.d.ts", + "lib/typert.remote-client.js", + "lib/typert.remote-client.d.ts" ], "license": "MIT", "peerDependencies": { @@ -55,14 +72,17 @@ "@deepseek-ai/dsh-subprocess": "workspace:*", "@deepseek-ai/dsh-user-questions": "workspace:*", "@deepseek-ai/dsh-attachment": "workspace:*", - "@deepseek-ai/dsh-llm": "workspace:*" + "@deepseek-ai/dsh-llm": "workspace:*", + "@deepseek-ai/dsh-scope": "workspace:*", + "@deepseek-ai/dsh-agent-preset-registry": "workspace:*" }, "dependencies": { "@deepseek-ai/dsh-util-values": "workspace:*", "@deepseek-ai/schemastery": "workspace:~", "zod": "^4.4.3", "@deepseek-ai/dsh-brand": "workspace:*", - "@deepseek-ai/dsh-atomic-write": "workspace:*" + "@deepseek-ai/dsh-atomic-write": "workspace:*", + "@deepseek-ai/dsh-typert-protocol": "workspace:*" }, "devDependencies": { "@deepseek-ai/dsh-working-directory": "workspace:*", @@ -89,11 +109,13 @@ "@deepseek-ai/dsh-sandbox-policy": "workspace:*", "sharp": "^0.35.3", "@deepseek-ai/dsh-ptc-runtime": "workspace:*", - "@deepseek-ai/dsh-user-questions": "workspace:*", "@deepseek-ai/dsh-settings": "workspace:*", + "@deepseek-ai/dsh-user-questions": "workspace:*", "@deepseek-ai/dsh-app-boot": "workspace:*", "@deepseek-ai/dsh-config-editor": "workspace:*", "@deepseek-ai/dsh-hmr": "workspace:*", - "@deepseek-ai/cordis-plugin-timer": "workspace:~" + "@deepseek-ai/cordis-plugin-timer": "workspace:~", + "@deepseek-ai/dsh-scope": "workspace:*", + "@deepseek-ai/dsh-agent-preset-registry": "workspace:*" } } diff --git a/packages/experimental/cua/src/access-types.ts b/packages/experimental/cua/src/access-types.ts new file mode 100644 index 0000000000..1124ff5023 --- /dev/null +++ b/packages/experimental/cua/src/access-types.ts @@ -0,0 +1,30 @@ +/** Client-safe Computer Use permission observations and saved application decisions. */ +import type { CuaApplicationGrant } from './application-types.ts' +export type { CuaApplicationGrant, CuaApplicationId } from './application-types.ts' + +/** macOS privileges used by the native provider. */ +export type CuaSystemPermission = 'accessibility' | 'screenRecording' +/** Observed process; neither the helper nor its parent is assumed to be the TCC owner. */ +export interface CuaPermissionProcess { + pid: number + path?: string | undefined + name?: string | undefined +} +/** Fresh permission observations and startup metadata from the shared native helper. */ +export interface CuaSystemPermissionState { + accessibility: boolean + screenRecording: boolean + helper: CuaPermissionProcess + parent: CuaPermissionProcess + /** Application macOS reports as responsible for the helper, captured at startup; absent when it cannot be identified. */ + launchApplication?: { name: string; path: string; icon?: string | undefined } | undefined +} +/** Saved grants and refused applications, independent of native permission checks and icons. */ +export interface CuaApplicationState { + /** Owning configuration entry for permission updates; null when this composition has no Loader entry. */ + settingsNamespace: string | null + allowedApps: readonly CuaApplicationGrant[] + /** Refusals from any session, excluding saved grants; other sessions may still hold session access. */ + blockedApps: readonly CuaApplicationGrant[] + persistentAvailable: boolean +} diff --git a/packages/experimental/cua/src/access.ts b/packages/experimental/cua/src/access.ts new file mode 100644 index 0000000000..2553286a8b --- /dev/null +++ b/packages/experimental/cua/src/access.ts @@ -0,0 +1,142 @@ +/** Browser controls for the provider's application decisions and native permission work. */ +import type { Context } from '@deepseek-ai/cordis' +import { Remote, TypertRemoteService } from '@deepseek-ai/dsh-typert-protocol' +import { ApplicationAccess, type AccessOptions } from './application-access.ts' +import type { JsonValue } from '@deepseek-ai/dsh-util-values' +import { z } from 'zod' +import { ApplicationIcon } from './protocol.ts' +import { NativeClient, type NativeClientConfig, type NativeChannelFactory } from './native-client.ts' +import type { CuaApplicationId, CuaApplicationState, CuaSystemPermission, CuaSystemPermissionState } from './access-types.ts' +export { ApplicationAuthorizationResult } from './application-access.ts' +export type * from './access-types.ts' + +interface ControlOptions extends AccessOptions { + native: NativeClientConfig + createChannel: NativeChannelFactory + settingsNamespace: string | null +} +type Control = (method: string, params: Record) => Promise +const permissionProcess = z.object({ pid: z.number().int().positive(), path: z.string().optional(), name: z.string().optional() }) +const systemPermissions = z.object({ + accessibility: z.boolean(), screenRecording: z.boolean(), helper: permissionProcess, parent: permissionProcess, + launchApplication: z.object({ name: z.string().min(1), path: z.string().min(1), icon: ApplicationIcon.optional() }).optional(), +}) + +declare module '@deepseek-ai/cordis' { + interface Context { + /** Native permission controls and executor-owned application consent. */ + cuaAccess: CuaAccess + } +} + +/** Browser controls for provider-owned native permission and application decisions. */ +export class CuaAccess extends TypertRemoteService { + /** + * @param ctx Provider context. + * @param policy Application decision owner. + * @param settingsNamespace Owning configuration entry. + * @param control Serialized native permission dispatch. + */ + constructor(ctx: Context, private readonly policy: ApplicationAccess, + private readonly settingsNamespace: string | null, private readonly control: Control) { super(ctx, 'cuaAccess') } + + /** + * Read application decisions without waiting for the native helper. + * @returns Saved grants and refused applications with their persistence owner. + */ + @Remote + applications(): CuaApplicationState { + return { settingsNamespace: this.settingsNamespace, ...this.policy.status() } + } + + /** + * Inspect system privileges independently of application decisions and icons. + * @returns Fresh native preflight and launch metadata. + */ + @Remote + async systemStatus(): Promise { + return systemPermissions.parse(await this.control('permissionStatus', {})) + } + + /** + * Load presentation icons separately from application decisions. + * @returns Optional native PNG icons keyed by executable path for saved and blocked applications. + */ + @Remote + async applicationIcons(): Promise> { + const state = this.policy.status() + const paths = [...new Set([...state.allowedApps, ...state.blockedApps].map(app => app.executablePath))] + return paths.length === 0 ? {} : z.record(z.string(), ApplicationIcon).parse(await this.control('applicationIcons', { paths })) + } + + /** + * Request a system privilege explicitly selected by the user. + * @param permission macOS privilege to request. + * @returns Preflight state after requesting; a false value never implies consent. + */ + @Remote + async requestPermission(permission: CuaSystemPermission): Promise { + return systemPermissions.parse(await this.control('requestPermission', { permission })) + } + + /** + * Open the macOS privacy pane selected by the user. + * @param permission Corresponding system privilege. + * @returns Completion after macOS accepted the open request. + */ + @Remote + async openSettings(permission: CuaSystemPermission): Promise { + z.object({ opened: z.literal(true) }).parse(await this.control('openPermissionSettings', { permission })) + } + + /** + * Remove saved and session access and join affected operations. + * @param id Application identity displayed on the permission page. + * @returns Completion after persistence and cancellation; dispatched input cannot be undone. + */ + @Remote + revoke(id: CuaApplicationId): Promise { return this.policy.revoke(id) } + + /** + * Clear this application's refusals across sessions so each session may ask for access again. + * @param id Refused or revoked application displayed on the permission page. + * @returns Completion after clearing refusals; no permission is granted and no action is replayed. + */ + /* oxlint-disable typescript/require-await -- Remote callers receive policy failures as rejected promises. */ + @Remote + async retryApproval(id: CuaApplicationId): Promise { this.policy.retryApproval(id) } + /* oxlint-enable typescript/require-await */ +} + +/** + * Install human-facing controls while keeping executor capabilities private to the provider. + * @param ctx Provider context that owns decisions and operation lifetimes. + * @param options Native transport and profile persistence. + * @returns Consent selection, guarded dispatch, and task cleanup capabilities. + */ +export function installCuaAccess(ctx: Context, options: ControlOptions): Pick { + const policy = new ApplicationAccess(ctx, options) + const lifetime = new AbortController() + let client: NativeClient | undefined + let tail: Promise = Promise.resolve() + const control: Control = (method, params) => { + lifetime.signal.throwIfAborted() + const perform = async () => { + lifetime.signal.throwIfAborted() + if (client?.disposed) await client.dispose() + lifetime.signal.throwIfAborted() + if (!client || client.disposed) client = new NativeClient(ctx, options.native, options.createChannel) + return client.request(method, params, lifetime.signal) + } + const operation = tail.then(perform, perform) + tail = operation + return operation + } + const dispose = ctx.effect(() => async () => { + lifetime.abort() + await Promise.allSettled([policy.dispose(), tail]) + await client?.dispose() + }, 'cua-access.dispose') + new CuaAccess(ctx, policy, options.settingsNamespace, control) + return { select: policy.select.bind(policy), run: policy.run.bind(policy), forget: policy.forget.bind(policy), dispose } +} diff --git a/packages/experimental/cua/src/application-access.ts b/packages/experimental/cua/src/application-access.ts index 1412e0cc99..4b34223012 100644 --- a/packages/experimental/cua/src/application-access.ts +++ b/packages/experimental/cua/src/application-access.ts @@ -6,6 +6,7 @@ import { z } from 'zod' import { NativeError, type NativeClient } from './native-client.ts' import type { CuaAppHandle } from './types.ts' import type { CuaApplicationGrant, CuaApplicationId } from './application-types.ts' +import type { CuaApplicationState } from './access-types.ts' /** Native identity admission occurs before any application observation or input. */ export const ApplicationAuthorizationResult = z.object({ @@ -19,7 +20,7 @@ export interface AccessOptions { } interface Selection { identity: CuaApplicationGrant - scope: 'task' | 'persistent' + scope: 'session' | 'persistent' lifetime: AbortController } interface ClientSelections { @@ -37,9 +38,9 @@ interface ActiveOperation { /** Owns application consent and cancels operations when saved access is removed. */ export class ApplicationAccess { private readonly clients = new Map() - private readonly rejected = new Map>() + private readonly decisions = new Map>() private readonly active = new Set() - private readonly selections = new WeakMap>>() + private readonly selections = new Map>>() private readonly lifetime = new AbortController() private mutationTail: Promise = Promise.resolve() private disposal: Promise | undefined @@ -59,13 +60,66 @@ export class ApplicationAccess { this.lifetime.abort() await Promise.allSettled([this.mutationTail, ...[...this.active].map(operation => operation.done.promise)]) this.clients.clear() - this.rejected.clear() + this.decisions.clear() })() } + /** + * Project application decisions without requesting native permissions. + * @returns Current saved grants and refused applications for the permission page. + */ + status(): Omit { + const allowedApps = [...this.options.grants()] + const blockedApps = new Map() + for (const decisions of this.decisions.values()) for (const [id, decision] of decisions) { + if (decision.allowed || allowedApps.some(grant => grant.id === id)) continue + blockedApps.set(id, decision.identity) + } + return { allowedApps, + blockedApps: [...blockedApps.values()], persistentAvailable: this.options.save !== undefined } + } + + /** + * Remove saved and session consent and cancel affected native operations. + * @param id Application identity shown in the permission page. + * @returns Completion after persistence and affected operations settle; already dispatched input cannot be undone. + */ + async revoke(id: CuaApplicationId): Promise { + this.lifetime.signal.throwIfAborted() + let clients: readonly NativeClient[] = [] + await this.mutate(async () => { + const grants = this.options.grants() + if (grants.some(grant => grant.id === id)) { + const save = this.options.save + if (!save) throw new Error('Application permissions cannot be saved in this composition') + await save(grants.filter(grant => grant.id !== id)) + } + clients = this.invalidate(id) + }) + const results = await Promise.allSettled([ + ...clients.map(client => client.dispose()), + ...[...this.active].filter(operation => operation.identity.id === id).map(operation => operation.done.promise), + ]) + const errors: unknown[] = [] + for (const result of results) if (result.status === 'rejected') errors.push(result.reason) + if (errors.length) throw new AggregateError(errors, 'Application access was revoked, but helper cleanup failed') + } + + /** + * Clear explicit refusal; the next getApp still requires a new user decision. + * @param id Application identity selected by the human. + */ + retryApproval(id: CuaApplicationId): void { + this.lifetime.signal.throwIfAborted() + for (const decisions of this.decisions.values()) if (decisions.get(id)?.allowed === false) decisions.delete(id) + for (const state of this.clients.values()) for (const [handle, selected] of state.handles) { + if (selected.identity.id === id && selected.lifetime.signal.aborted) state.handles.delete(handle) + } + } + /** * Obtain consent for a freshly selected native handle. Free text never grants access. - * @param agent Exact live task requesting access. + * @param agent Live agent whose session owns the decision. * @param client Helper that issued the handle. * @param handle Helper handle for a running instance or an approved pending launch. * @param identity Native executable/signing identity. @@ -79,19 +133,21 @@ export class ApplicationAccess { const existing = this.clients.get(client)?.handles.get(handle) if (!client.disposed && existing && this.permitted(existing)) return const saved = identity.persistable && this.options.grants().some(grant => grant.id === identity.id) - let scope: Selection['scope'] = 'persistent' - if (!saved && this.rejected.get(agent)?.has(identity.id)) throw this.denied(identity) + const decision = this.decisions.get(agent.id)?.get(identity.id) + let scope: Selection['scope'] = saved ? 'persistent' : 'session' + if (!saved && decision?.allowed === false) throw this.denied(identity) if (client.disposed) throw new NativeError('NATIVE_CLOSED', 'Helper is closed; acquire fresh application handles') - if (!saved) { + if (!saved && !decision?.allowed) { const questions = this.ctx.get('userQuestions') if (!questions) throw new NativeError('APP_APPROVAL_UNAVAILABLE', 'No human answerer is available for application access') - const options = [{ label: 'Allow for this task' }, { label: 'Deny' }] + const options = [{ label: 'Allow for this session' }, { label: 'Deny' }] if (identity.persistable && this.options.save) options.push({ label: 'Always allow' }) let result: AskUserQuestionAnswer try { result = await questions.ask({ agent, signal, questions: [{ id: 'cua-app-access', question: `Allow Computer Use to launch if needed, read, capture, and control ${identity.name}?`, - detail: `Application: ${identity.executablePath}\nTask access applies to the selected application instance, including its launch if needed. macOS permissions and approval of sensitive actions remain separate. Saved access can be revoked by removing its allowedApps entry from this provider configuration. Task access ends with the task.`, options }] }) + detail: 'Session access lasts until DSH restarts or Computer Use is disabled.', + options }] }) } catch (error) { if (error instanceof UserQuestionError && error.code === 'ASK_CANCELLED') { this.invalidate(identity.id, agent) @@ -105,8 +161,9 @@ export class ApplicationAccess { signal.throwIfAborted() const answer = result.answers.find(answer => answer.id === 'cua-app-access') const chosen = answer && !answer.custom?.trim() && answer.selected.length === 1 ? answer.selected[0] : undefined - if (chosen === 'Allow for this task') scope = 'task' + if (chosen === 'Allow for this session') scope = 'session' else if (chosen === 'Always allow' && identity.persistable && this.options.save) { + scope = 'persistent' const { persistable: _persistable, ...grant } = identity await this.persist((grants) => { signal.throwIfAborted() @@ -129,7 +186,8 @@ export class ApplicationAccess { } state.handles.set(handle, { identity: grant, scope, lifetime: new AbortController() }) this.clients.set(client, state) - this.rejected.get(agent)?.delete(identity.id) + if (scope === 'session') this.decide(agent.id, grant, true) + else this.decisions.get(agent.id)?.delete(identity.id) })) } @@ -155,11 +213,14 @@ export class ApplicationAccess { } private async selectInOrder(agent: Agent, id: CuaApplicationId, select: () => Promise): Promise { - const selections = this.selections.get(agent) ?? new Map>() - this.selections.set(agent, selections) + const selections = this.selections.get(agent.id) ?? new Map>() + this.selections.set(agent.id, selections) const operation = (selections.get(id) ?? Promise.resolve()).then(select, select) selections.set(id, operation) - try { await operation } finally { if (selections.get(id) === operation) selections.delete(id) } + try { await operation } finally { + if (selections.get(id) === operation) selections.delete(id) + if (selections.size === 0) this.selections.delete(agent.id) + } } private async withOperation(agent: Agent, client: NativeClient, identity: CuaApplicationGrant, signal: AbortSignal, @@ -172,6 +233,9 @@ export class ApplicationAccess { const result = await dispatch(lifetime) lifetime.throwIfAborted() return result + } catch (error) { + if (operation.controller.signal.aborted) throw operation.controller.signal.reason + throw error } finally { this.active.delete(operation) operation.done.resolve() @@ -179,40 +243,44 @@ export class ApplicationAccess { } /** - * Forget the disposed task's transient decisions and selected handles. - * @param agent Disposed task whose transient decisions are released. + * Release the disposed agent's handles while retaining its session decisions. + * @param agent Disposed agent whose selected handles are released. */ forget(agent: Agent): void { for (const [client, state] of this.clients) if (state.agent === agent) this.clients.delete(client) - this.rejected.delete(agent) } private permitted(selection: Selection): boolean { return !selection.lifetime.signal.aborted - && (selection.scope === 'task' || this.options.grants().some(grant => grant.id === selection.identity.id)) + && (selection.scope === 'session' || this.options.grants().some(grant => grant.id === selection.identity.id)) } private denied(identity: CuaApplicationGrant): NativeError { return new NativeError('APP_ACCESS_DENIED', `The user denied access to ${identity.name}. Do not retry or use another tool to bypass this decision.`) } - private reject(agent: Agent, identity: CuaApplicationGrant): void { - const rejected = this.rejected.get(agent) ?? new Map() - rejected.set(identity.id, { id: identity.id, name: identity.name, executablePath: identity.executablePath }) - this.rejected.set(agent, rejected) + private decide(sessionId: Agent['id'], identity: CuaApplicationGrant, allowed: boolean): void { + const decisions = this.decisions.get(sessionId) ?? new Map() + decisions.set(identity.id, { identity: { id: identity.id, name: identity.name, executablePath: identity.executablePath }, allowed }) + this.decisions.set(sessionId, decisions) } - private invalidate(id: CuaApplicationId, agent?: Agent): void { + private invalidate(id: CuaApplicationId, agent?: Agent): NativeClient[] { const clients = new Set() + for (const [sessionId, decisions] of this.decisions) { + const decision = decisions.get(id) + if (decision && (agent === undefined || sessionId === agent.id)) this.decide(sessionId, decision.identity, false) + } for (const [client, state] of this.clients) for (const selection of state.handles.values()) if (selection.identity.id === id - && (agent === undefined || state.agent === agent)) { + && (agent === undefined || state.agent.id === agent.id)) { selection.lifetime.abort(new NativeError('APP_ACCESS_REVOKED', 'Application access was revoked; stop using this application')) - this.reject(state.agent, selection.identity) + this.decide(state.agent.id, selection.identity, false) clients.add(client) } - for (const operation of this.active) if (operation.identity.id === id && (agent === undefined || operation.agent === agent)) { - this.reject(operation.agent, operation.identity) + for (const operation of this.active) if (operation.identity.id === id && (agent === undefined || operation.agent.id === agent.id)) { + this.decide(operation.agent.id, operation.identity, false) operation.controller.abort(this.denied(operation.identity)) clients.add(operation.client) } for (const client of clients) client.invalidate() + return [...clients] } private reconcile(): void { for (const state of this.clients.values()) for (const selection of state.handles.values()) { diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index 131e15c3ec..6aa023962c 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -13,22 +13,28 @@ import type { ContentBlock } from '@deepseek-ai/dsh-llm' import type { JsonValue } from '@deepseek-ai/dsh-util-values' import { z } from 'zod' import { NativeClient, NativeError, type NativeClientConfig, type NativeChannelFactory } from './native-client.ts' -import { ApplicationAuthorizationResult, ApplicationAccess } from './application-access.ts' -import type { CuaApplicationGrant, CuaApplicationId } from './application-types.ts' +import { ApplicationAuthorizationResult, installCuaAccess } from './access.ts' +import type { CuaApplicationGrant, CuaApplicationId } from './access-types.ts' import type { CuaAppHandle, CuaOperation } from './types.ts' import { AppResult, NativeScreenshotResult, Request, ScreenshotResult, StateResult, WindowsResult } from './protocol.ts' import { CuaCoordinates } from './coordinates.ts' -import type {} from '@deepseek-ai/dsh-system-prompt' -import type { ToolExecution } from '@deepseek-ai/dsh-tools' +import { createScope, type Scope } from '@deepseek-ai/dsh-scope' +import type {} from '@deepseek-ai/dsh-agent-preset-registry' +import type { PromptSection } from '@deepseek-ai/dsh-system-prompt' +import type { ToolDefinition, ToolExecution } from '@deepseek-ai/dsh-tools' import type {} from '@deepseek-ai/dsh-subprocess' import type {} from '@deepseek-ai/dsh-computer-use' import type {} from '@deepseek-ai/dsh-ptc-runtime' import type {} from '@deepseek-ai/dsh-settings' import type {} from '@deepseek-ai/cordis-plugin-loader' +export { CuaAccess } from './access.ts' +export type * from './access-types.ts' /** Native executable and per-operation resource bounds. */ export interface CuaConfig { + /** Register tools for Agents using this preset; defaults to ptc. False enables global registration. */ + agentPreset: string | false /** Signed applications explicitly approved for future tasks in this profile. */ allowedApps: Volatile & { /** Unbranded digest at the profile configuration boundary; admitted by the Config schema. */ @@ -72,6 +78,7 @@ export interface CuaConfig { /** Validated host configuration; no driver is downloaded at runtime. */ export const CuaConfig = Schema.object({ + agentPreset: Schema.union([Schema.string().min(1), Schema.const(false)]).default('ptc'), allowedApps: Schema.array(Schema.object({ id: Schema.string().pattern(/^[a-f0-9]{64}$/).required(), name: Schema.string().required(), executablePath: Schema.string().min(1).required(), @@ -157,7 +164,9 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) } const entryId = ctx.fiber.entry?.options.id ctx.inject(['settings'], scope => scope.effect(() => scope.settings.configure({ auto: false }, ctx.fiber))) - const access = new ApplicationAccess(ctx, { + const access = installCuaAccess(ctx, { + settingsNamespace: entryId ?? null, + native, createChannel: backend.createChannel, grants: () => config.allowedApps.get().map(grant => ({ ...grant, id: grant.id as CuaApplicationId })), get save() { const settings = ctx.get('settings') @@ -256,8 +265,10 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) : await requestNative() if (method === 'getApp') { const prepared = AppResult.extend({ pid: AppResult.shape.pid.nullable() }).parse(result) - const { authorization } = z.object({ authorization: ApplicationAuthorizationResult }).parse(result) - await access.select(agent, client, prepared.appHandle, authorization, signal) + const { authorization, applicationIcon } = z.object({ authorization: ApplicationAuthorizationResult, + applicationIcon: ApplicationIcon.optional(), + }).parse(result) + await access.select(agent, client, prepared.appHandle, authorization, signal, applicationIcon) signal.throwIfAborted() const selected = prepared.pid === null ? AppResult.parse(await access.run(client, prepared.appHandle, signal, @@ -276,7 +287,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) if (method === 'listWindows') execution.coordinates.observe(parsedParams.appHandle as CuaAppHandle, WindowsResult.parse(result)) return result } - yield ctx.tools.register({ + const tool: ToolDefinition = { name: 'cua', description: backend.description, parameters: { ...z.record(z.string(), z.json()).parse(z.toJSONSchema(Request)), type: 'object' }, output: { schema: {}, render: renderResult }, @@ -319,14 +330,47 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) pending.delete(operation) } }, - }) - yield ctx.systemPrompt.section({ + } + const prompt: PromptSection = { name: 'computer-use:dsh-cua', order: ctx.systemPrompt.getSectionOrder('TOOL_COMPUTER_USE'), text: () => ctx.get('ptcRuntime')?.programFormat !== 'repl' ? backend.guidance : `${backend.guidance} Persistent PTC initialization: var { connectCua } = await import(${JSON.stringify(new URL('../lib/bridge.js', import.meta.url).href)}); var cua = connectCua(() => tools.cua, text => console.log(text));`, - }) + } + const register = (scope: Context): void => { + scope.tools.register(tool) + scope.systemPrompt.section(prompt) + } + if (config.agentPreset === false) register(ctx) + else { + const consumer = ctx.inject(['agentPresets'], async (presetCtx) => { + const scopes = new Map() + const sync = async (agent: Agent): Promise => { + const previous = scopes.get(agent) + if (presetCtx.agentPresets.composedPreset(agent.ctx) !== config.agentPreset) { + scopes.delete(agent) + await previous?.dispose() + return + } + if (previous !== undefined) return + const scope = createScope(agent.ctx, agent) + scopes.set(agent, scope) + scope.ctx.effect(() => () => { + if (scopes.get(agent) === scope) scopes.delete(agent) + }, 'cua.agent-tools') + register(scope.ctx) + } + presetCtx.effect(() => async () => { await Promise.all([...scopes.values()].map(scope => scope.dispose())) }, 'cua.agent-tools') + presetCtx.on('agent/created', async ({ agent }) => { await sync(agent) }) + presetCtx.on('agent-preset/selected', (id) => { + const agent = presetCtx.agents.get(id) + if (agent !== undefined) void sync(agent).catch((error: unknown) => { presetCtx.logger.warn(`CUA tool scope update failed: ${String(error)}`) }) + }) + await Promise.all(presetCtx.agents.list().map(sync)) + }) + yield () => consumer.dispose() + } }, 'dsh-cua.runtime') } diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index 46eddd2e1c..f732e1bf82 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -78,8 +78,13 @@ export const NativeResourceParameters = { }).strict(), findApps: z.object({ maxAppMatches: z.number().int().min(1).max(1000) }).strict(), } -/** Executor-only continuation after application consent; absent from the model Request union. */ -export const NativeOnlyOperations = [operation('launchApp', app)] as const +/** Host-only launch continuation and user-facing system permission controls; absent from model arguments. */ +export const NativeOnlyOperations = [ + operation('launchApp', app), operation('permissionStatus', {}), + operation('applicationIcons', { paths: z.array(z.string().min(1)) }), + operation('requestPermission', { permission: z.enum(['accessibility', 'screenRecording']) }), + operation('openPermissionSettings', { permission: z.enum(['accessibility', 'screenRecording']) }), +] as const /** Process identity returned only after a successful app selection. */ export const AppResult = z.object({ @@ -124,3 +129,6 @@ export const ActionResult = z.object({ dispatched: z.literal(true), verified: z. export const HoverResult = ActionResult.extend({ hoverRoute: z.enum(['accessibility', 'system-pointer']), systemPointerMoved: z.boolean() }) /** Background restoration dispatch and sampled foreground changes; null means the observation was unavailable. */ export const ShowResult = ActionResult.extend({ windowId: id.nullable(), foregroundChanged: z.boolean().nullable() }) + +/** Bounded native PNG thumbnail for human presentation, excluded from model results and saved grants. */ +export const ApplicationIcon = z.string().max(65536).regex(/^data:image\/png;base64,[A-Za-z0-9+/]+={0,2}$/) diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index bb465c5e36..a413854609 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -1,4 +1,5 @@ /** Native application objects independent of evaluator ownership and persistence. */ +export type * from './access-types.ts' import type { Branded } from '@deepseek-ai/dsh-brand' import type { JsonValue } from '@deepseek-ai/dsh-util-values' import type { z } from 'zod' diff --git a/packages/experimental/cua/tests/access.spec.ts b/packages/experimental/cua/tests/access.spec.ts new file mode 100644 index 0000000000..473ba6acaa --- /dev/null +++ b/packages/experimental/cua/tests/access.spec.ts @@ -0,0 +1,398 @@ +/** Application decisions are checked at dispatch and revocation cancels owned work. */ +import { setImmediate } from 'node:timers/promises' +import { Context } from '@deepseek-ai/cordis' +import { mountAgentLoopTestDependencies, mountAgentLoopTestHarness } from '@deepseek-ai/dsh-agent-loop-testkit' +import { SessionId } from '@deepseek-ai/dsh-session' +import UserQuestions, { type AskUserQuestionAnswer } from '@deepseek-ai/dsh-user-questions' +import { expect, it, onTestFinished, vi } from 'vitest' +import { ApplicationAuthorizationResult, installCuaAccess } from '../src/access.ts' +import type { CuaApplicationGrant } from '../src/access-types.ts' +import type { CuaAppHandle } from '../src/types.ts' +import { NativeClient } from '../src/native-client.ts' + +const native = vi.hoisted(() => ({ instances: [] as Array<{ + disposed: boolean + request: ReturnType + dispose: ReturnType Promise>> +}> })) + +vi.mock('../src/native-client.ts', async original => ({ + ...await original(), + NativeClient: class { + disposed = false + request = vi.fn(async (method: string) => method === 'openPermissionSettings' ? { opened: true } + : method === 'applicationIcons' ? {} : { accessibility: false, screenRecording: false, helper: { pid: 9 }, parent: { pid: 8 } }) + invalidate() { this.disposed = true } + dispose = vi.fn(async () => { this.disposed = true }) + constructor() { native.instances.push(this) } + }, +})) + +const unusedConnector = () => { throw new Error('Mocked native client does not connect') } + +const nativeConfig = { helperPath: '/fixture', cwd: '/', timeoutMs: 1000, connectTimeoutMs: 5000, screenshotTimeoutMs: 5000, axTimeoutMs: 100, + maxRequestBytes: 4096, maxResponseBytes: 4096, graceMs: 100, textEventIntervalMs: 1 } +const identity = ApplicationAuthorizationResult.parse({ id: 'a'.repeat(64), name: 'Test application', executablePath: '/Applications/Fixture.app/Contents/MacOS/Fixture', persistable: true }) +const handle = 'fixture-app' as CuaAppHandle +const signal = new AbortController().signal + +async function setup(choice = 'Allow for this session', options: { writable?: boolean; questions?: boolean; initial?: readonly CuaApplicationGrant[] } = {}) { + native.instances.length = 0 + const ctx = new Context() + onTestFinished(() => ctx.fiber.dispose()) + await mountAgentLoopTestDependencies(ctx) + await mountAgentLoopTestHarness(ctx) + if (options.questions !== false) await ctx.plugin(UserQuestions) + const owner = await ctx.agents.create({ sessionId: SessionId('cua-access-test') }) + const answer = vi.fn<() => Promise>(async () => ({ answers: [{ id: 'cua-app-access', selected: [choice] }] })) + ctx.on('user-questions/request', answer) + let grants: readonly CuaApplicationGrant[] = options.initial ?? [] + const save = vi.fn(async (next: readonly CuaApplicationGrant[]) => { grants = next }) + let writable = options.writable !== false + const access = installCuaAccess(ctx, { native: nativeConfig, createChannel: unusedConnector, settingsNamespace: 'cua', grants: () => grants, get save() { return writable ? save : undefined } }) + const client = new NativeClient(ctx, nativeConfig, unusedConnector) + return { ctx, owner, agent: owner.agent, answer, save, access, client, grants: () => grants, + setWritable(value: boolean) { writable = value }, + } +} + +it('requires another user decision after the permission page permits asking again', async () => { + const f = await setup('Deny') + await expect(f.access.select(f.agent, f.client, handle, identity, signal)).rejects.toThrow('APP_ACCESS_DENIED') + expect(f.ctx.cuaAccess.applications().blockedApps).toHaveLength(1) + await f.ctx.cuaAccess.retryApproval(identity.id) + expect(f.ctx.cuaAccess.applications().blockedApps).toEqual([]) + await expect(f.access.run(f.client, handle, signal, async () => null)).rejects.toThrow('APP_ACCESS_REQUIRED') + f.answer.mockResolvedValueOnce({ answers: [{ id: 'cua-app-access', selected: ['Allow for this session'] }] }) + const retryClient = new NativeClient(f.ctx, nativeConfig, unusedConnector) + await f.access.select(f.agent, retryClient, handle, identity, signal) + expect(f.answer).toHaveBeenCalledTimes(2) + await f.ctx.cuaAccess.revoke(identity.id) + await f.ctx.cuaAccess.retryApproval(identity.id) + await expect(f.access.run(f.client, handle, signal, async () => null)).rejects.toThrow('APP_ACCESS_REQUIRED') +}) + +it('persists a signed identity and revokes it before further dispatch', async () => { + const f = await setup('Always allow') + await f.access.select(f.agent, f.client, handle, identity, signal) + expect(f.grants()).toEqual([{ id: identity.id, name: identity.name, executablePath: identity.executablePath }]) + await f.ctx.cuaAccess.revoke(identity.id) + expect(f.grants()).toEqual([]) + await expect(f.access.run(f.client, handle, signal, async () => null)).rejects.toThrow('APP_ACCESS_REVOKED') + await expect(f.access.select(f.agent, f.client, handle, identity, signal)).rejects.toThrow('APP_ACCESS_DENIED') +}) + +it('joins an in-flight native operation when application access is revoked', async () => { + const f = await setup() + await f.access.select(f.agent, f.client, handle, identity, signal) + const entered = Promise.withResolvers() + const pending = f.access.run(f.client, handle, signal, async (operationSignal) => { + const done = Promise.withResolvers() + operationSignal.addEventListener('abort', () => { done.reject(new Error('native cancelled')) }, { once: true }) + entered.resolve(undefined) + return done.promise + }) + const rejected = expect(pending).rejects.toThrow('APP_ACCESS_DENIED') + await entered.promise + await f.ctx.cuaAccess.revoke(identity.id) + await rejected + await expect(f.access.run(f.client, handle, signal, async () => null)).rejects.toThrow('APP_ACCESS_REVOKED') +}) + +it.each(['revoke', 'dispose'] as const)('waits for native cancellation to finish during %s', async (action) => { + const f = await setup() + await f.access.select(f.agent, f.client, handle, identity, signal) + const entered = Promise.withResolvers(), cancelled = Promise.withResolvers() + const release = Promise.withResolvers() + onTestFinished(() => { release.resolve(undefined) }) + const pending = f.access.run(f.client, handle, signal, async (operationSignal) => { + operationSignal.addEventListener('abort', () => { cancelled.resolve(undefined) }, { once: true }) + entered.resolve(operationSignal) + await release.promise + }) + const rejected = expect(pending).rejects.toThrow() + const operationSignal = await entered.promise + let finished = false + const closing = (action === 'revoke' ? f.ctx.cuaAccess.revoke(identity.id) : f.access.dispose()) + .then(() => { finished = true }) + try { + await cancelled.promise + await setImmediate() + expect(operationSignal.aborted).toBe(true) + expect(finished).toBe(false) + release.resolve(undefined) + await closing + await rejected + expect(finished).toBe(true) + } finally { + release.resolve(undefined) + await Promise.all([closing, rejected]) + } +}) + +it('fails closed without a human answerer and never offers persistence without its owner', async () => { + const f = await setup('Always allow', { writable: false, questions: false }) + await expect(f.access.select(f.agent, f.client, handle, identity, signal)).rejects.toThrow('APP_APPROVAL_UNAVAILABLE') + await f.ctx.plugin(UserQuestions) + await expect(f.access.select(f.agent, f.client, handle, identity, signal)).rejects.toThrow('APP_ACCESS_DENIED') + expect(f.save).not.toHaveBeenCalled() + expect(f.ctx.cuaAccess.applications().persistentAvailable).toBe(false) +}) + +it('projects task decisions, observes live config removal, and forgets a disposed task', async () => { + const f = await setup('Always allow') + await f.access.select(f.agent, f.client, handle, identity, signal) + expect(f.ctx.cuaAccess.applications().allowedApps).toHaveLength(1) + await f.save([]) + f.ctx.emit('loader/volatile-update', [['allowedApps']]) + await expect(f.access.run(f.client, handle, signal, async () => null)).rejects.toThrow('APP_ACCESS_REVOKED') + await f.owner.dispose() + f.access.forget(f.agent) +}) + +it('refuses to promise revocation of a saved decision without persistence', async () => { + const { persistable: _persistable, ...grant } = identity + const f = await setup('Deny', { initial: [grant], writable: false }) + await expect(f.ctx.cuaAccess.revoke(identity.id)).rejects.toThrow('cannot be saved') + expect(f.grants()).toEqual([grant]) +}) + +it('checks native permissions without converting an OS request into a grant', async () => { + const f = await setup() + await f.access.select(f.agent, f.client, handle, identity, signal) + const before = f.ctx.cuaAccess.applications() + expect(before).toEqual({ settingsNamespace: 'cua', allowedApps: [], blockedApps: [], persistentAvailable: true }) + expect(await f.ctx.cuaAccess.systemStatus()).toMatchObject({ accessibility: false, screenRecording: false }) + expect(await f.ctx.cuaAccess.requestPermission('accessibility')).toMatchObject({ accessibility: false }) + await f.ctx.cuaAccess.openSettings('screenRecording') + const transport = native.instances[1]! + expect(transport.request).toHaveBeenCalledWith('openPermissionSettings', { permission: 'screenRecording' }, expect.any(AbortSignal)) + transport.request.mockRejectedValueOnce(new Error('transport ended')) + await expect(f.ctx.cuaAccess.systemStatus()).rejects.toThrow('transport ended') + transport.disposed = true + expect((await f.ctx.cuaAccess.systemStatus()).screenRecording).toBe(false) + expect(transport.dispose).toHaveBeenCalledOnce() + await f.access.dispose() + await expect(f.ctx.cuaAccess.systemStatus()).rejects.toThrow() + await expect(f.ctx.cuaAccess.revoke(identity.id)).rejects.toThrow() +}) + +it('does not open a replacement control connection after provider disposal begins', async () => { + const f = await setup() + await f.ctx.cuaAccess.systemStatus() + const control = native.instances.at(-1)! + control.disposed = true + const released = Promise.withResolvers(), entered = Promise.withResolvers() + onTestFinished(() => { released.resolve(undefined) }) + control.dispose.mockImplementationOnce(async () => { entered.resolve(undefined); await released.promise }) + const pending = f.ctx.cuaAccess.systemStatus() + const rejected = expect(pending).rejects.toThrow() + await entered.promise + const count = native.instances.length + const closing = f.access.dispose() + released.resolve(undefined) + await Promise.all([closing, rejected]) + expect(native.instances).toHaveLength(count) +}) + + +it('isolates other tasks and applications while revoking and forgetting one decision', async () => { + const f = await setup() + const owner = await f.ctx.agents.create({ sessionId: SessionId('another-cua-task') }) + const otherClient = new NativeClient(f.ctx, nativeConfig, unusedConnector) + const other = { ...identity, id: 'b'.repeat(64) as typeof identity.id } + const otherHandle = 'other-app' as CuaAppHandle + await f.access.select(f.agent, f.client, handle, identity, signal) + await f.access.select(f.agent, f.client, otherHandle, other, signal) + await f.access.select(owner.agent, otherClient, otherHandle, other, signal) + f.ctx.emit('loader/volatile-update', [['allowedApps']]) + await f.ctx.cuaAccess.retryApproval(identity.id) + const started: PromiseWithResolvers = Promise.withResolvers() + const release: PromiseWithResolvers = Promise.withResolvers() + onTestFinished(() => { release.resolve() }) + const continuing = f.access.run(otherClient, otherHandle, signal, async (operationSignal) => { + started.resolve() + await release.promise + return !operationSignal.aborted + }) + await started.promise + await f.ctx.cuaAccess.revoke(identity.id) + release.resolve() + expect(await continuing).toBe(true) + f.access.forget(f.agent) + await expect(f.access.run(otherClient, otherHandle, signal, async () => 'still allowed')).resolves.toBe('still allowed') + await owner.dispose() +}) + +it('removes a grant whose save is already pending when another task revokes the application', async () => { + const f = await setup() + await f.access.select(f.agent, f.client, handle, identity, signal) + const owner = await f.ctx.agents.create({ sessionId: SessionId('cua-concurrent-save') }) + const client = new NativeClient(f.ctx, nativeConfig, unusedConnector) + const entered: PromiseWithResolvers = Promise.withResolvers() + const release: PromiseWithResolvers = Promise.withResolvers() + onTestFinished(() => { release.resolve() }) + const save = f.save.getMockImplementation()! + f.save.mockImplementationOnce(async (grants) => { entered.resolve(); await release.promise; await save(grants) }) + f.answer.mockResolvedValueOnce({ answers: [{ id: 'cua-app-access', selected: ['Always allow'] }] }) + const selection = f.access.select(owner.agent, client, handle, identity, signal) + const settledSelection = Promise.allSettled([selection]) + await entered.promise + const revoked = f.ctx.cuaAccess.revoke(identity.id) + release.resolve() + await revoked + await settledSelection + expect(f.grants()).toEqual([]) + expect(f.save).toHaveBeenCalledTimes(2) + expect(client.disposed).toBe(true) + expect(f.client.disposed).toBe(true) + const replacement = new NativeClient(f.ctx, nativeConfig, unusedConnector) + await expect(f.access.select(owner.agent, replacement, handle, identity, signal)).rejects.toThrow('APP_ACCESS_DENIED') +}) + +it('keeps revoked helper handles invalid after the application is approved again', async () => { + const f = await setup() + await f.access.select(f.agent, f.client, handle, identity, signal) + await f.ctx.cuaAccess.revoke(identity.id) + expect(f.client.disposed).toBe(true) + await f.ctx.cuaAccess.retryApproval(identity.id) + await expect(f.access.select(f.agent, f.client, handle, identity, signal)).rejects.toThrow('NATIVE_CLOSED') + const replacement = new NativeClient(f.ctx, nativeConfig, unusedConnector) + const replacementHandle = 'replacement-app' as CuaAppHandle + await f.access.select(f.agent, replacement, replacementHandle, identity, signal) + const oldDispatch = vi.fn(async () => 'stale window') + await expect(f.access.run(f.client, handle, signal, oldDispatch)).rejects.toThrow('APP_ACCESS_REQUIRED') + expect(oldDispatch).not.toHaveBeenCalled() + await expect(f.access.run(replacement, replacementHandle, signal, async () => 'fresh window')).resolves.toBe('fresh window') +}) + +it('projects restored persistent grants without showing the same application as blocked', async () => { + const f = await setup('Deny') + await expect(f.access.select(f.agent, f.client, handle, identity, signal)).rejects.toThrow('APP_ACCESS_DENIED') + const { persistable: _persistable, ...grant } = identity + await f.save([grant]) + expect(f.ctx.cuaAccess.applications().blockedApps).toEqual([]) + const replacement = new NativeClient(f.ctx, nativeConfig, unusedConnector) + await f.access.select(f.agent, replacement, handle, identity, signal) + expect(f.ctx.cuaAccess.applications().allowedApps).toEqual([grant]) + expect(f.answer).toHaveBeenCalledOnce() +}) + +it('keeps access revoked when stopping the helper reports a cleanup failure', async () => { + const f = await setup('Always allow') + await f.access.select(f.agent, f.client, handle, identity, signal) + vi.spyOn(f.client, 'dispose').mockRejectedValueOnce(new Error('process range did not exit')) + await expect(f.ctx.cuaAccess.revoke(identity.id)).rejects.toThrow('helper cleanup failed') + expect(f.grants()).toEqual([]) + expect(f.client.disposed).toBe(true) + const dispatch = vi.fn(async () => 'unexpected input') + await expect(f.access.run(f.client, handle, signal, dispatch)).rejects.toThrow('APP_ACCESS_REVOKED') + expect(dispatch).not.toHaveBeenCalled() +}) + +it('rejects an answer for a helper that exits while the human question is pending', async () => { + const f = await setup() + const answer = Promise.withResolvers() + const entered: PromiseWithResolvers = Promise.withResolvers() + onTestFinished(() => { answer.resolve({ answers: [{ id: 'cua-app-access', selected: ['Deny'] }] }) }) + f.answer.mockImplementationOnce(() => { entered.resolve(); return answer.promise }) + const selected = f.access.select(f.agent, f.client, handle, identity, signal) + const rejected = expect(selected).rejects.toThrow('NATIVE_CLOSED') + await entered.promise + f.client.invalidate() + answer.resolve({ answers: [{ id: 'cua-app-access', selected: ['Allow for this session'] }] }) + await rejected + const replacement = new NativeClient(f.ctx, nativeConfig, unusedConnector) + await f.access.select(f.agent, replacement, handle, identity, signal) + expect(f.answer).toHaveBeenCalledTimes(2) +}) + +it('keeps other session grants after revocation while requiring fresh native handles', async () => { + const f = await setup() + const other = { ...identity, id: 'b'.repeat(64) as typeof identity.id } + const otherHandle = 'other-application' as CuaAppHandle + await f.access.select(f.agent, f.client, handle, identity, signal) + await f.access.select(f.agent, f.client, otherHandle, other, signal) + await f.ctx.cuaAccess.revoke(identity.id) + await expect(f.access.run(f.client, otherHandle, signal, async () => 'old handle')).rejects.toThrow('NATIVE_CLOSED') + const state = f.ctx.cuaAccess.applications() + expect(state.blockedApps.map(app => app.id)).toEqual([identity.id]) + const replacement = new NativeClient(f.ctx, nativeConfig, unusedConnector) + await f.access.select(f.agent, replacement, otherHandle, other, signal) + await expect(f.access.run(replacement, otherHandle, signal, async () => 'new handle')).resolves.toBe('new handle') + expect(f.answer).toHaveBeenCalledTimes(2) +}) + +it('does not let a stale revoked task invalidate another task after a saved grant is restored', async () => { + const f = await setup() + await f.access.select(f.agent, f.client, handle, identity, signal) + await f.ctx.cuaAccess.revoke(identity.id) + const { persistable: _persistable, ...grant } = identity + await f.save([grant]) + const owner = await f.ctx.agents.create({ sessionId: SessionId('cua-restored-grant') }) + const replacement = new NativeClient(f.ctx, nativeConfig, unusedConnector) + await f.access.select(owner.agent, replacement, handle, identity, signal) + await expect(f.access.run(f.client, handle, signal, async () => 'stale')).rejects.toThrow('APP_ACCESS_REVOKED') + f.ctx.emit('loader/volatile-update', [['allowedApps']]) + expect(replacement.disposed).toBe(false) + await expect(f.access.run(replacement, handle, signal, async () => 'restored')).resolves.toBe('restored') +}) + +it('reads application decisions without native work and retrieves icons separately', async () => { + const f = await setup('Always allow') + await f.access.select(f.agent, f.client, handle, identity, signal) + expect(f.ctx.cuaAccess.applications().allowedApps).toEqual(f.grants()) + expect(native.instances).toHaveLength(1) + await f.ctx.cuaAccess.systemStatus() + const control = native.instances.at(-1)! + const pending = Promise.withResolvers>() + onTestFinished(() => { pending.resolve({}) }) + control.request.mockReturnValueOnce(pending.promise) + const icons = f.ctx.cuaAccess.applicationIcons() + expect(f.ctx.cuaAccess.applications().allowedApps).toEqual(f.grants()) + pending.resolve({ [identity.executablePath]: 'data:image/png;base64,aWNvbg==' }) + expect(await icons).toEqual({ [identity.executablePath]: 'data:image/png;base64,aWNvbg==' }) + expect(f.grants()[0]).not.toHaveProperty('icon') +}) + +it('skips native icon work for an empty application list and propagates icon failures independently', async () => { + const f = await setup() + expect(await f.ctx.cuaAccess.applicationIcons()).toEqual({}) + expect(native.instances).toHaveLength(1) + await f.access.select(f.agent, f.client, handle, identity, signal) + expect(await f.ctx.cuaAccess.applicationIcons()).toEqual({}) + expect(native.instances).toHaveLength(1) + await f.save([{ id: identity.id, name: identity.name, executablePath: identity.executablePath }]) + await f.ctx.cuaAccess.systemStatus() + native.instances.at(-1)!.request.mockRejectedValueOnce(new Error('icons unavailable')) + await expect(f.ctx.cuaAccess.applicationIcons()).rejects.toThrow('icons unavailable') + expect(f.ctx.cuaAccess.applications().allowedApps).toHaveLength(1) + expect((await f.ctx.cuaAccess.systemStatus()).accessibility).toBe(false) +}) + +it('preserves captured launch metadata separately from the current helper parent', async () => { + const f = await setup() + await f.ctx.cuaAccess.systemStatus() + const control = native.instances.at(-1)! + const launchApplication = { name: 'Visual Studio Code', path: '/Applications/Visual Studio Code.app', icon: 'data:image/png;base64,aWNvbg==' } + control.request.mockResolvedValueOnce({ accessibility: false, screenRecording: false, + helper: { pid: 9 }, parent: { pid: 1, path: '/sbin/launchd' }, launchApplication }) + expect(await f.ctx.cuaAccess.systemStatus()).toMatchObject({ parent: { pid: 1 }, launchApplication }) +}) + + +it('omits session-only grants from the permission page and retains their access after another session saves a grant', async () => { + const f = await setup() + await f.access.select(f.agent, f.client, handle, identity, signal) + const other = { ...identity, id: 'b'.repeat(64) as typeof identity.id, name: 'Another application', executablePath: '/Applications/Other.app' } + await f.access.select(f.agent, f.client, 'other-application' as CuaAppHandle, other, signal) + expect(f.ctx.cuaAccess.applications()).toEqual({ settingsNamespace: 'cua', allowedApps: [], blockedApps: [], persistentAvailable: true }) + const owner = await f.ctx.agents.create({ sessionId: SessionId('cua-always-allow') }) + const client = new NativeClient(f.ctx, nativeConfig, unusedConnector) + f.answer.mockResolvedValueOnce({ answers: [{ id: 'cua-app-access', selected: ['Always allow'] }] }) + await f.access.select(owner.agent, client, handle, identity, signal) + const state = f.ctx.cuaAccess.applications() + expect(state.allowedApps.map(app => app.id)).toEqual([identity.id]) + expect(state.blockedApps).toEqual([]) + await expect(f.access.run(f.client, handle, signal, async () => 'still allowed')).resolves.toBe('still allowed') +}) diff --git a/packages/experimental/cua/tests/agent-scopes.spec.ts b/packages/experimental/cua/tests/agent-scopes.spec.ts new file mode 100644 index 0000000000..c561d11a6e --- /dev/null +++ b/packages/experimental/cua/tests/agent-scopes.spec.ts @@ -0,0 +1,95 @@ +/** CUA tool and prompt ownership follows live Agent presets and provider disposal. */ +import { resolve } from 'node:path' +import { Context } from '@deepseek-ai/cordis' +import Loader from '@deepseek-ai/cordis-plugin-loader' +import AgentPresets from '@deepseek-ai/dsh-agent-preset-registry' +import { mountAgentLoopTestDependencies, mountAgentLoopTestHarness } from '@deepseek-ai/dsh-agent-loop-testkit' +import ComputerUseRegistry from '@deepseek-ai/dsh-computer-use' +import Subprocess from '@deepseek-ai/dsh-subprocess' +import { SessionId } from '@deepseek-ai/dsh-session' +import { expect, it, onTestFinished, vi } from 'vitest' +import { CuaConfig, installCua } from '../src/index.ts' + +const provider = { + name: 'test-cua-scopes', + inject: ['computerUse', 'subprocess', 'tools', 'systemPrompt', 'agents'], + Config: CuaConfig, + apply(ctx: Context, config: CuaConfig) { + installCua(ctx, config, { + helperPath: resolve('unused-cua-helper'), + createChannel() { throw new Error('Scope tests must not start a native helper') }, + description: 'Fixture computer interaction', guidance: 'Fixture CUA guidance', + }) + }, +} + +async function setup() { + const ctx = new Context() + onTestFinished(() => ctx.fiber.dispose()) + ctx.baseUrl = import.meta.url + await ctx.plugin(Loader) + await mountAgentLoopTestDependencies(ctx) + await mountAgentLoopTestHarness(ctx) + await ctx.plugin(ComputerUseRegistry) + await ctx.plugin(Subprocess, []) + await ctx.plugin(AgentPresets, { default: 'standard' }) + for (const id of ['standard', 'ptc']) { + await ctx.plugin({ inject: ['agentPresets'], async* apply(owner: Context) { + yield await owner.agentPresets.register({ id, plugins: [] }) + } }) + } + const create = (id: string, preset: string) => ctx.agents.create({ + sessionId: SessionId(id), setup: async (owner) => { await ctx.agentPresets.mount(owner, preset) }, + }) + return { ctx, create } +} + +it('adds and removes CUA for existing and new PTC Agents without changing other presets', async () => { + const { ctx, create } = await setup() + const existing = await create('existing', 'ptc') + const standard = await create('standard', 'standard') + expect(ctx.tools.get('cua', existing.agent)).toBeUndefined() + const mounted = await ctx.plugin(provider, {}) + await expect.poll(() => ctx.tools.get('cua', existing.agent)).toBeDefined() + expect(ctx.tools.get('cua', standard.agent)).toBeUndefined() + expect((await ctx.systemPrompt.assemble({ scope: existing.agent })).sections) + .toContainEqual(expect.objectContaining({ name: 'computer-use:dsh-cua', text: 'Fixture CUA guidance' })) + await ctx.agentPresets.select(existing.agent, 'ptc') + expect(ctx.tools.schemas(existing.agent).filter(tool => tool.name === 'cua')).toHaveLength(1) + const fresh = await create('fresh', 'ptc') + expect(ctx.tools.get('cua', fresh.agent)).toBeDefined() + await ctx.agentPresets.select(existing.agent, 'standard') + await expect.poll(() => ctx.tools.get('cua', existing.agent)).toBeUndefined() + expect((await ctx.systemPrompt.assemble({ scope: existing.agent })).sections) + .not.toContainEqual(expect.objectContaining({ name: 'computer-use:dsh-cua' })) + await ctx.agentPresets.select(existing.agent, 'ptc') + await expect.poll(() => ctx.tools.get('cua', existing.agent)).toBeDefined() + await fresh.dispose() + expect(ctx.tools.get('cua', fresh.agent)).toBeUndefined() + await mounted.dispose() + expect(ctx.tools.get('cua', existing.agent)).toBeUndefined() + expect((await ctx.systemPrompt.assemble({ scope: existing.agent })).sections) + .not.toContainEqual(expect.objectContaining({ name: 'computer-use:dsh-cua' })) + const reloaded = await ctx.plugin(provider, {}) + await expect.poll(() => ctx.tools.get('cua', existing.agent)).toBeDefined() + expect(ctx.tools.get('cua', standard.agent)).toBeUndefined() + await reloaded.dispose() +}) + +it('ignores unloaded Session selections and reports a failed live scope update', async () => { + const { ctx, create } = await setup() + const handle = await create('active', 'ptc') + const mounted = await ctx.plugin(provider, {}) + await expect.poll(() => ctx.tools.get('cua', handle.agent)).toBeDefined() + const warn = vi.spyOn(mounted.ctx.logger, 'warn').mockImplementation(() => {}) + const lookup = vi.spyOn(ctx.agentPresets, 'composedPreset') + onTestFinished(() => { lookup.mockRestore(); warn.mockRestore() }) + ctx.emit('agent-preset/selected', SessionId('unloaded'), 'ptc') + expect(lookup).not.toHaveBeenCalled() + lookup.mockImplementationOnce(() => { throw new Error('Preset lookup failed during reload') }) + ctx.emit('agent-preset/selected', handle.agent.id, 'ptc') + await expect.poll(() => warn.mock.calls).toEqual([['CUA tool scope update failed: Error: Preset lookup failed during reload']]) + expect(ctx.tools.get('cua', handle.agent)).toBeDefined() + await ctx.agentPresets.select(handle.agent, 'standard') + await expect.poll(() => ctx.tools.get('cua', handle.agent)).toBeUndefined() +}) diff --git a/packages/experimental/cua/tests/application-access.spec.ts b/packages/experimental/cua/tests/application-access.spec.ts index fc77988a11..9e0eac7822 100644 --- a/packages/experimental/cua/tests/application-access.spec.ts +++ b/packages/experimental/cua/tests/application-access.spec.ts @@ -36,7 +36,7 @@ const identity = ApplicationAuthorizationResult.parse({ id: 'a'.repeat(64), name const handle = 'fixture-app' as CuaAppHandle const signal = new AbortController().signal -async function setup(choice = 'Allow for this task', options: { writable?: boolean; questions?: boolean; initial?: readonly CuaApplicationGrant[] } = {}) { +async function setup(choice = 'Allow for this session', options: { writable?: boolean; questions?: boolean; initial?: readonly CuaApplicationGrant[] } = {}) { native.instances.length = 0 const ctx = new Context() onTestFinished(() => ctx.fiber.dispose()) @@ -63,7 +63,7 @@ it('refuses raw handle dispatch until getApp obtains consent', async () => { expect(dispatch).not.toHaveBeenCalled() }) -it('retains task consent for the exact helper handle without saving it', async () => { +it('reuses session consent across application handles and connections without saving it', async () => { const f = await setup() await f.access.select(f.agent, f.client, handle, identity, signal) await expect(f.access.run(f.client, handle, signal, async () => 'read')).resolves.toBe('read') @@ -72,6 +72,11 @@ it('retains task consent for the exact helper handle without saving it', async ( expect(f.save).not.toHaveBeenCalled() const replacement = new NativeClient(f.ctx, nativeConfig, unusedConnector) await expect(f.access.run(replacement, handle, signal, async () => null)).rejects.toThrow('APP_ACCESS_REQUIRED') + await f.access.select(f.agent, replacement, 'fresh-app' as CuaAppHandle, identity, signal) + await f.access.select(f.agent, replacement, 'restarted-app' as CuaAppHandle, identity, signal) + expect(f.answer).toHaveBeenCalledOnce() + await expect(f.access.run(f.client, handle, signal, async () => null)).rejects.toThrow('APP_ACCESS_REQUIRED') + await expect(f.access.run(replacement, 'restarted-app' as CuaAppHandle, signal, async () => 'read')).resolves.toBe('read') }) it('does not repeatedly ask after an application is denied', async () => { @@ -120,10 +125,10 @@ it('reuses only saved signed identities when a task or helper is replaced', asyn it.each([ { answers: [] }, - { answers: [{ id: 'another-question', selected: ['Allow for this task'] }] }, - { answers: [{ id: 'cua-app-access', selected: ['Allow for this task', 'Deny'] }] }, + { answers: [{ id: 'another-question', selected: ['Allow for this session'] }] }, + { answers: [{ id: 'cua-app-access', selected: ['Allow for this session', 'Deny'] }] }, { answers: [{ id: 'cua-app-access', selected: [], custom: 'Always allow' }] }, - { answers: [{ id: 'cua-app-access', selected: ['Allow for this task'], custom: 'different scope' }] }, + { answers: [{ id: 'cua-app-access', selected: ['Allow for this session'], custom: 'different scope' }] }, ])('does not interpret an incomplete or free-text answer as consent: %j', async (answer) => { const f = await setup() f.answer.mockResolvedValueOnce(answer) @@ -199,8 +204,8 @@ it('reuses one accepted question for duplicate selections', async () => { f.answer.mockImplementationOnce(() => { entered.resolve(); return answer.promise }) const first = f.access.select(f.agent, f.client, handle, identity, signal) await entered.promise - const second = f.access.select(f.agent, f.client, handle, identity, signal) - answer.resolve({ answers: [{ id: 'cua-app-access', selected: ['Allow for this task'] }] }) + const second = f.access.select(f.agent, f.client, 'another-handle' as CuaAppHandle, identity, signal) + answer.resolve({ answers: [{ id: 'cua-app-access', selected: ['Allow for this session'] }] }) await Promise.all([first, second]) expect(f.answer).toHaveBeenCalledOnce() await expect(f.access.run(f.client, handle, signal, async () => 'allowed')).resolves.toBe('allowed') @@ -244,7 +249,7 @@ it('cancels native work when saved consent is removed from configuration', async entered.resolve(undefined) return done.promise }) - const rejected = expect(pending).rejects.toThrow('native cancelled') + const rejected = expect(pending).rejects.toThrow('APP_ACCESS_DENIED') await entered.promise await f.save([]) f.ctx.emit('loader/volatile-update', [['allowedApps']]) @@ -279,7 +284,7 @@ it('rejects an answer for a helper that exits while the human question is pendin const rejected = expect(selected).rejects.toThrow('NATIVE_CLOSED') await entered.promise f.client.invalidate() - answer.resolve({ answers: [{ id: 'cua-app-access', selected: ['Allow for this task'] }] }) + answer.resolve({ answers: [{ id: 'cua-app-access', selected: ['Allow for this session'] }] }) await rejected }) @@ -317,3 +322,57 @@ it('keeps an active task operation running when a different task denies the same await expect(pending).resolves.toBeNull() } finally { release.resolve(null); await pending } }) + + +it.each(['Allow for this session', 'Deny'])('retains %s when the same session gets a new agent', async (choice) => { + const f = await setup(choice) + const select = f.access.select(f.agent, f.client, handle, identity, signal) + if (choice === 'Deny') await expect(select).rejects.toThrow('APP_ACCESS_DENIED') + else await select + await f.owner.dispose() + f.access.forget(f.agent) + const owner = await f.ctx.agents.create({ sessionId: f.agent.id }) + const client = new NativeClient(f.ctx, nativeConfig, unusedConnector) + const resumed = f.access.select(owner.agent, client, handle, identity, signal) + if (choice === 'Deny') await expect(resumed).rejects.toThrow('APP_ACCESS_DENIED') + else { + await resumed + await expect(f.access.run(client, handle, signal, async () => 'resumed')).resolves.toBe('resumed') + } + expect(f.answer).toHaveBeenCalledOnce() +}) + +it('asks separately for another session and a changed application identity', async () => { + const f = await setup() + await f.access.select(f.agent, f.client, handle, identity, signal) + const owner = await f.ctx.agents.create({ sessionId: SessionId('separate-session') }) + const client = new NativeClient(f.ctx, nativeConfig, unusedConnector) + await f.access.select(owner.agent, client, handle, identity, signal) + expect(f.answer).toHaveBeenCalledTimes(2) + await f.access.select(f.agent, f.client, 'changed-app' as CuaAppHandle, + { ...identity, id: 'b'.repeat(64) as typeof identity.id }, signal) + expect(f.answer).toHaveBeenCalledTimes(3) +}) + +it('revokes session consent even after its agent and connection are released', async () => { + const f = await setup() + await f.access.select(f.agent, f.client, handle, identity, signal) + f.access.forget(f.agent) + await f.access.revoke(identity.id) + const replacement = new NativeClient(f.ctx, nativeConfig, unusedConnector) + await expect(f.access.select(f.agent, replacement, handle, identity, signal)).rejects.toThrow('APP_ACCESS_DENIED') + expect(f.answer).toHaveBeenCalledOnce() + f.access.retryApproval(identity.id) + await f.access.select(f.agent, replacement, handle, identity, signal) + expect(f.answer).toHaveBeenCalledTimes(2) +}) + +it('clears session consent when the provider is disabled and recreated', async () => { + const f = await setup() + await f.access.select(f.agent, f.client, handle, identity, signal) + await f.access.dispose() + const access = new ApplicationAccess(f.ctx, { grants: f.grants, save: f.save }) + const client = new NativeClient(f.ctx, nativeConfig, unusedConnector) + await access.select(f.agent, client, handle, identity, signal) + expect(f.answer).toHaveBeenCalledTimes(2) +}) diff --git a/packages/experimental/cua/tsconfig.json b/packages/experimental/cua/tsconfig.json index a73c2cfb2e..cb086d3e6f 100644 --- a/packages/experimental/cua/tsconfig.json +++ b/packages/experimental/cua/tsconfig.json @@ -41,6 +41,9 @@ { "path": "../../ptc-runtime/ptc-runtime" }, + { + "path": "../../typert/protocol" + }, { "path": "../../settings/settings" }, @@ -55,6 +58,12 @@ }, { "path": "../../llm/llm" + }, + { + "path": "../../core/scope" + }, + { + "path": "../../preset/agent-preset-registry" } ] } diff --git a/packages/experimental/cua/tsdown.config.ts b/packages/experimental/cua/tsdown.config.ts index ccc53ca43d..f0d407249f 100644 --- a/packages/experimental/cua/tsdown.config.ts +++ b/packages/experimental/cua/tsdown.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from 'tsdown' export default defineConfig(Object.entries({ - index: 'lib/types/index.js', api: 'lib/types/api.js', bridge: 'lib/types/bridge.js', + index: 'lib/types/index.js', types: 'lib/types/types.js', api: 'lib/types/api.js', bridge: 'lib/types/bridge.js', 'native-client': 'lib/types/native-client.js', protocol: 'lib/types/protocol.js', }).map(([name, path]) => ({ entry: { [name]: path }, diff --git a/packages/extensions/cordis-client-runner/src/client/slot-catalog.ts b/packages/extensions/cordis-client-runner/src/client/slot-catalog.ts index 50bcbe529a..ba5123c9b4 100644 --- a/packages/extensions/cordis-client-runner/src/client/slot-catalog.ts +++ b/packages/extensions/cordis-client-runner/src/client/slot-catalog.ts @@ -2281,11 +2281,12 @@ export const CLIENT_SLOT_API: readonly ClientSlotEntry[] = [ 'useSessionAccess: (sessionId: SessionId, action: RemoteEndpoint) => boolean', 'useWorkspaceAccess: (workspaceId: WorkspaceId, action: RemoteEndpoint) => boolean', ], - keyDomain: 'open: any string the owner dispatches (no compile-time key set), already taken: @deepseek-ai/dsh-experimental-voice-input-bundle', + keyDomain: 'open: any string the owner dispatches (no compile-time key set), already taken: @deepseek-ai/dsh-cua-bundle, @deepseek-ai/dsh-experimental-voice-input-bundle', hookContext: '', slotInject: '', declaredBy: 'an entry in \'main\' (client-ui-plugin-manager), so it exists while that entry is mounted', occupants: [ + 'client-ui-cua PermissionSetup key \'@deepseek-ai/dsh-cua-bundle\'', 'experimental-client-ui-voice-input VoiceSetupPrompt key \'@deepseek-ai/dsh-experimental-voice-input-bundle\'', ], replaceRisk: 'shadows-shipped-ui', @@ -2323,12 +2324,13 @@ export const CLIENT_SLOT_API: readonly ClientSlotEntry[] = [ 'useSessionAccess: (sessionId: SessionId, action: RemoteEndpoint) => boolean', 'useWorkspaceAccess: (workspaceId: WorkspaceId, action: RemoteEndpoint) => boolean', ], - keyDomain: 'open: any string the owner dispatches (no compile-time key set), already taken: @deepseek-ai/dsh-experimental-cot-translation-bundle, @deepseek-ai/dsh-experimental-voice-input-bundle', + keyDomain: 'open: any string the owner dispatches (no compile-time key set), already taken: @deepseek-ai/dsh-cua-bundle, @deepseek-ai/dsh-experimental-cot-translation-bundle, @deepseek-ai/dsh-experimental-voice-input-bundle', hookContext: '', slotInject: '', declaredBy: 'an entry in \'main\' (client-ui-plugin-manager), so it exists while that entry is mounted', occupants: [ 'experimental-client-ui-cot-translation TranslationSettings key \'@deepseek-ai/dsh-experimental-cot-translation-bundle\'', + 'client-ui-cua PermissionPanel key \'@deepseek-ai/dsh-cua-bundle\'', 'experimental-client-ui-voice-input VoicePreparation key \'@deepseek-ai/dsh-experimental-voice-input-bundle\'', ], replaceRisk: 'shadows-shipped-ui', @@ -3399,6 +3401,7 @@ export const CLIENT_SLOT_API: readonly ClientSlotEntry[] = [ 'client-ui-workspace SessionRenameDialog id \'workspace.session-rename\'', 'client-ui-workspace SessionArchiveConfirmDialog id \'workspace.session-archive\'', 'client-ui-workspace RowActionToast id \'workspace.row-toast\'', + 'client-ui-cua PermissionToast id \'cua-permission-outcome\'', ], replaceRisk: 'none', example: 'return {\n inject: [\'slots\'],\n apply(ctx) {\n ctx.slots.inject(\'shell.overlay\', () => ctx.slots.register(\n { name: \'shell.overlay\', id: \'my-entry\', order: 100, label: \'My entry\' },\n () => React.createElement(\'div\', null, \'hello\'),\n ))\n },\n}', diff --git a/packages/extensions/tool-cordis/src/api-catalog.ts b/packages/extensions/tool-cordis/src/api-catalog.ts index cca1649f96..51fb6586dc 100644 --- a/packages/extensions/tool-cordis/src/api-catalog.ts +++ b/packages/extensions/tool-cordis/src/api-catalog.ts @@ -910,6 +910,55 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ }, ], }, + { + key: 'cuaAccess', + summary: 'Browser controls for provider-owned native permission and application decisions.', + description: 'Browser controls for provider-owned native permission and application decisions.', + methods: [ + { + signature: '@Remote applications(): CuaApplicationState', + description: 'Read application decisions without waiting for the native helper.', + parameters: [], + returns: 'Saved grants and refused applications with their persistence owner.', + }, + { + signature: '@Remote async systemStatus(): Promise', + description: 'Inspect system privileges independently of application decisions and icons.', + parameters: [], + returns: 'Fresh native preflight and launch metadata.', + }, + { + signature: '@Remote async applicationIcons(): Promise>', + description: 'Load presentation icons separately from application decisions.', + parameters: [], + returns: 'Optional native PNG icons keyed by executable path for saved and blocked applications.', + }, + { + signature: '@Remote async requestPermission(permission: CuaSystemPermission): Promise', + description: 'Request a system privilege explicitly selected by the user.', + parameters: [{ name: 'permission', description: 'macOS privilege to request.' }], + returns: 'Preflight state after requesting; a false value never implies consent.', + }, + { + signature: '@Remote async openSettings(permission: CuaSystemPermission): Promise', + description: 'Open the macOS privacy pane selected by the user.', + parameters: [{ name: 'permission', description: 'Corresponding system privilege.' }], + returns: 'Completion after macOS accepted the open request.', + }, + { + signature: '@Remote revoke(id: CuaApplicationId): Promise', + description: 'Remove saved and session access and join affected operations.', + parameters: [{ name: 'id', description: 'Application identity displayed on the permission page.' }], + returns: 'Completion after persistence and cancellation; dispatched input cannot be undone.', + }, + { + signature: '@Remote async retryApproval(id: CuaApplicationId): Promise', + description: 'Clear this application\'s refusals across sessions so each session may ask for access again.', + parameters: [{ name: 'id', description: 'Refused or revoked application displayed on the permission page.' }], + returns: 'Completion after clearing refusals; no permission is granted and no action is replayed.', + }, + ], + }, { key: 'deepseekAccount', summary: 'Account operations; only Host consumers can obtain a request credential.', @@ -5340,6 +5389,30 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'CronScheduleRecord', declaration: 'export interface CronScheduleRecord {\n readonly id: ScheduleId;\n readonly kind: \'cron\';\n readonly title: string;\n readonly prompt: string;\n readonly expression: string;\n readonly timeZone: string;\n readonly scheduledAt: string;\n}', }, + { + name: 'CuaApplicationGrant', + declaration: 'export interface CuaApplicationGrant {\n id: CuaApplicationId;\n name: string;\n executablePath: string;\n}', + }, + { + name: 'CuaApplicationId', + declaration: 'export type CuaApplicationId = Branded<\'CuaApplicationId\'>;', + }, + { + name: 'CuaApplicationState', + declaration: 'export interface CuaApplicationState {\n settingsNamespace: string | null;\n allowedApps: readonly CuaApplicationGrant[];\n blockedApps: readonly CuaApplicationGrant[];\n persistentAvailable: boolean;\n}', + }, + { + name: 'CuaPermissionProcess', + declaration: 'export interface CuaPermissionProcess {\n pid: number;\n path?: string | undefined;\n name?: string | undefined;\n}', + }, + { + name: 'CuaSystemPermission', + declaration: 'export type CuaSystemPermission = \'accessibility\' | \'screenRecording\';', + }, + { + name: 'CuaSystemPermissionState', + declaration: 'export interface CuaSystemPermissionState {\n accessibility: boolean;\n screenRecording: boolean;\n helper: CuaPermissionProcess;\n parent: CuaPermissionProcess;\n launchApplication?: {\n name: string;\n path: string;\n icon?: string | undefined;\n } | undefined;\n}', + }, { name: 'DailyInput', declaration: 'export interface DailyInput {\n readonly time: string;\n readonly time_zone: string;\n}', diff --git a/packages/preset/agent-preset/skills/cordis-composition-reference/references/packages.md b/packages/preset/agent-preset/skills/cordis-composition-reference/references/packages.md index cd38b17726..51e9d4b72d 100644 --- a/packages/preset/agent-preset/skills/cordis-composition-reference/references/packages.md +++ b/packages/preset/agent-preset/skills/cordis-composition-reference/references/packages.md @@ -184,6 +184,7 @@ Every package below exports a Cordis plugin that a bundle patch can name in a Lo | Package | Config | Description | |---|---|---| +| `@deepseek-ai/dsh-client-ui-cua` | no | Computer Use permission setup and application access controls | | `@deepseek-ai/dsh-experimental-agent-team` | yes | Implicit-root Agent Teams roster, direct peer messaging, and shared task DAG | | `@deepseek-ai/dsh-experimental-api-speech-to-text` | yes | Authenticated experimental speech transcription for browser clients | | `@deepseek-ai/dsh-experimental-auto-review` | no | Per-tool LLM authorization review for the DeepSeek Harness Auto permission preset | diff --git a/packages/session/session-persistence-jsonl/tests/retired-content-admission.spec.ts b/packages/session/session-persistence-jsonl/tests/retired-content-admission.spec.ts index 1d10cb2fde..ddf49e64e4 100644 --- a/packages/session/session-persistence-jsonl/tests/retired-content-admission.spec.ts +++ b/packages/session/session-persistence-jsonl/tests/retired-content-admission.spec.ts @@ -1,13 +1,13 @@ /** Interpreted retired content must fail before migration publication or recoverable native-tail suppression. */ import { Context } from '@deepseek-ai/cordis' -import { SessionId } from '@deepseek-ai/dsh-session' +import { SESSION_FORMAT_VERSION, SessionId } from '@deepseek-ai/dsh-session' import { SessionFormatUnsupportedError, SessionPersistenceCorruptionError } from '@deepseek-ai/dsh-session-persistence' import * as JsonlSessionPersistence from '@deepseek-ai/dsh-session-persistence-jsonl' import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { basename, dirname, join } from 'node:path' import { describe, expect, it, onTestFinished } from 'vitest' -import { generationLogPath } from '../src/format.ts' +import { generationLogPath, SessionLogScanner } from '../src/format.ts' import { compressZstdFrame } from '../src/zstd.ts' const id = SessionId('retired-content-admission') @@ -55,3 +55,12 @@ describe.each(modes)('retired content in JSONL ($compression, $access)', ({ comp expect(await readFile(path)).toEqual(bytes) }) }) + +it.each(['strict', 'recoverable'] as const)('classifies retired native PTC tags as unsupported during %s scanning', (recovery) => { + const header = { type: 'session', version: SESSION_FORMAT_VERSION, id, createdAt: 1, isSeeded: false, delegationDepth: 0 } + for (const type of ['tool/code-dispatch-start', 'tool/code-dispatch']) { + const scanner = new SessionLogScanner(Buffer.from(JSON.stringify(header) + '\n'), recovery) + const row = { type, seq: 0, time: 1, data: {} } + expect(() => { scanner.write(Buffer.from(JSON.stringify(row) + '\n')) }).toThrow(SessionFormatUnsupportedError) + } +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index da43e12301..37e833d594 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -36,6 +36,9 @@ importers: '@deepseek-ai/dsh-cua': specifier: workspace:* version: link:packages/experimental/cua + '@deepseek-ai/dsh-cua-bundle': + specifier: workspace:* + version: link:packages/experimental/cua-bundle '@deepseek-ai/dsh-experimental-browser-use-chrome-devtools-mcp': specifier: workspace:* version: link:packages/experimental/browser-use-chrome-devtools-mcp @@ -306,6 +309,9 @@ importers: '@deepseek-ai/dsh-cordis-client-runner': specifier: workspace:* version: link:../../packages/extensions/cordis-client-runner + '@deepseek-ai/dsh-cua-bundle': + specifier: workspace:* + version: link:../../packages/experimental/cua-bundle '@deepseek-ai/dsh-experimental-agent-team-profile': specifier: workspace:* version: link:../../packages/experimental/agent-team-profile @@ -8295,6 +8301,60 @@ importers: specifier: ^18.2.0 version: 18.3.1(react@18.3.1) + packages/experimental/client-ui-cua: + devDependencies: + '@deepseek-ai/cordis': + specifier: workspace:~ + version: link:../../../vendor/cordis + '@deepseek-ai/dsh-api-gateway': + specifier: workspace:* + version: link:../../api/gateway + '@deepseek-ai/dsh-api-remotes': + specifier: workspace:* + version: link:../../api/remotes + '@deepseek-ai/dsh-client-locale': + specifier: workspace:* + version: link:../../client/locale + '@deepseek-ai/dsh-client-store': + specifier: workspace:* + version: link:../../client/store + '@deepseek-ai/dsh-client-test-runtime': + specifier: workspace:* + version: link:../../test-support/client-runtime + '@deepseek-ai/dsh-client-ui-layout': + specifier: workspace:* + version: link:../../client/ui-layout + '@deepseek-ai/dsh-client-ui-plugin-manager': + specifier: workspace:* + version: link:../../client/ui-plugin-manager + '@deepseek-ai/dsh-client-ui-primitives': + specifier: workspace:* + version: link:../../client/ui-primitives + '@deepseek-ai/dsh-client-ui-renderer': + specifier: workspace:* + version: link:../../client/ui-renderer + '@deepseek-ai/dsh-client-ui-settings': + specifier: workspace:* + version: link:../../client/ui-settings + '@deepseek-ai/dsh-client-ui-slots': + specifier: workspace:* + version: link:../../client/ui-slots + '@deepseek-ai/dsh-cua': + specifier: workspace:* + version: link:../cua + '@deepseek-ai/dsh-typert-protocol': + specifier: workspace:* + version: link:../../typert/protocol + '@testing-library/react': + specifier: ^16.1.0 + version: 16.3.3(@testing-library/dom@10.4.2)(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + react: + specifier: ^18.2.0 + version: 18.3.1 + react-dom: + specifier: ^18.2.0 + version: 18.3.1(react@18.3.1) + packages/experimental/client-ui-voice-input: devDependencies: '@deepseek-ai/cordis': @@ -8489,6 +8549,9 @@ importers: '@deepseek-ai/dsh-brand': specifier: workspace:* version: link:../../util/brand + '@deepseek-ai/dsh-typert-protocol': + specifier: workspace:* + version: link:../../typert/protocol '@deepseek-ai/dsh-util-values': specifier: workspace:* version: link:../../util/values @@ -8520,6 +8583,9 @@ importers: '@deepseek-ai/dsh-agent-loop-testkit': specifier: workspace:* version: link:../../test-support/agent-loop-testkit + '@deepseek-ai/dsh-agent-preset-registry': + specifier: workspace:* + version: link:../../preset/agent-preset-registry '@deepseek-ai/dsh-app-boot': specifier: workspace:* version: link:../../boot/app-boot @@ -8559,6 +8625,9 @@ importers: '@deepseek-ai/dsh-sandbox-policy': specifier: workspace:* version: link:../../sandbox/sandbox-policy + '@deepseek-ai/dsh-scope': + specifier: workspace:* + version: link:../../core/scope '@deepseek-ai/dsh-session': specifier: workspace:* version: link:../../core/session @@ -8590,6 +8659,31 @@ importers: specifier: ^0.35.3 version: 0.35.5(@types/node@22.20.1) + packages/experimental/cua-bundle: + dependencies: + '@deepseek-ai/dsh-client-ui-cua': + specifier: workspace:* + version: link:../client-ui-cua + '@deepseek-ai/dsh-computer-use': + specifier: workspace:* + version: link:../../computer-use/computer-use + '@deepseek-ai/dsh-experimental-cua-macos': + specifier: workspace:* + version: link:../cua-macos + '@deepseek-ai/dsh-ptc-runtime-node': + specifier: workspace:* + version: link:../../ptc-runtime/ptc-runtime-node + devDependencies: + '@deepseek-ai/cordis': + specifier: workspace:~ + version: link:../../../vendor/cordis + '@deepseek-ai/cordis-plugin-include': + specifier: workspace:~ + version: link:../../../vendor/include + js-yaml: + specifier: ^4.2.0 + version: 4.3.2 + packages/experimental/cua-macos: dependencies: '@deepseek-ai/dsh-atomic-write': diff --git a/scripts/check-workspace-constraints.ts b/scripts/check-workspace-constraints.ts index c02b46ed3b..f346e4e9f6 100644 --- a/scripts/check-workspace-constraints.ts +++ b/scripts/check-workspace-constraints.ts @@ -181,7 +181,7 @@ const packageFileExtras: Readonly> = { // The embedded OAR implementation retains its upstream license. '@deepseek-ai/dsh-experimental-subagent-oar': ['LICENSE-OAR'], // CUA publishes its client API and attributed native source for local macOS builds. - '@deepseek-ai/dsh-cua': ['lib/api.js', 'lib/bridge.js', 'lib/native-client.js', 'lib/protocol.js'], + '@deepseek-ai/dsh-cua': ['lib/types.js', 'lib/api.js', 'lib/bridge.js', 'lib/native-client.js', 'lib/protocol.js'], '@deepseek-ai/dsh-experimental-cua-macos': ['native/bin', 'native/Package.swift', 'native/Sources', 'native/tests/swift', 'native/tests/test_swift.py', 'native/LICENSE.cua-driver', 'native/upstream.json', 'native/README.md', 'native/README.zh.md', 'native/protocol.schema.json'], '@deepseek-ai/dsh-webhook-github': ['examples/github-review/cordis.yml', 'examples/github-review/github-ready-review-rule.mjs'], // Owned Worker bundles import this public bootstrap before their business entry. diff --git a/scripts/experimental-package-policy.ts b/scripts/experimental-package-policy.ts index 466acaed2a..68c63fcd7e 100644 --- a/scripts/experimental-package-policy.ts +++ b/scripts/experimental-package-policy.ts @@ -3,6 +3,8 @@ /** Existing npm names retained by capabilities now housed under experimental/. */ export const EXPERIMENTAL_PACKAGE_NAME_EXCEPTIONS: Readonly> = { 'packages/experimental/cua': '@deepseek-ai/dsh-cua', + 'packages/experimental/cua-bundle': '@deepseek-ai/dsh-cua-bundle', + 'packages/experimental/client-ui-cua': '@deepseek-ai/dsh-client-ui-cua', 'packages/experimental/tool-session-query': '@deepseek-ai/dsh-tool-session-query', 'packages/experimental/session-title-all-prompts-llm': '@deepseek-ai/dsh-session-title-all-prompts-llm', 'packages/experimental/tool-terminal': '@deepseek-ai/dsh-tool-terminal', diff --git a/scripts/gen-cordis-catalog.ts b/scripts/gen-cordis-catalog.ts index de705e4bb2..df1833ce5e 100644 --- a/scripts/gen-cordis-catalog.ts +++ b/scripts/gen-cordis-catalog.ts @@ -79,6 +79,7 @@ export const SERVICE_PAGE: Record = { ptcRuntime: 'ptc-runtime.md', browserUse: 'browser-use.md', computerUse: 'computer-use.md', + cuaAccess: 'computer-use.md', commands: 'commands.md', compaction: 'compaction.md', cordisInspect: 'extensions.md', @@ -324,6 +325,12 @@ export const LINK_MAP: Readonly> = { InspectOptions: 'boot.md', BrowserUseProviderName: 'browser-use.md', ComputerUseProviderName: 'computer-use.md', + CuaApplicationId: 'computer-use.md', + CuaApplicationGrant: 'computer-use.md', + CuaPermissionProcess: 'computer-use.md', + CuaApplicationState: 'computer-use.md', + CuaSystemPermission: 'computer-use.md', + CuaSystemPermissionState: 'computer-use.md', RenderedDocumentBytes: 'office-to-pdf.md', OfficeToPdfRequest: 'office-to-pdf.md', OfficeToPdfResult: 'office-to-pdf.md', diff --git a/scripts/gen-doc-graphs.ts b/scripts/gen-doc-graphs.ts index 7b4b9df647..9123b5211e 100644 --- a/scripts/gen-doc-graphs.ts +++ b/scripts/gen-doc-graphs.ts @@ -223,6 +223,11 @@ const SERVICE_ROLES: ServiceRole[] = [ consumers: ['experimental-computer-use-cua-driver-mcp', 'experimental-computer-use-cua-driver-native'], note: 'One provider-owned name per service instance. Each provider also owns its model tools; the service has no common action API, runtime selection, or Session workflow lock.', }, + { + key: 'cuaAccess', pkg: 'cua', title: 'Computer Use permission controls', + mode: 'core', consumers: ['client-ui-cua'], + note: 'The native provider owns application consent and cancellation; the optional Client page queries and changes those decisions through its Remote.', + }, { key: 'officeToPdf', pkg: 'office-to-pdf', title: 'Office to PDF conversion', mode: 'core', consumers: ['client-ui-sidebar-documentpreview'], diff --git a/scripts/release/families.spec.ts b/scripts/release/families.spec.ts index 4a6a12c0a0..c60e4aef1a 100644 --- a/scripts/release/families.spec.ts +++ b/scripts/release/families.spec.ts @@ -62,10 +62,12 @@ describe('release families', () => { '@deepseek-ai/dsh-experimental-client-ui-agent-team', '@deepseek-ai/dsh-experimental-client-ui-claude-code-mods', '@deepseek-ai/dsh-experimental-client-ui-cot-translation', + '@deepseek-ai/dsh-client-ui-cua', '@deepseek-ai/dsh-experimental-client-ui-voice-input', '@deepseek-ai/dsh-experimental-computer-use-cua-driver-mcp', '@deepseek-ai/dsh-experimental-computer-use-cua-driver-native', '@deepseek-ai/dsh-experimental-cot-translation-bundle', + '@deepseek-ai/dsh-cua-bundle', '@deepseek-ai/dsh-experimental-cua-macos', '@deepseek-ai/dsh-cua', '@deepseek-ai/dsh-hook-protocol', diff --git a/scripts/verify-default-product-isolation.spec.ts b/scripts/verify-default-product-isolation.spec.ts index 1c7f0a9229..9cd21e12d4 100644 --- a/scripts/verify-default-product-isolation.spec.ts +++ b/scripts/verify-default-product-isolation.spec.ts @@ -113,15 +113,16 @@ describe('default product isolation', () => { expect(verifyDefaultProductIsolation(root).failures).toEqual([]) }) - it.each([true, false])('ships an optional bundle switched off with legacy icon=%s', (legacy) => { + it.each([[true, false], [false, false], [true, true], [false, true]])('ships an optional bundle switched off with legacy icon=%s and metadata=%s', (legacy, declared) => { const root = fixture() - const layer = '@deepseek-ai/dsh-experimental-layer' - write(root, 'packages/experimental/layer/package.json', { + const layer = declared ? '@deepseek-ai/dsh-prototype-layer' : '@deepseek-ai/dsh-experimental-layer' + const directory = declared ? 'packages/bundle/prototype-layer' : 'packages/experimental/layer' + write(root, `${directory}/package.json`, { name: layer, ...legacy ? { icon: './icon.svg' } : {}, exports: { ...legacy ? {} : { './icon': './icon.svg' }, './locale/*.json': './locale/*.json' }, - dependencies: { [experimental]: 'workspace:^' }, dsh: { bundle: { patch: './cordis.patch.yml' } }, + dependencies: { [experimental]: 'workspace:^' }, dsh: { bundle: { patch: './cordis.patch.yml', ...declared ? { experimental: true } : {} } }, }) - write(root, 'packages/experimental/layer/cordis.patch.yml', [{ insert: [{ name: experimental }] }]) + write(root, `${directory}/cordis.patch.yml`, [{ insert: [{ name: experimental }] }]) manifest(root, 'apps/cli/package.json', { dependencies: { [core]: 'workspace:^', [layer]: 'workspace:^' } }) write(root, profile, `export const PROFILE_TEMPLATES = { web: { bundles: ['${base}'] } }\n` + `export const DEFAULT_PROFILE_BUNDLES = ['${base}']\n` diff --git a/scripts/verify-default-product-isolation.ts b/scripts/verify-default-product-isolation.ts index 0120368412..66e6c369d4 100644 --- a/scripts/verify-default-product-isolation.ts +++ b/scripts/verify-default-product-isolation.ts @@ -107,7 +107,8 @@ export function verifyDefaultProductIsolation(root: string): ProductIsolationRes const sources = new Set() const configs = new Set() let webPluginCount = 0 - const isExperimental = (pkg: Package): boolean => isExperimentalPackageName(pkg.manifest.name) + const isExperimental = (pkg: Package): boolean => pkg.manifest.dsh?.bundle?.experimental === true + || isExperimentalPackageName(pkg.manifest.name) || display(pkg.directory).startsWith('packages/experimental/') const add = (pkg: Package, origin: string): void => { if (isExperimental(pkg)) { diff --git a/scripts/verify-package-readme-model-experience.ts b/scripts/verify-package-readme-model-experience.ts index 2c313a3dea..bed5aa89d8 100644 --- a/scripts/verify-package-readme-model-experience.ts +++ b/scripts/verify-package-readme-model-experience.ts @@ -144,6 +144,7 @@ const SENTENCE_MODEL_EXPERIENCE: Readonly> = { 'packages/client/ui-permission-presets': { kind: 'indirect', reason: 'The picker submits the host /permission command; the knob events it appends own the model-visible effect through the sandbox/approval consumers.' }, 'packages/client/ui-settings-plugins': { kind: 'none', reason: 'Browser-side settings surface; registers no model surface.' }, 'packages/client/ui-settings-shell': { kind: 'none', reason: 'Browser-side settings surface; registers no model surface.' }, + 'packages/experimental/client-ui-cua': { kind: 'none', reason: 'Browser permission controls; the native provider owns tools and guidance.' }, 'packages/client/ui-settings-session-log': { kind: 'none', reason: 'Browser preference controls request metadata, not model input.' }, 'packages/client/ui-settings-agent-loop': { kind: 'none', reason: 'Browser-side settings surface; registers no model surface.' }, 'packages/client/ui-settings-subagent': { kind: 'none', reason: 'Browser-side settings surface; registers no model surface.' }, diff --git a/snapshots/session/cua-native/consent-fixture.mjs b/snapshots/session/cua-native/consent-fixture.mjs deleted file mode 100644 index e0b0fe035e..0000000000 --- a/snapshots/session/cua-native/consent-fixture.mjs +++ /dev/null @@ -1,14 +0,0 @@ -/** Scripted human consent for the native application replay. */ -import { join } from 'node:path' -import { startFixtureHost } from './fixture-host.mjs' -export const name = 'cua-consent-fixture' -export const inject = ['userQuestions'] - -/** Answer through the existing human-question service while native dispatch stays real. */ -export async function apply(ctx) { - const close = await startFixtureHost(join(process.cwd(), 'native-helper.mjs')) - ctx.effect(() => close) - ctx.on('user-questions/request', async request => ({ answers: request.questions.map(question => ({ - id: question.id, selected: ['Allow for this task'], - })) })) -} diff --git a/snapshots/session/cua-native/cordis.snapshot.yml b/snapshots/session/cua-native/cordis.snapshot.yml index f6708af93d..958f41ca53 100644 --- a/snapshots/session/cua-native/cordis.snapshot.yml +++ b/snapshots/session/cua-native/cordis.snapshot.yml @@ -50,11 +50,13 @@ - id: cua-native name: '@deepseek-ai/dsh-experimental-cua-macos' config: + agentPreset: false helperPath: !!js process.cwd() + '/native-helper.mjs' + allowedApps: [{"id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "name": "Fixture", "executablePath": "/fixture/app"}] - id: user-questions name: '@deepseek-ai/dsh-user-questions' - insert: - - id: cua-consent-fixture - name: './consent-fixture.mjs' + - id: cua-permission-fixture + name: './permission-fixture.mjs' diff --git a/snapshots/session/cua-native/cordis.yml b/snapshots/session/cua-native/cordis.yml index b60d7872d4..c25eadc41d 100644 --- a/snapshots/session/cua-native/cordis.yml +++ b/snapshots/session/cua-native/cordis.yml @@ -32,11 +32,13 @@ - id: cua-native name: '@deepseek-ai/dsh-experimental-cua-macos' config: + agentPreset: false helperPath: !!js process.cwd() + '/native-helper.mjs' + allowedApps: [{"id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "name": "Fixture", "executablePath": "/fixture/app"}] - id: user-questions name: '@deepseek-ai/dsh-user-questions' - insert: - - id: cua-consent-fixture - name: './consent-fixture.mjs' + - id: cua-permission-fixture + name: './permission-fixture.mjs' diff --git a/snapshots/session/cua-native/permission-fixture.mjs b/snapshots/session/cua-native/permission-fixture.mjs new file mode 100644 index 0000000000..e1580ceedc --- /dev/null +++ b/snapshots/session/cua-native/permission-fixture.mjs @@ -0,0 +1,23 @@ +/** Scripted human answers and a permission-page revocation in the recorded CUA scenario. */ +import { join } from 'node:path' +import { startFixtureHost } from './fixture-host.mjs' +export const name = 'cua-permission-fixture' +export const inject = ['userQuestions', 'cuaAccess', 'tools'] + +/** Supply external human decisions while retaining the real permission policy and tool pipeline. */ +export async function apply(ctx) { + const close = await startFixtureHost(join(process.cwd(), 'native-helper.mjs')) + ctx.effect(() => close) + ctx.on('user-questions/request', async (request) => ({ answers: request.questions.map(question => ({ + id: question.id, + selected: [question.question.includes('Denied Fixture') ? 'Deny' : 'Allow for this session'], + })) })) + ctx.on('tools/post-execute', async (execution, _result, next) => { + const decision = await next() + if (execution.name === 'cua' && execution.arguments.method === 'getApp' + && execution.arguments.params.selector === 'Revocable Fixture') { + await ctx.cuaAccess.revoke('c'.repeat(64)) + } + return decision + }) +} diff --git a/snapshots/session/cua-native/replay.override.json b/snapshots/session/cua-native/replay.override.json index 916561a389..57e1373f80 100644 --- a/snapshots/session/cua-native/replay.override.json +++ b/snapshots/session/cua-native/replay.override.json @@ -32,6 +32,105 @@ } ] }, + { + "kind": "chunks", + "chunks": [ + { + "type": "block-start", + "index": 0, + "blockType": "tool-call" + }, + { + "type": "block-end", + "index": 0, + "block": { + "type": "tool-call", + "id": "cua-unauthorized-getState", + "name": "cua", + "arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}" + } + }, + { + "type": "usage", + "usage": { + "inputTokens": 3, + "outputTokens": 3 + } + }, + { + "type": "finish", + "reason": { + "kind": "tool-calls" + } + } + ] + }, + { + "kind": "chunks", + "chunks": [ + { + "type": "block-start", + "index": 0, + "blockType": "tool-call" + }, + { + "type": "block-end", + "index": 0, + "block": { + "type": "tool-call", + "id": "cua-unauthorized-screenshot", + "name": "cua", + "arguments": "{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}" + } + }, + { + "type": "usage", + "usage": { + "inputTokens": 3, + "outputTokens": 3 + } + }, + { + "type": "finish", + "reason": { + "kind": "tool-calls" + } + } + ] + }, + { + "kind": "chunks", + "chunks": [ + { + "type": "block-start", + "index": 0, + "blockType": "tool-call" + }, + { + "type": "block-end", + "index": 0, + "block": { + "type": "tool-call", + "id": "cua-unauthorized-click", + "name": "cua", + "arguments": "{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}" + } + }, + { + "type": "usage", + "usage": { + "inputTokens": 3, + "outputTokens": 3 + } + }, + { + "type": "finish", + "reason": { + "kind": "tool-calls" + } + } + ] + }, { "kind": "chunks", "chunks": [ @@ -1616,6 +1715,105 @@ } ] }, + { + "kind": "chunks", + "chunks": [ + { + "type": "block-start", + "index": 0, + "blockType": "tool-call" + }, + { + "type": "block-end", + "index": 0, + "block": { + "type": "tool-call", + "id": "cua-consent-denied", + "name": "cua", + "arguments": "{\"method\":\"getApp\",\"params\":{\"selector\":\"Denied Fixture\"}}" + } + }, + { + "type": "usage", + "usage": { + "inputTokens": 3, + "outputTokens": 3 + } + }, + { + "type": "finish", + "reason": { + "kind": "tool-calls" + } + } + ] + }, + { + "kind": "chunks", + "chunks": [ + { + "type": "block-start", + "index": 0, + "blockType": "tool-call" + }, + { + "type": "block-end", + "index": 0, + "block": { + "type": "tool-call", + "id": "cua-consent-revocable", + "name": "cua", + "arguments": "{\"method\":\"getApp\",\"params\":{\"selector\":\"Revocable Fixture\"}}" + } + }, + { + "type": "usage", + "usage": { + "inputTokens": 3, + "outputTokens": 3 + } + }, + { + "type": "finish", + "reason": { + "kind": "tool-calls" + } + } + ] + }, + { + "kind": "chunks", + "chunks": [ + { + "type": "block-start", + "index": 0, + "blockType": "tool-call" + }, + { + "type": "block-end", + "index": 0, + "block": { + "type": "tool-call", + "id": "cua-consent-revoked", + "name": "cua", + "arguments": "{\"method\":\"getState\",\"params\":{\"appHandle\":\"revoked-app\"}}" + } + }, + { + "type": "usage", + "usage": { + "inputTokens": 3, + "outputTokens": 3 + } + }, + { + "type": "finish", + "reason": { + "kind": "tool-calls" + } + } + ] + }, { "kind": "chunks", "chunks": [ diff --git a/snapshots/session/cua-native/session.v4.jsonl b/snapshots/session/cua-native/session.v4.jsonl index b608bb43c4..2b978c0888 100644 --- a/snapshots/session/cua-native/session.v4.jsonl +++ b/snapshots/session/cua-native/session.v4.jsonl @@ -2,261 +2,291 @@ {"type":"permission/preset","data":{"preset":"danger-full-access"}} {"type":"sandbox/mode","data":{"mode":"danger-full-access"}} {"type":"approval/policy","data":{"policy":"never"}} -{"type":"agent/inbox/spliced","data":{"target":"next-turn","start":0,"inserted":[{"content":[{"type":"text","text":"Inspect CUA capabilities, select the Fixture application, increment its counter once, then read fresh state and capture the window. Also inspect the AXless canvas window: capture it, dispatch one coordinate click at [20,20], and read its fresh empty AX state."}],"source":{"kind":"user"},"role":"user","id":"{{message:1}}"}]}} +{"type":"agent/inbox/spliced","data":{"target":"next-turn","start":0,"inserted":[{"content":[{"type":"text","text":"Inspect CUA capabilities, select the Fixture application, increment its counter once, then read fresh state and capture the window. Also inspect the AXless canvas window: capture it, dispatch one coordinate click at [20,20], and read its fresh empty AX state. Finally verify a denied application selection and refusal of a handle after application access is revoked."}],"source":{"kind":"user"},"role":"user","id":"{{message:1}}"}]}} {"type":"turn/start","data":{"turn":1}} {"type":"agent/inbox/spliced","data":{"target":"next-turn","start":0,"removedCount":1,"inserted":[]}} {"type":"step/start","data":{"turn":1,"step":1}} {"type":"system/message","data":{"turn":1,"step":1,"message":{"role":"system","content":[{"type":"text","text":"{{system}}"}],"source":{"kind":"system-prompt"},"id":"{{message:2}}"}},"surfaceOp":"append"} -{"type":"user/message","data":{"content":[{"type":"text","text":"Inspect CUA capabilities, select the Fixture application, increment its counter once, then read fresh state and capture the window. Also inspect the AXless canvas window: capture it, dispatch one coordinate click at [20,20], and read its fresh empty AX state."}],"source":{"kind":"user"},"role":"user","id":"{{message:1}}"},"surfaceOp":"append"} +{"type":"user/message","data":{"content":[{"type":"text","text":"Inspect CUA capabilities, select the Fixture application, increment its counter once, then read fresh state and capture the window. Also inspect the AXless canvas window: capture it, dispatch one coordinate click at [20,20], and read its fresh empty AX state. Finally verify a denied application selection and refusal of a handle after application access is revoked."}],"source":{"kind":"user"},"role":"user","id":"{{message:1}}"},"surfaceOp":"append"} {"type":"user/message","data":{"content":[{"type":"text","text":"Current runtime context. This snapshot supersedes earlier runtime-context snapshots.\n\nCurrent working directory: \"{{cwd}}\".\n\nCurrent DSH file policy: danger-full-access. The DSH file sandbox does not restrict file modifications by available operations.\n\nApproval prompts are disabled in this session: actions that require approval are rejected automatically — do not request sandbox escalation (do not set `sandbox_permissions`)."}],"source":{"kind":"runtime-context","form":"snapshot","sections":[{"name":"working-directory:current","text":"Current working directory: \"{{cwd}}\"."},{"name":"sandbox:policy","text":"Current DSH file policy: danger-full-access. The DSH file sandbox does not restrict file modifications by available operations."},{"name":"approval:policy","text":"Approval prompts are disabled in this session: actions that require approval are rejected automatically — do not request sandbox escalation (do not set `sandbox_permissions`)."}]},"role":"user","id":"{{message:3}}"},"surfaceOp":"append"} {"type":"request/header","data":{"header":{"config":{"provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"tools":"{{tools}}"},"reason":"initial"}} {"type":"request/context","data":{"provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"}} {"type":"session/title","data":{"title":"Inspect CUA capabilities, select the","messageSeqs":[8],"source":{"kind":"fallback"}}} -{"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:4}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554024,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554025,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}}},{"type":"chunk","time":1791947554025,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554025,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:4}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947688973,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947688973,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}}},{"type":"chunk","time":1791947688973,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947688973,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":1,"callId":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}} {"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-1"},"toolCallId":"cua-native-1","content":[{"type":"text","text":"{\"platform\":\"macos\",\"sessionLocked\":false,\"permissions\":{\"accessibility\":true,\"screenRecording\":true},\"methods\":[\"getCapabilities\",\"listApps\",\"getApp\",\"activate\",\"show\",\"listWindows\",\"getState\",\"hover\",\"click\",\"setValue\",\"typeText\",\"paste\",\"pressKey\",\"scroll\",\"drag\",\"screenshot\"],\"coordinateSpace\":\"screenshot-pixels\",\"browser\":false}"}],"isError":false,"id":"{{message:5}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":1}} {"type":"step/start","data":{"turn":1,"step":2}} -{"type":"assistant/message","data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-2","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"test.fixture\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:6}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554613,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554614,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-2","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"test.fixture\"}}"}}},{"type":"chunk","time":1791947554614,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554614,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":2,"callId":"cua-native-2","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"test.fixture\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":2,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-2"},"toolCallId":"cua-native-2","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"name\":\"Fixture\",\"bundleId\":\"test.fixture\"}"}],"isError":false,"id":"{{message:7}}"}},"sourceEventSeqs":[19],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-unauthorized-getState","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:6}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689573,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689573,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-unauthorized-getState","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1791947689573,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689573,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":2,"callId":"cua-unauthorized-getState","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":2,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-unauthorized-getState"},"toolCallId":"cua-unauthorized-getState","content":[{"type":"text","text":"Error: APP_ACCESS_REQUIRED: Select the application with getApp and obtain user consent before reading or controlling it"}],"isError":true,"id":"{{message:7}}"}},"sourceEventSeqs":[19],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":2}} {"type":"step/start","data":{"turn":1,"step":3}} -{"type":"assistant/message","data":{"turn":1,"step":3,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-3","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"includeMenus\":true,\"details\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:8}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554635,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554635,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-3","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"includeMenus\":true,\"details\":true}}"}}},{"type":"chunk","time":1791947554635,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554635,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":3,"callId":"cua-native-3","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"includeMenus\":true,\"details\":true}}"}} -{"type":"tool/result","data":{"turn":1,"step":3,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-3"},"toolCallId":"cua-native-3","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-1\",\"tree\":\"AX scope=window offsets=UTF-16 bounds=global-points\\n- [0] AXButton \\\"Increment\\\" id=\\\"increment\\\" actions=[\\\"AXPress\\\"] bounds=[60,80,100,20]\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 0\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\n - [3] AXMenuItem \\\"Fixture command\\\"\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:9}}"}},"sourceEventSeqs":[24],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":3,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-unauthorized-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:8}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689596,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689596,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-unauthorized-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947689596,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689596,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":3,"callId":"cua-unauthorized-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":3,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-unauthorized-screenshot"},"toolCallId":"cua-unauthorized-screenshot","content":[{"type":"text","text":"Error: APP_ACCESS_REQUIRED: Select the application with getApp and obtain user consent before reading or controlling it"}],"isError":true,"id":"{{message:9}}"}},"sourceEventSeqs":[24],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":3}} {"type":"step/start","data":{"turn":1,"step":4}} -{"type":"assistant/message","data":{"turn":1,"step":4,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:10}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554651,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554651,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}}},{"type":"chunk","time":1791947554651,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554651,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":4,"callId":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}} -{"type":"tool/result","data":{"turn":1,"step":4,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-4"},"toolCallId":"cua-native-4","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:11}}"}},"sourceEventSeqs":[29],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":4,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-unauthorized-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:10}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689615,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689615,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-unauthorized-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}}},{"type":"chunk","time":1791947689615,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689615,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":4,"callId":"cua-unauthorized-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}} +{"type":"tool/result","data":{"turn":1,"step":4,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-unauthorized-click"},"toolCallId":"cua-unauthorized-click","content":[{"type":"text","text":"Error: APP_ACCESS_REQUIRED: Select the application with getApp and obtain user consent before reading or controlling it"}],"isError":true,"id":"{{message:11}}"}},"sourceEventSeqs":[29],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":4}} {"type":"step/start","data":{"turn":1,"step":5}} -{"type":"assistant/message","data":{"turn":1,"step":5,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-5","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:12}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554667,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554667,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-5","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947554667,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554667,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":5,"callId":"cua-native-5","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":5,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-5"},"toolCallId":"cua-native-5","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-2\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 1\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:13}}"}},"sourceEventSeqs":[34],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":5,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-2","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"test.fixture\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:12}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689634,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689634,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-2","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"test.fixture\"}}"}}},{"type":"chunk","time":1791947689634,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689634,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":5,"callId":"cua-native-2","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"test.fixture\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":5,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-2"},"toolCallId":"cua-native-2","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"name\":\"Fixture\",\"bundleId\":\"test.fixture\"}"}],"isError":false,"id":"{{message:13}}"}},"sourceEventSeqs":[34],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":5}} {"type":"step/start","data":{"turn":1,"step":6}} -{"type":"assistant/message","data":{"turn":1,"step":6,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-6","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:14}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554684,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554684,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-6","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947554684,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554684,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":6,"callId":"cua-native-6","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":6,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-6"},"toolCallId":"cua-native-6","content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"isError":false,"id":"{{message:15}}"}},"sourceEventSeqs":[39],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":6,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-3","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"includeMenus\":true,\"details\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:14}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689657,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689657,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-3","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"includeMenus\":true,\"details\":true}}"}}},{"type":"chunk","time":1791947689657,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689657,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":6,"callId":"cua-native-3","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"includeMenus\":true,\"details\":true}}"}} +{"type":"tool/result","data":{"turn":1,"step":6,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-3"},"toolCallId":"cua-native-3","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-1\",\"tree\":\"AX scope=window offsets=UTF-16 bounds=global-points\\n- [0] AXButton \\\"Increment\\\" id=\\\"increment\\\" actions=[\\\"AXPress\\\"] bounds=[60,80,100,20]\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 0\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\n - [3] AXMenuItem \\\"Fixture command\\\"\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:15}}"}},"sourceEventSeqs":[39],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":6}} {"type":"step/start","data":{"turn":1,"step":7}} -{"type":"assistant/message","data":{"turn":1,"step":7,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-7","name":"cua","arguments":"{\"method\":\"listWindows\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:16}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554733,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554733,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-7","name":"cua","arguments":"{\"method\":\"listWindows\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1791947554733,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554733,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":7,"callId":"cua-native-7","name":"cua","arguments":"{\"method\":\"listWindows\",\"params\":{\"appHandle\":\"fixture-app\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":7,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-7"},"toolCallId":"cua-native-7","content":[{"type":"text","text":"[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true},{\"windowId\":8,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"AXless canvas\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}]"}],"isError":false,"id":"{{message:17}}"}},"sourceEventSeqs":[44],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":7,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:16}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689683,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689683,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}}},{"type":"chunk","time":1791947689683,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689683,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":7,"callId":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}} +{"type":"tool/result","data":{"turn":1,"step":7,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-4"},"toolCallId":"cua-native-4","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:17}}"}},"sourceEventSeqs":[44],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":7}} {"type":"step/start","data":{"turn":1,"step":8}} -{"type":"assistant/message","data":{"turn":1,"step":8,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-8","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:18}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554749,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554749,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-8","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1791947554749,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554749,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":8,"callId":"cua-native-8","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}} -{"type":"tool/result","data":{"turn":1,"step":8,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-8"},"toolCallId":"cua-native-8","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"tree\":\"\",\"windows\":[{\"windowId\":8,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"AXless canvas\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:19}}"}},"sourceEventSeqs":[49],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":8,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-5","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:18}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689706,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689706,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-5","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947689706,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689706,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":8,"callId":"cua-native-5","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":8,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-5"},"toolCallId":"cua-native-5","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-2\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 1\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:19}}"}},"sourceEventSeqs":[49],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":8}} {"type":"step/start","data":{"turn":1,"step":9}} -{"type":"assistant/message","data":{"turn":1,"step":9,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-9","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:20}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554765,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554765,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-9","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1791947554765,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554765,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":9,"callId":"cua-native-9","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}} -{"type":"tool/result","data":{"turn":1,"step":9,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-9"},"toolCallId":"cua-native-9","content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":8,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"isError":false,"id":"{{message:21}}"}},"sourceEventSeqs":[54],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":9,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-6","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:20}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689729,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689729,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-6","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947689729,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689729,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":9,"callId":"cua-native-6","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":9,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-6"},"toolCallId":"cua-native-6","content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"isError":false,"id":"{{message:21}}"}},"sourceEventSeqs":[54],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":9}} {"type":"step/start","data":{"turn":1,"step":10}} -{"type":"assistant/message","data":{"turn":1,"step":10,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-10","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":0,\"y\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:22}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554791,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554791,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-10","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":0,\"y\":0}}"}}},{"type":"chunk","time":1791947554791,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554791,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":10,"callId":"cua-native-10","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":0,\"y\":0}}"}} -{"type":"tool/result","data":{"turn":1,"step":10,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-10"},"toolCallId":"cua-native-10","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:23}}"}},"sourceEventSeqs":[59],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":10,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-7","name":"cua","arguments":"{\"method\":\"listWindows\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:22}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689792,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689792,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-7","name":"cua","arguments":"{\"method\":\"listWindows\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1791947689792,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689792,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":10,"callId":"cua-native-7","name":"cua","arguments":"{\"method\":\"listWindows\",\"params\":{\"appHandle\":\"fixture-app\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":10,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-7"},"toolCallId":"cua-native-7","content":[{"type":"text","text":"[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true},{\"windowId\":8,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"AXless canvas\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}]"}],"isError":false,"id":"{{message:23}}"}},"sourceEventSeqs":[59],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":10}} {"type":"step/start","data":{"turn":1,"step":11}} -{"type":"assistant/message","data":{"turn":1,"step":11,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-11","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:24}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554807,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554807,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-11","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1791947554807,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554807,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":11,"callId":"cua-native-11","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}} -{"type":"tool/result","data":{"turn":1,"step":11,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-11"},"toolCallId":"cua-native-11","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"tree\":\"\",\"windows\":[{\"windowId\":8,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"AXless canvas\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:25}}"}},"sourceEventSeqs":[64],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":11,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-8","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:24}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689834,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689834,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-8","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1791947689834,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689834,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":11,"callId":"cua-native-8","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}} +{"type":"tool/result","data":{"turn":1,"step":11,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-8"},"toolCallId":"cua-native-8","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"tree\":\"\",\"windows\":[{\"windowId\":8,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"AXless canvas\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:25}}"}},"sourceEventSeqs":[64],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":11}} {"type":"step/start","data":{"turn":1,"step":12}} -{"type":"assistant/message","data":{"turn":1,"step":12,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-show","name":"cua","arguments":"{\"method\":\"show\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:26}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554823,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554823,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-show","name":"cua","arguments":"{\"method\":\"show\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947554823,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554823,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":12,"callId":"cua-native-show","name":"cua","arguments":"{\"method\":\"show\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":12,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-show"},"toolCallId":"cua-native-show","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false,\"windowId\":7,\"foregroundChanged\":false}"}],"isError":false,"id":"{{message:27}}"}},"sourceEventSeqs":[69],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":12,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-9","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:26}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689861,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689861,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-9","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1791947689861,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689861,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":12,"callId":"cua-native-9","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}} +{"type":"tool/result","data":{"turn":1,"step":12,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-9"},"toolCallId":"cua-native-9","content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":8,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"isError":false,"id":"{{message:27}}"}},"sourceEventSeqs":[69],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":12}} {"type":"step/start","data":{"turn":1,"step":13}} -{"type":"assistant/message","data":{"turn":1,"step":13,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:28}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554839,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554839,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1791947554839,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554839,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":13,"callId":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":13,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-activate-first"},"toolCallId":"cua-native-activate-first","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false,\"foreground\":{\"requestId\":\"cua-native-activate-first\",\"operation\":\"activate\",\"targetPid\":9,\"targetWindowId\":null,\"before\":{\"frontPid\":42,\"focusedWindowId\":7,\"frontWindowId\":99,\"frontWindowPid\":42},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":true,\"raiseRequested\":false}}"}],"isError":false,"id":"{{message:29}}"}},"sourceEventSeqs":[74],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":13,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-10","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":0,\"y\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:28}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689896,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689896,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-10","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":0,\"y\":0}}"}}},{"type":"chunk","time":1791947689896,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689896,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":13,"callId":"cua-native-10","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":0,\"y\":0}}"}} +{"type":"tool/result","data":{"turn":1,"step":13,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-10"},"toolCallId":"cua-native-10","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:29}}"}},"sourceEventSeqs":[74],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":13}} {"type":"step/start","data":{"turn":1,"step":14}} -{"type":"assistant/message","data":{"turn":1,"step":14,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:30}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554856,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554856,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1791947554856,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554856,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":14,"callId":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":14,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-activate-repeat"},"toolCallId":"cua-native-activate-repeat","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false,\"foreground\":{\"requestId\":\"cua-native-activate-repeat\",\"operation\":\"activate\",\"targetPid\":9,\"targetWindowId\":null,\"before\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":false,\"raiseRequested\":false}}"}],"isError":false,"id":"{{message:31}}"}},"sourceEventSeqs":[79],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":14,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-11","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:30}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689918,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689918,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-11","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1791947689918,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689918,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":14,"callId":"cua-native-11","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}} +{"type":"tool/result","data":{"turn":1,"step":14,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-11"},"toolCallId":"cua-native-11","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"tree\":\"\",\"windows\":[{\"windowId\":8,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"AXless canvas\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:31}}"}},"sourceEventSeqs":[79],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":14}} {"type":"step/start","data":{"turn":1,"step":15}} -{"type":"assistant/message","data":{"turn":1,"step":15,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-transition","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"Return\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:32}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554872,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554872,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-transition","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"Return\"}}"}}},{"type":"chunk","time":1791947554872,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554872,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":15,"callId":"cua-native-transition","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"Return\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":15,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-transition"},"toolCallId":"cua-native-transition","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:33}}"}},"sourceEventSeqs":[84],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":15,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-show","name":"cua","arguments":"{\"method\":\"show\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:32}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689940,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689940,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-show","name":"cua","arguments":"{\"method\":\"show\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947689940,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689940,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":15,"callId":"cua-native-show","name":"cua","arguments":"{\"method\":\"show\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":15,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-show"},"toolCallId":"cua-native-show","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false,\"windowId\":7,\"foregroundChanged\":false}"}],"isError":false,"id":"{{message:33}}"}},"sourceEventSeqs":[84],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":15}} {"type":"step/start","data":{"turn":1,"step":16}} -{"type":"assistant/message","data":{"turn":1,"step":16,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-stale-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:34}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554888,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554888,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-stale-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947554888,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554888,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":16,"callId":"cua-native-stale-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":16,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-stale-read"},"toolCallId":"cua-native-stale-read","content":[{"type":"text","text":"Error: STALE_SNAPSHOT: Accessibility tree changed during observation; read fresh getAXState"}],"isError":true,"id":"{{message:35}}"}},"sourceEventSeqs":[89],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":16,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:34}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689961,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689961,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1791947689961,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689961,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":16,"callId":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":16,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-activate-first"},"toolCallId":"cua-native-activate-first","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false,\"foreground\":{\"requestId\":\"cua-native-activate-first\",\"operation\":\"activate\",\"targetPid\":9,\"targetWindowId\":null,\"before\":{\"frontPid\":42,\"focusedWindowId\":7,\"frontWindowId\":99,\"frontWindowPid\":42},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":true,\"raiseRequested\":false}}"}],"isError":false,"id":"{{message:35}}"}},"sourceEventSeqs":[89],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":16}} {"type":"step/start","data":{"turn":1,"step":17}} -{"type":"assistant/message","data":{"turn":1,"step":17,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-stale-index","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-2\",\"elementIndex\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:36}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554904,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554904,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-stale-index","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-2\",\"elementIndex\":0}}"}}},{"type":"chunk","time":1791947554904,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554904,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":17,"callId":"cua-native-stale-index","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-2\",\"elementIndex\":0}}"}} -{"type":"tool/result","data":{"turn":1,"step":17,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-stale-index"},"toolCallId":"cua-native-stale-index","content":[{"type":"text","text":"Error: STALE_SNAPSHOT: Read fresh state before acting"}],"isError":true,"id":"{{message:37}}"}},"sourceEventSeqs":[94],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":17,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:36}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689983,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689983,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1791947689983,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689983,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":17,"callId":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":17,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-activate-repeat"},"toolCallId":"cua-native-activate-repeat","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false,\"foreground\":{\"requestId\":\"cua-native-activate-repeat\",\"operation\":\"activate\",\"targetPid\":9,\"targetWindowId\":null,\"before\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":false,\"raiseRequested\":false}}"}],"isError":false,"id":"{{message:37}}"}},"sourceEventSeqs":[94],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":17}} {"type":"step/start","data":{"turn":1,"step":18}} -{"type":"assistant/message","data":{"turn":1,"step":18,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:38}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554920,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554920,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0}}"}}},{"type":"chunk","time":1791947554920,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554920,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":18,"callId":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0}}"}} -{"type":"tool/result","data":{"turn":1,"step":18,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-other-window"},"toolCallId":"cua-native-other-window","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:39}}"}},"sourceEventSeqs":[99],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":18,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-transition","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"Return\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:38}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690002,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690002,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-transition","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"Return\"}}"}}},{"type":"chunk","time":1791947690002,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690002,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":18,"callId":"cua-native-transition","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"Return\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":18,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-transition"},"toolCallId":"cua-native-transition","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:39}}"}},"sourceEventSeqs":[99],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":18}} {"type":"step/start","data":{"turn":1,"step":19}} -{"type":"assistant/message","data":{"turn":1,"step":19,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-fresh-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:40}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554936,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554936,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-fresh-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947554936,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554936,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":19,"callId":"cua-native-fresh-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":19,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-fresh-read"},"toolCallId":"cua-native-fresh-read","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-5\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 1\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:41}}"}},"sourceEventSeqs":[104],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":19,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-stale-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:40}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690019,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690019,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-stale-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947690019,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690019,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":19,"callId":"cua-native-stale-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":19,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-stale-read"},"toolCallId":"cua-native-stale-read","content":[{"type":"text","text":"Error: STALE_SNAPSHOT: Accessibility tree changed during observation; read fresh getAXState"}],"isError":true,"id":"{{message:41}}"}},"sourceEventSeqs":[104],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":19}} {"type":"step/start","data":{"turn":1,"step":20}} -{"type":"assistant/message","data":{"turn":1,"step":20,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-fresh-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-5\",\"elementIndex\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:42}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554954,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554954,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-fresh-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-5\",\"elementIndex\":0}}"}}},{"type":"chunk","time":1791947554954,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554954,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":20,"callId":"cua-native-fresh-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-5\",\"elementIndex\":0}}"}} -{"type":"tool/result","data":{"turn":1,"step":20,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-fresh-click"},"toolCallId":"cua-native-fresh-click","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:43}}"}},"sourceEventSeqs":[109],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":20,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-stale-index","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-2\",\"elementIndex\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:42}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690040,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690040,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-stale-index","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-2\",\"elementIndex\":0}}"}}},{"type":"chunk","time":1791947690040,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690040,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":20,"callId":"cua-native-stale-index","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-2\",\"elementIndex\":0}}"}} +{"type":"tool/result","data":{"turn":1,"step":20,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-stale-index"},"toolCallId":"cua-native-stale-index","content":[{"type":"text","text":"Error: STALE_SNAPSHOT: Read fresh state before acting"}],"isError":true,"id":"{{message:43}}"}},"sourceEventSeqs":[109],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":20}} {"type":"step/start","data":{"turn":1,"step":21}} -{"type":"assistant/message","data":{"turn":1,"step":21,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:44}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554970,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554970,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947554970,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554970,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":21,"callId":"cua-native-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":21,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-verify"},"toolCallId":"cua-native-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-6\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 2\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:45}}"}},"sourceEventSeqs":[114],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":21,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:44}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690064,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690064,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0}}"}}},{"type":"chunk","time":1791947690064,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690064,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":21,"callId":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0}}"}} +{"type":"tool/result","data":{"turn":1,"step":21,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-other-window"},"toolCallId":"cua-native-other-window","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:45}}"}},"sourceEventSeqs":[114],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":21}} {"type":"step/start","data":{"turn":1,"step":22}} -{"type":"assistant/message","data":{"turn":1,"step":22,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:46}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947554987,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947554987,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}}},{"type":"chunk","time":1791947554987,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947554987,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":22,"callId":"cua-native-geometry-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}} -{"type":"tool/result","data":{"turn":1,"step":22,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-observe"},"toolCallId":"cua-native-geometry-observe","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":9,\"snapshotId\":\"fixture-snapshot-7\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 2\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":9,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"Transformed window\",\"bounds\":{\"x\":16,\"y\":512,\"width\":131,\"height\":145},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:47}}"}},"sourceEventSeqs":[119],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":22,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-fresh-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:46}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690087,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690087,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-fresh-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947690087,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690087,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":22,"callId":"cua-native-fresh-read","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":22,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-fresh-read"},"toolCallId":"cua-native-fresh-read","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-5\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 1\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:47}}"}},"sourceEventSeqs":[119],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":22}} {"type":"step/start","data":{"turn":1,"step":23}} -{"type":"assistant/message","data":{"turn":1,"step":23,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:48}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555003,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555003,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}}},{"type":"chunk","time":1791947555003,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555003,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":23,"callId":"cua-native-geometry-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}} -{"type":"tool/result","data":{"turn":1,"step":23,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-capture"},"toolCallId":"cua-native-geometry-capture","content":[{"type":"text","text":"Error: WINDOW_CHANGED: Window display and accessibility bounds disagree. Read fresh state; if unchanged, use AX actions or explicitly restore the window before capture or coordinate input"}],"isError":true,"id":"{{message:49}}"}},"sourceEventSeqs":[124],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":23,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-fresh-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-5\",\"elementIndex\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:48}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690111,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690111,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-fresh-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-5\",\"elementIndex\":0}}"}}},{"type":"chunk","time":1791947690111,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690111,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":23,"callId":"cua-native-fresh-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-5\",\"elementIndex\":0}}"}} +{"type":"tool/result","data":{"turn":1,"step":23,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-fresh-click"},"toolCallId":"cua-native-fresh-click","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:49}}"}},"sourceEventSeqs":[124],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":23}} {"type":"step/start","data":{"turn":1,"step":24}} -{"type":"assistant/message","data":{"turn":1,"step":24,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"snapshotId\":\"fixture-snapshot-7\",\"x\":20,\"y\":20,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:50}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555020,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555020,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"snapshotId\":\"fixture-snapshot-7\",\"x\":20,\"y\":20,\"relative\":true}}"}}},{"type":"chunk","time":1791947555020,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555020,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":24,"callId":"cua-native-geometry-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"snapshotId\":\"fixture-snapshot-7\",\"x\":20,\"y\":20,\"relative\":true}}"}} -{"type":"tool/result","data":{"turn":1,"step":24,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-click"},"toolCallId":"cua-native-geometry-click","content":[{"type":"text","text":"Error: WINDOW_CHANGED: Window display and accessibility bounds disagree. Read fresh state; if unchanged, use AX actions or explicitly restore the window before capture or coordinate input"}],"isError":true,"id":"{{message:51}}"}},"sourceEventSeqs":[129],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":24,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:50}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690141,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690141,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947690141,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690141,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":24,"callId":"cua-native-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":24,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-verify"},"toolCallId":"cua-native-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-6\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 2\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:51}}"}},"sourceEventSeqs":[129],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":24}} {"type":"step/start","data":{"turn":1,"step":25}} -{"type":"assistant/message","data":{"turn":1,"step":25,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-scroll","name":"cua","arguments":"{\"method\":\"scroll\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"x\":20,\"y\":20,\"deltaLinesX\":0,\"deltaLinesY\":-1,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:52}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555037,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555037,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-scroll","name":"cua","arguments":"{\"method\":\"scroll\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"x\":20,\"y\":20,\"deltaLinesX\":0,\"deltaLinesY\":-1,\"relative\":true}}"}}},{"type":"chunk","time":1791947555037,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555037,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":25,"callId":"cua-native-geometry-scroll","name":"cua","arguments":"{\"method\":\"scroll\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"x\":20,\"y\":20,\"deltaLinesX\":0,\"deltaLinesY\":-1,\"relative\":true}}"}} -{"type":"tool/result","data":{"turn":1,"step":25,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-scroll"},"toolCallId":"cua-native-geometry-scroll","content":[{"type":"text","text":"Error: WINDOW_CHANGED: Window display and accessibility bounds disagree. Read fresh state; if unchanged, use AX actions or explicitly restore the window before capture or coordinate input"}],"isError":true,"id":"{{message:53}}"}},"sourceEventSeqs":[134],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":25,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:52}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690162,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690162,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}}},{"type":"chunk","time":1791947690162,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690162,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":25,"callId":"cua-native-geometry-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}} +{"type":"tool/result","data":{"turn":1,"step":25,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-observe"},"toolCallId":"cua-native-geometry-observe","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":9,\"snapshotId\":\"fixture-snapshot-7\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 2\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":9,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"Transformed window\",\"bounds\":{\"x\":16,\"y\":512,\"width\":131,\"height\":145},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:53}}"}},"sourceEventSeqs":[134],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":25}} {"type":"step/start","data":{"turn":1,"step":26}} -{"type":"assistant/message","data":{"turn":1,"step":26,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-drag","name":"cua","arguments":"{\"method\":\"drag\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"fromX\":20,\"fromY\":20,\"toX\":30,\"toY\":30,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:54}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555054,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555054,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-drag","name":"cua","arguments":"{\"method\":\"drag\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"fromX\":20,\"fromY\":20,\"toX\":30,\"toY\":30,\"relative\":true}}"}}},{"type":"chunk","time":1791947555054,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555054,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":26,"callId":"cua-native-geometry-drag","name":"cua","arguments":"{\"method\":\"drag\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"fromX\":20,\"fromY\":20,\"toX\":30,\"toY\":30,\"relative\":true}}"}} -{"type":"tool/result","data":{"turn":1,"step":26,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-drag"},"toolCallId":"cua-native-geometry-drag","content":[{"type":"text","text":"Error: WINDOW_CHANGED: Window display and accessibility bounds disagree. Read fresh state; if unchanged, use AX actions or explicitly restore the window before capture or coordinate input"}],"isError":true,"id":"{{message:55}}"}},"sourceEventSeqs":[139],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":26,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:54}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690186,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690186,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}}},{"type":"chunk","time":1791947690186,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690186,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":26,"callId":"cua-native-geometry-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}} +{"type":"tool/result","data":{"turn":1,"step":26,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-capture"},"toolCallId":"cua-native-geometry-capture","content":[{"type":"text","text":"Error: WINDOW_CHANGED: Window display and accessibility bounds disagree. Read fresh state; if unchanged, use AX actions or explicitly restore the window before capture or coordinate input"}],"isError":true,"id":"{{message:55}}"}},"sourceEventSeqs":[139],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":26}} {"type":"step/start","data":{"turn":1,"step":27}} -{"type":"assistant/message","data":{"turn":1,"step":27,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-ax","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-7\",\"elementIndex\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:56}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555071,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555071,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-ax","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-7\",\"elementIndex\":0}}"}}},{"type":"chunk","time":1791947555071,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555071,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":27,"callId":"cua-native-geometry-ax","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-7\",\"elementIndex\":0}}"}} -{"type":"tool/result","data":{"turn":1,"step":27,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-ax"},"toolCallId":"cua-native-geometry-ax","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:57}}"}},"sourceEventSeqs":[144],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":27,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"snapshotId\":\"fixture-snapshot-7\",\"x\":20,\"y\":20,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:56}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690221,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690221,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"snapshotId\":\"fixture-snapshot-7\",\"x\":20,\"y\":20,\"relative\":true}}"}}},{"type":"chunk","time":1791947690221,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690221,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":27,"callId":"cua-native-geometry-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"snapshotId\":\"fixture-snapshot-7\",\"x\":20,\"y\":20,\"relative\":true}}"}} +{"type":"tool/result","data":{"turn":1,"step":27,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-click"},"toolCallId":"cua-native-geometry-click","content":[{"type":"text","text":"Error: WINDOW_CHANGED: Window display and accessibility bounds disagree. Read fresh state; if unchanged, use AX actions or explicitly restore the window before capture or coordinate input"}],"isError":true,"id":"{{message:57}}"}},"sourceEventSeqs":[144],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":27}} {"type":"step/start","data":{"turn":1,"step":28}} -{"type":"assistant/message","data":{"turn":1,"step":28,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:58}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555087,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555087,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}}},{"type":"chunk","time":1791947555087,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555087,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":28,"callId":"cua-native-geometry-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}} -{"type":"tool/result","data":{"turn":1,"step":28,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-verify"},"toolCallId":"cua-native-geometry-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":9,\"snapshotId\":\"fixture-snapshot-8\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 3\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":9,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"Transformed window\",\"bounds\":{\"x\":16,\"y\":512,\"width\":131,\"height\":145},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:59}}"}},"sourceEventSeqs":[149],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":28,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-scroll","name":"cua","arguments":"{\"method\":\"scroll\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"x\":20,\"y\":20,\"deltaLinesX\":0,\"deltaLinesY\":-1,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:58}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690248,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690248,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-scroll","name":"cua","arguments":"{\"method\":\"scroll\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"x\":20,\"y\":20,\"deltaLinesX\":0,\"deltaLinesY\":-1,\"relative\":true}}"}}},{"type":"chunk","time":1791947690248,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690248,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":28,"callId":"cua-native-geometry-scroll","name":"cua","arguments":"{\"method\":\"scroll\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"x\":20,\"y\":20,\"deltaLinesX\":0,\"deltaLinesY\":-1,\"relative\":true}}"}} +{"type":"tool/result","data":{"turn":1,"step":28,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-scroll"},"toolCallId":"cua-native-geometry-scroll","content":[{"type":"text","text":"Error: WINDOW_CHANGED: Window display and accessibility bounds disagree. Read fresh state; if unchanged, use AX actions or explicitly restore the window before capture or coordinate input"}],"isError":true,"id":"{{message:59}}"}},"sourceEventSeqs":[149],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":28}} {"type":"step/start","data":{"turn":1,"step":29}} -{"type":"assistant/message","data":{"turn":1,"step":29,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-busy-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F11\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:60}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555103,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555103,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-busy-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F11\"}}"}}},{"type":"chunk","time":1791947555103,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555103,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":29,"callId":"cua-capture-busy-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F11\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":29,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-capture-busy-arm"},"toolCallId":"cua-capture-busy-arm","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:61}}"}},"sourceEventSeqs":[154],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":29,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-drag","name":"cua","arguments":"{\"method\":\"drag\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"fromX\":20,\"fromY\":20,\"toX\":30,\"toY\":30,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:60}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690267,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690267,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-drag","name":"cua","arguments":"{\"method\":\"drag\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"fromX\":20,\"fromY\":20,\"toX\":30,\"toY\":30,\"relative\":true}}"}}},{"type":"chunk","time":1791947690267,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690267,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":29,"callId":"cua-native-geometry-drag","name":"cua","arguments":"{\"method\":\"drag\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9,\"fromX\":20,\"fromY\":20,\"toX\":30,\"toY\":30,\"relative\":true}}"}} +{"type":"tool/result","data":{"turn":1,"step":29,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-drag"},"toolCallId":"cua-native-geometry-drag","content":[{"type":"text","text":"Error: WINDOW_CHANGED: Window display and accessibility bounds disagree. Read fresh state; if unchanged, use AX actions or explicitly restore the window before capture or coordinate input"}],"isError":true,"id":"{{message:61}}"}},"sourceEventSeqs":[154],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":29}} {"type":"step/start","data":{"turn":1,"step":30}} -{"type":"assistant/message","data":{"turn":1,"step":30,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-busy","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:62}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555120,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555120,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-busy","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947555120,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555120,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":30,"callId":"cua-capture-busy","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":30,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-capture-busy"},"toolCallId":"cua-capture-busy","content":[{"type":"text","text":"Error: CAPTURE_BUSY: A previous system capture is still pending; other native operations remain available\nScreenshot diagnostics:\n[cua-screenshot] {\"requestId\":\"cua-capture-busy\",\"stage\":\"helper.failed\",\"elapsedMs\":1,\"errorCode\":\"CAPTURE_BUSY\"}"}],"isError":true,"id":"{{message:63}}"}},"sourceEventSeqs":[159],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":30,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-ax","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-7\",\"elementIndex\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:62}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690294,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690294,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-ax","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-7\",\"elementIndex\":0}}"}}},{"type":"chunk","time":1791947690294,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690294,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":30,"callId":"cua-native-geometry-ax","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-7\",\"elementIndex\":0}}"}} +{"type":"tool/result","data":{"turn":1,"step":30,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-ax"},"toolCallId":"cua-native-geometry-ax","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:63}}"}},"sourceEventSeqs":[159],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":30}} {"type":"step/start","data":{"turn":1,"step":31}} -{"type":"assistant/message","data":{"turn":1,"step":31,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-busy-ax","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:64}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555141,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555141,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-busy-ax","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947555141,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555141,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":31,"callId":"cua-capture-busy-ax","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":31,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-capture-busy-ax"},"toolCallId":"cua-capture-busy-ax","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-9\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 3\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:65}}"}},"sourceEventSeqs":[164],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":31,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-geometry-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:64}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690314,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690314,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-geometry-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}}},{"type":"chunk","time":1791947690314,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690314,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":31,"callId":"cua-native-geometry-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":9}}"}} +{"type":"tool/result","data":{"turn":1,"step":31,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-geometry-verify"},"toolCallId":"cua-native-geometry-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":9,\"snapshotId\":\"fixture-snapshot-8\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 3\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":9,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"Transformed window\",\"bounds\":{\"x\":16,\"y\":512,\"width\":131,\"height\":145},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:65}}"}},"sourceEventSeqs":[164],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":31}} {"type":"step/start","data":{"turn":1,"step":32}} -{"type":"assistant/message","data":{"turn":1,"step":32,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-callback","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F10\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:66}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555157,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555157,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-callback","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F10\"}}"}}},{"type":"chunk","time":1791947555157,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555157,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":32,"callId":"cua-capture-callback","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F10\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":32,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-capture-callback"},"toolCallId":"cua-capture-callback","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:67}}"}},"sourceEventSeqs":[169],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":32,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-busy-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F11\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:66}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690340,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690340,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-busy-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F11\"}}"}}},{"type":"chunk","time":1791947690340,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690340,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":32,"callId":"cua-capture-busy-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F11\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":32,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-capture-busy-arm"},"toolCallId":"cua-capture-busy-arm","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:67}}"}},"sourceEventSeqs":[169],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":32}} {"type":"step/start","data":{"turn":1,"step":33}} -{"type":"assistant/message","data":{"turn":1,"step":33,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-recovered","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:68}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555173,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555173,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-recovered","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947555173,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555173,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":33,"callId":"cua-capture-recovered","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":33,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-capture-recovered"},"toolCallId":"cua-capture-recovered","content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"isError":false,"id":"{{message:69}}"}},"sourceEventSeqs":[174],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":33,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-busy","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:68}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690365,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690365,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-busy","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947690365,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690365,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":33,"callId":"cua-capture-busy","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":33,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-capture-busy"},"toolCallId":"cua-capture-busy","content":[{"type":"text","text":"Error: CAPTURE_BUSY: A previous system capture is still pending; other native operations remain available\nScreenshot diagnostics:\n[cua-screenshot] {\"requestId\":\"cua-capture-busy\",\"stage\":\"helper.failed\",\"elapsedMs\":1,\"errorCode\":\"CAPTURE_BUSY\"}"}],"isError":true,"id":"{{message:69}}"}},"sourceEventSeqs":[174],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":33}} {"type":"step/start","data":{"turn":1,"step":34}} -{"type":"assistant/message","data":{"turn":1,"step":34,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-enter","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-9\",\"elementIndex\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:70}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555199,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555199,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-enter","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-9\",\"elementIndex\":0}}"}}},{"type":"chunk","time":1791947555199,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555199,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":34,"callId":"cua-hover-enter","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-9\",\"elementIndex\":0}}"}} -{"type":"tool/result","data":{"turn":1,"step":34,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-enter"},"toolCallId":"cua-hover-enter","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false,\"hoverRoute\":\"accessibility\",\"systemPointerMoved\":false}"}],"isError":false,"id":"{{message:71}}"}},"sourceEventSeqs":[179],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":34,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-busy-ax","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:70}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690394,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690394,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-busy-ax","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947690394,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690394,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":34,"callId":"cua-capture-busy-ax","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":34,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-capture-busy-ax"},"toolCallId":"cua-capture-busy-ax","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-9\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 3\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:71}}"}},"sourceEventSeqs":[179],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":34}} {"type":"step/start","data":{"turn":1,"step":35}} -{"type":"assistant/message","data":{"turn":1,"step":35,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:72}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555215,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555215,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947555215,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555216,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":35,"callId":"cua-hover-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":35,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-observe"},"toolCallId":"cua-hover-observe","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-10\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 3\\\"\\n- AXStaticText \\\"Hover information\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:73}}"}},"sourceEventSeqs":[184],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":35,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-callback","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F10\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:72}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690414,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690414,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-callback","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F10\"}}"}}},{"type":"chunk","time":1791947690414,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690414,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":35,"callId":"cua-capture-callback","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F10\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":35,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-capture-callback"},"toolCallId":"cua-capture-callback","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:73}}"}},"sourceEventSeqs":[184],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":35}} {"type":"step/start","data":{"turn":1,"step":36}} -{"type":"assistant/message","data":{"turn":1,"step":36,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:74}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555231,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555231,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947555231,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555231,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":36,"callId":"cua-hover-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":36,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-capture"},"toolCallId":"cua-hover-capture","content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"isError":false,"id":"{{message:75}}"}},"sourceEventSeqs":[189],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":36,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-capture-recovered","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:74}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690434,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690434,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-capture-recovered","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947690434,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690434,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":36,"callId":"cua-capture-recovered","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":36,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-capture-recovered"},"toolCallId":"cua-capture-recovered","content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"isError":false,"id":"{{message:75}}"}},"sourceEventSeqs":[189],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":36}} {"type":"step/start","data":{"turn":1,"step":37}} -{"type":"assistant/message","data":{"turn":1,"step":37,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-popup","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:76}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555256,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555256,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-popup","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1791947555256,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555256,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":37,"callId":"cua-hover-popup","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}} -{"type":"tool/result","data":{"turn":1,"step":37,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-popup"},"toolCallId":"cua-hover-popup","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false,\"hoverRoute\":\"accessibility\",\"systemPointerMoved\":false}"}],"isError":false,"id":"{{message:77}}"}},"sourceEventSeqs":[194],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":37,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-enter","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-9\",\"elementIndex\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:76}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690466,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690466,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-enter","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-9\",\"elementIndex\":0}}"}}},{"type":"chunk","time":1791947690466,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690466,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":37,"callId":"cua-hover-enter","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-9\",\"elementIndex\":0}}"}} +{"type":"tool/result","data":{"turn":1,"step":37,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-enter"},"toolCallId":"cua-hover-enter","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false,\"hoverRoute\":\"accessibility\",\"systemPointerMoved\":false}"}],"isError":false,"id":"{{message:77}}"}},"sourceEventSeqs":[194],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":37}} {"type":"step/start","data":{"turn":1,"step":38}} -{"type":"assistant/message","data":{"turn":1,"step":38,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-10\",\"x\":100,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:78}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555273,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555273,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-10\",\"x\":100,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1791947555273,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555273,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":38,"callId":"cua-hover-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-10\",\"x\":100,\"y\":200,\"relative\":true}}"}} -{"type":"tool/result","data":{"turn":1,"step":38,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-click"},"toolCallId":"cua-hover-click","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:79}}"}},"sourceEventSeqs":[199],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":38,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:78}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690495,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690495,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947690495,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690495,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":38,"callId":"cua-hover-observe","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":38,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-observe"},"toolCallId":"cua-hover-observe","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-10\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 3\\\"\\n- AXStaticText \\\"Hover information\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:79}}"}},"sourceEventSeqs":[199],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":38}} {"type":"step/start","data":{"turn":1,"step":39}} -{"type":"assistant/message","data":{"turn":1,"step":39,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-exit","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:80}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555291,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555291,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-exit","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1791947555291,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555291,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":39,"callId":"cua-hover-exit","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"}} -{"type":"tool/result","data":{"turn":1,"step":39,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-exit"},"toolCallId":"cua-hover-exit","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false,\"hoverRoute\":\"accessibility\",\"systemPointerMoved\":false}"}],"isError":false,"id":"{{message:81}}"}},"sourceEventSeqs":[204],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":39,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:80}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690517,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690517,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947690517,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690517,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":39,"callId":"cua-hover-capture","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":39,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-capture"},"toolCallId":"cua-hover-capture","content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"isError":false,"id":"{{message:81}}"}},"sourceEventSeqs":[204],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":39}} {"type":"step/start","data":{"turn":1,"step":40}} -{"type":"assistant/message","data":{"turn":1,"step":40,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:82}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555307,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555307,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947555307,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555307,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":40,"callId":"cua-hover-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":40,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-verify"},"toolCallId":"cua-hover-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-11\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:83}}"}},"sourceEventSeqs":[209],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":40,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-popup","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:82}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690565,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690565,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-popup","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1791947690565,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690565,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":40,"callId":"cua-hover-popup","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":100,\"y\":200,\"relative\":true}}"}} +{"type":"tool/result","data":{"turn":1,"step":40,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-popup"},"toolCallId":"cua-hover-popup","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false,\"hoverRoute\":\"accessibility\",\"systemPointerMoved\":false}"}],"isError":false,"id":"{{message:83}}"}},"sourceEventSeqs":[209],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":40}} {"type":"step/start","data":{"turn":1,"step":41}} -{"type":"assistant/message","data":{"turn":1,"step":41,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-background","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F9\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:84}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555323,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555323,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-background","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F9\"}}"}}},{"type":"chunk","time":1791947555323,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555323,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":41,"callId":"cua-paste-background","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F9\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":41,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-background"},"toolCallId":"cua-paste-background","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:85}}"}},"sourceEventSeqs":[214],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":41,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-10\",\"x\":100,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:84}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690590,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690590,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-10\",\"x\":100,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1791947690590,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690590,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":41,"callId":"cua-hover-click","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-10\",\"x\":100,\"y\":200,\"relative\":true}}"}} +{"type":"tool/result","data":{"turn":1,"step":41,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-click"},"toolCallId":"cua-hover-click","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:85}}"}},"sourceEventSeqs":[214],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":41}} {"type":"step/start","data":{"turn":1,"step":42}} -{"type":"assistant/message","data":{"turn":1,"step":42,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-unread","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:86}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555339,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555339,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-unread","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}}},{"type":"chunk","time":1791947555339,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555339,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":42,"callId":"cua-paste-unread","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":42,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-unread"},"toolCallId":"cua-paste-unread","content":[{"type":"text","text":"Error: PASTE_NOT_CONSUMED: Paste command was dispatched, but clipboard data was not requested within the read deadline. Bring the target app to the foreground and focus its text input. Inspect the contents before deciding whether to retry; a delayed paste may still run."}],"isError":true,"id":"{{message:87}}"}},"sourceEventSeqs":[219],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":42,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-exit","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:86}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690613,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690613,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-exit","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"}}},{"type":"chunk","time":1791947690613,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690613,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":42,"callId":"cua-hover-exit","name":"cua","arguments":"{\"method\":\"hover\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7,\"x\":0,\"y\":200,\"relative\":true}}"}} +{"type":"tool/result","data":{"turn":1,"step":42,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-exit"},"toolCallId":"cua-hover-exit","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false,\"hoverRoute\":\"accessibility\",\"systemPointerMoved\":false}"}],"isError":false,"id":"{{message:87}}"}},"sourceEventSeqs":[219],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":42}} {"type":"step/start","data":{"turn":1,"step":43}} -{"type":"assistant/message","data":{"turn":1,"step":43,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-inspect","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:88}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555356,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555356,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-inspect","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947555356,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555356,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":43,"callId":"cua-paste-inspect","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":43,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-inspect"},"toolCallId":"cua-paste-inspect","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-12\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:89}}"}},"sourceEventSeqs":[224],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":43,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-hover-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:88}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690633,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690633,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-hover-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947690633,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690633,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":43,"callId":"cua-hover-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":43,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-hover-verify"},"toolCallId":"cua-hover-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-11\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:89}}"}},"sourceEventSeqs":[224],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":43}} {"type":"step/start","data":{"turn":1,"step":44}} -{"type":"assistant/message","data":{"turn":1,"step":44,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:90}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555372,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555372,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1791947555372,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555372,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":44,"callId":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":44,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-activate"},"toolCallId":"cua-paste-activate","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false,\"foreground\":{\"requestId\":\"cua-paste-activate\",\"operation\":\"activate\",\"targetPid\":9,\"targetWindowId\":null,\"before\":{\"frontPid\":42,\"focusedWindowId\":7,\"frontWindowId\":99,\"frontWindowPid\":42},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":true,\"raiseRequested\":false}}"}],"isError":false,"id":"{{message:91}}"}},"sourceEventSeqs":[229],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":44,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-background","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F9\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:90}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690653,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690653,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-background","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F9\"}}"}}},{"type":"chunk","time":1791947690653,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690653,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":44,"callId":"cua-paste-background","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F9\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":44,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-background"},"toolCallId":"cua-paste-background","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:91}}"}},"sourceEventSeqs":[229],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":44}} {"type":"step/start","data":{"turn":1,"step":45}} -{"type":"assistant/message","data":{"turn":1,"step":45,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:92}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555388,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555388,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947555388,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555388,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":45,"callId":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":45,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-focus"},"toolCallId":"cua-paste-focus","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-13\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:93}}"}},"sourceEventSeqs":[234],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":45,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-unread","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:92}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690672,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690672,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-unread","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}}},{"type":"chunk","time":1791947690672,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690672,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":45,"callId":"cua-paste-unread","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":45,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-unread"},"toolCallId":"cua-paste-unread","content":[{"type":"text","text":"Error: PASTE_NOT_CONSUMED: Paste command was dispatched, but clipboard data was not requested within the read deadline. Bring the target app to the foreground and focus its text input. Inspect the contents before deciding whether to retry; a delayed paste may still run."}],"isError":true,"id":"{{message:93}}"}},"sourceEventSeqs":[234],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":45}} {"type":"step/start","data":{"turn":1,"step":46}} -{"type":"assistant/message","data":{"turn":1,"step":46,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-consumed","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:94}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555404,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555404,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-consumed","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}}},{"type":"chunk","time":1791947555404,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555404,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":46,"callId":"cua-paste-consumed","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":46,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-consumed"},"toolCallId":"cua-paste-consumed","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:95}}"}},"sourceEventSeqs":[239],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":46,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-inspect","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:94}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690696,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690696,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-inspect","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947690696,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690696,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":46,"callId":"cua-paste-inspect","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":46,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-inspect"},"toolCallId":"cua-paste-inspect","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-12\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:95}}"}},"sourceEventSeqs":[239],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":46}} {"type":"step/start","data":{"turn":1,"step":47}} -{"type":"assistant/message","data":{"turn":1,"step":47,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:96}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555421,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555421,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947555421,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555421,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":47,"callId":"cua-paste-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":47,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-verify"},"toolCallId":"cua-paste-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-14\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"fixture paste\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:97}}"}},"sourceEventSeqs":[244],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":47,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:96}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690714,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690714,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1791947690714,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690714,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":47,"callId":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":47,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-activate"},"toolCallId":"cua-paste-activate","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false,\"foreground\":{\"requestId\":\"cua-paste-activate\",\"operation\":\"activate\",\"targetPid\":9,\"targetWindowId\":null,\"before\":{\"frontPid\":42,\"focusedWindowId\":7,\"frontWindowId\":99,\"frontWindowPid\":42},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":true,\"raiseRequested\":false}}"}],"isError":false,"id":"{{message:97}}"}},"sourceEventSeqs":[244],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":47}} {"type":"step/start","data":{"turn":1,"step":48}} -{"type":"assistant/message","data":{"turn":1,"step":48,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-debug-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F12\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:98}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555437,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555437,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-debug-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F12\"}}"}}},{"type":"chunk","time":1791947555437,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555437,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":48,"callId":"cua-debug-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F12\"}}"}} -{"type":"tool/result","data":{"turn":1,"step":48,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-debug-arm"},"toolCallId":"cua-debug-arm","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:99}}"}},"sourceEventSeqs":[249],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":48,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:98}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690734,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690734,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947690734,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690734,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":48,"callId":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":48,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-focus"},"toolCallId":"cua-paste-focus","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-13\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:99}}"}},"sourceEventSeqs":[249],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":48}} {"type":"step/start","data":{"turn":1,"step":49}} -{"type":"assistant/message","data":{"turn":1,"step":49,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-debug-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:100}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555453,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947555453,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-debug-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947555453,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555453,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":49,"callId":"cua-debug-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} -{"type":"tool/result","data":{"turn":1,"step":49,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-debug-screenshot"},"toolCallId":"cua-debug-screenshot","content":[{"type":"text","text":"Error: NATIVE_PROTOCOL: Helper returned an invalid response; acquire fresh application handles\nScreenshot diagnostics:\n[cua-screenshot] {\"requestId\":\"cua-debug-screenshot\",\"stage\":\"capture-image.begin\",\"elapsedMs\":1}"}],"isError":true,"id":"{{message:101}}"}},"sourceEventSeqs":[254],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":49,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-consumed","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:100}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690757,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690757,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-consumed","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}}},{"type":"chunk","time":1791947690757,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690757,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":49,"callId":"cua-paste-consumed","name":"cua","arguments":"{\"method\":\"paste\",\"params\":{\"appHandle\":\"fixture-app\",\"text\":\"fixture paste\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":49,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-consumed"},"toolCallId":"cua-paste-consumed","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:101}}"}},"sourceEventSeqs":[254],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":49}} {"type":"step/start","data":{"turn":1,"step":50}} -{"type":"assistant/message","data":{"turn":1,"step":50,"message":{"role":"assistant","content":[{"type":"text","text":"Observed Count: 4 after hovering, observing and clicking the revealed control. Moving away dismissed the hover information. Geometry conflicts rejected capture and coordinate gestures; the advertised AX button action remained available. No automatic activation or input retry was requested."}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:102}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947555473,"chunk":{"type":"block-start","index":0,"blockType":"text"}},{"type":"chunk","time":1791947555473,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"Observed Count: 4 after hovering, observing and clicking the revealed control. Moving away dismissed the hover information. Geometry conflicts rejected capture and coordinate gestures; the advertised AX button action remained available. No automatic activation or input retry was requested."}}},{"type":"chunk","time":1791947555473,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947555473,"chunk":{"type":"finish","reason":{"kind":"stop"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":50,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:102}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690778,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690778,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947690778,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690778,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":50,"callId":"cua-paste-verify","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":50,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-verify"},"toolCallId":"cua-paste-verify","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-14\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"fixture paste\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 4\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:103}}"}},"sourceEventSeqs":[259],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":50}} +{"type":"step/start","data":{"turn":1,"step":51}} +{"type":"assistant/message","data":{"turn":1,"step":51,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-debug-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F12\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:104}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690801,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690801,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-debug-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F12\"}}"}}},{"type":"chunk","time":1791947690801,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690801,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":51,"callId":"cua-debug-arm","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"F12\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":51,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-debug-arm"},"toolCallId":"cua-debug-arm","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:105}}"}},"sourceEventSeqs":[264],"surfaceOp":"append"} +{"type":"step/end","data":{"turn":1,"step":51}} +{"type":"step/start","data":{"turn":1,"step":52}} +{"type":"assistant/message","data":{"turn":1,"step":52,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-debug-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:106}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690821,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690821,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-debug-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947690821,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690821,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":52,"callId":"cua-debug-screenshot","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}} +{"type":"tool/result","data":{"turn":1,"step":52,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-debug-screenshot"},"toolCallId":"cua-debug-screenshot","content":[{"type":"text","text":"Error: NATIVE_PROTOCOL: Helper returned an invalid response; acquire fresh application handles\nScreenshot diagnostics:\n[cua-screenshot] {\"requestId\":\"cua-debug-screenshot\",\"stage\":\"capture-image.begin\",\"elapsedMs\":1}"}],"isError":true,"id":"{{message:107}}"}},"sourceEventSeqs":[269],"surfaceOp":"append"} +{"type":"step/end","data":{"turn":1,"step":52}} +{"type":"step/start","data":{"turn":1,"step":53}} +{"type":"assistant/message","data":{"turn":1,"step":53,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-consent-denied","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Denied Fixture\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:108}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690852,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690852,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-consent-denied","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Denied Fixture\"}}"}}},{"type":"chunk","time":1791947690852,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690852,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":53,"callId":"cua-consent-denied","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Denied Fixture\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":53,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-consent-denied"},"toolCallId":"cua-consent-denied","content":[{"type":"text","text":"Error: APP_ACCESS_DENIED: The user denied access to Denied Fixture. Do not retry or use another tool to bypass this decision."}],"isError":true,"id":"{{message:109}}"}},"sourceEventSeqs":[274],"surfaceOp":"append"} +{"type":"step/end","data":{"turn":1,"step":53}} +{"type":"step/start","data":{"turn":1,"step":54}} +{"type":"assistant/message","data":{"turn":1,"step":54,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-consent-revocable","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Revocable Fixture\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:110}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947691022,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947691022,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-consent-revocable","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Revocable Fixture\"}}"}}},{"type":"chunk","time":1791947691022,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947691022,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":54,"callId":"cua-consent-revocable","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"Revocable Fixture\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":54,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-consent-revocable"},"toolCallId":"cua-consent-revocable","content":[{"type":"text","text":"{\"appHandle\":\"revoked-app\",\"pid\":17,\"name\":\"Revocable Fixture\"}"}],"isError":false,"id":"{{message:111}}"}},"sourceEventSeqs":[279],"surfaceOp":"append"} +{"type":"step/end","data":{"turn":1,"step":54}} +{"type":"step/start","data":{"turn":1,"step":55}} +{"type":"assistant/message","data":{"turn":1,"step":55,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-consent-revoked","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"revoked-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:112}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947691192,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947691192,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-consent-revoked","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"revoked-app\"}}"}}},{"type":"chunk","time":1791947691192,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947691192,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":55,"callId":"cua-consent-revoked","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"revoked-app\"}}"}} +{"type":"tool/result","data":{"turn":1,"step":55,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-consent-revoked"},"toolCallId":"cua-consent-revoked","content":[{"type":"text","text":"Error: APP_ACCESS_REVOKED: Application access was revoked; stop using this application"}],"isError":true,"id":"{{message:113}}"}},"sourceEventSeqs":[284],"surfaceOp":"append"} +{"type":"step/end","data":{"turn":1,"step":55}} +{"type":"step/start","data":{"turn":1,"step":56}} +{"type":"assistant/message","data":{"turn":1,"step":56,"message":{"role":"assistant","content":[{"type":"text","text":"Observed Count: 4 after hovering, observing and clicking the revealed control. Moving away dismissed the hover information. Geometry conflicts rejected capture and coordinate gestures; the advertised AX button action remained available. No automatic activation or input retry was requested."}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:114}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947691212,"chunk":{"type":"block-start","index":0,"blockType":"text"}},{"type":"chunk","time":1791947691212,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"Observed Count: 4 after hovering, observing and clicking the revealed control. Moving away dismissed the hover information. Geometry conflicts rejected capture and coordinate gestures; the advertised AX button action remained available. No automatic activation or input retry was requested."}}},{"type":"chunk","time":1791947691212,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947691212,"chunk":{"type":"finish","reason":{"kind":"stop"}}}]},"surfaceOp":"append"} +{"type":"step/end","data":{"turn":1,"step":56}} {"type":"turn/end","data":{"turn":1,"reason":{"kind":"completed"}}} diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index fa48bb45ab..c43ddf6dd8 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -29,7 +29,7 @@ Use the workflow tool ONLY when the user explicitly asks for a workflow or for l Start independent delegations with `subagent` or `subagent_fork` together in one assistant message and continue useful work while they run. -Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. +Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user allows another request on the permission page. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again. @@ -66,7 +66,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to complete the Computer Use permission setup for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. diff --git a/snapshots/session/cua-native/workspace/native-helper.mjs b/snapshots/session/cua-native/workspace/native-helper.mjs index c4510416c3..2bbaa83188 100755 --- a/snapshots/session/cua-native/workspace/native-helper.mjs +++ b/snapshots/session/cua-native/workspace/native-helper.mjs @@ -125,6 +125,12 @@ function createApplicationState(socket, record, application, captureBroker, conn function refuse(code, message) { throw Object.assign(new Error(message), { code }) } function call(requestId, method, params) { record('call', { requestId, method, params }) + if (method === 'applicationIcons') return Object.fromEntries(params.paths.map(path => [path, `data:image/png;base64,${png}`])) + if (method === 'openPermissionSettings') return { opened: true } + if (method === 'permissionStatus' || method === 'requestPermission') { + const state = JSON.parse(readFileSync(new URL('./permissions.json', import.meta.url), 'utf8')) + return { ...state, helper: { pid: process.pid }, parent: { pid: process.ppid } } + } if (method === 'getCapabilities') return { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: true }, methods, coordinateSpace: 'window-points', browser: false } if (method === 'listApps') return [{ pid: app.pid, name: app.name, bundleId: app.bundleId }] if (method === 'getApp') { @@ -136,7 +142,7 @@ function createApplicationState(socket, record, application, captureBroker, conn } if (!['Fixture', 'test.fixture', '9'].includes(params.selector)) refuse('APP_NOT_FOUND', 'Select the running Fixture application') acquired = true - return { ...app, authorization: {"id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "name": "Fixture", "executablePath": "/fixture/app", "persistable": true} } + return { ...app, applicationIcon: `data:image/png;base64,${png}`, authorization: {"id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "name": "Fixture", "executablePath": "/fixture/app", "persistable": true} } } if (!acquired || params.appHandle !== app.appHandle) refuse('APP_NOT_FOUND', 'Acquire an application in this helper first') if (method === 'show') return { dispatched: true, verified: false, windowId: params.windowId ?? window.windowId, foregroundChanged: false } diff --git a/snapshots/session/cua-ptc/consent-fixture.mjs b/snapshots/session/cua-ptc/consent-fixture.mjs index 3e6dd4064c..85e83bcf69 100644 --- a/snapshots/session/cua-ptc/consent-fixture.mjs +++ b/snapshots/session/cua-ptc/consent-fixture.mjs @@ -15,6 +15,6 @@ export async function apply(ctx) { try { await close() } finally { await unlink(helper) } }) ctx.on('user-questions/request', async request => ({ answers: request.questions.map(question => ({ - id: question.id, selected: ['Allow for this task'], + id: question.id, selected: ['Allow for this session'], })) })) } diff --git a/snapshots/session/cua-ptc/cordis.snapshot.yml b/snapshots/session/cua-ptc/cordis.snapshot.yml index c2f9b23c4b..e42a3ce149 100644 --- a/snapshots/session/cua-ptc/cordis.snapshot.yml +++ b/snapshots/session/cua-ptc/cordis.snapshot.yml @@ -50,6 +50,7 @@ - id: cua-native name: '@deepseek-ai/dsh-experimental-cua-macos' config: + agentPreset: false helperPath: !!js process.cwd() + '/native-helper.mjs' - id: user-questions diff --git a/snapshots/session/cua-ptc/cordis.yml b/snapshots/session/cua-ptc/cordis.yml index 60a3be9c9e..ac932bb223 100644 --- a/snapshots/session/cua-ptc/cordis.yml +++ b/snapshots/session/cua-ptc/cordis.yml @@ -32,6 +32,7 @@ - id: cua-native name: '@deepseek-ai/dsh-experimental-cua-macos' config: + agentPreset: false helperPath: !!js process.cwd() + '/native-helper.mjs' - id: user-questions diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index 724b9bc2f3..1836fffaed 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -31,7 +31,7 @@ Use the workflow tool ONLY when the user explicitly asks for a workflow or for l Start independent delegations with `subagent` or `subagent_fork` together in one assistant message and continue useful work while they run. -Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. +Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user allows another request on the permission page. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again. @@ -68,7 +68,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to complete the Computer Use permission setup for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. diff --git a/tsconfig.base.json b/tsconfig.base.json index 2a92208c57..ab576e768b 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -302,6 +302,9 @@ "@deepseek-ai/dsh-experimental-client-ui-agent-team/client": ["./packages/experimental/client-ui-agent-team/src/client/index.ts"], "@deepseek-ai/dsh-client-ui-settings-plugins": ["./packages/client/ui-settings-plugins/src"], "@deepseek-ai/dsh-client-ui-settings-shell": ["./packages/client/ui-settings-shell/src"], + "@deepseek-ai/dsh-client-ui-cua": ["./packages/experimental/client-ui-cua/src/index.ts"], + "@deepseek-ai/dsh-client-ui-cua/client": ["./packages/experimental/client-ui-cua/src/client/index.ts"], + "@deepseek-ai/dsh-cua/types": ["./packages/experimental/cua/src/types.ts"], "@deepseek-ai/dsh-client-ui-settings-agent-loop": ["./packages/client/ui-settings-agent-loop/src"], "@deepseek-ai/dsh-client-ui-settings-session-log": ["./packages/client/ui-settings-session-log/src"], "@deepseek-ai/dsh-client-ui-settings-subagent": ["./packages/client/ui-settings-subagent/src"], @@ -398,6 +401,7 @@ "@deepseek-ai/dsh-config-editor": ["./packages/boot/config-editor/src"], "@deepseek-ai/dsh-credentials": ["./packages/credentials/credentials/src"], "@deepseek-ai/dsh-credentials-local": ["./packages/credentials/credentials-local/src"], + "@deepseek-ai/dsh-cua-bundle": ["./packages/experimental/cua-bundle/src"], "@deepseek-ai/dsh-deepseek-llm-api-extensions": ["./packages/llm/deepseek-llm-api-extensions/src"], "@deepseek-ai/dsh-deque": ["./packages/util/deque/src"], "@deepseek-ai/dsh-file-reference": ["./packages/context/file-reference/src"], diff --git a/tsconfig.client.json b/tsconfig.client.json index b2f0423c7c..7bf256becb 100644 --- a/tsconfig.client.json +++ b/tsconfig.client.json @@ -131,6 +131,7 @@ { "path": "./packages/client/ui-settings-plugins" }, { "path": "./packages/client/ui-settings-shell" }, { "path": "./packages/client/ui-settings-agent-loop" }, + { "path": "./packages/experimental/client-ui-cua" }, { "path": "./packages/client/ui-settings-session-log" }, { "path": "./packages/client/ui-settings-subagent" }, { "path": "./packages/client/ui-settings-web-search" }, diff --git a/tsconfig.host.json b/tsconfig.host.json index deca327660..ea1ffecb2a 100644 --- a/tsconfig.host.json +++ b/tsconfig.host.json @@ -74,6 +74,9 @@ "apps/web/tests/plugin-config.e2e.ts", "apps/web/tests/input-plugin-extensions.e2e.ts", "apps/web/tests/plugin-manager.e2e.ts", + "apps/web/tests/cua-optin.e2e.ts", + "apps/web/tests/bundle-platform.e2e.ts", + "apps/web/tests/cua-permissions.e2e.ts", "apps/web/tests/schedule-header.e2e.ts", "apps/web/tests/worktree-opt-in.e2e.ts", "apps/web/tests/session-inspector.e2e.ts", @@ -275,6 +278,7 @@ { "path": "./packages/experimental/computer-use-cua-driver-mcp" }, { "path": "./packages/experimental/computer-use-cua-driver-native" }, { "path": "./packages/experimental/cua" }, + { "path": "./packages/experimental/cua-bundle" }, { "path": "./packages/experimental/cua-macos" }, { "path": "./packages/client/ui-sidebar-documentpreview/tsconfig.host.json" }, { "path": "./packages/client/ui-sidebar-browser/tsconfig.host.json" }, diff --git a/vitest.config.ts b/vitest.config.ts index 9c717c0ced..7857266cd1 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -331,6 +331,9 @@ export default defineConfig({ // The speech entry also imports generated Remote definitions; voice-input.e2e.ts // exercises the built entry, while source tests cover mountVoiceInput. 'packages/experimental/client-ui-voice-input/src/client/index.ts', + // The permission entry binds generated Remote code; cua-permissions.e2e.ts covers + // its built entry, while source tests exercise mountCuaPermissions and its UI. + 'packages/experimental/client-ui-cua/src/client/index.ts', // The mods band entry imports the bridge's generated Remote contribution, which exists only in lib; // the Web snapshot exercises the built entry, while source tests cover mountModsBand. 'packages/experimental/client-ui-claude-code-mods/src/client/index.ts', From 8ad4e308a75fc7f590836b5e1cffc862975b4165 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 11:12:33 +0800 Subject: [PATCH 06/55] fix(cua): complete basic approval extraction --- .../experimental/client-ui-cua/tests/panel.client.spec.tsx | 4 +--- .../experimental/cua-macos/tests/provider-lifecycle.spec.ts | 1 + packages/experimental/cua/src/index.ts | 6 ++---- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx b/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx index 905321e79c..d6e833e2d5 100644 --- a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx +++ b/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx @@ -310,7 +310,7 @@ it('fills application names, system permissions, and icons as their observations it.each(['applications', 'icons'] as const)('contains a failed %s query in the application section', (key) => { const f = fixture() f.publish({ ...observe(value), [key]: { value: null, busy: false, error: 'offline' } }) - const view = render() + render() const requirements = screen.getByRole('region', { name: en.title }) const access = screen.getByRole('region', { name: en.applicationsTitle }) expect(within(requirements).getByText(en.granted)).toBeTruthy() @@ -318,8 +318,6 @@ it.each(['applications', 'icons'] as const)('contains a failed %s query in the a expect(within(access).getByRole('alert').textContent).toContain('offline') fireEvent.click(within(access).getByRole('button', { name: en.retry })) expect(f.props.refresh).toHaveBeenCalledTimes(2) - view.rerender() - expect(screen.queryByRole('alert')).toBeNull() }) it('finishes activation guidance without waiting for application icons', () => { diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index 06d3f8a846..efef014ef3 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -4,6 +4,7 @@ import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' import { setImmediate } from 'node:timers/promises' import { Context, FiberState } from '@deepseek-ai/cordis' +import Loader from '@deepseek-ai/cordis-plugin-loader' import { mountAgentLoopTestDependencies, mountAgentLoopTestHarness } from '@deepseek-ai/dsh-agent-loop-testkit' import ComputerUseRegistry from '@deepseek-ai/dsh-computer-use' import LocalAttachmentStore from '@deepseek-ai/dsh-attachment-local' diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index 6aa023962c..1e70320792 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -265,10 +265,8 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) : await requestNative() if (method === 'getApp') { const prepared = AppResult.extend({ pid: AppResult.shape.pid.nullable() }).parse(result) - const { authorization, applicationIcon } = z.object({ authorization: ApplicationAuthorizationResult, - applicationIcon: ApplicationIcon.optional(), - }).parse(result) - await access.select(agent, client, prepared.appHandle, authorization, signal, applicationIcon) + const { authorization } = z.object({ authorization: ApplicationAuthorizationResult }).parse(result) + await access.select(agent, client, prepared.appHandle, authorization, signal) signal.throwIfAborted() const selected = prepared.pid === null ? AppResult.parse(await access.run(client, prepared.appHandle, signal, From 9e0adb34ae73cd064d79ad03a89920312136c9f3 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 11:14:05 +0800 Subject: [PATCH 07/55] docs(cua): align basic configuration catalog --- docs/config-catalog.md | 2 +- docs/config-catalog.zh.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/config-catalog.md b/docs/config-catalog.md index da39766988..f5f4a22fbf 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -2482,7 +2482,7 @@ export interface PlanModeConfig { ## `@deepseek-ai/dsh-plugin-manager` - `inject`: `loader` · `profileContext` -- `source`: [`packages/boot/plugin-manager/src/index.ts:43`](../packages/boot/plugin-manager/src/index.ts) +- `source`: [`packages/boot/plugin-manager/src/index.ts:42`](../packages/boot/plugin-manager/src/index.ts) ```ts config-catalog /** The pnpm executable, registries, and limits for diagnostics, lookups and connection checks. */ diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index df608d7977..022ea67eb1 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -2484,7 +2484,7 @@ export interface PlanModeConfig { ## `@deepseek-ai/dsh-plugin-manager` - `inject`: `loader` · `profileContext` -- `source`: [`packages/boot/plugin-manager/src/index.ts:43`](../packages/boot/plugin-manager/src/index.ts) +- `source`: [`packages/boot/plugin-manager/src/index.ts:42`](../packages/boot/plugin-manager/src/index.ts) ```ts config-catalog /** The pnpm executable, registries, and limits for diagnostics, lookups and connection checks. */ From 8e726f5a39fbe8e80c58d5579a4e4168ae91784a Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 11:15:00 +0800 Subject: [PATCH 08/55] refactor(cua): retain existing experimental package classification --- scripts/verify-default-product-isolation.spec.ts | 11 +++++------ scripts/verify-default-product-isolation.ts | 3 +-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/scripts/verify-default-product-isolation.spec.ts b/scripts/verify-default-product-isolation.spec.ts index 9cd21e12d4..1c7f0a9229 100644 --- a/scripts/verify-default-product-isolation.spec.ts +++ b/scripts/verify-default-product-isolation.spec.ts @@ -113,16 +113,15 @@ describe('default product isolation', () => { expect(verifyDefaultProductIsolation(root).failures).toEqual([]) }) - it.each([[true, false], [false, false], [true, true], [false, true]])('ships an optional bundle switched off with legacy icon=%s and metadata=%s', (legacy, declared) => { + it.each([true, false])('ships an optional bundle switched off with legacy icon=%s', (legacy) => { const root = fixture() - const layer = declared ? '@deepseek-ai/dsh-prototype-layer' : '@deepseek-ai/dsh-experimental-layer' - const directory = declared ? 'packages/bundle/prototype-layer' : 'packages/experimental/layer' - write(root, `${directory}/package.json`, { + const layer = '@deepseek-ai/dsh-experimental-layer' + write(root, 'packages/experimental/layer/package.json', { name: layer, ...legacy ? { icon: './icon.svg' } : {}, exports: { ...legacy ? {} : { './icon': './icon.svg' }, './locale/*.json': './locale/*.json' }, - dependencies: { [experimental]: 'workspace:^' }, dsh: { bundle: { patch: './cordis.patch.yml', ...declared ? { experimental: true } : {} } }, + dependencies: { [experimental]: 'workspace:^' }, dsh: { bundle: { patch: './cordis.patch.yml' } }, }) - write(root, `${directory}/cordis.patch.yml`, [{ insert: [{ name: experimental }] }]) + write(root, 'packages/experimental/layer/cordis.patch.yml', [{ insert: [{ name: experimental }] }]) manifest(root, 'apps/cli/package.json', { dependencies: { [core]: 'workspace:^', [layer]: 'workspace:^' } }) write(root, profile, `export const PROFILE_TEMPLATES = { web: { bundles: ['${base}'] } }\n` + `export const DEFAULT_PROFILE_BUNDLES = ['${base}']\n` diff --git a/scripts/verify-default-product-isolation.ts b/scripts/verify-default-product-isolation.ts index 66e6c369d4..0120368412 100644 --- a/scripts/verify-default-product-isolation.ts +++ b/scripts/verify-default-product-isolation.ts @@ -107,8 +107,7 @@ export function verifyDefaultProductIsolation(root: string): ProductIsolationRes const sources = new Set() const configs = new Set() let webPluginCount = 0 - const isExperimental = (pkg: Package): boolean => pkg.manifest.dsh?.bundle?.experimental === true - || isExperimentalPackageName(pkg.manifest.name) + const isExperimental = (pkg: Package): boolean => isExperimentalPackageName(pkg.manifest.name) || display(pkg.directory).startsWith('packages/experimental/') const add = (pkg: Package, origin: string): void => { if (isExperimental(pkg)) { From 26282b91d4edb49356848b100281a5e0c971ab26 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 11:17:08 +0800 Subject: [PATCH 09/55] test(cua): match existing optional bundle presentation --- apps/web/tests/expected/plugin-config/official.expected.md | 2 +- apps/web/tests/expected/plugin-manager/live-enabled.expected.md | 2 +- apps/web/tests/expected/plugin-manager/manager.expected.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/web/tests/expected/plugin-config/official.expected.md b/apps/web/tests/expected/plugin-config/official.expected.md index 71c7f9218e..27c2e72887 100644 --- a/apps/web/tests/expected/plugin-config/official.expected.md +++ b/apps/web/tests/expected/plugin-config/official.expected.md @@ -40,7 +40,7 @@ - switch "启用 开发者工具" - listitem: - button "查看 Computer Use": Computer Use - - text: 实验性 目前仅支持 macOS，Windows 支持正在开发中。为已有及新建 PTC 会话加载 Computer Use。请切换到 PTC 模式来使用，并授予辅助功能和屏幕录制权限。 + - text: 目前仅支持 macOS，Windows 支持正在开发中。为已有及新建 PTC 会话加载 Computer Use。请切换到 PTC 模式来使用，并授予辅助功能和屏幕录制权限。 - switch "启用 Computer Use" - heading "已安装插件" [level=3] - list: diff --git a/apps/web/tests/expected/plugin-manager/live-enabled.expected.md b/apps/web/tests/expected/plugin-manager/live-enabled.expected.md index 7298240404..cfe553e2bb 100644 --- a/apps/web/tests/expected/plugin-manager/live-enabled.expected.md +++ b/apps/web/tests/expected/plugin-manager/live-enabled.expected.md @@ -40,7 +40,7 @@ - switch "启用 开发者工具" - listitem: - button "查看 Computer Use": Computer Use - - text: 实验性 目前仅支持 macOS，Windows 支持正在开发中。为已有及新建 PTC 会话加载 Computer Use。请切换到 PTC 模式来使用，并授予辅助功能和屏幕录制权限。 + - text: 目前仅支持 macOS，Windows 支持正在开发中。为已有及新建 PTC 会话加载 Computer Use。请切换到 PTC 模式来使用，并授予辅助功能和屏幕录制权限。 - switch "启用 Computer Use" - heading "已安装插件" [level=3] - list: diff --git a/apps/web/tests/expected/plugin-manager/manager.expected.md b/apps/web/tests/expected/plugin-manager/manager.expected.md index cb09a9e29e..d58ad62b03 100644 --- a/apps/web/tests/expected/plugin-manager/manager.expected.md +++ b/apps/web/tests/expected/plugin-manager/manager.expected.md @@ -40,7 +40,7 @@ - switch "启用 开发者工具" - listitem: - button "查看 Computer Use": Computer Use - - text: 实验性 目前仅支持 macOS，Windows 支持正在开发中。为已有及新建 PTC 会话加载 Computer Use。请切换到 PTC 模式来使用，并授予辅助功能和屏幕录制权限。 + - text: 目前仅支持 macOS，Windows 支持正在开发中。为已有及新建 PTC 会话加载 Computer Use。请切换到 PTC 模式来使用，并授予辅助功能和屏幕录制权限。 - switch "启用 Computer Use" - heading "已安装插件" [level=3] - list: From d4129cb83c8be000e6732536fd5bd72a80cc1235 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 11:17:44 +0800 Subject: [PATCH 10/55] test(cua): compare runtime identities without inspecting service proxies --- apps/web/tests/cua-optin.e2e.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/web/tests/cua-optin.e2e.ts b/apps/web/tests/cua-optin.e2e.ts index 4c954f1cbe..60bf3799a2 100644 --- a/apps/web/tests/cua-optin.e2e.ts +++ b/apps/web/tests/cua-optin.e2e.ts @@ -51,9 +51,9 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv await setup.getByRole('button', { name: '稍后', exact: true }).click() expect(selected()).toContain('@deepseek-ai/dsh-cua-bundle') expect(await readFile(settingsPath, 'utf8')).toBe(before) - expect(scaffold.ctx.ptcRuntime).toBe(originalRuntime) + expect(scaffold.ctx.ptcRuntime === originalRuntime).toBe(true) expect(scaffold.ctx.agentPresets.defaultId).toBe(initialDefault) - expect(scaffold.ctx.agents.get(originalAgent.id)).toBe(originalAgent) + expect(scaffold.ctx.agents.get(originalAgent.id) === originalAgent).toBe(true) expect(scaffold.ctx.tools.schemas(originalAgent).some(tool => tool.name === 'cua')).toBe(false) expect(scaffold.ctx.tools.schemas(ptcAgent).some(tool => tool.name === 'cua')).toBe(true) expect((await scaffold.ctx.systemPrompt.assemble({ scope: ptcAgent })).tools.map(tool => tool.name)).toEqual(['run_code']) @@ -65,7 +65,7 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv await expect.poll(() => toggle.getAttribute('aria-checked')).toBe('false') expect(scaffold.ctx.get('computerUse')).toBeUndefined() for (const agent of [ptcAgent, fresh.agent]) expect(scaffold.ctx.tools.schemas(agent).some(tool => tool.name === 'cua')).toBe(false) - expect(scaffold.ctx.ptcRuntime).toBe(originalRuntime) + expect(scaffold.ctx.ptcRuntime === originalRuntime).toBe(true) expect(await readFile(settingsPath, 'utf8')).toBe(before) await card.getByRole('button', { name: '查看 Computer Use', exact: true }).click() const detailToggle = page.locator('[data-plugin-panel]').getByRole('switch', { name: '启用 Computer Use', exact: true }) @@ -78,7 +78,7 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv await defaultSetting.getByRole('button', { name: '设为 PTC', exact: true }).click() await expect.poll(() => scaffold.ctx.agentPresets.defaultId).toBe('ptc') expect(scaffold.ctx.agentPresets.composedPreset(originalAgent.ctx)).toBe(initialDefault) - expect(scaffold.ctx.ptcRuntime).toBe(originalRuntime) + expect(scaffold.ctx.ptcRuntime === originalRuntime).toBe(true) const confirmed = await scaffold.ctx.agents.create({ sessionId: SessionId('cua-confirmed-default'), setup: async (ctx) => { await scaffold.ctx.agentPresets.mount(ctx) }, }) @@ -89,7 +89,7 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv await detailToggle.click() await expect.poll(() => detailToggle.getAttribute('aria-checked')).toBe('false') expect(await readFile(settingsPath, 'utf8')).toBe(saved) - expect(scaffold.ctx.ptcRuntime).toBe(originalRuntime) + expect(scaffold.ctx.ptcRuntime === originalRuntime).toBe(true) } finally { try { await browser?.close() } finally { try { await ownedScaffold?.close() } finally { await rm(directory, { recursive: true, force: true }) } From ff2c6bb6be07d4820a4ef4e8894e8f31290320a4 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 11:20:28 +0800 Subject: [PATCH 11/55] test(cua): observe the PTC provider lifecycle directly --- apps/web/tests/cua-optin.e2e.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/apps/web/tests/cua-optin.e2e.ts b/apps/web/tests/cua-optin.e2e.ts index 60bf3799a2..97fd6f763e 100644 --- a/apps/web/tests/cua-optin.e2e.ts +++ b/apps/web/tests/cua-optin.e2e.ts @@ -33,8 +33,10 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv const ptcAgent = (await scaffold.ctx.agents.create({ sessionId: SessionId('cua-existing-ptc'), setup: async (ctx) => { await scaffold.ctx.agentPresets.mount(ctx, 'ptc') }, })).agent - const originalRuntime = scaffold.ctx.ptcRuntime - expect(originalRuntime.programFormat).toBe('function-body') + const runtimeFiber = () => [...scaffold.ctx.loader.entries()].find(entry => entry.options.id === 'ptc-runtime')?.fiber + const originalRuntimeFiber = runtimeFiber() + expect(originalRuntimeFiber).toBeDefined() + expect(scaffold.ctx.ptcRuntime.programFormat).toBe('function-body') const settingsPath = join(scaffold.harnessHome, 'profiles/scaffold/cordis.patch.yml') const before = await readFile(settingsPath, 'utf8') const selected = () => readProfileManifest('scaffold', join(scaffold.harnessHome, 'profiles/scaffold')).dsh?.profile?.bundles @@ -51,7 +53,8 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv await setup.getByRole('button', { name: '稍后', exact: true }).click() expect(selected()).toContain('@deepseek-ai/dsh-cua-bundle') expect(await readFile(settingsPath, 'utf8')).toBe(before) - expect(scaffold.ctx.ptcRuntime === originalRuntime).toBe(true) + expect(runtimeFiber() === originalRuntimeFiber).toBe(true) + expect(scaffold.ctx.ptcRuntime.programFormat).toBe('function-body') expect(scaffold.ctx.agentPresets.defaultId).toBe(initialDefault) expect(scaffold.ctx.agents.get(originalAgent.id) === originalAgent).toBe(true) expect(scaffold.ctx.tools.schemas(originalAgent).some(tool => tool.name === 'cua')).toBe(false) @@ -65,7 +68,8 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv await expect.poll(() => toggle.getAttribute('aria-checked')).toBe('false') expect(scaffold.ctx.get('computerUse')).toBeUndefined() for (const agent of [ptcAgent, fresh.agent]) expect(scaffold.ctx.tools.schemas(agent).some(tool => tool.name === 'cua')).toBe(false) - expect(scaffold.ctx.ptcRuntime === originalRuntime).toBe(true) + expect(runtimeFiber() === originalRuntimeFiber).toBe(true) + expect(scaffold.ctx.ptcRuntime.programFormat).toBe('function-body') expect(await readFile(settingsPath, 'utf8')).toBe(before) await card.getByRole('button', { name: '查看 Computer Use', exact: true }).click() const detailToggle = page.locator('[data-plugin-panel]').getByRole('switch', { name: '启用 Computer Use', exact: true }) @@ -78,7 +82,8 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv await defaultSetting.getByRole('button', { name: '设为 PTC', exact: true }).click() await expect.poll(() => scaffold.ctx.agentPresets.defaultId).toBe('ptc') expect(scaffold.ctx.agentPresets.composedPreset(originalAgent.ctx)).toBe(initialDefault) - expect(scaffold.ctx.ptcRuntime === originalRuntime).toBe(true) + expect(runtimeFiber() === originalRuntimeFiber).toBe(true) + expect(scaffold.ctx.ptcRuntime.programFormat).toBe('function-body') const confirmed = await scaffold.ctx.agents.create({ sessionId: SessionId('cua-confirmed-default'), setup: async (ctx) => { await scaffold.ctx.agentPresets.mount(ctx) }, }) @@ -89,7 +94,8 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv await detailToggle.click() await expect.poll(() => detailToggle.getAttribute('aria-checked')).toBe('false') expect(await readFile(settingsPath, 'utf8')).toBe(saved) - expect(scaffold.ctx.ptcRuntime === originalRuntime).toBe(true) + expect(runtimeFiber() === originalRuntimeFiber).toBe(true) + expect(scaffold.ctx.ptcRuntime.programFormat).toBe('function-body') } finally { try { await browser?.close() } finally { try { await ownedScaffold?.close() } finally { await rm(directory, { recursive: true, force: true }) } From ef15f31682dcd02eb4e528c6fd1cc045b93365ac Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 11:30:56 +0800 Subject: [PATCH 12/55] test(cua): include the configured click hold guidance --- packages/experimental/cua/tests/agent-scopes.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/experimental/cua/tests/agent-scopes.spec.ts b/packages/experimental/cua/tests/agent-scopes.spec.ts index c561d11a6e..26568bca49 100644 --- a/packages/experimental/cua/tests/agent-scopes.spec.ts +++ b/packages/experimental/cua/tests/agent-scopes.spec.ts @@ -53,7 +53,7 @@ it('adds and removes CUA for existing and new PTC Agents without changing other await expect.poll(() => ctx.tools.get('cua', existing.agent)).toBeDefined() expect(ctx.tools.get('cua', standard.agent)).toBeUndefined() expect((await ctx.systemPrompt.assemble({ scope: existing.agent })).sections) - .toContainEqual(expect.objectContaining({ name: 'computer-use:dsh-cua', text: 'Fixture CUA guidance' })) + .toContainEqual(expect.objectContaining({ name: 'computer-use:dsh-cua', text: 'Fixture CUA guidance\n\nMaximum click holdMs: 5000 ms.' })) await ctx.agentPresets.select(existing.agent, 'ptc') expect(ctx.tools.schemas(existing.agent).filter(tool => tool.name === 'cua')).toHaveLength(1) const fresh = await create('fresh', 'ptc') From f722f6e376c5a5ac09da3e04424b2434c35389a2 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 11:31:23 +0800 Subject: [PATCH 13/55] test(cua): supply click hold bounds in permission fixtures --- packages/experimental/cua/tests/access.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/experimental/cua/tests/access.spec.ts b/packages/experimental/cua/tests/access.spec.ts index 473ba6acaa..84289fb246 100644 --- a/packages/experimental/cua/tests/access.spec.ts +++ b/packages/experimental/cua/tests/access.spec.ts @@ -31,7 +31,7 @@ vi.mock('../src/native-client.ts', async original => ({ const unusedConnector = () => { throw new Error('Mocked native client does not connect') } const nativeConfig = { helperPath: '/fixture', cwd: '/', timeoutMs: 1000, connectTimeoutMs: 5000, screenshotTimeoutMs: 5000, axTimeoutMs: 100, - maxRequestBytes: 4096, maxResponseBytes: 4096, graceMs: 100, textEventIntervalMs: 1 } + maxRequestBytes: 4096, maxResponseBytes: 4096, graceMs: 100, textEventIntervalMs: 1, maxClickHoldMs: 5000 } const identity = ApplicationAuthorizationResult.parse({ id: 'a'.repeat(64), name: 'Test application', executablePath: '/Applications/Fixture.app/Contents/MacOS/Fixture', persistable: true }) const handle = 'fixture-app' as CuaAppHandle const signal = new AbortController().signal From 0a21831883fabf21ddf1fbf62b782f489929a9bd Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 11:34:08 +0800 Subject: [PATCH 14/55] test(cua): retain permission scenarios in click hold replay --- snapshots/session/cua-native/session.v4.jsonl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/snapshots/session/cua-native/session.v4.jsonl b/snapshots/session/cua-native/session.v4.jsonl index 2b978c0888..e7587237ab 100644 --- a/snapshots/session/cua-native/session.v4.jsonl +++ b/snapshots/session/cua-native/session.v4.jsonl @@ -42,8 +42,8 @@ {"type":"tool/result","data":{"turn":1,"step":6,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-3"},"toolCallId":"cua-native-3","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-1\",\"tree\":\"AX scope=window offsets=UTF-16 bounds=global-points\\n- [0] AXButton \\\"Increment\\\" id=\\\"increment\\\" actions=[\\\"AXPress\\\"] bounds=[60,80,100,20]\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 0\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\n - [3] AXMenuItem \\\"Fixture command\\\"\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:15}}"}},"sourceEventSeqs":[39],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":6}} {"type":"step/start","data":{"turn":1,"step":7}} -{"type":"assistant/message","data":{"turn":1,"step":7,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:16}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689683,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689683,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}}},{"type":"chunk","time":1791947689683,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689683,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":7,"callId":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}} +{"type":"assistant/message","data":{"turn":1,"step":7,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0,\"holdMs\":800}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:16}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689683,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689683,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0,\"holdMs\":800}}"}}},{"type":"chunk","time":1791947689683,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689683,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":7,"callId":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0,\"holdMs\":800}}"}} {"type":"tool/result","data":{"turn":1,"step":7,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-4"},"toolCallId":"cua-native-4","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:17}}"}},"sourceEventSeqs":[44],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":7}} {"type":"step/start","data":{"turn":1,"step":8}} @@ -112,8 +112,8 @@ {"type":"tool/result","data":{"turn":1,"step":20,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-stale-index"},"toolCallId":"cua-native-stale-index","content":[{"type":"text","text":"Error: STALE_SNAPSHOT: Read fresh state before acting"}],"isError":true,"id":"{{message:43}}"}},"sourceEventSeqs":[109],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":20}} {"type":"step/start","data":{"turn":1,"step":21}} -{"type":"assistant/message","data":{"turn":1,"step":21,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:44}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690064,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690064,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0}}"}}},{"type":"chunk","time":1791947690064,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690064,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":21,"callId":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0}}"}} +{"type":"assistant/message","data":{"turn":1,"step":21,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0,\"holdMs\":800}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:44}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690064,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690064,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0,\"holdMs\":800}}"}}},{"type":"chunk","time":1791947690064,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690064,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":21,"callId":"cua-native-other-window","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"x\":0,\"y\":0,\"holdMs\":800}}"}} {"type":"tool/result","data":{"turn":1,"step":21,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-other-window"},"toolCallId":"cua-native-other-window","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:45}}"}},"sourceEventSeqs":[114],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":21}} {"type":"step/start","data":{"turn":1,"step":22}} From ba96a9aaea88820e3d7f60e4915b5e1fca11cda4 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:04:57 +0800 Subject: [PATCH 15/55] test(cua): restore consent coverage and basic preview expectations --- apps/web/tests/preview-boot.e2e.ts | 14 ++-- .../tests/provider-lifecycle.spec.ts | 83 ++++++++++++++++++- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/apps/web/tests/preview-boot.e2e.ts b/apps/web/tests/preview-boot.e2e.ts index a6d1faaadb..cb81357a42 100644 --- a/apps/web/tests/preview-boot.e2e.ts +++ b/apps/web/tests/preview-boot.e2e.ts @@ -682,17 +682,19 @@ async function toggleOptionalBundles(page: Page): Promise { }) expect(result.optional.map(entry => entry.name).sort()).toEqual([...OPTIONAL_BUNDLES].sort()) expect(result.optional.filter(entry => entry.enabled || entry.availability !== 'installation')).toEqual([]) - expect(result.optional.filter(entry => entry.error !== null)).toEqual([ - expect.objectContaining({ name: '@deepseek-ai/dsh-cua-bundle', error: { code: 'unsupported-platform' } }), - ]) + expect(result.optional.filter(entry => entry.error !== null)).toEqual([]) // Metadata resolves through the Worker's Node internal modules: every optional bundle shows its title and icon. expect(result.optional.filter(entry => entry.metaError !== null || entry.title !== 'object' || entry.icon !== 'string')).toEqual([]) expect(result.inventoryErrors).toEqual([]) console.info(`preview boot: optional bundle toggles ${JSON.stringify(result.toggles.map(({ name, enable, disable, error }) => ({ name, enable, disable, error })))}`) - expect(result.toggles.find(toggle => toggle.name === '@deepseek-ai/dsh-cua-bundle')).toMatchObject({ - enable: 'failed', disable: 'applied', error: { code: 'unsupported-platform' }, - changed: 0, inactive: [], restored: true, + const cua = result.toggles.find(toggle => toggle.name === '@deepseek-ai/dsh-cua-bundle')! + expect(cua).toMatchObject({ + enable: 'failed', disable: 'applied', + error: { code: 'operation-error', diagnostic: expect.stringContaining('Computer Use currently requires macOS') }, + restored: true, }) + expect(cua.changed).toBeGreaterThan(0) + expect(cua.inactive).toContain('@deepseek-ai/dsh-experimental-cua-macos true failed') const supported = result.toggles.filter(toggle => toggle.name !== '@deepseek-ai/dsh-cua-bundle') expect(supported.filter(toggle => toggle.enable !== 'applied' || toggle.disable !== 'applied') .map(({ name, enable, disable }) => ({ name, enable, disable }))).toEqual([]) diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index 5c5d68c908..9f3c57851b 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -1,10 +1,16 @@ /** Native helper replacement is synchronized with Agent and provider disposal. */ -import { mkdtemp, rm } from 'node:fs/promises' +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' import { setImmediate } from 'node:timers/promises' import { Context, FiberState } from '@deepseek-ai/cordis' import Loader from '@deepseek-ai/cordis-plugin-loader' +import Timer from '@deepseek-ai/cordis-plugin-timer' +import { initProfile, mountRootInclude, readProfilePatches, type ProfileContext } from '@deepseek-ai/dsh-app-boot' +import ConfigEditor from '@deepseek-ai/dsh-config-editor' +import Settings from '@deepseek-ai/dsh-settings' +import Hmr from '@deepseek-ai/dsh-hmr' import { mountAgentLoopTestDependencies, mountAgentLoopTestHarness } from '@deepseek-ai/dsh-agent-loop-testkit' import ComputerUseRegistry from '@deepseek-ai/dsh-computer-use' import LocalAttachmentStore from '@deepseek-ai/dsh-attachment-local' @@ -645,3 +651,78 @@ it('rejects an unsupported native host before starting preparation', async () => await expect(NativeProvider.apply(f.ctx, { ...automaticConfig, allowedApps: { get: () => [] } })).rejects.toThrow('unsupported host') expect(native.prepare).not.toHaveBeenCalled() }) + +it('stores signed application consent in the current profile and refuses unavailable settings', async () => { + const f = await setup() + await f.provider.dispose() + const home = await mkdtemp(join(tmpdir(), 'cua-grant-profile-')), dir = join(home, 'profile') + onTestFinished(async () => { await f.ctx.fiber.dispose(); await rm(home, { recursive: true, force: true }) }) + initProfile(dir, ['permission-fixture-bundle']) + const bundle = join(dir, 'node_modules', 'permission-fixture-bundle') + await mkdir(bundle, { recursive: true }) + await writeFile(join(home, 'package.json'), '{"name":"test-installation"}\n') + await writeFile(join(bundle, 'package.json'), JSON.stringify({ name: 'permission-fixture-bundle', version: '1.0.0', + dsh: { bundle: { patch: 'cordis.patch.yml' } } })) + await writeFile(join(bundle, 'cordis.patch.yml'), JSON.stringify([{ insert: [{ id: 'cua', name: 'cordis:cua', config }] }])) + await writeFile(join(dir, 'cordis.yml'), '[]\n') + const profile: ProfileContext = { name: 'test', startedBundles: ['permission-fixture-bundle'], dir, + patchPath: join(dir, 'cordis.patch.yml'), installAnchor: join(home, 'package.json'), cwd: home, home, + overlays: [], telemetryDisabledEnv: undefined } + f.ctx.baseUrl = pathToFileURL(home).href + '/' + f.ctx.provide('profileContext', profile) + f.ctx.provide('appReady', { onReady: (listener: () => void) => { listener(); return () => {} } }) + await f.ctx.plugin(Loader) + f.ctx.loader.builtins.cua = NativeProvider + await f.ctx.plugin(ConfigEditor) + await mountRootInclude(f.ctx, join(dir, 'cordis.yml'), readProfilePatches('test', profile)) + await f.ctx.loader.await() + await f.ctx.plugin(UserQuestions) + const answer = vi.fn(async () => ({ answers: [{ id: 'cua-app-access', selected: ['Allow for this session'] }] })) + f.ctx.on('user-questions/request', answer) + native.request.mockResolvedValue({ appHandle: 'task-app', pid: 17, name: 'Fixture', + authorization: { id: 'a'.repeat(64), name: 'Fixture', executablePath: '/fixture/app', persistable: true } }) + expect((await f.execute('getApp', { selector: 'Fixture' })).isError).toBe(false) + await f.ctx.plugin(Settings) + await f.ctx.plugin(Timer) + await f.ctx.plugin(Hmr, { root: [], ignored: [], debounce: 0 }) + await f.ctx.hmr.runExclusive(async () => {}) + expect(f.ctx.settings.describe().find(row => row.ns === 'cua')?.autoGenerate).toBe(false) + answer.mockResolvedValue({ answers: [{ id: 'cua-app-access', selected: ['Always allow'] }] }) + const authorization = { id: 'b'.repeat(64), name: 'Fixture', executablePath: '/fixture/app', persistable: true } + native.request.mockResolvedValue({ appHandle: 'signed-app', pid: 17, name: 'Fixture', authorization }) + expect((await f.execute('getApp', { selector: 'Fixture' })).isError).toBe(false) + expect(await readFile(profile.patchPath, 'utf8')).toContain(authorization.id) + const settings = f.ctx.settings.describe().find(row => row.ns === 'cua')! + await f.ctx.settings.mutate('cua', [{ op: 'set', path: ['allowedApps'], value: [] }], settings.revision) + expect(await readFile(profile.patchPath, 'utf8')).toContain('allowedApps: []') + const revoked = await f.execute('getState', { appHandle: 'signed-app' }) + expect(revoked.isError).toBe(true) + expect(JSON.stringify(revoked.content)).toContain('APP_ACCESS_REVOKED') + vi.spyOn(f.ctx.settings, 'describe').mockReturnValue([]) + native.request.mockResolvedValue({ appHandle: 'different-app', pid: 18, name: 'Other fixture', authorization: { ...authorization, id: 'c'.repeat(64) } }) + const unavailable = await f.execute('getApp', { selector: 'Fixture' }) + expect(unavailable.isError).toBe(true) + expect(JSON.stringify(unavailable.content)).toContain('APP_ACCESS_DENIED') +}) + +it('rejects an application handle before any helper or consent has been acquired', async () => { + const f = await setup() + const result = await f.execute('getState', { appHandle: 'unselected-app' }) + expect(result.isError).toBe(true) + expect(JSON.stringify(result.content)).toContain('APP_ACCESS_REQUIRED') + expect(native.instances).toHaveLength(0) + expect(native.request).not.toHaveBeenCalled() +}) + +it('uses configured signed application grants without requesting a human answer', async () => { + const f = await setup() + await f.provider.dispose() + const grant = { id: 'a'.repeat(64), name: 'Fixture', executablePath: '/fixture/app' } + await f.ctx.plugin(NativeProvider, { ...config, allowedApps: [grant] }) + native.request.mockResolvedValueOnce({ appHandle: 'saved-app', pid: 17, name: 'Fixture', authorization: { ...grant, persistable: true } }) + expect((await f.execute('getApp', { selector: 'Fixture' })).isError).toBe(false) + native.request.mockResolvedValue({ dispatched: true, verified: false }) + expect((await f.execute('activate', { appHandle: 'saved-app' })).isError).toBe(false) + expect(native.instances).toHaveLength(1) + expect(native.request).toHaveBeenCalledTimes(2) +}) From 029f3d7a438488998b4aca2578501d102e2275e9 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:12:35 +0800 Subject: [PATCH 16/55] test(cua): avoid unsafe matcher assignment in preview assertions --- apps/web/tests/preview-boot.e2e.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/web/tests/preview-boot.e2e.ts b/apps/web/tests/preview-boot.e2e.ts index cb81357a42..b96dbd45a2 100644 --- a/apps/web/tests/preview-boot.e2e.ts +++ b/apps/web/tests/preview-boot.e2e.ts @@ -690,9 +690,10 @@ async function toggleOptionalBundles(page: Page): Promise { const cua = result.toggles.find(toggle => toggle.name === '@deepseek-ai/dsh-cua-bundle')! expect(cua).toMatchObject({ enable: 'failed', disable: 'applied', - error: { code: 'operation-error', diagnostic: expect.stringContaining('Computer Use currently requires macOS') }, + error: { code: 'operation-error' }, restored: true, }) + expect(cua.error).toHaveProperty('diagnostic', expect.stringContaining('Computer Use currently requires macOS')) expect(cua.changed).toBeGreaterThan(0) expect(cua.inactive).toContain('@deepseek-ai/dsh-experimental-cua-macos true failed') const supported = result.toggles.filter(toggle => toggle.name !== '@deepseek-ai/dsh-cua-bundle') From 373080de5e80c26a762d8fe3244958ae139c252b Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:48:58 +0800 Subject: [PATCH 17/55] test(cua): remove unrelated split remnants --- apps/web/tsconfig.json | 1 - .../tests/retired-content-admission.spec.ts | 13 ++----------- tsconfig.host.json | 1 - 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json index 666db22e82..9d25a02899 100644 --- a/apps/web/tsconfig.json +++ b/apps/web/tsconfig.json @@ -83,7 +83,6 @@ "tests/input-plugin-extensions.e2e.ts", "tests/plugin-manager.e2e.ts", "tests/cua-optin.e2e.ts", - "tests/bundle-platform.e2e.ts", "tests/cua-permissions.e2e.ts", "tests/schedule-header.e2e.ts", "tests/worktree-opt-in.e2e.ts", diff --git a/packages/session/session-persistence-jsonl/tests/retired-content-admission.spec.ts b/packages/session/session-persistence-jsonl/tests/retired-content-admission.spec.ts index ddf49e64e4..1d10cb2fde 100644 --- a/packages/session/session-persistence-jsonl/tests/retired-content-admission.spec.ts +++ b/packages/session/session-persistence-jsonl/tests/retired-content-admission.spec.ts @@ -1,13 +1,13 @@ /** Interpreted retired content must fail before migration publication or recoverable native-tail suppression. */ import { Context } from '@deepseek-ai/cordis' -import { SESSION_FORMAT_VERSION, SessionId } from '@deepseek-ai/dsh-session' +import { SessionId } from '@deepseek-ai/dsh-session' import { SessionFormatUnsupportedError, SessionPersistenceCorruptionError } from '@deepseek-ai/dsh-session-persistence' import * as JsonlSessionPersistence from '@deepseek-ai/dsh-session-persistence-jsonl' import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { basename, dirname, join } from 'node:path' import { describe, expect, it, onTestFinished } from 'vitest' -import { generationLogPath, SessionLogScanner } from '../src/format.ts' +import { generationLogPath } from '../src/format.ts' import { compressZstdFrame } from '../src/zstd.ts' const id = SessionId('retired-content-admission') @@ -55,12 +55,3 @@ describe.each(modes)('retired content in JSONL ($compression, $access)', ({ comp expect(await readFile(path)).toEqual(bytes) }) }) - -it.each(['strict', 'recoverable'] as const)('classifies retired native PTC tags as unsupported during %s scanning', (recovery) => { - const header = { type: 'session', version: SESSION_FORMAT_VERSION, id, createdAt: 1, isSeeded: false, delegationDepth: 0 } - for (const type of ['tool/code-dispatch-start', 'tool/code-dispatch']) { - const scanner = new SessionLogScanner(Buffer.from(JSON.stringify(header) + '\n'), recovery) - const row = { type, seq: 0, time: 1, data: {} } - expect(() => { scanner.write(Buffer.from(JSON.stringify(row) + '\n')) }).toThrow(SessionFormatUnsupportedError) - } -}) diff --git a/tsconfig.host.json b/tsconfig.host.json index ea1ffecb2a..a267f50f63 100644 --- a/tsconfig.host.json +++ b/tsconfig.host.json @@ -75,7 +75,6 @@ "apps/web/tests/input-plugin-extensions.e2e.ts", "apps/web/tests/plugin-manager.e2e.ts", "apps/web/tests/cua-optin.e2e.ts", - "apps/web/tests/bundle-platform.e2e.ts", "apps/web/tests/cua-permissions.e2e.ts", "apps/web/tests/schedule-header.e2e.ts", "apps/web/tests/worktree-opt-in.e2e.ts", From a1c0a68c80268677a95d9f15a031507289547b1b Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:49:15 +0800 Subject: [PATCH 18/55] docs(cua): place preset migration under the current release --- .../cua-preset-scope/guide.i18n.yaml | 6 +++--- .../cua-preset-scope/guide.md | 0 .../cua-preset-scope/guide.zh.md | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename docs/upgrade-guide/{v0.2.1-alpha.2 => v0.2.1-rc.2}/cua-preset-scope/guide.i18n.yaml (86%) rename docs/upgrade-guide/{v0.2.1-alpha.2 => v0.2.1-rc.2}/cua-preset-scope/guide.md (100%) rename docs/upgrade-guide/{v0.2.1-alpha.2 => v0.2.1-rc.2}/cua-preset-scope/guide.zh.md (100%) diff --git a/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.i18n.yaml b/docs/upgrade-guide/v0.2.1-rc.2/cua-preset-scope/guide.i18n.yaml similarity index 86% rename from docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.i18n.yaml rename to docs/upgrade-guide/v0.2.1-rc.2/cua-preset-scope/guide.i18n.yaml index bb91afebfd..7313c9c61a 100644 --- a/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.i18n.yaml +++ b/docs/upgrade-guide/v0.2.1-rc.2/cua-preset-scope/guide.i18n.yaml @@ -1,13 +1,13 @@ # Bilingual-pair consistency record for guide.md (docs/i18n/README.md): per heading # section, a hash of its English and Chinese blocks outside code blocks and generated regions. # After editing either side, bring the other along and re-record with: -# pnpm run verify-translation-pairing --write docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.md +# pnpm run verify-translation-pairing --write docs/upgrade-guide/v0.2.1-rc.2/cua-preset-scope/guide.md /: en: 36d6b79b48c08dd5 zh: 814caece159e9b0e /cua-defaults-to-ptc-session-scopes: - en: 1d6eee8020ebed3b - zh: d14279f527983c3c + en: a53b6884785bdf56 + zh: 7d27fc9b029e0e3e /cua-defaults-to-ptc-session-scopes/change: en: dac96a29e3751249 zh: d48cb030f43c07e4 diff --git a/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.md b/docs/upgrade-guide/v0.2.1-rc.2/cua-preset-scope/guide.md similarity index 100% rename from docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.md rename to docs/upgrade-guide/v0.2.1-rc.2/cua-preset-scope/guide.md diff --git a/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.zh.md b/docs/upgrade-guide/v0.2.1-rc.2/cua-preset-scope/guide.zh.md similarity index 100% rename from docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.zh.md rename to docs/upgrade-guide/v0.2.1-rc.2/cua-preset-scope/guide.zh.md From a2484cb02d1685e37d6fa2fab1282527bca235f6 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:49:53 +0800 Subject: [PATCH 19/55] fix(cua): keep permission guidance independent of the UI --- packages/experimental/cua-macos/README.md | 4 ++-- packages/experimental/cua-macos/README.zh.md | 4 ++-- packages/experimental/cua-macos/src/guidance.ts | 4 ++-- snapshots/session/cua-native/system-prompt.expected.md | 4 ++-- snapshots/session/cua-ptc/system-prompt.expected.md | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 3909b8f97a..960177c233 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -152,7 +152,7 @@ The plugin contributes one `cua` operation tool with method-specific parameters. ##### Operation guidance (verbatim) ```markdown -Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user allows another request on the permission page. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. +Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly permits another authorization request. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again. @@ -189,7 +189,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to complete the Computer Use permission setup for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant the required macOS system permissions for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index b01c0488d7..dab1ca6499 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -152,7 +152,7 @@ AppKit sheet 可以通过其窗口 ID 单独选择，也会出现在所属文档 ##### 操作指导文本（原文） ```markdown -Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user allows another request on the permission page. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. +Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly permits another authorization request. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again. @@ -189,7 +189,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to complete the Computer Use permission setup for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant the required macOS system permissions for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index 6de6ce0565..0147beefbe 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -1,6 +1,6 @@ /** macOS operation guidance supplied with the shared CUA object API. */ /** Instructions for raw operation calls and optional object-API bindings. */ -export const GUIDANCE = `Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user allows another request on the permission page. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. +export const GUIDANCE = `Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly permits another authorization request. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again. @@ -37,7 +37,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to complete the Computer Use permission setup for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant the required macOS system permissions for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index 82bccf01cb..d2daf65630 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -29,7 +29,7 @@ Use the workflow tool ONLY when the user explicitly asks for a workflow or for l Start independent delegations with `subagent` or `subagent_fork` together in one assistant message and continue useful work while they run. -Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user allows another request on the permission page. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. +Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly permits another authorization request. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again. @@ -66,7 +66,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to complete the Computer Use permission setup for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant the required macOS system permissions for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index 3c9c10466b..cf5222fd7f 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -31,7 +31,7 @@ Use the workflow tool ONLY when the user explicitly asks for a workflow or for l Start independent delegations with `subagent` or `subagent_fork` together in one assistant message and continue useful work while they run. -Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user allows another request on the permission page. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. +Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly permits another authorization request. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again. @@ -68,7 +68,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to complete the Computer Use permission setup for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant the required macOS system permissions for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. From eaea1ec5b9eb12945329f1b3191b956292cd12e7 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:50:07 +0800 Subject: [PATCH 20/55] perf(cua): omit unused icons from application selection --- packages/experimental/cua-macos/README.i18n.yaml | 4 ++-- packages/experimental/cua-macos/README.md | 2 +- packages/experimental/cua-macos/README.zh.md | 2 +- .../native/Sources/CuaNative/RequestCoordinator.swift | 7 +------ snapshots/session/cua-native/workspace/native-helper.mjs | 2 +- 5 files changed, 6 insertions(+), 11 deletions(-) diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index 69c1f545a8..58f4fc662c 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -12,8 +12,8 @@ en: 4549f02fc53854a5 zh: 6f74f1130b6623a8 /deepseek-ai-dsh-experimental-cua-macos/table-of-contents: - en: 867d9ec544719acf - zh: 92348599582dc33a + en: 9353d17b05f2f88c + zh: 6b2c6b4ee0046294 /deepseek-ai-dsh-experimental-cua-macos/use-this-package: en: 5f070508770a1a48 zh: 07d6e582c01b0ea3 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 960177c233..a65220d4b1 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -20,7 +20,7 @@ Inspect macOS permissions, select running applications and windows, read accessi - [Known Limitations and Deferred Work](#known-limitations-and-deferred-work) - [Dev Note](#dev-note) -The helper supplies bounded 64-pixel PNG application icons for permission prompts and saved-application lists. Icon reads do not launch an application or inspect its windows; executable and signing checks still determine authorization. +The helper supplies bounded 64-pixel PNG application icons for permission-page application lists. Icon reads do not launch an application or inspect its windows; executable and signing checks still determine authorization. The shared helper queries the process macOS considers responsible for it and captures that application’s name and icon before announcing readiness. A Desktop app launched from Terminal can therefore identify Terminal even when another GUI application sits between it and the helper. Permission queries retain this metadata after the responsible application exits and across connections sharing the helper. If the system query or application metadata is unavailable, the field is omitted and the UI uses the system prompt for guidance. diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index dab1ca6499..d57c404c6e 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -20,7 +20,7 @@ kind: "package-reference" - [已知限制与后续工作](#known-limitations-and-deferred-work) - [开发备注](#dev-note) -helper 为授权提示和已授权应用列表提供 64 像素 PNG 图标。读取图标不会启动应用或检查窗口，授权仍由可执行文件和签名校验决定。 +helper 为权限页的应用列表提供 64 像素 PNG 图标。读取图标不会启动应用或检查窗口，授权仍由可执行文件和签名校验决定。 共享 helper 查询 macOS 为它记录的责任进程，在发布就绪状态前保存对应应用的名称和图标。因此，从 Terminal 启动 Desktop 时，即使中间还有另一个 GUI 应用，也能识别到 Terminal。责任应用退出后，权限查询仍保留这些信息，后续复用该 helper 的连接读取同一份记录。系统查询或应用信息不可用时省略该字段，由界面引导用户按系统弹窗选择授权对象。 diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift index 280cd358ad..99ea8634a9 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift @@ -195,12 +195,7 @@ import Darwin let outcome = try await worker.execute(request.method, params: params, catalog: catalog, installed: installed, trace: trace) trace?.mark("worker.end") switch outcome { - case .result(let value): - if request.method == "getApp", var fields = value.object { - let path = installed?.path ?? catalog.applications.first { Double($0.pid) == value["pid"].number }?.path - if let path, let icon = ApplicationCatalog.icon(at: path) { fields["applicationIcon"] = .string(icon) } - result = .object(fields) - } else { result = value } + case .result(let value): result = value case .capture(let target): let image = try await capture.capture(target, cancellation: cancellation, trace: trace) result = try await worker.finishCapture(target, image: image, trace: trace) diff --git a/snapshots/session/cua-native/workspace/native-helper.mjs b/snapshots/session/cua-native/workspace/native-helper.mjs index 2bbaa83188..e36445ca94 100755 --- a/snapshots/session/cua-native/workspace/native-helper.mjs +++ b/snapshots/session/cua-native/workspace/native-helper.mjs @@ -142,7 +142,7 @@ function createApplicationState(socket, record, application, captureBroker, conn } if (!['Fixture', 'test.fixture', '9'].includes(params.selector)) refuse('APP_NOT_FOUND', 'Select the running Fixture application') acquired = true - return { ...app, applicationIcon: `data:image/png;base64,${png}`, authorization: {"id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "name": "Fixture", "executablePath": "/fixture/app", "persistable": true} } + return { ...app, authorization: {"id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "name": "Fixture", "executablePath": "/fixture/app", "persistable": true} } } if (!acquired || params.appHandle !== app.appHandle) refuse('APP_NOT_FOUND', 'Acquire an application in this helper first') if (method === 'show') return { dispatched: true, verified: false, windowId: params.windowId ?? window.windowId, foregroundChanged: false } From 5d0fe5430911f465bab527a11b405ffa21da459e Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:50:17 +0800 Subject: [PATCH 21/55] test(cua): cover each private native operation once --- .../cua-macos/tests/protocol-generation.spec.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts index 54b4bf3a19..220aed84a4 100644 --- a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts +++ b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts @@ -18,12 +18,7 @@ describe('native protocol generation', () => { { method: 'paste', params: { appHandle: 'app', text: 'text', pasteReadTimeoutMs: 1000 } }, { method: 'findApps', params: { query: 'Editor', maxAppMatches: 1 } }, { method: 'launchApp', params: { appHandle: 'pending' } }, - { method: 'permissionStatus', params: {} }, - { method: 'requestPermission', params: { permission: 'accessibility' } }, - { method: 'openPermissionSettings', params: { permission: 'screenRecording' } }, - { method: 'permissionStatus', params: {} }, - { method: 'requestPermission', params: { permission: 'accessibility' } }, - { method: 'openPermissionSettings', params: { permission: 'screenRecording' } }, + { method: 'applicationIcons', params: { paths: ['/Applications/Fixture.app'] } }, { method: 'permissionStatus', params: {} }, { method: 'requestPermission', params: { permission: 'accessibility' } }, { method: 'openPermissionSettings', params: { permission: 'screenRecording' } }, From d07ad00f86e9694e096edf47e2b78b5f21369bcf Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:50:57 +0800 Subject: [PATCH 22/55] fix(cua): keep the basic bundle independent of PTC configuration --- packages/experimental/README.i18n.yaml | 4 ++-- packages/experimental/README.md | 2 +- packages/experimental/README.zh.md | 2 +- packages/experimental/cua-bundle/package.json | 1 - pnpm-lock.yaml | 3 --- 5 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/experimental/README.i18n.yaml b/packages/experimental/README.i18n.yaml index 58388301cb..77c06a747b 100644 --- a/packages/experimental/README.i18n.yaml +++ b/packages/experimental/README.i18n.yaml @@ -18,8 +18,8 @@ en: b55aeb927737b90d zh: 0b215fcb5c2c7908 /packages-experimental/packages: - en: e12eeae4540a53e0 - zh: 44f65caca730252d + en: e20594b1b7e506d7 + zh: ef76c3c732ddb547 /packages-experimental/related-documentation: en: 1cb9d3448d44e48d zh: cd326d70a746b896 diff --git a/packages/experimental/README.md b/packages/experimental/README.md index ed093206a6..5f052b8e5e 100644 --- a/packages/experimental/README.md +++ b/packages/experimental/README.md @@ -71,7 +71,7 @@ Promotion requires a named product role, documented public behavior and limitati | [`ptc-runtime-python`](ptc-runtime-python/README.md) | CPython subprocess backend for the PTC execution seam | `ctx.ptcRuntime` | | [CUA](cua/README.md) | Shared application/window objects, consent, and logged execution. | Library | | [macOS CUA](cua-macos/README.md) | Swift native execution and platform preparation. | `ctx.computerUse` | -| [`cua-bundle`](cua-bundle/README.md) | Optional Computer Use switch with persistent PTC | — | +| [`cua-bundle`](cua-bundle/README.md) | Optional Computer Use switch for PTC sessions | — | | [`client-ui-cua`](client-ui-cua/README.md) | Computer Use permission setup and application access controls | — | | [`computer-use-cua-driver-mcp`](computer-use-cua-driver-mcp/README.md) | Use an installed Cua Driver through MCP | `ctx.computerUse` | | [`computer-use-cua-driver-native`](computer-use-cua-driver-native/README.md) | Embed the Cua Driver native npm runtime | `ctx.computerUse` | diff --git a/packages/experimental/README.zh.md b/packages/experimental/README.zh.md index edc4efb886..5a74a36072 100644 --- a/packages/experimental/README.zh.md +++ b/packages/experimental/README.zh.md @@ -71,7 +71,7 @@ kind: "package-group" | [`ptc-runtime-python`](ptc-runtime-python/README.zh.md) | PTC 执行 seam 的 CPython 子进程后端 | `ctx.ptcRuntime` | | [CUA](cua/README.zh.md) | 共享应用与窗口对象、授权和已记录的执行。 | 库 | | [macOS CUA](cua-macos/README.zh.md) | Swift 原生执行与平台组件准备。 | `ctx.computerUse` | -| [`cua-bundle`](cua-bundle/README.zh.md) | 默认关闭的 Computer Use 开关与 persistent PTC 组合 | — | +| [`cua-bundle`](cua-bundle/README.zh.md) | 面向 PTC 会话、默认关闭的 Computer Use 开关 | — | | [`client-ui-cua`](client-ui-cua/README.zh.md) | Computer Use 权限设置与应用授权管理 | — | | [`computer-use-cua-driver-mcp`](computer-use-cua-driver-mcp/README.zh.md) | 通过 MCP 使用已安装的 Cua Driver | `ctx.computerUse` | | [`computer-use-cua-driver-native`](computer-use-cua-driver-native/README.zh.md) | 嵌入 Cua Driver 原生 npm 运行时 | `ctx.computerUse` | diff --git a/packages/experimental/cua-bundle/package.json b/packages/experimental/cua-bundle/package.json index 96935054cc..155345ecc6 100644 --- a/packages/experimental/cua-bundle/package.json +++ b/packages/experimental/cua-bundle/package.json @@ -40,7 +40,6 @@ "dependencies": { "@deepseek-ai/dsh-experimental-cua-macos": "workspace:*", "@deepseek-ai/dsh-computer-use": "workspace:*", - "@deepseek-ai/dsh-ptc-runtime-node": "workspace:*", "@deepseek-ai/dsh-client-ui-cua": "workspace:*" }, "peerDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 37e833d594..ed0602a480 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8670,9 +8670,6 @@ importers: '@deepseek-ai/dsh-experimental-cua-macos': specifier: workspace:* version: link:../cua-macos - '@deepseek-ai/dsh-ptc-runtime-node': - specifier: workspace:* - version: link:../../ptc-runtime/ptc-runtime-node devDependencies: '@deepseek-ai/cordis': specifier: workspace:~ From a86170da75b63904b8198a6250859f5ed6fed177 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:51:17 +0800 Subject: [PATCH 23/55] refactor(cua): remove unused permission observation pause controls --- .../src/client/PermissionPanel.tsx | 10 ++-- .../client-ui-cua/src/client/controller.ts | 20 +------ .../tests/controller.client.spec.ts | 55 +------------------ .../client-ui-cua/tests/mount.client.spec.ts | 1 - .../client-ui-cua/tests/panel.client.spec.tsx | 2 +- 5 files changed, 9 insertions(+), 79 deletions(-) diff --git a/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx b/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx index 4c9ae041fd..b80edfee6c 100644 --- a/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx +++ b/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx @@ -94,7 +94,7 @@ function PermissionDescription({ application, permission, loading, t }: Pick value) const observed = state.system.value const apps = state.applications.value @@ -103,10 +103,10 @@ export function PermissionPanel(props: PermissionPanelProps) { const loading = observed === null && (busy || state.system.error === null) const application = !busy && state.system.error !== null ? undefined : observed?.launchApplication useEffect(() => { - setActive(true) + refresh() window.addEventListener('focus', refresh) return () => { window.removeEventListener('focus', refresh) } - }, [refresh, setActive]) + }, [refresh]) return

{t('title')}

@@ -158,9 +158,9 @@ export function PermissionPanel(props: PermissionPanelProps) { * @param props Permission observation and activation navigation callbacks. * @returns Setup guidance when system permissions need attention. */ -export function PermissionSetup({ t, setActive, useCuaPermissions, onDismiss, onOpenDetails }: PermissionSetupProps) { +export function PermissionSetup({ t, refresh, useCuaPermissions, onDismiss, onOpenDetails }: PermissionSetupProps) { const state = useCuaPermissions(value => value.system) - useEffect(() => { setActive(true) }, [setActive]) + useEffect(() => { refresh() }, [refresh]) const ready = state.value !== null && state.error === null && state.value.accessibility && state.value.screenRecording useEffect(() => { if (ready && !state.busy) onDismiss() }, [ready, state.busy, onDismiss]) diff --git a/packages/experimental/client-ui-cua/src/client/controller.ts b/packages/experimental/client-ui-cua/src/client/controller.ts index 552dcefbba..044f431e2e 100644 --- a/packages/experimental/client-ui-cua/src/client/controller.ts +++ b/packages/experimental/client-ui-cua/src/client/controller.ts @@ -31,8 +31,6 @@ export type PermissionView = { [K in keyof PermissionData]: Query } - /** Pause observation during bundle changes, or resume with a fresh read. */ - setActive: (active: boolean) => void refresh: () => void request: (permission: CuaSystemPermission) => void openSettings: (permission: CuaSystemPermission) => void @@ -53,8 +51,6 @@ export class PermissionController { private tail: Promise = Promise.resolve() private reads: Promise = Promise.resolve() private closed = false - private active = true - private epoch = 0 private sequence = 0 /** @param actions Browser-owned Remote calls that reject on failed responses. */ constructor(private readonly actions: PermissionActions) {} @@ -65,17 +61,6 @@ export class PermissionController { face(): PermissionFace { return { hooks: { cuaPermissions: this.state }, - setActive: (active) => { - if (this.closed) return - if (active !== this.active) { this.active = active; this.epoch++ } - if (active) this.enqueue() - else this.state.update((state) => { - state.busy = true - for (const key of ['system', 'applications', 'icons'] as const) { - state[key].busy = true; state[key].error = null - } - }) - }, refresh: () => { this.enqueue() }, request: (permission) => { this.enqueue(() => this.actions.request(permission), 'requested') }, openSettings: (permission) => { this.enqueue(() => this.actions.openSettings(permission), 'opened') }, @@ -87,9 +72,8 @@ export class PermissionController { /** Prevent publication after UI withdrawal and join the accepted Remote work. */ async dispose(): Promise { this.closed = true; await this.tail; await this.reads } private enqueue(action?: () => Promise, kind?: 'requested' | 'opened' | 'revoked' | 'retryAllowed'): void { - if (this.closed || !this.active) return - const epoch = this.epoch - const current = () => !this.closed && this.active && epoch === this.epoch + if (this.closed) return + const current = () => !this.closed const sequence = ++this.sequence this.state.update((state) => { state.busy = true diff --git a/packages/experimental/client-ui-cua/tests/controller.client.spec.ts b/packages/experimental/client-ui-cua/tests/controller.client.spec.ts index 6c12499c16..a04253ef4a 100644 --- a/packages/experimental/client-ui-cua/tests/controller.client.spec.ts +++ b/packages/experimental/client-ui-cua/tests/controller.client.spec.ts @@ -105,7 +105,7 @@ it('ignores stale toast completion after the permission UI is disposed', async ( await f.controller.dispose() const before = f.controller.state.getSnapshot() f.face.dismissToast() - f.face.setActive(true) + f.face.refresh() expect(f.controller.state.getSnapshot()).toBe(before) }) @@ -161,45 +161,6 @@ it('keeps a user-action failure visible when a refresh is queued behind it', asy }) -it.each(['success', 'failure'])('drops a permission read %s from the previous bundle activation', async (outcome) => { - const f = fixture() - f.face.setActive(true) - await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) - const pending = Promise.withResolvers(), entered = Promise.withResolvers() - onTestFinished(async () => { pending.resolve(value); await f.controller.dispose() }) - f.actions.readSystem.mockImplementationOnce(() => { entered.resolve(undefined); return pending.promise }) - f.face.refresh() - await entered.promise - f.face.setActive(false) - f.face.setActive(false) - expect(f.controller.state.getSnapshot()).toMatchObject({ busy: true, system: { error: null, value } }) - f.face.request('accessibility') - expect(f.actions.request).not.toHaveBeenCalled() - if (outcome === 'failure') pending.reject(new Error('old helper stopped')) - else pending.resolve({ ...value, screenRecording: true }) - await Promise.allSettled([pending.promise]) - expect(f.controller.state.getSnapshot()).toMatchObject({ busy: true, system: { error: null, value } }) - f.face.setActive(true) - await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) - expect(f.controller.state.getSnapshot()).toMatchObject({ system: { error: null, value } }) - expect(f.actions.readSystem).toHaveBeenCalledTimes(3) -}) - -it.each(['success', 'failure'])('does not publish a user-action %s after its bundle is replaced', async (outcome) => { - const f = fixture(), pending = Promise.withResolvers(), entered = Promise.withResolvers() - onTestFinished(async () => { pending.resolve(undefined); await f.controller.dispose() }) - f.actions.request.mockImplementationOnce(() => { entered.resolve(undefined); return pending.promise }) - f.face.request('accessibility') - await entered.promise - f.face.setActive(false) - if (outcome === 'failure') pending.reject(new Error('old request cancelled')) - else pending.resolve(undefined) - f.face.setActive(true) - await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) - expect(f.controller.state.getSnapshot()).toMatchObject({ toast: null, system: { error: null, value } }) - expect(f.actions.readSystem).toHaveBeenCalledOnce() -}) - it('publishes each observation before the other queries finish', async () => { const f = fixture() const system = Promise.withResolvers() @@ -237,17 +198,3 @@ it('keeps application decisions available when system inspection fails', async ( }) await f.controller.dispose() }) - -it.each(['success', 'failure'])('drops old icon %s after disabling and reenabling the bundle', async (outcome) => { - const f = fixture(), pending = Promise.withResolvers>() - onTestFinished(async () => { pending.resolve({}); await f.controller.dispose() }) - f.actions.readIcons.mockReturnValueOnce(pending.promise) - f.face.refresh() - await vi.waitFor(() => { expect(f.controller.state.getSnapshot().system.busy).toBe(false) }) - f.face.setActive(false) - f.face.setActive(true) - if (outcome === 'success') pending.resolve({ '/old/app': 'old-icon' }) - else pending.reject(new Error('old helper stopped')) - await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) - expect(f.controller.state.getSnapshot().icons).toEqual({ value: {}, error: null, busy: false }) -}) diff --git a/packages/experimental/client-ui-cua/tests/mount.client.spec.ts b/packages/experimental/client-ui-cua/tests/mount.client.spec.ts index c79bfed54b..df6732f404 100644 --- a/packages/experimental/client-ui-cua/tests/mount.client.spec.ts +++ b/packages/experimental/client-ui-cua/tests/mount.client.spec.ts @@ -18,7 +18,6 @@ const value: CuaApplicationState & { system: CuaSystemPermissionState } = { allowedApps: [], blockedApps: [], settingsNamespace: 'cua', persistentAvailable: true } function actionsOf(value: Record): asserts value is Record & PermissionFace { - assert(typeof value.setActive === 'function') assert(typeof value.refresh === 'function' && typeof value.request === 'function' && typeof value.openSettings === 'function') assert(typeof value.revoke === 'function' && typeof value.dismissToast === 'function') assert(typeof value.retryApproval === 'function') diff --git a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx b/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx index d6e833e2d5..5f21465344 100644 --- a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx +++ b/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx @@ -32,7 +32,7 @@ function fixture() { const refresh = vi.fn() const props = { useCuaPermissions: bindSnapshotSelector(state), t: makeTranslate(en), useCuaDefaultPreset: bindSnapshotSelector(preset), setPtcDefault: vi.fn(async () => true), - refresh, setActive: vi.fn((active: boolean) => { if (active) refresh() }), request: vi.fn(), openSettings: vi.fn(), revoke: vi.fn(), + refresh, request: vi.fn(), openSettings: vi.fn(), revoke: vi.fn(), retryApproval: vi.fn(), dismissToast: vi.fn(), } satisfies PermissionPanelProps return { state, preset, props, From 2d1609920003f9947d39b39e74249df4537533f0 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:51:36 +0800 Subject: [PATCH 24/55] fix(cua): localize complete permission instructions --- .../client-ui-cua/src/client/PermissionPanel.tsx | 7 ++++--- .../experimental/client-ui-cua/src/client/locales.ts | 10 ++++------ .../client-ui-cua/tests/panel.client.spec.tsx | 5 ++--- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx b/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx index b80edfee6c..faa83a783d 100644 --- a/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx +++ b/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx @@ -80,15 +80,16 @@ function PermissionDescription({ application, permission, loading, t }: Pick{t(permission === 'accessibility' ? 'accessibilityUse' : 'screenRecordingUse')}

+ const [before, after] = t(permission === 'accessibility' ? 'accessibilityPermission' : 'screenRecordingPermission').split('{application}') return

- {t('permissionPrefix')}{' '} + {before} {application === undefined ? : <> {application.icon !== undefined && } {application.name} } - {' '} - {t(permission === 'accessibility' ? 'accessibilityAction' : 'screenRecordingAction')} + + {after}

} diff --git a/packages/experimental/client-ui-cua/src/client/locales.ts b/packages/experimental/client-ui-cua/src/client/locales.ts index 03e86f33d0..52f286700e 100644 --- a/packages/experimental/client-ui-cua/src/client/locales.ts +++ b/packages/experimental/client-ui-cua/src/client/locales.ts @@ -17,9 +17,8 @@ export const en = { openSetup: 'Open settings', later: 'Later', close: 'Close', accessibility: 'Accessibility', screenRecording: 'Screen Recording', accessibilityUse: 'Read interface controls, click, and type.', screenRecordingUse: 'Capture the selected application window.', - permissionPrefix: 'In System Settings, allow', - accessibilityAction: 'to read interface controls, click, and type.', - screenRecordingAction: 'to capture application windows.', + accessibilityPermission: 'In System Settings, allow {application} to read interface controls, click, and type.', + screenRecordingPermission: 'In System Settings, allow {application} to capture application windows.', permissionHint: 'Enable the application in System Settings. Follow the macOS prompt if it names another application.', permissionHintUnknown: 'Enable these permissions for the application named in the macOS prompt.', granted: 'Allowed', missing: 'Not allowed', unknown: 'Unknown', request: 'Request permission', settings: 'Open System Settings', retry: 'Retry', @@ -48,9 +47,8 @@ export const zh: Record = { openSetup: '前往设置', later: '稍后', close: '关闭', accessibility: '辅助功能', screenRecording: '屏幕录制', accessibilityUse: '读取界面控件、点击和输入文字', screenRecordingUse: '截取所选应用的窗口', - permissionPrefix: '在系统设置中允许', - accessibilityAction: '读取界面、点击和输入', - screenRecordingAction: '截取应用窗口', + accessibilityPermission: '在系统设置中允许 {application} 读取界面、点击和输入。', + screenRecordingPermission: '在系统设置中允许 {application} 截取应用窗口。', permissionHint: '在系统设置中开启对应应用；若 macOS 提示其他应用，以系统提示为准', permissionHintUnknown: '在系统设置中，为 macOS 提示的应用开启对应权限', granted: '已允许', missing: '未允许', unknown: '未确认', request: '申请权限', settings: '打开系统设置', retry: '重试', diff --git a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx b/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx index 5f21465344..e29768cdf9 100644 --- a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx +++ b/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx @@ -229,13 +229,12 @@ it.each([en, zh])('shows three requirements with the target application beside e expect(within(requirements).getAllByRole('group')).toHaveLength(3) expect(screen.getByRole('region', { name: dictionary.applicationsTitle })).toBeTruthy() const accessibility = within(requirements).getByRole('group', { name: dictionary.accessibility }) - expect(accessibility.textContent).toContain(dictionary.permissionPrefix) expect(within(accessibility).getByText(application.name)).toBeTruthy() - expect(accessibility.textContent).toContain(dictionary.accessibilityAction) + expect(accessibility.textContent).toContain(dictionary.accessibilityPermission.replace('{application}', application.name)) expect(accessibility.querySelector('img')?.getAttribute('src')).toBe(application.icon) const recording = within(requirements).getByRole('group', { name: dictionary.screenRecording }) expect(within(recording).getByText(application.name)).toBeTruthy() - expect(recording.textContent).toContain(dictionary.screenRecordingAction) + expect(recording.textContent).toContain(dictionary.screenRecordingPermission.replace('{application}', application.name)) expect(within(requirements).getByText(dictionary.permissionHint)).toBeTruthy() f.publish({ system: { ...f.state.getSnapshot().system, error: 'offline' } }) expect(screen.queryByText(application.name)).toBeNull() From 49794a9ad2f5d030af2d694a55cad4a205891b87 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:52:06 +0800 Subject: [PATCH 25/55] docs(cua): define preset filtering and cover late registration --- packages/experimental/cua-macos/README.i18n.yaml | 4 ++-- packages/experimental/cua-macos/README.md | 1 + packages/experimental/cua-macos/README.zh.md | 1 + packages/experimental/cua/README.i18n.yaml | 4 ++-- packages/experimental/cua/README.md | 2 +- packages/experimental/cua/README.zh.md | 2 +- .../experimental/cua/tests/agent-scopes.spec.ts | 15 +++++++++++++-- 7 files changed, 21 insertions(+), 8 deletions(-) diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index 58f4fc662c..8e064caef0 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -15,8 +15,8 @@ en: 9353d17b05f2f88c zh: 6b2c6b4ee0046294 /deepseek-ai-dsh-experimental-cua-macos/use-this-package: - en: 5f070508770a1a48 - zh: 07d6e582c01b0ea3 + en: 107cc4d4e72e248d + zh: 1aec124d4018d2a4 /deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment: en: e7909c9b43e44fbc zh: 98949877d4ea3a86 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index a65220d4b1..7700c69ed1 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -42,6 +42,7 @@ The dump command checks profile composition without a model request. For a real | Field | Default | Meaning | | --- | --- | --- | +| `agentPreset` | `ptc` | Register tools and guidance only for Agents bound to this preset; `false` registers globally. | | `allowedApps` | `[]` | Live profile-local application consent, keyed by executable path and verified signing requirement. | | `helperPath` | package-owned executable | Optional absolute native executable override. | | `buildTimeoutMs` | `300000` | Maximum native preparation time. | diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index d57c404c6e..ccd844dcf7 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -42,6 +42,7 @@ pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis. | 字段 | 默认值 | 含义 | | --- | --- | --- | +| `agentPreset` | `ptc` | 仅为绑定该 preset 的 Agent 注册工具和指引；`false` 表示全局注册。 | | `allowedApps` | `[]` | 当前配置档的动态应用授权，绑定可执行文件路径和已验证的代码签名要求。 | | `helperPath` | 包内可执行文件 | 可选的原生可执行文件绝对路径覆盖。 | | `buildTimeoutMs` | `300000` | 原生组件准备的最长时间。 | diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index e5ee2a0f75..00480ee607 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -18,8 +18,8 @@ en: 577e1b18dd220c73 zh: bde0692c6695d749 /deepseek-ai-dsh-cua/understand-the-implementation: - en: 751fe580f79dfaae - zh: 3811db65f634ef5a + en: 5f8f14fda4d69173 + zh: 5493e3ba74ffb7e0 /deepseek-ai-dsh-cua/further-exploration: en: b8b0aece886fadaa zh: ecd8c88d601f2179 diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index 11cbd47457..24abc4a510 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -51,7 +51,7 @@ Actions return `dispatched:true, verified:false`; read fresh state to verify the The provider calls [installCua](src/index.ts) with its prepared helper, connection factory, and guidance. [NativeClient](src/native-client.ts) owns request correlation, byte limits, cancellation acknowledgement, and failure settlement. The platform connector owns startup and connection cleanup. The shared executor retains one connection per Agent and joins in-flight image writes during disposal. -`agentPreset` defaults to `ptc`, restricting tools and guidance to Agents bound to that preset. The provider uses the existing `agent/created` and `agent-preset/selected` events and Agent-owned scopes, including Agents already present when it loads. Provider unload and Agent disposal release those registrations. Set `agentPreset: false` for global registration in standalone compositions without presets. The Host continues to own permissions, native connections and cancellation; preset definitions remain unchanged. +`agentPreset` defaults to `ptc`, restricting tools and guidance to Agents bound to that preset. The provider uses the existing `agent/created` and `agent-preset/selected` events and Agent-owned scopes, including Agents already present when it loads. Provider unload and Agent disposal release those registrations. Preset definitions may register after the provider; a preset name filters bound Agents and supplies no tools until a matching Agent exists. Set `agentPreset: false` for global registration in standalone compositions without presets. The Host continues to own permissions, native connections and cancellation; preset definitions remain unchanged. [The operation schema](src/protocol.ts) is the authority for model parameters and separately declared Host-only fields. Native implementations enforce target identity, window geometry, resource limits, and operating-system access at execution. No invariant companion is published because this library has no separately projected registry or durable state. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index 3b58f03187..6de835777a 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -51,7 +51,7 @@ kind: "package-library" 提供方将准备好的 helper、连接工厂和指引传给 [installCua](src/index.ts)。[NativeClient](src/native-client.ts) 负责请求关联、字节上限、取消确认和失败处理。平台连接器负责启动和连接清理。共享执行器为每个 Agent 保留一个连接，并在销毁时等待进行中的图像写入。 -`agentPreset` 默认为 `ptc`，将工具和指引限制到绑定该 preset 的 Agent。提供方使用现有 `agent/created`、`agent-preset/selected` 事件及 Agent 拥有的作用域，也处理启用前已存在的 Agent。提供方卸载和 Agent 销毁都会释放这些注册。不使用 preset 的独立组合可设置 `agentPreset: false` 来全局注册。Host 仍拥有权限、本机连接和取消操作，preset 定义保持不变。 +`agentPreset` 默认为 `ptc`，将工具和指引限制到绑定该 preset 的 Agent。提供方使用现有 `agent/created`、`agent-preset/selected` 事件及 Agent 拥有的作用域，也处理启用前已存在的 Agent。提供方卸载和 Agent 销毁都会释放这些注册。preset 定义可以晚于提供方注册；名称用于筛选已绑定的 Agent，在匹配的 Agent 出现前不提供工具。不使用 preset 的独立组合可设置 `agentPreset: false` 来全局注册。Host 仍拥有权限、本机连接和取消操作，preset 定义保持不变。 [操作 schema](src/protocol.ts) 定义模型参数，并单独声明 Host 专用字段。原生实现在执行时检查目标身份、窗口几何、资源上限和操作系统访问条件。本库没有独立投影的注册表或持久状态，因此不发布 invariant 模块。 diff --git a/packages/experimental/cua/tests/agent-scopes.spec.ts b/packages/experimental/cua/tests/agent-scopes.spec.ts index 26568bca49..fd1a349130 100644 --- a/packages/experimental/cua/tests/agent-scopes.spec.ts +++ b/packages/experimental/cua/tests/agent-scopes.spec.ts @@ -23,7 +23,7 @@ const provider = { }, } -async function setup() { +async function setup(presetIds = ['standard', 'ptc']) { const ctx = new Context() onTestFinished(() => ctx.fiber.dispose()) ctx.baseUrl = import.meta.url @@ -33,7 +33,7 @@ async function setup() { await ctx.plugin(ComputerUseRegistry) await ctx.plugin(Subprocess, []) await ctx.plugin(AgentPresets, { default: 'standard' }) - for (const id of ['standard', 'ptc']) { + for (const id of presetIds) { await ctx.plugin({ inject: ['agentPresets'], async* apply(owner: Context) { yield await owner.agentPresets.register({ id, plugins: [] }) } }) @@ -93,3 +93,14 @@ it('ignores unloaded Session selections and reports a failed live scope update', await ctx.agentPresets.select(handle.agent, 'standard') await expect.poll(() => ctx.tools.get('cua', handle.agent)).toBeUndefined() }) + +it('accepts a matching preset registered after the CUA provider', async () => { + const { ctx, create } = await setup([]) + await ctx.plugin(provider, {}) + expect(ctx.tools.get('cua')).toBeUndefined() + await ctx.plugin({ inject: ['agentPresets'], async* apply(owner: Context) { + yield await owner.agentPresets.register({ id: 'ptc', plugins: [] }) + } }) + const handle = await create('late-preset', 'ptc') + expect(ctx.tools.get('cua', handle.agent)).toBeDefined() +}) From fe14a0972c0d17b59eb36e6ef85af4d187413e72 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:52:38 +0800 Subject: [PATCH 26/55] docs(cua): disclose session consent cache retention --- packages/experimental/cua/README.i18n.yaml | 4 ++-- packages/experimental/cua/README.md | 1 + packages/experimental/cua/README.zh.md | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index 00480ee607..14142829c4 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -39,8 +39,8 @@ en: 2d5df52dafa1d940 zh: 06e815759b9150c3 /deepseek-ai-dsh-cua/known-limitations-and-deferred-work: - en: da6aefcb279aea83 - zh: b45b7bb300b9f792 + en: ae98f1b2718b3b94 + zh: e6bf4e96ae08e669 /deepseek-ai-dsh-cua/known-limitations-and-deferred-work/dev-note: en: 0a1253c6f7b5e8b9 zh: 9f56528562c7c914 diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index 24abc4a510..cadf0e6a8d 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -88,6 +88,7 @@ Results append to history. Retaining objects does not rewrite previous messages; +- In-memory session decisions have no capacity limit or eviction. They remain until the provider unloads, so a long-running Host retains entries for every session/application pair that requested access. - Native execution currently uses the macOS provider. Windows support is planned as a separate provider package with a C# helper, retaining the shared app/window API, authorization, snapshot validity, and action-result semantics. Its implementation must document system-specific limitations and distinguish cross-compilation evidence from Windows desktop validation. - Persistent variables do not reserve an application across a sequence of operations. Other Agents and the user can change its state. Hover and observations do not restore the previous pointer position; [continuous interaction](../../../.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.md) uses no cross-call ownership. - The package stays in `experimental` until Windows support and platform validation permit promotion to the `computer-use` group. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index 6de835777a..35e37feb67 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -88,6 +88,7 @@ kind: "package-library" +- 会话授权决策保存在内存中，没有容量上限或淘汰机制，直到提供方卸载才清空。长期运行的 Host 会保留每个曾请求访问的会话与应用组合。 - 当前原生执行使用 macOS 提供方。Windows 支持计划使用独立的提供方包和 C# helper，保留共享应用与窗口 API、授权、快照有效性和动作结果语义。实现须说明系统特有的限制，并区分交叉编译证据和 Windows 桌面验证。 - 持久变量不会在一组连续操作期间独占应用。其他 Agent 和用户可能改变其状态。 悬停和观察不恢复先前的鼠标位置；[连续交互](../../../.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.zh.md)不分配跨调用所有权。 - 本包保留在 `experimental`，Windows 支持和平台验证完成后可迁至 `computer-use` 组。 From dd436a6c9cb7afd331760c7be60a7c032449e9e3 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:53:15 +0800 Subject: [PATCH 27/55] docs(cua): explain explicit package naming exceptions --- ...21-experimental-capabilities-as-optional-bundles.i18n.yaml | 4 ++-- ...026-09-21-experimental-capabilities-as-optional-bundles.md | 2 +- ...-09-21-experimental-capabilities-as-optional-bundles.zh.md | 2 +- scripts/experimental-package-policy.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.i18n.yaml b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.i18n.yaml index a2748e8846..345eec989b 100644 --- a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.i18n.yaml @@ -9,8 +9,8 @@ en: 2ba6c43b20a3d347 zh: 5647179ddaa38ad9 /agent-note-ship-experimental-capabilities-as-optional-bundles/decision: - en: 69df6602f92a3e09 - zh: 2e2ed04760086f4c + en: 6566c6fb7287d748 + zh: fe41c0a9da97f340 /agent-note-ship-experimental-capabilities-as-optional-bundles/alternatives-considered: en: d6f40bf02df37ab7 zh: e5b85ce9d41e8754 diff --git a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md index f0e6ce7337..a2dded7934 100644 --- a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md +++ b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md @@ -18,7 +18,7 @@ Bundles insert ordinary Host rows; optional model tools register in the global t Claude Code and Codex install on demand through the separate `ON_DEMAND_BUNDLES` catalog; the [on-demand native-bundle decision](2026-10-05-official-on-demand-bundles.md) owns the cost, version, and offline-discovery trade-offs. -Computer Use is a shipped optional bundle in the experimental group. Its repository-owned native component is prepared when the provider loads, without an external Cua Driver runtime. +Computer Use is a shipped optional bundle in the experimental group. Its bundle and client plugin use the explicit naming exceptions `@deepseek-ai/dsh-cua-bundle` and `@deepseek-ai/dsh-client-ui-cua`, alongside `@deepseek-ai/dsh-cua`. These names preserve profile references when the capability moves package groups; the experimental classification still controls isolation and promotion. Its repository-owned native component is prepared when the provider loads, without an external Cua Driver runtime. ## Alternatives considered diff --git a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.zh.md b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.zh.md index c90290b20d..db1b4a5506 100644 --- a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.zh.md +++ b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.zh.md @@ -18,7 +18,7 @@ Bundle 插入普通 Host 行；可选模型工具注册到全局工具层，因 Claude Code 与 Codex 通过独立的 `ON_DEMAND_BUNDLES` 目录按需安装，其成本、版本与离线发现取舍由[按需原生 bundle 决策](2026-10-05-official-on-demand-bundles.zh.md)管理。 -Computer Use 是 experimental 包组内随安装提供的可选组合包。提供方加载时准备仓库自有的本机组件，无需外部 Cua Driver 运行时。 +Computer Use 是 experimental 包组内随安装提供的可选组合包。组合包和前端插件与 `@deepseek-ai/dsh-cua` 一起采用显式命名例外 `@deepseek-ai/dsh-cua-bundle`、`@deepseek-ai/dsh-client-ui-cua`，在能力移动分组时保留配置引用。依赖隔离和晋升仍由实验性分类约束。提供方加载时准备仓库自有的本机组件，无需外部 Cua Driver 运行时。 ## 考虑过的替代方案 diff --git a/scripts/experimental-package-policy.ts b/scripts/experimental-package-policy.ts index 68c63fcd7e..5763dbdedc 100644 --- a/scripts/experimental-package-policy.ts +++ b/scripts/experimental-package-policy.ts @@ -1,6 +1,6 @@ /** Experimental package identities, publication policy, and retained npm names. */ -/** Existing npm names retained by capabilities now housed under experimental/. */ +/** Explicit npm identities retained across experimental classification and package-role moves. */ export const EXPERIMENTAL_PACKAGE_NAME_EXCEPTIONS: Readonly> = { 'packages/experimental/cua': '@deepseek-ai/dsh-cua', 'packages/experimental/cua-bundle': '@deepseek-ai/dsh-cua-bundle', From b39ce22a09a0fb785143c0fe89b15c69c9e48037 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:55:18 +0800 Subject: [PATCH 28/55] test(cua): cover permission actions settling after disposal --- .../client-ui-cua/tests/controller.client.spec.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/experimental/client-ui-cua/tests/controller.client.spec.ts b/packages/experimental/client-ui-cua/tests/controller.client.spec.ts index a04253ef4a..8b776837f9 100644 --- a/packages/experimental/client-ui-cua/tests/controller.client.spec.ts +++ b/packages/experimental/client-ui-cua/tests/controller.client.spec.ts @@ -109,6 +109,20 @@ it('ignores stale toast completion after the permission UI is disposed', async ( expect(f.controller.state.getSnapshot()).toBe(before) }) +it.each(['success', 'failure'])('settles an in-flight action %s after disposal without publishing', async (outcome) => { + const f = fixture(), pending = Promise.withResolvers(), entered = Promise.withResolvers() + onTestFinished(async () => { pending.resolve(undefined); await f.controller.dispose() }) + f.actions.request.mockImplementationOnce(() => { entered.resolve(undefined); return pending.promise }) + f.face.request('accessibility') + await entered.promise + const before = f.controller.state.getSnapshot(), closed = f.controller.dispose() + if (outcome === 'failure') pending.reject(new Error('permission request ended after withdrawal')) + else pending.resolve(undefined) + await closed + expect(f.controller.state.getSnapshot()).toBe(before) + expect(f.actions.readSystem).not.toHaveBeenCalled() +}) + it('contains a late read rejection and supports errors without an Error instance', async () => { const f = fixture() f.actions.readSystem.mockRejectedValueOnce('offline') From b0e4907b104d13e5744118250d8517996f76d540 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:58:28 +0800 Subject: [PATCH 29/55] test(cua): report permission panel overflow details --- apps/web/tests/cua-permissions.e2e.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/web/tests/cua-permissions.e2e.ts b/apps/web/tests/cua-permissions.e2e.ts index 3490749230..30441517cd 100644 --- a/apps/web/tests/cua-permissions.e2e.ts +++ b/apps/web/tests/cua-permissions.e2e.ts @@ -97,7 +97,13 @@ it.skipIf(process.platform !== 'darwin')('keeps system requests pending until fr for (const colorScheme of ['light', 'dark'] as const) { await page.emulateMedia({ colorScheme }) await expect.poll(() => page.evaluate(() => document.body.hasAttribute('data-ds-dark-theme'))).toBe(colorScheme === 'dark') - expect(await panel.evaluate(element => element.scrollWidth <= element.clientWidth)).toBe(true) + const layout = await panel.evaluate(element => ({ + clientWidth: element.clientWidth, scrollWidth: element.scrollWidth, + overflowing: [...element.querySelectorAll('*')].filter(child => + child.getBoundingClientRect().right > element.getBoundingClientRect().right + 1, + ).map(child => ({ tag: child.tagName, className: child.className, text: child.textContent?.slice(0, 80) })), + })) + expect(layout.scrollWidth, JSON.stringify({ width, colorScheme, ...layout })).toBeLessThanOrEqual(layout.clientWidth) if (webSnapshotMode() === 'refresh') { const directory = fileURLToPath(new URL('../../../.artifacts', import.meta.url)) await mkdir(directory, { recursive: true }) From 1aabce5547b1e62f45f2847c01e68a92fe0863bb Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 13:00:45 +0800 Subject: [PATCH 30/55] test(cua): await responsive layout before measuring overflow --- apps/web/tests/cua-permissions.e2e.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/web/tests/cua-permissions.e2e.ts b/apps/web/tests/cua-permissions.e2e.ts index 30441517cd..4e06bf0532 100644 --- a/apps/web/tests/cua-permissions.e2e.ts +++ b/apps/web/tests/cua-permissions.e2e.ts @@ -97,13 +97,16 @@ it.skipIf(process.platform !== 'darwin')('keeps system requests pending until fr for (const colorScheme of ['light', 'dark'] as const) { await page.emulateMedia({ colorScheme }) await expect.poll(() => page.evaluate(() => document.body.hasAttribute('data-ds-dark-theme'))).toBe(colorScheme === 'dark') - const layout = await panel.evaluate(element => ({ + const measure = () => panel.evaluate(element => ({ clientWidth: element.clientWidth, scrollWidth: element.scrollWidth, overflowing: [...element.querySelectorAll('*')].filter(child => child.getBoundingClientRect().right > element.getBoundingClientRect().right + 1, ).map(child => ({ tag: child.tagName, className: child.className, text: child.textContent?.slice(0, 80) })), })) - expect(layout.scrollWidth, JSON.stringify({ width, colorScheme, ...layout })).toBeLessThanOrEqual(layout.clientWidth) + await expect.poll(async () => { + const layout = await measure() + return layout.scrollWidth <= layout.clientWidth ? true : { width, colorScheme, ...layout } + }).toBe(true) if (webSnapshotMode() === 'refresh') { const directory = fileURLToPath(new URL('../../../.artifacts', import.meta.url)) await mkdir(directory, { recursive: true }) From 69f5a19ba79dfd2c166b6ea3602d74d89e072189 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 14:30:34 +0800 Subject: [PATCH 31/55] test(cua): await toggle completion under the browser deadline --- apps/web/tests/cua-optin.e2e.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/web/tests/cua-optin.e2e.ts b/apps/web/tests/cua-optin.e2e.ts index 97fd6f763e..6ceccf4aae 100644 --- a/apps/web/tests/cua-optin.e2e.ts +++ b/apps/web/tests/cua-optin.e2e.ts @@ -45,8 +45,9 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv const toggle = card.getByRole('switch', { name: '启用 Computer Use', exact: true }) expect(await toggle.getAttribute('aria-checked')).toBe('false') expect(await card.textContent()).toContain('请切换到 PTC 模式') + // Bundle transitions await native cleanup under the browser action deadline. await toggle.click() - await expect.poll(() => toggle.getAttribute('aria-checked')).toBe('true') + await toggle.and(page.locator('[aria-checked="true"]')).waitFor({ state: 'visible' }) const setup = page.getByRole('dialog', { name: '使用 Computer Use 前需要设置', exact: true }) await setup.waitFor() expect(await setup.getByText('Computer Use 插件已开启，但暂时无法检测系统权限。请前往插件详情页查看原因并重试。', { exact: true }).count()).toBe(1) @@ -65,7 +66,7 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv }) expect(scaffold.ctx.tools.schemas(fresh.agent).some(tool => tool.name === 'cua')).toBe(true) await toggle.click() - await expect.poll(() => toggle.getAttribute('aria-checked')).toBe('false') + await toggle.and(page.locator('[aria-checked="false"]')).waitFor({ state: 'visible' }) expect(scaffold.ctx.get('computerUse')).toBeUndefined() for (const agent of [ptcAgent, fresh.agent]) expect(scaffold.ctx.tools.schemas(agent).some(tool => tool.name === 'cua')).toBe(false) expect(runtimeFiber() === originalRuntimeFiber).toBe(true) @@ -74,7 +75,7 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv await card.getByRole('button', { name: '查看 Computer Use', exact: true }).click() const detailToggle = page.locator('[data-plugin-panel]').getByRole('switch', { name: '启用 Computer Use', exact: true }) await detailToggle.click() - await expect.poll(() => detailToggle.getAttribute('aria-checked')).toBe('true') + await detailToggle.and(page.locator('[aria-checked="true"]')).waitFor({ state: 'visible' }) const panel = page.locator('[data-cua-settings]') await panel.waitFor() const defaultSetting = panel.getByRole('group', { name: '新会话默认模式', exact: true }) @@ -92,7 +93,7 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv expect(saved).toContain('selectedDefault: ptc') expect(saved).not.toContain('executionMode:') await detailToggle.click() - await expect.poll(() => detailToggle.getAttribute('aria-checked')).toBe('false') + await detailToggle.and(page.locator('[aria-checked="false"]')).waitFor({ state: 'visible' }) expect(await readFile(settingsPath, 'utf8')).toBe(saved) expect(runtimeFiber() === originalRuntimeFiber).toBe(true) expect(scaffold.ctx.ptcRuntime.programFormat).toBe('function-body') From 7d6a8f052f6a80596504b89364017e339b5793c1 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 15:14:09 +0800 Subject: [PATCH 32/55] feat(cua): add an explicit persistent PTC setting --- apps/web/tests/cua-optin.e2e.ts | 29 +++++++++-- apps/web/tests/cua-permissions.e2e.ts | 2 +- .../cua-permissions/missing.expected.md | 4 ++ .../client-ui-cua/README.i18n.yaml | 8 +-- packages/experimental/client-ui-cua/README.md | 8 +-- .../experimental/client-ui-cua/README.zh.md | 8 +-- .../src/client/PermissionPanel.tsx | 50 +++++++++++++------ .../client-ui-cua/src/client/locales.ts | 14 ++++++ .../client-ui-cua/src/client/mount.ts | 4 +- .../client-ui-cua/tests/mount.client.spec.ts | 10 +++- .../client-ui-cua/tests/panel.client.spec.tsx | 47 +++++++++++++++-- .../experimental/cua-bundle/README.i18n.yaml | 4 +- packages/experimental/cua-bundle/README.md | 2 +- packages/experimental/cua-bundle/README.zh.md | 2 +- 14 files changed, 147 insertions(+), 45 deletions(-) diff --git a/apps/web/tests/cua-optin.e2e.ts b/apps/web/tests/cua-optin.e2e.ts index 6ceccf4aae..b6fa5de69a 100644 --- a/apps/web/tests/cua-optin.e2e.ts +++ b/apps/web/tests/cua-optin.e2e.ts @@ -1,4 +1,4 @@ -/** Optional CUA uses existing plugin activation without changing the shared PTC runtime. */ +/** Optional CUA uses existing plugin activation and offers a separate persistent PTC action. */ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -12,7 +12,7 @@ import { expect, it } from 'vitest' import { launchWebScaffold, type WebScaffold } from './scaffold.ts' import { connectFreshWorkspaceZh, ZH_BROWSER_LOCALE } from './support.ts' -it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserving default %s and runtime configuration', async (selectedDefault) => { +it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserving default %s until the explicit persistent-runtime action', async (selectedDefault) => { const directory = await mkdtemp(join(tmpdir(), 'cua-optin-web-')) let ownedScaffold: WebScaffold | undefined let browser: Browser | undefined @@ -92,11 +92,30 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv const saved = await readFile(settingsPath, 'utf8') expect(saved).toContain('selectedDefault: ptc') expect(saved).not.toContain('executionMode:') + const persistent = panel.getByRole('group', { name: 'PTC 执行方式', exact: true }) + await persistent.getByRole('button', { name: '设为持久化', exact: true }).click() + await persistent.getByRole('button', { name: '已设为持久化', exact: true }).waitFor() + expect(scaffold.ctx.ptcRuntime.programFormat).toBe('repl') + expect(runtimeFiber() === originalRuntimeFiber).toBe(false) + const persistentRuntimeFiber = runtimeFiber() + const runtime = scaffold.ctx.ptcRuntime, owner = {} + const program = 'globalThis.counter = (globalThis.counter ?? 0) + 1' + expect((await runtime.run(runtime.resolve({ program, bindings: [], stateOwner: owner }))).value).toBe(1) + const configured = await readFile(settingsPath, 'utf8') + expect(configured).toContain('executionMode: persistent-in-memory') + expect(scaffold.ctx.agentPresets.defaultId).toBe('ptc') + expect(scaffold.ctx.agentPresets.composedPreset(originalAgent.ctx)).toBe(initialDefault) await detailToggle.click() await detailToggle.and(page.locator('[aria-checked="false"]')).waitFor({ state: 'visible' }) - expect(await readFile(settingsPath, 'utf8')).toBe(saved) - expect(runtimeFiber() === originalRuntimeFiber).toBe(true) - expect(scaffold.ctx.ptcRuntime.programFormat).toBe('function-body') + expect(await readFile(settingsPath, 'utf8')).toBe(configured) + expect(runtimeFiber() === persistentRuntimeFiber).toBe(true) + expect((await runtime.run(runtime.resolve({ program, bindings: [], stateOwner: owner }))).value).toBe(2) + await detailToggle.click() + await detailToggle.and(page.locator('[aria-checked="true"]')).waitFor({ state: 'visible' }) + expect(await readFile(settingsPath, 'utf8')).toBe(configured) + expect(runtimeFiber() === persistentRuntimeFiber).toBe(true) + expect((await runtime.run(runtime.resolve({ program, bindings: [], stateOwner: owner }))).value).toBe(3) + await runtime.release(owner) } finally { try { await browser?.close() } finally { try { await ownedScaffold?.close() } finally { await rm(directory, { recursive: true, force: true }) } diff --git a/apps/web/tests/cua-permissions.e2e.ts b/apps/web/tests/cua-permissions.e2e.ts index 4e06bf0532..3379f2ecf1 100644 --- a/apps/web/tests/cua-permissions.e2e.ts +++ b/apps/web/tests/cua-permissions.e2e.ts @@ -80,7 +80,7 @@ it.skipIf(process.platform !== 'darwin')('keeps system requests pending until fr const panel = page.locator('[data-cua-settings]') const accessibility = panel.getByRole('group', { name: 'Accessibility', exact: true }) await accessibility.getByText('Not allowed', { exact: true }).waitFor() - expect(await panel.getByRole('region', { name: 'Authorize Computer Use', exact: true }).getByRole('group').count()).toBe(3) + expect(await panel.getByRole('region', { name: 'Authorize Computer Use', exact: true }).getByRole('group').count()).toBe(4) expect(await panel.getByRole('region', { name: 'Manage applications Computer Use can access', exact: true }).count()).toBe(1) await expect.poll(() => panel.locator('li img').evaluate((image: HTMLImageElement) => image.naturalWidth)).toBe(1) expect(await panel.getByText('/fixture/app', { exact: true }).count()).toBe(0) diff --git a/apps/web/tests/expected/cua-permissions/missing.expected.md b/apps/web/tests/expected/cua-permissions/missing.expected.md index d8fafe9400..25af1a8f27 100644 --- a/apps/web/tests/expected/cua-permissions/missing.expected.md +++ b/apps/web/tests/expected/cua-permissions/missing.expected.md @@ -4,6 +4,10 @@ - strong: Default mode for new sessions - paragraph: Computer Use is available only in PTC mode. - button "Set to PTC" + - group "PTC execution mode": + - strong: PTC execution mode + - paragraph: Keep variables and application objects across tool calls. Switching restarts the PTC runtime. + - button "Use persistent mode" - group "Accessibility": - strong: Accessibility - paragraph: In System Settings, allow Visual Studio Code.app to read interface controls, click, and type. diff --git a/packages/experimental/client-ui-cua/README.i18n.yaml b/packages/experimental/client-ui-cua/README.i18n.yaml index 957097ba2c..b5602c9448 100644 --- a/packages/experimental/client-ui-cua/README.i18n.yaml +++ b/packages/experimental/client-ui-cua/README.i18n.yaml @@ -15,11 +15,11 @@ en: 6658ab477751154b zh: 9b371809a6fe38dc /deepseek-ai-dsh-client-ui-cua/use-this-package: - en: d366fc19d6806a9f - zh: 0f6b02f19139c3f6 + en: 57485612d3ebe444 + zh: b1886ab8e3c313af /deepseek-ai-dsh-client-ui-cua/understand-the-implementation: - en: 7e661a9d00f98068 - zh: a0508c9b1d48cbb7 + en: 6ec7497752efaf72 + zh: 6bcd181aa26ce289 /deepseek-ai-dsh-client-ui-cua/further-exploration: en: f0fde9f736ade61a zh: 43a534a21bcc1c13 diff --git a/packages/experimental/client-ui-cua/README.md b/packages/experimental/client-ui-cua/README.md index 912d387180..0528df4203 100644 --- a/packages/experimental/client-ui-cua/README.md +++ b/packages/experimental/client-ui-cua/README.md @@ -27,11 +27,11 @@ The permission-page lists show native icons and application names. Paths remain ## Use this package -The [Computer Use bundle](../cua-bundle/README.md) mounts this page with its native provider. **Authorize Computer Use** groups the new-session default, Accessibility, and Screen Recording inside one border, with dividers between the rows. Each system-permission row shows the system-reported responsible application’s `.app` bundle filename and icon together in a bordered label. **Manage applications Computer Use can access** places the application lists inside a separate bordered group. Choose **Request permission**, find that application in the macOS permission list, and enable its switch; if macOS names another application, follow that prompt. With no detected source, the page directs the user to the application named by macOS. **Open System Settings** opens the corresponding privacy pane. The page refreshes on entry, when the window regains focus, and after permission actions. A failed query offers **Retry**. Follow any macOS quit-and-reopen prompt and restart DSH as needed. +The [Computer Use bundle](../cua-bundle/README.md) mounts this page with its native provider. **Authorize Computer Use** groups the new-session default, PTC execution mode, Accessibility, and Screen Recording inside one border, with dividers between the rows. Each system-permission row shows the system-reported responsible application’s `.app` bundle filename and icon together in a bordered label. **Manage applications Computer Use can access** places the application lists inside a separate bordered group. Choose **Request permission**, find that application in the macOS permission list, and enable its switch; if macOS names another application, follow that prompt. With no detected source, the page directs the user to the application named by macOS. **Open System Settings** opens the corresponding privacy pane. The page refreshes on entry, when the window regains focus, and after permission actions. A failed query offers **Retry**. Follow any macOS quit-and-reopen prompt and restart DSH as needed. System-permission operations affect the computer running DSH. When accessing DSH remotely, complete authorization on that computer. -Enabling from the plugin list shows setup guidance when system permissions need attention, with **Later** and **Open settings** actions. Enabling from the detail page displays the permission controls directly. The detail page provides **Set to PTC** under **Default mode for new sessions**; only this action saves the default. Existing sessions keep their preset. +Enabling from the plugin list shows setup guidance when system permissions need attention, with **Later** and **Open settings** actions. Enabling from the detail page displays the permission controls directly. The detail page provides **Set to PTC** under **Default mode for new sessions**; only this action saves the default. Existing sessions keep their preset. **Use persistent mode** in the **PTC execution mode** row separately saves `ptc-runtime.executionMode: persistent-in-memory`. Switching reloads the shared PTC runtime and may interrupt an active call; subsequent calls retain variables and application objects in memory. Already-persistent runtimes show a disabled **Persistent mode enabled** button. The page lists always-allowed applications and refusals from any session. A refusal in one session does not cancel another session’s grant or prevent a new session from asking. Session-only grants remain effective within their sessions and are omitted from this page. **Revoke** removes the selected application's access and cancels affected work. Input already delivered to an application cannot be undone; observe the application before any further action. **Allow asking again** clears that application’s refusal records across all sessions; the next application selection still requires a new decision. @@ -45,9 +45,9 @@ This plugin has no configurable fields.
Implementation internals -The optional browser contribution mounts the native provider's permission Remote and reuses the Plugins activation and configuration slots. The three setup rows remain visible while unknown fields use placeholders. The default-mode form, system checks, application decisions, and application icons fill in as each query completes. Application names and actions do not wait for icons; icon positions retain their space. User actions run in order without waiting for pending observation queries. Disposal waits for both actions and queries to settle before releasing the separately owned Remote mount. Query errors stay with the affected section, and activation guidance depends only on system checks. Late results after disposal are discarded. One controller projects queries into the page and setup dialog; reconnects and updates to the Host-reported settings namespace refresh it. Operation notices live in the shell overlay, so closing the page does not remove them. The provider owns permission checks, persisted decisions, and cancellation; the browser owns no authorization state. +The optional browser contribution mounts the native provider's permission Remote and reuses the Plugins activation and configuration slots. The four setup rows remain visible while unknown fields use placeholders. The default-mode and execution-mode forms, system checks, application decisions, and application icons fill in as each query completes. Application names and actions do not wait for icons; icon positions retain their space. User actions run in order without waiting for pending observation queries. Disposal waits for both actions and queries to settle before releasing the separately owned Remote mount. Query errors stay with the affected section, and activation guidance depends only on system checks. Late results after disposal are discarded. One controller projects queries into the page and setup dialog; reconnects and updates to the Host-reported settings namespace refresh it. Operation notices live in the shell overlay, so closing the page does not remove them. The provider owns permission checks, persisted decisions, and cancellation; the browser owns no authorization state. -The detail page reads the effective default (`selectedDefault ?? default`) from the existing `configForms` service. Its button writes only `selectedDefault` through the form's revision-checked settings path. Loading, read-only and unavailable settings cannot submit a change; a failed write remains beside the setting for retry. +The detail page reads the effective default (`selectedDefault ?? default`) from the existing `configForms` service. The default button writes only `selectedDefault`; the execution-mode button writes only `executionMode` on `ptc-runtime`. Both use the form's revision-checked settings path and existing HMR. Loading, read-only and unavailable settings cannot submit a change; a failed write remains beside the setting for retry.
diff --git a/packages/experimental/client-ui-cua/README.zh.md b/packages/experimental/client-ui-cua/README.zh.md index 3f75fcf364..7cb9cd0532 100644 --- a/packages/experimental/client-ui-cua/README.zh.md +++ b/packages/experimental/client-ui-cua/README.zh.md @@ -27,11 +27,11 @@ kind: "package-reference" ## 使用本包 -[Computer Use 组合](../cua-bundle/README.zh.md)将本页与原生 provider 一起挂载。“给 Computer Use 授权”把新会话默认模式、辅助功能和屏幕录制放在同一个边框内，三排之间使用分隔线。两条系统权限要求在能识别责任应用时，把该应用的 `.app` 包名称和图标放在同一个标签中。“管理 Computer Use 可访问的应用”使用独立边框包住应用列表。点击“申请权限”，在 macOS 权限列表中找到该应用并开启对应开关；若 macOS 显示其他名称，以系统提示为准。来源无法识别时，页面引导用户查找系统提示的应用。“打开系统设置”会打开对应的隐私设置页。进入详情页、返回窗口或完成授权操作后会自动刷新；查询失败时可点击“重试”。若系统要求退出并重新打开应用，请按提示操作并重新启动 DSH。 +[Computer Use 组合](../cua-bundle/README.zh.md)将本页与原生 provider 一起挂载。“给 Computer Use 授权”把新会话默认模式、PTC 执行方式、辅助功能和屏幕录制放在同一个边框内，四排之间使用分隔线。两条系统权限要求在能识别责任应用时，把该应用的 `.app` 包名称和图标放在同一个标签中。“管理 Computer Use 可访问的应用”使用独立边框包住应用列表。点击“申请权限”，在 macOS 权限列表中找到该应用并开启对应开关；若 macOS 显示其他名称，以系统提示为准。来源无法识别时，页面引导用户查找系统提示的应用。“打开系统设置”会打开对应的隐私设置页。进入详情页、返回窗口或完成授权操作后会自动刷新；查询失败时可点击“重试”。若系统要求退出并重新打开应用，请按提示操作并重新启动 DSH。 系统权限操作作用于运行 DSH 的电脑。远程访问时，请在该电脑上完成授权。 -从插件列表启用后，系统权限尚未就绪时会显示设置引导，提供“稍后”和“前往设置”。从详情页启用后直接显示权限控件。详情页的“新会话默认模式”提供“设为 PTC”按钮，仅点击此按钮才保存默认值。已有会话保留原 preset。 +从插件列表启用后，系统权限尚未就绪时会显示设置引导，提供“稍后”和“前往设置”。从详情页启用后直接显示权限控件。详情页的“新会话默认模式”提供“设为 PTC”按钮，仅点击此按钮才保存默认值。已有会话保留原 preset。“PTC 执行方式”一行的“设为持久化”单独保存 `ptc-runtime.executionMode: persistent-in-memory`。切换会重载共享 PTC 执行器，可能打断当前调用；后续调用在内存中保留变量和应用对象。已经使用持久化模式时，按钮显示“已设为持久化”并禁用。 页面展示“始终允许的应用”和“不允许访问的应用”。后者汇总任一会话的拒绝记录，不取消其他会话已有的授权，也不阻止新会话询问授权。会话授权仍在对应会话中生效，详情页不展示这类应用。“撤销”会移除对应应用的访问许可并取消相关操作。已发送到应用的输入无法撤回，继续操作前需重新观察应用。“允许再次询问”清除该应用在所有会话中的拒绝记录；下次选择应用时仍需用户重新决定。 @@ -45,9 +45,9 @@ kind: "package-reference"
实现细节 -可选浏览器插件挂载原生 provider 的权限 Remote，复用插件页的启用引导和配置插槽。三排使用条件始终可见，未加载的字段保留占位。默认模式、系统权限、应用列表和应用图标分别查询，结果返回后各自显示。应用名称和操作按钮不等待图标，图标位置保留空位。用户操作按顺序执行，不等待尚未完成的查询；卸载时等待操作和查询全部结束，再释放独立持有的 Remote 挂载。查询错误只显示在对应部分，启用引导只等待系统权限结果。卸载后的迟到结果会被丢弃。一个控制器把查询结果投影到页面和设置弹窗；重新连接或 Host 返回的配置 namespace 更新时刷新状态。操作提示挂载到 shell overlay，关闭页面后仍可显示。provider 负责权限检查、持久授权和取消；浏览器不保存授权决定。 +可选浏览器插件挂载原生 provider 的权限 Remote，复用插件页的启用引导和配置插槽。四排使用条件始终可见，未加载的字段保留占位。默认模式、执行方式、系统权限、应用列表和应用图标分别查询，结果返回后各自显示。应用名称和操作按钮不等待图标，图标位置保留空位。用户操作按顺序执行，不等待尚未完成的查询；卸载时等待操作和查询全部结束，再释放独立持有的 Remote 挂载。查询错误只显示在对应部分，启用引导只等待系统权限结果。卸载后的迟到结果会被丢弃。一个控制器把查询结果投影到页面和设置弹窗；重新连接或 Host 返回的配置 namespace 更新时刷新状态。操作提示挂载到 shell overlay，关闭页面后仍可显示。provider 负责权限检查、持久授权和取消；浏览器不保存授权决定。 -详情页通过现有 `configForms` 服务读取实际默认值（`selectedDefault ?? default`）。按钮仅写入 `selectedDefault`，沿用表单带版本检查的设置写入流程。设置仍在加载、只读或不可用时不能提交；写入失败时在设置旁显示错误，供用户重试。 +详情页通过现有 `configForms` 服务读取实际默认值（`selectedDefault ?? default`）。默认模式按钮仅写入 `selectedDefault`；执行方式按钮仅写入 `ptc-runtime` 的 `executionMode`。两者沿用表单带版本检查的设置写入流程和现有 HMR。设置仍在加载、只读或不可用时不能提交；写入失败时在设置旁显示错误，供用户重试。
diff --git a/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx b/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx index faa83a783d..b8312fcfe5 100644 --- a/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx +++ b/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx @@ -3,7 +3,7 @@ import { useEffect, useState } from 'react' import { Button, Modal, Toast } from '@deepseek-ai/dsh-client-ui-primitives' import type { InjectFace, PropsLocale, PropsRuntime } from '@deepseek-ai/dsh-client-ui-slots' import type {} from '@deepseek-ai/dsh-client-ui-plugin-manager/client' -import type { ConfigForm } from '@deepseek-ai/dsh-client-ui-settings/client' +import type { ConfigForm, ConfigFormSnapshot } from '@deepseek-ai/dsh-client-ui-settings/client' import type { CuaApplicationGrant, CuaSystemPermissionState } from '@deepseek-ai/dsh-cua/types' import type { PermissionFace } from './controller.ts' import type { NS } from './locales.ts' @@ -21,34 +21,53 @@ type DefaultPresetProps = PropsLocale & InjectFace<{ setPtcDefault: () => Promise }> -/** The detail page owns the optional new-session default action. */ -export type PermissionPanelProps = PermissionProps & DefaultPresetProps +type PersistentPtcProps = PropsLocale & InjectFace<{ + hooks: { cuaPtcRuntime: ConfigForm<{ executionMode?: 'per-call' | 'persistent-in-memory' }> } + setPersistentPtc: () => Promise +}> + +/** The detail page owns explicit preset and execution-mode settings. */ +export type PermissionPanelProps = PermissionProps & DefaultPresetProps & PersistentPtcProps function DefaultPresetSetting(props: DefaultPresetProps) { - const { t } = props const preset = props.useCuaDefaultPreset(value => value) - const isPtc = (preset.value?.selectedDefault ?? preset.value?.default) === 'ptc' - const loading = preset.value === undefined && preset.status === 'loading' + return +} + +function PersistentPtcSetting(props: PersistentPtcProps) { + const runtime = props.useCuaPtcRuntime(value => value) + return +} + +function ConfigurationSetting({ kind, setting, configured, save, t }: PropsLocale & { + kind: 'default' | 'persistent' + setting: ConfigFormSnapshot + configured: boolean + save: () => Promise +}) { + const loading = setting.value === undefined && setting.status === 'loading' const [saving, setSaving] = useState(false) const [error, setError] = useState(null) - const setDefault = async (): Promise => { + const submit = async (): Promise => { setSaving(true) setError(null) try { - if (!await props.setPtcDefault()) setError(t('defaultNotSaved')) + if (!await save()) setError(t(`${kind}NotSaved`)) } catch (error) { - setError(t('defaultFailed', { message: error instanceof Error ? error.message : String(error) })) + setError(t(`${kind}Failed`, { message: error instanceof Error ? error.message : String(error) })) } finally { setSaving(false) } } - return
+ return
-
{t('defaultTitle')}

{t('defaultBody')}

+
{t(`${kind}Title`)}

{t(`${kind}Body`)}

{loading ? - : } + : }
- {!isPtc && (preset.status === 'unavailable' ?

{t('defaultUnavailable')}

- : preset.status === 'ready' && !preset.writable &&

{t('defaultReadOnly')}

)} + {!configured && (setting.status === 'unavailable' ?

{t(`${kind}Unavailable`)}

+ : setting.status === 'ready' && !setting.writable &&

{t(`${kind}ReadOnly`)}

)} {error !== null &&

{error}

}
} @@ -113,6 +132,7 @@ export function PermissionPanel(props: PermissionPanelProps) {

{t('title')}

+ {!busy && state.system.error !== null &&
{t('queryFailed', { message: state.system.error })} diff --git a/packages/experimental/client-ui-cua/src/client/locales.ts b/packages/experimental/client-ui-cua/src/client/locales.ts index 52f286700e..9d28f99a5e 100644 --- a/packages/experimental/client-ui-cua/src/client/locales.ts +++ b/packages/experimental/client-ui-cua/src/client/locales.ts @@ -14,6 +14,13 @@ export const en = { defaultReadOnly: 'Default-mode settings are read-only.', defaultNotSaved: 'The default mode was not saved. Try again.', defaultFailed: 'Could not change the default mode: {message}', + persistentTitle: 'PTC execution mode', + persistentBody: 'Keep variables and application objects across tool calls. Switching restarts the PTC runtime.', + persistentConfirm: 'Use persistent mode', persistentSet: 'Persistent mode enabled', + persistentUnavailable: 'Could not read the PTC execution mode. Check it in Settings.', + persistentReadOnly: 'PTC execution settings are read-only.', + persistentNotSaved: 'The PTC execution mode was not saved. Try again.', + persistentFailed: 'Could not change the PTC execution mode: {message}', openSetup: 'Open settings', later: 'Later', close: 'Close', accessibility: 'Accessibility', screenRecording: 'Screen Recording', accessibilityUse: 'Read interface controls, click, and type.', screenRecordingUse: 'Capture the selected application window.', @@ -44,6 +51,13 @@ export const zh: Record = { defaultReadOnly: '默认模式设置为只读', defaultNotSaved: '默认模式未保存，请重试', defaultFailed: '无法切换默认模式：{message}', + persistentTitle: 'PTC 执行方式', + persistentBody: '跨工具调用保留变量和应用对象。切换会重启 PTC 执行器。', + persistentConfirm: '设为持久化', persistentSet: '已设为持久化', + persistentUnavailable: '无法读取 PTC 执行方式，请在设置中检查', + persistentReadOnly: 'PTC 执行方式设置为只读', + persistentNotSaved: 'PTC 执行方式未保存，请重试', + persistentFailed: '无法切换 PTC 执行方式：{message}', openSetup: '前往设置', later: '稍后', close: '关闭', accessibility: '辅助功能', screenRecording: '屏幕录制', accessibilityUse: '读取界面控件、点击和输入文字', screenRecordingUse: '截取所选应用的窗口', diff --git a/packages/experimental/client-ui-cua/src/client/mount.ts b/packages/experimental/client-ui-cua/src/client/mount.ts index a475c43d8e..66270c9dc5 100644 --- a/packages/experimental/client-ui-cua/src/client/mount.ts +++ b/packages/experimental/client-ui-cua/src/client/mount.ts @@ -53,10 +53,12 @@ export async function mountCuaPermissions(ctx: Context, contribution: TypertRemo }) const face = controller.face() const preset = ctx.configForms.get<{ default: string; selectedDefault?: string }>('agent-preset-registry') + const runtime = ctx.configForms.get<{ executionMode?: 'per-call' | 'persistent-in-memory' }>('ptc-runtime') const settings = { ...face, - hooks: { ...face.hooks, cuaDefaultPreset: preset }, + hooks: { ...face.hooks, cuaDefaultPreset: preset, cuaPtcRuntime: runtime }, setPtcDefault: () => preset.set('selectedDefault', 'ptc'), + setPersistentPtc: () => runtime.set('executionMode', 'persistent-in-memory'), } ctx.effect(() => () => controller.dispose()) ctx.on('connection/reset', face.refresh) diff --git a/packages/experimental/client-ui-cua/tests/mount.client.spec.ts b/packages/experimental/client-ui-cua/tests/mount.client.spec.ts index df6732f404..8118a1a6e1 100644 --- a/packages/experimental/client-ui-cua/tests/mount.client.spec.ts +++ b/packages/experimental/client-ui-cua/tests/mount.client.spec.ts @@ -37,10 +37,11 @@ async function fixture(fail = false) { $on(_event: string, listener: (namespace: string, revision: number) => void) { settingsListener = listener; return unsubscribe } } new Remote() - const setDefault = vi.fn(async () => true) + const setDefault = vi.fn(async () => true), setPersistent = vi.fn(async () => true) const preset = createSnapshotStore({ status: 'ready' as const, value: { default: 'standard' }, base: {}, user: {}, revision: 1, writable: true }) ctx.provide('configForms', { get: (namespace: string) => { + if (namespace === 'ptc-runtime') return { ...preset, set: setPersistent } expect(namespace).toBe('agent-preset-registry') return { ...preset, set: setDefault } } }) @@ -61,7 +62,7 @@ async function fixture(fail = false) { } } as never, () => null) if (fail) vi.spyOn(ctx.slots, 'inject').mockImplementationOnce(() => { throw new Error('slot unavailable') }) return { ctx, unmount, unsubscribe, status, systemStatus, applicationIcons, - requestPermission, openSettings, revoke, retryApproval, setDefault, + requestPermission, openSettings, revoke, retryApproval, setDefault, setPersistent, updateSettings(namespace: string) { settingsListener!(namespace, 1) }, } } @@ -115,6 +116,11 @@ it('withdraws the permission Remote and all controls with the optional plugin', expect(f.setDefault).not.toHaveBeenCalled() await (settings.setPtcDefault as () => Promise)() expect(f.setDefault).toHaveBeenCalledExactlyOnceWith('selectedDefault', 'ptc') + assert(typeof settings.setPersistentPtc === 'function') + expect(f.ctx.slots.entries('plugins.bundle.activation')[0]!.inject!()).not.toHaveProperty('setPersistentPtc') + expect(f.setPersistent).not.toHaveBeenCalled() + await (settings.setPersistentPtc as () => Promise)() + expect(f.setPersistent).toHaveBeenCalledExactlyOnceWith('executionMode', 'persistent-in-memory') await fiber.dispose() expect(f.ctx.slots.entries('conversation.composer')).toHaveLength(0) expect(f.ctx.slots.entries('plugins.bundle.config')).toHaveLength(0) diff --git a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx b/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx index e29768cdf9..de824f0bcc 100644 --- a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx +++ b/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx @@ -29,13 +29,17 @@ function fixture() { status: 'ready', value: { default: 'standard', selectedDefault: 'ptc' }, base: {}, user: {}, revision: 1, writable: true, }) + const runtime = createSnapshotStore>({ + status: 'ready', value: { executionMode: 'per-call' }, base: {}, user: {}, revision: 1, writable: true, + }) const refresh = vi.fn() const props = { useCuaPermissions: bindSnapshotSelector(state), t: makeTranslate(en), + useCuaPtcRuntime: bindSnapshotSelector(runtime), setPersistentPtc: vi.fn(async () => true), useCuaDefaultPreset: bindSnapshotSelector(preset), setPtcDefault: vi.fn(async () => true), refresh, request: vi.fn(), openSettings: vi.fn(), revoke: vi.fn(), retryApproval: vi.fn(), dismissToast: vi.fn(), } satisfies PermissionPanelProps - return { state, preset, props, + return { state, preset, runtime, props, publish(patch: Partial) { act(() => { state.set({ ...state.getSnapshot(), ...patch }) }) }, } } @@ -220,13 +224,13 @@ it('shows application icons and names while keeping the executable path in the t expect(identity.querySelector('img')?.getAttribute('src')).toBe('data:image/png;base64,aWNvbg==') }) -it.each([en, zh])('shows three requirements with the target application beside each system permission', (dictionary) => { +it.each([en, zh])('shows four requirements with the target application beside each system permission', (dictionary) => { const f = fixture() render() const application = { name: 'Visual Studio Code.app', path: '/Applications/Visual Studio Code.app', icon: 'data:image/png;base64,aWNvbg==' } f.publish({ ...observe({ ...value, system: { ...value.system, parent: { pid: 1, path: '/sbin/launchd' }, launchApplication: application } }) }) const requirements = screen.getByRole('region', { name: dictionary.title }) - expect(within(requirements).getAllByRole('group')).toHaveLength(3) + expect(within(requirements).getAllByRole('group')).toHaveLength(4) expect(screen.getByRole('region', { name: dictionary.applicationsTitle })).toBeTruthy() const accessibility = within(requirements).getByRole('group', { name: dictionary.accessibility }) expect(within(accessibility).getByText(application.name)).toBeTruthy() @@ -271,11 +275,11 @@ it('omits setup instructions once both system permissions are granted', () => { }) -it('keeps all three requirements visible before the first permission response', () => { +it('keeps all four requirements visible before the first permission response', () => { const f = fixture() render() const requirements = screen.getByRole('region', { name: en.title }) - expect(within(requirements).getAllByRole('group')).toHaveLength(3) + expect(within(requirements).getAllByRole('group')).toHaveLength(4) expect(screen.queryByRole('alert')).toBeNull() expect(screen.queryByRole('button', { name: en.request })).toBeNull() for (const button of screen.getAllByRole('button', { name: en.settings })) expect(button.hasAttribute('disabled')).toBe(true) @@ -327,3 +331,36 @@ it('finishes activation guidance without waiting for application icons', () => { expect(screen.queryByRole('dialog')).toBeNull() expect(f.state.getSnapshot().icons.busy).toBe(true) }) + +it.each([en, zh])('saves persistent PTC only from its own row and retains failures for retry', async (dictionary) => { + const f = fixture() + render() + const row = screen.getByRole('group', { name: dictionary.persistentTitle }) + expect(f.props.setPersistentPtc).not.toHaveBeenCalled() + const write = Promise.withResolvers() + f.props.setPersistentPtc.mockReturnValueOnce(write.promise) + fireEvent.click(within(row).getByRole('button', { name: dictionary.persistentConfirm })) + expect(within(row).getByRole('button').hasAttribute('disabled')).toBe(true) + await act(async () => { write.resolve(false); await write.promise }) + expect(within(row).getByRole('alert').textContent).toBe(dictionary.persistentNotSaved) + fireEvent.click(within(row).getByRole('button')) + await waitFor(() => { expect(within(row).queryByRole('alert')).toBeNull() }) + act(() => { f.runtime.set({ ...f.runtime.getSnapshot(), value: { executionMode: 'persistent-in-memory' } }) }) + expect(within(row).getByRole('button', { name: dictionary.persistentSet }).hasAttribute('disabled')).toBe(true) + expect(f.props.setPtcDefault).not.toHaveBeenCalled() +}) + +it('loads the runtime setting independently and reports unavailable and read-only configurations', () => { + const f = fixture() + f.runtime.set({ ...f.runtime.getSnapshot(), status: 'loading', value: undefined }) + render() + const row = screen.getByRole('group', { name: en.persistentTitle }) + expect(within(row).queryByRole('button')).toBeNull() + expect(screen.getByRole('button', { name: en.defaultSet })).toBeTruthy() + act(() => { f.runtime.set({ ...f.runtime.getSnapshot(), status: 'unavailable', writable: false }) }) + expect(within(row).getByRole('alert').textContent).toBe(en.persistentUnavailable) + act(() => { f.runtime.set({ ...f.runtime.getSnapshot(), status: 'ready', value: {}, writable: false }) }) + expect(within(row).getByRole('alert').textContent).toBe(en.persistentReadOnly) + expect(within(row).getByRole('button').hasAttribute('disabled')).toBe(true) + expect(f.props.setPersistentPtc).not.toHaveBeenCalled() +}) diff --git a/packages/experimental/cua-bundle/README.i18n.yaml b/packages/experimental/cua-bundle/README.i18n.yaml index e379b108fa..f84fa9dcaa 100644 --- a/packages/experimental/cua-bundle/README.i18n.yaml +++ b/packages/experimental/cua-bundle/README.i18n.yaml @@ -15,8 +15,8 @@ en: b563e4ec4727d0de zh: d0228da978238e05 /deepseek-ai-dsh-cua-bundle/use-this-package: - en: 7d378196959b1629 - zh: e735e217d1e35216 + en: c50ddfc2865ed10f + zh: f0ee5fd582c7e034 /deepseek-ai-dsh-cua-bundle/understand-the-implementation: en: 057771fe13aaa337 zh: 4fdb2b98ff3e7a78 diff --git a/packages/experimental/cua-bundle/README.md b/packages/experimental/cua-bundle/README.md index d87df12f07..921bc05f20 100644 --- a/packages/experimental/cua-bundle/README.md +++ b/packages/experimental/cua-bundle/README.md @@ -33,7 +33,7 @@ pnpm dsh web For the source Desktop launcher and its separate development profile, follow [Desktop development](../../../apps/desktop/README.md). Enable Computer Use in each profile that needs it. -The switch saves bundle selection in the profile’s `package.json`. Computer Use is installed only in PTC sessions. Enabling from the plugin list offers setup guidance when system permissions need attention; enabling from its detail page displays the permission controls directly. The detail page offers **Set to PTC** for the new-session default; existing sessions keep their preset. The switch leaves PTC execution mode and other settings unchanged. Native preparation runs when the provider loads and reuses the source-hashed executable when available. +The switch saves bundle selection in the profile’s `package.json`. Computer Use is installed only in PTC sessions. Enabling from the plugin list offers setup guidance when system permissions need attention; enabling from its detail page displays the permission controls directly. The detail page offers **Set to PTC** for the new-session default; existing sessions keep their preset. The switch leaves PTC execution mode and other settings unchanged. The detail page’s **Use persistent mode** action separately enables in-memory state across PTC calls; its effects and reload behavior are described in the [permission page](../client-ui-cua/README.md). Native preparation runs when the provider loads and reuses the source-hashed executable when available. Disabling unloads the provider and withdraws CUA tools from its Agent scopes. Runtime configuration, the new-session default and saved grants remain. Session-only application decisions end with provider disposal. diff --git a/packages/experimental/cua-bundle/README.zh.md b/packages/experimental/cua-bundle/README.zh.md index b6446f65f8..6c551c5a09 100644 --- a/packages/experimental/cua-bundle/README.zh.md +++ b/packages/experimental/cua-bundle/README.zh.md @@ -33,7 +33,7 @@ pnpm dsh web 源码 Desktop 的启动方式和独立开发配置档见[桌面开发说明](../../../apps/desktop/README.zh.md)。需要使用 Computer Use 的配置档分别开启该功能。 -开关将组合包选择保存到 profile 的 `package.json`。Computer Use 仅安装在 PTC 会话中。从插件列表启用时，系统权限未就绪会显示设置引导；从详情页启用后直接显示权限控件。详情页提供“设为 PTC”按钮来修改新会话默认模式，已有会话保留原 preset。开关保持 PTC 执行方式和其他设置不变。提供方加载时准备本机组件，已有与源码摘要匹配的可执行文件时直接复用。 +开关将组合包选择保存到 profile 的 `package.json`。Computer Use 仅安装在 PTC 会话中。从插件列表启用时，系统权限未就绪会显示设置引导；从详情页启用后直接显示权限控件。详情页提供“设为 PTC”按钮来修改新会话默认模式，已有会话保留原 preset。开关保持 PTC 执行方式和其他设置不变。详情页的“设为持久化”按钮单独开启 PTC 跨调用的内存状态保留，作用范围和重载行为见[权限页面](../client-ui-cua/README.zh.md)。提供方加载时准备本机组件，已有与源码摘要匹配的可执行文件时直接复用。 关闭会卸载提供方，并从其 Agent 作用域中撤回 CUA 工具。执行器配置、新会话默认值和已保存的授权保留。会话级应用授权随提供方卸载而失效。 From b524c7cc19df2208684ef574eb5affe03c107d55 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 15:24:37 +0800 Subject: [PATCH 33/55] fix(cua): use the profile editor for persistent runtime changes --- docs/subsystems/computer-use.md | 10 +++++++ docs/subsystems/computer-use.zh.md | 10 +++++++ .../client-ui-cua/README.i18n.yaml | 4 +-- packages/experimental/client-ui-cua/README.md | 4 +-- .../experimental/client-ui-cua/README.zh.md | 2 +- .../src/client/PermissionPanel.tsx | 29 +++++++++---------- .../client-ui-cua/src/client/controller.ts | 6 +++- .../client-ui-cua/src/client/mount.ts | 12 ++++++-- .../tests/controller.client.spec.ts | 4 ++- .../client-ui-cua/tests/mount.client.spec.ts | 8 +++-- .../client-ui-cua/tests/panel.client.spec.tsx | 16 +++++----- packages/experimental/cua/README.i18n.yaml | 4 +-- packages/experimental/cua/README.md | 2 ++ packages/experimental/cua/README.zh.md | 2 ++ packages/experimental/cua/src/access.ts | 25 ++++++++++++++++ .../experimental/cua/tests/access.spec.ts | 21 ++++++++++++++ .../extensions/tool-cordis/src/api-catalog.ts | 12 ++++++++ 17 files changed, 132 insertions(+), 39 deletions(-) diff --git a/docs/subsystems/computer-use.md b/docs/subsystems/computer-use.md index efff79fb54..a00e0f6e04 100644 --- a/docs/subsystems/computer-use.md +++ b/docs/subsystems/computer-use.md @@ -74,6 +74,16 @@ Source: [`packages/computer-use/computer-use/src/index.ts`](../../packages/compu Browser controls for provider-owned native permission and application decisions. ```ts cordis-catalog +/** Read PTC execution state separately from native permission checks. + * @returns Whether the shared runtime retains memory and has an editable profile entry. + */ +@Remote ptcConfiguration(): { persistent: boolean; writable: boolean } + +/** Persist in-memory PTC execution through the existing profile editor and Loader lifecycle. + * @returns Completion after the shared runtime reload; already-persistent runtimes are left intact. + */ +@Remote async enablePersistentPtc(): Promise + /** * Read application decisions without waiting for the native helper. * @returns Saved grants and refused applications with their persistence owner. diff --git a/docs/subsystems/computer-use.zh.md b/docs/subsystems/computer-use.zh.md index 997bf7523c..00ff655df2 100644 --- a/docs/subsystems/computer-use.zh.md +++ b/docs/subsystems/computer-use.zh.md @@ -74,6 +74,16 @@ Source: [`packages/computer-use/computer-use/src/index.ts`](../../packages/compu Browser controls for provider-owned native permission and application decisions. ```ts cordis-catalog +/** Read PTC execution state separately from native permission checks. + * @returns Whether the shared runtime retains memory and has an editable profile entry. + */ +@Remote ptcConfiguration(): { persistent: boolean; writable: boolean } + +/** Persist in-memory PTC execution through the existing profile editor and Loader lifecycle. + * @returns Completion after the shared runtime reload; already-persistent runtimes are left intact. + */ +@Remote async enablePersistentPtc(): Promise + /** * Read application decisions without waiting for the native helper. * @returns Saved grants and refused applications with their persistence owner. diff --git a/packages/experimental/client-ui-cua/README.i18n.yaml b/packages/experimental/client-ui-cua/README.i18n.yaml index b5602c9448..9d1ae66b95 100644 --- a/packages/experimental/client-ui-cua/README.i18n.yaml +++ b/packages/experimental/client-ui-cua/README.i18n.yaml @@ -18,8 +18,8 @@ en: 57485612d3ebe444 zh: b1886ab8e3c313af /deepseek-ai-dsh-client-ui-cua/understand-the-implementation: - en: 6ec7497752efaf72 - zh: 6bcd181aa26ce289 + en: f16ebf34970a7720 + zh: f1d79b3a60df361c /deepseek-ai-dsh-client-ui-cua/further-exploration: en: f0fde9f736ade61a zh: 43a534a21bcc1c13 diff --git a/packages/experimental/client-ui-cua/README.md b/packages/experimental/client-ui-cua/README.md index 0528df4203..b0c9ae7a12 100644 --- a/packages/experimental/client-ui-cua/README.md +++ b/packages/experimental/client-ui-cua/README.md @@ -45,9 +45,9 @@ This plugin has no configurable fields.
Implementation internals -The optional browser contribution mounts the native provider's permission Remote and reuses the Plugins activation and configuration slots. The four setup rows remain visible while unknown fields use placeholders. The default-mode and execution-mode forms, system checks, application decisions, and application icons fill in as each query completes. Application names and actions do not wait for icons; icon positions retain their space. User actions run in order without waiting for pending observation queries. Disposal waits for both actions and queries to settle before releasing the separately owned Remote mount. Query errors stay with the affected section, and activation guidance depends only on system checks. Late results after disposal are discarded. One controller projects queries into the page and setup dialog; reconnects and updates to the Host-reported settings namespace refresh it. Operation notices live in the shell overlay, so closing the page does not remove them. The provider owns permission checks, persisted decisions, and cancellation; the browser owns no authorization state. +The optional browser contribution mounts the native provider's permission Remote and reuses the Plugins activation and configuration slots. The four setup rows remain visible while unknown fields use placeholders. The default-mode form, runtime configuration, system checks, application decisions, and application icons fill in as each query completes. Application names and actions do not wait for icons; icon positions retain their space. User actions run in order without waiting for pending observation queries. Disposal waits for both actions and queries to settle before releasing the separately owned Remote mount. Query errors stay with the affected section, and activation guidance depends only on system checks. Late results after disposal are discarded. One controller projects queries into the page and setup dialog; reconnects and updates to the Host-reported settings namespace refresh it. Operation notices live in the shell overlay, so closing the page does not remove them. The provider owns permission checks, persisted decisions, and cancellation; the browser owns no authorization state. -The detail page reads the effective default (`selectedDefault ?? default`) from the existing `configForms` service. The default button writes only `selectedDefault`; the execution-mode button writes only `executionMode` on `ptc-runtime`. Both use the form's revision-checked settings path and existing HMR. Loading, read-only and unavailable settings cannot submit a change; a failed write remains beside the setting for retry. +The detail page reads the effective default (`selectedDefault ?? default`) from the existing `configForms` service. The default button writes only `selectedDefault` through the form's revision-checked settings path. The execution-mode row uses CUA's `ptcConfiguration()` and `enablePersistentPtc()` controls: the Host edits only `ptc-runtime.executionMode` through the existing configuration editor, preserving its ordinary Loader reload lifecycle. Permission checks and runtime state settle independently. Loading, read-only and unavailable settings cannot submit a change; a failed write remains beside the setting for retry.
diff --git a/packages/experimental/client-ui-cua/README.zh.md b/packages/experimental/client-ui-cua/README.zh.md index 7cb9cd0532..09a011c6d8 100644 --- a/packages/experimental/client-ui-cua/README.zh.md +++ b/packages/experimental/client-ui-cua/README.zh.md @@ -47,7 +47,7 @@ kind: "package-reference" 可选浏览器插件挂载原生 provider 的权限 Remote，复用插件页的启用引导和配置插槽。四排使用条件始终可见，未加载的字段保留占位。默认模式、执行方式、系统权限、应用列表和应用图标分别查询，结果返回后各自显示。应用名称和操作按钮不等待图标，图标位置保留空位。用户操作按顺序执行，不等待尚未完成的查询；卸载时等待操作和查询全部结束，再释放独立持有的 Remote 挂载。查询错误只显示在对应部分，启用引导只等待系统权限结果。卸载后的迟到结果会被丢弃。一个控制器把查询结果投影到页面和设置弹窗；重新连接或 Host 返回的配置 namespace 更新时刷新状态。操作提示挂载到 shell overlay，关闭页面后仍可显示。provider 负责权限检查、持久授权和取消；浏览器不保存授权决定。 -详情页通过现有 `configForms` 服务读取实际默认值（`selectedDefault ?? default`）。默认模式按钮仅写入 `selectedDefault`；执行方式按钮仅写入 `ptc-runtime` 的 `executionMode`。两者沿用表单带版本检查的设置写入流程和现有 HMR。设置仍在加载、只读或不可用时不能提交；写入失败时在设置旁显示错误，供用户重试。 +详情页通过现有 `configForms` 服务读取实际默认值（`selectedDefault ?? default`）。默认模式按钮仅写入 `selectedDefault`，沿用表单带版本检查的设置写入流程。执行方式一行调用 CUA 的 `ptcConfiguration()` 和 `enablePersistentPtc()`：Host 通过现有配置编辑器仅修改 `ptc-runtime.executionMode`，保留普通 Loader 重载规则。权限检查和执行方式状态独立返回。设置仍在加载、只读或不可用时不能提交；写入失败时在设置旁显示错误，供用户重试。 diff --git a/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx b/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx index b8312fcfe5..535e22b666 100644 --- a/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx +++ b/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx @@ -3,7 +3,7 @@ import { useEffect, useState } from 'react' import { Button, Modal, Toast } from '@deepseek-ai/dsh-client-ui-primitives' import type { InjectFace, PropsLocale, PropsRuntime } from '@deepseek-ai/dsh-client-ui-slots' import type {} from '@deepseek-ai/dsh-client-ui-plugin-manager/client' -import type { ConfigForm, ConfigFormSnapshot } from '@deepseek-ai/dsh-client-ui-settings/client' +import type { ConfigForm } from '@deepseek-ai/dsh-client-ui-settings/client' import type { CuaApplicationGrant, CuaSystemPermissionState } from '@deepseek-ai/dsh-cua/types' import type { PermissionFace } from './controller.ts' import type { NS } from './locales.ts' @@ -21,33 +21,32 @@ type DefaultPresetProps = PropsLocale & InjectFace<{ setPtcDefault: () => Promise }> -type PersistentPtcProps = PropsLocale & InjectFace<{ - hooks: { cuaPtcRuntime: ConfigForm<{ executionMode?: 'per-call' | 'persistent-in-memory' }> } - setPersistentPtc: () => Promise -}> +type PersistentPtcProps = PermissionProps & { setPersistentPtc: () => Promise } /** The detail page owns explicit preset and execution-mode settings. */ export type PermissionPanelProps = PermissionProps & DefaultPresetProps & PersistentPtcProps function DefaultPresetSetting(props: DefaultPresetProps) { const preset = props.useCuaDefaultPreset(value => value) - return } function PersistentPtcSetting(props: PersistentPtcProps) { - const runtime = props.useCuaPtcRuntime(value => value) - return + const runtime = props.useCuaPermissions(value => value.runtime) + return } -function ConfigurationSetting({ kind, setting, configured, save, t }: PropsLocale & { +function ConfigurationSetting({ kind, status, loading, writable, configured, save, t }: PropsLocale & { kind: 'default' | 'persistent' - setting: ConfigFormSnapshot + status: 'loading' | 'ready' | 'unavailable' + loading: boolean + writable: boolean configured: boolean save: () => Promise }) { - const loading = setting.value === undefined && setting.status === 'loading' const [saving, setSaving] = useState(false) const [error, setError] = useState(null) const submit = async (): Promise => { @@ -63,11 +62,11 @@ function ConfigurationSetting({ kind, setting, configured, save, t }: PropsLocal
{t(`${kind}Title`)}

{t(`${kind}Body`)}

{loading ? - : }
- {!configured && (setting.status === 'unavailable' ?

{t(`${kind}Unavailable`)}

- : setting.status === 'ready' && !setting.writable &&

{t(`${kind}ReadOnly`)}

)} + {!configured && (status === 'unavailable' ?

{t(`${kind}Unavailable`)}

+ : status === 'ready' && !writable &&

{t(`${kind}ReadOnly`)}

)} {error !== null &&

{error}

}
} diff --git a/packages/experimental/client-ui-cua/src/client/controller.ts b/packages/experimental/client-ui-cua/src/client/controller.ts index 044f431e2e..9a29b04436 100644 --- a/packages/experimental/client-ui-cua/src/client/controller.ts +++ b/packages/experimental/client-ui-cua/src/client/controller.ts @@ -4,6 +4,7 @@ import type { CuaApplicationId, CuaApplicationState, CuaSystemPermission, CuaSys /** Host operations available to the permission UI. */ export interface PermissionActions { + readRuntime: () => Promise<{ persistent: boolean; writable: boolean }> readSystem: () => Promise readApplications: () => Promise readIcons: () => Promise> @@ -13,6 +14,7 @@ export interface PermissionActions { retryApproval: (id: CuaApplicationId) => Promise } interface PermissionData { + runtime: { persistent: boolean; writable: boolean } system: CuaSystemPermissionState applications: CuaApplicationState icons: Record @@ -43,6 +45,7 @@ export interface PermissionFace { export class PermissionController { /** Last observed permissions and the lifetime of the current refresh/action. */ readonly state = createSnapshotStore({ + runtime: { value: null, busy: false, error: null }, system: { value: null, busy: false, error: null }, applications: { value: null, busy: false, error: null }, icons: { value: null, busy: false, error: null }, @@ -77,7 +80,7 @@ export class PermissionController { const sequence = ++this.sequence this.state.update((state) => { state.busy = true - for (const key of ['system', 'applications', 'icons'] as const) state[key].busy = true + for (const key of ['runtime', 'system', 'applications', 'icons'] as const) state[key].busy = true }) this.tail = this.tail.then(async () => { if (!current()) return @@ -92,6 +95,7 @@ export class PermissionController { } const latest = () => current() && this.sequence === sequence const refresh = Promise.all([ + this.read('runtime', this.actions.readRuntime, latest), this.read('system', this.actions.readSystem, latest), this.read('applications', this.actions.readApplications, latest), this.read('icons', this.actions.readIcons, latest), diff --git a/packages/experimental/client-ui-cua/src/client/mount.ts b/packages/experimental/client-ui-cua/src/client/mount.ts index 66270c9dc5..e01415b383 100644 --- a/packages/experimental/client-ui-cua/src/client/mount.ts +++ b/packages/experimental/client-ui-cua/src/client/mount.ts @@ -33,6 +33,9 @@ export async function mountCuaPermissions(ctx: Context, contribution: TypertRemo const ui = ctx.inject(['remote.cuaAccess', 'slots', 'locale'], (ctx) => { ctx.effect(() => ctx.locale.register(NS, { en, zh })) const controller = new PermissionController({ + readRuntime: async () => { + const result = await ctx.remote.cuaAccess.ptcConfiguration(); if (!result.ok) throw result.error; return result.value + }, readSystem: async () => { const result = await ctx.remote.cuaAccess.systemStatus(); if (!result.ok) throw result.error; return result.value }, @@ -53,12 +56,15 @@ export async function mountCuaPermissions(ctx: Context, contribution: TypertRemo }) const face = controller.face() const preset = ctx.configForms.get<{ default: string; selectedDefault?: string }>('agent-preset-registry') - const runtime = ctx.configForms.get<{ executionMode?: 'per-call' | 'persistent-in-memory' }>('ptc-runtime') const settings = { ...face, - hooks: { ...face.hooks, cuaDefaultPreset: preset, cuaPtcRuntime: runtime }, + hooks: { ...face.hooks, cuaDefaultPreset: preset }, setPtcDefault: () => preset.set('selectedDefault', 'ptc'), - setPersistentPtc: () => runtime.set('executionMode', 'persistent-in-memory'), + setPersistentPtc: async () => { + const result = await ctx.remote.cuaAccess.enablePersistentPtc(); if (!result.ok) throw result.error + face.refresh() + return true + }, } ctx.effect(() => () => controller.dispose()) ctx.on('connection/reset', face.refresh) diff --git a/packages/experimental/client-ui-cua/tests/controller.client.spec.ts b/packages/experimental/client-ui-cua/tests/controller.client.spec.ts index 8b776837f9..e05f2d5882 100644 --- a/packages/experimental/client-ui-cua/tests/controller.client.spec.ts +++ b/packages/experimental/client-ui-cua/tests/controller.client.spec.ts @@ -8,7 +8,9 @@ const value: CuaSystemPermissionState = { accessibility: true, screenRecording: const applications: CuaApplicationState = { allowedApps: [], blockedApps: [], settingsNamespace: 'cua', persistentAvailable: true } function fixture() { - const actions = { readSystem: vi.fn(async () => value), readApplications: vi.fn(async () => applications), + const actions = { + readRuntime: vi.fn(async () => ({ persistent: false, writable: true })), + readSystem: vi.fn(async () => value), readApplications: vi.fn(async () => applications), readIcons: vi.fn(async (): Promise> => ({})), request: vi.fn(async () => {}), openSettings: vi.fn(async () => {}), revoke: vi.fn(async () => {}), retryApproval: vi.fn(async () => {}) } const controller = new PermissionController(actions) diff --git a/packages/experimental/client-ui-cua/tests/mount.client.spec.ts b/packages/experimental/client-ui-cua/tests/mount.client.spec.ts index 8118a1a6e1..5e9e29cc32 100644 --- a/packages/experimental/client-ui-cua/tests/mount.client.spec.ts +++ b/packages/experimental/client-ui-cua/tests/mount.client.spec.ts @@ -41,7 +41,6 @@ async function fixture(fail = false) { const preset = createSnapshotStore({ status: 'ready' as const, value: { default: 'standard' }, base: {}, user: {}, revision: 1, writable: true }) ctx.provide('configForms', { get: (namespace: string) => { - if (namespace === 'ptc-runtime') return { ...preset, set: setPersistent } expect(namespace).toBe('agent-preset-registry') return { ...preset, set: setDefault } } }) @@ -53,7 +52,10 @@ async function fixture(fail = false) { const openSettings = vi.fn<() => Promise>>().mockResolvedValue({ ok: true, value: undefined }) const revoke = vi.fn<() => Promise>>().mockResolvedValue({ ok: true, value: undefined }) const retryApproval = vi.fn<() => Promise>>().mockResolvedValue({ ok: true, value: undefined }) - ctx.provide('remote.cuaAccess', { applications: status, systemStatus, applicationIcons, requestPermission, openSettings, revoke, retryApproval }) + ctx.provide('remote.cuaAccess', { + ptcConfiguration: vi.fn(async () => ({ ok: true, value: { persistent: false, writable: true } })), + enablePersistentPtc: async () => { await setPersistent(); return { ok: true, value: undefined } }, + applications: status, systemStatus, applicationIcons, requestPermission, openSettings, revoke, retryApproval }) ctx.provide('locale', new LocaleRuntime(ctx)) await ctx.plugin(SlotRegistry) ctx.slots.register({ name: 'root', children: { @@ -120,7 +122,7 @@ it('withdraws the permission Remote and all controls with the optional plugin', expect(f.ctx.slots.entries('plugins.bundle.activation')[0]!.inject!()).not.toHaveProperty('setPersistentPtc') expect(f.setPersistent).not.toHaveBeenCalled() await (settings.setPersistentPtc as () => Promise)() - expect(f.setPersistent).toHaveBeenCalledExactlyOnceWith('executionMode', 'persistent-in-memory') + expect(f.setPersistent).toHaveBeenCalledExactlyOnceWith() await fiber.dispose() expect(f.ctx.slots.entries('conversation.composer')).toHaveLength(0) expect(f.ctx.slots.entries('plugins.bundle.config')).toHaveLength(0) diff --git a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx b/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx index de824f0bcc..0d47ad25b6 100644 --- a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx +++ b/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx @@ -22,6 +22,7 @@ function observe(value: CuaApplicationState & { system: CuaSystemPermissionState function fixture() { const state = createSnapshotStore({ + runtime: { value: { persistent: false, writable: true }, busy: false, error: null }, system: { value: null, busy: true, error: null }, applications: { value: null, busy: true, error: null }, icons: { value: null, busy: true, error: null }, busy: true, toast: null, }) @@ -29,17 +30,14 @@ function fixture() { status: 'ready', value: { default: 'standard', selectedDefault: 'ptc' }, base: {}, user: {}, revision: 1, writable: true, }) - const runtime = createSnapshotStore>({ - status: 'ready', value: { executionMode: 'per-call' }, base: {}, user: {}, revision: 1, writable: true, - }) const refresh = vi.fn() const props = { useCuaPermissions: bindSnapshotSelector(state), t: makeTranslate(en), - useCuaPtcRuntime: bindSnapshotSelector(runtime), setPersistentPtc: vi.fn(async () => true), + setPersistentPtc: vi.fn(async () => true), useCuaDefaultPreset: bindSnapshotSelector(preset), setPtcDefault: vi.fn(async () => true), refresh, request: vi.fn(), openSettings: vi.fn(), revoke: vi.fn(), retryApproval: vi.fn(), dismissToast: vi.fn(), } satisfies PermissionPanelProps - return { state, preset, runtime, props, + return { state, preset, props, publish(patch: Partial) { act(() => { state.set({ ...state.getSnapshot(), ...patch }) }) }, } } @@ -345,21 +343,21 @@ it.each([en, zh])('saves persistent PTC only from its own row and retains failur expect(within(row).getByRole('alert').textContent).toBe(dictionary.persistentNotSaved) fireEvent.click(within(row).getByRole('button')) await waitFor(() => { expect(within(row).queryByRole('alert')).toBeNull() }) - act(() => { f.runtime.set({ ...f.runtime.getSnapshot(), value: { executionMode: 'persistent-in-memory' } }) }) + f.publish({ runtime: { value: { persistent: true, writable: true }, busy: false, error: null } }) expect(within(row).getByRole('button', { name: dictionary.persistentSet }).hasAttribute('disabled')).toBe(true) expect(f.props.setPtcDefault).not.toHaveBeenCalled() }) it('loads the runtime setting independently and reports unavailable and read-only configurations', () => { const f = fixture() - f.runtime.set({ ...f.runtime.getSnapshot(), status: 'loading', value: undefined }) + f.publish({ runtime: { value: null, busy: true, error: null } }) render() const row = screen.getByRole('group', { name: en.persistentTitle }) expect(within(row).queryByRole('button')).toBeNull() expect(screen.getByRole('button', { name: en.defaultSet })).toBeTruthy() - act(() => { f.runtime.set({ ...f.runtime.getSnapshot(), status: 'unavailable', writable: false }) }) + f.publish({ runtime: { value: null, busy: false, error: 'offline' } }) expect(within(row).getByRole('alert').textContent).toBe(en.persistentUnavailable) - act(() => { f.runtime.set({ ...f.runtime.getSnapshot(), status: 'ready', value: {}, writable: false }) }) + f.publish({ runtime: { value: { persistent: false, writable: false }, busy: false, error: null } }) expect(within(row).getByRole('alert').textContent).toBe(en.persistentReadOnly) expect(within(row).getByRole('button').hasAttribute('disabled')).toBe(true) expect(f.props.setPersistentPtc).not.toHaveBeenCalled() diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index 14142829c4..5bdc7c755d 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -18,8 +18,8 @@ en: 577e1b18dd220c73 zh: bde0692c6695d749 /deepseek-ai-dsh-cua/understand-the-implementation: - en: 5f8f14fda4d69173 - zh: 5493e3ba74ffb7e0 + en: 727a4ec9b5cd1789 + zh: 000a6a34b29e1008 /deepseek-ai-dsh-cua/further-exploration: en: b8b0aece886fadaa zh: ecd8c88d601f2179 diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index cadf0e6a8d..99c32a8c92 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -49,6 +49,8 @@ Actions return `dispatched:true, verified:false`; read fresh state to verify the
Shared execution and platform ownership +The permission Remote also exposes `ptcConfiguration()` and `enablePersistentPtc()` for the detail-page runtime setting. Reads inspect the existing runtime and editable profile entry. Explicit writes use `configEditor.edit()` to preserve other fields and apply normal Loader reload; an already-persistent runtime is left intact. This setting is separate from system and application authorization. + The provider calls [installCua](src/index.ts) with its prepared helper, connection factory, and guidance. [NativeClient](src/native-client.ts) owns request correlation, byte limits, cancellation acknowledgement, and failure settlement. The platform connector owns startup and connection cleanup. The shared executor retains one connection per Agent and joins in-flight image writes during disposal. `agentPreset` defaults to `ptc`, restricting tools and guidance to Agents bound to that preset. The provider uses the existing `agent/created` and `agent-preset/selected` events and Agent-owned scopes, including Agents already present when it loads. Provider unload and Agent disposal release those registrations. Preset definitions may register after the provider; a preset name filters bound Agents and supplies no tools until a matching Agent exists. Set `agentPreset: false` for global registration in standalone compositions without presets. The Host continues to own permissions, native connections and cancellation; preset definitions remain unchanged. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index 35e37feb67..ad5251c485 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -49,6 +49,8 @@ kind: "package-library"
共享执行与平台职责 +权限 Remote 还通过 `ptcConfiguration()` 和 `enablePersistentPtc()` 支持详情页的执行方式设置。查询读取现有执行器和可编辑的 profile 条目。显式写入使用 `configEditor.edit()` 保留其他字段并按正常 Loader 规则重载；已是持久化模式的执行器保持不变。此设置独立于系统权限和应用授权。 + 提供方将准备好的 helper、连接工厂和指引传给 [installCua](src/index.ts)。[NativeClient](src/native-client.ts) 负责请求关联、字节上限、取消确认和失败处理。平台连接器负责启动和连接清理。共享执行器为每个 Agent 保留一个连接，并在销毁时等待进行中的图像写入。 `agentPreset` 默认为 `ptc`，将工具和指引限制到绑定该 preset 的 Agent。提供方使用现有 `agent/created`、`agent-preset/selected` 事件及 Agent 拥有的作用域，也处理启用前已存在的 Agent。提供方卸载和 Agent 销毁都会释放这些注册。preset 定义可以晚于提供方注册；名称用于筛选已绑定的 Agent，在匹配的 Agent 出现前不提供工具。不使用 preset 的独立组合可设置 `agentPreset: false` 来全局注册。Host 仍拥有权限、本机连接和取消操作，preset 定义保持不变。 diff --git a/packages/experimental/cua/src/access.ts b/packages/experimental/cua/src/access.ts index 2553286a8b..15e1a33b83 100644 --- a/packages/experimental/cua/src/access.ts +++ b/packages/experimental/cua/src/access.ts @@ -1,5 +1,7 @@ /** Browser controls for the provider's application decisions and native permission work. */ import type { Context } from '@deepseek-ai/cordis' +import type {} from '@deepseek-ai/dsh-config-editor' +import type {} from '@deepseek-ai/dsh-ptc-runtime' import { Remote, TypertRemoteService } from '@deepseek-ai/dsh-typert-protocol' import { ApplicationAccess, type AccessOptions } from './application-access.ts' import type { JsonValue } from '@deepseek-ai/dsh-util-values' @@ -40,6 +42,29 @@ export class CuaAccess extends TypertRemoteService { constructor(ctx: Context, private readonly policy: ApplicationAccess, private readonly settingsNamespace: string | null, private readonly control: Control) { super(ctx, 'cuaAccess') } + /** Read PTC execution state separately from native permission checks. + * @returns Whether the shared runtime retains memory and has an editable profile entry. + */ + @Remote + ptcConfiguration(): { persistent: boolean; writable: boolean } { + return { + persistent: this.ctx.get('ptcRuntime')?.programFormat === 'repl', + writable: this.ctx.get('configEditor')?.entries().some(entry => entry.options.id === 'ptc-runtime') ?? false, + } + } + + /** Persist in-memory PTC execution through the existing profile editor and Loader lifecycle. + * @returns Completion after the shared runtime reload; already-persistent runtimes are left intact. + */ + @Remote + async enablePersistentPtc(): Promise { + if (this.ptcConfiguration().persistent) return + const editor = this.ctx.get('configEditor') + const entry = editor?.entries().find(entry => entry.options.id === 'ptc-runtime') + if (editor === undefined || entry === undefined) throw new Error('PTC runtime has no editable profile entry') + await editor.edit(entry, current => ({ ...current, executionMode: 'persistent-in-memory' })) + } + /** * Read application decisions without waiting for the native helper. * @returns Saved grants and refused applications with their persistence owner. diff --git a/packages/experimental/cua/tests/access.spec.ts b/packages/experimental/cua/tests/access.spec.ts index 84289fb246..8cffbcc4e9 100644 --- a/packages/experimental/cua/tests/access.spec.ts +++ b/packages/experimental/cua/tests/access.spec.ts @@ -396,3 +396,24 @@ it('omits session-only grants from the permission page and retains their access expect(state.blockedApps).toEqual([]) await expect(f.access.run(f.client, handle, signal, async () => 'still allowed')).resolves.toBe('still allowed') }) + +it('configures persistent PTC through the existing profile editor only on explicit request', async () => { + const f = await setup() + expect(f.ctx.cuaAccess.ptcConfiguration()).toEqual({ persistent: false, writable: false }) + await expect(f.ctx.cuaAccess.enablePersistentPtc()).rejects.toThrow('no editable profile entry') + const entry = { options: { id: 'ptc-runtime' } } + const runtime = { programFormat: 'function-body' } + const edit = vi.fn(async (received: object, change: (current: Record) => Record) => { + expect(received).toBe(entry) + expect(change({ timeoutMs: 4200 })).toEqual({ timeoutMs: 4200, executionMode: 'persistent-in-memory' }) + runtime.programFormat = 'repl' + }) + f.ctx.provide('ptcRuntime', runtime) + f.ctx.provide('configEditor', { entries: () => [entry], edit }) + expect(f.ctx.cuaAccess.ptcConfiguration()).toEqual({ persistent: false, writable: true }) + expect(edit).not.toHaveBeenCalled() + await f.ctx.cuaAccess.enablePersistentPtc() + expect(f.ctx.cuaAccess.ptcConfiguration()).toEqual({ persistent: true, writable: true }) + await f.ctx.cuaAccess.enablePersistentPtc() + expect(edit).toHaveBeenCalledOnce() +}) diff --git a/packages/extensions/tool-cordis/src/api-catalog.ts b/packages/extensions/tool-cordis/src/api-catalog.ts index 1645a26ba0..05bf106378 100644 --- a/packages/extensions/tool-cordis/src/api-catalog.ts +++ b/packages/extensions/tool-cordis/src/api-catalog.ts @@ -915,6 +915,18 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ summary: 'Browser controls for provider-owned native permission and application decisions.', description: 'Browser controls for provider-owned native permission and application decisions.', methods: [ + { + signature: '@Remote ptcConfiguration(): { persistent: boolean; writable: boolean }', + description: 'Read PTC execution state separately from native permission checks.', + parameters: [], + returns: 'Whether the shared runtime retains memory and has an editable profile entry.', + }, + { + signature: '@Remote async enablePersistentPtc(): Promise', + description: 'Persist in-memory PTC execution through the existing profile editor and Loader lifecycle.', + parameters: [], + returns: 'Completion after the shared runtime reload; already-persistent runtimes are left intact.', + }, { signature: '@Remote applications(): CuaApplicationState', description: 'Read application decisions without waiting for the native helper.', From aa52dc48b5fc5edcdda46626b9519693c9168196 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 15:27:05 +0800 Subject: [PATCH 34/55] test(cua): assert the replaced runtime service instance --- apps/web/tests/cua-optin.e2e.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/web/tests/cua-optin.e2e.ts b/apps/web/tests/cua-optin.e2e.ts index b6fa5de69a..36a0c7a0e2 100644 --- a/apps/web/tests/cua-optin.e2e.ts +++ b/apps/web/tests/cua-optin.e2e.ts @@ -34,6 +34,7 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv setup: async (ctx) => { await scaffold.ctx.agentPresets.mount(ctx, 'ptc') }, })).agent const runtimeFiber = () => [...scaffold.ctx.loader.entries()].find(entry => entry.options.id === 'ptc-runtime')?.fiber + const originalRuntime = scaffold.ctx.ptcRuntime const originalRuntimeFiber = runtimeFiber() expect(originalRuntimeFiber).toBeDefined() expect(scaffold.ctx.ptcRuntime.programFormat).toBe('function-body') @@ -96,7 +97,7 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv await persistent.getByRole('button', { name: '设为持久化', exact: true }).click() await persistent.getByRole('button', { name: '已设为持久化', exact: true }).waitFor() expect(scaffold.ctx.ptcRuntime.programFormat).toBe('repl') - expect(runtimeFiber() === originalRuntimeFiber).toBe(false) + expect(scaffold.ctx.ptcRuntime === originalRuntime).toBe(false) const persistentRuntimeFiber = runtimeFiber() const runtime = scaffold.ctx.ptcRuntime, owner = {} const program = 'globalThis.counter = (globalThis.counter ?? 0) + 1' From 4c966c65ea54ba37607b550aabee4801d543948e Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 15:41:44 +0800 Subject: [PATCH 35/55] test(cua): cover runtime configuration transport failures --- .../client-ui-cua/tests/mount.client.spec.ts | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/experimental/client-ui-cua/tests/mount.client.spec.ts b/packages/experimental/client-ui-cua/tests/mount.client.spec.ts index 5e9e29cc32..904a8453fc 100644 --- a/packages/experimental/client-ui-cua/tests/mount.client.spec.ts +++ b/packages/experimental/client-ui-cua/tests/mount.client.spec.ts @@ -37,7 +37,10 @@ async function fixture(fail = false) { $on(_event: string, listener: (namespace: string, revision: number) => void) { settingsListener = listener; return unsubscribe } } new Remote() - const setDefault = vi.fn(async () => true), setPersistent = vi.fn(async () => true) + const setDefault = vi.fn(async () => true) + const setPersistent = vi.fn<() => Promise>>().mockResolvedValue({ ok: true, value: undefined }) + const runtimeConfiguration = vi.fn<() => Promise>>() + .mockResolvedValue({ ok: true, value: { persistent: false, writable: true } }) const preset = createSnapshotStore({ status: 'ready' as const, value: { default: 'standard' }, base: {}, user: {}, revision: 1, writable: true }) ctx.provide('configForms', { get: (namespace: string) => { @@ -53,8 +56,7 @@ async function fixture(fail = false) { const revoke = vi.fn<() => Promise>>().mockResolvedValue({ ok: true, value: undefined }) const retryApproval = vi.fn<() => Promise>>().mockResolvedValue({ ok: true, value: undefined }) ctx.provide('remote.cuaAccess', { - ptcConfiguration: vi.fn(async () => ({ ok: true, value: { persistent: false, writable: true } })), - enablePersistentPtc: async () => { await setPersistent(); return { ok: true, value: undefined } }, + ptcConfiguration: runtimeConfiguration, enablePersistentPtc: setPersistent, applications: status, systemStatus, applicationIcons, requestPermission, openSettings, revoke, retryApproval }) ctx.provide('locale', new LocaleRuntime(ctx)) await ctx.plugin(SlotRegistry) @@ -64,7 +66,7 @@ async function fixture(fail = false) { } } as never, () => null) if (fail) vi.spyOn(ctx.slots, 'inject').mockImplementationOnce(() => { throw new Error('slot unavailable') }) return { ctx, unmount, unsubscribe, status, systemStatus, applicationIcons, - requestPermission, openSettings, revoke, retryApproval, setDefault, setPersistent, + requestPermission, openSettings, revoke, retryApproval, setDefault, setPersistent, runtimeConfiguration, updateSettings(namespace: string) { settingsListener!(namespace, 1) }, } } @@ -90,7 +92,8 @@ it('withdraws the permission Remote and all controls with the optional plugin', await vi.waitFor(() => { expect(actions.hooks.cuaPermissions.getSnapshot().toast?.kind).toBe('retryAllowed') }) const failed = { ok: false as const, error: new RemoteError('gateway/internal', 'offline', {}) } for (const [target, invoke] of [ - [f.status, actions.refresh], [f.systemStatus, actions.refresh], [f.applicationIcons, actions.refresh], [f.requestPermission, () => { actions.request('accessibility') }], + [f.runtimeConfiguration, actions.refresh], [f.status, actions.refresh], [f.systemStatus, actions.refresh], + [f.applicationIcons, actions.refresh], [f.requestPermission, () => { actions.request('accessibility') }], [f.openSettings, () => { actions.openSettings('accessibility') }], [f.revoke, () => { actions.revoke('a'.repeat(64) as import('@deepseek-ai/dsh-cua/types').CuaApplicationId) }], [f.retryApproval, () => { actions.retryApproval('a'.repeat(64) as import('@deepseek-ai/dsh-cua/types').CuaApplicationId) }], @@ -101,8 +104,9 @@ it('withdraws the permission Remote and all controls with the optional plugin', invoke() await vi.waitFor(() => { const state = actions.hooks.cuaPermissions.getSnapshot() - const error = target === f.status ? state.applications.error : target === f.systemStatus ? state.system.error - : target === f.applicationIcons ? state.icons.error : state.toast?.kind === 'failed' ? state.toast.message : null + const error = target === f.runtimeConfiguration ? state.runtime.error : target === f.status ? state.applications.error + : target === f.systemStatus ? state.system.error + : target === f.applicationIcons ? state.icons.error : state.toast?.kind === 'failed' ? state.toast.message : null expect(error).toContain('offline') }) } @@ -123,6 +127,8 @@ it('withdraws the permission Remote and all controls with the optional plugin', expect(f.setPersistent).not.toHaveBeenCalled() await (settings.setPersistentPtc as () => Promise)() expect(f.setPersistent).toHaveBeenCalledExactlyOnceWith() + f.setPersistent.mockResolvedValueOnce(failed) + await expect((settings.setPersistentPtc as () => Promise)()).rejects.toThrow('offline') await fiber.dispose() expect(f.ctx.slots.entries('conversation.composer')).toHaveLength(0) expect(f.ctx.slots.entries('plugins.bundle.config')).toHaveLength(0) From 67a4ab78be8a4c57da71a8defc3a41cecb1fc4dd Mon Sep 17 00:00:00 2001 From: turtle1999 Date: Mon, 19 Oct 2026 16:59:18 +0800 Subject: [PATCH 36/55] fix(file-upload): release the Agent resolver when its owner reloads session-controller registers its resolver through `ctx.fileUploads`. Inside that call Cordis returns a new proxy for every function-valued property read from `this`, so the disposer's identity check never matched and the slot was never cleared. Any restart of session-controller, such as a profile patch that changes session-query-sqlite, then failed with "Agent resolver is already registered" and left every sessionController dependent pending. Store the resolver in a plain holder object, as the commands receipt resolver already does, and test registration and disposal through a plugin fiber. --- .../tests/commands-upload-file.host.spec.ts | 27 ++++++++++++++----- packages/client/file-upload/src/index.ts | 13 +++++---- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/packages/api/session-controller/tests/commands-upload-file.host.spec.ts b/packages/api/session-controller/tests/commands-upload-file.host.spec.ts index aa7422918e..48ba896f01 100644 --- a/packages/api/session-controller/tests/commands-upload-file.host.spec.ts +++ b/packages/api/session-controller/tests/commands-upload-file.host.spec.ts @@ -1,4 +1,4 @@ -import { Context } from '@deepseek-ai/cordis' +import { Context, FiberState } from '@deepseek-ai/cordis' import AgentRegistry from '@deepseek-ai/dsh-agent' import SessionProjectionRegistry from '@deepseek-ai/dsh-session-projection' import { createInboxStub } from '@deepseek-ai/dsh-agent-loop-testkit' @@ -240,26 +240,39 @@ describe('Session file uploads', () => { await expect(uploading).rejects.toMatchObject({ code: 'session/not-found' }) }) - it('resolves a cold ordinary Agent and releases the resolver registration', async () => { + it('resolves a cold ordinary Agent and releases the resolver with its registering fiber', async () => { const { ctx, uploads, agent, disposeAgent } = await uploadHarness() await disposeAgent() const resolveAgent = vi.fn(async () => { await ctx.agents.register(agent) return agent }) - const disposeResolver = uploads.registerAgentResolver(resolveAgent) - expect(() => { uploads.registerAgentResolver(resolveAgent) }).toThrow('already registered') + const resolverOwner = { + name: 'resolver-owner', + inject: ['fileUploads'], + apply: (owner: Context) => { + owner.effect(() => owner.fileUploads.registerAgentResolver(resolveAgent), 'test: Agent resolver') + }, + } + const first = ctx.plugin(resolverOwner) + await first + expect(() => { ctx.fileUploads.registerAgentResolver(resolveAgent) }).toThrow('already registered') await expect(uploads.uploadStream({ sessionId: SESSION, data: (async function* (): AsyncIterable { yield Uint8Array.of(1) })(), })).resolves.toMatchObject({ file: { bytes: 1 } }) expect(resolveAgent).toHaveBeenCalledWith(SESSION) + await first.dispose() + const second = ctx.plugin(resolverOwner) + await second + expect(second.state).toBe(FiberState.ACTIVE) + await second.dispose() + const disposeResolver = ctx.fileUploads.registerAgentResolver(resolveAgent) disposeResolver() const replacement = vi.fn(async () => agent) - const disposeReplacement = uploads.registerAgentResolver(replacement) + const disposeReplacement = ctx.fileUploads.registerAgentResolver(replacement) disposeResolver() - expect(() => { uploads.registerAgentResolver(replacement) }).toThrow('already registered') - expect(disposeReplacement).toBeTypeOf('function') + expect(() => { ctx.fileUploads.registerAgentResolver(replacement) }).toThrow('already registered') disposeReplacement() }) diff --git a/packages/client/file-upload/src/index.ts b/packages/client/file-upload/src/index.ts index a784096461..acf99f22d2 100644 --- a/packages/client/file-upload/src/index.ts +++ b/packages/client/file-upload/src/index.ts @@ -57,7 +57,10 @@ export class FileUploads extends TypertRemoteService { static inject = ['agents', 'attachments', 'commands', 'connection'] private readonly stagedFiles = new WeakMap>() - private agentResolver: AgentResolver | undefined + // Inside a method called through `ctx.fileUploads`, Cordis returns a new proxy + // for every function-valued property read from `this`, so `===` against the + // registered resolver would always be false; a plain object is returned as is. + private readonly agentResolution: { resolver: AgentResolver | undefined } = { resolver: undefined } /** @param ctx - Host context carrying Agent, attachment, command, and Connection services. */ constructor(ctx: Context) { @@ -87,10 +90,10 @@ export class FileUploads extends TypertRemoteService { * @returns disposer removing this resolver. */ registerAgentResolver(resolve: AgentResolver): () => void { - if (this.agentResolver !== undefined) throw new Error('file-upload: Agent resolver is already registered') - this.agentResolver = resolve + if (this.agentResolution.resolver !== undefined) throw new Error('file-upload: Agent resolver is already registered') + this.agentResolution.resolver = resolve return () => { - if (this.agentResolver === resolve) this.agentResolver = undefined + if (this.agentResolution.resolver === resolve) this.agentResolution.resolver = undefined } } @@ -212,7 +215,7 @@ export class FileUploads extends TypertRemoteService { private async resolveAgent(sessionId: SessionId): Promise { const live = this.ctx.agents.get(sessionId) if (live !== undefined) return live - const resolver = this.agentResolver + const resolver = this.agentResolution.resolver if (resolver === undefined) { throw new RemoteError('session/not-found', `session "${sessionId}" is not attached`, { sessionId }) } From b40d335e3d6449a208a473a49e4354edf33f2263 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 19:46:13 +0800 Subject: [PATCH 37/55] refactor(cua): colocate permission UI and narrow package exports --- ...capabilities-as-optional-bundles.i18n.yaml | 4 +- ...mental-capabilities-as-optional-bundles.md | 2 +- ...tal-capabilities-as-optional-bundles.zh.md | 2 +- ...26-09-26-cua-application-consent.i18n.yaml | 4 +- .../2026-09-26-cua-application-consent.md | 2 +- .../2026-09-26-cua-application-consent.zh.md | 2 +- docs/capability-seams.i18n.yaml | 4 +- docs/capability-seams.md | 5 +- docs/capability-seams.zh.md | 5 +- docs/config-catalog.md | 3 +- docs/config-catalog.zh.md | 3 +- docs/module-graph.md | 2 - docs/module-graph.zh.md | 2 - docs/subsystems/computer-use.i18n.yaml | 4 +- docs/subsystems/computer-use.md | 2 +- docs/subsystems/computer-use.zh.md | 2 +- packages/experimental/README.i18n.yaml | 4 +- packages/experimental/README.md | 1 - packages/experimental/README.zh.md | 1 - .../client-ui-cua/README.i18n.yaml | 37 ------ packages/experimental/client-ui-cua/README.md | 92 --------------- .../experimental/client-ui-cua/README.zh.md | 92 --------------- .../experimental/client-ui-cua/package.json | 72 ------------ .../experimental/client-ui-cua/src/index.ts | 2 - .../client-ui-cua/tsdown.config.ts | 3 - .../experimental/cua-bundle/README.i18n.yaml | 4 +- packages/experimental/cua-bundle/README.md | 4 +- packages/experimental/cua-bundle/README.zh.md | 4 +- .../experimental/cua-bundle/cordis.patch.yml | 2 +- packages/experimental/cua-bundle/package.json | 2 +- .../experimental/cua-macos/README.i18n.yaml | 4 +- packages/experimental/cua-macos/README.md | 2 +- packages/experimental/cua-macos/README.zh.md | 2 +- packages/experimental/cua-macos/tsconfig.json | 2 +- packages/experimental/cua/README.i18n.yaml | 15 ++- packages/experimental/cua/README.md | 20 +++- packages/experimental/cua/README.zh.md | 20 +++- packages/experimental/cua/package.json | 47 +++++++- .../src/client/PermissionPanel.module.css | 0 .../src/client/PermissionPanel.tsx | 2 +- .../src/client/controller.ts | 2 +- .../src/client/index.ts | 0 .../src/client/locales.ts | 0 .../src/client/mount.ts | 0 .../src/css-modules.d.ts | 0 packages/experimental/cua/src/index.ts | 4 +- packages/experimental/cua/src/remote-types.ts | 3 + packages/experimental/cua/src/types.ts | 1 - .../tests/controller.client.spec.ts | 2 +- .../cua/tests/export-types.spec.ts | 18 +++ .../cua/tests/exports.built.e2e.ts | 18 +++ .../cua/tests/fixtures/exports.mjs | 28 +++++ .../tests/mount.client.spec.ts | 12 +- .../tests/panel.client.spec.tsx | 2 +- .../tsconfig.client.json} | 13 ++- packages/experimental/cua/tsconfig.host.json | 73 ++++++++++++ packages/experimental/cua/tsconfig.json | 64 +---------- packages/experimental/cua/tsdown.config.ts | 11 +- .../src/client/slot-catalog.ts | 6 +- .../references/packages.md | 1 - pnpm-lock.yaml | 105 ++++++++---------- scripts/check-workspace-constraints.ts | 2 +- scripts/experimental-package-policy.ts | 1 - scripts/gen-doc-graphs.ts | 2 +- scripts/release/families.spec.ts | 1 - .../verify-package-readme-model-experience.ts | 1 - tsconfig.base.json | 5 +- tsconfig.client.json | 2 +- tsconfig.host.json | 2 +- vitest.config.ts | 2 +- 70 files changed, 354 insertions(+), 509 deletions(-) delete mode 100644 packages/experimental/client-ui-cua/README.i18n.yaml delete mode 100644 packages/experimental/client-ui-cua/README.md delete mode 100644 packages/experimental/client-ui-cua/README.zh.md delete mode 100644 packages/experimental/client-ui-cua/package.json delete mode 100644 packages/experimental/client-ui-cua/src/index.ts delete mode 100644 packages/experimental/client-ui-cua/tsdown.config.ts rename packages/experimental/{client-ui-cua => cua}/src/client/PermissionPanel.module.css (100%) rename packages/experimental/{client-ui-cua => cua}/src/client/PermissionPanel.tsx (99%) rename packages/experimental/{client-ui-cua => cua}/src/client/controller.ts (98%) rename packages/experimental/{client-ui-cua => cua}/src/client/index.ts (100%) rename packages/experimental/{client-ui-cua => cua}/src/client/locales.ts (100%) rename packages/experimental/{client-ui-cua => cua}/src/client/mount.ts (100%) rename packages/experimental/{client-ui-cua => cua}/src/css-modules.d.ts (100%) create mode 100644 packages/experimental/cua/src/remote-types.ts rename packages/experimental/{client-ui-cua => cua}/tests/controller.client.spec.ts (99%) create mode 100644 packages/experimental/cua/tests/export-types.spec.ts create mode 100644 packages/experimental/cua/tests/exports.built.e2e.ts create mode 100644 packages/experimental/cua/tests/fixtures/exports.mjs rename packages/experimental/{client-ui-cua => cua}/tests/mount.client.spec.ts (96%) rename packages/experimental/{client-ui-cua => cua}/tests/panel.client.spec.tsx (99%) rename packages/experimental/{client-ui-cua/tsconfig.json => cua/tsconfig.client.json} (78%) create mode 100644 packages/experimental/cua/tsconfig.host.json diff --git a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.i18n.yaml b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.i18n.yaml index 345eec989b..5d8a536a53 100644 --- a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.i18n.yaml @@ -9,8 +9,8 @@ en: 2ba6c43b20a3d347 zh: 5647179ddaa38ad9 /agent-note-ship-experimental-capabilities-as-optional-bundles/decision: - en: 6566c6fb7287d748 - zh: fe41c0a9da97f340 + en: d6328ba2b218f2ac + zh: 3731561313b1ad5e /agent-note-ship-experimental-capabilities-as-optional-bundles/alternatives-considered: en: d6f40bf02df37ab7 zh: e5b85ce9d41e8754 diff --git a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md index a2dded7934..a2d39b0ffb 100644 --- a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md +++ b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md @@ -18,7 +18,7 @@ Bundles insert ordinary Host rows; optional model tools register in the global t Claude Code and Codex install on demand through the separate `ON_DEMAND_BUNDLES` catalog; the [on-demand native-bundle decision](2026-10-05-official-on-demand-bundles.md) owns the cost, version, and offline-discovery trade-offs. -Computer Use is a shipped optional bundle in the experimental group. Its bundle and client plugin use the explicit naming exceptions `@deepseek-ai/dsh-cua-bundle` and `@deepseek-ai/dsh-client-ui-cua`, alongside `@deepseek-ai/dsh-cua`. These names preserve profile references when the capability moves package groups; the experimental classification still controls isolation and promotion. Its repository-owned native component is prepared when the provider loads, without an external Cua Driver runtime. +Computer Use is a shipped optional bundle in the experimental group. Its bundle uses the explicit naming exception `@deepseek-ai/dsh-cua-bundle`, alongside `@deepseek-ai/dsh-cua`. These names preserve profile references when the capability moves package groups; the experimental classification still controls isolation and promotion. Its repository-owned native component is prepared when the provider loads, without an external Cua Driver runtime. ## Alternatives considered diff --git a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.zh.md b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.zh.md index db1b4a5506..15bc65b56f 100644 --- a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.zh.md +++ b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.zh.md @@ -18,7 +18,7 @@ Bundle 插入普通 Host 行；可选模型工具注册到全局工具层，因 Claude Code 与 Codex 通过独立的 `ON_DEMAND_BUNDLES` 目录按需安装，其成本、版本与离线发现取舍由[按需原生 bundle 决策](2026-10-05-official-on-demand-bundles.zh.md)管理。 -Computer Use 是 experimental 包组内随安装提供的可选组合包。组合包和前端插件与 `@deepseek-ai/dsh-cua` 一起采用显式命名例外 `@deepseek-ai/dsh-cua-bundle`、`@deepseek-ai/dsh-client-ui-cua`，在能力移动分组时保留配置引用。依赖隔离和晋升仍由实验性分类约束。提供方加载时准备仓库自有的本机组件，无需外部 Cua Driver 运行时。 +Computer Use 是 experimental 包组内随安装提供的可选组合包。组合包与 `@deepseek-ai/dsh-cua` 一起采用显式命名例外 `@deepseek-ai/dsh-cua-bundle`，在能力移动分组时保留配置引用。依赖隔离和晋升仍由实验性分类约束。提供方加载时准备仓库自有的本机组件，无需外部 Cua Driver 运行时。 ## 考虑过的替代方案 diff --git a/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.i18n.yaml b/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.i18n.yaml index 2cac3e4e63..2f89f011b6 100644 --- a/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.i18n.yaml @@ -9,8 +9,8 @@ en: b32a7ce6483b9b48 zh: ec0938ad032b99d8 /agent-note-application-consent-for-native-computer-use/decision: - en: 22b3e4b770d2517f - zh: 2250b31a5a50758a + en: 36c560c24c376f12 + zh: 5e4ff72f66574e38 /agent-note-application-consent-for-native-computer-use/alternatives-considered: en: 80cd5fd93a42d937 zh: 2a0e241b1b8c2e1d diff --git a/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.md b/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.md index d69ba1218c..389a994714 100644 --- a/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.md +++ b/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.md @@ -12,7 +12,7 @@ macOS privileges authorize an execution host, not a user's choice of target appl The CUA executor owns application consent for every object-API and native-tool dispatch. It uses the existing User Questions service for human decisions and the provider's existing profile configuration for saved grants. The existing question UI presents the offered labels; the executor rejects custom text or answers that do not select exactly one offered label. Permission-page icons and names are presentation metadata, excluded from saved identities and model results. General tool approval and operating-system privileges remain separate requirements. Arbitrary persistent PTC code and separately authorized shell tools retain their own execution authority. -System-permission buttons call the native helper on the DSH host and remain available to remote browsers. The [usage documentation](../../../../packages/experimental/client-ui-cua/README.md#use-this-package) explains where to complete authorization; the page has no persistent remote-access notice. +System-permission buttons call the native helper on the DSH host and remain available to remote browsers. The [usage documentation](../../../../packages/experimental/cua/README.md#use-this-package) explains where to complete authorization; the page has no persistent remote-access notice. A saved identity combines the canonical executable path with a verified designated signing requirement, represented by a SHA-256 digest. The path prevents authorization from following a copied application, and the signing requirement distinguishes replacements while allowing matching signed updates. Unverifiable and ad-hoc applications receive session access only, tied to a process instance. Labels do not establish identity. diff --git a/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.zh.md b/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.zh.md index b758b796c9..e7311dfc6d 100644 --- a/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.zh.md +++ b/.agents/notes/implemented/architecture/2026-09-26-cua-application-consent.zh.md @@ -12,7 +12,7 @@ macOS 权限授权执行宿主，不能表达用户允许操作哪些目标应 CUA 执行器在每次对象 API 和原生工具调用时检查应用授权。人工决定复用 User Questions 服务，长期许可保存在 provider 的现有 profile 配置中。通用工具审批和操作系统权限仍是独立要求。任意 persistent PTC 代码和另行授权的 shell 工具保留各自的执行权限。现有问答界面展示授权选项；执行器拒绝自由文本或未准确选中一个所给选项的回答。权限页图标和名称仅用于展示，不进入保存的身份信息或模型结果。 -系统权限按钮调用运行 DSH 的电脑上的原生 helper，远程访问时仍可使用。授权地点写在[使用说明](../../../../packages/experimental/client-ui-cua/README.zh.md#use-this-package)中，页面不常驻远程访问提示。 +系统权限按钮调用运行 DSH 的电脑上的原生 helper，远程访问时仍可使用。授权地点写在[使用说明](../../../../packages/experimental/cua/README.zh.md#use-this-package)中，页面不常驻远程访问提示。 长期身份由规范化可执行文件路径和经过验证的签名指定要求组成，以 SHA-256 摘要表示。路径阻止许可跟随应用副本，签名要求区分替换程序，并允许符合相同签名要求的更新。签名不可验证或使用 ad-hoc 签名的应用只能获得会话授权，许可绑定进程实例。显示名称不用于确定身份。 diff --git a/docs/capability-seams.i18n.yaml b/docs/capability-seams.i18n.yaml index 3443e528f0..ae1d83cdf3 100644 --- a/docs/capability-seams.i18n.yaml +++ b/docs/capability-seams.i18n.yaml @@ -6,5 +6,5 @@ en: 41d772192075e133 zh: 5ef3da69b4de1d8a /capability-seams-and-core-services: - en: b360360db9a3f6a4 - zh: d14808d61000bdff + en: 753b3da3abc6aeea + zh: aab02ec16c3f744b diff --git a/docs/capability-seams.md b/docs/capability-seams.md index 93b22913dc..f2b52ade47 100644 --- a/docs/capability-seams.md +++ b/docs/capability-seams.md @@ -54,7 +54,6 @@ flowchart LR pkg_experimental_computer_use_cua_driver_native["experimental-computer-use-cua-driver-native"] pkg_cua["cua"] svc_cuaAccess["ctx.cuaAccess
Computer Use permission controls"] - pkg_client_ui_cua["client-ui-cua"] pkg_office_to_pdf["office-to-pdf"] svc_officeToPdf["ctx.officeToPdf
Office to PDF conversion"] pkg_client_ui_sidebar_documentpreview["client-ui-sidebar-documentpreview"] @@ -488,7 +487,7 @@ flowchart LR svc_credentials --> pkg_api_settings_controller svc_credentials --> pkg_llm_deepseek svc_credentials --> pkg_llm_pi_ai - svc_cuaAccess --> pkg_client_ui_cua + svc_cuaAccess --> pkg_cua svc_deepseekAccount --> pkg_api_account_controller svc_deepseekAccount --> pkg_llm_deepseek svc_deepseekLlmApiExtensions --> pkg_llm_deepseek @@ -633,7 +632,7 @@ flowchart LR | `ctx.mcpResources` | `seam` | [`mcp-resources`](../packages/mcp/mcp-resources) | [`mcp-client`](../packages/mcp/mcp-client) | [`mcp-resources`](../packages/mcp/mcp-resources) | - | Connection-owned providers serve shared resource tools in the calling agent scope. | | `ctx.browserUse` | `seam` | [`browser-use`](../packages/browser-use/browser-use) | [`experimental-browser-use-playwright-mcp`](../packages/experimental/browser-use-playwright-mcp), [`experimental-browser-use-chrome-devtools-mcp`](../packages/experimental/browser-use-chrome-devtools-mcp), [`experimental-browser-use-stagehand-native`](../packages/experimental/browser-use-stagehand-native) | [`experimental-browser-use-playwright-mcp`](../packages/experimental/browser-use-playwright-mcp), [`experimental-browser-use-chrome-devtools-mcp`](../packages/experimental/browser-use-chrome-devtools-mcp), [`experimental-browser-use-stagehand-native`](../packages/experimental/browser-use-stagehand-native) | - | One provider-owned name per service instance. Providers own their tools and browser resources per live Session; the shared service has no browser operation API. | | `ctx.computerUse` | `seam` | [`computer-use`](../packages/computer-use/computer-use) | [`experimental-computer-use-cua-driver-mcp`](../packages/experimental/computer-use-cua-driver-mcp), [`experimental-computer-use-cua-driver-native`](../packages/experimental/computer-use-cua-driver-native) | [`experimental-computer-use-cua-driver-mcp`](../packages/experimental/computer-use-cua-driver-mcp), [`experimental-computer-use-cua-driver-native`](../packages/experimental/computer-use-cua-driver-native) | - | One provider-owned name per service instance. Each provider also owns its model tools; the service has no common action API, runtime selection, or Session workflow lock. | -| `ctx.cuaAccess` | `core` | [`cua`](../packages/experimental/cua) | - | [`client-ui-cua`](../packages/experimental/client-ui-cua) | - | The native provider owns application consent and cancellation; the optional Client page queries and changes those decisions through its Remote. | +| `ctx.cuaAccess` | `core` | [`cua`](../packages/experimental/cua) | - | [`cua`](../packages/experimental/cua) | - | The native provider owns application consent and cancellation; the optional Client page queries and changes those decisions through its Remote. | | `ctx.officeToPdf` | `core` | [`office-to-pdf`](../packages/document/office-to-pdf) | - | [`client-ui-sidebar-documentpreview`](../packages/client/ui-sidebar-documentpreview) | - | Authorized Office bytes are converted on the Host using the declared native target engine, or Node WASM when no native target is declared. | | `ctx.attachments` | `seam` | [`attachment`](../packages/attachment/attachment) | [`attachment-local`](../packages/attachment/attachment-local) | [`api-session-controller`](../packages/api/session-controller), [`tool-fs`](../packages/fs/tool-fs), [`llm-pi-ai`](../packages/llm/llm-pi-ai), [`llm-deepseek`](../packages/llm/llm-deepseek) | - | The host commits accepted images before session events; provider adapters resolve authorized durable references into provider-native content. | | `ctx.fileUploads` | `core` | [`client-file-upload`](../packages/client/file-upload) | - | [`api-session-controller`](../packages/api/session-controller) | - | Owns streaming intake, durable storage, and staged receipt lifetime; the Session controller binds receipts to accepted submissions. | diff --git a/docs/capability-seams.zh.md b/docs/capability-seams.zh.md index fb70a985bf..fede454e27 100644 --- a/docs/capability-seams.zh.md +++ b/docs/capability-seams.zh.md @@ -56,7 +56,6 @@ flowchart LR pkg_experimental_computer_use_cua_driver_native["experimental-computer-use-cua-driver-native"] pkg_cua["cua"] svc_cuaAccess["ctx.cuaAccess
Computer Use permission controls"] - pkg_client_ui_cua["client-ui-cua"] pkg_office_to_pdf["office-to-pdf"] svc_officeToPdf["ctx.officeToPdf
Office to PDF conversion"] pkg_client_ui_sidebar_documentpreview["client-ui-sidebar-documentpreview"] @@ -490,7 +489,7 @@ flowchart LR svc_credentials --> pkg_api_settings_controller svc_credentials --> pkg_llm_deepseek svc_credentials --> pkg_llm_pi_ai - svc_cuaAccess --> pkg_client_ui_cua + svc_cuaAccess --> pkg_cua svc_deepseekAccount --> pkg_api_account_controller svc_deepseekAccount --> pkg_llm_deepseek svc_deepseekLlmApiExtensions --> pkg_llm_deepseek @@ -635,7 +634,7 @@ flowchart LR | `ctx.mcpResources` | `seam` | [`mcp-resources`](../packages/mcp/mcp-resources) | [`mcp-client`](../packages/mcp/mcp-client) | [`mcp-resources`](../packages/mcp/mcp-resources) | - | 连接所有者提供的操作在调用 agent 的作用域内服务于共享资源工具。 | | `ctx.browserUse` | `seam` | [`browser-use`](../packages/browser-use/browser-use) | [`experimental-browser-use-playwright-mcp`](../packages/experimental/browser-use-playwright-mcp), [`experimental-browser-use-chrome-devtools-mcp`](../packages/experimental/browser-use-chrome-devtools-mcp), [`experimental-browser-use-stagehand-native`](../packages/experimental/browser-use-stagehand-native) | [`experimental-browser-use-playwright-mcp`](../packages/experimental/browser-use-playwright-mcp), [`experimental-browser-use-chrome-devtools-mcp`](../packages/experimental/browser-use-chrome-devtools-mcp), [`experimental-browser-use-stagehand-native`](../packages/experimental/browser-use-stagehand-native) | - | 每个服务实例注册一个提供方拥有的名称。提供方按实时 Session 拥有自己的工具与浏览器资源；共享服务不提供浏览器操作 API。 | | `ctx.computerUse` | `seam` | [`computer-use`](../packages/computer-use/computer-use) | [`experimental-computer-use-cua-driver-mcp`](../packages/experimental/computer-use-cua-driver-mcp), [`experimental-computer-use-cua-driver-native`](../packages/experimental/computer-use-cua-driver-native) | [`experimental-computer-use-cua-driver-mcp`](../packages/experimental/computer-use-cua-driver-mcp), [`experimental-computer-use-cua-driver-native`](../packages/experimental/computer-use-cua-driver-native) | - | 每个服务实例只注册一个提供方自定的名称。各提供方也拥有自己的模型工具；服务不提供通用操作 API、运行时选择或 Session 流程锁。 | -| `ctx.cuaAccess` | `core` | [`cua`](../packages/experimental/cua) | - | [`client-ui-cua`](../packages/experimental/client-ui-cua) | - | 原生 provider 负责应用授权和取消操作；可选的 Client 权限页面通过 Remote 查询和修改授权。 | +| `ctx.cuaAccess` | `core` | [`cua`](../packages/experimental/cua) | - | [`cua`](../packages/experimental/cua) | - | 原生 provider 负责应用授权和取消操作；可选的 Client 权限页面通过 Remote 查询和修改授权。 | | `ctx.officeToPdf` | `core` | [`office-to-pdf`](../packages/document/office-to-pdf) | - | [`client-ui-sidebar-documentpreview`](../packages/client/ui-sidebar-documentpreview) | - | 已授权的 Office 字节在宿主上使用已声明的原生目标引擎转换；未声明原生目标时使用 Node WASM。 | | `ctx.attachments` | `seam` | [`attachment`](../packages/attachment/attachment) | [`attachment-local`](../packages/attachment/attachment-local) | [`api-session-controller`](../packages/api/session-controller), [`tool-fs`](../packages/fs/tool-fs), [`llm-pi-ai`](../packages/llm/llm-pi-ai), [`llm-deepseek`](../packages/llm/llm-deepseek) | - | 宿主会在会话事件之前提交已接受的图片；提供方适配器将已授权的持久引用解析为提供方原生内容。 | | `ctx.fileUploads` | `core` | [`client-file-upload`](../packages/client/file-upload) | - | [`api-session-controller`](../packages/api/session-controller) | - | 负责流式接收、持久存储和暂存回执生命周期；Session Controller 将回执绑定到已接受的提交。 | diff --git a/docs/config-catalog.md b/docs/config-catalog.md index bc4eec2935..0ac5edbca0 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -4684,7 +4684,6 @@ These load from a `cordis.yml` entry with no `config:` block; they declare no co | `@deepseek-ai/dsh-client-ui-commands` | — | [`packages/client/ui-commands/src/index.ts`](../packages/client/ui-commands/src/index.ts) | | `@deepseek-ai/dsh-client-ui-conversation` | — | [`packages/client/ui-conversation/src/index.ts`](../packages/client/ui-conversation/src/index.ts) | | `@deepseek-ai/dsh-client-ui-cordis` | — | [`packages/extensions/ui-cordis/src/index.ts`](../packages/extensions/ui-cordis/src/index.ts) | -| `@deepseek-ai/dsh-client-ui-cua` | — | [`packages/experimental/client-ui-cua/src/index.ts`](../packages/experimental/client-ui-cua/src/index.ts) | | `@deepseek-ai/dsh-client-ui-deepseek-model-label-1017` | — | [`packages/client/ui-deepseek-model-label-1017/src/index.ts`](../packages/client/ui-deepseek-model-label-1017/src/index.ts) | | `@deepseek-ai/dsh-client-ui-deliverables` | `systemPrompt` · `connection` · `sessionQuery` · `sessionController` · `workspaceFiles` · `fs` · `sandboxPolicy` · `workspaceChanges` | [`packages/client/ui-deliverables/src/index.ts`](../packages/client/ui-deliverables/src/index.ts) | | `@deepseek-ai/dsh-client-ui-directory-picker-browse` | — | [`packages/client/ui-directory-picker-browse/src/index.ts`](../packages/client/ui-directory-picker-browse/src/index.ts) | @@ -4734,6 +4733,7 @@ These load from a `cordis.yml` entry with no `config:` block; they declare no co | `@deepseek-ai/dsh-computer-use` | — | [`packages/computer-use/computer-use/src/index.ts`](../packages/computer-use/computer-use/src/index.ts) | | `@deepseek-ai/dsh-config-editor` | `loader` · `profileContext` | [`packages/boot/config-editor/src/index.ts`](../packages/boot/config-editor/src/index.ts) | | `@deepseek-ai/dsh-cordis-client-runner` | — | [`packages/extensions/cordis-client-runner/src/index.ts`](../packages/extensions/cordis-client-runner/src/index.ts) | +| `@deepseek-ai/dsh-cua` | — | [`packages/experimental/cua/src/index.ts`](../packages/experimental/cua/src/index.ts) | | `@deepseek-ai/dsh-deepseek-llm-api-extensions` | — | [`packages/llm/deepseek-llm-api-extensions/src/index.ts`](../packages/llm/deepseek-llm-api-extensions/src/index.ts) | | `@deepseek-ai/dsh-experimental-auto-review` | `approval` · `llm` · `permissionPresets` · `sessions` · `tools` · `workingDirectory` | [`packages/experimental/auto-review/src/index.ts`](../packages/experimental/auto-review/src/index.ts) | | `@deepseek-ai/dsh-experimental-client-ui-agent-team` | — | [`packages/experimental/client-ui-agent-team/src/index.ts`](../packages/experimental/client-ui-agent-team/src/index.ts) | @@ -4821,7 +4821,6 @@ Imported as libraries by other packages; a `cordis.yml` cannot load them. | `@deepseek-ai/dsh-client-ui-slots` | — | [`packages/client/ui-slots/src/index.ts`](../packages/client/ui-slots/src/index.ts) | | `@deepseek-ai/dsh-client-web` | — | [`packages/client/web/src/index.ts`](../packages/client/web/src/index.ts) | | `@deepseek-ai/dsh-cmdline` | — | [`packages/boot/cmdline/src/index.ts`](../packages/boot/cmdline/src/index.ts) | -| `@deepseek-ai/dsh-cua` | — | [`packages/experimental/cua/src/index.ts`](../packages/experimental/cua/src/index.ts) | | `@deepseek-ai/dsh-cua-bundle` | — | [`packages/experimental/cua-bundle/src/index.ts`](../packages/experimental/cua-bundle/src/index.ts) | | `@deepseek-ai/dsh-deque` | — | [`packages/util/deque/src/index.ts`](../packages/util/deque/src/index.ts) | | `@deepseek-ai/dsh-experimental-agent-team-profile` | — | [`packages/experimental/agent-team-profile/src/index.ts`](../packages/experimental/agent-team-profile/src/index.ts) | diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index 88327a30f4..f8cfb1749c 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -4686,7 +4686,6 @@ export interface Config { | `@deepseek-ai/dsh-client-ui-commands` | — | [`packages/client/ui-commands/src/index.ts`](../packages/client/ui-commands/src/index.ts) | | `@deepseek-ai/dsh-client-ui-conversation` | — | [`packages/client/ui-conversation/src/index.ts`](../packages/client/ui-conversation/src/index.ts) | | `@deepseek-ai/dsh-client-ui-cordis` | — | [`packages/extensions/ui-cordis/src/index.ts`](../packages/extensions/ui-cordis/src/index.ts) | -| `@deepseek-ai/dsh-client-ui-cua` | — | [`packages/experimental/client-ui-cua/src/index.ts`](../packages/experimental/client-ui-cua/src/index.ts) | | `@deepseek-ai/dsh-client-ui-deepseek-model-label-1017` | — | [`packages/client/ui-deepseek-model-label-1017/src/index.ts`](../packages/client/ui-deepseek-model-label-1017/src/index.ts) | | `@deepseek-ai/dsh-client-ui-deliverables` | `systemPrompt` · `connection` · `sessionQuery` · `sessionController` · `workspaceFiles` · `fs` · `sandboxPolicy` · `workspaceChanges` | [`packages/client/ui-deliverables/src/index.ts`](../packages/client/ui-deliverables/src/index.ts) | | `@deepseek-ai/dsh-client-ui-directory-picker-browse` | — | [`packages/client/ui-directory-picker-browse/src/index.ts`](../packages/client/ui-directory-picker-browse/src/index.ts) | @@ -4736,6 +4735,7 @@ export interface Config { | `@deepseek-ai/dsh-computer-use` | — | [`packages/computer-use/computer-use/src/index.ts`](../packages/computer-use/computer-use/src/index.ts) | | `@deepseek-ai/dsh-config-editor` | `loader` · `profileContext` | [`packages/boot/config-editor/src/index.ts`](../packages/boot/config-editor/src/index.ts) | | `@deepseek-ai/dsh-cordis-client-runner` | — | [`packages/extensions/cordis-client-runner/src/index.ts`](../packages/extensions/cordis-client-runner/src/index.ts) | +| `@deepseek-ai/dsh-cua` | — | [`packages/experimental/cua/src/index.ts`](../packages/experimental/cua/src/index.ts) | | `@deepseek-ai/dsh-deepseek-llm-api-extensions` | — | [`packages/llm/deepseek-llm-api-extensions/src/index.ts`](../packages/llm/deepseek-llm-api-extensions/src/index.ts) | | `@deepseek-ai/dsh-experimental-auto-review` | `approval` · `llm` · `permissionPresets` · `sessions` · `tools` · `workingDirectory` | [`packages/experimental/auto-review/src/index.ts`](../packages/experimental/auto-review/src/index.ts) | | `@deepseek-ai/dsh-experimental-client-ui-agent-team` | — | [`packages/experimental/client-ui-agent-team/src/index.ts`](../packages/experimental/client-ui-agent-team/src/index.ts) | @@ -4823,7 +4823,6 @@ export interface Config { | `@deepseek-ai/dsh-client-ui-slots` | — | [`packages/client/ui-slots/src/index.ts`](../packages/client/ui-slots/src/index.ts) | | `@deepseek-ai/dsh-client-web` | — | [`packages/client/web/src/index.ts`](../packages/client/web/src/index.ts) | | `@deepseek-ai/dsh-cmdline` | — | [`packages/boot/cmdline/src/index.ts`](../packages/boot/cmdline/src/index.ts) | -| `@deepseek-ai/dsh-cua` | — | [`packages/experimental/cua/src/index.ts`](../packages/experimental/cua/src/index.ts) | | `@deepseek-ai/dsh-cua-bundle` | — | [`packages/experimental/cua-bundle/src/index.ts`](../packages/experimental/cua-bundle/src/index.ts) | | `@deepseek-ai/dsh-deque` | — | [`packages/util/deque/src/index.ts`](../packages/util/deque/src/index.ts) | | `@deepseek-ai/dsh-experimental-agent-team-profile` | — | [`packages/experimental/agent-team-profile/src/index.ts`](../packages/experimental/agent-team-profile/src/index.ts) | diff --git a/docs/module-graph.md b/docs/module-graph.md index 39941991c8..ab621eb709 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -246,7 +246,6 @@ flowchart TD pkg_office_to_pdf["office-to-pdf"] end subgraph group_experimental["packages/experimental"] - pkg_client_ui_cua["client-ui-cua"] pkg_cua["cua"] pkg_cua_bundle["cua-bundle"] pkg_experimental_agent_team["experimental-agent-team"] @@ -1544,7 +1543,6 @@ flowchart TD | [`client-web`](../packages/client/web) | `client` | — | | [`credentials`](../packages/credentials/credentials) | `credentials` | — | | [`office-to-pdf`](../packages/document/office-to-pdf) | `document` | — | -| [`client-ui-cua`](../packages/experimental/client-ui-cua) | `experimental` | — | | [`cua-bundle`](../packages/experimental/cua-bundle) | `experimental` | — | | [`experimental-agent-team-profile`](../packages/experimental/agent-team-profile) | `experimental` | — | | [`experimental-badge-skill-bundle`](../packages/experimental/badge-skill-bundle) | `experimental` | — | diff --git a/docs/module-graph.zh.md b/docs/module-graph.zh.md index 97e302bb3d..531c608183 100644 --- a/docs/module-graph.zh.md +++ b/docs/module-graph.zh.md @@ -248,7 +248,6 @@ flowchart TD pkg_office_to_pdf["office-to-pdf"] end subgraph group_experimental["packages/experimental"] - pkg_client_ui_cua["client-ui-cua"] pkg_cua["cua"] pkg_cua_bundle["cua-bundle"] pkg_experimental_agent_team["experimental-agent-team"] @@ -1546,7 +1545,6 @@ flowchart TD | [`client-web`](../packages/client/web) | `client` | — | | [`credentials`](../packages/credentials/credentials) | `credentials` | — | | [`office-to-pdf`](../packages/document/office-to-pdf) | `document` | — | -| [`client-ui-cua`](../packages/experimental/client-ui-cua) | `experimental` | — | | [`cua-bundle`](../packages/experimental/cua-bundle) | `experimental` | — | | [`experimental-agent-team-profile`](../packages/experimental/agent-team-profile) | `experimental` | — | | [`experimental-badge-skill-bundle`](../packages/experimental/badge-skill-bundle) | `experimental` | — | diff --git a/docs/subsystems/computer-use.i18n.yaml b/docs/subsystems/computer-use.i18n.yaml index 77ba44e680..fc82ebf720 100644 --- a/docs/subsystems/computer-use.i18n.yaml +++ b/docs/subsystems/computer-use.i18n.yaml @@ -3,8 +3,8 @@ # After editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/subsystems/computer-use.md /computer-use: - en: 6f40d55bef2d7753 - zh: 0fc4eb79afb3cd73 + en: aff84e3ac8c41594 + zh: bee7fc09854bfa8b /computer-use/choose-a-provider: en: fa2e0d7b41c0450b zh: af9a799711f901e6 diff --git a/docs/subsystems/computer-use.md b/docs/subsystems/computer-use.md index a00e0f6e04..bc8f1b0491 100644 --- a/docs/subsystems/computer-use.md +++ b/docs/subsystems/computer-use.md @@ -8,7 +8,7 @@ Computer use lets a model observe and operate the local desktop through a config The [`macOS CUA provider`](../../packages/experimental/cua-macos/README.md) uses the [shared CUA objects and executor](../../packages/experimental/cua/README.md), which enforce application consent through User Questions and profile configuration. -The CUA provider owns `ctx.cuaAccess` for system-permission status, explicit system requests, and application revocation. Its [permission page](../../packages/experimental/client-ui-cua/README.md) projects these controls; app/window operations enforce the provider's consent policy before dispatch. +The CUA provider owns `ctx.cuaAccess` for system-permission status, explicit system requests, and application revocation. Its [permission page](../../packages/experimental/cua/README.md) projects these controls; app/window operations enforce the provider's consent policy before dispatch. | Permission type | Meaning | |---|---| diff --git a/docs/subsystems/computer-use.zh.md b/docs/subsystems/computer-use.zh.md index 00ff655df2..1323c6b033 100644 --- a/docs/subsystems/computer-use.zh.md +++ b/docs/subsystems/computer-use.zh.md @@ -8,7 +8,7 @@ [macOS CUA 提供方](../../packages/experimental/cua-macos/README.zh.md)使用[共享 CUA 对象与执行器](../../packages/experimental/cua/README.zh.md)，通过 User Questions 和配置档设置执行应用授权。 -CUA provider 通过 `ctx.cuaAccess` 提供系统权限状态、用户主动申请和应用授权撤销。[权限页面](../../packages/experimental/client-ui-cua/README.zh.md)投影这些控件；app/window 操作在分派前执行 provider 的授权检查。 +CUA provider 通过 `ctx.cuaAccess` 提供系统权限状态、用户主动申请和应用授权撤销。[权限页面](../../packages/experimental/cua/README.zh.md)投影这些控件；app/window 操作在分派前执行 provider 的授权检查。 | 权限类型 | 含义 | |---|---| diff --git a/packages/experimental/README.i18n.yaml b/packages/experimental/README.i18n.yaml index 77c06a747b..ce1021f7ee 100644 --- a/packages/experimental/README.i18n.yaml +++ b/packages/experimental/README.i18n.yaml @@ -18,8 +18,8 @@ en: b55aeb927737b90d zh: 0b215fcb5c2c7908 /packages-experimental/packages: - en: e20594b1b7e506d7 - zh: ef76c3c732ddb547 + en: 94cc18a191e52f26 + zh: 6a5ce630fa730239 /packages-experimental/related-documentation: en: 1cb9d3448d44e48d zh: cd326d70a746b896 diff --git a/packages/experimental/README.md b/packages/experimental/README.md index 5f052b8e5e..0a01b71eeb 100644 --- a/packages/experimental/README.md +++ b/packages/experimental/README.md @@ -72,7 +72,6 @@ Promotion requires a named product role, documented public behavior and limitati | [CUA](cua/README.md) | Shared application/window objects, consent, and logged execution. | Library | | [macOS CUA](cua-macos/README.md) | Swift native execution and platform preparation. | `ctx.computerUse` | | [`cua-bundle`](cua-bundle/README.md) | Optional Computer Use switch for PTC sessions | — | -| [`client-ui-cua`](client-ui-cua/README.md) | Computer Use permission setup and application access controls | — | | [`computer-use-cua-driver-mcp`](computer-use-cua-driver-mcp/README.md) | Use an installed Cua Driver through MCP | `ctx.computerUse` | | [`computer-use-cua-driver-native`](computer-use-cua-driver-native/README.md) | Embed the Cua Driver native npm runtime | `ctx.computerUse` | | [`browser-use-playwright-mcp`](browser-use-playwright-mcp/README.md) | Playwright browser tools over MCP | `ctx.browserUse` | diff --git a/packages/experimental/README.zh.md b/packages/experimental/README.zh.md index 5a74a36072..6d20549870 100644 --- a/packages/experimental/README.zh.md +++ b/packages/experimental/README.zh.md @@ -72,7 +72,6 @@ kind: "package-group" | [CUA](cua/README.zh.md) | 共享应用与窗口对象、授权和已记录的执行。 | 库 | | [macOS CUA](cua-macos/README.zh.md) | Swift 原生执行与平台组件准备。 | `ctx.computerUse` | | [`cua-bundle`](cua-bundle/README.zh.md) | 面向 PTC 会话、默认关闭的 Computer Use 开关 | — | -| [`client-ui-cua`](client-ui-cua/README.zh.md) | Computer Use 权限设置与应用授权管理 | — | | [`computer-use-cua-driver-mcp`](computer-use-cua-driver-mcp/README.zh.md) | 通过 MCP 使用已安装的 Cua Driver | `ctx.computerUse` | | [`computer-use-cua-driver-native`](computer-use-cua-driver-native/README.zh.md) | 嵌入 Cua Driver 原生 npm 运行时 | `ctx.computerUse` | | [`browser-use-playwright-mcp`](browser-use-playwright-mcp/README.zh.md) | 通过 MCP 提供 Playwright 浏览器工具 | `ctx.browserUse` | diff --git a/packages/experimental/client-ui-cua/README.i18n.yaml b/packages/experimental/client-ui-cua/README.i18n.yaml deleted file mode 100644 index 9d1ae66b95..0000000000 --- a/packages/experimental/client-ui-cua/README.i18n.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# Bilingual-pair consistency record for README.md (docs/i18n/README.md): per heading -# section, a hash of its English and Chinese blocks outside code blocks and generated regions. -# After editing either side, bring the other along and re-record with: -# pnpm run verify-translation-pairing --write packages/experimental/client-ui-cua/README.md -/: - en: e5ff656e81db5fc8 - zh: 43a8e818159176b3 -/deepseek-ai-dsh-client-ui-cua: - en: 43859860b79a3757 - zh: e8e0829c36613320 -/deepseek-ai-dsh-client-ui-cua/summary: - en: 4e62e511b5d14986 - zh: 1fce04dbad7515f7 -/deepseek-ai-dsh-client-ui-cua/table-of-contents: - en: 6658ab477751154b - zh: 9b371809a6fe38dc -/deepseek-ai-dsh-client-ui-cua/use-this-package: - en: 57485612d3ebe444 - zh: b1886ab8e3c313af -/deepseek-ai-dsh-client-ui-cua/understand-the-implementation: - en: f16ebf34970a7720 - zh: f1d79b3a60df361c -/deepseek-ai-dsh-client-ui-cua/further-exploration: - en: f0fde9f736ade61a - zh: 43a534a21bcc1c13 -/deepseek-ai-dsh-client-ui-cua/model-experience: - en: 7cfbaa62f70c1916 - zh: 384ec816a50fc7c5 -/deepseek-ai-dsh-client-ui-cua/model-experience/kv-cache-effect: - en: f04fc6e6f503960f - zh: f080408ae6729fee -/deepseek-ai-dsh-client-ui-cua/known-limitations-and-deferred-work: - en: c032612d923e1367 - zh: eb77d474e21ca5d9 -/deepseek-ai-dsh-client-ui-cua/known-limitations-and-deferred-work/dev-note: - en: b836f01bd0bceb4a - zh: a6ea1361c4a48df0 diff --git a/packages/experimental/client-ui-cua/README.md b/packages/experimental/client-ui-cua/README.md deleted file mode 100644 index b0c9ae7a12..0000000000 --- a/packages/experimental/client-ui-cua/README.md +++ /dev/null @@ -1,92 +0,0 @@ ---- -description: "Check macOS Computer Use permissions and revoke application access." -kind: "package-reference" ---- - -# @deepseek-ai/dsh-client-ui-cua - -English | [中文](README.zh.md) - -## Summary - -Open Computer Use in Plugins to check Accessibility and Screen Recording, request missing system permissions, and revoke application access. Explicit activation probes permissions and may start a dedicated native helper. Missing privileges show setup guidance; a failed probe directs the user to diagnostics and retry in the details page. System prompts still require the user's decision. - -## Table of Contents - -- [Use this package](#use-this-package) -- [Understand the implementation](#understand-the-implementation) -- [Further Exploration](#further-exploration) -- [Model Experience](#model-experience) -- [Known Limitations and Deferred Work](#known-limitations-and-deferred-work) -- [Dev Note](#dev-note) - -The permission-page lists show native icons and application names. Paths remain available on hover. Icons are presentation metadata and are never saved with grants. Application access uses the existing User Questions interface, with session access, refusal and, for verifiable applications with writable settings, persistent access. - ------ - - -## Use this package - -The [Computer Use bundle](../cua-bundle/README.md) mounts this page with its native provider. **Authorize Computer Use** groups the new-session default, PTC execution mode, Accessibility, and Screen Recording inside one border, with dividers between the rows. Each system-permission row shows the system-reported responsible application’s `.app` bundle filename and icon together in a bordered label. **Manage applications Computer Use can access** places the application lists inside a separate bordered group. Choose **Request permission**, find that application in the macOS permission list, and enable its switch; if macOS names another application, follow that prompt. With no detected source, the page directs the user to the application named by macOS. **Open System Settings** opens the corresponding privacy pane. The page refreshes on entry, when the window regains focus, and after permission actions. A failed query offers **Retry**. Follow any macOS quit-and-reopen prompt and restart DSH as needed. - -System-permission operations affect the computer running DSH. When accessing DSH remotely, complete authorization on that computer. - -Enabling from the plugin list shows setup guidance when system permissions need attention, with **Later** and **Open settings** actions. Enabling from the detail page displays the permission controls directly. The detail page provides **Set to PTC** under **Default mode for new sessions**; only this action saves the default. Existing sessions keep their preset. **Use persistent mode** in the **PTC execution mode** row separately saves `ptc-runtime.executionMode: persistent-in-memory`. Switching reloads the shared PTC runtime and may interrupt an active call; subsequent calls retain variables and application objects in memory. Already-persistent runtimes show a disabled **Persistent mode enabled** button. - -The page lists always-allowed applications and refusals from any session. A refusal in one session does not cancel another session’s grant or prevent a new session from asking. Session-only grants remain effective within their sessions and are omitted from this page. **Revoke** removes the selected application's access and cancels affected work. Input already delivered to an application cannot be undone; observe the application before any further action. **Allow asking again** clears that application’s refusal records across all sessions; the next application selection still requires a new decision. - -This plugin has no configurable fields. - ------ - - -## Understand the implementation - -
-Implementation internals - -The optional browser contribution mounts the native provider's permission Remote and reuses the Plugins activation and configuration slots. The four setup rows remain visible while unknown fields use placeholders. The default-mode form, runtime configuration, system checks, application decisions, and application icons fill in as each query completes. Application names and actions do not wait for icons; icon positions retain their space. User actions run in order without waiting for pending observation queries. Disposal waits for both actions and queries to settle before releasing the separately owned Remote mount. Query errors stay with the affected section, and activation guidance depends only on system checks. Late results after disposal are discarded. One controller projects queries into the page and setup dialog; reconnects and updates to the Host-reported settings namespace refresh it. Operation notices live in the shell overlay, so closing the page does not remove them. The provider owns permission checks, persisted decisions, and cancellation; the browser owns no authorization state. - -The detail page reads the effective default (`selectedDefault ?? default`) from the existing `configForms` service. The default button writes only `selectedDefault` through the form's revision-checked settings path. The execution-mode row uses CUA's `ptcConfiguration()` and `enablePersistentPtc()` controls: the Host edits only `ptc-runtime.executionMode` through the existing configuration editor, preserving its ordinary Loader reload lifecycle. Permission checks and runtime state settle independently. Loading, read-only and unavailable settings cannot submit a change; a failed write remains beside the setting for retry. - -
- ------ - - -## Further Exploration - -- [Native provider](../cua/README.md): executable identity and operation checks. -- [Plugin manager](../../client/ui-plugin-manager/README.md): activation and page slots. -- [UI primitives](../../client/ui-primitives/README.md): shared modal, buttons, and toast. - ------ - - -## Model Experience - -None, as this package registers browser controls and no model-facing tools or prompt sections. - -#### KV Cache effect - -None; the native provider owns model-visible guidance and operation results. - -## Known Limitations and Deferred Work - - - -This CUA-specific UI remains experimental while Windows support is in development; promotion returns it to the `client` group as `ui-cua`. - -- macOS may require restarting the launching application after a system permission change. A pending request is not an authorization grant. -- Application attribution depends on an optional native system query and available application metadata. If either is unavailable, the page uses generic system-prompt guidance. Packaged Desktop distribution and permission attribution remain deferred. -- No invariant companion is published: this UI projects the provider's query and stores no independent authorization decisions. - - -### Dev Note - -
-Working context for maintainers - -None. - -
diff --git a/packages/experimental/client-ui-cua/README.zh.md b/packages/experimental/client-ui-cua/README.zh.md deleted file mode 100644 index 09a011c6d8..0000000000 --- a/packages/experimental/client-ui-cua/README.zh.md +++ /dev/null @@ -1,92 +0,0 @@ ---- -description: "检查 macOS Computer Use 权限并撤销应用访问权限。" -kind: "package-reference" ---- - -# @deepseek-ai/dsh-client-ui-cua - -[English](README.md) | 中文 - -## 概述 - -在插件页打开 Computer Use，可以检查辅助功能和屏幕录制权限、申请缺失的系统权限，并撤销应用授权。用户显式启用时会检查权限，可能启动专用原生 helper。缺少权限时显示设置引导，查询失败时引导用户到详情页查看原因并重试。系统授权仍由用户决定。 - -## 目录 - -- [使用本包](#use-this-package) -- [理解实现](#understand-the-implementation) -- [延伸阅读](#further-exploration) -- [模型体验](#model-experience) -- [已知限制与后续工作](#known-limitations-and-deferred-work) -- [开发备注](#dev-note) - -权限页列表显示本机图标与应用名称，悬停可查看路径。图标仅用于展示，不随授权保存。应用授权沿用现有 User Questions 界面，提供本会话允许和拒绝；应用签名可验证且配置可写时，还提供始终允许。 - ------ - - -## 使用本包 - -[Computer Use 组合](../cua-bundle/README.zh.md)将本页与原生 provider 一起挂载。“给 Computer Use 授权”把新会话默认模式、PTC 执行方式、辅助功能和屏幕录制放在同一个边框内，四排之间使用分隔线。两条系统权限要求在能识别责任应用时，把该应用的 `.app` 包名称和图标放在同一个标签中。“管理 Computer Use 可访问的应用”使用独立边框包住应用列表。点击“申请权限”，在 macOS 权限列表中找到该应用并开启对应开关；若 macOS 显示其他名称，以系统提示为准。来源无法识别时，页面引导用户查找系统提示的应用。“打开系统设置”会打开对应的隐私设置页。进入详情页、返回窗口或完成授权操作后会自动刷新；查询失败时可点击“重试”。若系统要求退出并重新打开应用，请按提示操作并重新启动 DSH。 - -系统权限操作作用于运行 DSH 的电脑。远程访问时，请在该电脑上完成授权。 - -从插件列表启用后，系统权限尚未就绪时会显示设置引导，提供“稍后”和“前往设置”。从详情页启用后直接显示权限控件。详情页的“新会话默认模式”提供“设为 PTC”按钮，仅点击此按钮才保存默认值。已有会话保留原 preset。“PTC 执行方式”一行的“设为持久化”单独保存 `ptc-runtime.executionMode: persistent-in-memory`。切换会重载共享 PTC 执行器，可能打断当前调用；后续调用在内存中保留变量和应用对象。已经使用持久化模式时，按钮显示“已设为持久化”并禁用。 - -页面展示“始终允许的应用”和“不允许访问的应用”。后者汇总任一会话的拒绝记录，不取消其他会话已有的授权，也不阻止新会话询问授权。会话授权仍在对应会话中生效，详情页不展示这类应用。“撤销”会移除对应应用的访问许可并取消相关操作。已发送到应用的输入无法撤回，继续操作前需重新观察应用。“允许再次询问”清除该应用在所有会话中的拒绝记录；下次选择应用时仍需用户重新决定。 - -本插件没有配置字段。 - ------ - - -## 理解实现 - -
-实现细节 - -可选浏览器插件挂载原生 provider 的权限 Remote，复用插件页的启用引导和配置插槽。四排使用条件始终可见，未加载的字段保留占位。默认模式、执行方式、系统权限、应用列表和应用图标分别查询，结果返回后各自显示。应用名称和操作按钮不等待图标，图标位置保留空位。用户操作按顺序执行，不等待尚未完成的查询；卸载时等待操作和查询全部结束，再释放独立持有的 Remote 挂载。查询错误只显示在对应部分，启用引导只等待系统权限结果。卸载后的迟到结果会被丢弃。一个控制器把查询结果投影到页面和设置弹窗；重新连接或 Host 返回的配置 namespace 更新时刷新状态。操作提示挂载到 shell overlay，关闭页面后仍可显示。provider 负责权限检查、持久授权和取消；浏览器不保存授权决定。 - -详情页通过现有 `configForms` 服务读取实际默认值（`selectedDefault ?? default`）。默认模式按钮仅写入 `selectedDefault`，沿用表单带版本检查的设置写入流程。执行方式一行调用 CUA 的 `ptcConfiguration()` 和 `enablePersistentPtc()`：Host 通过现有配置编辑器仅修改 `ptc-runtime.executionMode`，保留普通 Loader 重载规则。权限检查和执行方式状态独立返回。设置仍在加载、只读或不可用时不能提交；写入失败时在设置旁显示错误，供用户重试。 - -
- ------ - - -## 延伸阅读 - -- [原生 provider](../cua/README.zh.md)：可执行文件身份和操作检查。 -- [插件管理](../../client/ui-plugin-manager/README.zh.md)：启用引导和页面插槽。 -- [界面基础组件](../../client/ui-primitives/README.zh.md)：共用弹窗、按钮和提示。 - ------ - - -## 模型体验 - -无。本包注册浏览器控件，不注册模型工具或提示词段落。 - -#### KV 缓存影响 - -无。模型可见的指引和操作结果由原生 provider 负责。 - -## 已知限制与后续工作 - - - -这套 CUA 专用界面随 Windows 支持一起开发，暂放在 `experimental`；转为正式功能后迁回 `client` 组的 `ui-cua`。 - -- 系统权限变更后，macOS 可能要求重启启动程序。申请尚未完成时，不能视为已授权。 -- 应用归属依赖可选的本机系统查询和可读取的应用信息；不可用时，页面使用通用指引，让用户按系统弹窗完成授权。桌面安装包的分发和权限归属留待后续处理。 -- 不提供 invariant companion：本界面投影 provider 查询，不独立保存授权决定。 - - -### 开发备注 - -
-维护者工作记录 - -无。 - -
diff --git a/packages/experimental/client-ui-cua/package.json b/packages/experimental/client-ui-cua/package.json deleted file mode 100644 index 6554c14387..0000000000 --- a/packages/experimental/client-ui-cua/package.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "name": "@deepseek-ai/dsh-client-ui-cua", - "description": "Computer Use permission setup and application access controls", - "version": "0.2.1-rc.2", - "publishConfig": { - "access": "public" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/deepseek-ai/deepseek-harness.git", - "directory": "packages/experimental/client-ui-cua" - }, - "type": "module", - "main": "lib/index.js", - "types": "lib/types/index.d.ts", - "exports": { - ".": { - "types": "./lib/types/index.d.ts", - "default": "./lib/index.js" - }, - "./client": { - "types": "./lib/types/client/index.d.ts", - "default": "./lib/client.js" - }, - "./src/*": "./src/*", - "./package.json": "./package.json" - }, - "dsh": { - "client": { - "inject": [ - "@deepseek-ai/dsh-api-remotes", - "@deepseek-ai/dsh-client-locale", - "@deepseek-ai/dsh-client-ui-plugin-manager", - "@deepseek-ai/dsh-client-ui-settings", - "@deepseek-ai/dsh-client-ui-layout" - ], - "platform": "web" - } - }, - "scripts": { - "bundle": "tsdown", - "watch": "tsdown --watch" - }, - "license": "MIT", - "peerDependencies": { - "@deepseek-ai/cordis": "workspace:~" - }, - "devDependencies": { - "@deepseek-ai/cordis": "workspace:~", - "@deepseek-ai/dsh-api-remotes": "workspace:*", - "@deepseek-ai/dsh-api-gateway": "workspace:*", - "@deepseek-ai/dsh-client-locale": "workspace:*", - "@deepseek-ai/dsh-client-store": "workspace:*", - "@deepseek-ai/dsh-client-ui-primitives": "workspace:*", - "@deepseek-ai/dsh-client-ui-renderer": "workspace:*", - "@deepseek-ai/dsh-client-ui-slots": "workspace:*", - "@deepseek-ai/dsh-client-ui-plugin-manager": "workspace:*", - "@deepseek-ai/dsh-client-ui-settings": "workspace:*", - "@deepseek-ai/dsh-client-ui-layout": "workspace:*", - "@deepseek-ai/dsh-cua": "workspace:*", - "@deepseek-ai/dsh-client-test-runtime": "workspace:*", - "react": "^18.2.0", - "react-dom": "^18.2.0", - "@testing-library/react": "^16.1.0", - "@deepseek-ai/dsh-typert-protocol": "workspace:*" - }, - "files": [ - "lib/index.js", - "lib/client.js", - "lib/types/**/*.d.ts" - ] -} diff --git a/packages/experimental/client-ui-cua/src/index.ts b/packages/experimental/client-ui-cua/src/index.ts deleted file mode 100644 index ceee879f3f..0000000000 --- a/packages/experimental/client-ui-cua/src/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -/** Host marker; the client export owns the optional permission UI, while dsh-cua owns native policy. */ -export function apply(): void {} diff --git a/packages/experimental/client-ui-cua/tsdown.config.ts b/packages/experimental/client-ui-cua/tsdown.config.ts deleted file mode 100644 index 71d5bd77f8..0000000000 --- a/packages/experimental/client-ui-cua/tsdown.config.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { clientBundle } from '../../client/tsdown.client.ts' - -export default clientBundle(import.meta.dirname, ['lib/types/index.js']) diff --git a/packages/experimental/cua-bundle/README.i18n.yaml b/packages/experimental/cua-bundle/README.i18n.yaml index f84fa9dcaa..a2b0c1f5fe 100644 --- a/packages/experimental/cua-bundle/README.i18n.yaml +++ b/packages/experimental/cua-bundle/README.i18n.yaml @@ -15,8 +15,8 @@ en: b563e4ec4727d0de zh: d0228da978238e05 /deepseek-ai-dsh-cua-bundle/use-this-package: - en: c50ddfc2865ed10f - zh: f0ee5fd582c7e034 + en: d171cfb240ae4503 + zh: b65e31e613f7b000 /deepseek-ai-dsh-cua-bundle/understand-the-implementation: en: 057771fe13aaa337 zh: 4fdb2b98ff3e7a78 diff --git a/packages/experimental/cua-bundle/README.md b/packages/experimental/cua-bundle/README.md index 921bc05f20..fc74a014cd 100644 --- a/packages/experimental/cua-bundle/README.md +++ b/packages/experimental/cua-bundle/README.md @@ -33,7 +33,7 @@ pnpm dsh web For the source Desktop launcher and its separate development profile, follow [Desktop development](../../../apps/desktop/README.md). Enable Computer Use in each profile that needs it. -The switch saves bundle selection in the profile’s `package.json`. Computer Use is installed only in PTC sessions. Enabling from the plugin list offers setup guidance when system permissions need attention; enabling from its detail page displays the permission controls directly. The detail page offers **Set to PTC** for the new-session default; existing sessions keep their preset. The switch leaves PTC execution mode and other settings unchanged. The detail page’s **Use persistent mode** action separately enables in-memory state across PTC calls; its effects and reload behavior are described in the [permission page](../client-ui-cua/README.md). Native preparation runs when the provider loads and reuses the source-hashed executable when available. +The switch saves bundle selection in the profile’s `package.json`. Computer Use is installed only in PTC sessions. Enabling from the plugin list offers setup guidance when system permissions need attention; enabling from its detail page displays the permission controls directly. The detail page offers **Set to PTC** for the new-session default; existing sessions keep their preset. The switch leaves PTC execution mode and other settings unchanged. The detail page’s **Use persistent mode** action separately enables in-memory state across PTC calls; its effects and reload behavior are described in the [permission page](../cua/README.md). Native preparation runs when the provider loads and reuses the source-hashed executable when available. Disabling unloads the provider and withdraws CUA tools from its Agent scopes. Runtime configuration, the new-session default and saved grants remain. Session-only application decisions end with provider disposal. @@ -41,7 +41,7 @@ Computer Use requires a macOS DSH Host. Other operating systems can display the If preparation fails, the saved selection remains and CUA operations stay unavailable. Read the Host startup diagnostic, correct its cause, then restart DSH. -Explicit activation runs a read-only permission check, which may start a helper without opening a system dialog. Open the [permission page](../client-ui-cua/README.md) for system requests, settings navigation and automatic refresh. Tasks ask separately before accessing an application. Saved application decisions use the existing profile configuration; session decisions remain in memory across turns and connections until DSH restarts or Computer Use is disabled. Revoke access on the same page. +Explicit activation runs a read-only permission check, which may start a helper without opening a system dialog. Open the [permission page](../cua/README.md) for system requests, settings navigation and automatic refresh. Tasks ask separately before accessing an application. Saved application decisions use the existing profile configuration; session decisions remain in memory across turns and connections until DSH restarts or Computer Use is disabled. Revoke access on the same page. ----- diff --git a/packages/experimental/cua-bundle/README.zh.md b/packages/experimental/cua-bundle/README.zh.md index 6c551c5a09..febad6d82b 100644 --- a/packages/experimental/cua-bundle/README.zh.md +++ b/packages/experimental/cua-bundle/README.zh.md @@ -33,7 +33,7 @@ pnpm dsh web 源码 Desktop 的启动方式和独立开发配置档见[桌面开发说明](../../../apps/desktop/README.zh.md)。需要使用 Computer Use 的配置档分别开启该功能。 -开关将组合包选择保存到 profile 的 `package.json`。Computer Use 仅安装在 PTC 会话中。从插件列表启用时，系统权限未就绪会显示设置引导；从详情页启用后直接显示权限控件。详情页提供“设为 PTC”按钮来修改新会话默认模式，已有会话保留原 preset。开关保持 PTC 执行方式和其他设置不变。详情页的“设为持久化”按钮单独开启 PTC 跨调用的内存状态保留，作用范围和重载行为见[权限页面](../client-ui-cua/README.zh.md)。提供方加载时准备本机组件，已有与源码摘要匹配的可执行文件时直接复用。 +开关将组合包选择保存到 profile 的 `package.json`。Computer Use 仅安装在 PTC 会话中。从插件列表启用时，系统权限未就绪会显示设置引导；从详情页启用后直接显示权限控件。详情页提供“设为 PTC”按钮来修改新会话默认模式，已有会话保留原 preset。开关保持 PTC 执行方式和其他设置不变。详情页的“设为持久化”按钮单独开启 PTC 跨调用的内存状态保留，作用范围和重载行为见[权限页面](../cua/README.zh.md)。提供方加载时准备本机组件，已有与源码摘要匹配的可执行文件时直接复用。 关闭会卸载提供方，并从其 Agent 作用域中撤回 CUA 工具。执行器配置、新会话默认值和已保存的授权保留。会话级应用授权随提供方卸载而失效。 @@ -41,7 +41,7 @@ Computer Use 要求 DSH Host 运行在 macOS 上。其他系统可以显示入 准备失败时，已保存的选择保留，CUA 操作不可用。查看 Host 的启动诊断，修复原因后重启 DSH。 -显式启用会进行只读权限检查，可能启动 helper，但不会打开系统授权弹窗。[权限页面](../client-ui-cua/README.zh.md) 提供系统申请入口、设置跳转和自动检测。任务访问应用前会单独询问用户。长期应用授权保存在现有 profile 配置中，会话授权保存在内存中，跨 turn 和连接复用，重启 DSH 或关闭 Computer Use 后失效。可以在同一页面撤销授权。 +显式启用会进行只读权限检查，可能启动 helper，但不会打开系统授权弹窗。[权限页面](../cua/README.zh.md) 提供系统申请入口、设置跳转和自动检测。任务访问应用前会单独询问用户。长期应用授权保存在现有 profile 配置中，会话授权保存在内存中，跨 turn 和连接复用，重启 DSH 或关闭 Computer Use 后失效。可以在同一页面撤销授权。 ----- diff --git a/packages/experimental/cua-bundle/cordis.patch.yml b/packages/experimental/cua-bundle/cordis.patch.yml index e468011f06..99af9b3ccb 100644 --- a/packages/experimental/cua-bundle/cordis.patch.yml +++ b/packages/experimental/cua-bundle/cordis.patch.yml @@ -6,4 +6,4 @@ config: agentPreset: ptc - id: ui-cua - name: '@deepseek-ai/dsh-client-ui-cua' + name: '@deepseek-ai/dsh-cua' diff --git a/packages/experimental/cua-bundle/package.json b/packages/experimental/cua-bundle/package.json index 155345ecc6..546c66db40 100644 --- a/packages/experimental/cua-bundle/package.json +++ b/packages/experimental/cua-bundle/package.json @@ -40,7 +40,7 @@ "dependencies": { "@deepseek-ai/dsh-experimental-cua-macos": "workspace:*", "@deepseek-ai/dsh-computer-use": "workspace:*", - "@deepseek-ai/dsh-client-ui-cua": "workspace:*" + "@deepseek-ai/dsh-cua": "workspace:*" }, "peerDependencies": { "@deepseek-ai/cordis": "workspace:~" diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index 8e064caef0..e7d55edf7c 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -45,8 +45,8 @@ en: a2c9acb5a4320dda zh: e8d1586af60215c4 /deepseek-ai-dsh-experimental-cua-macos/known-limitations-and-deferred-work: - en: 1c42b446c186b01b - zh: 0f1ecea4fd0a1a13 + en: de72c8c1184cb8dd + zh: 33fd21a93e51fee1 /deepseek-ai-dsh-experimental-cua-macos/known-limitations-and-deferred-work/dev-note: en: b836f01bd0bceb4a zh: 8122fa08b2aa95fa diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 7700c69ed1..a91c37b885 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -216,7 +216,7 @@ The queue orders individual native requests. Holding an app object does not rese - macOS Stage Manager can reduce background window bounds and screenshots to thumbnails. Full-window observation and coordinate input in this state are not guaranteed; model guidance requires explaining foreground needs and avoiding repeated focus reclamation. - Background input depends on private SkyLight symbols; macOS updates can make them unavailable. Validate each target OS version. - Observed failures include dispatched Unicode text not appearing in Blender and background Command+A not selecting all text in AppKit. Inspect application state after input; a successful tool result confirms dispatch only. -- Native binaries require a local build. Signing and Desktop distribution remain future work. The [permission page](../client-ui-cua/README.md) offers macOS prompts and app revocation; packaged Desktop permission attribution remains deferred. +- Native binaries require a local build. Signing and Desktop distribution remain future work. The [permission page](../cua/README.md) offers macOS prompts and app revocation; packaged Desktop permission attribution remains deferred. - Persistent PTC is an external dependency assumption, not an implementation in this package. Real-model task success requires separate evaluation beyond native and protocol fixtures. ----- diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index ccd844dcf7..03ff814e8a 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -216,7 +216,7 @@ Use selectText only on controls advertising selectionWritable. It selects a uniq - macOS 台前调度可能将后台窗口的尺寸与截图变为缩略图，当前无法保证这种状态下的完整画面与坐标输入；模型指引要求说明前台需求并避免反复抢回焦点。 - 后台输入依赖私有 SkyLight 符号，macOS 升级可能使其不可用，需在目标系统版本验证。 - 实测 Blender Unicode 输入曾出现分派成功但界面未显示文本，后台 AppKit 的 Command+A 曾未选中全文。输入后必须检查应用状态；工具返回成功只表示已分派。 -- 原生二进制需要本地构建，签名和桌面分发留待后续。[权限页面](../client-ui-cua/README.zh.md)提供 macOS 申请入口和应用授权撤销；桌面安装包的权限归属留待后续处理。 +- 原生二进制需要本地构建，签名和桌面分发留待后续。[权限页面](../cua/README.zh.md)提供 macOS 申请入口和应用授权撤销；桌面安装包的权限归属留待后续处理。 - 持久 PTC 是对外部运行环境的假设，本包不实现它。真实模型任务成功率需要在原生测试和协议测试之外单独评估。 ----- diff --git a/packages/experimental/cua-macos/tsconfig.json b/packages/experimental/cua-macos/tsconfig.json index 05a6c912a2..986873b01b 100644 --- a/packages/experimental/cua-macos/tsconfig.json +++ b/packages/experimental/cua-macos/tsconfig.json @@ -12,7 +12,7 @@ "path": "../../../vendor/cordis" }, { - "path": "../cua" + "path": "../cua/tsconfig.host.json" }, { "path": "../../subprocess/subprocess" diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index 5bdc7c755d..d00649e0f8 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -3,8 +3,8 @@ # After editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/experimental/cua/README.md /: - en: aa3d316ef17feae7 - zh: 5a30d743ce257d5b + en: ed2539c6d12f814a + zh: 55a6c9d7f0d21fb7 /deepseek-ai-dsh-cua: en: 0ef7ee882840cf1b zh: 01b9e768eff2d302 @@ -15,11 +15,14 @@ en: 4081cf444a77bf20 zh: def91d58969ec281 /deepseek-ai-dsh-cua/use-this-package: - en: 577e1b18dd220c73 - zh: bde0692c6695d749 + en: ceb54d964dc355b6 + zh: 0892066fbceef7b9 +/deepseek-ai-dsh-cua/use-this-package/permission-controls: + en: 6873925322180290 + zh: 218cada140ec90b1 /deepseek-ai-dsh-cua/understand-the-implementation: - en: 727a4ec9b5cd1789 - zh: 000a6a34b29e1008 + en: 68fc12448f1f1fc0 + zh: ea1d489c263c6d71 /deepseek-ai-dsh-cua/further-exploration: en: b8b0aece886fadaa zh: ecd8c88d601f2179 diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index 99c32a8c92..d143d12d18 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -1,6 +1,6 @@ --- description: "Reuse Computer Use application and window objects with authorized, logged platform operations." -kind: "package-library" +kind: "package-reference" --- # @deepseek-ai/dsh-cua @@ -41,6 +41,18 @@ Actions return `dispatched:true, verified:false`; read fresh state to verify the `click(target, { holdMs: 800 })` holds the mouse button for 800 milliseconds before release. `holdMs` accepts nonnegative integers and applies to each press of a double-click; zero releases without an additional hold. The provider config `maxClickHoldMs` limits explicit holds and defaults to 5000 ms. Explicit holds use pointer input for both AX indices and coordinates; omission preserves ordinary click behavior. The operation deadline includes the hold, and cancellation or timeout releases the button. Received input cannot be undone; forced helper termination cannot guarantee release. +### Permission controls + +The [Computer Use bundle](../cua-bundle/README.md) mounts this page with its native provider. **Authorize Computer Use** groups the new-session default, PTC execution mode, Accessibility, and Screen Recording inside one border, with dividers between the rows. Each system-permission row shows the system-reported responsible application’s `.app` bundle filename and icon together in a bordered label. **Manage applications Computer Use can access** places the application lists inside a separate bordered group. Choose **Request permission**, find that application in the macOS permission list, and enable its switch; if macOS names another application, follow that prompt. With no detected source, the page directs the user to the application named by macOS. **Open System Settings** opens the corresponding privacy pane. The page refreshes on entry, when the window regains focus, and after permission actions. A failed query offers **Retry**. Follow any macOS quit-and-reopen prompt and restart DSH as needed. + +System-permission operations affect the computer running DSH. When accessing DSH remotely, complete authorization on that computer. + +Enabling from the plugin list shows setup guidance when system permissions need attention, with **Later** and **Open settings** actions. Enabling from the detail page displays the permission controls directly. The detail page provides **Set to PTC** under **Default mode for new sessions**; only this action saves the default. Existing sessions keep their preset. **Use persistent mode** in the **PTC execution mode** row separately saves `ptc-runtime.executionMode: persistent-in-memory`. Switching reloads the shared PTC runtime and may interrupt an active call; subsequent calls retain variables and application objects in memory. Already-persistent runtimes show a disabled **Persistent mode enabled** button. + +The page lists always-allowed applications and refusals from any session. A refusal in one session does not cancel another session’s grant or prevent a new session from asking. Session-only grants remain effective within their sessions and are omitted from this page. **Revoke** removes the selected application's access and cancels affected work. Input already delivered to an application cannot be undone; observe the application before any further action. **Allow asking again** clears that application’s refusal records across all sessions; the next application selection still requires a new decision. + +The Host marker has no configurable fields; platform providers own native configuration. + ----- @@ -57,6 +69,12 @@ The provider calls [installCua](src/index.ts) with its prepared helper, connecti [The operation schema](src/protocol.ts) is the authority for model parameters and separately declared Host-only fields. Native implementations enforce target identity, window geometry, resource limits, and operating-system access at execution. No invariant companion is published because this library has no separately projected registry or durable state. + +The root Host entry exports `installCua`, `CuaConfig`, and the Typert-discovered `CuaAccess` service; its `apply()` only marks the optional Client contribution. `/client` loads the permission UI through the existing module loader. Separate Host and Client compiler projects keep React, CSS, and browser registrations out of headless imports. `/api` owns the app/window API and its types; `/bridge`, `/native-client`, and `/protocol` retain runtime and platform consumers. `/types` exports only the four named boundary types required by generated Remote declarations. The Client uses package-local types. `/typert` supplies Host reflection and `/remote` supplies generated browser descriptors and codecs; permission authority remains on the Host. + +The optional browser contribution mounts the native provider's permission Remote and reuses the Plugins activation and configuration slots. The four setup rows remain visible while unknown fields use placeholders. The default-mode form, runtime configuration, system checks, application decisions, and application icons fill in as each query completes. Application names and actions do not wait for icons; icon positions retain their space. User actions run in order without waiting for pending observation queries. Disposal waits for both actions and queries to settle before releasing the separately owned Remote mount. Query errors stay with the affected section, and activation guidance depends only on system checks. Late results after disposal are discarded. One controller projects queries into the page and setup dialog; reconnects and updates to the Host-reported settings namespace refresh it. Operation notices live in the shell overlay, so closing the page does not remove them. The provider owns permission checks, persisted decisions, and cancellation; the browser owns no authorization state. + +The detail page reads the effective default (`selectedDefault ?? default`) from the existing `configForms` service. The default button writes only `selectedDefault` through the form's revision-checked settings path. The execution-mode row uses CUA's `ptcConfiguration()` and `enablePersistentPtc()` controls: the Host edits only `ptc-runtime.executionMode` through the existing configuration editor, preserving its ordinary Loader reload lifecycle. Permission checks and runtime state settle independently. Loading, read-only and unavailable settings cannot submit a change; a failed write remains beside the setting for retry.
----- diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index ad5251c485..0bfcaf6214 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -1,6 +1,6 @@ --- description: "复用 Computer Use 应用和窗口对象，通过已授权并记录日志的平台操作完成任务。" -kind: "package-library" +kind: "package-reference" --- # @deepseek-ai/dsh-cua @@ -41,6 +41,18 @@ kind: "package-library" `click(target, { holdMs: 800 })` 按住鼠标按钮 800 毫秒后松开。`holdMs` 接受非负整数，双击时应用于每次按下；0 表示按下后立即松开，不额外等待。提供方配置 `maxClickHoldMs` 限制显式按住时长，默认 5000 毫秒。指定时长后，AX 索引和坐标目标都使用鼠标事件；省略时保留普通点击行为。按住时间计入操作超时，取消或超时会松开按钮。已接收的输入无法撤销；辅助进程被强制终止时无法保证松开。 +### 权限控件 + +[Computer Use 组合](../cua-bundle/README.zh.md)将本页与原生 provider 一起挂载。“给 Computer Use 授权”把新会话默认模式、PTC 执行方式、辅助功能和屏幕录制放在同一个边框内，四排之间使用分隔线。两条系统权限要求在能识别责任应用时，把该应用的 `.app` 包名称和图标放在同一个标签中。“管理 Computer Use 可访问的应用”使用独立边框包住应用列表。点击“申请权限”，在 macOS 权限列表中找到该应用并开启对应开关；若 macOS 显示其他名称，以系统提示为准。来源无法识别时，页面引导用户查找系统提示的应用。“打开系统设置”会打开对应的隐私设置页。进入详情页、返回窗口或完成授权操作后会自动刷新；查询失败时可点击“重试”。若系统要求退出并重新打开应用，请按提示操作并重新启动 DSH。 + +系统权限操作作用于运行 DSH 的电脑。远程访问时，请在该电脑上完成授权。 + +从插件列表启用后，系统权限尚未就绪时会显示设置引导，提供“稍后”和“前往设置”。从详情页启用后直接显示权限控件。详情页的“新会话默认模式”提供“设为 PTC”按钮，仅点击此按钮才保存默认值。已有会话保留原 preset。“PTC 执行方式”一行的“设为持久化”单独保存 `ptc-runtime.executionMode: persistent-in-memory`。切换会重载共享 PTC 执行器，可能打断当前调用；后续调用在内存中保留变量和应用对象。已经使用持久化模式时，按钮显示“已设为持久化”并禁用。 + +页面展示“始终允许的应用”和“不允许访问的应用”。后者汇总任一会话的拒绝记录，不取消其他会话已有的授权，也不阻止新会话询问授权。会话授权仍在对应会话中生效，详情页不展示这类应用。“撤销”会移除对应应用的访问许可并取消相关操作。已发送到应用的输入无法撤回，继续操作前需重新观察应用。“允许再次询问”清除该应用在所有会话中的拒绝记录；下次选择应用时仍需用户重新决定。 + +本插件没有配置字段。 + ----- @@ -57,6 +69,12 @@ kind: "package-library" [操作 schema](src/protocol.ts) 定义模型参数，并单独声明 Host 专用字段。原生实现在执行时检查目标身份、窗口几何、资源上限和操作系统访问条件。本库没有独立投影的注册表或持久状态，因此不发布 invariant 模块。 + +根 Host 入口导出 `installCua`、`CuaConfig` 和供 Typert 发现的 `CuaAccess` 服务，`apply()` 只标记可选 Client 贡献。`/client` 通过现有模块加载器加载权限界面。Host 和 Client 使用独立编译项目，headless 导入不会加载 React、CSS 或浏览器注册。`/api` 提供应用、窗口 API 及其类型；`/bridge`、`/native-client` 和 `/protocol` 保留运行时和平台消费者。`/types` 仅导出生成 Remote 声明所需的四个命名类型，Client 使用包内类型。`/typert` 提供 Host 反射，`/remote` 提供浏览器描述和编解码，权限判断仍由 Host 执行。 + +可选浏览器插件挂载原生 provider 的权限 Remote，复用插件页的启用引导和配置插槽。四排使用条件始终可见，未加载的字段保留占位。默认模式、执行方式、系统权限、应用列表和应用图标分别查询，结果返回后各自显示。应用名称和操作按钮不等待图标，图标位置保留空位。用户操作按顺序执行，不等待尚未完成的查询；卸载时等待操作和查询全部结束，再释放独立持有的 Remote 挂载。查询错误只显示在对应部分，启用引导只等待系统权限结果。卸载后的迟到结果会被丢弃。一个控制器把查询结果投影到页面和设置弹窗；重新连接或 Host 返回的配置 namespace 更新时刷新状态。操作提示挂载到 shell overlay，关闭页面后仍可显示。provider 负责权限检查、持久授权和取消；浏览器不保存授权决定。 + +详情页通过现有 `configForms` 服务读取实际默认值（`selectedDefault ?? default`）。默认模式按钮仅写入 `selectedDefault`，沿用表单带版本检查的设置写入流程。执行方式一行调用 CUA 的 `ptcConfiguration()` 和 `enablePersistentPtc()`：Host 通过现有配置编辑器仅修改 `ptc-runtime.executionMode`，保留普通 Loader 重载规则。权限检查和执行方式状态独立返回。设置仍在加载、只读或不可用时不能提交；写入失败时在设置旁显示错误，供用户重试。
----- diff --git a/packages/experimental/cua/package.json b/packages/experimental/cua/package.json index a28b368b46..a5c7ff5142 100644 --- a/packages/experimental/cua/package.json +++ b/packages/experimental/cua/package.json @@ -37,8 +37,8 @@ "default": "./lib/typert.remote-client.js" }, "./types": { - "types": "./lib/types/types.d.ts", - "default": "./lib/types.js" + "types": "./lib/types/remote-types.d.ts", + "default": "./lib/remote-types.js" }, "./native-client": { "types": "./lib/types/native-client.d.ts", @@ -47,11 +47,14 @@ "./protocol": { "types": "./lib/types/protocol.d.ts", "default": "./lib/protocol.js" + }, + "./client": { + "types": "./lib/types/client/index.d.ts", + "default": "./lib/client.js" } }, "files": [ "lib/index.js", - "lib/types.js", "lib/api.js", "lib/bridge.js", "lib/native-client.js", @@ -60,7 +63,9 @@ "lib/typert.host.js", "lib/typert.host.d.ts", "lib/typert.remote-client.js", - "lib/typert.remote-client.d.ts" + "lib/typert.remote-client.d.ts", + "lib/remote-types.js", + "lib/client.js" ], "license": "MIT", "peerDependencies": { @@ -116,6 +121,38 @@ "@deepseek-ai/dsh-hmr": "workspace:*", "@deepseek-ai/cordis-plugin-timer": "workspace:~", "@deepseek-ai/dsh-scope": "workspace:*", - "@deepseek-ai/dsh-agent-preset-registry": "workspace:*" + "@deepseek-ai/dsh-agent-preset-registry": "workspace:*", + "@deepseek-ai/dsh-api-remotes": "workspace:*", + "@deepseek-ai/dsh-api-gateway": "workspace:*", + "@deepseek-ai/dsh-client-locale": "workspace:*", + "@deepseek-ai/dsh-client-store": "workspace:*", + "@deepseek-ai/dsh-client-ui-primitives": "workspace:*", + "@deepseek-ai/dsh-client-ui-renderer": "workspace:*", + "@deepseek-ai/dsh-client-ui-slots": "workspace:*", + "@deepseek-ai/dsh-client-ui-plugin-manager": "workspace:*", + "@deepseek-ai/dsh-client-ui-settings": "workspace:*", + "@deepseek-ai/dsh-client-ui-layout": "workspace:*", + "@deepseek-ai/dsh-client-test-runtime": "workspace:*", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "@testing-library/react": "^16.1.0", + "@deepseek-ai/dsh-typert-protocol": "workspace:*", + "@deepseek-ai/dsh-loader-smoke": "workspace:*" + }, + "dsh": { + "client": { + "inject": [ + "@deepseek-ai/dsh-api-remotes", + "@deepseek-ai/dsh-client-locale", + "@deepseek-ai/dsh-client-ui-plugin-manager", + "@deepseek-ai/dsh-client-ui-settings", + "@deepseek-ai/dsh-client-ui-layout" + ], + "platform": "web" + } + }, + "scripts": { + "bundle": "tsdown", + "watch": "tsdown --watch" } } diff --git a/packages/experimental/client-ui-cua/src/client/PermissionPanel.module.css b/packages/experimental/cua/src/client/PermissionPanel.module.css similarity index 100% rename from packages/experimental/client-ui-cua/src/client/PermissionPanel.module.css rename to packages/experimental/cua/src/client/PermissionPanel.module.css diff --git a/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx b/packages/experimental/cua/src/client/PermissionPanel.tsx similarity index 99% rename from packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx rename to packages/experimental/cua/src/client/PermissionPanel.tsx index 535e22b666..8c304ef81d 100644 --- a/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx +++ b/packages/experimental/cua/src/client/PermissionPanel.tsx @@ -4,7 +4,7 @@ import { Button, Modal, Toast } from '@deepseek-ai/dsh-client-ui-primitives' import type { InjectFace, PropsLocale, PropsRuntime } from '@deepseek-ai/dsh-client-ui-slots' import type {} from '@deepseek-ai/dsh-client-ui-plugin-manager/client' import type { ConfigForm } from '@deepseek-ai/dsh-client-ui-settings/client' -import type { CuaApplicationGrant, CuaSystemPermissionState } from '@deepseek-ai/dsh-cua/types' +import type { CuaApplicationGrant, CuaSystemPermissionState } from '../access-types.ts' import type { PermissionFace } from './controller.ts' import type { NS } from './locales.ts' import css from './PermissionPanel.module.css' diff --git a/packages/experimental/client-ui-cua/src/client/controller.ts b/packages/experimental/cua/src/client/controller.ts similarity index 98% rename from packages/experimental/client-ui-cua/src/client/controller.ts rename to packages/experimental/cua/src/client/controller.ts index 9a29b04436..c23497c4d7 100644 --- a/packages/experimental/client-ui-cua/src/client/controller.ts +++ b/packages/experimental/cua/src/client/controller.ts @@ -1,6 +1,6 @@ /** One query projection for the permission page and activation guidance. */ import { createSnapshotStore, type SnapshotStore } from '@deepseek-ai/dsh-client-store' -import type { CuaApplicationId, CuaApplicationState, CuaSystemPermission, CuaSystemPermissionState } from '@deepseek-ai/dsh-cua/types' +import type { CuaApplicationId, CuaApplicationState, CuaSystemPermission, CuaSystemPermissionState } from '../access-types.ts' /** Host operations available to the permission UI. */ export interface PermissionActions { diff --git a/packages/experimental/client-ui-cua/src/client/index.ts b/packages/experimental/cua/src/client/index.ts similarity index 100% rename from packages/experimental/client-ui-cua/src/client/index.ts rename to packages/experimental/cua/src/client/index.ts diff --git a/packages/experimental/client-ui-cua/src/client/locales.ts b/packages/experimental/cua/src/client/locales.ts similarity index 100% rename from packages/experimental/client-ui-cua/src/client/locales.ts rename to packages/experimental/cua/src/client/locales.ts diff --git a/packages/experimental/client-ui-cua/src/client/mount.ts b/packages/experimental/cua/src/client/mount.ts similarity index 100% rename from packages/experimental/client-ui-cua/src/client/mount.ts rename to packages/experimental/cua/src/client/mount.ts diff --git a/packages/experimental/client-ui-cua/src/css-modules.d.ts b/packages/experimental/cua/src/css-modules.d.ts similarity index 100% rename from packages/experimental/client-ui-cua/src/css-modules.d.ts rename to packages/experimental/cua/src/css-modules.d.ts diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index 708830bf2c..1cec00f09a 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -29,7 +29,6 @@ import type {} from '@deepseek-ai/dsh-settings' import type {} from '@deepseek-ai/cordis-plugin-loader' export { CuaAccess } from './access.ts' -export type * from './access-types.ts' /** Native executable and per-operation resource bounds. */ export interface CuaConfig { @@ -376,3 +375,6 @@ var cua = connectCua(() => tools.cua, text => console.log(text));`, } }, 'dsh-cua.runtime') } + +/** Host marker for the optional Client contribution; platform providers install native execution separately. */ +export function apply(): void {} diff --git a/packages/experimental/cua/src/remote-types.ts b/packages/experimental/cua/src/remote-types.ts new file mode 100644 index 0000000000..6c16dde03d --- /dev/null +++ b/packages/experimental/cua/src/remote-types.ts @@ -0,0 +1,3 @@ +/** Named Remote boundary types required by Typert's generated Client declarations. */ +export type { CuaApplicationId } from './application-types.ts' +export type { CuaApplicationState, CuaSystemPermission, CuaSystemPermissionState } from './access-types.ts' diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index 2ca68b96a3..1c0511f0ab 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -1,5 +1,4 @@ /** Native application objects independent of evaluator ownership and persistence. */ -export type * from './access-types.ts' import type { Branded } from '@deepseek-ai/dsh-brand' import type { JsonValue } from '@deepseek-ai/dsh-util-values' import type { z } from 'zod' diff --git a/packages/experimental/client-ui-cua/tests/controller.client.spec.ts b/packages/experimental/cua/tests/controller.client.spec.ts similarity index 99% rename from packages/experimental/client-ui-cua/tests/controller.client.spec.ts rename to packages/experimental/cua/tests/controller.client.spec.ts index e05f2d5882..82262df31f 100644 --- a/packages/experimental/client-ui-cua/tests/controller.client.spec.ts +++ b/packages/experimental/cua/tests/controller.client.spec.ts @@ -1,7 +1,7 @@ /** Permission refreshes preserve observations and join user actions before withdrawal. */ import { setImmediate } from 'node:timers/promises' import { expect, it, onTestFinished, vi } from 'vitest' -import type { CuaApplicationId, CuaApplicationState, CuaSystemPermissionState } from '@deepseek-ai/dsh-cua/types' +import type { CuaApplicationId, CuaApplicationState, CuaSystemPermissionState } from '../src/access-types.ts' import { PermissionController } from '../src/client/controller.ts' const value: CuaSystemPermissionState = { accessibility: true, screenRecording: false, helper: { pid: 9 }, parent: { pid: 8 } } diff --git a/packages/experimental/cua/tests/export-types.spec.ts b/packages/experimental/cua/tests/export-types.spec.ts new file mode 100644 index 0000000000..d71093f281 --- /dev/null +++ b/packages/experimental/cua/tests/export-types.spec.ts @@ -0,0 +1,18 @@ +/** CUA's public types distinguish object consumers from its generated permission transport. */ +import { expectTypeOf, it } from 'vitest' +import type { CuaApi } from '@deepseek-ai/dsh-cua/api' +import type { CuaApplicationState } from '@deepseek-ai/dsh-cua/types' +// @ts-expect-error Permission payloads are not part of the platform-provider root entry. +import type { CuaSystemPermissionState as RemovedRoot } from '@deepseek-ai/dsh-cua' +// @ts-expect-error The object API does not expose permission-page state. +import type { CuaSystemPermissionState as RemovedApi } from '@deepseek-ai/dsh-cua/api' +// @ts-expect-error Object interfaces use /api; /types is the generated Remote type entry. +import type { CuaApi as RemovedTypes } from '@deepseek-ai/dsh-cua/types' +// @ts-expect-error The nested grant structure is internal, not an independently exported type. +import type { CuaApplicationGrant as RemovedGrant } from '@deepseek-ai/dsh-cua/types' + +it('keeps application operations and the necessary Remote payload types available', () => { + expectTypeOf().toHaveProperty('getApp') + expectTypeOf().toHaveProperty('allowedApps') + expectTypeOf<[RemovedRoot, RemovedApi, RemovedTypes, RemovedGrant]>().toBeArray() +}) diff --git a/packages/experimental/cua/tests/exports.built.e2e.ts b/packages/experimental/cua/tests/exports.built.e2e.ts new file mode 100644 index 0000000000..9300b946d5 --- /dev/null +++ b/packages/experimental/cua/tests/exports.built.e2e.ts @@ -0,0 +1,18 @@ +/** Published CUA entries retain headless isolation and the existing generated transport. */ +import { execFile } from 'node:child_process' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' +import { resolveExampleLaunch } from '@deepseek-ai/dsh-loader-smoke' +import { expect, it, onTestFinished } from 'vitest' + +it('loads the built Host, object API and native exports without Client dependencies', async () => { + const fixture = fileURLToPath(new URL('./fixtures/exports.mjs', import.meta.url)) + const launch = resolveExampleLaunch({ srcBin: fixture, libBin: fixture, mode: 'lib' }) + const controller = new AbortController() + const child = promisify(execFile)(launch.command, launch.args, { signal: controller.signal }) + onTestFinished(async () => { + controller.abort() + await child.catch(() => { /* A failed or cancelled fixture is asserted by the test. */ }) + }) + expect((await child).stdout).toBe('headless exports and generated Remote passed\n') +}) diff --git a/packages/experimental/cua/tests/fixtures/exports.mjs b/packages/experimental/cua/tests/fixtures/exports.mjs new file mode 100644 index 0000000000..aa1625007c --- /dev/null +++ b/packages/experimental/cua/tests/fixtures/exports.mjs @@ -0,0 +1,28 @@ +/** Built package smoke: headless consumers must not import the browser implementation. */ +import assert from 'node:assert/strict' +import { registerHooks } from 'node:module' + +const guard = registerHooks({ + resolve(specifier, context, next) { + assert(!/^(?:react(?:-dom)?(?:\/|$)|@deepseek-ai\/dsh-client-)/.test(specifier), `Headless import reached ${specifier}`) + const result = next(specifier, context) + assert(!/\/cua\/lib\/(?:client(?:\/|\.)|types\/client\/)/.test(result.url), `Headless import reached ${result.url}`) + return result + }, +}) +const { Context } = await import('@deepseek-ai/cordis') +const host = await import('@deepseek-ai/dsh-cua') +const { createCuaApi } = await import('@deepseek-ai/dsh-cua/api') +const { connectCua } = await import('@deepseek-ai/dsh-cua/bridge') +const { NativeClient } = await import('@deepseek-ai/dsh-cua/native-client') +const { Request } = await import('@deepseek-ai/dsh-cua/protocol') +for (const callable of [host.apply, host.installCua, host.CuaAccess, createCuaApi, connectCua, NativeClient]) assert.equal(typeof callable, 'function') +assert.equal(Request.parse({ method: 'getCapabilities', params: {} }).method, 'getCapabilities') +const ctx = new Context() +try { await ctx.plugin(host); assert.equal(ctx.get('cuaAccess'), undefined) } +finally { await ctx.fiber.dispose() } +guard.deregister() +const remote = await import('@deepseek-ai/dsh-cua/remote') +assert.equal(remote.default.package, '@deepseek-ai/dsh-cua') +assert.equal(remote.default.descriptors.length, 9) +process.stdout.write('headless exports and generated Remote passed\n') diff --git a/packages/experimental/client-ui-cua/tests/mount.client.spec.ts b/packages/experimental/cua/tests/mount.client.spec.ts similarity index 96% rename from packages/experimental/client-ui-cua/tests/mount.client.spec.ts rename to packages/experimental/cua/tests/mount.client.spec.ts index 904a8453fc..dbf16c9a3b 100644 --- a/packages/experimental/client-ui-cua/tests/mount.client.spec.ts +++ b/packages/experimental/cua/tests/mount.client.spec.ts @@ -6,7 +6,7 @@ import { LocaleRuntime } from '@deepseek-ai/dsh-client-locale/client' import { SlotRegistry } from '@deepseek-ai/dsh-client-ui-renderer/client' import { createSnapshotStore } from '@deepseek-ai/dsh-client-store' import { RemoteError, type RemoteResult, type TypertRemoteContribution } from '@deepseek-ai/dsh-typert-protocol' -import type { CuaApplicationState, CuaSystemPermissionState } from '@deepseek-ai/dsh-cua/types' +import type { CuaApplicationState, CuaSystemPermissionState } from '../src/access-types.ts' import { expect, it, vi } from 'vitest' import type { PermissionFace } from '../src/client/controller.ts' import { mountCuaPermissions, inject } from '../src/client/mount.ts' @@ -86,17 +86,17 @@ it('withdraws the permission Remote and all controls with the optional plugin', await vi.waitFor(() => { expect(actions.hooks.cuaPermissions.getSnapshot().toast?.kind).toBe('requested') }) actions.openSettings('screenRecording') await vi.waitFor(() => { expect(actions.hooks.cuaPermissions.getSnapshot().toast?.kind).toBe('opened') }) - actions.revoke('a'.repeat(64) as import('@deepseek-ai/dsh-cua/types').CuaApplicationId) + actions.revoke('a'.repeat(64) as import('../src/access-types.ts').CuaApplicationId) await vi.waitFor(() => { expect(actions.hooks.cuaPermissions.getSnapshot().toast?.kind).toBe('revoked') }) - actions.retryApproval('a'.repeat(64) as import('@deepseek-ai/dsh-cua/types').CuaApplicationId) + actions.retryApproval('a'.repeat(64) as import('../src/access-types.ts').CuaApplicationId) await vi.waitFor(() => { expect(actions.hooks.cuaPermissions.getSnapshot().toast?.kind).toBe('retryAllowed') }) const failed = { ok: false as const, error: new RemoteError('gateway/internal', 'offline', {}) } for (const [target, invoke] of [ [f.runtimeConfiguration, actions.refresh], [f.status, actions.refresh], [f.systemStatus, actions.refresh], [f.applicationIcons, actions.refresh], [f.requestPermission, () => { actions.request('accessibility') }], [f.openSettings, () => { actions.openSettings('accessibility') }], - [f.revoke, () => { actions.revoke('a'.repeat(64) as import('@deepseek-ai/dsh-cua/types').CuaApplicationId) }], - [f.retryApproval, () => { actions.retryApproval('a'.repeat(64) as import('@deepseek-ai/dsh-cua/types').CuaApplicationId) }], + [f.revoke, () => { actions.revoke('a'.repeat(64) as import('../src/access-types.ts').CuaApplicationId) }], + [f.retryApproval, () => { actions.retryApproval('a'.repeat(64) as import('../src/access-types.ts').CuaApplicationId) }], ] as const) { actions.refresh() await vi.waitFor(() => { expect(actions.hooks.cuaPermissions.getSnapshot().busy).toBe(false) }) @@ -156,7 +156,7 @@ it.each(['plugin', 'root'])('keeps the Remote mounted until an accepted revoke c const fiber = await f.ctx.plugin({ inject: [...inject], apply: ctx => mountCuaPermissions(ctx, contribution) }) const actions = f.ctx.slots.entries('plugins.bundle.config')[0]!.inject!() actionsOf(actions) - actions.revoke('a'.repeat(64) as import('@deepseek-ai/dsh-cua/types').CuaApplicationId) + actions.revoke('a'.repeat(64) as import('../src/access-types.ts').CuaApplicationId) await entered.promise let closed = false closing = Promise.resolve(owner === 'plugin' ? fiber.dispose() : f.ctx.fiber.dispose()).then(() => { closed = true }) diff --git a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx b/packages/experimental/cua/tests/panel.client.spec.tsx similarity index 99% rename from packages/experimental/client-ui-cua/tests/panel.client.spec.tsx rename to packages/experimental/cua/tests/panel.client.spec.tsx index 0d47ad25b6..d0117edee6 100644 --- a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx +++ b/packages/experimental/cua/tests/panel.client.spec.tsx @@ -3,7 +3,7 @@ import { act, cleanup, fireEvent, render, screen, waitFor, within } from '@testing-library/react' import { bindSnapshotSelector, makeTranslate } from '@deepseek-ai/dsh-client-test-runtime' import { createSnapshotStore } from '@deepseek-ai/dsh-client-store' -import type { CuaApplicationId, CuaApplicationState, CuaSystemPermissionState } from '@deepseek-ai/dsh-cua/types' +import type { CuaApplicationId, CuaApplicationState, CuaSystemPermissionState } from '../src/access-types.ts' import type { ConfigFormSnapshot } from '@deepseek-ai/dsh-client-ui-settings/client' import { afterEach, expect, it, vi } from 'vitest' import { PermissionPanel, PermissionSetup, PermissionToast, type PermissionPanelProps } from '../src/client/PermissionPanel.tsx' diff --git a/packages/experimental/client-ui-cua/tsconfig.json b/packages/experimental/cua/tsconfig.client.json similarity index 78% rename from packages/experimental/client-ui-cua/tsconfig.json rename to packages/experimental/cua/tsconfig.client.json index 66cf034b76..03db4207e8 100644 --- a/packages/experimental/client-ui-cua/tsconfig.json +++ b/packages/experimental/cua/tsconfig.client.json @@ -2,10 +2,15 @@ "extends": "../../../tsconfig.base.client.json", "compilerOptions": { "rootDir": "src", - "outDir": "lib/types" + "outDir": "lib/types", + "tsBuildInfoFile": "lib/tsconfig.client.tsbuildinfo" }, "include": [ - "src" + "src/client", + "src/css-modules.d.ts", + "src/access-types.ts", + "src/application-types.ts", + "src/remote-types.ts" ], "references": [ { @@ -42,10 +47,10 @@ "path": "../../api/gateway/tsconfig.client.json" }, { - "path": "../cua" + "path": "../../typert/protocol" }, { - "path": "../../typert/protocol" + "path": "../../util/brand" } ] } diff --git a/packages/experimental/cua/tsconfig.host.json b/packages/experimental/cua/tsconfig.host.json new file mode 100644 index 0000000000..ebeaed4918 --- /dev/null +++ b/packages/experimental/cua/tsconfig.host.json @@ -0,0 +1,73 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib/types", + "tsBuildInfoFile": "lib/tsconfig.host.tsbuildinfo" + }, + "include": [ + "src/*.ts" + ], + "references": [ + { + "path": "../../../vendor/cordis" + }, + { + "path": "../../../vendor/schemastery" + }, + { + "path": "../../computer-use/computer-use" + }, + { + "path": "../../core/tools" + }, + { + "path": "../../core/system-prompt" + }, + { + "path": "../../core/agent" + }, + { + "path": "../../subprocess/subprocess" + }, + { + "path": "../../util/values" + }, + { + "path": "../../util/brand" + }, + { + "path": "../../util/atomic-write" + }, + { + "path": "../../ptc-runtime/ptc-runtime" + }, + { + "path": "../../typert/protocol" + }, + { + "path": "../../settings/settings" + }, + { + "path": "../../interaction/user-questions" + }, + { + "path": "../../../vendor/loader" + }, + { + "path": "../../attachment/attachment" + }, + { + "path": "../../llm/llm" + }, + { + "path": "../../core/scope" + }, + { + "path": "../../preset/agent-preset-registry" + } + ], + "exclude": [ + "src/css-modules.d.ts" + ] +} diff --git a/packages/experimental/cua/tsconfig.json b/packages/experimental/cua/tsconfig.json index cb086d3e6f..2eca820546 100644 --- a/packages/experimental/cua/tsconfig.json +++ b/packages/experimental/cua/tsconfig.json @@ -1,69 +1,11 @@ { - "extends": "../../../tsconfig.base.json", - "compilerOptions": { - "rootDir": "src", - "outDir": "lib/types" - }, - "include": [ - "src" - ], + "files": [], "references": [ { - "path": "../../../vendor/cordis" + "path": "./tsconfig.host.json" }, { - "path": "../../../vendor/schemastery" - }, - { - "path": "../../computer-use/computer-use" - }, - { - "path": "../../core/tools" - }, - { - "path": "../../core/system-prompt" - }, - { - "path": "../../core/agent" - }, - { - "path": "../../subprocess/subprocess" - }, - { - "path": "../../util/values" - }, - { - "path": "../../util/brand" - }, - { - "path": "../../util/atomic-write" - }, - { - "path": "../../ptc-runtime/ptc-runtime" - }, - { - "path": "../../typert/protocol" - }, - { - "path": "../../settings/settings" - }, - { - "path": "../../interaction/user-questions" - }, - { - "path": "../../../vendor/loader" - }, - { - "path": "../../attachment/attachment" - }, - { - "path": "../../llm/llm" - }, - { - "path": "../../core/scope" - }, - { - "path": "../../preset/agent-preset-registry" + "path": "./tsconfig.client.json" } ] } diff --git a/packages/experimental/cua/tsdown.config.ts b/packages/experimental/cua/tsdown.config.ts index f0d407249f..5dac622abb 100644 --- a/packages/experimental/cua/tsdown.config.ts +++ b/packages/experimental/cua/tsdown.config.ts @@ -1,10 +1,13 @@ -import { defineConfig } from 'tsdown' +import type { UserConfig } from 'tsdown' +import { clientBundle } from '../../client/tsdown.client.ts' -export default defineConfig(Object.entries({ - index: 'lib/types/index.js', types: 'lib/types/types.js', api: 'lib/types/api.js', bridge: 'lib/types/bridge.js', +const runtime: UserConfig[] = Object.entries({ + 'remote-types': 'lib/types/remote-types.js', api: 'lib/types/api.js', bridge: 'lib/types/bridge.js', 'native-client': 'lib/types/native-client.js', protocol: 'lib/types/protocol.js', }).map(([name, path]) => ({ entry: { [name]: path }, outDir: 'lib', format: ['esm'], platform: 'node', target: 'es2024', fixedExtension: false, dts: false, clean: false, -}))) +})) + +export default clientBundle(import.meta.dirname, ['lib/types/index.js'], { hostPhase: true, companions: runtime }) diff --git a/packages/extensions/cordis-client-runner/src/client/slot-catalog.ts b/packages/extensions/cordis-client-runner/src/client/slot-catalog.ts index ba5123c9b4..98ab21a6b9 100644 --- a/packages/extensions/cordis-client-runner/src/client/slot-catalog.ts +++ b/packages/extensions/cordis-client-runner/src/client/slot-catalog.ts @@ -2286,8 +2286,8 @@ export const CLIENT_SLOT_API: readonly ClientSlotEntry[] = [ slotInject: '', declaredBy: 'an entry in \'main\' (client-ui-plugin-manager), so it exists while that entry is mounted', occupants: [ - 'client-ui-cua PermissionSetup key \'@deepseek-ai/dsh-cua-bundle\'', 'experimental-client-ui-voice-input VoiceSetupPrompt key \'@deepseek-ai/dsh-experimental-voice-input-bundle\'', + 'cua PermissionSetup key \'@deepseek-ai/dsh-cua-bundle\'', ], replaceRisk: 'shadows-shipped-ui', example: 'return {\n inject: [\'slots\'],\n apply(ctx) {\n ctx.slots.inject(\'plugins.bundle.activation\', () => ctx.slots.register(\n { name: \'plugins.bundle.activation\', key: \'\' },\n () => React.createElement(\'div\', null, \'hello\'),\n ))\n },\n}', @@ -2330,8 +2330,8 @@ export const CLIENT_SLOT_API: readonly ClientSlotEntry[] = [ declaredBy: 'an entry in \'main\' (client-ui-plugin-manager), so it exists while that entry is mounted', occupants: [ 'experimental-client-ui-cot-translation TranslationSettings key \'@deepseek-ai/dsh-experimental-cot-translation-bundle\'', - 'client-ui-cua PermissionPanel key \'@deepseek-ai/dsh-cua-bundle\'', 'experimental-client-ui-voice-input VoicePreparation key \'@deepseek-ai/dsh-experimental-voice-input-bundle\'', + 'cua PermissionPanel key \'@deepseek-ai/dsh-cua-bundle\'', ], replaceRisk: 'shadows-shipped-ui', example: 'return {\n inject: [\'slots\'],\n apply(ctx) {\n ctx.slots.inject(\'plugins.bundle.config\', () => ctx.slots.register(\n { name: \'plugins.bundle.config\', key: \'\' },\n () => React.createElement(\'div\', null, \'hello\'),\n ))\n },\n}', @@ -3401,7 +3401,7 @@ export const CLIENT_SLOT_API: readonly ClientSlotEntry[] = [ 'client-ui-workspace SessionRenameDialog id \'workspace.session-rename\'', 'client-ui-workspace SessionArchiveConfirmDialog id \'workspace.session-archive\'', 'client-ui-workspace RowActionToast id \'workspace.row-toast\'', - 'client-ui-cua PermissionToast id \'cua-permission-outcome\'', + 'cua PermissionToast id \'cua-permission-outcome\'', ], replaceRisk: 'none', example: 'return {\n inject: [\'slots\'],\n apply(ctx) {\n ctx.slots.inject(\'shell.overlay\', () => ctx.slots.register(\n { name: \'shell.overlay\', id: \'my-entry\', order: 100, label: \'My entry\' },\n () => React.createElement(\'div\', null, \'hello\'),\n ))\n },\n}', diff --git a/packages/preset/agent-preset/skills/cordis-composition-reference/references/packages.md b/packages/preset/agent-preset/skills/cordis-composition-reference/references/packages.md index 51e9d4b72d..cd38b17726 100644 --- a/packages/preset/agent-preset/skills/cordis-composition-reference/references/packages.md +++ b/packages/preset/agent-preset/skills/cordis-composition-reference/references/packages.md @@ -184,7 +184,6 @@ Every package below exports a Cordis plugin that a bundle patch can name in a Lo | Package | Config | Description | |---|---|---| -| `@deepseek-ai/dsh-client-ui-cua` | no | Computer Use permission setup and application access controls | | `@deepseek-ai/dsh-experimental-agent-team` | yes | Implicit-root Agent Teams roster, direct peer messaging, and shared task DAG | | `@deepseek-ai/dsh-experimental-api-speech-to-text` | yes | Authenticated experimental speech transcription for browser clients | | `@deepseek-ai/dsh-experimental-auto-review` | no | Per-tool LLM authorization review for the DeepSeek Harness Auto permission preset | diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5c0a6f3c23..95dc195b0e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8301,60 +8301,6 @@ importers: specifier: ^18.2.0 version: 18.3.1(react@18.3.1) - packages/experimental/client-ui-cua: - devDependencies: - '@deepseek-ai/cordis': - specifier: workspace:~ - version: link:../../../vendor/cordis - '@deepseek-ai/dsh-api-gateway': - specifier: workspace:* - version: link:../../api/gateway - '@deepseek-ai/dsh-api-remotes': - specifier: workspace:* - version: link:../../api/remotes - '@deepseek-ai/dsh-client-locale': - specifier: workspace:* - version: link:../../client/locale - '@deepseek-ai/dsh-client-store': - specifier: workspace:* - version: link:../../client/store - '@deepseek-ai/dsh-client-test-runtime': - specifier: workspace:* - version: link:../../test-support/client-runtime - '@deepseek-ai/dsh-client-ui-layout': - specifier: workspace:* - version: link:../../client/ui-layout - '@deepseek-ai/dsh-client-ui-plugin-manager': - specifier: workspace:* - version: link:../../client/ui-plugin-manager - '@deepseek-ai/dsh-client-ui-primitives': - specifier: workspace:* - version: link:../../client/ui-primitives - '@deepseek-ai/dsh-client-ui-renderer': - specifier: workspace:* - version: link:../../client/ui-renderer - '@deepseek-ai/dsh-client-ui-settings': - specifier: workspace:* - version: link:../../client/ui-settings - '@deepseek-ai/dsh-client-ui-slots': - specifier: workspace:* - version: link:../../client/ui-slots - '@deepseek-ai/dsh-cua': - specifier: workspace:* - version: link:../cua - '@deepseek-ai/dsh-typert-protocol': - specifier: workspace:* - version: link:../../typert/protocol - '@testing-library/react': - specifier: ^16.1.0 - version: 16.3.3(@testing-library/dom@10.4.2)(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) - react: - specifier: ^18.2.0 - version: 18.3.1 - react-dom: - specifier: ^18.2.0 - version: 18.3.1(react@18.3.1) - packages/experimental/client-ui-voice-input: devDependencies: '@deepseek-ai/cordis': @@ -8586,6 +8532,12 @@ importers: '@deepseek-ai/dsh-agent-preset-registry': specifier: workspace:* version: link:../../preset/agent-preset-registry + '@deepseek-ai/dsh-api-gateway': + specifier: workspace:* + version: link:../../api/gateway + '@deepseek-ai/dsh-api-remotes': + specifier: workspace:* + version: link:../../api/remotes '@deepseek-ai/dsh-app-boot': specifier: workspace:* version: link:../../boot/app-boot @@ -8595,6 +8547,33 @@ importers: '@deepseek-ai/dsh-attachment-local': specifier: workspace:* version: link:../../attachment/attachment-local + '@deepseek-ai/dsh-client-locale': + specifier: workspace:* + version: link:../../client/locale + '@deepseek-ai/dsh-client-store': + specifier: workspace:* + version: link:../../client/store + '@deepseek-ai/dsh-client-test-runtime': + specifier: workspace:* + version: link:../../test-support/client-runtime + '@deepseek-ai/dsh-client-ui-layout': + specifier: workspace:* + version: link:../../client/ui-layout + '@deepseek-ai/dsh-client-ui-plugin-manager': + specifier: workspace:* + version: link:../../client/ui-plugin-manager + '@deepseek-ai/dsh-client-ui-primitives': + specifier: workspace:* + version: link:../../client/ui-primitives + '@deepseek-ai/dsh-client-ui-renderer': + specifier: workspace:* + version: link:../../client/ui-renderer + '@deepseek-ai/dsh-client-ui-settings': + specifier: workspace:* + version: link:../../client/ui-settings + '@deepseek-ai/dsh-client-ui-slots': + specifier: workspace:* + version: link:../../client/ui-slots '@deepseek-ai/dsh-computer-use': specifier: workspace:* version: link:../../computer-use/computer-use @@ -8613,6 +8592,9 @@ importers: '@deepseek-ai/dsh-llm-deepseek': specifier: workspace:* version: link:../../llm/llm-deepseek + '@deepseek-ai/dsh-loader-smoke': + specifier: workspace:* + version: link:../../test-support/loader-smoke '@deepseek-ai/dsh-ptc-runtime': specifier: workspace:* version: link:../../ptc-runtime/ptc-runtime @@ -8655,18 +8637,27 @@ importers: '@deepseek-ai/dsh-working-directory': specifier: workspace:* version: link:../../session/working-directory + '@testing-library/react': + specifier: ^16.1.0 + version: 16.3.3(@testing-library/dom@10.4.2)(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + react: + specifier: ^18.2.0 + version: 18.3.1 + react-dom: + specifier: ^18.2.0 + version: 18.3.1(react@18.3.1) sharp: specifier: ^0.35.3 version: 0.35.5(@types/node@22.20.1) packages/experimental/cua-bundle: dependencies: - '@deepseek-ai/dsh-client-ui-cua': - specifier: workspace:* - version: link:../client-ui-cua '@deepseek-ai/dsh-computer-use': specifier: workspace:* version: link:../../computer-use/computer-use + '@deepseek-ai/dsh-cua': + specifier: workspace:* + version: link:../cua '@deepseek-ai/dsh-experimental-cua-macos': specifier: workspace:* version: link:../cua-macos diff --git a/scripts/check-workspace-constraints.ts b/scripts/check-workspace-constraints.ts index f346e4e9f6..108a345794 100644 --- a/scripts/check-workspace-constraints.ts +++ b/scripts/check-workspace-constraints.ts @@ -181,7 +181,7 @@ const packageFileExtras: Readonly> = { // The embedded OAR implementation retains its upstream license. '@deepseek-ai/dsh-experimental-subagent-oar': ['LICENSE-OAR'], // CUA publishes its client API and attributed native source for local macOS builds. - '@deepseek-ai/dsh-cua': ['lib/types.js', 'lib/api.js', 'lib/bridge.js', 'lib/native-client.js', 'lib/protocol.js'], + '@deepseek-ai/dsh-cua': ['lib/remote-types.js', 'lib/api.js', 'lib/bridge.js', 'lib/native-client.js', 'lib/protocol.js'], '@deepseek-ai/dsh-experimental-cua-macos': ['native/bin', 'native/Package.swift', 'native/Sources', 'native/tests/swift', 'native/tests/test_swift.py', 'native/LICENSE.cua-driver', 'native/upstream.json', 'native/README.md', 'native/README.zh.md', 'native/protocol.schema.json'], '@deepseek-ai/dsh-webhook-github': ['examples/github-review/cordis.yml', 'examples/github-review/github-ready-review-rule.mjs'], // Owned Worker bundles import this public bootstrap before their business entry. diff --git a/scripts/experimental-package-policy.ts b/scripts/experimental-package-policy.ts index 5763dbdedc..44fbd52ebe 100644 --- a/scripts/experimental-package-policy.ts +++ b/scripts/experimental-package-policy.ts @@ -4,7 +4,6 @@ export const EXPERIMENTAL_PACKAGE_NAME_EXCEPTIONS: Readonly> = { 'packages/experimental/cua': '@deepseek-ai/dsh-cua', 'packages/experimental/cua-bundle': '@deepseek-ai/dsh-cua-bundle', - 'packages/experimental/client-ui-cua': '@deepseek-ai/dsh-client-ui-cua', 'packages/experimental/tool-session-query': '@deepseek-ai/dsh-tool-session-query', 'packages/experimental/session-title-all-prompts-llm': '@deepseek-ai/dsh-session-title-all-prompts-llm', 'packages/experimental/tool-terminal': '@deepseek-ai/dsh-tool-terminal', diff --git a/scripts/gen-doc-graphs.ts b/scripts/gen-doc-graphs.ts index 9123b5211e..94d7ff7c34 100644 --- a/scripts/gen-doc-graphs.ts +++ b/scripts/gen-doc-graphs.ts @@ -225,7 +225,7 @@ const SERVICE_ROLES: ServiceRole[] = [ }, { key: 'cuaAccess', pkg: 'cua', title: 'Computer Use permission controls', - mode: 'core', consumers: ['client-ui-cua'], + mode: 'core', consumers: ['cua'], note: 'The native provider owns application consent and cancellation; the optional Client page queries and changes those decisions through its Remote.', }, { diff --git a/scripts/release/families.spec.ts b/scripts/release/families.spec.ts index c60e4aef1a..5ead068208 100644 --- a/scripts/release/families.spec.ts +++ b/scripts/release/families.spec.ts @@ -62,7 +62,6 @@ describe('release families', () => { '@deepseek-ai/dsh-experimental-client-ui-agent-team', '@deepseek-ai/dsh-experimental-client-ui-claude-code-mods', '@deepseek-ai/dsh-experimental-client-ui-cot-translation', - '@deepseek-ai/dsh-client-ui-cua', '@deepseek-ai/dsh-experimental-client-ui-voice-input', '@deepseek-ai/dsh-experimental-computer-use-cua-driver-mcp', '@deepseek-ai/dsh-experimental-computer-use-cua-driver-native', diff --git a/scripts/verify-package-readme-model-experience.ts b/scripts/verify-package-readme-model-experience.ts index bed5aa89d8..2c313a3dea 100644 --- a/scripts/verify-package-readme-model-experience.ts +++ b/scripts/verify-package-readme-model-experience.ts @@ -144,7 +144,6 @@ const SENTENCE_MODEL_EXPERIENCE: Readonly> = { 'packages/client/ui-permission-presets': { kind: 'indirect', reason: 'The picker submits the host /permission command; the knob events it appends own the model-visible effect through the sandbox/approval consumers.' }, 'packages/client/ui-settings-plugins': { kind: 'none', reason: 'Browser-side settings surface; registers no model surface.' }, 'packages/client/ui-settings-shell': { kind: 'none', reason: 'Browser-side settings surface; registers no model surface.' }, - 'packages/experimental/client-ui-cua': { kind: 'none', reason: 'Browser permission controls; the native provider owns tools and guidance.' }, 'packages/client/ui-settings-session-log': { kind: 'none', reason: 'Browser preference controls request metadata, not model input.' }, 'packages/client/ui-settings-agent-loop': { kind: 'none', reason: 'Browser-side settings surface; registers no model surface.' }, 'packages/client/ui-settings-subagent': { kind: 'none', reason: 'Browser-side settings surface; registers no model surface.' }, diff --git a/tsconfig.base.json b/tsconfig.base.json index ab576e768b..95528a8c0b 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -302,9 +302,8 @@ "@deepseek-ai/dsh-experimental-client-ui-agent-team/client": ["./packages/experimental/client-ui-agent-team/src/client/index.ts"], "@deepseek-ai/dsh-client-ui-settings-plugins": ["./packages/client/ui-settings-plugins/src"], "@deepseek-ai/dsh-client-ui-settings-shell": ["./packages/client/ui-settings-shell/src"], - "@deepseek-ai/dsh-client-ui-cua": ["./packages/experimental/client-ui-cua/src/index.ts"], - "@deepseek-ai/dsh-client-ui-cua/client": ["./packages/experimental/client-ui-cua/src/client/index.ts"], - "@deepseek-ai/dsh-cua/types": ["./packages/experimental/cua/src/types.ts"], + "@deepseek-ai/dsh-cua/client": ["./packages/experimental/cua/src/client/index.ts"], + "@deepseek-ai/dsh-cua/types": ["./packages/experimental/cua/src/remote-types.ts"], "@deepseek-ai/dsh-client-ui-settings-agent-loop": ["./packages/client/ui-settings-agent-loop/src"], "@deepseek-ai/dsh-client-ui-settings-session-log": ["./packages/client/ui-settings-session-log/src"], "@deepseek-ai/dsh-client-ui-settings-subagent": ["./packages/client/ui-settings-subagent/src"], diff --git a/tsconfig.client.json b/tsconfig.client.json index 7bf256becb..c335a29e83 100644 --- a/tsconfig.client.json +++ b/tsconfig.client.json @@ -131,7 +131,7 @@ { "path": "./packages/client/ui-settings-plugins" }, { "path": "./packages/client/ui-settings-shell" }, { "path": "./packages/client/ui-settings-agent-loop" }, - { "path": "./packages/experimental/client-ui-cua" }, + { "path": "./packages/experimental/cua/tsconfig.client.json" }, { "path": "./packages/client/ui-settings-session-log" }, { "path": "./packages/client/ui-settings-subagent" }, { "path": "./packages/client/ui-settings-web-search" }, diff --git a/tsconfig.host.json b/tsconfig.host.json index a267f50f63..1ae120dd29 100644 --- a/tsconfig.host.json +++ b/tsconfig.host.json @@ -276,7 +276,7 @@ { "path": "./packages/computer-use/computer-use" }, { "path": "./packages/experimental/computer-use-cua-driver-mcp" }, { "path": "./packages/experimental/computer-use-cua-driver-native" }, - { "path": "./packages/experimental/cua" }, + { "path": "./packages/experimental/cua/tsconfig.host.json" }, { "path": "./packages/experimental/cua-bundle" }, { "path": "./packages/experimental/cua-macos" }, { "path": "./packages/client/ui-sidebar-documentpreview/tsconfig.host.json" }, diff --git a/vitest.config.ts b/vitest.config.ts index 7857266cd1..56d206808f 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -333,7 +333,7 @@ export default defineConfig({ 'packages/experimental/client-ui-voice-input/src/client/index.ts', // The permission entry binds generated Remote code; cua-permissions.e2e.ts covers // its built entry, while source tests exercise mountCuaPermissions and its UI. - 'packages/experimental/client-ui-cua/src/client/index.ts', + 'packages/experimental/cua/src/client/index.ts', // The mods band entry imports the bridge's generated Remote contribution, which exists only in lib; // the Web snapshot exercises the built entry, while source tests cover mountModsBand. 'packages/experimental/client-ui-claude-code-mods/src/client/index.ts', From 4da281c93393c51d9e64f670ada15be3ecb91339 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 19:47:55 +0800 Subject: [PATCH 38/55] docs(cua): classify the combined package as a Host and Client plugin --- .../skills/cordis-composition-reference/references/packages.md | 1 + scripts/doc-standard.spec.ts | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/preset/agent-preset/skills/cordis-composition-reference/references/packages.md b/packages/preset/agent-preset/skills/cordis-composition-reference/references/packages.md index cd38b17726..f6690231e2 100644 --- a/packages/preset/agent-preset/skills/cordis-composition-reference/references/packages.md +++ b/packages/preset/agent-preset/skills/cordis-composition-reference/references/packages.md @@ -184,6 +184,7 @@ Every package below exports a Cordis plugin that a bundle patch can name in a Lo | Package | Config | Description | |---|---|---| +| `@deepseek-ai/dsh-cua` | no | Shared Computer Use application objects, authorization, and logged tool execution | | `@deepseek-ai/dsh-experimental-agent-team` | yes | Implicit-root Agent Teams roster, direct peer messaging, and shared task DAG | | `@deepseek-ai/dsh-experimental-api-speech-to-text` | yes | Authenticated experimental speech transcription for browser clients | | `@deepseek-ai/dsh-experimental-auto-review` | no | Per-tool LLM authorization review for the DeepSeek Harness Auto permission preset | diff --git a/scripts/doc-standard.spec.ts b/scripts/doc-standard.spec.ts index 215048297e..c7118c778e 100644 --- a/scripts/doc-standard.spec.ts +++ b/scripts/doc-standard.spec.ts @@ -55,7 +55,6 @@ const KIND_TEMPLATES: Readonly> = { * library; the check re-derives the entry shape so a stale entry fails loud. */ const PACKAGE_LIBRARIES: Readonly> = { - 'packages/experimental/cua': 'Shared CUA objects and installer consumed by platform providers; no plugin entry.', 'packages/experimental/browser-use-runtime': 'Provider-owned browser resource management and MCP integration helpers; no plugin entry.', 'packages/boot/app-boot': 'Boot library the app bins import; plain helper exports.', 'packages/boot/cmdline': 'Command-line library the app bins import; plain module exports.', From 57af330bf5ba7ce7fa16ee6568d1aa28ad54f18d Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 19:48:35 +0800 Subject: [PATCH 39/55] test(cua): keep Host marker checks out of the Client compiler --- .../experimental/cua/tests/export-types.spec.ts | 13 ++++++++++++- .../experimental/cua/tests/mount.client.spec.ts | 2 -- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/experimental/cua/tests/export-types.spec.ts b/packages/experimental/cua/tests/export-types.spec.ts index d71093f281..399fd84355 100644 --- a/packages/experimental/cua/tests/export-types.spec.ts +++ b/packages/experimental/cua/tests/export-types.spec.ts @@ -1,5 +1,7 @@ /** CUA's public types distinguish object consumers from its generated permission transport. */ -import { expectTypeOf, it } from 'vitest' +import { expect, expectTypeOf, it } from 'vitest' +import { Context } from '@deepseek-ai/cordis' +import * as Cua from '../src/index.ts' import type { CuaApi } from '@deepseek-ai/dsh-cua/api' import type { CuaApplicationState } from '@deepseek-ai/dsh-cua/types' // @ts-expect-error Permission payloads are not part of the platform-provider root entry. @@ -16,3 +18,12 @@ it('keeps application operations and the necessary Remote payload types availabl expectTypeOf().toHaveProperty('allowedApps') expectTypeOf<[RemovedRoot, RemovedApi, RemovedTypes, RemovedGrant]>().toBeArray() }) + +it('mounts the Host marker without installing native or browser services', async () => { + const ctx = new Context() + try { + await ctx.plugin(Cua) + expect(ctx.get('cuaAccess')).toBeUndefined() + expect(ctx.get('computerUse')).toBeUndefined() + } finally { await ctx.fiber.dispose() } +}) diff --git a/packages/experimental/cua/tests/mount.client.spec.ts b/packages/experimental/cua/tests/mount.client.spec.ts index dbf16c9a3b..02145516f6 100644 --- a/packages/experimental/cua/tests/mount.client.spec.ts +++ b/packages/experimental/cua/tests/mount.client.spec.ts @@ -10,7 +10,6 @@ import type { CuaApplicationState, CuaSystemPermissionState } from '../src/acces import { expect, it, vi } from 'vitest' import type { PermissionFace } from '../src/client/controller.ts' import { mountCuaPermissions, inject } from '../src/client/mount.ts' -import { apply as hostApply } from '../src/index.ts' const contribution: TypertRemoteContribution = { package: '@deepseek-ai/dsh-cua', descriptors: [] } const value: CuaApplicationState & { system: CuaSystemPermissionState } = { @@ -72,7 +71,6 @@ async function fixture(fail = false) { } it('withdraws the permission Remote and all controls with the optional plugin', async () => { - hostApply() const f = await fixture() try { const fiber = f.ctx.plugin({ inject: [...inject], apply: ctx => mountCuaPermissions(ctx, contribution) }) From 630a693b70ed8e4fe6e9b119230f9bd5bc6ec5cd Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 19:52:35 +0800 Subject: [PATCH 40/55] fix(cua): align combined package publication inventory --- packages/experimental/README.i18n.yaml | 4 ++-- packages/experimental/README.md | 2 +- packages/experimental/README.zh.md | 2 +- packages/experimental/cua/package.json | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/experimental/README.i18n.yaml b/packages/experimental/README.i18n.yaml index ce1021f7ee..2f9ac3db9b 100644 --- a/packages/experimental/README.i18n.yaml +++ b/packages/experimental/README.i18n.yaml @@ -18,8 +18,8 @@ en: b55aeb927737b90d zh: 0b215fcb5c2c7908 /packages-experimental/packages: - en: 94cc18a191e52f26 - zh: 6a5ce630fa730239 + en: b16e94185c9b8b24 + zh: 784faa79e027cd14 /packages-experimental/related-documentation: en: 1cb9d3448d44e48d zh: cd326d70a746b896 diff --git a/packages/experimental/README.md b/packages/experimental/README.md index 0a01b71eeb..ba4fabaddf 100644 --- a/packages/experimental/README.md +++ b/packages/experimental/README.md @@ -69,7 +69,7 @@ Promotion requires a named product role, documented public behavior and limitati | [`claude-code-mods`](claude-code-mods/README.md) | Run Claude Code mods as plugins: their hook chains on harness extension points and a band above the prompt | `ctx.claudeCodeMods` | | [`client-ui-claude-code-mods`](client-ui-claude-code-mods/README.md) | The Web band that draws a mod's tree above the prompt and sends button clicks back | — | | [`ptc-runtime-python`](ptc-runtime-python/README.md) | CPython subprocess backend for the PTC execution seam | `ctx.ptcRuntime` | -| [CUA](cua/README.md) | Shared application/window objects, consent, and logged execution. | Library | +| [CUA](cua/README.md) | Application/window objects, authorized execution, and permission controls. | `ctx.cuaAccess` | | [macOS CUA](cua-macos/README.md) | Swift native execution and platform preparation. | `ctx.computerUse` | | [`cua-bundle`](cua-bundle/README.md) | Optional Computer Use switch for PTC sessions | — | | [`computer-use-cua-driver-mcp`](computer-use-cua-driver-mcp/README.md) | Use an installed Cua Driver through MCP | `ctx.computerUse` | diff --git a/packages/experimental/README.zh.md b/packages/experimental/README.zh.md index 6d20549870..eb99c614fb 100644 --- a/packages/experimental/README.zh.md +++ b/packages/experimental/README.zh.md @@ -69,7 +69,7 @@ kind: "package-group" | [`claude-code-mods`](claude-code-mods/README.zh.md) | 把 Claude Code 模组作为插件运行：钩子链落在 harness 扩展点上，并在提示框上方绘制横幅 | `ctx.claudeCodeMods` | | [`client-ui-claude-code-mods`](client-ui-claude-code-mods/README.zh.md) | 在提示框上方绘制模组树并把按钮点击发回的 Web 横幅 | — | | [`ptc-runtime-python`](ptc-runtime-python/README.zh.md) | PTC 执行 seam 的 CPython 子进程后端 | `ctx.ptcRuntime` | -| [CUA](cua/README.zh.md) | 共享应用与窗口对象、授权和已记录的执行。 | 库 | +| [CUA](cua/README.zh.md) | 应用与窗口对象、授权执行和权限控件。 | `ctx.cuaAccess` | | [macOS CUA](cua-macos/README.zh.md) | Swift 原生执行与平台组件准备。 | `ctx.computerUse` | | [`cua-bundle`](cua-bundle/README.zh.md) | 面向 PTC 会话、默认关闭的 Computer Use 开关 | — | | [`computer-use-cua-driver-mcp`](computer-use-cua-driver-mcp/README.zh.md) | 通过 MCP 使用已安装的 Cua Driver | `ctx.computerUse` | diff --git a/packages/experimental/cua/package.json b/packages/experimental/cua/package.json index a5c7ff5142..1b91f37483 100644 --- a/packages/experimental/cua/package.json +++ b/packages/experimental/cua/package.json @@ -55,6 +55,8 @@ }, "files": [ "lib/index.js", + "lib/client.js", + "lib/remote-types.js", "lib/api.js", "lib/bridge.js", "lib/native-client.js", @@ -63,9 +65,7 @@ "lib/typert.host.js", "lib/typert.host.d.ts", "lib/typert.remote-client.js", - "lib/typert.remote-client.d.ts", - "lib/remote-types.js", - "lib/client.js" + "lib/typert.remote-client.d.ts" ], "license": "MIT", "peerDependencies": { From e49ef798849b5e97330e1f01775f2dc210f96c35 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 20:06:23 +0800 Subject: [PATCH 41/55] build(cua): classify approved Host imports for the combined package --- docs/module-graph.md | 15 ++++----------- docs/module-graph.zh.md | 15 ++++----------- packages/experimental/cua/package.json | 20 ++++++-------------- pnpm-lock.yaml | 16 ++++++++-------- scripts/package-dependency-policy.ts | 5 +++++ 5 files changed, 27 insertions(+), 44 deletions(-) diff --git a/docs/module-graph.md b/docs/module-graph.md index ab621eb709..e1fa9de809 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -759,6 +759,9 @@ flowchart TD pkg_deepseek_account_platform --> pkg_credentials pkg_deepseek_account_platform --> pkg_deepseek_account pkg_deepseek_account_platform --> pkg_host_webserver + pkg_cua --> pkg_attachment + pkg_cua --> pkg_scope + pkg_cua --> pkg_user_questions pkg_experimental_ptc_runtime_python --> pkg_ptc_runtime pkg_experimental_ptc_runtime_python --> pkg_sandbox pkg_experimental_ptc_runtime_python --> pkg_sandbox_policy @@ -1069,16 +1072,6 @@ flowchart TD pkg_api_settings_controller --> pkg_settings pkg_api_settings_controller --> pkg_typert_protocol pkg_web_app --> pkg_shell_env - pkg_cua --> pkg_agent - pkg_cua --> pkg_agent_preset_registry - pkg_cua --> pkg_attachment - pkg_cua --> pkg_computer_use - pkg_cua --> pkg_llm - pkg_cua --> pkg_scope - pkg_cua --> pkg_subprocess - pkg_cua --> pkg_system_prompt - pkg_cua --> pkg_tools - pkg_cua --> pkg_user_questions pkg_experimental_auto_review --> pkg_agent pkg_experimental_auto_review --> pkg_agent_instructions pkg_experimental_auto_review --> pkg_llm @@ -1681,6 +1674,7 @@ flowchart TD | [`api-account-controller`](../packages/api/account-controller) | `api` | [`agent`](../packages/core/agent), [`deepseek-account`](../packages/credentials/deepseek-account), [`typert-protocol`](../packages/typert/protocol) | | [`compaction`](../packages/compaction/compaction) | `compaction` | [`brand`](../packages/util/brand), [`commands`](../packages/interaction/commands), [`llm`](../packages/llm/llm), [`session`](../packages/core/session) | | [`deepseek-account-platform`](../packages/credentials/deepseek-account-platform) | `credentials` | [`authorization`](../packages/credentials/authorization), [`credentials`](../packages/credentials/credentials), [`deepseek-account`](../packages/credentials/deepseek-account), [`host-webserver`](../packages/host/webserver) | +| [`cua`](../packages/experimental/cua) | `experimental` | [`attachment`](../packages/attachment/attachment), [`scope`](../packages/core/scope), [`user-questions`](../packages/interaction/user-questions) | | [`experimental-ptc-runtime-python`](../packages/experimental/ptc-runtime-python) | `experimental` | [`ptc-runtime`](../packages/ptc-runtime/ptc-runtime), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`timeout`](../packages/util/timeout), [`util-values`](../packages/util/values) | | [`experimental-worktree`](../packages/experimental/worktree) | `experimental` | [`agent`](../packages/core/agent), [`fs`](../packages/fs/fs), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`subprocess`](../packages/subprocess/subprocess), [`working-directory`](../packages/session/working-directory) | | [`command-feedback`](../packages/feedback/command-feedback) | `feedback` | [`anonymous-user-id`](../packages/identity/anonymous-user-id), [`commands`](../packages/interaction/commands), [`session`](../packages/core/session), [`typert-protocol`](../packages/typert/protocol) | @@ -1740,7 +1734,6 @@ flowchart TD | [`session-query`](../packages/session-query/session-query) | `session-query` | [`brand`](../packages/util/brand), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-persistence`](../packages/session/session-persistence), [`session-projection`](../packages/session/session-projection), [`session-projection-cache`](../packages/session/session-projection-cache), [`session-title`](../packages/session/session-title), [`tool-todo`](../packages/todo/tool-todo) | | [`api-settings-controller`](../packages/api/settings-controller) | `api` | [`agent-preset-registry`](../packages/preset/agent-preset-registry), [`credentials`](../packages/credentials/credentials), [`native-command`](../packages/util/native-command), [`session`](../packages/core/session), [`settings`](../packages/settings/settings), [`typert-protocol`](../packages/typert/protocol) | | [`web-app`](../packages/bundle/web-app) | `bundle` | [`shell-env`](../packages/shell/shell-env) | -| [`cua`](../packages/experimental/cua) | `experimental` | [`agent`](../packages/core/agent), [`agent-preset-registry`](../packages/preset/agent-preset-registry), [`attachment`](../packages/attachment/attachment), [`computer-use`](../packages/computer-use/computer-use), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-questions`](../packages/interaction/user-questions) | | [`experimental-auto-review`](../packages/experimental/auto-review) | `experimental` | [`agent`](../packages/core/agent), [`agent-instructions`](../packages/context/agent-instructions), [`llm`](../packages/llm/llm), [`permission-presets`](../packages/interaction/permission-presets), [`session`](../packages/core/session), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) | | [`webhook`](../packages/experimental/webhook) | `experimental` | [`agent`](../packages/core/agent), [`agent-default-model`](../packages/core/agent-default-model), [`agent-preset-registry`](../packages/preset/agent-preset-registry), [`llm`](../packages/llm/llm), [`permission-presets`](../packages/interaction/permission-presets), [`session`](../packages/core/session), [`session-title`](../packages/session/session-title), [`workspace`](../packages/workspace/workspace) | | [`tool-cordis`](../packages/extensions/tool-cordis) | `extensions` | [`agent`](../packages/core/agent), [`app-boot`](../packages/boot/app-boot), [`cordis-host-runner`](../packages/extensions/cordis-host-runner), [`tools`](../packages/core/tools) | diff --git a/docs/module-graph.zh.md b/docs/module-graph.zh.md index 531c608183..7985a9744d 100644 --- a/docs/module-graph.zh.md +++ b/docs/module-graph.zh.md @@ -761,6 +761,9 @@ flowchart TD pkg_deepseek_account_platform --> pkg_credentials pkg_deepseek_account_platform --> pkg_deepseek_account pkg_deepseek_account_platform --> pkg_host_webserver + pkg_cua --> pkg_attachment + pkg_cua --> pkg_scope + pkg_cua --> pkg_user_questions pkg_experimental_ptc_runtime_python --> pkg_ptc_runtime pkg_experimental_ptc_runtime_python --> pkg_sandbox pkg_experimental_ptc_runtime_python --> pkg_sandbox_policy @@ -1071,16 +1074,6 @@ flowchart TD pkg_api_settings_controller --> pkg_settings pkg_api_settings_controller --> pkg_typert_protocol pkg_web_app --> pkg_shell_env - pkg_cua --> pkg_agent - pkg_cua --> pkg_agent_preset_registry - pkg_cua --> pkg_attachment - pkg_cua --> pkg_computer_use - pkg_cua --> pkg_llm - pkg_cua --> pkg_scope - pkg_cua --> pkg_subprocess - pkg_cua --> pkg_system_prompt - pkg_cua --> pkg_tools - pkg_cua --> pkg_user_questions pkg_experimental_auto_review --> pkg_agent pkg_experimental_auto_review --> pkg_agent_instructions pkg_experimental_auto_review --> pkg_llm @@ -1683,6 +1676,7 @@ flowchart TD | [`api-account-controller`](../packages/api/account-controller) | `api` | [`agent`](../packages/core/agent), [`deepseek-account`](../packages/credentials/deepseek-account), [`typert-protocol`](../packages/typert/protocol) | | [`compaction`](../packages/compaction/compaction) | `compaction` | [`brand`](../packages/util/brand), [`commands`](../packages/interaction/commands), [`llm`](../packages/llm/llm), [`session`](../packages/core/session) | | [`deepseek-account-platform`](../packages/credentials/deepseek-account-platform) | `credentials` | [`authorization`](../packages/credentials/authorization), [`credentials`](../packages/credentials/credentials), [`deepseek-account`](../packages/credentials/deepseek-account), [`host-webserver`](../packages/host/webserver) | +| [`cua`](../packages/experimental/cua) | `experimental` | [`attachment`](../packages/attachment/attachment), [`scope`](../packages/core/scope), [`user-questions`](../packages/interaction/user-questions) | | [`experimental-ptc-runtime-python`](../packages/experimental/ptc-runtime-python) | `experimental` | [`ptc-runtime`](../packages/ptc-runtime/ptc-runtime), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`timeout`](../packages/util/timeout), [`util-values`](../packages/util/values) | | [`experimental-worktree`](../packages/experimental/worktree) | `experimental` | [`agent`](../packages/core/agent), [`fs`](../packages/fs/fs), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`subprocess`](../packages/subprocess/subprocess), [`working-directory`](../packages/session/working-directory) | | [`command-feedback`](../packages/feedback/command-feedback) | `feedback` | [`anonymous-user-id`](../packages/identity/anonymous-user-id), [`commands`](../packages/interaction/commands), [`session`](../packages/core/session), [`typert-protocol`](../packages/typert/protocol) | @@ -1742,7 +1736,6 @@ flowchart TD | [`session-query`](../packages/session-query/session-query) | `session-query` | [`brand`](../packages/util/brand), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-persistence`](../packages/session/session-persistence), [`session-projection`](../packages/session/session-projection), [`session-projection-cache`](../packages/session/session-projection-cache), [`session-title`](../packages/session/session-title), [`tool-todo`](../packages/todo/tool-todo) | | [`api-settings-controller`](../packages/api/settings-controller) | `api` | [`agent-preset-registry`](../packages/preset/agent-preset-registry), [`credentials`](../packages/credentials/credentials), [`native-command`](../packages/util/native-command), [`session`](../packages/core/session), [`settings`](../packages/settings/settings), [`typert-protocol`](../packages/typert/protocol) | | [`web-app`](../packages/bundle/web-app) | `bundle` | [`shell-env`](../packages/shell/shell-env) | -| [`cua`](../packages/experimental/cua) | `experimental` | [`agent`](../packages/core/agent), [`agent-preset-registry`](../packages/preset/agent-preset-registry), [`attachment`](../packages/attachment/attachment), [`computer-use`](../packages/computer-use/computer-use), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-questions`](../packages/interaction/user-questions) | | [`experimental-auto-review`](../packages/experimental/auto-review) | `experimental` | [`agent`](../packages/core/agent), [`agent-instructions`](../packages/context/agent-instructions), [`llm`](../packages/llm/llm), [`permission-presets`](../packages/interaction/permission-presets), [`session`](../packages/core/session), [`tools`](../packages/core/tools), [`working-directory`](../packages/session/working-directory) | | [`webhook`](../packages/experimental/webhook) | `experimental` | [`agent`](../packages/core/agent), [`agent-default-model`](../packages/core/agent-default-model), [`agent-preset-registry`](../packages/preset/agent-preset-registry), [`llm`](../packages/llm/llm), [`permission-presets`](../packages/interaction/permission-presets), [`session`](../packages/core/session), [`session-title`](../packages/session/session-title), [`workspace`](../packages/workspace/workspace) | | [`tool-cordis`](../packages/extensions/tool-cordis) | `extensions` | [`agent`](../packages/core/agent), [`app-boot`](../packages/boot/app-boot), [`cordis-host-runner`](../packages/extensions/cordis-host-runner), [`tools`](../packages/core/tools) | diff --git a/packages/experimental/cua/package.json b/packages/experimental/cua/package.json index 1b91f37483..adfdd19789 100644 --- a/packages/experimental/cua/package.json +++ b/packages/experimental/cua/package.json @@ -70,24 +70,16 @@ "license": "MIT", "peerDependencies": { "@deepseek-ai/cordis": "workspace:~", - "@deepseek-ai/dsh-computer-use": "workspace:*", - "@deepseek-ai/dsh-system-prompt": "workspace:*", - "@deepseek-ai/dsh-tools": "workspace:*", - "@deepseek-ai/dsh-agent": "workspace:*", - "@deepseek-ai/dsh-subprocess": "workspace:*", "@deepseek-ai/dsh-user-questions": "workspace:*", "@deepseek-ai/dsh-attachment": "workspace:*", - "@deepseek-ai/dsh-llm": "workspace:*", - "@deepseek-ai/dsh-scope": "workspace:*", - "@deepseek-ai/dsh-agent-preset-registry": "workspace:*" + "@deepseek-ai/dsh-scope": "workspace:*" }, "dependencies": { - "@deepseek-ai/dsh-util-values": "workspace:*", "@deepseek-ai/schemastery": "workspace:~", "zod": "^4.4.3", - "@deepseek-ai/dsh-brand": "workspace:*", "@deepseek-ai/dsh-atomic-write": "workspace:*", - "@deepseek-ai/dsh-typert-protocol": "workspace:*" + "@deepseek-ai/dsh-typert-protocol": "workspace:*", + "@deepseek-ai/dsh-computer-use": "workspace:*" }, "devDependencies": { "@deepseek-ai/dsh-working-directory": "workspace:*", @@ -99,7 +91,6 @@ "@deepseek-ai/dsh-agent-loop-testkit": "workspace:*", "@deepseek-ai/dsh-attachment": "workspace:*", "@deepseek-ai/dsh-attachment-local": "workspace:*", - "@deepseek-ai/dsh-computer-use": "workspace:*", "@deepseek-ai/dsh-llm": "workspace:*", "@deepseek-ai/dsh-session": "workspace:*", "@deepseek-ai/dsh-session-projection": "workspace:*", @@ -136,8 +127,9 @@ "react": "^18.2.0", "react-dom": "^18.2.0", "@testing-library/react": "^16.1.0", - "@deepseek-ai/dsh-typert-protocol": "workspace:*", - "@deepseek-ai/dsh-loader-smoke": "workspace:*" + "@deepseek-ai/dsh-loader-smoke": "workspace:*", + "@deepseek-ai/dsh-util-values": "workspace:*", + "@deepseek-ai/dsh-brand": "workspace:*" }, "dsh": { "client": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 95dc195b0e..06583be24f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8492,15 +8492,12 @@ importers: '@deepseek-ai/dsh-atomic-write': specifier: workspace:* version: link:../../util/atomic-write - '@deepseek-ai/dsh-brand': + '@deepseek-ai/dsh-computer-use': specifier: workspace:* - version: link:../../util/brand + version: link:../../computer-use/computer-use '@deepseek-ai/dsh-typert-protocol': specifier: workspace:* version: link:../../typert/protocol - '@deepseek-ai/dsh-util-values': - specifier: workspace:* - version: link:../../util/values '@deepseek-ai/schemastery': specifier: link:../../../vendor/schemastery version: link:../../../vendor/schemastery @@ -8547,6 +8544,9 @@ importers: '@deepseek-ai/dsh-attachment-local': specifier: workspace:* version: link:../../attachment/attachment-local + '@deepseek-ai/dsh-brand': + specifier: workspace:* + version: link:../../util/brand '@deepseek-ai/dsh-client-locale': specifier: workspace:* version: link:../../client/locale @@ -8574,9 +8574,6 @@ importers: '@deepseek-ai/dsh-client-ui-slots': specifier: workspace:* version: link:../../client/ui-slots - '@deepseek-ai/dsh-computer-use': - specifier: workspace:* - version: link:../../computer-use/computer-use '@deepseek-ai/dsh-config-editor': specifier: workspace:* version: link:../../boot/config-editor @@ -8634,6 +8631,9 @@ importers: '@deepseek-ai/dsh-user-questions': specifier: workspace:* version: link:../../interaction/user-questions + '@deepseek-ai/dsh-util-values': + specifier: workspace:* + version: link:../../util/values '@deepseek-ai/dsh-working-directory': specifier: workspace:* version: link:../../session/working-directory diff --git a/scripts/package-dependency-policy.ts b/scripts/package-dependency-policy.ts index ef15d1bbf7..95e498cdfc 100644 --- a/scripts/package-dependency-policy.ts +++ b/scripts/package-dependency-policy.ts @@ -51,6 +51,8 @@ const DUPLICATE_SAFE_PACKAGES: readonly string[] = [ * prominent heading in the pull request description. */ const SAFE_HOST_DEPENDENCY_EXPORTS = { + '@deepseek-ai/dsh-atomic-write': ['withFileLock'], + '@deepseek-ai/dsh-computer-use/brand': ['ComputerUseProviderName'], '@deepseek-ai/dsh-credentials': ['credentialKey'], '@deepseek-ai/dsh-deque': ['Deque'], '@deepseek-ai/dsh-llm': ['callConfigEquals'], @@ -61,6 +63,9 @@ const SAFE_HOST_DEPENDENCY_EXPORTS = { /** Runtime exports that require every consumer to resolve the provider's shared peer instance. */ const PEER_REQUIRED_HOST_EXPORTS = { + // CUA checks UserQuestionError and preserves AttachmentError from image admission. + '@deepseek-ai/dsh-attachment': ['admitEncodedImages'], + '@deepseek-ai/dsh-user-questions': ['UserQuestionError'], // `instanceof` checks need the translator's own class instance. '@deepseek-ai/dsh-experimental-translator': ['TranslationError'], '@deepseek-ai/dsh-scope': ['carrierKeyOf', 'createScope', 'scopeOf', 'scopeTarget'], From ecc90620a411bfdadbcb69a1d9add19ad58b8366 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 21:04:06 +0800 Subject: [PATCH 42/55] feat(cua): use one action per system permission row --- apps/web/tests/cua-permissions.e2e.ts | 11 +++-- .../cua-permissions/missing.expected.md | 4 -- packages/experimental/cua/README.i18n.yaml | 4 +- packages/experimental/cua/README.md | 2 +- packages/experimental/cua/README.zh.md | 2 +- .../cua/src/client/PermissionPanel.module.css | 7 ++- .../cua/src/client/PermissionPanel.tsx | 31 +++++++++---- .../experimental/cua/src/client/locales.ts | 4 +- .../cua/tests/panel.client.spec.tsx | 44 ++++++++++++++++--- 9 files changed, 78 insertions(+), 31 deletions(-) diff --git a/apps/web/tests/cua-permissions.e2e.ts b/apps/web/tests/cua-permissions.e2e.ts index 3379f2ecf1..3fdbc863c1 100644 --- a/apps/web/tests/cua-permissions.e2e.ts +++ b/apps/web/tests/cua-permissions.e2e.ts @@ -79,7 +79,7 @@ it.skipIf(process.platform !== 'darwin')('keeps system requests pending until fr await openDetails.click() const panel = page.locator('[data-cua-settings]') const accessibility = panel.getByRole('group', { name: 'Accessibility', exact: true }) - await accessibility.getByText('Not allowed', { exact: true }).waitFor() + await accessibility.getByRole('button', { name: 'Request permission', exact: true }).waitFor() expect(await panel.getByRole('region', { name: 'Authorize Computer Use', exact: true }).getByRole('group').count()).toBe(4) expect(await panel.getByRole('region', { name: 'Manage applications Computer Use can access', exact: true }).count()).toBe(1) await expect.poll(() => panel.locator('li img').evaluate((image: HTMLImageElement) => image.naturalWidth)).toBe(1) @@ -122,13 +122,16 @@ it.skipIf(process.platform !== 'darwin')('keeps system requests pending until fr } await accessibility.getByRole('button', { name: 'Request permission', exact: true }).click() await expect.poll(() => panel.getAttribute('aria-busy')).toBe('false') - expect(await accessibility.getByText('Not allowed', { exact: true }).count()).toBe(1) + expect(await accessibility.getByRole('button', { name: 'Request permission', exact: true }).count()).toBe(1) await writeFile(state, JSON.stringify({ accessibility: true, screenRecording: false, launchApplication })) // Headless pages retain document focus; model returning from System Settings at the window-event boundary. await page.evaluate(() => { window.dispatchEvent(new Event('focus')) }) await accessibility.getByText('Allowed', { exact: true }).waitFor() const screen = panel.getByRole('group', { name: 'Screen Recording', exact: true }) - await screen.getByRole('button', { name: 'Open System Settings', exact: true }).click() + expect(await accessibility.getByRole('button').count()).toBe(1) + await accessibility.getByRole('button', { name: 'Allowed', exact: true }).click() + await page.getByRole('menuitem', { name: 'Open System Settings', exact: true }).click() + expect(await screen.getByRole('button').count()).toBe(1) await expect.poll(() => panel.getAttribute('aria-busy')).toBe('false') await panel.getByRole('button', { name: 'Revoke', exact: true }).click() await panel.getByRole('heading', { name: 'Always-allowed applications', exact: true }).waitFor({ state: 'hidden' }) @@ -141,7 +144,7 @@ it.skipIf(process.platform !== 'darwin')('keeps system requests pending until fr .filter(record => record.event === 'call').map(({ method, params }) => ({ method, params })) expect(events).toContainEqual({ method: 'requestPermission', params: { permission: 'accessibility' } }) expect(events.filter(event => event.method === 'openPermissionSettings')).toEqual([ - { method: 'openPermissionSettings', params: { permission: 'screenRecording' } }, + { method: 'openPermissionSettings', params: { permission: 'accessibility' } }, ]) expect(events.every(event => ['permissionStatus', 'requestPermission', 'openPermissionSettings', 'applicationIcons', 'getApp'].includes(event.method))).toBe(true) expect(tripwire.pageErrors).toEqual([]) diff --git a/apps/web/tests/expected/cua-permissions/missing.expected.md b/apps/web/tests/expected/cua-permissions/missing.expected.md index 25af1a8f27..4255b31e9f 100644 --- a/apps/web/tests/expected/cua-permissions/missing.expected.md +++ b/apps/web/tests/expected/cua-permissions/missing.expected.md @@ -11,15 +11,11 @@ - group "Accessibility": - strong: Accessibility - paragraph: In System Settings, allow Visual Studio Code.app to read interface controls, click, and type. - - text: Not allowed - button "Request permission" - - button "Open System Settings" - group "Screen Recording": - strong: Screen Recording - paragraph: In System Settings, allow Visual Studio Code.app to capture application windows. - - text: Not allowed - button "Request permission" - - button "Open System Settings" - paragraph: Enable the application in System Settings. Follow the macOS prompt if it names another application. - region "Manage applications Computer Use can access": - heading "Manage applications Computer Use can access" [level=3] diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index d00649e0f8..51e7f7b5da 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -18,8 +18,8 @@ en: ceb54d964dc355b6 zh: 0892066fbceef7b9 /deepseek-ai-dsh-cua/use-this-package/permission-controls: - en: 6873925322180290 - zh: 218cada140ec90b1 + en: 096915a67d0d5fd9 + zh: 999c004cc923577a /deepseek-ai-dsh-cua/understand-the-implementation: en: 68fc12448f1f1fc0 zh: ea1d489c263c6d71 diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index d143d12d18..50a696813a 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -43,7 +43,7 @@ Actions return `dispatched:true, verified:false`; read fresh state to verify the ### Permission controls -The [Computer Use bundle](../cua-bundle/README.md) mounts this page with its native provider. **Authorize Computer Use** groups the new-session default, PTC execution mode, Accessibility, and Screen Recording inside one border, with dividers between the rows. Each system-permission row shows the system-reported responsible application’s `.app` bundle filename and icon together in a bordered label. **Manage applications Computer Use can access** places the application lists inside a separate bordered group. Choose **Request permission**, find that application in the macOS permission list, and enable its switch; if macOS names another application, follow that prompt. With no detected source, the page directs the user to the application named by macOS. **Open System Settings** opens the corresponding privacy pane. The page refreshes on entry, when the window regains focus, and after permission actions. A failed query offers **Retry**. Follow any macOS quit-and-reopen prompt and restart DSH as needed. +The [Computer Use bundle](../cua-bundle/README.md) mounts this page with its native provider. **Authorize Computer Use** groups the new-session default, PTC execution mode, Accessibility, and Screen Recording inside one border, with dividers between the rows. Each system-permission row shows the system-reported responsible application’s `.app` bundle filename and icon together in a bordered label. **Manage applications Computer Use can access** places the application lists inside a separate bordered group. Choose **Request permission**, find that application in the macOS permission list, and enable its switch; if macOS names another application, follow that prompt. With no detected source, the page directs the user to the application named by macOS. Each system-permission row has one control: **Request permission** when access is missing, or an **Allowed** dropdown after authorization. The dropdown’s **Open System Settings** item opens the corresponding privacy pane, where the user can revoke access. Unknown status uses an **Unknown** dropdown; pending initial observations retain a single placeholder. The page refreshes on entry, when the window regains focus, and after permission actions. A failed query offers **Retry**. Follow any macOS quit-and-reopen prompt and restart DSH as needed. System-permission operations affect the computer running DSH. When accessing DSH remotely, complete authorization on that computer. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index 0bfcaf6214..9b0d007b3e 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -43,7 +43,7 @@ kind: "package-reference" ### 权限控件 -[Computer Use 组合](../cua-bundle/README.zh.md)将本页与原生 provider 一起挂载。“给 Computer Use 授权”把新会话默认模式、PTC 执行方式、辅助功能和屏幕录制放在同一个边框内，四排之间使用分隔线。两条系统权限要求在能识别责任应用时，把该应用的 `.app` 包名称和图标放在同一个标签中。“管理 Computer Use 可访问的应用”使用独立边框包住应用列表。点击“申请权限”，在 macOS 权限列表中找到该应用并开启对应开关；若 macOS 显示其他名称，以系统提示为准。来源无法识别时，页面引导用户查找系统提示的应用。“打开系统设置”会打开对应的隐私设置页。进入详情页、返回窗口或完成授权操作后会自动刷新；查询失败时可点击“重试”。若系统要求退出并重新打开应用，请按提示操作并重新启动 DSH。 +[Computer Use 组合](../cua-bundle/README.zh.md)将本页与原生 provider 一起挂载。“给 Computer Use 授权”把新会话默认模式、PTC 执行方式、辅助功能和屏幕录制放在同一个边框内，四排之间使用分隔线。两条系统权限要求在能识别责任应用时，把该应用的 `.app` 包名称和图标放在同一个标签中。“管理 Computer Use 可访问的应用”使用独立边框包住应用列表。点击“申请权限”，在 macOS 权限列表中找到该应用并开启对应开关；若 macOS 显示其他名称，以系统提示为准。来源无法识别时，页面引导用户查找系统提示的应用。每条系统权限要求只显示一个控件：未授权时为“申请权限”，已授权时为“已允许”下拉按钮。菜单中的“打开系统设置”进入对应隐私设置页，由用户撤销权限。无法确认状态时显示“未确认”下拉按钮，首次查询期间保留一个占位。进入详情页、返回窗口或完成授权操作后会自动刷新；查询失败时可点击“重试”。若系统要求退出并重新打开应用，请按提示操作并重新启动 DSH。 系统权限操作作用于运行 DSH 的电脑。远程访问时，请在该电脑上完成授权。 diff --git a/packages/experimental/cua/src/client/PermissionPanel.module.css b/packages/experimental/cua/src/client/PermissionPanel.module.css index f4fb0334d0..11ce22b9f7 100644 --- a/packages/experimental/cua/src/client/PermissionPanel.module.css +++ b/packages/experimental/cua/src/client/PermissionPanel.module.css @@ -10,7 +10,7 @@ .permission { min-height: 80px; box-sizing: border-box; } .defaultSetting > p { margin: 0 16px 16px; } .permission > div:first-child, .apps li > div { flex: 1; min-width: 140px; } -.actions, .notice { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; } +.notice { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; } .group { display: grid; gap: 4px; } .apps { display: grid; padding: 0; margin: 0; list-style: none; } .apps li { padding: 12px 0; } @@ -33,15 +33,14 @@ } .applicationBadge img { flex-shrink: 0; object-fit: contain; } .applicationBadge > span { min-width: 0; overflow-wrap: anywhere; } -.permissionStatus { min-width: 72px; } -.buttonPlaceholder, .statusPlaceholder, .appPlaceholder, .appsPlaceholder { +.permissionAction { min-width: 120px; } +.buttonPlaceholder, .appPlaceholder, .appsPlaceholder { display: inline-block; max-width: 100%; border-radius: var(--dsw-radius-xs); background: var(--dsw-alias-bg-skeleton); } .buttonPlaceholder { width: 120px; height: 28px; } -.statusPlaceholder { width: 64px; height: 12px; } .appPlaceholder { width: 160px; height: 20px; } .appsPlaceholder { width: 240px; height: 40px; } diff --git a/packages/experimental/cua/src/client/PermissionPanel.tsx b/packages/experimental/cua/src/client/PermissionPanel.tsx index 8c304ef81d..c6fb69d3e8 100644 --- a/packages/experimental/cua/src/client/PermissionPanel.tsx +++ b/packages/experimental/cua/src/client/PermissionPanel.tsx @@ -1,6 +1,6 @@ /** Native permission status and revocable application consent on the existing plugin page. */ import { useEffect, useState } from 'react' -import { Button, Modal, Toast } from '@deepseek-ai/dsh-client-ui-primitives' +import { Button, IconChevronDownOutlineRegular, Menu, Modal, Toast } from '@deepseek-ai/dsh-client-ui-primitives' import type { InjectFace, PropsLocale, PropsRuntime } from '@deepseek-ai/dsh-client-ui-slots' import type {} from '@deepseek-ai/dsh-client-ui-plugin-manager/client' import type { ConfigForm } from '@deepseek-ai/dsh-client-ui-settings/client' @@ -111,6 +111,26 @@ function PermissionDescription({ application, permission, loading, t }: Pick } +function PermissionAction({ permission, granted, loading, busy, request, openSettings, t }: Pick & { + permission: 'accessibility' | 'screenRecording' + granted: boolean | null + loading: boolean + busy: boolean +}) { + const [open, setOpen] = useState(false) + useEffect(() => { setOpen(false) }, [busy, loading, granted]) + if (loading) return + if (granted === false) return + return { setOpen(false) }} align="end" portal autoFocus + anchor={} + items={[{ id: 'settings', label: t('settings') }]} + onSelect={() => { setOpen(false); openSettings(permission) }} /> +} + /** @param props Host observation and explicit user actions. @returns Permission controls without automatic system prompts. */ export function PermissionPanel(props: PermissionPanelProps) { const { t, refresh } = props @@ -142,13 +162,8 @@ export function PermissionPanel(props: PermissionPanelProps) { {t(permission)}
- {loading ? - : t(observed === null || state.system.error !== null ? 'unknown' : observed[permission] ? 'granted' : 'missing')} -
- {loading ? - : observed?.[permission] === false && } - -
+
)}
{observed !== null && (!observed.accessibility || !observed.screenRecording) &&

diff --git a/packages/experimental/cua/src/client/locales.ts b/packages/experimental/cua/src/client/locales.ts index 9d28f99a5e..0702e20597 100644 --- a/packages/experimental/cua/src/client/locales.ts +++ b/packages/experimental/cua/src/client/locales.ts @@ -28,7 +28,7 @@ export const en = { screenRecordingPermission: 'In System Settings, allow {application} to capture application windows.', permissionHint: 'Enable the application in System Settings. Follow the macOS prompt if it names another application.', permissionHintUnknown: 'Enable these permissions for the application named in the macOS prompt.', - granted: 'Allowed', missing: 'Not allowed', unknown: 'Unknown', request: 'Request permission', settings: 'Open System Settings', retry: 'Retry', + granted: 'Allowed', unknown: 'Unknown', request: 'Request permission', settings: 'Open System Settings', retry: 'Retry', allowedApps: 'Always-allowed applications', blockedApps: 'Applications not allowed', retryApproval: 'Allow asking again', retryAllowed: 'The next application selection will ask again; access has not been granted', @@ -65,7 +65,7 @@ export const zh: Record = { screenRecordingPermission: '在系统设置中允许 {application} 截取应用窗口。', permissionHint: '在系统设置中开启对应应用；若 macOS 提示其他应用，以系统提示为准', permissionHintUnknown: '在系统设置中，为 macOS 提示的应用开启对应权限', - granted: '已允许', missing: '未允许', unknown: '未确认', request: '申请权限', settings: '打开系统设置', retry: '重试', + granted: '已允许', unknown: '未确认', request: '申请权限', settings: '打开系统设置', retry: '重试', allowedApps: '始终允许的应用', blockedApps: '不允许访问的应用', retryApproval: '允许再次询问', retryAllowed: '下次选择此应用时将重新询问，当前尚未授权', diff --git a/packages/experimental/cua/tests/panel.client.spec.tsx b/packages/experimental/cua/tests/panel.client.spec.tsx index d0117edee6..5c131659b3 100644 --- a/packages/experimental/cua/tests/panel.client.spec.tsx +++ b/packages/experimental/cua/tests/panel.client.spec.tsx @@ -49,10 +49,11 @@ it('requests only a missing privilege and leaves OS attribution to the system pr f.publish({ ...observe(value) }) fireEvent.click(screen.getByRole('button', { name: en.request })) expect(f.props.request).toHaveBeenCalledWith('accessibility') - const controls = screen.getAllByRole('button', { name: en.settings }) - fireEvent.click(controls[0]!) - fireEvent.click(controls[1]!) - expect(f.props.openSettings.mock.calls).toEqual([['accessibility'], ['screenRecording']]) + expect(screen.queryByRole('button', { name: en.settings })).toBeNull() + fireEvent.click(screen.getByRole('button', { name: en.granted })) + expect(f.props.openSettings).not.toHaveBeenCalled() + fireEvent.click(screen.getByRole('menuitem', { name: en.settings })) + expect(f.props.openSettings.mock.calls).toEqual([['screenRecording']]) fireEvent.focus(window) expect(f.props.refresh).toHaveBeenCalledTimes(2) shown.unmount() @@ -280,7 +281,7 @@ it('keeps all four requirements visible before the first permission response', ( expect(within(requirements).getAllByRole('group')).toHaveLength(4) expect(screen.queryByRole('alert')).toBeNull() expect(screen.queryByRole('button', { name: en.request })).toBeNull() - for (const button of screen.getAllByRole('button', { name: en.settings })) expect(button.hasAttribute('disabled')).toBe(true) + for (const permission of [en.accessibility, en.screenRecording]) expect(within(screen.getByRole('group', { name: permission })).queryByRole('button')).toBeNull() }) it('fills application names, system permissions, and icons as their observations arrive', () => { @@ -362,3 +363,36 @@ it('loads the runtime setting independently and reports unavailable and read-onl expect(within(row).getByRole('button').hasAttribute('disabled')).toBe(true) expect(f.props.setPersistentPtc).not.toHaveBeenCalled() }) + + +it.each([en, zh])('keeps one control per permission row and closes a granted menu on refresh or revocation', (dictionary) => { + const f = fixture() + f.publish(observe(value)) + render() + const missing = screen.getByRole('group', { name: dictionary.accessibility }) + const granted = screen.getByRole('group', { name: dictionary.screenRecording }) + expect(within(missing).getAllByRole('button')).toHaveLength(1) + expect(within(missing).getByRole('button', { name: dictionary.request })).toBeTruthy() + const trigger = within(granted).getByRole('button', { name: dictionary.granted }) + expect(within(granted).getAllByRole('button')).toHaveLength(1) + trigger.focus() + fireEvent.click(trigger) + expect(trigger.getAttribute('aria-expanded')).toBe('true') + expect(screen.getByRole('menuitem', { name: dictionary.settings })).toBeTruthy() + fireEvent.keyDown(screen.getByRole('menu'), { key: 'Escape' }) + expect(screen.queryByRole('menu')).toBeNull() + expect(document.activeElement).toBe(trigger) + fireEvent.click(trigger) + f.publish({ system: { ...f.state.getSnapshot().system, busy: true }, busy: true }) + expect(screen.queryByRole('menu')).toBeNull() + expect(trigger.hasAttribute('disabled')).toBe(true) + f.publish(observe(value)) + expect(screen.queryByRole('menu')).toBeNull() + fireEvent.click(trigger) + f.publish(observe({ ...value, system: { ...value.system, screenRecording: false } })) + expect(screen.queryByRole('menu')).toBeNull() + expect(within(granted).getAllByRole('button')).toHaveLength(1) + expect(within(granted).getByRole('button', { name: dictionary.request })).toBeTruthy() + expect(f.props.openSettings).not.toHaveBeenCalled() + expect(f.props.revoke).not.toHaveBeenCalled() +}) From 43aafb2e1982e8396527f651dbdbb3fbdc97298a Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 21:08:01 +0800 Subject: [PATCH 43/55] fix(cua): retain permission controls while stale errors refresh --- .../experimental/cua/src/client/PermissionPanel.tsx | 3 ++- .../experimental/cua/tests/panel.client.spec.tsx | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/experimental/cua/src/client/PermissionPanel.tsx b/packages/experimental/cua/src/client/PermissionPanel.tsx index c6fb69d3e8..525a313d05 100644 --- a/packages/experimental/cua/src/client/PermissionPanel.tsx +++ b/packages/experimental/cua/src/client/PermissionPanel.tsx @@ -162,7 +162,8 @@ export function PermissionPanel(props: PermissionPanelProps) { {t(permission)} - )} diff --git a/packages/experimental/cua/tests/panel.client.spec.tsx b/packages/experimental/cua/tests/panel.client.spec.tsx index 5c131659b3..a780d4a319 100644 --- a/packages/experimental/cua/tests/panel.client.spec.tsx +++ b/packages/experimental/cua/tests/panel.client.spec.tsx @@ -396,3 +396,15 @@ it.each([en, zh])('keeps one control per permission row and closes a granted men expect(f.props.openSettings).not.toHaveBeenCalled() expect(f.props.revoke).not.toHaveBeenCalled() }) + + +it('retains the previous single control while a stale query error is being refreshed', () => { + const f = fixture() + f.publish(observe(value)) + render() + f.publish({ system: { value: value.system, busy: true, error: 'stale error' }, busy: true }) + expect(screen.getByRole('button', { name: en.request }).hasAttribute('disabled')).toBe(true) + expect(screen.getByRole('button', { name: en.granted }).hasAttribute('disabled')).toBe(true) + expect(screen.queryByRole('alert')).toBeNull() + expect(screen.queryByRole('menu')).toBeNull() +}) From 03d3cf919659195830ee95dcc70004cf0b3230bb Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 21:11:13 +0800 Subject: [PATCH 44/55] refactor(cua): consume the generated permission Remote directly --- packages/experimental/cua/README.i18n.yaml | 4 +- packages/experimental/cua/README.md | 2 +- packages/experimental/cua/README.zh.md | 2 +- .../experimental/cua/src/client/controller.ts | 49 ++++---- packages/experimental/cua/src/client/mount.ts | 23 +--- .../cua/tests/controller.client.spec.ts | 107 ++++++++++++++---- 6 files changed, 113 insertions(+), 74 deletions(-) diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index 51e7f7b5da..6d415d746b 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -21,8 +21,8 @@ en: 096915a67d0d5fd9 zh: 999c004cc923577a /deepseek-ai-dsh-cua/understand-the-implementation: - en: 68fc12448f1f1fc0 - zh: ea1d489c263c6d71 + en: 1fde7df44f040131 + zh: 59a842fa402ccb35 /deepseek-ai-dsh-cua/further-exploration: en: b8b0aece886fadaa zh: ecd8c88d601f2179 diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index 50a696813a..29415cd575 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -72,7 +72,7 @@ The provider calls [installCua](src/index.ts) with its prepared helper, connecti The root Host entry exports `installCua`, `CuaConfig`, and the Typert-discovered `CuaAccess` service; its `apply()` only marks the optional Client contribution. `/client` loads the permission UI through the existing module loader. Separate Host and Client compiler projects keep React, CSS, and browser registrations out of headless imports. `/api` owns the app/window API and its types; `/bridge`, `/native-client`, and `/protocol` retain runtime and platform consumers. `/types` exports only the four named boundary types required by generated Remote declarations. The Client uses package-local types. `/typert` supplies Host reflection and `/remote` supplies generated browser descriptors and codecs; permission authority remains on the Host. -The optional browser contribution mounts the native provider's permission Remote and reuses the Plugins activation and configuration slots. The four setup rows remain visible while unknown fields use placeholders. The default-mode form, runtime configuration, system checks, application decisions, and application icons fill in as each query completes. Application names and actions do not wait for icons; icon positions retain their space. User actions run in order without waiting for pending observation queries. Disposal waits for both actions and queries to settle before releasing the separately owned Remote mount. Query errors stay with the affected section, and activation guidance depends only on system checks. Late results after disposal are discarded. One controller projects queries into the page and setup dialog; reconnects and updates to the Host-reported settings namespace refresh it. Operation notices live in the shell overlay, so closing the page does not remove them. The provider owns permission checks, persisted decisions, and cancellation; the browser owns no authorization state. +The optional browser contribution mounts the native provider's permission Remote and reuses the Plugins activation and configuration slots. The four setup rows remain visible while unknown fields use placeholders. The default-mode form, runtime configuration, system checks, application decisions, and application icons fill in as each query completes. Application names and actions do not wait for icons; icon positions retain their space. User actions run in order without waiting for pending observation queries. Disposal waits for both actions and queries to settle before releasing the separately owned Remote mount. Query errors stay with the affected section, and activation guidance depends only on system checks. Late results after disposal are discarded. The controller consumes the generated `ctx.remote.cuaAccess` namespace directly and handles unsuccessful Remote results in its existing query and action paths. It projects queries into the page and setup dialog; reconnects and updates to the Host-reported settings namespace refresh it. Operation notices live in the shell overlay, so closing the page does not remove them. The provider owns permission checks, persisted decisions, and cancellation; the browser owns no authorization state. The detail page reads the effective default (`selectedDefault ?? default`) from the existing `configForms` service. The default button writes only `selectedDefault` through the form's revision-checked settings path. The execution-mode row uses CUA's `ptcConfiguration()` and `enablePersistentPtc()` controls: the Host edits only `ptc-runtime.executionMode` through the existing configuration editor, preserving its ordinary Loader reload lifecycle. Permission checks and runtime state settle independently. Loading, read-only and unavailable settings cannot submit a change; a failed write remains beside the setting for retry. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index 9b0d007b3e..80b3ed3a6b 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -72,7 +72,7 @@ kind: "package-reference" 根 Host 入口导出 `installCua`、`CuaConfig` 和供 Typert 发现的 `CuaAccess` 服务，`apply()` 只标记可选 Client 贡献。`/client` 通过现有模块加载器加载权限界面。Host 和 Client 使用独立编译项目，headless 导入不会加载 React、CSS 或浏览器注册。`/api` 提供应用、窗口 API 及其类型；`/bridge`、`/native-client` 和 `/protocol` 保留运行时和平台消费者。`/types` 仅导出生成 Remote 声明所需的四个命名类型，Client 使用包内类型。`/typert` 提供 Host 反射，`/remote` 提供浏览器描述和编解码，权限判断仍由 Host 执行。 -可选浏览器插件挂载原生 provider 的权限 Remote，复用插件页的启用引导和配置插槽。四排使用条件始终可见，未加载的字段保留占位。默认模式、执行方式、系统权限、应用列表和应用图标分别查询，结果返回后各自显示。应用名称和操作按钮不等待图标，图标位置保留空位。用户操作按顺序执行，不等待尚未完成的查询；卸载时等待操作和查询全部结束，再释放独立持有的 Remote 挂载。查询错误只显示在对应部分，启用引导只等待系统权限结果。卸载后的迟到结果会被丢弃。一个控制器把查询结果投影到页面和设置弹窗；重新连接或 Host 返回的配置 namespace 更新时刷新状态。操作提示挂载到 shell overlay，关闭页面后仍可显示。provider 负责权限检查、持久授权和取消；浏览器不保存授权决定。 +可选浏览器插件挂载原生 provider 的权限 Remote，复用插件页的启用引导和配置插槽。四排使用条件始终可见，未加载的字段保留占位。默认模式、执行方式、系统权限、应用列表和应用图标分别查询，结果返回后各自显示。应用名称和操作按钮不等待图标，图标位置保留空位。用户操作按顺序执行，不等待尚未完成的查询；卸载时等待操作和查询全部结束，再释放独立持有的 Remote 挂载。查询错误只显示在对应部分，启用引导只等待系统权限结果。卸载后的迟到结果会被丢弃。控制器直接接收生成的 `ctx.remote.cuaAccess`，在现有查询和操作流程中处理失败响应，并把查询结果投影到页面和设置弹窗；重新连接或 Host 返回的配置 namespace 更新时刷新状态。操作提示挂载到 shell overlay，关闭页面后仍可显示。provider 负责权限检查、持久授权和取消；浏览器不保存授权决定。 详情页通过现有 `configForms` 服务读取实际默认值（`selectedDefault ?? default`）。默认模式按钮仅写入 `selectedDefault`，沿用表单带版本检查的设置写入流程。执行方式一行调用 CUA 的 `ptcConfiguration()` 和 `enablePersistentPtc()`：Host 通过现有配置编辑器仅修改 `ptc-runtime.executionMode`，保留普通 Loader 重载规则。权限检查和执行方式状态独立返回。设置仍在加载、只读或不可用时不能提交；写入失败时在设置旁显示错误，供用户重试。 diff --git a/packages/experimental/cua/src/client/controller.ts b/packages/experimental/cua/src/client/controller.ts index c23497c4d7..0ecdbdf4d0 100644 --- a/packages/experimental/cua/src/client/controller.ts +++ b/packages/experimental/cua/src/client/controller.ts @@ -1,18 +1,11 @@ /** One query projection for the permission page and activation guidance. */ +import type { Context } from '@deepseek-ai/cordis' +import type {} from '@deepseek-ai/dsh-api-remotes/client' +import type {} from '@deepseek-ai/dsh-cua/remote' +import type { RemoteResult } from '@deepseek-ai/dsh-typert-protocol' import { createSnapshotStore, type SnapshotStore } from '@deepseek-ai/dsh-client-store' import type { CuaApplicationId, CuaApplicationState, CuaSystemPermission, CuaSystemPermissionState } from '../access-types.ts' -/** Host operations available to the permission UI. */ -export interface PermissionActions { - readRuntime: () => Promise<{ persistent: boolean; writable: boolean }> - readSystem: () => Promise - readApplications: () => Promise - readIcons: () => Promise> - request: (permission: CuaSystemPermission) => Promise - openSettings: (permission: CuaSystemPermission) => Promise - revoke: (id: CuaApplicationId) => Promise - retryApproval: (id: CuaApplicationId) => Promise -} interface PermissionData { runtime: { persistent: boolean; writable: boolean } system: CuaSystemPermissionState @@ -55,8 +48,8 @@ export class PermissionController { private reads: Promise = Promise.resolve() private closed = false private sequence = 0 - /** @param actions Browser-owned Remote calls that reject on failed responses. */ - constructor(private readonly actions: PermissionActions) {} + /** @param remote Generated CUA Remote namespace; failed results are handled with query and action failures. */ + constructor(private readonly remote: Context['remote']['cuaAccess']) {} /** * Bind the permission projection to its user actions. * @returns Reactive state and controls shared by the page, prompt, and toast host. @@ -65,16 +58,16 @@ export class PermissionController { return { hooks: { cuaPermissions: this.state }, refresh: () => { this.enqueue() }, - request: (permission) => { this.enqueue(() => this.actions.request(permission), 'requested') }, - openSettings: (permission) => { this.enqueue(() => this.actions.openSettings(permission), 'opened') }, - revoke: (id) => { this.enqueue(() => this.actions.revoke(id), 'revoked') }, - retryApproval: (id) => { this.enqueue(() => this.actions.retryApproval(id), 'retryAllowed') }, + request: (permission) => { this.enqueue(() => this.remote.requestPermission(permission), 'requested') }, + openSettings: (permission) => { this.enqueue(() => this.remote.openSettings(permission), 'opened') }, + revoke: (id) => { this.enqueue(() => this.remote.revoke(id), 'revoked') }, + retryApproval: (id) => { this.enqueue(() => this.remote.retryApproval(id), 'retryAllowed') }, dismissToast: () => { if (!this.closed) this.state.update((state) => { state.toast = null }) }, } } /** Prevent publication after UI withdrawal and join the accepted Remote work. */ async dispose(): Promise { this.closed = true; await this.tail; await this.reads } - private enqueue(action?: () => Promise, kind?: 'requested' | 'opened' | 'revoked' | 'retryAllowed'): void { + private enqueue(action?: () => Promise>, kind?: 'requested' | 'opened' | 'revoked' | 'retryAllowed'): void { if (this.closed) return const current = () => !this.closed const sequence = ++this.sequence @@ -85,7 +78,10 @@ export class PermissionController { this.tail = this.tail.then(async () => { if (!current()) return try { - await action?.() + if (action) { + const result = await action() + if (!result.ok) throw result.error + } if (!current()) return if (kind) this.state.update((state) => { state.toast = { id: sequence, kind } }) } catch (error) { @@ -95,10 +91,10 @@ export class PermissionController { } const latest = () => current() && this.sequence === sequence const refresh = Promise.all([ - this.read('runtime', this.actions.readRuntime, latest), - this.read('system', this.actions.readSystem, latest), - this.read('applications', this.actions.readApplications, latest), - this.read('icons', this.actions.readIcons, latest), + this.read('runtime', this.remote.ptcConfiguration, latest), + this.read('system', this.remote.systemStatus, latest), + this.read('applications', this.remote.applications, latest), + this.read('icons', this.remote.applicationIcons, latest), ]).then(() => { if (latest()) this.state.update((state) => { state.busy = false }) }) @@ -107,13 +103,14 @@ export class PermissionController { } private async read( - key: K, read: () => Promise, current: () => boolean, + key: K, read: () => Promise>, current: () => boolean, ): Promise { if (!current()) return try { - const value = await read() + const result = await read() if (!current()) return - this.state.update((state) => { state[key].value = value; state[key].error = null; state[key].busy = false }) + if (!result.ok) throw result.error + this.state.update((state) => { state[key].value = result.value; state[key].error = null; state[key].busy = false }) } catch (error) { if (!current()) return this.state.update((state) => { diff --git a/packages/experimental/cua/src/client/mount.ts b/packages/experimental/cua/src/client/mount.ts index e01415b383..bc43e9bd8b 100644 --- a/packages/experimental/cua/src/client/mount.ts +++ b/packages/experimental/cua/src/client/mount.ts @@ -32,28 +32,7 @@ export async function mountCuaPermissions(ctx: Context, contribution: TypertRemo const transport = ctx.plugin(async (owner) => { await owner.remote.$mount(contribution) }) const ui = ctx.inject(['remote.cuaAccess', 'slots', 'locale'], (ctx) => { ctx.effect(() => ctx.locale.register(NS, { en, zh })) - const controller = new PermissionController({ - readRuntime: async () => { - const result = await ctx.remote.cuaAccess.ptcConfiguration(); if (!result.ok) throw result.error; return result.value - }, - readSystem: async () => { - const result = await ctx.remote.cuaAccess.systemStatus(); if (!result.ok) throw result.error; return result.value - }, - readApplications: async () => { - const result = await ctx.remote.cuaAccess.applications(); if (!result.ok) throw result.error; return result.value - }, - readIcons: async () => { - const result = await ctx.remote.cuaAccess.applicationIcons(); if (!result.ok) throw result.error; return result.value - }, - request: async (permission) => { - const result = await ctx.remote.cuaAccess.requestPermission(permission); if (!result.ok) throw result.error - }, - openSettings: async (permission) => { - const result = await ctx.remote.cuaAccess.openSettings(permission); if (!result.ok) throw result.error - }, - revoke: async (id) => { const result = await ctx.remote.cuaAccess.revoke(id); if (!result.ok) throw result.error }, - retryApproval: async (id) => { const result = await ctx.remote.cuaAccess.retryApproval(id); if (!result.ok) throw result.error }, - }) + const controller = new PermissionController(ctx.remote.cuaAccess) const face = controller.face() const preset = ctx.configForms.get<{ default: string; selectedDefault?: string }>('agent-preset-registry') const settings = { diff --git a/packages/experimental/cua/tests/controller.client.spec.ts b/packages/experimental/cua/tests/controller.client.spec.ts index 82262df31f..afcdb37c8a 100644 --- a/packages/experimental/cua/tests/controller.client.spec.ts +++ b/packages/experimental/cua/tests/controller.client.spec.ts @@ -1,5 +1,7 @@ /** Permission refreshes preserve observations and join user actions before withdrawal. */ import { setImmediate } from 'node:timers/promises' +import type { Context } from '@deepseek-ai/cordis' +import { RemoteError } from '@deepseek-ai/dsh-typert-protocol' import { expect, it, onTestFinished, vi } from 'vitest' import type { CuaApplicationId, CuaApplicationState, CuaSystemPermissionState } from '../src/access-types.ts' import { PermissionController } from '../src/client/controller.ts' @@ -7,12 +9,19 @@ import { PermissionController } from '../src/client/controller.ts' const value: CuaSystemPermissionState = { accessibility: true, screenRecording: false, helper: { pid: 9 }, parent: { pid: 8 } } const applications: CuaApplicationState = { allowedApps: [], blockedApps: [], settingsNamespace: 'cua', persistentAvailable: true } +type CuaRemote = Context['remote']['cuaAccess'] function fixture() { const actions = { - readRuntime: vi.fn(async () => ({ persistent: false, writable: true })), - readSystem: vi.fn(async () => value), readApplications: vi.fn(async () => applications), - readIcons: vi.fn(async (): Promise> => ({})), request: vi.fn(async () => {}), - openSettings: vi.fn(async () => {}), revoke: vi.fn(async () => {}), retryApproval: vi.fn(async () => {}) } + ptcConfiguration: vi.fn(async () => ({ ok: true, value: { persistent: false, writable: true } })), + systemStatus: vi.fn(async () => ({ ok: true, value })), + applications: vi.fn(async () => ({ ok: true, value: applications })), + applicationIcons: vi.fn(async () => ({ ok: true, value: {} })), + requestPermission: vi.fn(async () => ({ ok: true, value })), + openSettings: vi.fn(async () => ({ ok: true, value: undefined })), + revoke: vi.fn(async () => ({ ok: true, value: undefined })), + retryApproval: vi.fn(async () => ({ ok: true, value: undefined })), + enablePersistentPtc: vi.fn(async () => ({ ok: true, value: undefined })), + } const controller = new PermissionController(actions) return { actions, controller, face: controller.face() } } @@ -21,7 +30,7 @@ it('refreshes after a user request without treating the request itself as a gran const f = fixture() f.face.request('screenRecording') await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) - expect(f.actions.request).toHaveBeenCalledWith('screenRecording') + expect(f.actions.requestPermission).toHaveBeenCalledWith('screenRecording') expect(f.controller.state.getSnapshot().system.value?.screenRecording).toBe(false) expect(f.controller.state.getSnapshot().toast?.kind).toBe('requested') f.face.dismissToast() @@ -33,7 +42,7 @@ it('retains the last observation while marking a failed refresh', async () => { const f = fixture() f.face.refresh() await vi.waitFor(() => { expect(f.controller.state.getSnapshot().system.value).toEqual(value) }) - f.actions.readSystem.mockRejectedValueOnce(new Error('connection closed')) + f.actions.systemStatus.mockRejectedValueOnce(new Error('connection closed')) f.face.refresh() await vi.waitFor(() => { expect(f.controller.state.getSnapshot().system.error).toBe('connection closed') }) expect(f.controller.state.getSnapshot().system.value).toEqual(value) @@ -58,7 +67,10 @@ it('joins a pending read without publishing after disposal or starting queued ac const f = fixture() const pending = Promise.withResolvers() const entered = Promise.withResolvers() - f.actions.readSystem.mockImplementationOnce(() => { entered.resolve(undefined); return pending.promise }) + f.actions.systemStatus.mockImplementationOnce(async () => { + entered.resolve(undefined) + return { ok: true, value: await pending.promise } + }) f.face.refresh() await entered.promise f.face.request('accessibility') @@ -67,17 +79,20 @@ it('joins a pending read without publishing after disposal or starting queued ac pending.resolve(value) await closed expect(f.controller.state.getSnapshot()).toBe(before) - expect(f.actions.request).not.toHaveBeenCalled() + expect(f.actions.requestPermission).not.toHaveBeenCalled() f.face.refresh() - expect(f.actions.readSystem).toHaveBeenCalledOnce() + expect(f.actions.systemStatus).toHaveBeenCalledOnce() }) it('revokes access while icons are pending, serializes writes, and joins reads on disposal', async () => { const f = fixture(), icons = Promise.withResolvers>() const action = Promise.withResolvers(), entered = Promise.withResolvers() onTestFinished(async () => { icons.resolve({}); action.resolve(undefined); await f.controller.dispose() }) - f.actions.readIcons.mockReturnValueOnce(icons.promise) - f.actions.revoke.mockImplementationOnce(() => { entered.resolve(undefined); return action.promise }) + f.actions.applicationIcons.mockImplementationOnce(async () => ({ ok: true, value: await icons.promise })) + f.actions.revoke.mockImplementationOnce(async () => { + entered.resolve(undefined) + return { ok: true, value: await action.promise } + }) f.face.refresh() await vi.waitFor(() => { expect(f.controller.state.getSnapshot().applications.busy).toBe(false) }) expect(f.controller.state.getSnapshot().icons.busy).toBe(true) @@ -114,7 +129,11 @@ it('ignores stale toast completion after the permission UI is disposed', async ( it.each(['success', 'failure'])('settles an in-flight action %s after disposal without publishing', async (outcome) => { const f = fixture(), pending = Promise.withResolvers(), entered = Promise.withResolvers() onTestFinished(async () => { pending.resolve(undefined); await f.controller.dispose() }) - f.actions.request.mockImplementationOnce(() => { entered.resolve(undefined); return pending.promise }) + f.actions.requestPermission.mockImplementationOnce(async () => { + entered.resolve(undefined) + await pending.promise + return { ok: true, value } + }) f.face.request('accessibility') await entered.promise const before = f.controller.state.getSnapshot(), closed = f.controller.dispose() @@ -122,16 +141,19 @@ it.each(['success', 'failure'])('settles an in-flight action %s after disposal w else pending.resolve(undefined) await closed expect(f.controller.state.getSnapshot()).toBe(before) - expect(f.actions.readSystem).not.toHaveBeenCalled() + expect(f.actions.systemStatus).not.toHaveBeenCalled() }) it('contains a late read rejection and supports errors without an Error instance', async () => { const f = fixture() - f.actions.readSystem.mockRejectedValueOnce('offline') + f.actions.systemStatus.mockRejectedValueOnce('offline') f.face.refresh() await vi.waitFor(() => { expect(f.controller.state.getSnapshot().system.error).toBe('offline') }) const pending = Promise.withResolvers(), entered = Promise.withResolvers() - f.actions.readSystem.mockImplementationOnce(() => { entered.resolve(undefined); return pending.promise }) + f.actions.systemStatus.mockImplementationOnce(async () => { + entered.resolve(undefined) + return { ok: true, value: await pending.promise } + }) f.face.refresh() await entered.promise const before = f.controller.state.getSnapshot(), closed = f.controller.dispose() @@ -146,8 +168,14 @@ it.each(['success', 'failure'])('does not publish an older refresh %s while the const first = Promise.withResolvers(), firstEntered = Promise.withResolvers() const latest = Promise.withResolvers(), latestEntered = Promise.withResolvers() onTestFinished(async () => { first.resolve(value); latest.resolve(value); await f.controller.dispose() }) - f.actions.readSystem.mockImplementationOnce(() => { firstEntered.resolve(undefined); return first.promise }) - f.actions.readSystem.mockImplementationOnce(() => { latestEntered.resolve(undefined); return latest.promise }) + f.actions.systemStatus.mockImplementationOnce(async () => { + firstEntered.resolve(undefined) + return { ok: true, value: await first.promise } + }) + f.actions.systemStatus.mockImplementationOnce(async () => { + latestEntered.resolve(undefined) + return { ok: true, value: await latest.promise } + }) const observations: Array> = [] const unsubscribe = f.controller.state.subscribe(() => { observations.push(f.controller.state.getSnapshot()) }) onTestFinished(unsubscribe) @@ -166,7 +194,11 @@ it.each(['success', 'failure'])('does not publish an older refresh %s while the it('keeps a user-action failure visible when a refresh is queued behind it', async () => { const f = fixture(), action = Promise.withResolvers(), entered = Promise.withResolvers() onTestFinished(async () => { action.resolve(undefined); await f.controller.dispose() }) - f.actions.request.mockImplementationOnce(() => { entered.resolve(undefined); return action.promise }) + f.actions.requestPermission.mockImplementationOnce(async () => { + entered.resolve(undefined) + await action.promise + return { ok: true, value } + }) f.face.request('accessibility') await entered.promise f.face.refresh() @@ -183,9 +215,9 @@ it('publishes each observation before the other queries finish', async () => { const apps = Promise.withResolvers() const icons = Promise.withResolvers>() onTestFinished(async () => { system.resolve(value); apps.resolve(applications); icons.resolve({}); await f.controller.dispose() }) - f.actions.readSystem.mockReturnValueOnce(system.promise) - f.actions.readApplications.mockReturnValueOnce(apps.promise) - f.actions.readIcons.mockReturnValueOnce(icons.promise) + f.actions.systemStatus.mockImplementationOnce(async () => ({ ok: true, value: await system.promise })) + f.actions.applications.mockImplementationOnce(async () => ({ ok: true, value: await apps.promise })) + f.actions.applicationIcons.mockImplementationOnce(async () => ({ ok: true, value: await icons.promise })) f.face.refresh() apps.resolve(applications) await vi.waitFor(() => { @@ -205,7 +237,7 @@ it('publishes each observation before the other queries finish', async () => { it('keeps application decisions available when system inspection fails', async () => { const f = fixture() - f.actions.readSystem.mockRejectedValueOnce(new Error('helper offline')) + f.actions.systemStatus.mockRejectedValueOnce(new Error('helper offline')) f.face.refresh() await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) expect(f.controller.state.getSnapshot()).toMatchObject({ @@ -214,3 +246,34 @@ it('keeps application decisions available when system inspection fails', async ( }) await f.controller.dispose() }) + + +it.each([ + ['ptcConfiguration', 'runtime'], ['systemStatus', 'system'], + ['applications', 'applications'], ['applicationIcons', 'icons'], +] as const)('contains an unsuccessful %s response in its own query section', async (method, key) => { + const f = fixture() + onTestFinished(() => f.controller.dispose()) + f.face.refresh() + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) + const previous = f.controller.state.getSnapshot()[key].value + f.actions[method].mockResolvedValueOnce({ ok: false, error: new RemoteError('gateway/internal', 'query refused', {}) }) + f.face.refresh() + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) + expect(f.controller.state.getSnapshot()[key]).toEqual({ value: previous, busy: false, error: 'query refused' }) + expect(f.controller.state.getSnapshot().toast).toBeNull() +}) + +it.each(['requestPermission', 'openSettings', 'revoke', 'retryApproval'] as const)( + 'reports an unsuccessful %s response instead of a success notice', async (method) => { + const f = fixture() + onTestFinished(() => f.controller.dispose()) + f.actions[method].mockResolvedValueOnce({ ok: false, error: new RemoteError('gateway/internal', 'action refused', {}) }) + if (method === 'requestPermission') f.face.request('accessibility') + else if (method === 'openSettings') f.face.openSettings('accessibility') + else f.face[method]('a'.repeat(64) as CuaApplicationId) + await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) + expect(f.controller.state.getSnapshot().toast).toMatchObject({ kind: 'failed', message: 'action refused' }) + expect(f.actions.systemStatus).toHaveBeenCalledOnce() + }, +) From fdb0885ab0797d0bf5561efe66038b3678d9542b Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 21:13:41 +0800 Subject: [PATCH 45/55] test(cua): load the generated Remote type contribution in controller tests --- packages/experimental/cua/tests/controller.client.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/experimental/cua/tests/controller.client.spec.ts b/packages/experimental/cua/tests/controller.client.spec.ts index afcdb37c8a..59568eac9c 100644 --- a/packages/experimental/cua/tests/controller.client.spec.ts +++ b/packages/experimental/cua/tests/controller.client.spec.ts @@ -1,6 +1,7 @@ /** Permission refreshes preserve observations and join user actions before withdrawal. */ import { setImmediate } from 'node:timers/promises' import type { Context } from '@deepseek-ai/cordis' +import type {} from '@deepseek-ai/dsh-cua/remote' import { RemoteError } from '@deepseek-ai/dsh-typert-protocol' import { expect, it, onTestFinished, vi } from 'vitest' import type { CuaApplicationId, CuaApplicationState, CuaSystemPermissionState } from '../src/access-types.ts' From c2d384a4c181369f8f8133c6c39932a53673d5cf Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 21:29:50 +0800 Subject: [PATCH 46/55] style(cua): align permission controls and configured states --- .../experimental/cua/src/client/PermissionPanel.module.css | 5 +++-- packages/experimental/cua/src/client/PermissionPanel.tsx | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/experimental/cua/src/client/PermissionPanel.module.css b/packages/experimental/cua/src/client/PermissionPanel.module.css index 11ce22b9f7..d93d715e03 100644 --- a/packages/experimental/cua/src/client/PermissionPanel.module.css +++ b/packages/experimental/cua/src/client/PermissionPanel.module.css @@ -33,14 +33,15 @@ } .applicationBadge img { flex-shrink: 0; object-fit: contain; } .applicationBadge > span { min-width: 0; overflow-wrap: anywhere; } -.permissionAction { min-width: 120px; } +.permissionAction, .buttonPlaceholder { width: 168px; flex-shrink: 0; } +.permissionAction[data-configured]:disabled { opacity: 1; color: var(--dsw-alias-label-primary); cursor: default; } .buttonPlaceholder, .appPlaceholder, .appsPlaceholder { display: inline-block; max-width: 100%; border-radius: var(--dsw-radius-xs); background: var(--dsw-alias-bg-skeleton); } -.buttonPlaceholder { width: 120px; height: 28px; } +.buttonPlaceholder { height: 28px; } .appPlaceholder { width: 160px; height: 20px; } .appsPlaceholder { width: 240px; height: 40px; } diff --git a/packages/experimental/cua/src/client/PermissionPanel.tsx b/packages/experimental/cua/src/client/PermissionPanel.tsx index 525a313d05..d6d190f38b 100644 --- a/packages/experimental/cua/src/client/PermissionPanel.tsx +++ b/packages/experimental/cua/src/client/PermissionPanel.tsx @@ -62,7 +62,7 @@ function ConfigurationSetting({ kind, status, loading, writable, configured, sav

{t(`${kind}Title`)}

{t(`${kind}Body`)}

{loading ? - : }
{!configured && (status === 'unavailable' ?

{t(`${kind}Unavailable`)}

From 2e215f20f96e4881732c90d51432a262d072b16e Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 21:30:07 +0800 Subject: [PATCH 47/55] fix(cua): restrict native package delivery to macOS targets --- apps/desktop/README.i18n.yaml | 4 +- apps/desktop/README.md | 2 + apps/desktop/README.zh.md | 2 + apps/desktop/scripts/prepare-dsh.ts | 2 +- apps/desktop/scripts/prepare-package-set.ts | 34 ++++- .../tests/cua-optional-package.spec.ts | 118 ++++++++++++++++++ .../desktop/tests/prepare-package-set.spec.ts | 80 +++++++++++- .../experimental/cua-bundle/README.i18n.yaml | 4 +- packages/experimental/cua-bundle/README.md | 2 +- packages/experimental/cua-bundle/README.zh.md | 2 +- .../experimental/cua-bundle/cordis.patch.yml | 1 + packages/experimental/cua-bundle/package.json | 7 +- .../cua-bundle/tests/platform.spec.ts | 34 +++++ .../experimental/cua-macos/README.i18n.yaml | 4 +- packages/experimental/cua-macos/README.md | 2 + packages/experimental/cua-macos/README.zh.md | 2 + .../experimental/cua-macos/native/.npmignore | 3 + packages/experimental/cua-macos/package.json | 5 +- pnpm-lock.yaml | 10 +- scripts/verify-cordis-config.spec.ts | 5 +- scripts/verify-cordis-config.ts | 4 +- 21 files changed, 298 insertions(+), 29 deletions(-) create mode 100644 apps/desktop/tests/cua-optional-package.spec.ts create mode 100644 packages/experimental/cua-bundle/tests/platform.spec.ts create mode 100644 packages/experimental/cua-macos/native/.npmignore diff --git a/apps/desktop/README.i18n.yaml b/apps/desktop/README.i18n.yaml index 6060f79da9..bab17a6021 100644 --- a/apps/desktop/README.i18n.yaml +++ b/apps/desktop/README.i18n.yaml @@ -42,8 +42,8 @@ en: 98cf848762482d48 zh: 74f9100a39ec8c2f /deepseek-harness-desktop/develop: - en: fe6d23e6a3d3bc66 - zh: d60c8d3a55747916 + en: faf7676ba18bbf2f + zh: f1d7b75ad348c548 /deepseek-harness-desktop/develop/startup-onboarding: en: a51346b8d880ddba zh: 7913214c1c66063c diff --git a/apps/desktop/README.md b/apps/desktop/README.md index e0bb92c5b7..f00a0dc2bf 100644 --- a/apps/desktop/README.md +++ b/apps/desktop/README.md @@ -202,6 +202,8 @@ After an explicit build, `start:desktop` reconstructs the disposable project and pnpm run start:desktop ``` +Desktop package preparation filters optional native packages by the installer target’s OS and architecture. Windows package sets omit the macOS Computer Use tarball, including when their inputs were packed on macOS. + The Web counterparts are `pnpm run dev:web` and `pnpm run start:web`, documented in the [development guide](../../docs/development.md). Workspace development runs the current CLI and private Desktop Host packages under Electron RunAsNode. Plugin management and recovery use `$DSH_HOME/profiles/desktop`, separate from the disposable workspace runtime. The Host uses runtime module resolution in both development and packaged builds without creating official-package fallback links; developer-installed packages, including links, retain native priority. Use an unpacked application to exercise Electron RunAsNode, bundled pnpm, bundled dsh resources, plugin installation and repair paths. The [native/renderer keyboard tests](tests/keyboard.spec.ts) compile as part of the repository Client typecheck. Their Desktop imports are limited to Cordis-free input, persistence, IPC, browser-guest, and overlay modules. diff --git a/apps/desktop/README.zh.md b/apps/desktop/README.zh.md index 430ac4ad3d..4dfdeebc0f 100644 --- a/apps/desktop/README.zh.md +++ b/apps/desktop/README.zh.md @@ -202,6 +202,8 @@ pnpm run dev:desktop pnpm run start:desktop ``` +桌面包准备过程按安装包目标的操作系统和架构筛选可选原生包。Windows 的包集合不包含 macOS Computer Use tarball，即使输入包是在 macOS 上打包的。 + Web 侧的对应命令是 `pnpm run dev:web` 与 `pnpm run start:web`，见[开发指南](../../docs/development.zh.md)。Workspace 开发使用 Electron RunAsNode 运行当前 CLI 与私有 Desktop Host 包，插件管理和恢复使用 `$DSH_HOME/profiles/desktop`，与一次性工作区运行时分离。Host 在开发与打包构建中都使用 runtime 模块解析，不创建官方包的 fallback 链接；开发者安装的包（包括链接）保留原生优先级。需要验证 Electron RunAsNode、内置 pnpm、内置 dsh 资源、插件安装和修复时，应运行未封装安装器的应用目录。 [原生输入与渲染进程键盘测试](tests/keyboard.spec.ts)直接纳入仓库 Client 类型检查。它只导入不依赖 Cordis 的 Desktop 输入、持久化、IPC、浏览器 guest 和蒙层模块。 diff --git a/apps/desktop/scripts/prepare-dsh.ts b/apps/desktop/scripts/prepare-dsh.ts index 9de39d3ff2..365c5a45f2 100644 --- a/apps/desktop/scripts/prepare-dsh.ts +++ b/apps/desktop/scripts/prepare-dsh.ts @@ -134,7 +134,7 @@ async function main(): Promise { await packagingStep(process.env.DSH_DESKTOP_PACKAGING_RUN_DIR, 'runtime:install', () => runPnpm(['install', '--prod', '--frozen-lockfile', '--trust-lockfile'])) const packageSet = readDesktopCorePackageSet(BUILD_ROOT, release.version) const targetName = resolveDesktopBuildTarget() - const target = { platform: process.platform, arch: desktopTargetPlatform(targetName).arch } + const target = desktopTargetPlatform(targetName) const modules = join(BUILD_ROOT, 'node_modules') const officeManifest = JSON.parse(readFileSync(join(modules, '@deepseek-ai/libreoffice-kit/package.json'), 'utf8')) const officeEngine = selectOfficeEngine(officeManifest, target) diff --git a/apps/desktop/scripts/prepare-package-set.ts b/apps/desktop/scripts/prepare-package-set.ts index 7e7d62eea2..21df315715 100644 --- a/apps/desktop/scripts/prepare-package-set.ts +++ b/apps/desktop/scripts/prepare-package-set.ts @@ -25,7 +25,7 @@ import { } from '../src/core-package-set.ts' import { capture } from '../../../scripts/release/process.ts' import { tarballFiles } from '../../../scripts/release/tarball.ts' -import { resolveDesktopTargetBuildPaths } from './desktop-build-paths.mjs' +import { desktopTargetPlatform, resolveDesktopBuildTarget, resolveDesktopTargetBuildPaths } from './desktop-build-paths.mjs' const DSH_PACKAGE = '@deepseek-ai/dsh' const ROOT_PACKAGES = [DSH_PACKAGE, DESKTOP_HOST_PACKAGE] as const @@ -41,6 +41,17 @@ export interface PackedDesktopPackage { readonly manifest: Readonly> } +function supportsTarget(manifest: Readonly>, field: 'os' | 'cpu', target: string): boolean { + const values = manifest[field] + if (values === undefined) return true + if (!Array.isArray(values) || !values.every((value): value is string => typeof value === 'string')) { + throw new Error(`desktop package set: ${String(manifest.name)} has invalid ${field}`) + } + if (values.includes(`!${target}`)) return false + return values.length === 0 || values.includes('any') || values.includes(target) + || values.every(value => value.startsWith('!')) +} + function dependencyNames(manifest: Readonly>, section: string): string[] { const value = manifest[section] if (value === undefined) return [] @@ -54,10 +65,12 @@ function dependencyNames(manifest: Readonly>, section: s * Select workspace dependencies rooted at dsh and its private Host; npm resolves external packages. * Reads the repository workspace manifest and package manifests to distinguish required local packages from npm-resolved externals. * @param available - Packed packages indexed by package name. + * @param target - Platform and architecture of the installer payload. * @returns Selected packages sorted by name. */ export function selectDesktopPackageClosure( available: ReadonlyMap, + target: ReturnType, ): PackedDesktopPackage[] { const workspace = yaml.load(readFileSync(join(REPOSITORY_ROOT, 'pnpm-workspace.yaml'), 'utf8')) as { packages: string[] } const workspaceNames = new Set(globSync(workspace.packages.map(pattern => `${pattern}/package.json`), { cwd: REPOSITORY_ROOT }) @@ -77,7 +90,9 @@ export function selectDesktopPackageClosure( } } for (const dependency of dependencyNames(packed.manifest, OPTIONAL_DEPENDENCY_SECTION)) { - if (available.has(dependency)) visit(dependency) + const optional = available.get(dependency) + if (optional && supportsTarget(optional.manifest, 'os', target.platform) + && supportsTarget(optional.manifest, 'cpu', target.arch)) visit(dependency) } } for (const name of ROOT_PACKAGES) { @@ -127,9 +142,16 @@ export function assertDesktopHostPackageFiles(files: readonly string[]): void { } } -/** Prepare a package set from release tarball directories. */ -export function prepareDesktopPackageSet(inputs: readonly string[], output: string): void { - const selected = selectDesktopPackageClosure(packedPackages(inputs)) +/** + * Prepare a package set from release tarball directories. + * @param inputs - Release tarball directories. + * @param output - Destination package-set directory. + * @param target - Platform and architecture of the installer payload. + */ +export function prepareDesktopPackageSet( + inputs: readonly string[], output: string, target: ReturnType, +): void { + const selected = selectDesktopPackageClosure(packedPackages(inputs), target) const host = selected.find(packed => packed.manifest.name === DESKTOP_HOST_PACKAGE) if (host === undefined) throw new Error(`desktop package set: selected closure omits ${DESKTOP_HOST_PACKAGE}`) assertDesktopHostPackageFiles(tarballFiles(host.tarball)) @@ -171,7 +193,7 @@ function main(): void { }) const inputs = (values.from ?? defaultInputs).map(path => resolve(REPOSITORY_ROOT, path)) const output = values.out === undefined ? buildPaths.packageSet : resolve(REPOSITORY_ROOT, values.out) - prepareDesktopPackageSet(inputs, output) + prepareDesktopPackageSet(inputs, output, desktopTargetPlatform(resolveDesktopBuildTarget())) console.log(`desktop package set: prepared ${output}`) } diff --git a/apps/desktop/tests/cua-optional-package.spec.ts b/apps/desktop/tests/cua-optional-package.spec.ts new file mode 100644 index 0000000000..28ea98b403 --- /dev/null +++ b/apps/desktop/tests/cua-optional-package.spec.ts @@ -0,0 +1,118 @@ +/** Cold pnpm requests exercise shipped CUA dependency/platform metadata with inert package contents. */ +import { createHash } from 'node:crypto' +import { createServer } from 'node:http' +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { execa } from 'execa' +import { expect, it, onTestFinished } from 'vitest' +import { pnpmCommand } from '../../../scripts/release/process.ts' + +const repository = resolve(import.meta.dirname, '../../..') +const providerName = '@deepseek-ai/dsh-experimental-cua-macos' +const bundleName = '@deepseek-ai/dsh-cua-bundle' +const readManifest = async (path: string): Promise> => + JSON.parse(await readFile(join(repository, path, 'package.json'), 'utf8')) + +it('fetches the macOS optional tarball only for Darwin, with an empty cache for every target', async () => { + const root = await mkdtemp(join(tmpdir(), 'cua-cold-install-')) + onTestFinished(() => rm(root, { recursive: true, force: true })) + const userconfig = join(root, 'npmrc') + await writeFile(userconfig, '') + const environment = Object.fromEntries(Object.entries(process.env).filter(([key]) => + !/KEY|SECRET|TOKEN|PASSWORD/iu.test(key) && !/^(npm|pnpm|corepack)_/iu.test(key) + && key !== 'NODE_OPTIONS' && key !== 'NODE_PATH')) + const [pnpm, ...prefix] = pnpmCommand() + const command = async (args: string[], cwd: string) => { + const result = await execa(pnpm, [...prefix, `--config.userconfig=${userconfig}`, ...args], { + cwd, env: { ...environment, CI: 'true' }, extendEnv: false, reject: false, timeout: 60_000, + }) + expect(result.timedOut, result.stderr).toBe(false) + expect(result.signal, result.stderr).toBeUndefined() + expect(result.exitCode, `${result.stdout}\n${result.stderr}`).toBe(0) + return result.stdout + } + const cli = await readManifest('apps/cli') + const bundle = await readManifest('packages/experimental/cua-bundle') + const native = await readManifest('packages/experimental/cua-macos') + expect(cli.dependencies).toHaveProperty(bundleName) + expect(bundle.dependencies).not.toHaveProperty(providerName) + expect(bundle.optionalDependencies).toEqual({ [providerName]: 'workspace:*' }) + expect(native.os).toEqual(['darwin']) + const version = '1.0.0' + const bundleDependencies = Object.fromEntries(Object.keys(bundle.dependencies as Record).map(name => [name, version])) + const manifests = [ + { name: cli.name, version, dependencies: { [bundleName]: version } }, + { name: bundleName, version, dependencies: bundleDependencies, optionalDependencies: { [providerName]: version } }, + ...Object.keys(bundleDependencies).map(name => ({ name, version })), + { name: providerName, version, os: native.os, files: native.files }, + ] + const packages = new Map; bytes: Buffer }>() + for (const manifest of manifests) { + const name = String(manifest.name) + const directory = join(root, name.replace('@deepseek-ai/', '')) + await mkdir(directory) + await writeFile(join(directory, 'package.json'), JSON.stringify(manifest)) + if (name === providerName) { + await mkdir(join(directory, 'native/bin/darwin-arm64'), { recursive: true }) + for (const file of ['.gitignore', '.npmignore']) { + await writeFile(join(directory, 'native', file), await readFile(join(repository, 'packages/experimental/cua-macos/native', file))) + } + await writeFile(join(directory, 'native/bin/darwin-arm64/dsh-cua-native'), 'inert native payload') + } + await command(['pack', '--out', 'package.tgz'], directory) + const bytes = await readFile(join(directory, 'package.tgz')) + packages.set(name, { manifest, bytes }) + if (name === providerName) { + const result = await execa('tar', ['-tzf', join(directory, 'package.tgz')]) + expect(result.stdout).toContain('package/native/bin/darwin-arm64/dsh-cua-native') + } + } + const requests: string[] = [] + let omitOs = false + const server = createServer((request, response) => { + const path = decodeURIComponent(request.url ?? '') + requests.push(path) + const tarball = path.endsWith('.tgz') + const name = path.slice(1).replace(/\.tgz$/u, '') + const entry = packages.get(name) + if (!entry) { response.writeHead(404); response.end(); return } + if (tarball) { response.end(entry.bytes); return } + const manifest = { ...entry.manifest } + if (omitOs) delete manifest.os + response.setHeader('Content-Type', 'application/json') + response.end(JSON.stringify({ name, 'dist-tags': { latest: version }, versions: { + [version]: { ...manifest, dist: { + tarball: `http://${request.headers.host}/${name}.tgz`, + integrity: `sha512-${createHash('sha512').update(entry.bytes).digest('base64')}`, + } }, + } })) + }) + onTestFinished(async () => { + server.closeAllConnections() + await new Promise((resolve, reject) => { server.close(error => error ? reject(error) : resolve()) }) + }) + await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve) }) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('fixture registry has no TCP address') + for (const target of ['darwin', 'win32', 'linux', 'linux-without-os']) { + omitOs = target === 'linux-without-os' + const project = join(root, target) + await mkdir(project) + await writeFile(join(project, 'package.json'), JSON.stringify({ private: true, dependencies: { [String(cli.name)]: version } })) + await writeFile(join(project, 'pnpm-workspace.yaml'), [ + 'autoInstallPeers: false', 'supportedArchitectures:', ` os: [${omitOs ? 'linux' : target}]`, + ' cpu: [arm64]', 'minimumReleaseAge: 0', '', + ].join('\n')) + requests.length = 0 + await command([ + 'install', '--ignore-scripts', '--config.enable-global-virtual-store=false', + `--config.registry=http://127.0.0.1:${address.port}/`, `--config.store-dir=${join(project, 'store')}`, + `--config.cache-dir=${join(project, 'cache')}`, `--config.state-dir=${join(project, 'state')}`, + ], project) + expect(requests).toContain(`/${bundleName}.tgz`) + expect(requests).toContain('/@deepseek-ai/dsh-cua.tgz') + expect(requests.includes(`/${providerName}.tgz`), `${target}: ${JSON.stringify(requests)}`) + .toBe(target === 'darwin' || omitOs) + } +}, 120_000) diff --git a/apps/desktop/tests/prepare-package-set.spec.ts b/apps/desktop/tests/prepare-package-set.spec.ts index 6db1ccbd01..c81d0c694c 100644 --- a/apps/desktop/tests/prepare-package-set.spec.ts +++ b/apps/desktop/tests/prepare-package-set.spec.ts @@ -1,6 +1,12 @@ -import { afterEach, describe, expect, it, vi } from 'vitest' +import { afterEach, describe, expect, it, onTestFinished, vi } from 'vitest' +import { execFileSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { DESKTOP_HOST_RUNTIME_FILES, DESKTOP_PACKAGES_DIR } from '../src/core-package-set.ts' import { assertDesktopHostPackageFiles, + prepareDesktopPackageSet, selectDesktopPackageClosure, type PackedDesktopPackage, } from '../scripts/prepare-package-set.ts' @@ -37,7 +43,7 @@ describe('desktop package-set selection', () => { ['@deepseek-ai/platform-package', packed('@deepseek-ai/platform-package')], ['@deepseek-ai/unused', packed('@deepseek-ai/unused')], ]) - expect(selectDesktopPackageClosure(available).map(entry => entry.manifest.name)).toEqual([ + expect(selectDesktopPackageClosure(available, { platform: 'win32', arch: 'x64' }).map(entry => entry.manifest.name)).toEqual([ '@deepseek-ai/cordis', '@deepseek-ai/dsh', '@deepseek-ai/dsh-base', @@ -57,10 +63,10 @@ describe('desktop package-set selection', () => { dependencies: { '@deepseek-ai/dsh': '^1.0.0' }, })], ]) - expect(() => selectDesktopPackageClosure(available)).toThrow(/unpacked package/u) + expect(() => selectDesktopPackageClosure(available, { platform: 'win32', arch: 'x64' })).toThrow(/unpacked package/u) expect(() => selectDesktopPackageClosure(new Map([ ['@deepseek-ai/dsh', packed('@deepseek-ai/dsh')], - ]))).toThrow(/omit @deepseek-ai\/dsh-desktop-host/u) + ]), { platform: 'win32', arch: 'x64' })).toThrow(/omit @deepseek-ai\/dsh-desktop-host/u) }) it('leaves independently published Office packages to npm resolution', () => { @@ -73,7 +79,7 @@ describe('desktop package-set selection', () => { })], ['@deepseek-ai/dsh-desktop-host', packed('@deepseek-ai/dsh-desktop-host')], ]) - expect(selectDesktopPackageClosure(available).map(entry => entry.manifest.name)).toEqual([ + expect(selectDesktopPackageClosure(available, { platform: 'win32', arch: 'x64' }).map(entry => entry.manifest.name)).toEqual([ '@deepseek-ai/dsh', '@deepseek-ai/dsh-desktop-host', ]) }) @@ -92,3 +98,67 @@ describe('desktop package-set selection', () => { expect(() => { assertDesktopHostPackageFiles(files.slice(0, 1)) }).toThrow(/lib\/cli\.js/u) }) }) + +const macProvider = '@deepseek-ai/dsh-experimental-cua-macos' +const cuaBundle = '@deepseek-ai/dsh-cua-bundle' +const targetManifests = () => new Map([ + ['@deepseek-ai/dsh', packed('@deepseek-ai/dsh', { dependencies: { [cuaBundle]: '1.0.0' } })], + ['@deepseek-ai/dsh-desktop-host', packed('@deepseek-ai/dsh-desktop-host')], + [cuaBundle, packed(cuaBundle, { + optionalDependencies: { [macProvider]: '1.0.0' }, dependencies: { '@deepseek-ai/dsh-cua': '1.0.0' }, + })], + ['@deepseek-ai/dsh-cua', packed('@deepseek-ai/dsh-cua')], + [macProvider, packed(macProvider, { os: ['darwin'] })], +]) + +it.each([ + ['darwin', 'arm64', true], ['darwin', 'x64', true], ['win32', 'x64', false], +] as const)('selects the CUA native tarball for target %s/%s independently of the host', (platform, arch, included) => { + const selected = selectDesktopPackageClosure(targetManifests(), { platform, arch }).map(entry => entry.manifest.name) + expect(selected.includes(macProvider)).toBe(included) + expect(selected).toContain('@deepseek-ai/dsh-cua') + expect(selected).toContain(cuaBundle) +}) + +it.each([ + [{ cpu: ['arm64'] }, false], [{ os: ['!win32'] }, false], [{ os: ['!linux'] }, true], + [{ os: [] }, true], [{ os: ['any'] }, true], [{ os: ['any', '!win32'] }, false], + [{ cpu: ['x64'] }, true], +])('honors optional package platform constraints %j', (constraints, included) => { + const packages = targetManifests() + packages.set(macProvider, packed(macProvider, constraints)) + expect(selectDesktopPackageClosure(packages, { platform: 'win32', arch: 'x64' }) + .some(entry => entry.manifest.name === macProvider)).toBe(included) +}) + +it.each([{ os: 'darwin' }, { cpu: [42] }])('rejects malformed platform constraints %j', (constraints) => { + const packages = targetManifests() + packages.set(macProvider, packed(macProvider, constraints)) + expect(() => selectDesktopPackageClosure(packages, { platform: 'win32', arch: 'x64' })).toThrow(/has invalid/u) +}) + +it('copies no macOS tarball into the Windows package set', () => { + const root = mkdtempSync(join(tmpdir(), 'desktop-platform-packages-')) + onTestFinished(() => rmSync(root, { recursive: true, force: true })) + const inputs = join(root, 'inputs') + mkdirSync(inputs) + for (const [name, entry] of targetManifests()) { + const stem = name.replace('@deepseek-ai/', '') + const folder = join(root, stem, 'package') + mkdirSync(folder, { recursive: true }) + writeFileSync(join(folder, 'package.json'), JSON.stringify(entry.manifest)) + if (name === '@deepseek-ai/dsh-desktop-host') { + for (const file of DESKTOP_HOST_RUNTIME_FILES) { + mkdirSync(join(folder, file, '..'), { recursive: true }) + writeFileSync(join(folder, file), '') + } + } + execFileSync('tar', ['-czf', join(inputs, `${stem}.tgz`), '-C', join(folder, '..'), 'package']) + } + const output = join(root, 'output') + prepareDesktopPackageSet([inputs], output, { platform: 'win32', arch: 'x64' }) + expect(readdirSync(join(output, DESKTOP_PACKAGES_DIR))).not.toContain('dsh-experimental-cua-macos.tgz') + prepareDesktopPackageSet([inputs], output, { platform: 'darwin', arch: 'arm64' }) + expect(readFileSync(join(output, DESKTOP_PACKAGES_DIR, 'dsh-experimental-cua-macos.tgz'))) + .toEqual(readFileSync(join(inputs, 'dsh-experimental-cua-macos.tgz'))) +}) diff --git a/packages/experimental/cua-bundle/README.i18n.yaml b/packages/experimental/cua-bundle/README.i18n.yaml index a2b0c1f5fe..f2f7091bbf 100644 --- a/packages/experimental/cua-bundle/README.i18n.yaml +++ b/packages/experimental/cua-bundle/README.i18n.yaml @@ -15,8 +15,8 @@ en: b563e4ec4727d0de zh: d0228da978238e05 /deepseek-ai-dsh-cua-bundle/use-this-package: - en: d171cfb240ae4503 - zh: b65e31e613f7b000 + en: 32932a00acbfe4b6 + zh: 34b6fa6933e4e8c9 /deepseek-ai-dsh-cua-bundle/understand-the-implementation: en: 057771fe13aaa337 zh: 4fdb2b98ff3e7a78 diff --git a/packages/experimental/cua-bundle/README.md b/packages/experimental/cua-bundle/README.md index fc74a014cd..40e17fd90b 100644 --- a/packages/experimental/cua-bundle/README.md +++ b/packages/experimental/cua-bundle/README.md @@ -37,7 +37,7 @@ The switch saves bundle selection in the profile’s `package.json`. Computer Us Disabling unloads the provider and withdraws CUA tools from its Agent scopes. Runtime configuration, the new-session default and saved grants remain. Session-only application decisions end with provider disposal. -Computer Use requires a macOS DSH Host. Other operating systems can display the entry, but native provider startup fails. A browser on another operating system can connect to a supported Mac Host. +Computer Use requires a macOS DSH Host. On other operating systems, installation omits the optional macOS provider and the bundle skips its Loader entry. Shared CUA logic and the permission-page entry remain available, but Computer Use operations require the provider. A browser on another operating system can connect to a supported Mac Host. If preparation fails, the saved selection remains and CUA operations stay unavailable. Read the Host startup diagnostic, correct its cause, then restart DSH. diff --git a/packages/experimental/cua-bundle/README.zh.md b/packages/experimental/cua-bundle/README.zh.md index febad6d82b..96d7c1076c 100644 --- a/packages/experimental/cua-bundle/README.zh.md +++ b/packages/experimental/cua-bundle/README.zh.md @@ -37,7 +37,7 @@ pnpm dsh web 关闭会卸载提供方，并从其 Agent 作用域中撤回 CUA 工具。执行器配置、新会话默认值和已保存的授权保留。会话级应用授权随提供方卸载而失效。 -Computer Use 要求 DSH Host 运行在 macOS 上。其他系统可以显示入口，但原生提供方启动会失败。其他操作系统上的浏览器可以连接受支持的 Mac Host。 +Computer Use 要求 DSH Host 运行在 macOS 上。其他系统安装时跳过可选的 macOS 提供方，组合包也不加载该提供方。共享 CUA 逻辑和权限页入口仍保留，Computer Use 操作需要提供方才能使用。其他操作系统上的浏览器可以连接受支持的 Mac Host。 准备失败时，已保存的选择保留，CUA 操作不可用。查看 Host 的启动诊断，修复原因后重启 DSH。 diff --git a/packages/experimental/cua-bundle/cordis.patch.yml b/packages/experimental/cua-bundle/cordis.patch.yml index 99af9b3ccb..5d5fb48b6f 100644 --- a/packages/experimental/cua-bundle/cordis.patch.yml +++ b/packages/experimental/cua-bundle/cordis.patch.yml @@ -3,6 +3,7 @@ name: '@deepseek-ai/dsh-computer-use' - id: cua name: '@deepseek-ai/dsh-experimental-cua-macos' + disabled: !!js process.platform !== 'darwin' config: agentPreset: ptc - id: ui-cua diff --git a/packages/experimental/cua-bundle/package.json b/packages/experimental/cua-bundle/package.json index 546c66db40..d4e4438b46 100644 --- a/packages/experimental/cua-bundle/package.json +++ b/packages/experimental/cua-bundle/package.json @@ -38,7 +38,6 @@ } }, "dependencies": { - "@deepseek-ai/dsh-experimental-cua-macos": "workspace:*", "@deepseek-ai/dsh-computer-use": "workspace:*", "@deepseek-ai/dsh-cua": "workspace:*" }, @@ -48,6 +47,10 @@ "devDependencies": { "@deepseek-ai/cordis": "workspace:~", "@deepseek-ai/cordis-plugin-include": "workspace:~", - "js-yaml": "^4.2.0" + "js-yaml": "^4.2.0", + "@deepseek-ai/cordis-plugin-loader": "workspace:~" + }, + "optionalDependencies": { + "@deepseek-ai/dsh-experimental-cua-macos": "workspace:*" } } diff --git a/packages/experimental/cua-bundle/tests/platform.spec.ts b/packages/experimental/cua-bundle/tests/platform.spec.ts new file mode 100644 index 0000000000..4b3fd66138 --- /dev/null +++ b/packages/experimental/cua-bundle/tests/platform.spec.ts @@ -0,0 +1,34 @@ +/** The shipped patch leaves shared entries available while skipping an absent native provider. */ +import { readFile } from 'node:fs/promises' +import { Context } from '@deepseek-ai/cordis' +import Loader, { interpolate, type EntryOptions } from '@deepseek-ai/cordis-plugin-loader' +import { entryListSchema } from '@deepseek-ai/cordis-plugin-include' +import { load } from 'js-yaml' +import { expect, it, vi } from 'vitest' + +it.each(['darwin', 'win32', 'linux'])('loads the native provider only on Darwin for %s', async (platform) => { + const patch = load(await readFile(new URL('../cordis.patch.yml', import.meta.url), 'utf8'), { + schema: entryListSchema, + }) as { insert: EntryOptions[] }[] + const ctx = new Context() + await ctx.plugin(Loader) + const imported = vi.spyOn(ctx.loader.internal, 'import').mockImplementation(async (name) => { + if (platform !== 'darwin' && name === '@deepseek-ai/dsh-experimental-cua-macos') { + throw new Error('native provider is absent from this installation') + } + return { name, apply: () => {} } + }) + try { + for (const row of patch[0]!.insert) { + await ctx.loader.create({ ...row, disabled: interpolate({ process: { platform } }, row.disabled) }) + } + await ctx.loader.await() + const imports = imported.mock.calls.map(call => call[0]) + expect(imports).toContain('@deepseek-ai/dsh-cua') + expect(imports).toContain('@deepseek-ai/dsh-computer-use') + expect(imports.includes('@deepseek-ai/dsh-experimental-cua-macos')).toBe(platform === 'darwin') + } finally { + await ctx.fiber.dispose() + imported.mockRestore() + } +}) diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index e7d55edf7c..c2cb1bfdfa 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -15,8 +15,8 @@ en: 9353d17b05f2f88c zh: 6b2c6b4ee0046294 /deepseek-ai-dsh-experimental-cua-macos/use-this-package: - en: 107cc4d4e72e248d - zh: 1aec124d4018d2a4 + en: 7126e254aaf42772 + zh: fa05ac2346e022ff /deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment: en: e7909c9b43e44fbc zh: 98949877d4ea3a86 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index a91c37b885..5a15eb8dbe 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -29,6 +29,8 @@ The shared helper queries the process macOS considers responsible for it and cap ## Use this package +This package is an optional dependency restricted to macOS. Prepare the matching executable under `native/bin` before packing; that directory is included in the archive. Installed packages require the prepared helper and do not compile Swift. + For repository Web use, enable [Computer Use](../cua-bundle/README.md) in Plugins. This prepares the local executable on macOS 14 or later; Windows support is in development. Subsequent builds check it while the existing Web profile keeps the bundle selected. Disabled profiles do not prepare native components. Explicit source composition is also available: ```sh diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index 03ff814e8a..223079a130 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -29,6 +29,8 @@ helper 为权限页的应用列表提供 64 像素 PNG 图标。读取图标不 ## 使用本包 +本包是仅限 macOS 的可选依赖。打包前需在 `native/bin` 下准备对应架构的可执行文件，该目录会包含在发布包中。安装版需要准备好的 helper，不会编译 Swift。 + 从仓库运行 Web 时，在插件页开启 [Computer Use](../cua-bundle/README.zh.md)。首次启用会在 macOS 14 及以上准备本机组件，Windows 支持正在开发中。现有 Web profile 保持选中时，后续构建继续检查更新。未开启的 profile 不准备原生组件。也可以显式加载源码覆盖层： ```sh diff --git a/packages/experimental/cua-macos/native/.npmignore b/packages/experimental/cua-macos/native/.npmignore new file mode 100644 index 0000000000..3168a3629c --- /dev/null +++ b/packages/experimental/cua-macos/native/.npmignore @@ -0,0 +1,3 @@ +target/ +.build/ +.swiftpm/ diff --git a/packages/experimental/cua-macos/package.json b/packages/experimental/cua-macos/package.json index 03d3f568e2..a2cfd494dd 100644 --- a/packages/experimental/cua-macos/package.json +++ b/packages/experimental/cua-macos/package.json @@ -84,5 +84,8 @@ }, "scripts": { "build:native": "swift build --package-path native --configuration release" - } + }, + "os": [ + "darwin" + ] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 06583be24f..552891e818 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8658,9 +8658,6 @@ importers: '@deepseek-ai/dsh-cua': specifier: workspace:* version: link:../cua - '@deepseek-ai/dsh-experimental-cua-macos': - specifier: workspace:* - version: link:../cua-macos devDependencies: '@deepseek-ai/cordis': specifier: workspace:~ @@ -8668,9 +8665,16 @@ importers: '@deepseek-ai/cordis-plugin-include': specifier: workspace:~ version: link:../../../vendor/include + '@deepseek-ai/cordis-plugin-loader': + specifier: workspace:~ + version: link:../../../vendor/loader js-yaml: specifier: ^4.2.0 version: 4.3.2 + optionalDependencies: + '@deepseek-ai/dsh-experimental-cua-macos': + specifier: workspace:* + version: link:../cua-macos packages/experimental/cua-macos: dependencies: diff --git a/scripts/verify-cordis-config.spec.ts b/scripts/verify-cordis-config.spec.ts index ec0385af10..8c7209e86d 100644 --- a/scripts/verify-cordis-config.spec.ts +++ b/scripts/verify-cordis-config.spec.ts @@ -80,11 +80,14 @@ describe('workspace Bundle discovery and product dependency closures', () => { } const self = { file, name: '@deepseek-ai/dsh-subagent-example' } expect(bundlePluginDependencyErrors(manifestPath, manifest, [self])).toEqual([]) + expect(bundlePluginDependencyErrors(manifestPath, { + ...manifest, optionalDependencies: { '@deepseek-ai/dsh-platform-plugin': 'workspace:*' }, + }, [{ file, name: '@deepseek-ai/dsh-platform-plugin' }])).toEqual([]) expect(bundlePluginDependencyErrors(manifestPath, manifest, [ self, { file, name: '@deepseek-ai/dsh-missing-plugin' }, ])).toEqual([ - `${file}: @deepseek-ai/dsh-missing-plugin must be declared in ${manifestPath} dependencies`, + `${file}: @deepseek-ai/dsh-missing-plugin must be declared in ${manifestPath} dependencies or optionalDependencies`, ]) }) }) diff --git a/scripts/verify-cordis-config.ts b/scripts/verify-cordis-config.ts index 76e9bdba87..3de9914cdf 100644 --- a/scripts/verify-cordis-config.ts +++ b/scripts/verify-cordis-config.ts @@ -396,8 +396,8 @@ export function bundlePluginDependencyErrors( return missingPluginDependencies( // A Bundle may mount its own package (for example, its provider or runtime row). references.filter(reference => packageNameFromSpecifier(reference.name) !== manifest.name), - manifest.dependencies ?? {}, - `${manifestPath} dependencies`, + { ...manifest.dependencies, ...manifest.optionalDependencies }, + `${manifestPath} dependencies or optionalDependencies`, ) } From 587ee178538220103a360aa4a207732d1e6ec961 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 21:31:32 +0800 Subject: [PATCH 48/55] test(cua): isolate package manager caches and narrow Loader importer --- apps/desktop/tests/cua-optional-package.spec.ts | 8 +++++--- packages/experimental/cua-bundle/tests/platform.spec.ts | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/desktop/tests/cua-optional-package.spec.ts b/apps/desktop/tests/cua-optional-package.spec.ts index 28ea98b403..c1eecfbd98 100644 --- a/apps/desktop/tests/cua-optional-package.spec.ts +++ b/apps/desktop/tests/cua-optional-package.spec.ts @@ -25,7 +25,9 @@ it('fetches the macOS optional tarball only for Darwin, with an empty cache for const [pnpm, ...prefix] = pnpmCommand() const command = async (args: string[], cwd: string) => { const result = await execa(pnpm, [...prefix, `--config.userconfig=${userconfig}`, ...args], { - cwd, env: { ...environment, CI: 'true' }, extendEnv: false, reject: false, timeout: 60_000, + cwd, env: { ...environment, CI: 'true', XDG_CACHE_HOME: join(cwd, 'xdg-cache'), + XDG_DATA_HOME: join(cwd, 'xdg-data'), XDG_STATE_HOME: join(cwd, 'xdg-state'), + XDG_CONFIG_HOME: join(cwd, 'xdg-config') }, extendEnv: false, reject: false, timeout: 60_000, }) expect(result.timedOut, result.stderr).toBe(false) expect(result.signal, result.stderr).toBeUndefined() @@ -105,14 +107,14 @@ it('fetches the macOS optional tarball only for Darwin, with an empty cache for ' cpu: [arm64]', 'minimumReleaseAge: 0', '', ].join('\n')) requests.length = 0 - await command([ + const output = await command([ 'install', '--ignore-scripts', '--config.enable-global-virtual-store=false', `--config.registry=http://127.0.0.1:${address.port}/`, `--config.store-dir=${join(project, 'store')}`, `--config.cache-dir=${join(project, 'cache')}`, `--config.state-dir=${join(project, 'state')}`, ], project) expect(requests).toContain(`/${bundleName}.tgz`) expect(requests).toContain('/@deepseek-ai/dsh-cua.tgz') - expect(requests.includes(`/${providerName}.tgz`), `${target}: ${JSON.stringify(requests)}`) + expect(requests.includes(`/${providerName}.tgz`), `${target}: ${JSON.stringify(requests)}\n${output}`) .toBe(target === 'darwin' || omitOs) } }, 120_000) diff --git a/packages/experimental/cua-bundle/tests/platform.spec.ts b/packages/experimental/cua-bundle/tests/platform.spec.ts index 4b3fd66138..e899f09b3d 100644 --- a/packages/experimental/cua-bundle/tests/platform.spec.ts +++ b/packages/experimental/cua-bundle/tests/platform.spec.ts @@ -12,6 +12,7 @@ it.each(['darwin', 'win32', 'linux'])('loads the native provider only on Darwin }) as { insert: EntryOptions[] }[] const ctx = new Context() await ctx.plugin(Loader) + if (!ctx.loader.internal) throw new Error('Loader module importer is unavailable') const imported = vi.spyOn(ctx.loader.internal, 'import').mockImplementation(async (name) => { if (platform !== 'darwin' && name === '@deepseek-ai/dsh-experimental-cua-macos') { throw new Error('native provider is absent from this installation') From 35ba8ad66d3edef37bf4a9d5e72ce30389930602 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 21:33:04 +0800 Subject: [PATCH 49/55] test(cua): cover platform inference and disabled native provider in preview --- apps/desktop/scripts/prepare-package-set.ts | 2 +- apps/desktop/tests/cua-optional-package.spec.ts | 13 +++++++------ apps/web/tests/preview-boot.e2e.ts | 10 ++++++---- .../experimental/cua-bundle/tests/platform.spec.ts | 2 +- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/apps/desktop/scripts/prepare-package-set.ts b/apps/desktop/scripts/prepare-package-set.ts index 21df315715..4cffa7018b 100644 --- a/apps/desktop/scripts/prepare-package-set.ts +++ b/apps/desktop/scripts/prepare-package-set.ts @@ -48,7 +48,7 @@ function supportsTarget(manifest: Readonly>, field: 'os' throw new Error(`desktop package set: ${String(manifest.name)} has invalid ${field}`) } if (values.includes(`!${target}`)) return false - return values.length === 0 || values.includes('any') || values.includes(target) + return values.length === 0 || (values.length === 1 && values[0] === 'any') || values.includes(target) || values.every(value => value.startsWith('!')) } diff --git a/apps/desktop/tests/cua-optional-package.spec.ts b/apps/desktop/tests/cua-optional-package.spec.ts index c1eecfbd98..d174749d50 100644 --- a/apps/desktop/tests/cua-optional-package.spec.ts +++ b/apps/desktop/tests/cua-optional-package.spec.ts @@ -71,7 +71,7 @@ it('fetches the macOS optional tarball only for Darwin, with an empty cache for } } const requests: string[] = [] - let omitOs = false + let unrestricted = false const server = createServer((request, response) => { const path = decodeURIComponent(request.url ?? '') requests.push(path) @@ -81,7 +81,8 @@ it('fetches the macOS optional tarball only for Darwin, with an empty cache for if (!entry) { response.writeHead(404); response.end(); return } if (tarball) { response.end(entry.bytes); return } const manifest = { ...entry.manifest } - if (omitOs) delete manifest.os + // pnpm also infers macOS from this package name; an explicit any overrides that inference. + if (unrestricted && name === providerName) manifest.os = ['any'] response.setHeader('Content-Type', 'application/json') response.end(JSON.stringify({ name, 'dist-tags': { latest: version }, versions: { [version]: { ...manifest, dist: { @@ -97,13 +98,13 @@ it('fetches the macOS optional tarball only for Darwin, with an empty cache for await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve) }) const address = server.address() if (!address || typeof address === 'string') throw new Error('fixture registry has no TCP address') - for (const target of ['darwin', 'win32', 'linux', 'linux-without-os']) { - omitOs = target === 'linux-without-os' + for (const target of ['darwin', 'win32', 'linux', 'linux-unrestricted']) { + unrestricted = target === 'linux-unrestricted' const project = join(root, target) await mkdir(project) await writeFile(join(project, 'package.json'), JSON.stringify({ private: true, dependencies: { [String(cli.name)]: version } })) await writeFile(join(project, 'pnpm-workspace.yaml'), [ - 'autoInstallPeers: false', 'supportedArchitectures:', ` os: [${omitOs ? 'linux' : target}]`, + 'autoInstallPeers: false', 'supportedArchitectures:', ` os: [${unrestricted ? 'linux' : target}]`, ' cpu: [arm64]', 'minimumReleaseAge: 0', '', ].join('\n')) requests.length = 0 @@ -115,6 +116,6 @@ it('fetches the macOS optional tarball only for Darwin, with an empty cache for expect(requests).toContain(`/${bundleName}.tgz`) expect(requests).toContain('/@deepseek-ai/dsh-cua.tgz') expect(requests.includes(`/${providerName}.tgz`), `${target}: ${JSON.stringify(requests)}\n${output}`) - .toBe(target === 'darwin' || omitOs) + .toBe(target === 'darwin' || unrestricted) } }, 120_000) diff --git a/apps/web/tests/preview-boot.e2e.ts b/apps/web/tests/preview-boot.e2e.ts index b96dbd45a2..157d7b97c6 100644 --- a/apps/web/tests/preview-boot.e2e.ts +++ b/apps/web/tests/preview-boot.e2e.ts @@ -689,13 +689,15 @@ async function toggleOptionalBundles(page: Page): Promise { console.info(`preview boot: optional bundle toggles ${JSON.stringify(result.toggles.map(({ name, enable, disable, error }) => ({ name, enable, disable, error })))}`) const cua = result.toggles.find(toggle => toggle.name === '@deepseek-ai/dsh-cua-bundle')! expect(cua).toMatchObject({ - enable: 'failed', disable: 'applied', - error: { code: 'operation-error' }, + enable: 'applied', disable: 'applied', + error: null, restored: true, }) - expect(cua.error).toHaveProperty('diagnostic', expect.stringContaining('Computer Use currently requires macOS')) expect(cua.changed).toBeGreaterThan(0) - expect(cua.inactive).toContain('@deepseek-ai/dsh-experimental-cua-macos true failed') + expect(cua.inactive).toEqual([]) + expect(cua.plugins).toContainEqual(expect.objectContaining({ + moduleName: '@deepseek-ai/dsh-experimental-cua-macos', enabled: false, fiberPhase: null, + })) const supported = result.toggles.filter(toggle => toggle.name !== '@deepseek-ai/dsh-cua-bundle') expect(supported.filter(toggle => toggle.enable !== 'applied' || toggle.disable !== 'applied') .map(({ name, enable, disable }) => ({ name, enable, disable }))).toEqual([]) diff --git a/packages/experimental/cua-bundle/tests/platform.spec.ts b/packages/experimental/cua-bundle/tests/platform.spec.ts index e899f09b3d..9001d00c53 100644 --- a/packages/experimental/cua-bundle/tests/platform.spec.ts +++ b/packages/experimental/cua-bundle/tests/platform.spec.ts @@ -13,7 +13,7 @@ it.each(['darwin', 'win32', 'linux'])('loads the native provider only on Darwin const ctx = new Context() await ctx.plugin(Loader) if (!ctx.loader.internal) throw new Error('Loader module importer is unavailable') - const imported = vi.spyOn(ctx.loader.internal, 'import').mockImplementation(async (name) => { + const imported = vi.spyOn(ctx.loader.internal, 'import').mockImplementation(async (name: string) => { if (platform !== 'darwin' && name === '@deepseek-ai/dsh-experimental-cua-macos') { throw new Error('native provider is absent from this installation') } From f85a71505f1318aaef065377b43853341653ad04 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 21:37:05 +0800 Subject: [PATCH 50/55] fix(boot): include installed optional packages in profile resolution --- packages/boot/app-boot/README.i18n.yaml | 4 +-- packages/boot/app-boot/README.md | 2 +- packages/boot/app-boot/README.zh.md | 2 +- packages/boot/app-boot/src/profile.ts | 3 ++- .../tests/profile-resolution-service.spec.ts | 25 +++++++++++++++++++ 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/packages/boot/app-boot/README.i18n.yaml b/packages/boot/app-boot/README.i18n.yaml index c13fb647d1..be702d4775 100644 --- a/packages/boot/app-boot/README.i18n.yaml +++ b/packages/boot/app-boot/README.i18n.yaml @@ -24,8 +24,8 @@ en: 4a424cd401556f20 zh: 0b34dd18e0db6cea /deepseek-ai-dsh-app-boot/use-this-package/profiles: - en: d2d520668668292c - zh: b7deeab27032a3ad + en: 0a1b5f4f40d835a7 + zh: d8f4b54160aa033b /deepseek-ai-dsh-app-boot/use-this-package/previewing-the-effective-configuration: en: 646a0ccdd7881ad1 zh: e3696acb1573dd7b diff --git a/packages/boot/app-boot/README.md b/packages/boot/app-boot/README.md index 59c38476da..4b43180589 100644 --- a/packages/boot/app-boot/README.md +++ b/packages/boot/app-boot/README.md @@ -64,7 +64,7 @@ The enabled `dsh-hmr` plugin watches the profile manifest and both user patch fi Inserted plugin names may be absolute filesystem paths, file URLs, or package specifiers. Patch loading converts absolute paths and patch-relative `./` or `../` paths to file URLs within `insert` rows and their nested groups; existing-entry name assertions and replacement `config` values remain literal. -Before mounting profile rows, the `dsh` launcher computes one immutable runtime resolution from the installation and ordered bundle dependency graphs. Every profile launcher uses runtime resolution, including plain Node, packaged executables, and the Electron Host. It installs the runtime resolution through Node's ESM and CommonJS resolvers without creating fallback links. +Before mounting profile rows, the `dsh` launcher computes one immutable runtime resolution from the installation and ordered bundle dependency graphs. Every profile launcher uses runtime resolution, including plain Node, packaged executables, and the Electron Host. It installs the runtime resolution through Node's ESM and CommonJS resolvers without creating fallback links. Traversal includes installed optional dependencies and skips those absent from the target installation. `sanitizeProfile(binName, profileDir, bundles)` provides filesystem recovery without loading plugins or parsing patches. Desktop uses it for native fatal recovery. Call it only after stopping the profile and excluding concurrent profile writes. It renames the profile’s `cordis.patch.yml` to a unique `.bak-` sibling and restores the supplied bundle list, preserving installed packages and other manifest fields. The timestamp is Unix time in milliseconds; collisions append an ordinal (`-1`, `-2`, …) without changing it. It returns the backup path, or `undefined` when no patch exists; missing profiles remain absent. Profile initialization recreates an empty patch on the next launch. The home-level patch is unchanged. Invalid profile JSON fails before mutation; later errors propagate and retain completed changes for retry. diff --git a/packages/boot/app-boot/README.zh.md b/packages/boot/app-boot/README.zh.md index 73e383e04a..eee5749ba6 100644 --- a/packages/boot/app-boot/README.zh.md +++ b/packages/boot/app-boot/README.zh.md @@ -64,7 +64,7 @@ profile 导入插件前，DSH 会将其声明的 DSH 版本要求与 `getDshRunt 插入条目的插件名可以是绝对文件系统路径、文件 URL 或包标识符。patch 加载会把 `insert` 条目及其嵌套分组中的绝对路径以及相对于 patch 文件的 `./` 或 `../` 路径转换为文件 URL；对已有条目名称的断言及替换用的 `config` 值保持原样。 -挂载 profile 条目前，`dsh` launcher 会从安装依赖图与有序 bundle 依赖图计算一份不可变的 runtime resolution。普通 Node、打包可执行文件与 Electron Host 等所有 profile 启动器都使用 runtime 解析，将 runtime resolution 安装到 Node 的 ESM 与 CommonJS 解析器中，不创建 fallback 链接。 +挂载 profile 条目前，`dsh` launcher 会从安装依赖图与有序 bundle 依赖图计算一份不可变的 runtime resolution。普通 Node、打包可执行文件与 Electron Host 等所有 profile 启动器都使用 runtime 解析，将 runtime resolution 安装到 Node 的 ESM 与 CommonJS 解析器中，不创建 fallback 链接。依赖遍历包含已安装的可选依赖，跳过目标安装中缺席的包。 `sanitizeProfile(binName, profileDir, bundles)` 提供文件恢复，无需加载插件或解析 patch。Desktop 在原生致命错误恢复中调用它。调用前必须停止 profile 并排除并发 profile 写入。它将 profile 的 `cordis.patch.yml` 重命名为带唯一 `.bak-` 后缀的同目录备份，并恢复调用方指定的 bundle 列表，保留已安装包和其他 manifest 字段。时间戳为 Unix 毫秒数；同名备份已存在时追加序号（`-1`、`-2`、……），时间戳保持不变。返回值为备份路径；patch 不存在时返回 `undefined`，缺失的 profile 不会被创建。下次启动的 profile 初始化会重新创建空 patch。home 级 patch 不变。无效 profile JSON 在修改前报错；后续错误向调用方抛出，保留已完成的修改供重试。 diff --git a/packages/boot/app-boot/src/profile.ts b/packages/boot/app-boot/src/profile.ts index e5111b82dc..db23fbf232 100644 --- a/packages/boot/app-boot/src/profile.ts +++ b/packages/boot/app-boot/src/profile.ts @@ -375,7 +375,8 @@ function readPackageManifest(anchor: string): ProfileManifest { /** Return dependency names that may be imported by a loader-visible plugin. */ function profileDependencyNames(manifest: ProfileManifest): string[] { - return [...Object.keys(manifest.dependencies ?? {}), ...Object.keys(manifest.peerDependencies ?? {})] + return [...Object.keys(manifest.dependencies ?? {}), ...Object.keys(manifest.peerDependencies ?? {}), + ...Object.keys(manifest.optionalDependencies ?? {})] } /** Resolve the installation packages that the runtime resolver supplies to every profile. */ diff --git a/packages/boot/app-boot/tests/profile-resolution-service.spec.ts b/packages/boot/app-boot/tests/profile-resolution-service.spec.ts index 9ca8a7e0d6..bed3b6c8c1 100644 --- a/packages/boot/app-boot/tests/profile-resolution-service.spec.ts +++ b/packages/boot/app-boot/tests/profile-resolution-service.spec.ts @@ -334,3 +334,28 @@ describe('profile package metadata service', () => { expect(getEnvironmentData(key)).toBe(previous) }) }) + + +it.each(['installation', 'bundle'] as const)('exposes installed optional dependencies from %s and skips absent ones', async (scope) => { + const root = realpathSync.native(mkdtempSync(join(tmpdir(), 'dsh-optional-resolution-'))) + roots.push(root) + const installAnchor = join(root, 'install', 'package.json') + file(installAnchor, JSON.stringify({ name: 'installation', version: '1.0.0' })) + const profileDir = join(root, 'profiles', 'test') + initProfile(profileDir, ['optional-bundle']) + const bundleDir = join(profileDir, 'node_modules', 'optional-bundle') + const bundleManifest = { name: 'optional-bundle', version: '1.0.0', dsh: { bundle: { patch: './cordis.patch.yml' } } } + file(join(bundleDir, 'package.json'), JSON.stringify(bundleManifest)) + file(join(bundleDir, 'cordis.patch.yml'), '[]\n') + const anchor = scope === 'installation' ? installAnchor : join(bundleDir, 'package.json') + file(anchor, JSON.stringify({ ...JSON.parse(readFileSync(anchor, 'utf8')) as object, + optionalDependencies: { 'metadata-lib': '*', 'absent-platform': '*' }, + })) + const optionalDir = join(dirname(anchor), 'node_modules', 'metadata-lib') + pkg(optionalDir, '1.0.0') + const active = await createRuntimeResolution({ + installAnchor, home: root, profile: loadProfileDirectory('dsh', profileDir, installAnchor), + }) + expect(active.entries).toContainEqual(expect.objectContaining({ name: 'metadata-lib', packageDir: optionalDir })) + expect(active.entries.some(entry => entry.name === 'absent-platform')).toBe(false) +}) From 6d0ba11d2b11ec96b5b574aeef1e02bf89c3c744 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 21:38:08 +0800 Subject: [PATCH 51/55] test(cua): keep fixture opt-in coverage independent of host platform --- apps/web/tests/cua-optin.e2e.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/web/tests/cua-optin.e2e.ts b/apps/web/tests/cua-optin.e2e.ts index 36a0c7a0e2..7a35661fe4 100644 --- a/apps/web/tests/cua-optin.e2e.ts +++ b/apps/web/tests/cua-optin.e2e.ts @@ -19,10 +19,13 @@ it.each([undefined, 'minimal'] as const)('adds CUA to PTC sessions while preserv try { const patch = join(directory, 'native-fixture.yml') // The executable cannot answer native permission queries, so setup must offer retry in the detail page. - await writeFile(patch, JSON.stringify([{ id: 'cua', config: { helperPath: process.execPath } }])) - const scaffold = ownedScaffold = await launchWebScaffold({ harnessHome: join(directory, 'home'), extraOverlayPath: [ - fileURLToPath(new URL('./pin-browse-picker.overlay.yml', import.meta.url)), patch, - ] }) + await writeFile(patch, JSON.stringify([{ id: 'cua', disabled: false, config: { helperPath: process.execPath } }])) + // Install the test provider explicitly so the helper override also exercises session wiring on Linux. + const scaffold = ownedScaffold = await launchWebScaffold({ + profile: { packages: [{ dir: fileURLToPath(new URL('../../../packages/experimental/cua-macos', import.meta.url)), enabled: false }] }, + harnessHome: join(directory, 'home'), extraOverlayPath: [ + fileURLToPath(new URL('./pin-browse-picker.overlay.yml', import.meta.url)), patch, + ] }) const initialDefault = selectedDefault ?? 'standard' if (selectedDefault !== undefined) await scaffold.ctx.settings.update('agent-preset-registry', { selectedDefault }) browser = await chromium.launch() From ea016fa3dd01ef0c82f2606e42d86890d76adb1d Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 21:39:38 +0800 Subject: [PATCH 52/55] fix(manifest): describe optional dependency declarations --- packages/util/package-manifest/README.i18n.yaml | 4 ++-- packages/util/package-manifest/README.md | 2 +- packages/util/package-manifest/README.zh.md | 2 +- packages/util/package-manifest/src/types.ts | 2 ++ 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/util/package-manifest/README.i18n.yaml b/packages/util/package-manifest/README.i18n.yaml index 2ce3905fcf..bb73ae8f0f 100644 --- a/packages/util/package-manifest/README.i18n.yaml +++ b/packages/util/package-manifest/README.i18n.yaml @@ -15,8 +15,8 @@ en: d152484eb41ac6b4 zh: 1db86285abf6c499 /deepseek-ai-dsh-package-manifest/use-this-package: - en: f275964a8845888a - zh: b284e09aacea079b + en: 0f5e861b0c0c5ee2 + zh: 766eaa36b794a22b /deepseek-ai-dsh-package-manifest/understand-the-implementation: en: 41ec449dee927f04 zh: ec1b583bd9025d84 diff --git a/packages/util/package-manifest/README.md b/packages/util/package-manifest/README.md index d50312947e..a1d5e7f423 100644 --- a/packages/util/package-manifest/README.md +++ b/packages/util/package-manifest/README.md @@ -43,7 +43,7 @@ const manifest: DshPackageManifest = { } ``` -`DshPackageManifest` describes the package.json fields used by DSH, with required `name` and `version`; it is not an exhaustive npm schema. Local profile readers use `Partial` because profiles need no published version. `DshManifest` describes only public author fields under `dsh`. `DshBundleManifest.patch` is one patch file path or an ordered list of them, each relative to the package root; the launcher applies a list in order as one bundle layer. TypeScript checks the example and erases `import type`; these interfaces do not parse JSON or write a file. +`DshPackageManifest` describes the package.json fields used by DSH, with required `name` and `version`; it includes dependency, peer, and optional dependency declarations but is not an exhaustive npm schema. Local profile readers use `Partial` because profiles need no published version. `DshManifest` describes only public author fields under `dsh`. `DshBundleManifest.patch` is one patch file path or an ordered list of them, each relative to the package root; the launcher applies a list in order as one bundle layer. TypeScript checks the example and erases `import type`; these interfaces do not parse JSON or write a file. The following metadata fields are optional. Omitting them leaves the format version or compatible host versions undeclared; readers do not infer defaults. diff --git a/packages/util/package-manifest/README.zh.md b/packages/util/package-manifest/README.zh.md index 9a535a094a..d676ff9bf7 100644 --- a/packages/util/package-manifest/README.zh.md +++ b/packages/util/package-manifest/README.zh.md @@ -43,7 +43,7 @@ const manifest: DshPackageManifest = { } ``` -`DshPackageManifest` 描述 DSH 使用的 package.json 字段，其中 `name` 和 `version` 必填；它不是完整的 npm schema（模式）。本地 profile 读取方使用 `Partial`，因为 profile 无需发布版本。`DshManifest` 仅描述 `dsh` 下的公共作者字段。`DshBundleManifest.patch` 是一个 patch 文件路径，或一个有序的路径列表，均相对于包根目录；launcher 按列表顺序把它们作为同一个组合包层应用。TypeScript 检查示例并删除 `import type`；这些接口不解析 JSON，也不写入文件。 +`DshPackageManifest` 描述 DSH 使用的 package.json 字段，其中 `name` 和 `version` 必填；它包含普通依赖、peer 依赖及可选依赖声明，未覆盖 npm 的所有字段。本地 profile 读取方使用 `Partial`，因为 profile 无需发布版本。`DshManifest` 仅描述 `dsh` 下的公共作者字段。`DshBundleManifest.patch` 是一个 patch 文件路径，或一个有序的路径列表，均相对于包根目录；launcher 按列表顺序把它们作为同一个组合包层应用。TypeScript 检查示例并删除 `import type`；这些接口不解析 JSON，也不写入文件。 以下元数据字段均可选。省略时，格式版本或兼容的宿主版本保持未声明状态；读取方不推断默认值。 diff --git a/packages/util/package-manifest/src/types.ts b/packages/util/package-manifest/src/types.ts index e5289ad1e2..27de17acfe 100644 --- a/packages/util/package-manifest/src/types.ts +++ b/packages/util/package-manifest/src/types.ts @@ -18,6 +18,8 @@ export interface DshPackageManifest { private?: boolean /** Packages installed alongside this package. */ dependencies?: Record + /** Packages that may be absent when installation or platform checks skip them. */ + optionalDependencies?: Record /** Compatible versions of packages supplied by the consuming project. */ peerDependencies?: Record /** Runtime requirements; DSH compatibility is declarative until a reader enforces it. */ From 05713edefab1d2da8d1900f13e3fb198c4a76a31 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 21:41:33 +0800 Subject: [PATCH 53/55] refactor(cua): open system settings directly from allowed permission controls --- apps/web/tests/cua-permissions.e2e.ts | 1 - packages/experimental/cua/README.i18n.yaml | 4 +- packages/experimental/cua/README.md | 2 +- packages/experimental/cua/README.zh.md | 2 +- .../cua/src/client/PermissionPanel.tsx | 15 +++---- .../cua/tests/panel.client.spec.tsx | 40 ++----------------- 6 files changed, 12 insertions(+), 52 deletions(-) diff --git a/apps/web/tests/cua-permissions.e2e.ts b/apps/web/tests/cua-permissions.e2e.ts index 3fdbc863c1..18072ae8b1 100644 --- a/apps/web/tests/cua-permissions.e2e.ts +++ b/apps/web/tests/cua-permissions.e2e.ts @@ -130,7 +130,6 @@ it.skipIf(process.platform !== 'darwin')('keeps system requests pending until fr const screen = panel.getByRole('group', { name: 'Screen Recording', exact: true }) expect(await accessibility.getByRole('button').count()).toBe(1) await accessibility.getByRole('button', { name: 'Allowed', exact: true }).click() - await page.getByRole('menuitem', { name: 'Open System Settings', exact: true }).click() expect(await screen.getByRole('button').count()).toBe(1) await expect.poll(() => panel.getAttribute('aria-busy')).toBe('false') await panel.getByRole('button', { name: 'Revoke', exact: true }).click() diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index 6d415d746b..4c76d9b5e3 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -18,8 +18,8 @@ en: ceb54d964dc355b6 zh: 0892066fbceef7b9 /deepseek-ai-dsh-cua/use-this-package/permission-controls: - en: 096915a67d0d5fd9 - zh: 999c004cc923577a + en: be53792edc381ea1 + zh: 037407055d9f34a0 /deepseek-ai-dsh-cua/understand-the-implementation: en: 1fde7df44f040131 zh: 59a842fa402ccb35 diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index 29415cd575..8279d70d9b 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -43,7 +43,7 @@ Actions return `dispatched:true, verified:false`; read fresh state to verify the ### Permission controls -The [Computer Use bundle](../cua-bundle/README.md) mounts this page with its native provider. **Authorize Computer Use** groups the new-session default, PTC execution mode, Accessibility, and Screen Recording inside one border, with dividers between the rows. Each system-permission row shows the system-reported responsible application’s `.app` bundle filename and icon together in a bordered label. **Manage applications Computer Use can access** places the application lists inside a separate bordered group. Choose **Request permission**, find that application in the macOS permission list, and enable its switch; if macOS names another application, follow that prompt. With no detected source, the page directs the user to the application named by macOS. Each system-permission row has one control: **Request permission** when access is missing, or an **Allowed** dropdown after authorization. The dropdown’s **Open System Settings** item opens the corresponding privacy pane, where the user can revoke access. Unknown status uses an **Unknown** dropdown; pending initial observations retain a single placeholder. The page refreshes on entry, when the window regains focus, and after permission actions. A failed query offers **Retry**. Follow any macOS quit-and-reopen prompt and restart DSH as needed. +The [Computer Use bundle](../cua-bundle/README.md) mounts this page with its native provider. **Authorize Computer Use** groups the new-session default, PTC execution mode, Accessibility, and Screen Recording inside one border, with dividers between the rows. Each system-permission row shows the system-reported responsible application’s `.app` bundle filename and icon together in a bordered label. **Manage applications Computer Use can access** places the application lists inside a separate bordered group. Choose **Request permission**, find that application in the macOS permission list, and enable its switch; if macOS names another application, follow that prompt. With no detected source, the page directs the user to the application named by macOS. Each system-permission row has one control: **Request permission** when access is missing, or an outlined **Allowed** button with a settings icon after authorization. Clicking it opens the corresponding privacy pane, where the user can revoke access; hovering shows **Open System Settings**. Unknown status uses the same button with an **Unknown** label; pending initial observations retain a single placeholder. The page refreshes on entry, when the window regains focus, and after permission actions. A failed query offers **Retry**. Follow any macOS quit-and-reopen prompt and restart DSH as needed. System-permission operations affect the computer running DSH. When accessing DSH remotely, complete authorization on that computer. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index 80b3ed3a6b..5f3202407e 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -43,7 +43,7 @@ kind: "package-reference" ### 权限控件 -[Computer Use 组合](../cua-bundle/README.zh.md)将本页与原生 provider 一起挂载。“给 Computer Use 授权”把新会话默认模式、PTC 执行方式、辅助功能和屏幕录制放在同一个边框内，四排之间使用分隔线。两条系统权限要求在能识别责任应用时，把该应用的 `.app` 包名称和图标放在同一个标签中。“管理 Computer Use 可访问的应用”使用独立边框包住应用列表。点击“申请权限”，在 macOS 权限列表中找到该应用并开启对应开关；若 macOS 显示其他名称，以系统提示为准。来源无法识别时，页面引导用户查找系统提示的应用。每条系统权限要求只显示一个控件：未授权时为“申请权限”，已授权时为“已允许”下拉按钮。菜单中的“打开系统设置”进入对应隐私设置页，由用户撤销权限。无法确认状态时显示“未确认”下拉按钮，首次查询期间保留一个占位。进入详情页、返回窗口或完成授权操作后会自动刷新；查询失败时可点击“重试”。若系统要求退出并重新打开应用，请按提示操作并重新启动 DSH。 +[Computer Use 组合](../cua-bundle/README.zh.md)将本页与原生 provider 一起挂载。“给 Computer Use 授权”把新会话默认模式、PTC 执行方式、辅助功能和屏幕录制放在同一个边框内，四排之间使用分隔线。两条系统权限要求在能识别责任应用时，把该应用的 `.app` 包名称和图标放在同一个标签中。“管理 Computer Use 可访问的应用”使用独立边框包住应用列表。点击“申请权限”，在 macOS 权限列表中找到该应用并开启对应开关；若 macOS 显示其他名称，以系统提示为准。来源无法识别时，页面引导用户查找系统提示的应用。每条系统权限要求只显示一个控件：未授权时为“申请权限”，已授权时为带设置图标的“已允许”描边按钮，点击直接进入对应隐私设置页，由用户撤销权限，悬停提示“打开系统设置”。无法确认状态时使用同样的按钮，文字为“未确认”。首次查询期间保留一个占位。进入详情页、返回窗口或完成授权操作后会自动刷新；查询失败时可点击“重试”。若系统要求退出并重新打开应用，请按提示操作并重新启动 DSH。 系统权限操作作用于运行 DSH 的电脑。远程访问时，请在该电脑上完成授权。 diff --git a/packages/experimental/cua/src/client/PermissionPanel.tsx b/packages/experimental/cua/src/client/PermissionPanel.tsx index d6d190f38b..5abe3d5db9 100644 --- a/packages/experimental/cua/src/client/PermissionPanel.tsx +++ b/packages/experimental/cua/src/client/PermissionPanel.tsx @@ -1,6 +1,6 @@ /** Native permission status and revocable application consent on the existing plugin page. */ import { useEffect, useState } from 'react' -import { Button, IconChevronDownOutlineRegular, Menu, Modal, Toast } from '@deepseek-ai/dsh-client-ui-primitives' +import { Button, IconSettingsOutlineRegular, Modal, Toast } from '@deepseek-ai/dsh-client-ui-primitives' import type { InjectFace, PropsLocale, PropsRuntime } from '@deepseek-ai/dsh-client-ui-slots' import type {} from '@deepseek-ai/dsh-client-ui-plugin-manager/client' import type { ConfigForm } from '@deepseek-ai/dsh-client-ui-settings/client' @@ -117,18 +117,13 @@ function PermissionAction({ permission, granted, loading, busy, request, openSet loading: boolean busy: boolean }) { - const [open, setOpen] = useState(false) - useEffect(() => { setOpen(false) }, [busy, loading, granted]) if (loading) return if (granted === false) return - return { setOpen(false) }} align="end" portal autoFocus - anchor={} - items={[{ id: 'settings', label: t('settings') }]} - onSelect={() => { setOpen(false); openSettings(permission) }} /> + return } /** @param props Host observation and explicit user actions. @returns Permission controls without automatic system prompts. */ diff --git a/packages/experimental/cua/tests/panel.client.spec.tsx b/packages/experimental/cua/tests/panel.client.spec.tsx index a780d4a319..6c2b06668f 100644 --- a/packages/experimental/cua/tests/panel.client.spec.tsx +++ b/packages/experimental/cua/tests/panel.client.spec.tsx @@ -50,9 +50,9 @@ it('requests only a missing privilege and leaves OS attribution to the system pr fireEvent.click(screen.getByRole('button', { name: en.request })) expect(f.props.request).toHaveBeenCalledWith('accessibility') expect(screen.queryByRole('button', { name: en.settings })).toBeNull() - fireEvent.click(screen.getByRole('button', { name: en.granted })) - expect(f.props.openSettings).not.toHaveBeenCalled() - fireEvent.click(screen.getByRole('menuitem', { name: en.settings })) + const allowed = screen.getByRole('button', { name: en.granted }) + expect(allowed.getAttribute('title')).toBe(en.settings) + fireEvent.click(allowed) expect(f.props.openSettings.mock.calls).toEqual([['screenRecording']]) fireEvent.focus(window) expect(f.props.refresh).toHaveBeenCalledTimes(2) @@ -365,39 +365,6 @@ it('loads the runtime setting independently and reports unavailable and read-onl }) -it.each([en, zh])('keeps one control per permission row and closes a granted menu on refresh or revocation', (dictionary) => { - const f = fixture() - f.publish(observe(value)) - render() - const missing = screen.getByRole('group', { name: dictionary.accessibility }) - const granted = screen.getByRole('group', { name: dictionary.screenRecording }) - expect(within(missing).getAllByRole('button')).toHaveLength(1) - expect(within(missing).getByRole('button', { name: dictionary.request })).toBeTruthy() - const trigger = within(granted).getByRole('button', { name: dictionary.granted }) - expect(within(granted).getAllByRole('button')).toHaveLength(1) - trigger.focus() - fireEvent.click(trigger) - expect(trigger.getAttribute('aria-expanded')).toBe('true') - expect(screen.getByRole('menuitem', { name: dictionary.settings })).toBeTruthy() - fireEvent.keyDown(screen.getByRole('menu'), { key: 'Escape' }) - expect(screen.queryByRole('menu')).toBeNull() - expect(document.activeElement).toBe(trigger) - fireEvent.click(trigger) - f.publish({ system: { ...f.state.getSnapshot().system, busy: true }, busy: true }) - expect(screen.queryByRole('menu')).toBeNull() - expect(trigger.hasAttribute('disabled')).toBe(true) - f.publish(observe(value)) - expect(screen.queryByRole('menu')).toBeNull() - fireEvent.click(trigger) - f.publish(observe({ ...value, system: { ...value.system, screenRecording: false } })) - expect(screen.queryByRole('menu')).toBeNull() - expect(within(granted).getAllByRole('button')).toHaveLength(1) - expect(within(granted).getByRole('button', { name: dictionary.request })).toBeTruthy() - expect(f.props.openSettings).not.toHaveBeenCalled() - expect(f.props.revoke).not.toHaveBeenCalled() -}) - - it('retains the previous single control while a stale query error is being refreshed', () => { const f = fixture() f.publish(observe(value)) @@ -406,5 +373,4 @@ it('retains the previous single control while a stale query error is being refre expect(screen.getByRole('button', { name: en.request }).hasAttribute('disabled')).toBe(true) expect(screen.getByRole('button', { name: en.granted }).hasAttribute('disabled')).toBe(true) expect(screen.queryByRole('alert')).toBeNull() - expect(screen.queryByRole('menu')).toBeNull() }) From 539716d112781c5db05fee11720bd996fb952b51 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 22:00:42 +0800 Subject: [PATCH 54/55] test(cua): satisfy type-aware lint for permission and platform regressions --- apps/desktop/tests/cua-optional-package.spec.ts | 4 ++-- apps/desktop/tests/prepare-package-set.spec.ts | 2 +- packages/experimental/cua-bundle/tests/platform.spec.ts | 2 +- packages/experimental/cua/tests/controller.client.spec.ts | 3 ++- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/desktop/tests/cua-optional-package.spec.ts b/apps/desktop/tests/cua-optional-package.spec.ts index d174749d50..3f3f7ad572 100644 --- a/apps/desktop/tests/cua-optional-package.spec.ts +++ b/apps/desktop/tests/cua-optional-package.spec.ts @@ -12,7 +12,7 @@ const repository = resolve(import.meta.dirname, '../../..') const providerName = '@deepseek-ai/dsh-experimental-cua-macos' const bundleName = '@deepseek-ai/dsh-cua-bundle' const readManifest = async (path: string): Promise> => - JSON.parse(await readFile(join(repository, path, 'package.json'), 'utf8')) + JSON.parse(await readFile(join(repository, path, 'package.json'), 'utf8')) as Record it('fetches the macOS optional tarball only for Darwin, with an empty cache for every target', async () => { const root = await mkdtemp(join(tmpdir(), 'cua-cold-install-')) @@ -93,7 +93,7 @@ it('fetches the macOS optional tarball only for Darwin, with an empty cache for }) onTestFinished(async () => { server.closeAllConnections() - await new Promise((resolve, reject) => { server.close(error => error ? reject(error) : resolve()) }) + await new Promise((resolve, reject) => { server.close((error) => { if (error) reject(error); else resolve() }) }) }) await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve) }) const address = server.address() diff --git a/apps/desktop/tests/prepare-package-set.spec.ts b/apps/desktop/tests/prepare-package-set.spec.ts index c81d0c694c..c1b5edbd69 100644 --- a/apps/desktop/tests/prepare-package-set.spec.ts +++ b/apps/desktop/tests/prepare-package-set.spec.ts @@ -139,7 +139,7 @@ it.each([{ os: 'darwin' }, { cpu: [42] }])('rejects malformed platform constrain it('copies no macOS tarball into the Windows package set', () => { const root = mkdtempSync(join(tmpdir(), 'desktop-platform-packages-')) - onTestFinished(() => rmSync(root, { recursive: true, force: true })) + onTestFinished(() => { rmSync(root, { recursive: true, force: true }) }) const inputs = join(root, 'inputs') mkdirSync(inputs) for (const [name, entry] of targetManifests()) { diff --git a/packages/experimental/cua-bundle/tests/platform.spec.ts b/packages/experimental/cua-bundle/tests/platform.spec.ts index 9001d00c53..1a5cf24aec 100644 --- a/packages/experimental/cua-bundle/tests/platform.spec.ts +++ b/packages/experimental/cua-bundle/tests/platform.spec.ts @@ -21,7 +21,7 @@ it.each(['darwin', 'win32', 'linux'])('loads the native provider only on Darwin }) try { for (const row of patch[0]!.insert) { - await ctx.loader.create({ ...row, disabled: interpolate({ process: { platform } }, row.disabled) }) + await ctx.loader.create({ ...row, disabled: Boolean(interpolate({ process: { platform } }, row.disabled)) }) } await ctx.loader.await() const imports = imported.mock.calls.map(call => call[0]) diff --git a/packages/experimental/cua/tests/controller.client.spec.ts b/packages/experimental/cua/tests/controller.client.spec.ts index 59568eac9c..7a2f146ca2 100644 --- a/packages/experimental/cua/tests/controller.client.spec.ts +++ b/packages/experimental/cua/tests/controller.client.spec.ts @@ -92,7 +92,8 @@ it('revokes access while icons are pending, serializes writes, and joins reads o f.actions.applicationIcons.mockImplementationOnce(async () => ({ ok: true, value: await icons.promise })) f.actions.revoke.mockImplementationOnce(async () => { entered.resolve(undefined) - return { ok: true, value: await action.promise } + await action.promise + return { ok: true, value: undefined } }) f.face.refresh() await vi.waitFor(() => { expect(f.controller.state.getSnapshot().applications.busy).toBe(false) }) From 33381c86c4310615c84b36e98c1d673575be6a19 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 22:11:22 +0800 Subject: [PATCH 55/55] fix(release): preserve platform constraints in packed install verification --- scripts/release/verify-packed-install.spec.ts | 21 +++++++++- scripts/release/verify-packed-install.ts | 38 ++++++++++++++----- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/scripts/release/verify-packed-install.spec.ts b/scripts/release/verify-packed-install.spec.ts index b40f21ae92..4eb45db00d 100644 --- a/scripts/release/verify-packed-install.spec.ts +++ b/scripts/release/verify-packed-install.spec.ts @@ -38,7 +38,8 @@ const path = require('node:path') const consumer = process.env.INIT_CWD || process.cwd() const cache = path.join(consumer, '.npm-cache', '_cacache') const files = fs.existsSync(cache) ? fs.readdirSync(cache, { recursive: true }).filter(name => fs.statSync(path.join(cache, name)).isFile()) : [] -fs.writeFileSync(process.env.CACHE_EVIDENCE, JSON.stringify({ consumer, files })) +fs.writeFileSync(process.env.CACHE_EVIDENCE, JSON.stringify({ consumer, files, + installed: ['matching', 'wrong-os', 'wrong-cpu'].filter(name => fs.existsSync(path.join(consumer, 'node_modules', name))) })) ` writeFileSync(join(source, 'probe.cjs'), probe + (outcome === 'install failure' ? '\nprocess.exit(37)\n' : '')) writeFileSync(join(source, 'lib/bin.js'), outcome === 'entry failure' ? 'process.exit(37)\n' : 'console.log("0.0.1")\n') @@ -51,6 +52,21 @@ fs.writeFileSync(process.env.CACHE_EVIDENCE, JSON.stringify({ consumer, files }) expect(archive.error).toBeUndefined() expect(archive.signal).toBeNull() expect(archive.status, archive.stderr).toBe(0) + for (const [name, platform] of [ + ['matching', { os: [process.platform], cpu: ['any'] }], + ['wrong-os', { os: [`!${process.platform}`] }], + ['wrong-cpu', { cpu: [process.arch === 'arm64' ? 'x64' : 'arm64'] }], + ] as const) { + const packageDirectory = join(directory, name, 'package') + mkdirSync(packageDirectory, { recursive: true }) + writeFileSync(join(packageDirectory, 'package.json'), JSON.stringify({ name, version: '0.0.1', ...platform })) + const packedPlatform = spawnSync('tar', ['-czf', join(packed, `${name}.tgz`), '-C', join(directory, name), 'package'], { + encoding: 'utf8', timeout: 30_000, + }) + expect(packedPlatform.error).toBeUndefined() + expect(packedPlatform.signal).toBeNull() + expect(packedPlatform.status, packedPlatform.stderr).toBe(0) + } const environment = Object.fromEntries(Object.entries(process.env).filter(([key]) => !/^npm_config_/i.test(key))) Object.assign(environment, { npm_config_cache: ambient, @@ -68,7 +84,8 @@ fs.writeFileSync(process.env.CACHE_EVIDENCE, JSON.stringify({ consumer, files }) expect(result.error).toBeUndefined() expect(result.signal).toBeNull() expect(result.status, result.stdout + result.stderr).toBe(outcome === 'success' ? 0 : 1) - const observation = JSON.parse(readFileSync(evidence, 'utf8')) as { consumer: string; files: string[] } + const observation = JSON.parse(readFileSync(evidence, 'utf8')) as { consumer: string; files: string[]; installed: string[] } + expect(observation.installed).toEqual(['matching']) expect(observation.files.length, 'npm must write tarball content into the consumer-private cache').toBeGreaterThan(0) expect(observation.consumer).toContain('dsh-packed-dsh-') expect(existsSync(observation.consumer), 'the verifier finally must remove its consumer and cache').toBe(false) diff --git a/scripts/release/verify-packed-install.ts b/scripts/release/verify-packed-install.ts index 82e209cf49..08ab4860c0 100644 --- a/scripts/release/verify-packed-install.ts +++ b/scripts/release/verify-packed-install.ts @@ -24,7 +24,7 @@ import { pathToFileURL } from 'node:url' import { parseArgs } from 'node:util' import { releaseFamily } from './families.ts' import { capture, isEntry } from './process.ts' -import { packedIdentity } from './tarball.ts' +import { packedManifest } from './tarball.ts' import { verifyInstalledProductIsolation } from './installed-product-isolation.ts' /** @@ -45,30 +45,46 @@ function consumerEnvironment(consumerRoot: string): NodeJS.ProcessEnv { return environment } +function acceptsPlatform(value: unknown, current: string): boolean { + if (value === undefined) return true + if (!Array.isArray(value) || !value.every((tag): tag is string => typeof tag === 'string')) { + throw new Error('packed platform constraints must be arrays of strings') + } + let accepted = value.every(tag => tag.startsWith('!')) + for (const tag of value) { + if (tag === `!${current}`) return false + if (tag === current || (tag === 'any' && value.length === 1)) accepted = true + } + return accepted +} + /** - * Every packed tarball in the given directories, as `file:` dependency entries. + * Packed tarballs and their compatibility with the verification host, as `file:` dependency entries. * * The directories are read by their contents rather than a pack order file: a * directory here can hold tarballs packed only to satisfy a cross-sequence * dependency, which no release order describes. * @param directories - absolute directories holding packed tarballs. - * @returns Package name to tarball file URL, and the version each carries. + * @returns Package name to tarball URL, version, and host compatibility. */ -function packedDependencies(directories: readonly string[]): Map { - const dependencies = new Map() +function packedDependencies(directories: readonly string[]): Map { + const dependencies = new Map() for (const directory of directories) { const tarballs = readdirSync(directory).filter(name => name.endsWith('.tgz')).sort() if (tarballs.length === 0) throw new Error(`${directory} holds no packed tarball`) for (const filename of tarballs) { const tarball = join(directory, filename) - const { name, version } = packedIdentity(tarball) - dependencies.set(name, { url: pathToFileURL(tarball).href, version }) + const manifest = packedManifest(tarball) + const { name, version } = manifest + if (typeof name !== 'string' || typeof version !== 'string') throw new Error(`${tarball} manifest lacks name/version`) + const compatible = acceptsPlatform(manifest.os, process.platform) && acceptsPlatform(manifest.cpu, process.arch) + dependencies.set(name, { url: pathToFileURL(tarball).href, version, compatible }) } } return dependencies } -/** Install every tarball under `--from` and drive the `--family` entry. */ +/** Install platform-compatible tarballs under `--from` and drive the `--family` entry. */ function main(): void { const { values } = parseArgs({ options: { family: { type: 'string' }, from: { type: 'string', multiple: true } }, @@ -96,7 +112,11 @@ function main(): void { name: `dsh-packed-install-${family.id}`, version: '0.0.0', private: true, - dependencies: Object.fromEntries([...packed].map(([name, entryPacked]) => [name, entryPacked.url])), + dependencies: Object.fromEntries([...packed].filter(([, entryPacked]) => entryPacked.compatible) + .map(([name, entryPacked]) => [name, entryPacked.url])), + // Keep foreign-platform metadata local without turning those tarballs into required installations. + optionalDependencies: Object.fromEntries([...packed].filter(([, entryPacked]) => !entryPacked.compatible) + .map(([name, entryPacked]) => [name, entryPacked.url])), }, null, 2)}\n`) const environment = consumerEnvironment(consumerRoot)