From 373080de5e80c26a762d8fe3244958ae139c252b Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:48:58 +0800 Subject: [PATCH 01/12] test(cua): remove unrelated split remnants --- apps/web/tsconfig.json | 1 - .../tests/retired-content-admission.spec.ts | 13 ++----------- tsconfig.host.json | 1 - 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json index 666db22e82..9d25a02899 100644 --- a/apps/web/tsconfig.json +++ b/apps/web/tsconfig.json @@ -83,7 +83,6 @@ "tests/input-plugin-extensions.e2e.ts", "tests/plugin-manager.e2e.ts", "tests/cua-optin.e2e.ts", - "tests/bundle-platform.e2e.ts", "tests/cua-permissions.e2e.ts", "tests/schedule-header.e2e.ts", "tests/worktree-opt-in.e2e.ts", diff --git a/packages/session/session-persistence-jsonl/tests/retired-content-admission.spec.ts b/packages/session/session-persistence-jsonl/tests/retired-content-admission.spec.ts index ddf49e64e4..1d10cb2fde 100644 --- a/packages/session/session-persistence-jsonl/tests/retired-content-admission.spec.ts +++ b/packages/session/session-persistence-jsonl/tests/retired-content-admission.spec.ts @@ -1,13 +1,13 @@ /** Interpreted retired content must fail before migration publication or recoverable native-tail suppression. */ import { Context } from '@deepseek-ai/cordis' -import { SESSION_FORMAT_VERSION, SessionId } from '@deepseek-ai/dsh-session' +import { SessionId } from '@deepseek-ai/dsh-session' import { SessionFormatUnsupportedError, SessionPersistenceCorruptionError } from '@deepseek-ai/dsh-session-persistence' import * as JsonlSessionPersistence from '@deepseek-ai/dsh-session-persistence-jsonl' import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { basename, dirname, join } from 'node:path' import { describe, expect, it, onTestFinished } from 'vitest' -import { generationLogPath, SessionLogScanner } from '../src/format.ts' +import { generationLogPath } from '../src/format.ts' import { compressZstdFrame } from '../src/zstd.ts' const id = SessionId('retired-content-admission') @@ -55,12 +55,3 @@ describe.each(modes)('retired content in JSONL ($compression, $access)', ({ comp expect(await readFile(path)).toEqual(bytes) }) }) - -it.each(['strict', 'recoverable'] as const)('classifies retired native PTC tags as unsupported during %s scanning', (recovery) => { - const header = { type: 'session', version: SESSION_FORMAT_VERSION, id, createdAt: 1, isSeeded: false, delegationDepth: 0 } - for (const type of ['tool/code-dispatch-start', 'tool/code-dispatch']) { - const scanner = new SessionLogScanner(Buffer.from(JSON.stringify(header) + '\n'), recovery) - const row = { type, seq: 0, time: 1, data: {} } - expect(() => { scanner.write(Buffer.from(JSON.stringify(row) + '\n')) }).toThrow(SessionFormatUnsupportedError) - } -}) diff --git a/tsconfig.host.json b/tsconfig.host.json index ea1ffecb2a..a267f50f63 100644 --- a/tsconfig.host.json +++ b/tsconfig.host.json @@ -75,7 +75,6 @@ "apps/web/tests/input-plugin-extensions.e2e.ts", "apps/web/tests/plugin-manager.e2e.ts", "apps/web/tests/cua-optin.e2e.ts", - "apps/web/tests/bundle-platform.e2e.ts", "apps/web/tests/cua-permissions.e2e.ts", "apps/web/tests/schedule-header.e2e.ts", "apps/web/tests/worktree-opt-in.e2e.ts", From a1c0a68c80268677a95d9f15a031507289547b1b Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:49:15 +0800 Subject: [PATCH 02/12] docs(cua): place preset migration under the current release --- .../cua-preset-scope/guide.i18n.yaml | 6 +++--- .../cua-preset-scope/guide.md | 0 .../cua-preset-scope/guide.zh.md | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename docs/upgrade-guide/{v0.2.1-alpha.2 => v0.2.1-rc.2}/cua-preset-scope/guide.i18n.yaml (86%) rename docs/upgrade-guide/{v0.2.1-alpha.2 => v0.2.1-rc.2}/cua-preset-scope/guide.md (100%) rename docs/upgrade-guide/{v0.2.1-alpha.2 => v0.2.1-rc.2}/cua-preset-scope/guide.zh.md (100%) diff --git a/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.i18n.yaml b/docs/upgrade-guide/v0.2.1-rc.2/cua-preset-scope/guide.i18n.yaml similarity index 86% rename from docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.i18n.yaml rename to docs/upgrade-guide/v0.2.1-rc.2/cua-preset-scope/guide.i18n.yaml index bb91afebfd..7313c9c61a 100644 --- a/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.i18n.yaml +++ b/docs/upgrade-guide/v0.2.1-rc.2/cua-preset-scope/guide.i18n.yaml @@ -1,13 +1,13 @@ # Bilingual-pair consistency record for guide.md (docs/i18n/README.md): per heading # section, a hash of its English and Chinese blocks outside code blocks and generated regions. # After editing either side, bring the other along and re-record with: -# pnpm run verify-translation-pairing --write docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.md +# pnpm run verify-translation-pairing --write docs/upgrade-guide/v0.2.1-rc.2/cua-preset-scope/guide.md /: en: 36d6b79b48c08dd5 zh: 814caece159e9b0e /cua-defaults-to-ptc-session-scopes: - en: 1d6eee8020ebed3b - zh: d14279f527983c3c + en: a53b6884785bdf56 + zh: 7d27fc9b029e0e3e /cua-defaults-to-ptc-session-scopes/change: en: dac96a29e3751249 zh: d48cb030f43c07e4 diff --git a/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.md b/docs/upgrade-guide/v0.2.1-rc.2/cua-preset-scope/guide.md similarity index 100% rename from docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.md rename to docs/upgrade-guide/v0.2.1-rc.2/cua-preset-scope/guide.md diff --git a/docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.zh.md b/docs/upgrade-guide/v0.2.1-rc.2/cua-preset-scope/guide.zh.md similarity index 100% rename from docs/upgrade-guide/v0.2.1-alpha.2/cua-preset-scope/guide.zh.md rename to docs/upgrade-guide/v0.2.1-rc.2/cua-preset-scope/guide.zh.md From a2484cb02d1685e37d6fa2fab1282527bca235f6 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:49:53 +0800 Subject: [PATCH 03/12] fix(cua): keep permission guidance independent of the UI --- packages/experimental/cua-macos/README.md | 4 ++-- packages/experimental/cua-macos/README.zh.md | 4 ++-- packages/experimental/cua-macos/src/guidance.ts | 4 ++-- snapshots/session/cua-native/system-prompt.expected.md | 4 ++-- snapshots/session/cua-ptc/system-prompt.expected.md | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 3909b8f97a..960177c233 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -152,7 +152,7 @@ The plugin contributes one `cua` operation tool with method-specific parameters. ##### Operation guidance (verbatim) ```markdown -Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user allows another request on the permission page. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. +Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly permits another authorization request. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again. @@ -189,7 +189,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to complete the Computer Use permission setup for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant the required macOS system permissions for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index b01c0488d7..dab1ca6499 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -152,7 +152,7 @@ AppKit sheet 可以通过其窗口 ID 单独选择，也会出现在所属文档 ##### 操作指导文本（原文） ```markdown -Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user allows another request on the permission page. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. +Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly permits another authorization request. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again. @@ -189,7 +189,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to complete the Computer Use permission setup for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant the required macOS system permissions for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index 6de6ce0565..0147beefbe 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -1,6 +1,6 @@ /** macOS operation guidance supplied with the shared CUA object API. */ /** Instructions for raw operation calls and optional object-API bindings. */ -export const GUIDANCE = `Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user allows another request on the permission page. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. +export const GUIDANCE = `Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly permits another authorization request. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again. @@ -37,7 +37,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to complete the Computer Use permission setup for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant the required macOS system permissions for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index 82bccf01cb..d2daf65630 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -29,7 +29,7 @@ Use the workflow tool ONLY when the user explicitly asks for a workflow or for l Start independent delegations with `subagent` or `subagent_fork` together in one assistant message and continue useful work while they run. -Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user allows another request on the permission page. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. +Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly permits another authorization request. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again. @@ -66,7 +66,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to complete the Computer Use permission setup for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant the required macOS system permissions for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index 3c9c10466b..cf5222fd7f 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -31,7 +31,7 @@ Use the workflow tool ONLY when the user explicitly asks for a workflow or for l Start independent delegations with `subagent` or `subagent_fork` together in one assistant message and continue useful work while they run. -Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user allows another request on the permission page. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. +Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly permits another authorization request. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation. Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again. @@ -68,7 +68,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to complete the Computer Use permission setup for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant the required macOS system permissions for macOS Accessibility or Screen Recording. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. From eaea1ec5b9eb12945329f1b3191b956292cd12e7 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:50:07 +0800 Subject: [PATCH 04/12] perf(cua): omit unused icons from application selection --- packages/experimental/cua-macos/README.i18n.yaml | 4 ++-- packages/experimental/cua-macos/README.md | 2 +- packages/experimental/cua-macos/README.zh.md | 2 +- .../native/Sources/CuaNative/RequestCoordinator.swift | 7 +------ snapshots/session/cua-native/workspace/native-helper.mjs | 2 +- 5 files changed, 6 insertions(+), 11 deletions(-) diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index 69c1f545a8..58f4fc662c 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -12,8 +12,8 @@ en: 4549f02fc53854a5 zh: 6f74f1130b6623a8 /deepseek-ai-dsh-experimental-cua-macos/table-of-contents: - en: 867d9ec544719acf - zh: 92348599582dc33a + en: 9353d17b05f2f88c + zh: 6b2c6b4ee0046294 /deepseek-ai-dsh-experimental-cua-macos/use-this-package: en: 5f070508770a1a48 zh: 07d6e582c01b0ea3 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 960177c233..a65220d4b1 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -20,7 +20,7 @@ Inspect macOS permissions, select running applications and windows, read accessi - [Known Limitations and Deferred Work](#known-limitations-and-deferred-work) - [Dev Note](#dev-note) -The helper supplies bounded 64-pixel PNG application icons for permission prompts and saved-application lists. Icon reads do not launch an application or inspect its windows; executable and signing checks still determine authorization. +The helper supplies bounded 64-pixel PNG application icons for permission-page application lists. Icon reads do not launch an application or inspect its windows; executable and signing checks still determine authorization. The shared helper queries the process macOS considers responsible for it and captures that application’s name and icon before announcing readiness. A Desktop app launched from Terminal can therefore identify Terminal even when another GUI application sits between it and the helper. Permission queries retain this metadata after the responsible application exits and across connections sharing the helper. If the system query or application metadata is unavailable, the field is omitted and the UI uses the system prompt for guidance. diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index dab1ca6499..d57c404c6e 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -20,7 +20,7 @@ kind: "package-reference" - [已知限制与后续工作](#known-limitations-and-deferred-work) - [开发备注](#dev-note) -helper 为授权提示和已授权应用列表提供 64 像素 PNG 图标。读取图标不会启动应用或检查窗口，授权仍由可执行文件和签名校验决定。 +helper 为权限页的应用列表提供 64 像素 PNG 图标。读取图标不会启动应用或检查窗口，授权仍由可执行文件和签名校验决定。 共享 helper 查询 macOS 为它记录的责任进程，在发布就绪状态前保存对应应用的名称和图标。因此，从 Terminal 启动 Desktop 时，即使中间还有另一个 GUI 应用，也能识别到 Terminal。责任应用退出后，权限查询仍保留这些信息，后续复用该 helper 的连接读取同一份记录。系统查询或应用信息不可用时省略该字段，由界面引导用户按系统弹窗选择授权对象。 diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift index 280cd358ad..99ea8634a9 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/RequestCoordinator.swift @@ -195,12 +195,7 @@ import Darwin let outcome = try await worker.execute(request.method, params: params, catalog: catalog, installed: installed, trace: trace) trace?.mark("worker.end") switch outcome { - case .result(let value): - if request.method == "getApp", var fields = value.object { - let path = installed?.path ?? catalog.applications.first { Double($0.pid) == value["pid"].number }?.path - if let path, let icon = ApplicationCatalog.icon(at: path) { fields["applicationIcon"] = .string(icon) } - result = .object(fields) - } else { result = value } + case .result(let value): result = value case .capture(let target): let image = try await capture.capture(target, cancellation: cancellation, trace: trace) result = try await worker.finishCapture(target, image: image, trace: trace) diff --git a/snapshots/session/cua-native/workspace/native-helper.mjs b/snapshots/session/cua-native/workspace/native-helper.mjs index 2bbaa83188..e36445ca94 100755 --- a/snapshots/session/cua-native/workspace/native-helper.mjs +++ b/snapshots/session/cua-native/workspace/native-helper.mjs @@ -142,7 +142,7 @@ function createApplicationState(socket, record, application, captureBroker, conn } if (!['Fixture', 'test.fixture', '9'].includes(params.selector)) refuse('APP_NOT_FOUND', 'Select the running Fixture application') acquired = true - return { ...app, applicationIcon: `data:image/png;base64,${png}`, authorization: {"id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "name": "Fixture", "executablePath": "/fixture/app", "persistable": true} } + return { ...app, authorization: {"id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "name": "Fixture", "executablePath": "/fixture/app", "persistable": true} } } if (!acquired || params.appHandle !== app.appHandle) refuse('APP_NOT_FOUND', 'Acquire an application in this helper first') if (method === 'show') return { dispatched: true, verified: false, windowId: params.windowId ?? window.windowId, foregroundChanged: false } From 5d0fe5430911f465bab527a11b405ffa21da459e Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:50:17 +0800 Subject: [PATCH 05/12] test(cua): cover each private native operation once --- .../cua-macos/tests/protocol-generation.spec.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts index 54b4bf3a19..220aed84a4 100644 --- a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts +++ b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts @@ -18,12 +18,7 @@ describe('native protocol generation', () => { { method: 'paste', params: { appHandle: 'app', text: 'text', pasteReadTimeoutMs: 1000 } }, { method: 'findApps', params: { query: 'Editor', maxAppMatches: 1 } }, { method: 'launchApp', params: { appHandle: 'pending' } }, - { method: 'permissionStatus', params: {} }, - { method: 'requestPermission', params: { permission: 'accessibility' } }, - { method: 'openPermissionSettings', params: { permission: 'screenRecording' } }, - { method: 'permissionStatus', params: {} }, - { method: 'requestPermission', params: { permission: 'accessibility' } }, - { method: 'openPermissionSettings', params: { permission: 'screenRecording' } }, + { method: 'applicationIcons', params: { paths: ['/Applications/Fixture.app'] } }, { method: 'permissionStatus', params: {} }, { method: 'requestPermission', params: { permission: 'accessibility' } }, { method: 'openPermissionSettings', params: { permission: 'screenRecording' } }, From d07ad00f86e9694e096edf47e2b78b5f21369bcf Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:50:57 +0800 Subject: [PATCH 06/12] fix(cua): keep the basic bundle independent of PTC configuration --- packages/experimental/README.i18n.yaml | 4 ++-- packages/experimental/README.md | 2 +- packages/experimental/README.zh.md | 2 +- packages/experimental/cua-bundle/package.json | 1 - pnpm-lock.yaml | 3 --- 5 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/experimental/README.i18n.yaml b/packages/experimental/README.i18n.yaml index 58388301cb..77c06a747b 100644 --- a/packages/experimental/README.i18n.yaml +++ b/packages/experimental/README.i18n.yaml @@ -18,8 +18,8 @@ en: b55aeb927737b90d zh: 0b215fcb5c2c7908 /packages-experimental/packages: - en: e12eeae4540a53e0 - zh: 44f65caca730252d + en: e20594b1b7e506d7 + zh: ef76c3c732ddb547 /packages-experimental/related-documentation: en: 1cb9d3448d44e48d zh: cd326d70a746b896 diff --git a/packages/experimental/README.md b/packages/experimental/README.md index ed093206a6..5f052b8e5e 100644 --- a/packages/experimental/README.md +++ b/packages/experimental/README.md @@ -71,7 +71,7 @@ Promotion requires a named product role, documented public behavior and limitati | [`ptc-runtime-python`](ptc-runtime-python/README.md) | CPython subprocess backend for the PTC execution seam | `ctx.ptcRuntime` | | [CUA](cua/README.md) | Shared application/window objects, consent, and logged execution. | Library | | [macOS CUA](cua-macos/README.md) | Swift native execution and platform preparation. | `ctx.computerUse` | -| [`cua-bundle`](cua-bundle/README.md) | Optional Computer Use switch with persistent PTC | — | +| [`cua-bundle`](cua-bundle/README.md) | Optional Computer Use switch for PTC sessions | — | | [`client-ui-cua`](client-ui-cua/README.md) | Computer Use permission setup and application access controls | — | | [`computer-use-cua-driver-mcp`](computer-use-cua-driver-mcp/README.md) | Use an installed Cua Driver through MCP | `ctx.computerUse` | | [`computer-use-cua-driver-native`](computer-use-cua-driver-native/README.md) | Embed the Cua Driver native npm runtime | `ctx.computerUse` | diff --git a/packages/experimental/README.zh.md b/packages/experimental/README.zh.md index edc4efb886..5a74a36072 100644 --- a/packages/experimental/README.zh.md +++ b/packages/experimental/README.zh.md @@ -71,7 +71,7 @@ kind: "package-group" | [`ptc-runtime-python`](ptc-runtime-python/README.zh.md) | PTC 执行 seam 的 CPython 子进程后端 | `ctx.ptcRuntime` | | [CUA](cua/README.zh.md) | 共享应用与窗口对象、授权和已记录的执行。 | 库 | | [macOS CUA](cua-macos/README.zh.md) | Swift 原生执行与平台组件准备。 | `ctx.computerUse` | -| [`cua-bundle`](cua-bundle/README.zh.md) | 默认关闭的 Computer Use 开关与 persistent PTC 组合 | — | +| [`cua-bundle`](cua-bundle/README.zh.md) | 面向 PTC 会话、默认关闭的 Computer Use 开关 | — | | [`client-ui-cua`](client-ui-cua/README.zh.md) | Computer Use 权限设置与应用授权管理 | — | | [`computer-use-cua-driver-mcp`](computer-use-cua-driver-mcp/README.zh.md) | 通过 MCP 使用已安装的 Cua Driver | `ctx.computerUse` | | [`computer-use-cua-driver-native`](computer-use-cua-driver-native/README.zh.md) | 嵌入 Cua Driver 原生 npm 运行时 | `ctx.computerUse` | diff --git a/packages/experimental/cua-bundle/package.json b/packages/experimental/cua-bundle/package.json index 96935054cc..155345ecc6 100644 --- a/packages/experimental/cua-bundle/package.json +++ b/packages/experimental/cua-bundle/package.json @@ -40,7 +40,6 @@ "dependencies": { "@deepseek-ai/dsh-experimental-cua-macos": "workspace:*", "@deepseek-ai/dsh-computer-use": "workspace:*", - "@deepseek-ai/dsh-ptc-runtime-node": "workspace:*", "@deepseek-ai/dsh-client-ui-cua": "workspace:*" }, "peerDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 37e833d594..ed0602a480 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8670,9 +8670,6 @@ importers: '@deepseek-ai/dsh-experimental-cua-macos': specifier: workspace:* version: link:../cua-macos - '@deepseek-ai/dsh-ptc-runtime-node': - specifier: workspace:* - version: link:../../ptc-runtime/ptc-runtime-node devDependencies: '@deepseek-ai/cordis': specifier: workspace:~ From a86170da75b63904b8198a6250859f5ed6fed177 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:51:17 +0800 Subject: [PATCH 07/12] refactor(cua): remove unused permission observation pause controls --- .../src/client/PermissionPanel.tsx | 10 ++-- .../client-ui-cua/src/client/controller.ts | 20 +------ .../tests/controller.client.spec.ts | 55 +------------------ .../client-ui-cua/tests/mount.client.spec.ts | 1 - .../client-ui-cua/tests/panel.client.spec.tsx | 2 +- 5 files changed, 9 insertions(+), 79 deletions(-) diff --git a/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx b/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx index 4c9ae041fd..b80edfee6c 100644 --- a/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx +++ b/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx @@ -94,7 +94,7 @@ function PermissionDescription({ application, permission, loading, t }: Pick value) const observed = state.system.value const apps = state.applications.value @@ -103,10 +103,10 @@ export function PermissionPanel(props: PermissionPanelProps) { const loading = observed === null && (busy || state.system.error === null) const application = !busy && state.system.error !== null ? undefined : observed?.launchApplication useEffect(() => { - setActive(true) + refresh() window.addEventListener('focus', refresh) return () => { window.removeEventListener('focus', refresh) } - }, [refresh, setActive]) + }, [refresh]) return

{t('title')}

@@ -158,9 +158,9 @@ export function PermissionPanel(props: PermissionPanelProps) { * @param props Permission observation and activation navigation callbacks. * @returns Setup guidance when system permissions need attention. */ -export function PermissionSetup({ t, setActive, useCuaPermissions, onDismiss, onOpenDetails }: PermissionSetupProps) { +export function PermissionSetup({ t, refresh, useCuaPermissions, onDismiss, onOpenDetails }: PermissionSetupProps) { const state = useCuaPermissions(value => value.system) - useEffect(() => { setActive(true) }, [setActive]) + useEffect(() => { refresh() }, [refresh]) const ready = state.value !== null && state.error === null && state.value.accessibility && state.value.screenRecording useEffect(() => { if (ready && !state.busy) onDismiss() }, [ready, state.busy, onDismiss]) diff --git a/packages/experimental/client-ui-cua/src/client/controller.ts b/packages/experimental/client-ui-cua/src/client/controller.ts index 552dcefbba..044f431e2e 100644 --- a/packages/experimental/client-ui-cua/src/client/controller.ts +++ b/packages/experimental/client-ui-cua/src/client/controller.ts @@ -31,8 +31,6 @@ export type PermissionView = { [K in keyof PermissionData]: Query } - /** Pause observation during bundle changes, or resume with a fresh read. */ - setActive: (active: boolean) => void refresh: () => void request: (permission: CuaSystemPermission) => void openSettings: (permission: CuaSystemPermission) => void @@ -53,8 +51,6 @@ export class PermissionController { private tail: Promise = Promise.resolve() private reads: Promise = Promise.resolve() private closed = false - private active = true - private epoch = 0 private sequence = 0 /** @param actions Browser-owned Remote calls that reject on failed responses. */ constructor(private readonly actions: PermissionActions) {} @@ -65,17 +61,6 @@ export class PermissionController { face(): PermissionFace { return { hooks: { cuaPermissions: this.state }, - setActive: (active) => { - if (this.closed) return - if (active !== this.active) { this.active = active; this.epoch++ } - if (active) this.enqueue() - else this.state.update((state) => { - state.busy = true - for (const key of ['system', 'applications', 'icons'] as const) { - state[key].busy = true; state[key].error = null - } - }) - }, refresh: () => { this.enqueue() }, request: (permission) => { this.enqueue(() => this.actions.request(permission), 'requested') }, openSettings: (permission) => { this.enqueue(() => this.actions.openSettings(permission), 'opened') }, @@ -87,9 +72,8 @@ export class PermissionController { /** Prevent publication after UI withdrawal and join the accepted Remote work. */ async dispose(): Promise { this.closed = true; await this.tail; await this.reads } private enqueue(action?: () => Promise, kind?: 'requested' | 'opened' | 'revoked' | 'retryAllowed'): void { - if (this.closed || !this.active) return - const epoch = this.epoch - const current = () => !this.closed && this.active && epoch === this.epoch + if (this.closed) return + const current = () => !this.closed const sequence = ++this.sequence this.state.update((state) => { state.busy = true diff --git a/packages/experimental/client-ui-cua/tests/controller.client.spec.ts b/packages/experimental/client-ui-cua/tests/controller.client.spec.ts index 6c12499c16..a04253ef4a 100644 --- a/packages/experimental/client-ui-cua/tests/controller.client.spec.ts +++ b/packages/experimental/client-ui-cua/tests/controller.client.spec.ts @@ -105,7 +105,7 @@ it('ignores stale toast completion after the permission UI is disposed', async ( await f.controller.dispose() const before = f.controller.state.getSnapshot() f.face.dismissToast() - f.face.setActive(true) + f.face.refresh() expect(f.controller.state.getSnapshot()).toBe(before) }) @@ -161,45 +161,6 @@ it('keeps a user-action failure visible when a refresh is queued behind it', asy }) -it.each(['success', 'failure'])('drops a permission read %s from the previous bundle activation', async (outcome) => { - const f = fixture() - f.face.setActive(true) - await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) - const pending = Promise.withResolvers(), entered = Promise.withResolvers() - onTestFinished(async () => { pending.resolve(value); await f.controller.dispose() }) - f.actions.readSystem.mockImplementationOnce(() => { entered.resolve(undefined); return pending.promise }) - f.face.refresh() - await entered.promise - f.face.setActive(false) - f.face.setActive(false) - expect(f.controller.state.getSnapshot()).toMatchObject({ busy: true, system: { error: null, value } }) - f.face.request('accessibility') - expect(f.actions.request).not.toHaveBeenCalled() - if (outcome === 'failure') pending.reject(new Error('old helper stopped')) - else pending.resolve({ ...value, screenRecording: true }) - await Promise.allSettled([pending.promise]) - expect(f.controller.state.getSnapshot()).toMatchObject({ busy: true, system: { error: null, value } }) - f.face.setActive(true) - await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) - expect(f.controller.state.getSnapshot()).toMatchObject({ system: { error: null, value } }) - expect(f.actions.readSystem).toHaveBeenCalledTimes(3) -}) - -it.each(['success', 'failure'])('does not publish a user-action %s after its bundle is replaced', async (outcome) => { - const f = fixture(), pending = Promise.withResolvers(), entered = Promise.withResolvers() - onTestFinished(async () => { pending.resolve(undefined); await f.controller.dispose() }) - f.actions.request.mockImplementationOnce(() => { entered.resolve(undefined); return pending.promise }) - f.face.request('accessibility') - await entered.promise - f.face.setActive(false) - if (outcome === 'failure') pending.reject(new Error('old request cancelled')) - else pending.resolve(undefined) - f.face.setActive(true) - await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) - expect(f.controller.state.getSnapshot()).toMatchObject({ toast: null, system: { error: null, value } }) - expect(f.actions.readSystem).toHaveBeenCalledOnce() -}) - it('publishes each observation before the other queries finish', async () => { const f = fixture() const system = Promise.withResolvers() @@ -237,17 +198,3 @@ it('keeps application decisions available when system inspection fails', async ( }) await f.controller.dispose() }) - -it.each(['success', 'failure'])('drops old icon %s after disabling and reenabling the bundle', async (outcome) => { - const f = fixture(), pending = Promise.withResolvers>() - onTestFinished(async () => { pending.resolve({}); await f.controller.dispose() }) - f.actions.readIcons.mockReturnValueOnce(pending.promise) - f.face.refresh() - await vi.waitFor(() => { expect(f.controller.state.getSnapshot().system.busy).toBe(false) }) - f.face.setActive(false) - f.face.setActive(true) - if (outcome === 'success') pending.resolve({ '/old/app': 'old-icon' }) - else pending.reject(new Error('old helper stopped')) - await vi.waitFor(() => { expect(f.controller.state.getSnapshot().busy).toBe(false) }) - expect(f.controller.state.getSnapshot().icons).toEqual({ value: {}, error: null, busy: false }) -}) diff --git a/packages/experimental/client-ui-cua/tests/mount.client.spec.ts b/packages/experimental/client-ui-cua/tests/mount.client.spec.ts index c79bfed54b..df6732f404 100644 --- a/packages/experimental/client-ui-cua/tests/mount.client.spec.ts +++ b/packages/experimental/client-ui-cua/tests/mount.client.spec.ts @@ -18,7 +18,6 @@ const value: CuaApplicationState & { system: CuaSystemPermissionState } = { allowedApps: [], blockedApps: [], settingsNamespace: 'cua', persistentAvailable: true } function actionsOf(value: Record): asserts value is Record & PermissionFace { - assert(typeof value.setActive === 'function') assert(typeof value.refresh === 'function' && typeof value.request === 'function' && typeof value.openSettings === 'function') assert(typeof value.revoke === 'function' && typeof value.dismissToast === 'function') assert(typeof value.retryApproval === 'function') diff --git a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx b/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx index d6e833e2d5..5f21465344 100644 --- a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx +++ b/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx @@ -32,7 +32,7 @@ function fixture() { const refresh = vi.fn() const props = { useCuaPermissions: bindSnapshotSelector(state), t: makeTranslate(en), useCuaDefaultPreset: bindSnapshotSelector(preset), setPtcDefault: vi.fn(async () => true), - refresh, setActive: vi.fn((active: boolean) => { if (active) refresh() }), request: vi.fn(), openSettings: vi.fn(), revoke: vi.fn(), + refresh, request: vi.fn(), openSettings: vi.fn(), revoke: vi.fn(), retryApproval: vi.fn(), dismissToast: vi.fn(), } satisfies PermissionPanelProps return { state, preset, props, From 2d1609920003f9947d39b39e74249df4537533f0 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:51:36 +0800 Subject: [PATCH 08/12] fix(cua): localize complete permission instructions --- .../client-ui-cua/src/client/PermissionPanel.tsx | 7 ++++--- .../experimental/client-ui-cua/src/client/locales.ts | 10 ++++------ .../client-ui-cua/tests/panel.client.spec.tsx | 5 ++--- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx b/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx index b80edfee6c..faa83a783d 100644 --- a/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx +++ b/packages/experimental/client-ui-cua/src/client/PermissionPanel.tsx @@ -80,15 +80,16 @@ function PermissionDescription({ application, permission, loading, t }: Pick{t(permission === 'accessibility' ? 'accessibilityUse' : 'screenRecordingUse')}

+ const [before, after] = t(permission === 'accessibility' ? 'accessibilityPermission' : 'screenRecordingPermission').split('{application}') return

- {t('permissionPrefix')}{' '} + {before} {application === undefined ? : <> {application.icon !== undefined && } {application.name} } - {' '} - {t(permission === 'accessibility' ? 'accessibilityAction' : 'screenRecordingAction')} + + {after}

} diff --git a/packages/experimental/client-ui-cua/src/client/locales.ts b/packages/experimental/client-ui-cua/src/client/locales.ts index 03e86f33d0..52f286700e 100644 --- a/packages/experimental/client-ui-cua/src/client/locales.ts +++ b/packages/experimental/client-ui-cua/src/client/locales.ts @@ -17,9 +17,8 @@ export const en = { openSetup: 'Open settings', later: 'Later', close: 'Close', accessibility: 'Accessibility', screenRecording: 'Screen Recording', accessibilityUse: 'Read interface controls, click, and type.', screenRecordingUse: 'Capture the selected application window.', - permissionPrefix: 'In System Settings, allow', - accessibilityAction: 'to read interface controls, click, and type.', - screenRecordingAction: 'to capture application windows.', + accessibilityPermission: 'In System Settings, allow {application} to read interface controls, click, and type.', + screenRecordingPermission: 'In System Settings, allow {application} to capture application windows.', permissionHint: 'Enable the application in System Settings. Follow the macOS prompt if it names another application.', permissionHintUnknown: 'Enable these permissions for the application named in the macOS prompt.', granted: 'Allowed', missing: 'Not allowed', unknown: 'Unknown', request: 'Request permission', settings: 'Open System Settings', retry: 'Retry', @@ -48,9 +47,8 @@ export const zh: Record = { openSetup: '前往设置', later: '稍后', close: '关闭', accessibility: '辅助功能', screenRecording: '屏幕录制', accessibilityUse: '读取界面控件、点击和输入文字', screenRecordingUse: '截取所选应用的窗口', - permissionPrefix: '在系统设置中允许', - accessibilityAction: '读取界面、点击和输入', - screenRecordingAction: '截取应用窗口', + accessibilityPermission: '在系统设置中允许 {application} 读取界面、点击和输入。', + screenRecordingPermission: '在系统设置中允许 {application} 截取应用窗口。', permissionHint: '在系统设置中开启对应应用；若 macOS 提示其他应用，以系统提示为准', permissionHintUnknown: '在系统设置中，为 macOS 提示的应用开启对应权限', granted: '已允许', missing: '未允许', unknown: '未确认', request: '申请权限', settings: '打开系统设置', retry: '重试', diff --git a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx b/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx index 5f21465344..e29768cdf9 100644 --- a/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx +++ b/packages/experimental/client-ui-cua/tests/panel.client.spec.tsx @@ -229,13 +229,12 @@ it.each([en, zh])('shows three requirements with the target application beside e expect(within(requirements).getAllByRole('group')).toHaveLength(3) expect(screen.getByRole('region', { name: dictionary.applicationsTitle })).toBeTruthy() const accessibility = within(requirements).getByRole('group', { name: dictionary.accessibility }) - expect(accessibility.textContent).toContain(dictionary.permissionPrefix) expect(within(accessibility).getByText(application.name)).toBeTruthy() - expect(accessibility.textContent).toContain(dictionary.accessibilityAction) + expect(accessibility.textContent).toContain(dictionary.accessibilityPermission.replace('{application}', application.name)) expect(accessibility.querySelector('img')?.getAttribute('src')).toBe(application.icon) const recording = within(requirements).getByRole('group', { name: dictionary.screenRecording }) expect(within(recording).getByText(application.name)).toBeTruthy() - expect(recording.textContent).toContain(dictionary.screenRecordingAction) + expect(recording.textContent).toContain(dictionary.screenRecordingPermission.replace('{application}', application.name)) expect(within(requirements).getByText(dictionary.permissionHint)).toBeTruthy() f.publish({ system: { ...f.state.getSnapshot().system, error: 'offline' } }) expect(screen.queryByText(application.name)).toBeNull() From 49794a9ad2f5d030af2d694a55cad4a205891b87 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:52:06 +0800 Subject: [PATCH 09/12] docs(cua): define preset filtering and cover late registration --- packages/experimental/cua-macos/README.i18n.yaml | 4 ++-- packages/experimental/cua-macos/README.md | 1 + packages/experimental/cua-macos/README.zh.md | 1 + packages/experimental/cua/README.i18n.yaml | 4 ++-- packages/experimental/cua/README.md | 2 +- packages/experimental/cua/README.zh.md | 2 +- .../experimental/cua/tests/agent-scopes.spec.ts | 15 +++++++++++++-- 7 files changed, 21 insertions(+), 8 deletions(-) diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index 58f4fc662c..8e064caef0 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -15,8 +15,8 @@ en: 9353d17b05f2f88c zh: 6b2c6b4ee0046294 /deepseek-ai-dsh-experimental-cua-macos/use-this-package: - en: 5f070508770a1a48 - zh: 07d6e582c01b0ea3 + en: 107cc4d4e72e248d + zh: 1aec124d4018d2a4 /deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment: en: e7909c9b43e44fbc zh: 98949877d4ea3a86 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index a65220d4b1..7700c69ed1 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -42,6 +42,7 @@ The dump command checks profile composition without a model request. For a real | Field | Default | Meaning | | --- | --- | --- | +| `agentPreset` | `ptc` | Register tools and guidance only for Agents bound to this preset; `false` registers globally. | | `allowedApps` | `[]` | Live profile-local application consent, keyed by executable path and verified signing requirement. | | `helperPath` | package-owned executable | Optional absolute native executable override. | | `buildTimeoutMs` | `300000` | Maximum native preparation time. | diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index d57c404c6e..ccd844dcf7 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -42,6 +42,7 @@ pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis. | 字段 | 默认值 | 含义 | | --- | --- | --- | +| `agentPreset` | `ptc` | 仅为绑定该 preset 的 Agent 注册工具和指引；`false` 表示全局注册。 | | `allowedApps` | `[]` | 当前配置档的动态应用授权，绑定可执行文件路径和已验证的代码签名要求。 | | `helperPath` | 包内可执行文件 | 可选的原生可执行文件绝对路径覆盖。 | | `buildTimeoutMs` | `300000` | 原生组件准备的最长时间。 | diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index e5ee2a0f75..00480ee607 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -18,8 +18,8 @@ en: 577e1b18dd220c73 zh: bde0692c6695d749 /deepseek-ai-dsh-cua/understand-the-implementation: - en: 751fe580f79dfaae - zh: 3811db65f634ef5a + en: 5f8f14fda4d69173 + zh: 5493e3ba74ffb7e0 /deepseek-ai-dsh-cua/further-exploration: en: b8b0aece886fadaa zh: ecd8c88d601f2179 diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index 11cbd47457..24abc4a510 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -51,7 +51,7 @@ Actions return `dispatched:true, verified:false`; read fresh state to verify the The provider calls [installCua](src/index.ts) with its prepared helper, connection factory, and guidance. [NativeClient](src/native-client.ts) owns request correlation, byte limits, cancellation acknowledgement, and failure settlement. The platform connector owns startup and connection cleanup. The shared executor retains one connection per Agent and joins in-flight image writes during disposal. -`agentPreset` defaults to `ptc`, restricting tools and guidance to Agents bound to that preset. The provider uses the existing `agent/created` and `agent-preset/selected` events and Agent-owned scopes, including Agents already present when it loads. Provider unload and Agent disposal release those registrations. Set `agentPreset: false` for global registration in standalone compositions without presets. The Host continues to own permissions, native connections and cancellation; preset definitions remain unchanged. +`agentPreset` defaults to `ptc`, restricting tools and guidance to Agents bound to that preset. The provider uses the existing `agent/created` and `agent-preset/selected` events and Agent-owned scopes, including Agents already present when it loads. Provider unload and Agent disposal release those registrations. Preset definitions may register after the provider; a preset name filters bound Agents and supplies no tools until a matching Agent exists. Set `agentPreset: false` for global registration in standalone compositions without presets. The Host continues to own permissions, native connections and cancellation; preset definitions remain unchanged. [The operation schema](src/protocol.ts) is the authority for model parameters and separately declared Host-only fields. Native implementations enforce target identity, window geometry, resource limits, and operating-system access at execution. No invariant companion is published because this library has no separately projected registry or durable state. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index 3b58f03187..6de835777a 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -51,7 +51,7 @@ kind: "package-library" 提供方将准备好的 helper、连接工厂和指引传给 [installCua](src/index.ts)。[NativeClient](src/native-client.ts) 负责请求关联、字节上限、取消确认和失败处理。平台连接器负责启动和连接清理。共享执行器为每个 Agent 保留一个连接，并在销毁时等待进行中的图像写入。 -`agentPreset` 默认为 `ptc`，将工具和指引限制到绑定该 preset 的 Agent。提供方使用现有 `agent/created`、`agent-preset/selected` 事件及 Agent 拥有的作用域，也处理启用前已存在的 Agent。提供方卸载和 Agent 销毁都会释放这些注册。不使用 preset 的独立组合可设置 `agentPreset: false` 来全局注册。Host 仍拥有权限、本机连接和取消操作，preset 定义保持不变。 +`agentPreset` 默认为 `ptc`，将工具和指引限制到绑定该 preset 的 Agent。提供方使用现有 `agent/created`、`agent-preset/selected` 事件及 Agent 拥有的作用域，也处理启用前已存在的 Agent。提供方卸载和 Agent 销毁都会释放这些注册。preset 定义可以晚于提供方注册；名称用于筛选已绑定的 Agent，在匹配的 Agent 出现前不提供工具。不使用 preset 的独立组合可设置 `agentPreset: false` 来全局注册。Host 仍拥有权限、本机连接和取消操作，preset 定义保持不变。 [操作 schema](src/protocol.ts) 定义模型参数，并单独声明 Host 专用字段。原生实现在执行时检查目标身份、窗口几何、资源上限和操作系统访问条件。本库没有独立投影的注册表或持久状态，因此不发布 invariant 模块。 diff --git a/packages/experimental/cua/tests/agent-scopes.spec.ts b/packages/experimental/cua/tests/agent-scopes.spec.ts index 26568bca49..fd1a349130 100644 --- a/packages/experimental/cua/tests/agent-scopes.spec.ts +++ b/packages/experimental/cua/tests/agent-scopes.spec.ts @@ -23,7 +23,7 @@ const provider = { }, } -async function setup() { +async function setup(presetIds = ['standard', 'ptc']) { const ctx = new Context() onTestFinished(() => ctx.fiber.dispose()) ctx.baseUrl = import.meta.url @@ -33,7 +33,7 @@ async function setup() { await ctx.plugin(ComputerUseRegistry) await ctx.plugin(Subprocess, []) await ctx.plugin(AgentPresets, { default: 'standard' }) - for (const id of ['standard', 'ptc']) { + for (const id of presetIds) { await ctx.plugin({ inject: ['agentPresets'], async* apply(owner: Context) { yield await owner.agentPresets.register({ id, plugins: [] }) } }) @@ -93,3 +93,14 @@ it('ignores unloaded Session selections and reports a failed live scope update', await ctx.agentPresets.select(handle.agent, 'standard') await expect.poll(() => ctx.tools.get('cua', handle.agent)).toBeUndefined() }) + +it('accepts a matching preset registered after the CUA provider', async () => { + const { ctx, create } = await setup([]) + await ctx.plugin(provider, {}) + expect(ctx.tools.get('cua')).toBeUndefined() + await ctx.plugin({ inject: ['agentPresets'], async* apply(owner: Context) { + yield await owner.agentPresets.register({ id: 'ptc', plugins: [] }) + } }) + const handle = await create('late-preset', 'ptc') + expect(ctx.tools.get('cua', handle.agent)).toBeDefined() +}) From fe14a0972c0d17b59eb36e6ef85af4d187413e72 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:52:38 +0800 Subject: [PATCH 10/12] docs(cua): disclose session consent cache retention --- packages/experimental/cua/README.i18n.yaml | 4 ++-- packages/experimental/cua/README.md | 1 + packages/experimental/cua/README.zh.md | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index 00480ee607..14142829c4 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -39,8 +39,8 @@ en: 2d5df52dafa1d940 zh: 06e815759b9150c3 /deepseek-ai-dsh-cua/known-limitations-and-deferred-work: - en: da6aefcb279aea83 - zh: b45b7bb300b9f792 + en: ae98f1b2718b3b94 + zh: e6bf4e96ae08e669 /deepseek-ai-dsh-cua/known-limitations-and-deferred-work/dev-note: en: 0a1253c6f7b5e8b9 zh: 9f56528562c7c914 diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index 24abc4a510..cadf0e6a8d 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -88,6 +88,7 @@ Results append to history. Retaining objects does not rewrite previous messages; +- In-memory session decisions have no capacity limit or eviction. They remain until the provider unloads, so a long-running Host retains entries for every session/application pair that requested access. - Native execution currently uses the macOS provider. Windows support is planned as a separate provider package with a C# helper, retaining the shared app/window API, authorization, snapshot validity, and action-result semantics. Its implementation must document system-specific limitations and distinguish cross-compilation evidence from Windows desktop validation. - Persistent variables do not reserve an application across a sequence of operations. Other Agents and the user can change its state. Hover and observations do not restore the previous pointer position; [continuous interaction](../../../.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.md) uses no cross-call ownership. - The package stays in `experimental` until Windows support and platform validation permit promotion to the `computer-use` group. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index 6de835777a..35e37feb67 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -88,6 +88,7 @@ kind: "package-library" +- 会话授权决策保存在内存中，没有容量上限或淘汰机制，直到提供方卸载才清空。长期运行的 Host 会保留每个曾请求访问的会话与应用组合。 - 当前原生执行使用 macOS 提供方。Windows 支持计划使用独立的提供方包和 C# helper，保留共享应用与窗口 API、授权、快照有效性和动作结果语义。实现须说明系统特有的限制，并区分交叉编译证据和 Windows 桌面验证。 - 持久变量不会在一组连续操作期间独占应用。其他 Agent 和用户可能改变其状态。 悬停和观察不恢复先前的鼠标位置；[连续交互](../../../.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.zh.md)不分配跨调用所有权。 - 本包保留在 `experimental`，Windows 支持和平台验证完成后可迁至 `computer-use` 组。 From dd436a6c9cb7afd331760c7be60a7c032449e9e3 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:53:15 +0800 Subject: [PATCH 11/12] docs(cua): explain explicit package naming exceptions --- ...21-experimental-capabilities-as-optional-bundles.i18n.yaml | 4 ++-- ...026-09-21-experimental-capabilities-as-optional-bundles.md | 2 +- ...-09-21-experimental-capabilities-as-optional-bundles.zh.md | 2 +- scripts/experimental-package-policy.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.i18n.yaml b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.i18n.yaml index a2748e8846..345eec989b 100644 --- a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.i18n.yaml @@ -9,8 +9,8 @@ en: 2ba6c43b20a3d347 zh: 5647179ddaa38ad9 /agent-note-ship-experimental-capabilities-as-optional-bundles/decision: - en: 69df6602f92a3e09 - zh: 2e2ed04760086f4c + en: 6566c6fb7287d748 + zh: fe41c0a9da97f340 /agent-note-ship-experimental-capabilities-as-optional-bundles/alternatives-considered: en: d6f40bf02df37ab7 zh: e5b85ce9d41e8754 diff --git a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md index f0e6ce7337..a2dded7934 100644 --- a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md +++ b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.md @@ -18,7 +18,7 @@ Bundles insert ordinary Host rows; optional model tools register in the global t Claude Code and Codex install on demand through the separate `ON_DEMAND_BUNDLES` catalog; the [on-demand native-bundle decision](2026-10-05-official-on-demand-bundles.md) owns the cost, version, and offline-discovery trade-offs. -Computer Use is a shipped optional bundle in the experimental group. Its repository-owned native component is prepared when the provider loads, without an external Cua Driver runtime. +Computer Use is a shipped optional bundle in the experimental group. Its bundle and client plugin use the explicit naming exceptions `@deepseek-ai/dsh-cua-bundle` and `@deepseek-ai/dsh-client-ui-cua`, alongside `@deepseek-ai/dsh-cua`. These names preserve profile references when the capability moves package groups; the experimental classification still controls isolation and promotion. Its repository-owned native component is prepared when the provider loads, without an external Cua Driver runtime. ## Alternatives considered diff --git a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.zh.md b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.zh.md index c90290b20d..db1b4a5506 100644 --- a/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.zh.md +++ b/.agents/notes/implemented/architecture/2026-09-21-experimental-capabilities-as-optional-bundles.zh.md @@ -18,7 +18,7 @@ Bundle 插入普通 Host 行；可选模型工具注册到全局工具层，因 Claude Code 与 Codex 通过独立的 `ON_DEMAND_BUNDLES` 目录按需安装，其成本、版本与离线发现取舍由[按需原生 bundle 决策](2026-10-05-official-on-demand-bundles.zh.md)管理。 -Computer Use 是 experimental 包组内随安装提供的可选组合包。提供方加载时准备仓库自有的本机组件，无需外部 Cua Driver 运行时。 +Computer Use 是 experimental 包组内随安装提供的可选组合包。组合包和前端插件与 `@deepseek-ai/dsh-cua` 一起采用显式命名例外 `@deepseek-ai/dsh-cua-bundle`、`@deepseek-ai/dsh-client-ui-cua`，在能力移动分组时保留配置引用。依赖隔离和晋升仍由实验性分类约束。提供方加载时准备仓库自有的本机组件，无需外部 Cua Driver 运行时。 ## 考虑过的替代方案 diff --git a/scripts/experimental-package-policy.ts b/scripts/experimental-package-policy.ts index 68c63fcd7e..5763dbdedc 100644 --- a/scripts/experimental-package-policy.ts +++ b/scripts/experimental-package-policy.ts @@ -1,6 +1,6 @@ /** Experimental package identities, publication policy, and retained npm names. */ -/** Existing npm names retained by capabilities now housed under experimental/. */ +/** Explicit npm identities retained across experimental classification and package-role moves. */ export const EXPERIMENTAL_PACKAGE_NAME_EXCEPTIONS: Readonly> = { 'packages/experimental/cua': '@deepseek-ai/dsh-cua', 'packages/experimental/cua-bundle': '@deepseek-ai/dsh-cua-bundle', From b39ce22a09a0fb785143c0fe89b15c69c9e48037 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 19 Oct 2026 12:55:18 +0800 Subject: [PATCH 12/12] test(cua): cover permission actions settling after disposal --- .../client-ui-cua/tests/controller.client.spec.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/experimental/client-ui-cua/tests/controller.client.spec.ts b/packages/experimental/client-ui-cua/tests/controller.client.spec.ts index a04253ef4a..8b776837f9 100644 --- a/packages/experimental/client-ui-cua/tests/controller.client.spec.ts +++ b/packages/experimental/client-ui-cua/tests/controller.client.spec.ts @@ -109,6 +109,20 @@ it('ignores stale toast completion after the permission UI is disposed', async ( expect(f.controller.state.getSnapshot()).toBe(before) }) +it.each(['success', 'failure'])('settles an in-flight action %s after disposal without publishing', async (outcome) => { + const f = fixture(), pending = Promise.withResolvers(), entered = Promise.withResolvers() + onTestFinished(async () => { pending.resolve(undefined); await f.controller.dispose() }) + f.actions.request.mockImplementationOnce(() => { entered.resolve(undefined); return pending.promise }) + f.face.request('accessibility') + await entered.promise + const before = f.controller.state.getSnapshot(), closed = f.controller.dispose() + if (outcome === 'failure') pending.reject(new Error('permission request ended after withdrawal')) + else pending.resolve(undefined) + await closed + expect(f.controller.state.getSnapshot()).toBe(before) + expect(f.actions.readSystem).not.toHaveBeenCalled() +}) + it('contains a late read rejection and supports errors without an Error instance', async () => { const f = fixture() f.actions.readSystem.mockRejectedValueOnce('offline')