From b01b889632c63fb6c54fa231ffbbabd84dd9775f Mon Sep 17 00:00:00 2001 From: creatixchu Date: Wed, 14 Oct 2026 13:32:36 +0800 Subject: [PATCH] fix(cua): report readable argument errors and clarify coordinate targets --- packages/experimental/cua-macos/README.md | 2 +- packages/experimental/cua-macos/README.zh.md | 2 +- .../experimental/cua-macos/src/guidance.ts | 2 +- .../tests/persistent-bridge.built.e2e.ts | 3 +++ packages/experimental/cua/README.i18n.yaml | 4 ++-- packages/experimental/cua/README.md | 2 +- packages/experimental/cua/README.zh.md | 2 +- packages/experimental/cua/src/api.ts | 24 ++++++++++++++++++- packages/experimental/cua/src/protocol.ts | 3 ++- packages/experimental/cua/tests/api.spec.ts | 20 ++++++++++++++++ 10 files changed, 55 insertions(+), 9 deletions(-) diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index 2d676cbf8d..74887ae936 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -184,7 +184,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index 7b169bb834..43c37296bb 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -184,7 +184,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index c261b15d32..829706b8ab 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -37,7 +37,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. For click and hover, coordinate targets must be [x, y], for example hover([650, 400], {relative:true}). A bare number such as hover(650) is always an AX element index. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input. Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls. diff --git a/packages/experimental/cua-macos/tests/persistent-bridge.built.e2e.ts b/packages/experimental/cua-macos/tests/persistent-bridge.built.e2e.ts index c97c1c1e08..aa11a360c7 100644 --- a/packages/experimental/cua-macos/tests/persistent-bridge.built.e2e.ts +++ b/packages/experimental/cua-macos/tests/persistent-bridge.built.e2e.ts @@ -78,6 +78,9 @@ it.skipIf(process.platform === 'win32' || !existsSync(bridge) || !existsSync(boo expect(first.value).toEqual({ mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 7, byteLength: 90 }) expect(images).toBe(1) expect(JSON.stringify(first)).not.toContain('iVBORw0KGgo') + const invalid = await run('await cua.findApps("");', 'invalid') + expect(invalid.error?.message).toBe('findApps query: Expected a non-empty search string') + expect(calls.filter(call => call.phase === 'invalid')).toEqual([]) const second = await run('await win.getAXState({emit:false}); await win.click(0); await win.getAXState({emit:false});', 'second') expect(second.error).toBeUndefined() expect(second.value).toContain('Count: 2') diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index 7f50a8e20a..a99f1eaf24 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -15,8 +15,8 @@ en: d152484eb41ac6b4 zh: 1db86285abf6c499 /deepseek-ai-dsh-cua/use-this-package: - en: 4672d9e873336c60 - zh: 034fa7a7528a1d8b + en: 52cacbe2d2aeb2bc + zh: b3c65814d9a17052 /deepseek-ai-dsh-cua/understand-the-implementation: en: effcee57d8854c32 zh: d02c6b76e7249051 diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index 623168dea2..dcd8dc186d 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -27,7 +27,7 @@ Inspect applications, retain window objects across PTC calls, and send actions t Mount the [macOS provider](../cua-macos/README.md) to enable native operations. External PTC hosts import `connectCua` from `@deepseek-ai/dsh-cua/bridge` and supply the current `tools.cua` binding. Source compositions and native preparation belong to the provider. -Use App methods for a single-window task. `getApp` reads initial accessibility state; `getWindows` and `getWindow` select an explicit window when needed. Re-reading one app/window scope replaces its element indices without invalidating another window's snapshot. Keyboard input follows the application's focus. +Use App methods for a single-window task. `getApp` reads initial accessibility state; `getWindows` and `getWindow` select an explicit window when needed. Re-reading one app/window scope replaces its element indices without invalidating another window's snapshot. Keyboard input follows the application's focus. Invalid request arguments reject before dispatch with a readable method, parameter and requirement. Observation methods emit and return their result. Use `void await app.getAXState()` for one visible copy, or use `emit:false` and print the returned text once. Screenshots return `CuaScreenshot` with `bytes`, `mimeType`, `pixelWidth`, `pixelHeight`, and `windowId`. The image sink receives the same object. Use its MIME type when selecting a file extension or declaring an image; the format can be PNG, JPEG, WebP, or GIF. Each screenshot enters the attachment pipeline once; `emit:false` only suppresses the object sink. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index d8a40fbc0d..a526735ec5 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -27,7 +27,7 @@ kind: "package-library" 挂载 [macOS 提供方](../cua-macos/README.zh.md)以启用原生操作。外部 PTC 宿主从 `@deepseek-ai/dsh-cua/bridge` 导入 `connectCua`，并提供当前 `tools.cua` 绑定。源码组合与原生组件准备由提供方负责。 -单窗口任务直接使用 App 方法。`getApp` 读取初始辅助功能状态；需要指定窗口时使用 `getWindows` 和 `getWindow`。重新读取一个应用或窗口范围会替换其中的元素编号，不使其他窗口的快照失效。键盘输入跟随应用焦点。 +单窗口任务直接使用 App 方法。`getApp` 读取初始辅助功能状态；需要指定窗口时使用 `getWindows` 和 `getWindow`。重新读取一个应用或窗口范围会替换其中的元素编号，不使其他窗口的快照失效。键盘输入跟随应用焦点。请求参数校验失败时，在分派前返回包含方法、参数和要求的可读消息。 观察方法会输出并返回结果。使用 `void await app.getAXState()` 只展示一份内容，或使用 `emit:false` 后打印一次返回文本。截图返回包含 `bytes`、`mimeType`、`pixelWidth`、`pixelHeight` 和 `windowId` 的 `CuaScreenshot`，图片输出回调接收同一个对象。选择文件后缀或声明图片时使用其中的 MIME 类型；格式可能是 PNG、JPEG、WebP 或 GIF。每张截图只进入一次附件处理链；`emit:false` 只抑制对象层的输出。 diff --git a/packages/experimental/cua/src/api.ts b/packages/experimental/cua/src/api.ts index d2e3347075..0be64c2358 100644 --- a/packages/experimental/cua/src/api.ts +++ b/packages/experimental/cua/src/api.ts @@ -1,10 +1,28 @@ /** Object facade over logged native operations; its caller owns evaluator persistence. */ import { Buffer } from 'node:buffer' +import type { ZodIssue } from 'zod' import type { JsonValue } from '@deepseek-ai/dsh-util-values' import { ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from './protocol.ts' import type { CuaApi, CuaApp, CuaAppHandle, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaOperation, CuaScreenshot, CuaSurface, CuaWindow } from './types.ts' export type * from './types.ts' +function requestIssueMessage(issues: readonly ZodIssue[], parent: readonly PropertyKey[] = []): string { + return [...new Set(issues.map((issue) => { + const path = [...parent, ...issue.path] + if (issue.code === 'invalid_union') { + const fewest = Math.min(...issue.errors.map(branch => branch.length)) + return [...new Set(issue.errors.filter(branch => branch.length === fewest) + .map(branch => requestIssueMessage(branch, path)))].join(' or ') + } + // The facade supplies the method and wrapper; failures belong to params. + const fields = path.slice(1) + if (issue.code === 'unrecognized_keys') { + return `${issue.keys.map(key => [...fields, key].join('.')).join(', ')}: not accepted for these arguments` + } + return `${fields.join('.')}: ${issue.message}` + }))].join('; ') +} + function keyChord(chord: string): { key: string; modifiers: string[] } { const parts = chord.split('+') let key = parts.pop() @@ -33,7 +51,11 @@ function keyChord(chord: string): { key: string; modifiers: string[] } { * @returns Objects independent of variable storage and evaluator lifecycle. */ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): CuaApi { - const call = (method: CuaOperation['method'], params: Record) => dispatch(Request.parse({ method, params })) + const call = (method: CuaOperation['method'], params: Record) => { + const request = Request.safeParse({ method, params }) + if (!request.success) throw new Error(`${method} ${requestIssueMessage(request.error.issues)}`) + return dispatch(request.data) + } const action = async (method: CuaOperation['method'], params: Record) => ActionResult.parse(await call(method, params)) async function emit(value: object, options?: CuaObservationOptions): Promise { if (options?.emit !== false) await sink?.text(JSON.stringify(value)) diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index 5d1fb0d608..46eddd2e1c 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -29,7 +29,8 @@ const imageBase64 = z.string().min(1).refine(value => Buffer.from(value, 'base64 /** Model-submittable operation parameters; Host resource fields and private continuations are separate. */ export const Request = z.discriminatedUnion('method', [ - operation('getCapabilities', {}), operation('listApps', {}), operation('findApps', { query: z.string().min(1).regex(/\S/) }), + operation('getCapabilities', {}), operation('listApps', {}), + operation('findApps', { query: z.string().min(1, 'Expected a non-empty search string').regex(/\S/, 'Expected a non-empty search string') }), operation('getApp', { selector: z.string().min(1) }), operation('activate', app), operation('show', { ...app, windowId: id.optional() }), diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index abde95a5b6..6b2f04251e 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -94,6 +94,12 @@ describe('application discovery', () => { await expect(f.api.findApps(' ')).rejects.toThrow() }) + it.each(['', ' '])('names the invalid search parameter without exposing schema internals: %j', async (query) => { + const f = fixture() + await expect(f.api.findApps(query)).rejects.toThrow('findApps query: Expected a non-empty search string') + expect(f.dispatch).not.toHaveBeenCalled() + }) + it('rejects malformed discovery and capability results', async () => { const f = fixture() f.dispatch.mockResolvedValueOnce({ platform: 'macos' }) @@ -467,6 +473,20 @@ describe('input actions', () => { ['extra plus delimiter', app => app.pressKey('cmd+++')], ['unknown modifier', app => app.pressKey('unknown+a')], ] + it.each<[string, (app: CuaApp) => Promise]>([ + ['getState windowId: Too small: expected number to be >0', app => app.getWindow(0)], + ['selectText text: Too small: expected string to have >=1 characters', app => app.selectText(0, '')], + ['hover x: Invalid input: expected number, received NaN', app => app.hover([NaN, 0])], + ['hover x: Too big: expected number to be <=1000', app => app.hover([1001, 0], { relative: true })], + ['click relative: not accepted for these arguments', app => app.click(650, { relative: true })], + ])('reports a readable validation failure: %s', async (message, run) => { + const f = fixture() + const app = await f.api.getApp('Editor') + f.dispatch.mockClear() + await expect(run(app)).rejects.toThrow(message) + expect(f.dispatch).not.toHaveBeenCalled() + }) + it.each(invalidActions)('rejects %s without invoking native input', async (_label, run) => { const f = fixture() const app = await f.api.getApp('Editor')