diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts new file mode 100644 index 0000000000..8575a7926d --- /dev/null +++ b/packages/experimental/cua/src/protocol.ts @@ -0,0 +1,65 @@ +/** JSON validation shared by model tool admission and the native object facade. */ +import { z } from 'zod' +import type { CuaAppHandle, CuaSnapshotId } from './types.ts' + +const id = z.number().int().positive() +const point = z.number().nonnegative() +const app = { appHandle: z.string().min(1) } +const window = { ...app, windowId: id } +const snapshot = { ...app, snapshotId: z.string().min(1), elementIndex: z.number().int().nonnegative() } +const coordinate = { ...window, snapshotId: z.string().min(1).optional(), x: point, y: point } +const bounds = z.object({ + x: z.number(), y: z.number(), width: z.number().nonnegative(), height: z.number().nonnegative(), +}) +const operation = (method: M, params: S) => z.object({ + method: z.literal(method), params: z.object(params).strict(), +}).strict() + +/** Complete native request schema; resource bounds are supplied by the host. */ +export const Request = z.discriminatedUnion('method', [ + operation('getCapabilities', {}), operation('listApps', {}), + operation('getApp', { selector: z.string().min(1) }), + operation('listWindows', app), + operation('getState', { ...app, windowId: id.optional() }), + z.object({ method: z.literal('click'), params: z.union([ + z.object(snapshot).strict(), + z.object({ ...coordinate, mouseButton: z.enum(['left', 'right']).optional(), clickCount: z.union([z.literal(1), z.literal(2)]).optional() }).strict(), + ]) }).strict(), + operation('setValue', { ...snapshot, value: z.string() }), + operation('typeText', { ...app, text: z.string() }), + operation('pressKey', { ...app, key: z.string().min(1), modifiers: z.array(z.enum(['command', 'control', 'option', 'shift', 'fn'])).optional() }), + operation('scroll', { ...coordinate, deltaLinesX: z.number().int().min(-10).max(10), deltaLinesY: z.number().int().min(-10).max(10) }), + operation('drag', { ...window, snapshotId: z.string().min(1).optional(), fromX: point, fromY: point, toX: point, toY: point }), + operation('screenshot', window), +]) + +/** Process identity returned only after a successful app selection. */ +export const AppResult = z.object({ + appHandle: z.string().min(1).transform(value => value as CuaAppHandle), pid: id, name: z.string(), bundleId: z.string().optional(), +}) +/** Running application records from the helper. */ +export const AppsResult = z.array(z.object({ pid: id, name: z.string(), bundleId: z.string().optional() })) +/** Current windows and their global point bounds. */ +export const WindowsResult = z.array(z.object({ + windowId: id, pid: id, appName: z.string(), title: z.string(), bounds, + isOnScreen: z.boolean(), onCurrentSpace: z.boolean().nullable(), +})) +/** Observed state with window and process identity for stale-target rejection. */ +export const StateResult = z.object({ + appHandle: z.string().min(1).transform(value => value as CuaAppHandle), + pid: id, windowId: id.nullable(), + snapshotId: z.string().min(1).transform(value => value as CuaSnapshotId), + tree: z.string(), windows: WindowsResult, truncated: z.boolean(), +}) +/** PNG data and coordinate metadata; pixel dimensions do not define input coordinates. */ +export const ScreenshotResult = z.object({ + mimeType: z.literal('image/png'), data: z.base64().min(1), pixelWidth: id, pixelHeight: id, windowId: id, + bounds: bounds.extend({ width: z.number().positive(), height: z.number().positive() }), coordinateSpace: z.literal('window-points'), +}) +/** Permissions are reported without opening system permission dialogs. */ +export const CapabilitiesResult = z.object({ + platform: z.string(), permissions: z.object({ accessibility: z.boolean(), screenRecording: z.boolean() }), + methods: z.array(z.string()), coordinateSpace: z.literal('window-points'), browser: z.literal(false), +}) +/** An action has reached native dispatch; application state was not verified. */ +export const ActionResult = z.object({ dispatched: z.literal(true), verified: z.literal(false) }) diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts new file mode 100644 index 0000000000..707b7fc433 --- /dev/null +++ b/packages/experimental/cua/src/types.ts @@ -0,0 +1,165 @@ +/** Native application objects independent of evaluator ownership and persistence. */ +import type { Branded } from '@deepseek-ai/dsh-brand' +import type { JsonValue } from '@deepseek-ai/dsh-util-values' + +/** Helper-issued application identity, bound to one process lifetime. */ +export type CuaAppHandle = Branded<'CuaAppHandle'> +/** Helper-issued accessibility observation identity. */ +export type CuaSnapshotId = Branded<'CuaSnapshotId'> +/** One operation sent through an authorized and logged host dispatcher. */ +export interface CuaOperation { method: string; params: Record } +/** Dispatch exactly once; a failed action must not be retried implicitly. */ +export type CuaDispatch = (operation: CuaOperation) => Promise +/** Optional output owned by the caller; persistence and image admission are external. */ +export interface CuaObservationSink { + /** + * Emit text. + * @param value - observed application state. + */ + text(value: string): void | Promise + /** + * Emit an image. + * @param data - PNG bytes. + */ + image(data: Uint8Array): void | Promise +} +/** Observations emit by default. Returned values are unaffected by this option. */ +export interface CuaObservationOptions { emit?: boolean } +/** Diffing is not implemented; all reads return a complete tree. */ +export interface CuaAxOptions extends CuaObservationOptions { disableDiffing?: boolean } +/** Available permissions and operations reported by the selected native implementation. */ +export interface CuaCapabilities { + platform: string + permissions: { accessibility: boolean; screenRecording: boolean } + methods: string[] + coordinateSpace: 'window-points' + browser: false +} +/** App discovery result accepted by getApp. */ +export interface CuaAppInfo { id: string; displayName: string; isRunning: true; pid: number } +/** Window bounds in global macOS points. Local operations use an origin of [0,0]. */ +export interface CuaBounds { x: number; y: number; width: number; height: number } +/** Observed window metadata; identifiers are valid only while the window exists. */ +export interface CuaWindowInfo { + windowId: number + pid: number + appName: string + title: string + bounds: CuaBounds + isOnScreen: boolean + onCurrentSpace: boolean | null +} +/** Successful input dispatch; application outcome still requires observation. */ +export interface CuaActionResult { dispatched: true; verified: false } +/** AX element clicks support only one left click. */ +export interface CuaClickOptions { mouseButton?: 'left' | 'right'; clickCount?: 1 | 2 } +/** Positive deltas reveal content above/left; each axis is an integer from -10 to 10. */ +export interface CuaScrollOptions { x: number; y: number; deltaLinesX: number; deltaLinesY: number } +/** An observed AX tree and independently captured PNG; capture is not atomic. */ +export interface CuaObservation { state: string; screenshot: Uint8Array } +/** App-wide or explicitly scoped window observations and actions. */ +export interface CuaSurface { + /** + * Read a fresh full tree. + * @param options - emission and diff options. + * @returns AX text with element indices. + */ + getAXState(options?: CuaAxOptions): Promise + /** + * Capture the selected window. + * @param options - emission options. + * @returns PNG bytes, whose pixel dimensions may differ from window points. + */ + getScreenshot(options?: CuaObservationOptions): Promise + /** + * Read AX then capture. + * @param options - observation options. + * @returns both non-atomic observations. + */ + getAXStateAndScreenshot(options?: CuaAxOptions): Promise + /** + * Click a current AX index or window point. + * @param target - index or [x,y]. + * @param options - button and click count. + * @returns dispatch evidence. + */ + click(target: number | readonly [number, number], options?: CuaClickOptions): Promise + /** + * Set an observed control value. + * @param index - index from this surface's latest tree. + * @param value - replacement value. + * @returns dispatch evidence. + */ + setValue(index: number, value: string): Promise + /** + * Scroll at a window point. + * @param options - point and line deltas; at least one delta must be nonzero. + * @returns dispatch evidence. + */ + scroll(options: CuaScrollOptions): Promise + /** + * Drag between points in one window. + * @param from - starting window point. + * @param to - destination window point. + * @returns dispatch evidence. + */ + drag(from: readonly [number, number], to: readonly [number, number]): Promise +} +/** One explicit window; keyboard operations remain app-scoped because they follow focus. */ +export interface CuaWindow extends CuaSurface { readonly windowId: number } +/** Native app handle; implicit coordinate operations require one observed window. */ +export interface CuaApp extends CuaSurface { + readonly pid: number + readonly appHandle: CuaAppHandle + /** + * Enumerate current windows. + * @param options - output options. + * @returns current window records. + */ + getWindows(options?: CuaObservationOptions): Promise + /** + * Observe a specific window. + * @param windowId - ID obtained from getWindows. + * @returns a reusable window object with its initial AX state emitted. + */ + getWindow(windowId: number): Promise + /** + * Input at the application's current focus. + * @param text - literal text; cancellation may leave a prefix entered. + * @returns dispatch evidence. + */ + typeText(text: string): Promise + /** + * Send a key/chord at current focus. + * @param key - xdotool-style chord such as super+c or shift+Tab. + * @returns dispatch evidence. + */ + pressKey(key: string): Promise +} +/** Native app API; its caller owns global bindings and variable persistence. */ +export interface CuaApi { + /** + * Inspect permissions without requesting them. + * @param options - output options. + * @returns reported native capabilities. + */ + getCapabilities(options?: CuaObservationOptions): Promise + /** + * Enumerate running applications. + * @param options - output options. + * @returns identifiers accepted by getApp. + */ + listApps(options?: CuaObservationOptions): Promise + /** + * Discover apps. + * @param options - output options. + * @returns native apps and an empty browser list. + */ + getState(options?: CuaObservationOptions): Promise<{ apps: CuaAppInfo[]; browsers: [] }> + /** + * Select an existing app and emit initial AX state. + * @param selector - exact name, bundle ID or PID string. + * @returns a handle tied to this process lifetime. + */ + getApp(selector: string): Promise +} diff --git a/packages/experimental/cua/tests/protocol.spec.ts b/packages/experimental/cua/tests/protocol.spec.ts new file mode 100644 index 0000000000..9c06d68670 --- /dev/null +++ b/packages/experimental/cua/tests/protocol.spec.ts @@ -0,0 +1,26 @@ +/** Model-authored operation JSON cannot supply host configuration or bypass retained identities. */ +import { expect, it } from 'vitest' +import type { JsonValue } from '@deepseek-ai/dsh-util-values' +import { Request, ScreenshotResult } from '../src/protocol.ts' + +const rejectedRequests: Array<{ reason: string; request: JsonValue }> = [ + { reason: 'raw process ID instead of a selected app handle', request: { method: 'typeText', params: { pid: 17, text: 'hello' } } }, + { reason: 'unknown native method', request: { method: 'shutdown', params: {} } }, + { reason: 'host-owned traversal limit', request: { method: 'getState', params: { appHandle: 'app', maxElements: 1000000 } } }, + { reason: 'extra tool argument', request: { method: 'listApps', params: {}, helperPath: '/untrusted/helper' } }, + { reason: 'element click without an observed snapshot', request: { method: 'click', params: { appHandle: 'app', elementIndex: 0 } } }, + { reason: 'coordinate click without a window', request: { method: 'click', params: { appHandle: 'app', x: 1, y: 2 } } }, + { reason: 'mixed element and coordinate click', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementIndex: 0, windowId: 23, x: 1, y: 2 } } }, + { reason: 'unsupported modifier', request: { method: 'pressKey', params: { appHandle: 'app', key: 'a', modifiers: ['hyper'] } } }, +] + +it.each(rejectedRequests)('rejects $reason at operation admission', ({ request }) => { + expect(Request.safeParse(request).success).toBe(false) +}) + +it('rejects malformed screenshot encoding before decoding bytes', () => { + expect(ScreenshotResult.safeParse({ + mimeType: 'image/png', data: 'not base64!', pixelWidth: 1, pixelHeight: 1, windowId: 23, + bounds: { x: 0, y: 0, width: 10, height: 10 }, coordinateSpace: 'window-points', + }).success).toBe(false) +})