feat(cua): define application types and validate the operation protocol

Define the evaluator-independent app/window API and validate method-specific requests and native responses. Test invalid targets and parameter admission.

packages/experimental/cua/src/protocol.ts

@@ -0,0 +1,65 @@
+/** JSON validation shared by model tool admission and the native object facade. */
+import { z } from 'zod'
+import type { CuaAppHandle, CuaSnapshotId } from './types.ts'
+
+const id = z.number().int().positive()
+const point = z.number().nonnegative()
+const app = { appHandle: z.string().min(1) }
+const window = { ...app, windowId: id }
+const snapshot = { ...app, snapshotId: z.string().min(1), elementIndex: z.number().int().nonnegative() }
+const coordinate = { ...window, snapshotId: z.string().min(1).optional(), x: point, y: point }
+const bounds = z.object({
+  x: z.number(), y: z.number(), width: z.number().nonnegative(), height: z.number().nonnegative(),
+})
+const operation = <M extends string, S extends z.ZodRawShape>(method: M, params: S) => z.object({
+  method: z.literal(method), params: z.object(params).strict(),
+}).strict()
+
+/** Complete native request schema; resource bounds are supplied by the host. */
+export const Request = z.discriminatedUnion('method', [
+  operation('getCapabilities', {}), operation('listApps', {}),
+  operation('getApp', { selector: z.string().min(1) }),
+  operation('listWindows', app),
+  operation('getState', { ...app, windowId: id.optional() }),
+  z.object({ method: z.literal('click'), params: z.union([
+    z.object(snapshot).strict(),
+    z.object({ ...coordinate, mouseButton: z.enum(['left', 'right']).optional(), clickCount: z.union([z.literal(1), z.literal(2)]).optional() }).strict(),
+  ]) }).strict(),
+  operation('setValue', { ...snapshot, value: z.string() }),
+  operation('typeText', { ...app, text: z.string() }),
+  operation('pressKey', { ...app, key: z.string().min(1), modifiers: z.array(z.enum(['command', 'control', 'option', 'shift', 'fn'])).optional() }),
+  operation('scroll', { ...coordinate, deltaLinesX: z.number().int().min(-10).max(10), deltaLinesY: z.number().int().min(-10).max(10) }),
+  operation('drag', { ...window, snapshotId: z.string().min(1).optional(), fromX: point, fromY: point, toX: point, toY: point }),
+  operation('screenshot', window),
+])
+
+/** Process identity returned only after a successful app selection. */
+export const AppResult = z.object({
+  appHandle: z.string().min(1).transform(value => value as CuaAppHandle), pid: id, name: z.string(), bundleId: z.string().optional(),
+})
+/** Running application records from the helper. */
+export const AppsResult = z.array(z.object({ pid: id, name: z.string(), bundleId: z.string().optional() }))
+/** Current windows and their global point bounds. */
+export const WindowsResult = z.array(z.object({
+  windowId: id, pid: id, appName: z.string(), title: z.string(), bounds,
+  isOnScreen: z.boolean(), onCurrentSpace: z.boolean().nullable(),
+}))
+/** Observed state with window and process identity for stale-target rejection. */
+export const StateResult = z.object({
+  appHandle: z.string().min(1).transform(value => value as CuaAppHandle),
+  pid: id, windowId: id.nullable(),
+  snapshotId: z.string().min(1).transform(value => value as CuaSnapshotId),
+  tree: z.string(), windows: WindowsResult, truncated: z.boolean(),
+})
+/** PNG data and coordinate metadata; pixel dimensions do not define input coordinates. */
+export const ScreenshotResult = z.object({
+  mimeType: z.literal('image/png'), data: z.base64().min(1), pixelWidth: id, pixelHeight: id, windowId: id,
+  bounds: bounds.extend({ width: z.number().positive(), height: z.number().positive() }), coordinateSpace: z.literal('window-points'),
+})
+/** Permissions are reported without opening system permission dialogs. */
+export const CapabilitiesResult = z.object({
+  platform: z.string(), permissions: z.object({ accessibility: z.boolean(), screenRecording: z.boolean() }),
+  methods: z.array(z.string()), coordinateSpace: z.literal('window-points'), browser: z.literal(false),
+})
+/** An action has reached native dispatch; application state was not verified. */
+export const ActionResult = z.object({ dispatched: z.literal(true), verified: z.literal(false) })

packages/experimental/cua/src/types.ts

@@ -0,0 +1,165 @@
+/** Native application objects independent of evaluator ownership and persistence. */
+import type { Branded } from '@deepseek-ai/dsh-brand'
+import type { JsonValue } from '@deepseek-ai/dsh-util-values'
+
+/** Helper-issued application identity, bound to one process lifetime. */
+export type CuaAppHandle = Branded<'CuaAppHandle'>
+/** Helper-issued accessibility observation identity. */
+export type CuaSnapshotId = Branded<'CuaSnapshotId'>
+/** One operation sent through an authorized and logged host dispatcher. */
+export interface CuaOperation { method: string; params: Record<string, JsonValue> }
+/** Dispatch exactly once; a failed action must not be retried implicitly. */
+export type CuaDispatch = (operation: CuaOperation) => Promise<JsonValue>
+/** Optional output owned by the caller; persistence and image admission are external. */
+export interface CuaObservationSink {
+  /**
+   * Emit text.
+   * @param value - observed application state.
+   */
+  text(value: string): void | Promise<void>
+  /**
+   * Emit an image.
+   * @param data - PNG bytes.
+   */
+  image(data: Uint8Array): void | Promise<void>
+}
+/** Observations emit by default. Returned values are unaffected by this option. */
+export interface CuaObservationOptions { emit?: boolean }
+/** Diffing is not implemented; all reads return a complete tree. */
+export interface CuaAxOptions extends CuaObservationOptions { disableDiffing?: boolean }
+/** Available permissions and operations reported by the selected native implementation. */
+export interface CuaCapabilities {
+  platform: string
+  permissions: { accessibility: boolean; screenRecording: boolean }
+  methods: string[]
+  coordinateSpace: 'window-points'
+  browser: false
+}
+/** App discovery result accepted by getApp. */
+export interface CuaAppInfo { id: string; displayName: string; isRunning: true; pid: number }
+/** Window bounds in global macOS points. Local operations use an origin of [0,0]. */
+export interface CuaBounds { x: number; y: number; width: number; height: number }
+/** Observed window metadata; identifiers are valid only while the window exists. */
+export interface CuaWindowInfo {
+  windowId: number
+  pid: number
+  appName: string
+  title: string
+  bounds: CuaBounds
+  isOnScreen: boolean
+  onCurrentSpace: boolean | null
+}
+/** Successful input dispatch; application outcome still requires observation. */
+export interface CuaActionResult { dispatched: true; verified: false }
+/** AX element clicks support only one left click. */
+export interface CuaClickOptions { mouseButton?: 'left' | 'right'; clickCount?: 1 | 2 }
+/** Positive deltas reveal content above/left; each axis is an integer from -10 to 10. */
+export interface CuaScrollOptions { x: number; y: number; deltaLinesX: number; deltaLinesY: number }
+/** An observed AX tree and independently captured PNG; capture is not atomic. */
+export interface CuaObservation { state: string; screenshot: Uint8Array }
+/** App-wide or explicitly scoped window observations and actions. */
+export interface CuaSurface {
+  /**
+   * Read a fresh full tree.
+   * @param options - emission and diff options.
+   * @returns AX text with element indices.
+   */
+  getAXState(options?: CuaAxOptions): Promise<string>
+  /**
+   * Capture the selected window.
+   * @param options - emission options.
+   * @returns PNG bytes, whose pixel dimensions may differ from window points.
+   */
+  getScreenshot(options?: CuaObservationOptions): Promise<Uint8Array>
+  /**
+   * Read AX then capture.
+   * @param options - observation options.
+   * @returns both non-atomic observations.
+   */
+  getAXStateAndScreenshot(options?: CuaAxOptions): Promise<CuaObservation>
+  /**
+   * Click a current AX index or window point.
+   * @param target - index or [x,y].
+   * @param options - button and click count.
+   * @returns dispatch evidence.
+   */
+  click(target: number | readonly [number, number], options?: CuaClickOptions): Promise<CuaActionResult>
+  /**
+   * Set an observed control value.
+   * @param index - index from this surface's latest tree.
+   * @param value - replacement value.
+   * @returns dispatch evidence.
+   */
+  setValue(index: number, value: string): Promise<CuaActionResult>
+  /**
+   * Scroll at a window point.
+   * @param options - point and line deltas; at least one delta must be nonzero.
+   * @returns dispatch evidence.
+   */
+  scroll(options: CuaScrollOptions): Promise<CuaActionResult>
+  /**
+   * Drag between points in one window.
+   * @param from - starting window point.
+   * @param to - destination window point.
+   * @returns dispatch evidence.
+   */
+  drag(from: readonly [number, number], to: readonly [number, number]): Promise<CuaActionResult>
+}
+/** One explicit window; keyboard operations remain app-scoped because they follow focus. */
+export interface CuaWindow extends CuaSurface { readonly windowId: number }
+/** Native app handle; implicit coordinate operations require one observed window. */
+export interface CuaApp extends CuaSurface {
+  readonly pid: number
+  readonly appHandle: CuaAppHandle
+  /**
+   * Enumerate current windows.
+   * @param options - output options.
+   * @returns current window records.
+   */
+  getWindows(options?: CuaObservationOptions): Promise<CuaWindowInfo[]>
+  /**
+   * Observe a specific window.
+   * @param windowId - ID obtained from getWindows.
+   * @returns a reusable window object with its initial AX state emitted.
+   */
+  getWindow(windowId: number): Promise<CuaWindow>
+  /**
+   * Input at the application's current focus.
+   * @param text - literal text; cancellation may leave a prefix entered.
+   * @returns dispatch evidence.
+   */
+  typeText(text: string): Promise<CuaActionResult>
+  /**
+   * Send a key/chord at current focus.
+   * @param key - xdotool-style chord such as super+c or shift+Tab.
+   * @returns dispatch evidence.
+   */
+  pressKey(key: string): Promise<CuaActionResult>
+}
+/** Native app API; its caller owns global bindings and variable persistence. */
+export interface CuaApi {
+  /**
+   * Inspect permissions without requesting them.
+   * @param options - output options.
+   * @returns reported native capabilities.
+   */
+  getCapabilities(options?: CuaObservationOptions): Promise<CuaCapabilities>
+  /**
+   * Enumerate running applications.
+   * @param options - output options.
+   * @returns identifiers accepted by getApp.
+   */
+  listApps(options?: CuaObservationOptions): Promise<CuaAppInfo[]>
+  /**
+   * Discover apps.
+   * @param options - output options.
+   * @returns native apps and an empty browser list.
+   */
+  getState(options?: CuaObservationOptions): Promise<{ apps: CuaAppInfo[]; browsers: [] }>
+  /**
+   * Select an existing app and emit initial AX state.
+   * @param selector - exact name, bundle ID or PID string.
+   * @returns a handle tied to this process lifetime.
+   */
+  getApp(selector: string): Promise<CuaApp>
+}

packages/experimental/cua/tests/protocol.spec.ts

@@ -0,0 +1,26 @@
+/** Model-authored operation JSON cannot supply host configuration or bypass retained identities. */
+import { expect, it } from 'vitest'
+import type { JsonValue } from '@deepseek-ai/dsh-util-values'
+import { Request, ScreenshotResult } from '../src/protocol.ts'
+
+const rejectedRequests: Array<{ reason: string; request: JsonValue }> = [
+  { reason: 'raw process ID instead of a selected app handle', request: { method: 'typeText', params: { pid: 17, text: 'hello' } } },
+  { reason: 'unknown native method', request: { method: 'shutdown', params: {} } },
+  { reason: 'host-owned traversal limit', request: { method: 'getState', params: { appHandle: 'app', maxElements: 1000000 } } },
+  { reason: 'extra tool argument', request: { method: 'listApps', params: {}, helperPath: '/untrusted/helper' } },
+  { reason: 'element click without an observed snapshot', request: { method: 'click', params: { appHandle: 'app', elementIndex: 0 } } },
+  { reason: 'coordinate click without a window', request: { method: 'click', params: { appHandle: 'app', x: 1, y: 2 } } },
+  { reason: 'mixed element and coordinate click', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementIndex: 0, windowId: 23, x: 1, y: 2 } } },
+  { reason: 'unsupported modifier', request: { method: 'pressKey', params: { appHandle: 'app', key: 'a', modifiers: ['hyper'] } } },
+]
+
+it.each(rejectedRequests)('rejects $reason at operation admission', ({ request }) => {
+  expect(Request.safeParse(request).success).toBe(false)
+})
+
+it('rejects malformed screenshot encoding before decoding bytes', () => {
+  expect(ScreenshotResult.safeParse({
+    mimeType: 'image/png', data: 'not base64!', pixelWidth: 1, pixelHeight: 1, windowId: 23,
+    bounds: { x: 0, y: 0, width: 10, height: 10 }, coordinateSpace: 'window-points',
+  }).success).toBe(false)
+})