refactor(ptc): simplify image references and CUA bridge
This commit is contained in:
1 parent
4e72e43037
commit
3dbe1c126f
9 files changed
+67
-136
No files matched your search
@@ -257,7 +257,6 @@ function renderRunOutput(
|
||||
value: RunCodeOutput,
|
||||
guidance: string,
|
||||
images: PtcImageOutput[] = [],
|
||||
imageRefs?: Pick<WeakMap<PtcImage, Extract<ContentBlock, { type: 'image' }>['attachment']>, 'get'>,
|
||||
): ContentBlock[] {
|
||||
const content: ContentBlock[] = []
|
||||
const appendText = (text: string): void => { if (text.length > 0) content.push({ type: 'text', text }) }
|
||||
@@ -265,9 +264,7 @@ function renderRunOutput(
|
||||
for (const output of images) {
|
||||
appendText(value.logs.slice(position, output.afterLog).join('\n'))
|
||||
position = output.afterLog
|
||||
const attachment = imageRefs?.get(output.image)
|
||||
if (attachment === undefined) throw new Error('Inspected image is unavailable')
|
||||
content.push({ type: 'image', attachment })
|
||||
content.push({ type: 'image', attachment: output.attachment })
|
||||
}
|
||||
const remaining = value.logs.slice(position)
|
||||
if (value.result !== undefined) remaining.push(renderValue(value.result))
|
||||
@@ -325,7 +322,6 @@ export function createRunCodeTool(
|
||||
}
|
||||
|
||||
type ImageRef = Extract<ContentBlock, { type: 'image' }>['attachment']
|
||||
const imageRefs = new WeakMap<PtcImage, ImageRef>()
|
||||
const inspections = new WeakMap<ToolExecution, { value?: RunCodeOutput; images: PtcImageOutput[] }>()
|
||||
const programImage = (ref: ImageRef): PtcImage => {
|
||||
const attachments = ctx.get('attachments')
|
||||
@@ -333,9 +329,7 @@ export function createRunCodeTool(
|
||||
const access = attachments === undefined || fs === undefined ? undefined
|
||||
: resolveImageAttachmentAccess(attachments, path => fs.processPathFromHostPath(path), ref)
|
||||
if (access === undefined) throw new Error('Image storage must expose a readable path in the program filesystem')
|
||||
const image = new PtcImage(access.readonlyPath, ref.width, ref.height)
|
||||
imageRefs.set(image, ref)
|
||||
return image
|
||||
return new PtcImage(access.readonlyPath, ref)
|
||||
}
|
||||
const stateOwners = new WeakMap<object, WeakSet<PtcRuntime>>()
|
||||
const definition = defineTool({
|
||||
@@ -749,9 +743,6 @@ export function createRunCodeTool(
|
||||
}
|
||||
|
||||
if (result.images !== undefined && result.images.length > 0) {
|
||||
for (const output of result.images) {
|
||||
if (!imageRefs.has(output.image)) throw new Error('Runtime returned an image outside this executor')
|
||||
}
|
||||
inspections.set(exec, { images: result.images })
|
||||
}
|
||||
if (result.error) {
|
||||
@@ -791,14 +782,10 @@ export function createRunCodeTool(
|
||||
if (selected === undefined) return undefined
|
||||
if (result.isError) {
|
||||
if (selected.value !== undefined) return undefined
|
||||
return [...result.content, ...selected.images.map((output) => {
|
||||
const attachment = imageRefs.get(output.image)
|
||||
if (attachment === undefined) throw new Error('Inspected image is unavailable')
|
||||
return { type: 'image' as const, attachment }
|
||||
})]
|
||||
return [...result.content, ...selected.images.map(output => ({ type: 'image' as const, attachment: output.attachment }))]
|
||||
}
|
||||
if (selected.value === undefined || !isDeepStrictEqual(selected.value, result.value)) return undefined
|
||||
return renderRunOutput(selected.value, escalationGuidance(ctx.get('ptcRuntime')?.sandboxMode), selected.images, imageRefs)
|
||||
return renderRunOutput(selected.value, escalationGuidance(ctx.get('ptcRuntime')?.sandboxMode), selected.images)
|
||||
},
|
||||
// The model-authored description is the call's always-visible UI label
|
||||
// (the bash `description` precedent); the program itself rides rawInput.
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
/** Object facade over logged native operations; its caller owns evaluator persistence. */
|
||||
import { Buffer } from 'node:buffer'
|
||||
import type { z } from 'zod'
|
||||
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
|
||||
import { ActionNamesResult, ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from './protocol.ts'
|
||||
import type { CuaApi, CuaApp, CuaAppHandle, CuaCapabilities, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaOperation, CuaScreenshot, CuaSurface, CuaWindow } from './types.ts'
|
||||
import { ActionNamesResult, ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ShowResult, StateResult, WindowsResult } from './protocol.ts'
|
||||
import type { CuaApi, CuaApp, CuaAppHandle, CuaCapabilities, CuaDispatch, CuaObservationOptions, CuaOperation, CuaProgramImage, CuaSurface, CuaWindow } from './types.ts'
|
||||
export type * from './types.ts'
|
||||
|
||||
function requestIssueMessage(issues: readonly z.core.$ZodIssue[], parent: readonly PropertyKey[] = []): string {
|
||||
@@ -31,71 +30,49 @@ function validatedCall(dispatch: CuaDispatch) {
|
||||
}
|
||||
}
|
||||
|
||||
async function emit(sink: CuaObservationSink | undefined, value: object, options?: CuaObservationOptions): Promise<void> {
|
||||
if (options?.emit !== false) await sink?.text(JSON.stringify(value))
|
||||
async function emit(
|
||||
onText: ((text: string) => void | Promise<void>) | undefined, value: object, options?: CuaObservationOptions,
|
||||
): Promise<void> {
|
||||
if (options?.emit !== false) await onText?.(JSON.stringify(value))
|
||||
}
|
||||
|
||||
/**
|
||||
* Create app/window objects using the caller's authorized dispatcher.
|
||||
* @param dispatch - Current tool authority; rejects propagate without retries.
|
||||
* @param sink - Optional text/image output; emit false suppresses this sink only.
|
||||
* Create app/window objects using the caller's authorized program dispatcher.
|
||||
* @param dispatch - Current tool authority; screenshots resolve to program Images and rejections propagate without retries.
|
||||
* @param onText - Optional observation output; emit false suppresses this callback.
|
||||
* @returns Objects independent of variable storage and evaluator lifecycle.
|
||||
*/
|
||||
export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): CuaApi
|
||||
/**
|
||||
* Create app/window objects whose screenshot values are decoded by the caller.
|
||||
* @param dispatch - Current authorized tool dispatcher.
|
||||
* @param sink - Optional text output.
|
||||
* @param readImage - Decodes each screenshot result; the caller owns image output.
|
||||
* @returns App and window objects using the supplied image type.
|
||||
*/
|
||||
export function createCuaApi<TImage>(
|
||||
dispatch: CuaDispatch, sink: CuaObservationSink | undefined, readImage: (value: JsonValue, windowId: number) => TImage,
|
||||
): CuaApi<TImage>
|
||||
export function createCuaApi(
|
||||
dispatch: CuaDispatch, sink?: CuaObservationSink, readImage?: (value: JsonValue, windowId: number) => unknown,
|
||||
): CuaApi<unknown> {
|
||||
export function createCuaApi(dispatch: CuaDispatch, onText?: (text: string) => void | Promise<void>): CuaApi {
|
||||
const call = validatedCall(dispatch)
|
||||
let capabilities: CuaCapabilities | undefined
|
||||
async function currentCapabilities(): Promise<CuaCapabilities> {
|
||||
if (capabilities === undefined) capabilities = CapabilitiesResult.parse(await call('getCapabilities', {}))
|
||||
return capabilities
|
||||
}
|
||||
const screenshot = async (value: JsonValue, windowId: number, options?: CuaObservationOptions): Promise<unknown> => {
|
||||
if (readImage !== undefined) return readImage(value, windowId)
|
||||
const image = ScreenshotResult.parse(value)
|
||||
if (image.windowId !== windowId) throw new Error('Native screenshot belongs to another window')
|
||||
const screenshot: CuaScreenshot = {
|
||||
bytes: Uint8Array.from(Buffer.from(image.data, 'base64')),
|
||||
mimeType: image.mimeType, pixelWidth: image.pixelWidth, pixelHeight: image.pixelHeight, windowId: image.windowId,
|
||||
}
|
||||
if (options?.emit !== false) await sink?.image(screenshot)
|
||||
return screenshot
|
||||
}
|
||||
const objects = createCuaObjects(call, sink, screenshot)
|
||||
const objects = createCuaObjects(call, onText)
|
||||
|
||||
const api: CuaApi<unknown> = {
|
||||
const api: CuaApi = {
|
||||
async getCapabilities(options) {
|
||||
const result = CapabilitiesResult.parse(await call('getCapabilities', {}))
|
||||
capabilities = result
|
||||
await emit(sink, result, options)
|
||||
await emit(onText, result, options)
|
||||
return result
|
||||
},
|
||||
async listApps(options) {
|
||||
const result = AppsResult.parse(await call('listApps', {})).map(app => ({
|
||||
id: app.bundleId || String(app.pid), displayName: app.name, isRunning: true as const, pid: app.pid,
|
||||
}))
|
||||
await emit(sink, result, options)
|
||||
await emit(onText, result, options)
|
||||
return result
|
||||
},
|
||||
async findApps(query, options) {
|
||||
const result = FoundAppsResult.parse(await call('findApps', { query }))
|
||||
await emit(sink, result, options)
|
||||
await emit(onText, result, options)
|
||||
return result
|
||||
},
|
||||
async getState(options) {
|
||||
const result = { apps: await api.listApps({ emit: false }), browsers: [] as [] }
|
||||
await emit(sink, result, options)
|
||||
await emit(onText, result, options)
|
||||
return result
|
||||
},
|
||||
async getApp(selector) {
|
||||
@@ -113,9 +90,8 @@ export function createCuaApi(
|
||||
|
||||
type CuaScopeParams = (params: Record<string, JsonValue>) => Record<string, JsonValue>
|
||||
|
||||
function createCuaObjects<TImage>(call: ReturnType<typeof validatedCall>, sink: CuaObservationSink | undefined,
|
||||
readImage: (value: JsonValue, windowId: number, options?: CuaObservationOptions) => Promise<TImage>): {
|
||||
createCuaApp(appHandle: CuaAppHandle, capabilities: CuaCapabilities): CuaApp<TImage>
|
||||
function createCuaObjects(call: ReturnType<typeof validatedCall>, onText: ((text: string) => void | Promise<void>) | undefined): {
|
||||
createCuaApp(appHandle: CuaAppHandle, capabilities: CuaCapabilities): CuaApp
|
||||
} {
|
||||
const observed = new Map<CuaAppHandle, { snapshotId: string; windowId: number | undefined }>()
|
||||
function currentSnapshot(appHandle: CuaAppHandle): string {
|
||||
@@ -152,7 +128,7 @@ function createCuaObjects<TImage>(call: ReturnType<typeof validatedCall>, sink:
|
||||
throw new Error('Native observation belongs to another application or window')
|
||||
}
|
||||
observed.set(target.appHandle, { snapshotId: state.snapshotId, windowId: target.windowId })
|
||||
if (options?.emit !== false) await sink?.text(state.tree)
|
||||
if (options?.emit !== false) await onText?.(state.tree)
|
||||
return state.tree
|
||||
},
|
||||
async listActions(elementId) {
|
||||
@@ -202,7 +178,7 @@ function createCuaObjects<TImage>(call: ReturnType<typeof validatedCall>, sink:
|
||||
}
|
||||
return { surface, call: scopedCall, action, elementParams }
|
||||
}
|
||||
function createCuaWindow(appHandle: CuaAppHandle, windowId: number, capabilities: CuaCapabilities): CuaWindow<TImage> {
|
||||
function createCuaWindow(appHandle: CuaAppHandle, windowId: number, capabilities: CuaCapabilities): CuaWindow {
|
||||
const base = createCuaSurface({ appHandle, windowId },
|
||||
params => ({ ...params, appHandle, windowId }), () => undefined, () => {
|
||||
if (!capabilities.keyboard.includes('window')) {
|
||||
@@ -211,8 +187,13 @@ function createCuaObjects<TImage>(call: ReturnType<typeof validatedCall>, sink:
|
||||
})
|
||||
return {
|
||||
...base.surface, windowId,
|
||||
async getScreenshot(options) {
|
||||
return readImage(await base.call('screenshot', {}), windowId, options)
|
||||
async getScreenshot() {
|
||||
const value = await base.call('screenshot', {})
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)
|
||||
|| typeof value.path !== 'string' || typeof value.width !== 'number' || typeof value.height !== 'number') {
|
||||
throw new Error('The CUA program binding must return an Image for screenshots')
|
||||
}
|
||||
return value as CuaProgramImage
|
||||
},
|
||||
async mouseMove(targetPoint, options) {
|
||||
const relative = options?.relative === undefined ? {} : { relative: options.relative }
|
||||
@@ -255,7 +236,7 @@ function createCuaObjects<TImage>(call: ReturnType<typeof validatedCall>, sink:
|
||||
},
|
||||
}
|
||||
}
|
||||
function createCuaApp(appHandle: CuaAppHandle, capabilities: CuaCapabilities): CuaApp<TImage> {
|
||||
function createCuaApp(appHandle: CuaAppHandle, capabilities: CuaCapabilities): CuaApp {
|
||||
const base = createCuaSurface({ appHandle }, params => ({ ...params, appHandle }), () => {
|
||||
if (!capabilities.appLevelAccessibility) {
|
||||
throw new Error('Application-level accessibility is not supported on current platform; observe a window instead.')
|
||||
@@ -270,10 +251,10 @@ function createCuaObjects<TImage>(call: ReturnType<typeof validatedCall>, sink:
|
||||
async activate() { return base.action('activate', {}) },
|
||||
async getWindows(options) {
|
||||
const windows = WindowsResult.parse(await base.call('listWindows', {}))
|
||||
await emit(sink, windows, options)
|
||||
await emit(onText, windows, options)
|
||||
return windows
|
||||
},
|
||||
async getWindow(windowId): Promise<CuaWindow<TImage>> {
|
||||
async getWindow(windowId): Promise<CuaWindow> {
|
||||
const window = createCuaWindow(appHandle, windowId, capabilities)
|
||||
await window.getAccessibilityTree({ emit: false })
|
||||
return window
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
/** Connect retained application objects to the calling program's current logged tool binding. */
|
||||
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
|
||||
import { createCuaApi } from './api.ts'
|
||||
import type { CuaApi, CuaOperation, CuaProgramImage } from './types.ts'
|
||||
import type { CuaApi, CuaOperation } from './types.ts'
|
||||
|
||||
/** Standard tools.cua binding; screenshots resolve to stored program images. */
|
||||
export type CuaToolBinding = (operation: CuaOperation) => Promise<JsonValue>
|
||||
@@ -12,15 +12,6 @@ export type CuaToolBinding = (operation: CuaOperation) => Promise<JsonValue>
|
||||
* @param onText - Optional text output captured by the caller's execution environment.
|
||||
* @returns Application and window objects using the current tool binding. Rejections propagate without retries.
|
||||
*/
|
||||
export function connectCua(getTool: () => CuaToolBinding, onText?: (text: string) => void | Promise<void>): CuaApi<CuaProgramImage> {
|
||||
return createCuaApi(operation => getTool()(operation), {
|
||||
text: value => onText?.(value),
|
||||
image: () => {},
|
||||
}, (value) => {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)
|
||||
|| typeof value.path !== 'string' || typeof value.width !== 'number' || typeof value.height !== 'number') {
|
||||
throw new Error('The CUA program binding must return an Image for screenshots')
|
||||
}
|
||||
return value as CuaProgramImage
|
||||
})
|
||||
export function connectCua(getTool: () => CuaToolBinding, onText?: (text: string) => void | Promise<void>): CuaApi {
|
||||
return createCuaApi(operation => getTool()(operation), onText)
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
/** Async method catalog with consent and observation rules, grouped by observation, mouse, and keyboard. */
|
||||
export const COMMON_GUIDANCE = `Use cua to observe and operate native applications. Always use cua.getCapabilities() to confirm whether a specific API is available on the current platform; its reported capabilities take precedence over availability statements in these instructions. All methods are async; observation options default to { emit: true }, and emit: false suppresses sink output only, leaving returned values unchanged. Actions report dispatch evidence, not verified completion. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI actions and ask the user to unlock the desktop session; null or unknown means verify a usable unlocked desktop session first. cua.getApp requests user consent; if access is denied or revoked, stop using that application until the user explicitly permits another request, and never bypass a refusal or switch tools to avoid it. Never replay uncertain input without a fresh observation.
|
||||
|
||||
In persistent PTC, use the host-provided bridge initialization once and retain the returned objects; cua comes from that bridge and is not a preinstalled global. Observations emit through inspect; never stringify screenshot image bytes.
|
||||
In persistent PTC, use the host-provided bridge initialization once and retain the returned objects; cua comes from that bridge and is not a preinstalled global. Text observations emit through inspect.
|
||||
|
||||
Signatures show resolved return values after await. ActionResult = { dispatched: boolean, foregroundChanged?: boolean | null }; foregroundChanged is omitted when not sampled and null when the sample is unavailable. JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; numbers must be finite. App is the application object with readonly app.appHandle: string (opaque) and the methods listed below for app level. Window is the window object with readonly window.windowId: number and the methods listed below for window level. Neither object carries pid.
|
||||
|
||||
@@ -31,7 +31,7 @@ Element ids are opaque strings from the latest successful tree; the object facad
|
||||
|
||||
## Simulated mouse
|
||||
Use an explicit window for screenshots and pointer input.
|
||||
- getScreenshot(options?: { emit?: boolean }) -> Image — capture the selected window as a stored image with readonly path, width, and height. Pass selected screenshots to inspect(...values) to display them; read the window ID from window.windowId.
|
||||
- getScreenshot() -> Image — capture the selected window as a stored image with readonly path, width, and height. Pass selected screenshots to inspect(...values) to display them; read the window ID from window.windowId.
|
||||
- mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration.
|
||||
- click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press.
|
||||
- scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
import type { Branded } from '@deepseek-ai/dsh-brand'
|
||||
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
|
||||
import type { z } from 'zod'
|
||||
import type { ActionResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, WindowsResult } from './protocol.ts'
|
||||
import type { ActionResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ShowResult, WindowsResult } from './protocol.ts'
|
||||
|
||||
/** Helper-issued application selection; public app objects hold only this opaque handle. */
|
||||
export type CuaAppHandle = Branded<'CuaAppHandle'>
|
||||
@@ -17,23 +17,6 @@ type CuaParameters<M extends CuaOperation['method']> = Extract<CuaOperation, { m
|
||||
export type CuaDispatch = (operation: CuaOperation) => Promise<JsonValue>
|
||||
/** Stored screenshot returned to a program for explicit inspection. */
|
||||
export type CuaProgramImage = { readonly path: string; readonly width: number; readonly height: number }
|
||||
/** Encoded model-request image with the format, pixel dimensions and window reported by the Host. */
|
||||
export type CuaScreenshot = Pick<z.output<typeof ScreenshotResult>, 'mimeType' | 'pixelWidth' | 'pixelHeight' | 'windowId'> & {
|
||||
bytes: Uint8Array
|
||||
}
|
||||
/** Optional output owned by the caller; persistence and image admission are external. */
|
||||
export interface CuaObservationSink {
|
||||
/**
|
||||
* Emit text.
|
||||
* @param value - observed application state.
|
||||
*/
|
||||
text(value: string): void | Promise<void>
|
||||
/**
|
||||
* Emit an image.
|
||||
* @param screenshot - encoded image and its validated format, dimensions and window.
|
||||
*/
|
||||
image(screenshot: CuaScreenshot): void | Promise<void>
|
||||
}
|
||||
/** Observations emit by default. Returned values are unaffected by this option. */
|
||||
export interface CuaObservationOptions { emit?: boolean }
|
||||
/** Full observations of the requested content; each read replaces the application's current snapshot. */
|
||||
@@ -184,14 +167,13 @@ export interface CuaSurface {
|
||||
* Capture and pointer delivery reject known AX/display frame conflicts with WINDOW_CHANGED.
|
||||
* Coordinate input also rejects resizing since the latest successful AX read or screenshot.
|
||||
*/
|
||||
export interface CuaWindow<TImage = CuaScreenshot> extends CuaSurface {
|
||||
export interface CuaWindow extends CuaSurface {
|
||||
readonly windowId: number
|
||||
/**
|
||||
* Capture the selected window.
|
||||
* @param options - emission options.
|
||||
* @returns encoded model-request image with its format, pixel dimensions and window; use mimeType when saving or declaring the image.
|
||||
* @returns Stored Image with a readable path and pixel dimensions, ready for explicit inspection.
|
||||
*/
|
||||
getScreenshot(options?: CuaObservationOptions): Promise<TImage>
|
||||
getScreenshot(): Promise<CuaProgramImage>
|
||||
/**
|
||||
* Request a mouse move to an element or a point without pressing a button.
|
||||
* The provider reports the route and whether the system pointer moved.
|
||||
@@ -226,7 +208,7 @@ export interface CuaWindow<TImage = CuaScreenshot> extends CuaSurface {
|
||||
drag(from: readonly [number, number], to: readonly [number, number], options?: CuaDragOptions): Promise<CuaActionResult>
|
||||
}
|
||||
/** Application-level accessibility, window discovery and scope-dependent input operations. */
|
||||
export interface CuaApp<TImage = CuaScreenshot> extends CuaSurface {
|
||||
export interface CuaApp extends CuaSurface {
|
||||
readonly appHandle: CuaAppHandle
|
||||
/**
|
||||
* Activate this application explicitly, potentially changing the user's focused app and Space.
|
||||
@@ -244,10 +226,10 @@ export interface CuaApp<TImage = CuaScreenshot> extends CuaSurface {
|
||||
* @param windowId - ID obtained from getWindows.
|
||||
* @returns A window object with its initial accessibility tree read silently; this updates the application's current snapshot.
|
||||
*/
|
||||
getWindow(windowId: number): Promise<CuaWindow<TImage>>
|
||||
getWindow(windowId: number): Promise<CuaWindow>
|
||||
}
|
||||
/** Native app API; its caller owns global bindings and variable persistence. */
|
||||
export interface CuaApi<TImage = CuaScreenshot> {
|
||||
export interface CuaApi {
|
||||
/**
|
||||
* Inspect permissions without requesting them.
|
||||
* @param options - output options.
|
||||
@@ -278,5 +260,5 @@ export interface CuaApi<TImage = CuaScreenshot> {
|
||||
* @param selector - Exact name, bundle ID, absolute application path or running PID string; ambiguous names reject.
|
||||
* @returns An application handle tied to this process lifetime; its initial tree read becomes the current snapshot.
|
||||
*/
|
||||
getApp(selector: string): Promise<CuaApp<TImage>>
|
||||
getApp(selector: string): Promise<CuaApp>
|
||||
}
|
||||
@@ -8085,13 +8085,9 @@ export const TYPE_API: readonly TypeApiEntry[] = [
|
||||
name: 'PtcDispatchLog',
|
||||
declaration: 'export interface PtcDispatchLog {\n readonly exec: ToolExecution;\n readonly agent?: Agent;\n readonly subCallId: ToolCallId;\n readonly name: string;\n readonly isError: boolean;\n readonly content: ContentBlock[];\n}',
|
||||
},
|
||||
{
|
||||
name: 'PtcImage',
|
||||
declaration: 'export class PtcImage {\n readonly [key: string]: PtcJsonValue;\n readonly path: string;\n readonly width: number;\n readonly height: number;\n constructor(path: string, width: number, height: number);\n}',
|
||||
},
|
||||
{
|
||||
name: 'PtcImageOutput',
|
||||
declaration: 'export interface PtcImageOutput {\n afterLog: number;\n image: PtcImage;\n}',
|
||||
declaration: 'export interface PtcImageOutput {\n afterLog: number;\n attachment: ImageAttachmentRef;\n}',
|
||||
},
|
||||
{
|
||||
name: 'PtcJsonValue',
|
||||
|
||||
@@ -24,7 +24,9 @@ export class OutputLedger {
|
||||
/** Admit an image descriptor against the same outer-output budget as text. */
|
||||
admitImage(output: PtcImageOutput): boolean {
|
||||
const overhead = this.imageEntries === 0 ? 2 : 1
|
||||
const bytes = jsonValueBytesUpTo({ afterLog: output.afterLog, image: output.image }, this.maxBytes - this.bytes - overhead)
|
||||
const bytes = jsonValueBytesUpTo(
|
||||
{ afterLog: output.afterLog, attachment: { ...output.attachment } }, this.maxBytes - this.bytes - overhead,
|
||||
)
|
||||
if (bytes === undefined) return false
|
||||
this.bytes += bytes + overhead
|
||||
this.imageEntries += 1
|
||||
|
||||
@@ -33,7 +33,7 @@ interface Evaluation {
|
||||
pending: number
|
||||
pendingBytes: number
|
||||
admit(text: string): void
|
||||
image(image: PtcImage): void
|
||||
image(attachment: PtcImage['attachment']): void
|
||||
finish(error?: PtcRunFailure, value?: PtcJsonValue, discard?: boolean): void
|
||||
}
|
||||
|
||||
@@ -53,8 +53,7 @@ export class ProgramWorker {
|
||||
private termination: Promise<boolean> | undefined
|
||||
private starting?: Promise<void>
|
||||
private exitOutcome?: SubprocessOutcome
|
||||
private readonly images = new Map<string, PtcImage>()
|
||||
private readonly imageIds = new WeakMap<PtcImage, string>()
|
||||
private readonly images = new Map<string, PtcImage['attachment']>()
|
||||
private readonly controller = new AbortController()
|
||||
closed = false
|
||||
|
||||
@@ -160,14 +159,14 @@ export class ProgramWorker {
|
||||
evaluation.finish({ kind: 'output-limit', message: 'output limit exceeded' }, undefined, true)
|
||||
}
|
||||
},
|
||||
image: (image) => {
|
||||
image: (attachment) => {
|
||||
if (overflowed || evaluation.settled) return
|
||||
if (!output.admitImage({ afterLog: logs.length, image })) {
|
||||
if (!output.admitImage({ afterLog: logs.length, attachment })) {
|
||||
overflowed = true
|
||||
evaluation.finish({ kind: 'output-limit', message: 'output limit exceeded' }, undefined, true)
|
||||
return
|
||||
}
|
||||
images.push({ afterLog: logs.length, image })
|
||||
images.push({ afterLog: logs.length, attachment })
|
||||
},
|
||||
finish: (error, value, discard = false) => {
|
||||
if (evaluation.settled) return
|
||||
@@ -297,13 +296,9 @@ export class ProgramWorker {
|
||||
}
|
||||
|
||||
private registerImage(image: PtcImage): PtcImageWire {
|
||||
let id = this.imageIds.get(image)
|
||||
if (id === undefined) {
|
||||
id = randomUUID()
|
||||
this.imageIds.set(image, id)
|
||||
this.images.set(id, image)
|
||||
}
|
||||
return { kind: 'image', id, path: image.path, width: image.width, height: image.height }
|
||||
const id = randomUUID()
|
||||
this.images.set(id, image.attachment)
|
||||
return { kind: 'image', id, path: image.path, width: image.attachment.width, height: image.attachment.height }
|
||||
}
|
||||
|
||||
private async start(cwd: string): Promise<void> {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
/** Image values carried by program bindings and explicit inspection output. */
|
||||
import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
|
||||
import type { PtcJsonValue } from './types.ts'
|
||||
|
||||
/** A stored image readable from the program's execution environment. */
|
||||
@@ -6,21 +7,17 @@ export class PtcImage {
|
||||
readonly [key: string]: PtcJsonValue
|
||||
/** Absolute attachment file path readable in the execution world. */
|
||||
readonly path: string
|
||||
/** Stored pixel width. */
|
||||
readonly width: number
|
||||
/** Stored pixel height. */
|
||||
readonly height: number
|
||||
/** Stored image selected when the program inspects this value. */
|
||||
readonly attachment: Readonly<ImageAttachmentRef>
|
||||
|
||||
/**
|
||||
* Describe one immutable image file in the execution world.
|
||||
* @param path - Absolute path to the stored image.
|
||||
* @param width - Stored pixel width.
|
||||
* @param height - Stored pixel height.
|
||||
* @param attachment - Reference to the immutable stored image.
|
||||
*/
|
||||
constructor(path: string, width: number, height: number) {
|
||||
constructor(path: string, attachment: ImageAttachmentRef) {
|
||||
this.path = path
|
||||
this.width = width
|
||||
this.height = height
|
||||
this.attachment = attachment
|
||||
Object.freeze(this)
|
||||
}
|
||||
}
|
||||
@@ -29,6 +26,6 @@ export class PtcImage {
|
||||
export interface PtcImageOutput {
|
||||
/** Number of text entries emitted before this image. */
|
||||
afterLog: number
|
||||
/** Host-issued image selected by the program. */
|
||||
image: PtcImage
|
||||
/** Stored image selected by the program. */
|
||||
attachment: ImageAttachmentRef
|
||||
}
|
||||
Reference in new issue
Block a user