From 3dbe1c126fc5d006eb2e1ed697e6d8f63cc1d772 Mon Sep 17 00:00:00 2001 From: _Kerman Date: Mon, 9 Nov 2026 15:57:42 +0800 Subject: [PATCH] refactor(ptc): simplify image references and CUA bridge --- packages/core/tool-executor-ptc/src/ptc.ts | 21 +---- packages/experimental/cua/src/api.ts | 79 +++++++------------ packages/experimental/cua/src/bridge.ts | 15 +--- packages/experimental/cua/src/guidance.ts | 4 +- packages/experimental/cua/src/types.ts | 34 ++------ .../extensions/tool-cordis/src/api-catalog.ts | 6 +- .../ptc-runtime-node/src/output-ledger.ts | 4 +- .../ptc-runtime-node/src/worker.ts | 21 ++--- packages/ptc-runtime/ptc-runtime/src/image.ts | 19 ++--- 9 files changed, 67 insertions(+), 136 deletions(-) diff --git a/packages/core/tool-executor-ptc/src/ptc.ts b/packages/core/tool-executor-ptc/src/ptc.ts index 2337fb57b3..78af0917ad 100644 --- a/packages/core/tool-executor-ptc/src/ptc.ts +++ b/packages/core/tool-executor-ptc/src/ptc.ts @@ -257,7 +257,6 @@ function renderRunOutput( value: RunCodeOutput, guidance: string, images: PtcImageOutput[] = [], - imageRefs?: Pick['attachment']>, 'get'>, ): ContentBlock[] { const content: ContentBlock[] = [] const appendText = (text: string): void => { if (text.length > 0) content.push({ type: 'text', text }) } @@ -265,9 +264,7 @@ function renderRunOutput( for (const output of images) { appendText(value.logs.slice(position, output.afterLog).join('\n')) position = output.afterLog - const attachment = imageRefs?.get(output.image) - if (attachment === undefined) throw new Error('Inspected image is unavailable') - content.push({ type: 'image', attachment }) + content.push({ type: 'image', attachment: output.attachment }) } const remaining = value.logs.slice(position) if (value.result !== undefined) remaining.push(renderValue(value.result)) @@ -325,7 +322,6 @@ export function createRunCodeTool( } type ImageRef = Extract['attachment'] - const imageRefs = new WeakMap() const inspections = new WeakMap() const programImage = (ref: ImageRef): PtcImage => { const attachments = ctx.get('attachments') @@ -333,9 +329,7 @@ export function createRunCodeTool( const access = attachments === undefined || fs === undefined ? undefined : resolveImageAttachmentAccess(attachments, path => fs.processPathFromHostPath(path), ref) if (access === undefined) throw new Error('Image storage must expose a readable path in the program filesystem') - const image = new PtcImage(access.readonlyPath, ref.width, ref.height) - imageRefs.set(image, ref) - return image + return new PtcImage(access.readonlyPath, ref) } const stateOwners = new WeakMap>() const definition = defineTool({ @@ -749,9 +743,6 @@ export function createRunCodeTool( } if (result.images !== undefined && result.images.length > 0) { - for (const output of result.images) { - if (!imageRefs.has(output.image)) throw new Error('Runtime returned an image outside this executor') - } inspections.set(exec, { images: result.images }) } if (result.error) { @@ -791,14 +782,10 @@ export function createRunCodeTool( if (selected === undefined) return undefined if (result.isError) { if (selected.value !== undefined) return undefined - return [...result.content, ...selected.images.map((output) => { - const attachment = imageRefs.get(output.image) - if (attachment === undefined) throw new Error('Inspected image is unavailable') - return { type: 'image' as const, attachment } - })] + return [...result.content, ...selected.images.map(output => ({ type: 'image' as const, attachment: output.attachment }))] } if (selected.value === undefined || !isDeepStrictEqual(selected.value, result.value)) return undefined - return renderRunOutput(selected.value, escalationGuidance(ctx.get('ptcRuntime')?.sandboxMode), selected.images, imageRefs) + return renderRunOutput(selected.value, escalationGuidance(ctx.get('ptcRuntime')?.sandboxMode), selected.images) }, // The model-authored description is the call's always-visible UI label // (the bash `description` precedent); the program itself rides rawInput. diff --git a/packages/experimental/cua/src/api.ts b/packages/experimental/cua/src/api.ts index d93460ccc2..064770be5b 100644 --- a/packages/experimental/cua/src/api.ts +++ b/packages/experimental/cua/src/api.ts @@ -1,9 +1,8 @@ /** Object facade over logged native operations; its caller owns evaluator persistence. */ -import { Buffer } from 'node:buffer' import type { z } from 'zod' import type { JsonValue } from '@deepseek-ai/dsh-util-values' -import { ActionNamesResult, ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from './protocol.ts' -import type { CuaApi, CuaApp, CuaAppHandle, CuaCapabilities, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaOperation, CuaScreenshot, CuaSurface, CuaWindow } from './types.ts' +import { ActionNamesResult, ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ShowResult, StateResult, WindowsResult } from './protocol.ts' +import type { CuaApi, CuaApp, CuaAppHandle, CuaCapabilities, CuaDispatch, CuaObservationOptions, CuaOperation, CuaProgramImage, CuaSurface, CuaWindow } from './types.ts' export type * from './types.ts' function requestIssueMessage(issues: readonly z.core.$ZodIssue[], parent: readonly PropertyKey[] = []): string { @@ -31,71 +30,49 @@ function validatedCall(dispatch: CuaDispatch) { } } -async function emit(sink: CuaObservationSink | undefined, value: object, options?: CuaObservationOptions): Promise { - if (options?.emit !== false) await sink?.text(JSON.stringify(value)) +async function emit( + onText: ((text: string) => void | Promise) | undefined, value: object, options?: CuaObservationOptions, +): Promise { + if (options?.emit !== false) await onText?.(JSON.stringify(value)) } /** - * Create app/window objects using the caller's authorized dispatcher. - * @param dispatch - Current tool authority; rejects propagate without retries. - * @param sink - Optional text/image output; emit false suppresses this sink only. + * Create app/window objects using the caller's authorized program dispatcher. + * @param dispatch - Current tool authority; screenshots resolve to program Images and rejections propagate without retries. + * @param onText - Optional observation output; emit false suppresses this callback. * @returns Objects independent of variable storage and evaluator lifecycle. */ -export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): CuaApi -/** - * Create app/window objects whose screenshot values are decoded by the caller. - * @param dispatch - Current authorized tool dispatcher. - * @param sink - Optional text output. - * @param readImage - Decodes each screenshot result; the caller owns image output. - * @returns App and window objects using the supplied image type. - */ -export function createCuaApi( - dispatch: CuaDispatch, sink: CuaObservationSink | undefined, readImage: (value: JsonValue, windowId: number) => TImage, -): CuaApi -export function createCuaApi( - dispatch: CuaDispatch, sink?: CuaObservationSink, readImage?: (value: JsonValue, windowId: number) => unknown, -): CuaApi { +export function createCuaApi(dispatch: CuaDispatch, onText?: (text: string) => void | Promise): CuaApi { const call = validatedCall(dispatch) let capabilities: CuaCapabilities | undefined async function currentCapabilities(): Promise { if (capabilities === undefined) capabilities = CapabilitiesResult.parse(await call('getCapabilities', {})) return capabilities } - const screenshot = async (value: JsonValue, windowId: number, options?: CuaObservationOptions): Promise => { - if (readImage !== undefined) return readImage(value, windowId) - const image = ScreenshotResult.parse(value) - if (image.windowId !== windowId) throw new Error('Native screenshot belongs to another window') - const screenshot: CuaScreenshot = { - bytes: Uint8Array.from(Buffer.from(image.data, 'base64')), - mimeType: image.mimeType, pixelWidth: image.pixelWidth, pixelHeight: image.pixelHeight, windowId: image.windowId, - } - if (options?.emit !== false) await sink?.image(screenshot) - return screenshot - } - const objects = createCuaObjects(call, sink, screenshot) + const objects = createCuaObjects(call, onText) - const api: CuaApi = { + const api: CuaApi = { async getCapabilities(options) { const result = CapabilitiesResult.parse(await call('getCapabilities', {})) capabilities = result - await emit(sink, result, options) + await emit(onText, result, options) return result }, async listApps(options) { const result = AppsResult.parse(await call('listApps', {})).map(app => ({ id: app.bundleId || String(app.pid), displayName: app.name, isRunning: true as const, pid: app.pid, })) - await emit(sink, result, options) + await emit(onText, result, options) return result }, async findApps(query, options) { const result = FoundAppsResult.parse(await call('findApps', { query })) - await emit(sink, result, options) + await emit(onText, result, options) return result }, async getState(options) { const result = { apps: await api.listApps({ emit: false }), browsers: [] as [] } - await emit(sink, result, options) + await emit(onText, result, options) return result }, async getApp(selector) { @@ -113,9 +90,8 @@ export function createCuaApi( type CuaScopeParams = (params: Record) => Record -function createCuaObjects(call: ReturnType, sink: CuaObservationSink | undefined, - readImage: (value: JsonValue, windowId: number, options?: CuaObservationOptions) => Promise): { - createCuaApp(appHandle: CuaAppHandle, capabilities: CuaCapabilities): CuaApp +function createCuaObjects(call: ReturnType, onText: ((text: string) => void | Promise) | undefined): { + createCuaApp(appHandle: CuaAppHandle, capabilities: CuaCapabilities): CuaApp } { const observed = new Map() function currentSnapshot(appHandle: CuaAppHandle): string { @@ -152,7 +128,7 @@ function createCuaObjects(call: ReturnType, sink: throw new Error('Native observation belongs to another application or window') } observed.set(target.appHandle, { snapshotId: state.snapshotId, windowId: target.windowId }) - if (options?.emit !== false) await sink?.text(state.tree) + if (options?.emit !== false) await onText?.(state.tree) return state.tree }, async listActions(elementId) { @@ -202,7 +178,7 @@ function createCuaObjects(call: ReturnType, sink: } return { surface, call: scopedCall, action, elementParams } } - function createCuaWindow(appHandle: CuaAppHandle, windowId: number, capabilities: CuaCapabilities): CuaWindow { + function createCuaWindow(appHandle: CuaAppHandle, windowId: number, capabilities: CuaCapabilities): CuaWindow { const base = createCuaSurface({ appHandle, windowId }, params => ({ ...params, appHandle, windowId }), () => undefined, () => { if (!capabilities.keyboard.includes('window')) { @@ -211,8 +187,13 @@ function createCuaObjects(call: ReturnType, sink: }) return { ...base.surface, windowId, - async getScreenshot(options) { - return readImage(await base.call('screenshot', {}), windowId, options) + async getScreenshot() { + const value = await base.call('screenshot', {}) + if (value === null || typeof value !== 'object' || Array.isArray(value) + || typeof value.path !== 'string' || typeof value.width !== 'number' || typeof value.height !== 'number') { + throw new Error('The CUA program binding must return an Image for screenshots') + } + return value as CuaProgramImage }, async mouseMove(targetPoint, options) { const relative = options?.relative === undefined ? {} : { relative: options.relative } @@ -255,7 +236,7 @@ function createCuaObjects(call: ReturnType, sink: }, } } - function createCuaApp(appHandle: CuaAppHandle, capabilities: CuaCapabilities): CuaApp { + function createCuaApp(appHandle: CuaAppHandle, capabilities: CuaCapabilities): CuaApp { const base = createCuaSurface({ appHandle }, params => ({ ...params, appHandle }), () => { if (!capabilities.appLevelAccessibility) { throw new Error('Application-level accessibility is not supported on current platform; observe a window instead.') @@ -270,10 +251,10 @@ function createCuaObjects(call: ReturnType, sink: async activate() { return base.action('activate', {}) }, async getWindows(options) { const windows = WindowsResult.parse(await base.call('listWindows', {})) - await emit(sink, windows, options) + await emit(onText, windows, options) return windows }, - async getWindow(windowId): Promise> { + async getWindow(windowId): Promise { const window = createCuaWindow(appHandle, windowId, capabilities) await window.getAccessibilityTree({ emit: false }) return window diff --git a/packages/experimental/cua/src/bridge.ts b/packages/experimental/cua/src/bridge.ts index 860f4bf8b5..4b2a5b5002 100644 --- a/packages/experimental/cua/src/bridge.ts +++ b/packages/experimental/cua/src/bridge.ts @@ -1,7 +1,7 @@ /** Connect retained application objects to the calling program's current logged tool binding. */ import type { JsonValue } from '@deepseek-ai/dsh-util-values' import { createCuaApi } from './api.ts' -import type { CuaApi, CuaOperation, CuaProgramImage } from './types.ts' +import type { CuaApi, CuaOperation } from './types.ts' /** Standard tools.cua binding; screenshots resolve to stored program images. */ export type CuaToolBinding = (operation: CuaOperation) => Promise @@ -12,15 +12,6 @@ export type CuaToolBinding = (operation: CuaOperation) => Promise * @param onText - Optional text output captured by the caller's execution environment. * @returns Application and window objects using the current tool binding. Rejections propagate without retries. */ -export function connectCua(getTool: () => CuaToolBinding, onText?: (text: string) => void | Promise): CuaApi { - return createCuaApi(operation => getTool()(operation), { - text: value => onText?.(value), - image: () => {}, - }, (value) => { - if (value === null || typeof value !== 'object' || Array.isArray(value) - || typeof value.path !== 'string' || typeof value.width !== 'number' || typeof value.height !== 'number') { - throw new Error('The CUA program binding must return an Image for screenshots') - } - return value as CuaProgramImage - }) +export function connectCua(getTool: () => CuaToolBinding, onText?: (text: string) => void | Promise): CuaApi { + return createCuaApi(operation => getTool()(operation), onText) } diff --git a/packages/experimental/cua/src/guidance.ts b/packages/experimental/cua/src/guidance.ts index 21967391be..0b05e03ce0 100644 --- a/packages/experimental/cua/src/guidance.ts +++ b/packages/experimental/cua/src/guidance.ts @@ -2,7 +2,7 @@ /** Async method catalog with consent and observation rules, grouped by observation, mouse, and keyboard. */ export const COMMON_GUIDANCE = `Use cua to observe and operate native applications. Always use cua.getCapabilities() to confirm whether a specific API is available on the current platform; its reported capabilities take precedence over availability statements in these instructions. All methods are async; observation options default to { emit: true }, and emit: false suppresses sink output only, leaving returned values unchanged. Actions report dispatch evidence, not verified completion. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI actions and ask the user to unlock the desktop session; null or unknown means verify a usable unlocked desktop session first. cua.getApp requests user consent; if access is denied or revoked, stop using that application until the user explicitly permits another request, and never bypass a refusal or switch tools to avoid it. Never replay uncertain input without a fresh observation. -In persistent PTC, use the host-provided bridge initialization once and retain the returned objects; cua comes from that bridge and is not a preinstalled global. Observations emit through inspect; never stringify screenshot image bytes. +In persistent PTC, use the host-provided bridge initialization once and retain the returned objects; cua comes from that bridge and is not a preinstalled global. Text observations emit through inspect. Signatures show resolved return values after await. ActionResult = { dispatched: boolean, foregroundChanged?: boolean | null }; foregroundChanged is omitted when not sampled and null when the sample is unavailable. JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; numbers must be finite. App is the application object with readonly app.appHandle: string (opaque) and the methods listed below for app level. Window is the window object with readonly window.windowId: number and the methods listed below for window level. Neither object carries pid. @@ -31,7 +31,7 @@ Element ids are opaque strings from the latest successful tree; the object facad ## Simulated mouse Use an explicit window for screenshots and pointer input. -- getScreenshot(options?: { emit?: boolean }) -> Image — capture the selected window as a stored image with readonly path, width, and height. Pass selected screenshots to inspect(...values) to display them; read the window ID from window.windowId. +- getScreenshot() -> Image — capture the selected window as a stored image with readonly path, width, and height. Pass selected screenshots to inspect(...values) to display them; read the window ID from window.windowId. - mouseMove(target: string | [number, number], options?: { relative?: boolean }) -> { dispatched: boolean, foregroundChanged?: boolean | null, hoverRoute: 'accessibility' | 'system-pointer', systemPointerMoved: boolean } — request a mouse move to the target without pressing a button; inspect hoverRoute and systemPointerMoved, and expect no automatic pointer restoration. - click(target: string | [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', clickCount?: 1 | 2, modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], holdMs?: number, relative?: boolean }) -> ActionResult — click an element id or point, or perform a left-button double-click; holdMs controls the press duration of each press. - scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left. diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index 8683b2de66..10c950adc0 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -2,7 +2,7 @@ import type { Branded } from '@deepseek-ai/dsh-brand' import type { JsonValue } from '@deepseek-ai/dsh-util-values' import type { z } from 'zod' -import type { ActionResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, WindowsResult } from './protocol.ts' +import type { ActionResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ShowResult, WindowsResult } from './protocol.ts' /** Helper-issued application selection; public app objects hold only this opaque handle. */ export type CuaAppHandle = Branded<'CuaAppHandle'> @@ -17,23 +17,6 @@ type CuaParameters = Extract Promise /** Stored screenshot returned to a program for explicit inspection. */ export type CuaProgramImage = { readonly path: string; readonly width: number; readonly height: number } -/** Encoded model-request image with the format, pixel dimensions and window reported by the Host. */ -export type CuaScreenshot = Pick, 'mimeType' | 'pixelWidth' | 'pixelHeight' | 'windowId'> & { - bytes: Uint8Array -} -/** Optional output owned by the caller; persistence and image admission are external. */ -export interface CuaObservationSink { - /** - * Emit text. - * @param value - observed application state. - */ - text(value: string): void | Promise - /** - * Emit an image. - * @param screenshot - encoded image and its validated format, dimensions and window. - */ - image(screenshot: CuaScreenshot): void | Promise -} /** Observations emit by default. Returned values are unaffected by this option. */ export interface CuaObservationOptions { emit?: boolean } /** Full observations of the requested content; each read replaces the application's current snapshot. */ @@ -184,14 +167,13 @@ export interface CuaSurface { * Capture and pointer delivery reject known AX/display frame conflicts with WINDOW_CHANGED. * Coordinate input also rejects resizing since the latest successful AX read or screenshot. */ -export interface CuaWindow extends CuaSurface { +export interface CuaWindow extends CuaSurface { readonly windowId: number /** * Capture the selected window. - * @param options - emission options. - * @returns encoded model-request image with its format, pixel dimensions and window; use mimeType when saving or declaring the image. + * @returns Stored Image with a readable path and pixel dimensions, ready for explicit inspection. */ - getScreenshot(options?: CuaObservationOptions): Promise + getScreenshot(): Promise /** * Request a mouse move to an element or a point without pressing a button. * The provider reports the route and whether the system pointer moved. @@ -226,7 +208,7 @@ export interface CuaWindow extends CuaSurface { drag(from: readonly [number, number], to: readonly [number, number], options?: CuaDragOptions): Promise } /** Application-level accessibility, window discovery and scope-dependent input operations. */ -export interface CuaApp extends CuaSurface { +export interface CuaApp extends CuaSurface { readonly appHandle: CuaAppHandle /** * Activate this application explicitly, potentially changing the user's focused app and Space. @@ -244,10 +226,10 @@ export interface CuaApp extends CuaSurface { * @param windowId - ID obtained from getWindows. * @returns A window object with its initial accessibility tree read silently; this updates the application's current snapshot. */ - getWindow(windowId: number): Promise> + getWindow(windowId: number): Promise } /** Native app API; its caller owns global bindings and variable persistence. */ -export interface CuaApi { +export interface CuaApi { /** * Inspect permissions without requesting them. * @param options - output options. @@ -278,5 +260,5 @@ export interface CuaApi { * @param selector - Exact name, bundle ID, absolute application path or running PID string; ambiguous names reject. * @returns An application handle tied to this process lifetime; its initial tree read becomes the current snapshot. */ - getApp(selector: string): Promise> + getApp(selector: string): Promise } diff --git a/packages/extensions/tool-cordis/src/api-catalog.ts b/packages/extensions/tool-cordis/src/api-catalog.ts index 63fe3904f0..f4c4b2f157 100644 --- a/packages/extensions/tool-cordis/src/api-catalog.ts +++ b/packages/extensions/tool-cordis/src/api-catalog.ts @@ -8085,13 +8085,9 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'PtcDispatchLog', declaration: 'export interface PtcDispatchLog {\n readonly exec: ToolExecution;\n readonly agent?: Agent;\n readonly subCallId: ToolCallId;\n readonly name: string;\n readonly isError: boolean;\n readonly content: ContentBlock[];\n}', }, - { - name: 'PtcImage', - declaration: 'export class PtcImage {\n readonly [key: string]: PtcJsonValue;\n readonly path: string;\n readonly width: number;\n readonly height: number;\n constructor(path: string, width: number, height: number);\n}', - }, { name: 'PtcImageOutput', - declaration: 'export interface PtcImageOutput {\n afterLog: number;\n image: PtcImage;\n}', + declaration: 'export interface PtcImageOutput {\n afterLog: number;\n attachment: ImageAttachmentRef;\n}', }, { name: 'PtcJsonValue', diff --git a/packages/ptc-runtime/ptc-runtime-node/src/output-ledger.ts b/packages/ptc-runtime/ptc-runtime-node/src/output-ledger.ts index 907a9ba979..4d5cc8be89 100644 --- a/packages/ptc-runtime/ptc-runtime-node/src/output-ledger.ts +++ b/packages/ptc-runtime/ptc-runtime-node/src/output-ledger.ts @@ -24,7 +24,9 @@ export class OutputLedger { /** Admit an image descriptor against the same outer-output budget as text. */ admitImage(output: PtcImageOutput): boolean { const overhead = this.imageEntries === 0 ? 2 : 1 - const bytes = jsonValueBytesUpTo({ afterLog: output.afterLog, image: output.image }, this.maxBytes - this.bytes - overhead) + const bytes = jsonValueBytesUpTo( + { afterLog: output.afterLog, attachment: { ...output.attachment } }, this.maxBytes - this.bytes - overhead, + ) if (bytes === undefined) return false this.bytes += bytes + overhead this.imageEntries += 1 diff --git a/packages/ptc-runtime/ptc-runtime-node/src/worker.ts b/packages/ptc-runtime/ptc-runtime-node/src/worker.ts index 8f758f50bc..e1b880229f 100644 --- a/packages/ptc-runtime/ptc-runtime-node/src/worker.ts +++ b/packages/ptc-runtime/ptc-runtime-node/src/worker.ts @@ -33,7 +33,7 @@ interface Evaluation { pending: number pendingBytes: number admit(text: string): void - image(image: PtcImage): void + image(attachment: PtcImage['attachment']): void finish(error?: PtcRunFailure, value?: PtcJsonValue, discard?: boolean): void } @@ -53,8 +53,7 @@ export class ProgramWorker { private termination: Promise | undefined private starting?: Promise private exitOutcome?: SubprocessOutcome - private readonly images = new Map() - private readonly imageIds = new WeakMap() + private readonly images = new Map() private readonly controller = new AbortController() closed = false @@ -160,14 +159,14 @@ export class ProgramWorker { evaluation.finish({ kind: 'output-limit', message: 'output limit exceeded' }, undefined, true) } }, - image: (image) => { + image: (attachment) => { if (overflowed || evaluation.settled) return - if (!output.admitImage({ afterLog: logs.length, image })) { + if (!output.admitImage({ afterLog: logs.length, attachment })) { overflowed = true evaluation.finish({ kind: 'output-limit', message: 'output limit exceeded' }, undefined, true) return } - images.push({ afterLog: logs.length, image }) + images.push({ afterLog: logs.length, attachment }) }, finish: (error, value, discard = false) => { if (evaluation.settled) return @@ -297,13 +296,9 @@ export class ProgramWorker { } private registerImage(image: PtcImage): PtcImageWire { - let id = this.imageIds.get(image) - if (id === undefined) { - id = randomUUID() - this.imageIds.set(image, id) - this.images.set(id, image) - } - return { kind: 'image', id, path: image.path, width: image.width, height: image.height } + const id = randomUUID() + this.images.set(id, image.attachment) + return { kind: 'image', id, path: image.path, width: image.attachment.width, height: image.attachment.height } } private async start(cwd: string): Promise { diff --git a/packages/ptc-runtime/ptc-runtime/src/image.ts b/packages/ptc-runtime/ptc-runtime/src/image.ts index 96e48b6b60..06c80443a9 100644 --- a/packages/ptc-runtime/ptc-runtime/src/image.ts +++ b/packages/ptc-runtime/ptc-runtime/src/image.ts @@ -1,4 +1,5 @@ /** Image values carried by program bindings and explicit inspection output. */ +import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment' import type { PtcJsonValue } from './types.ts' /** A stored image readable from the program's execution environment. */ @@ -6,21 +7,17 @@ export class PtcImage { readonly [key: string]: PtcJsonValue /** Absolute attachment file path readable in the execution world. */ readonly path: string - /** Stored pixel width. */ - readonly width: number - /** Stored pixel height. */ - readonly height: number + /** Stored image selected when the program inspects this value. */ + readonly attachment: Readonly /** * Describe one immutable image file in the execution world. * @param path - Absolute path to the stored image. - * @param width - Stored pixel width. - * @param height - Stored pixel height. + * @param attachment - Reference to the immutable stored image. */ - constructor(path: string, width: number, height: number) { + constructor(path: string, attachment: ImageAttachmentRef) { this.path = path - this.width = width - this.height = height + this.attachment = attachment Object.freeze(this) } } @@ -29,6 +26,6 @@ export class PtcImage { export interface PtcImageOutput { /** Number of text entries emitted before this image. */ afterLog: number - /** Host-issued image selected by the program. */ - image: PtcImage + /** Stored image selected by the program. */ + attachment: ImageAttachmentRef }