diff --git a/packages/experimental/client-ui-perf/README.i18n.yaml b/packages/experimental/client-ui-perf/README.i18n.yaml index 4d1c44495d..18cb61aa05 100644 --- a/packages/experimental/client-ui-perf/README.i18n.yaml +++ b/packages/experimental/client-ui-perf/README.i18n.yaml @@ -9,8 +9,8 @@ en: 092989f386525568 zh: 4a6276cfc268c851 /deepseek-ai-dsh-experimental-client-ui-perf/summary: - en: e9a7a32dd3ede2d8 - zh: 7eda50b8dd62ecde + en: 0b5de167c933df06 + zh: eb703c1bc3f46b40 /deepseek-ai-dsh-experimental-client-ui-perf/table-of-contents: en: d152484eb41ac6b4 zh: 09388d293f9be9cb @@ -24,8 +24,8 @@ en: e001601f2f9779b1 zh: 077d3703b8c82774 /deepseek-ai-dsh-experimental-client-ui-perf/understand-the-implementation: - en: e88a036021aff9f4 - zh: 52b14b7b971278b5 + en: 419a3c1b9360721b + zh: 20af2a2534ac3652 /deepseek-ai-dsh-experimental-client-ui-perf/further-exploration: en: 285cfe1d7a99af34 zh: fbcee76e0c2f20fb @@ -36,8 +36,8 @@ en: 7605bb868f93b4a8 zh: 184ca10ce6680499 /deepseek-ai-dsh-experimental-client-ui-perf/known-limitations-and-deferred-work: - en: 9f52f1b4e4e75cf5 - zh: b6557b7415ce29af + en: f9e1b7b047937d51 + zh: 804a52a6815823e8 /deepseek-ai-dsh-experimental-client-ui-perf/known-limitations-and-deferred-work/dev-note: en: 3518dbaf30fb19a6 zh: c3c929308e955ebb diff --git a/packages/experimental/client-ui-perf/README.md b/packages/experimental/client-ui-perf/README.md index 79f31d2178..0bd215e0d3 100644 --- a/packages/experimental/client-ui-perf/README.md +++ b/packages/experimental/client-ui-perf/README.md @@ -9,7 +9,7 @@ English | [中文](README.zh.md) ## Summary -Adds a Performance tab beside Chat and Trajectory. Each turn is one row; expanding it lists its steps with context tokens, output tokens, cache hit rate, first-token latency, model time, output speed, tool time, and a total-time bar split into first token, generation, tools, and the rest. A running step shows elapsed time instead of a blank. Scrolling to the top pulls earlier turns through the same Session paging the other views use; there are no page controls. When a step has no tool calls, its answer label is followed by the first line of the model reply, with the full text on hover. The Question column classifies what opened the turn: a human prompt shows its text, while a team message, subagent settlement, goal round, or runtime/time context shows a localized trigger badge plus the triggering text. The panel reads Session events only; it registers no tool, prompt section, or service. +Adds a Performance tab beside Chat and Trajectory. Each turn is one row; expanding it lists its steps with context tokens, output tokens, cache hit rate, first-token latency, model time, output speed, tool time, and a total-time bar split into first token, generation, tools, and the rest. A running step shows elapsed time instead of a blank. Scrolling to the top pulls earlier turns through the same Session paging the other views use; there are no page controls. When a step has no tool calls, its answer label is followed by the first line of the model reply, with the full text on hover. The Question column classifies what opened the turn: a human prompt shows its text, while a team message, subagent settlement, goal round, or runtime/time context shows a localized trigger badge plus the triggering text. A prompt that attached images shows an image-count pill in the same style as the trigger badge. Hovering or keyboard-focusing the pill opens a small preview strip, and clicking a preview loads that image in the shared image viewer; an image-only prompt shows the pill instead of an empty cell. The panel reads Session events and the prompt's durable images; it registers no tool, prompt section, or service. ## Table of Contents @@ -54,8 +54,8 @@ Four Conversation Definitions fold the event window into nodes for one view targ | `src/client/perf-fold.ts` | Event folding, tool pairing, Code-Mode dispatch attribution, and per-step metrics | | `src/client/perf-aggregate.ts` | Turn aggregation and ordering, the interrupted-tail close, and the Code-Mode merge into the owning step | | `src/client/perf-snapshot-builder.ts` | The `perf` view target builder | -| `src/client/PerfView.tsx` | Scrollport with scroll-to-top paging, table, time bars, step accordion, and copy controls | -| `src/client/locales.ts` | Every visible string, in Chinese and English | +| `src/client/PerfView.tsx` | Scrollport with scroll-to-top paging, table, time bars, step accordion, copy controls, and the prompt image pill with its preview and viewer | +| `src/client/locales.ts` | Every visible string, in Chinese, English, and Japanese | Windows are bounded: when a step's `step/start` is outside the loaded window, its start-dependent columns show `—` with a `start unknown` marker rather than a guessed value. Scrolling to the top asks the Session for one older page; the view offsets the scroll position by the height it prepended, so the row being read stays in place. @@ -91,6 +91,7 @@ No effect. The package never constructs a model request and never mutates a prom - **Browser evidence is a test-only overlay.** Package tests cover folding, aggregation, formatting, and the jsdom table; `apps/web/tests/client-ui-perf-panel.e2e.ts` mounts the plugin through a test-only overlay over the shipped Web surface and compares a committed ARIA golden. No recorded-session scenario exists because no shipped profile mounts the package yet. - **Late re-deliveries are not turns.** Compaction can restate a pruned tool result under its original turn and step after that turn left the window; a turn with no loaded start, prompt, or end marker is dropped instead of appearing as an empty ghost row. - **Timing is derived, not sampled.** Total time is the last step's end minus the first step's start; queueing and scheduler waits land in the "rest" segment. The total-time bar draws tool calls as the union of their paired intervals, so parallel calls count once; the tool column keeps the summed per-call time. +- **Prompt images load through the shared image cache.** The pill and its preview resolve durable attachments with `uiConversation.imageUrl`, so the cache owns and revokes the browser URLs per Session binding; the view creates none of its own. A failed read shows a localized failure state with retry. Touch clients do not open the hover preview; tapping the pill loads the first image directly. ### Dev Note diff --git a/packages/experimental/client-ui-perf/README.zh.md b/packages/experimental/client-ui-perf/README.zh.md index cb8a943c35..aee469c007 100644 --- a/packages/experimental/client-ui-perf/README.zh.md +++ b/packages/experimental/client-ui-perf/README.zh.md @@ -9,7 +9,7 @@ kind: "package-reference" ## 概述 -在「对话」和「轨迹」旁边加一个「性能」页签。每一轮一行，展开后列出各步的上下文词数、输出词数、缓存命中率、首 token 延迟、模型用时、输出速度、工具调用用时，以及一条按首 token、生成、工具调用、其余四段拆开的总用时条。还在跑的步显示已用时间，而不是留空。页签只读会话事件，不注册工具、提示词段落或服务。 +在「对话」和「轨迹」旁边加一个「性能」页签。每一轮一行，展开后列出各步的上下文词数、输出词数、缓存命中率、首 token 延迟、模型用时、输出速度、工具调用用时，以及一条按首 token、生成、工具调用、其余四段拆开的总用时条。还在跑的步显示已用时间，而不是留空。用户问题列会说明这一轮由什么触发：人写的提问显示原文，团队消息、子代理完成、目标轮次或上下文唤醒显示本地化触发标签加触发文本。带图的提问显示与触发标签同样式的张数标签；悬停或键盘聚焦标签会浮出小图预览，点击预览用共用看图器加载该图，只有图片的提问显示标签而不是空单元格。页签只读会话事件和提问里的持久图片，不注册工具、提示词段落或服务。 ## 目录 @@ -54,8 +54,8 @@ kind: "package-reference" | `src/client/perf-fold.ts` | 事件折叠、工具配对、Code-Mode 内层调用归属、单步指标 | | `src/client/perf-aggregate.ts` | 轮次汇总与排序，含被中断尾部的收尾 | | `src/client/perf-snapshot-builder.ts` | `perf` 视图目标的增量 builder | -| `src/client/PerfView.tsx` | 表格、时间条、步骤手风琴与复制按钮 | -| `src/client/locales.ts` | 全部可见文案，中英两份 | +| `src/client/PerfView.tsx` | 表格、时间条、步骤手风琴、复制按钮，以及提问图片标签、预览与看图器 | +| `src/client/locales.ts` | 全部可见文案，中、英、日三份 | 会话窗口是有界的：某一步的 `step/start` 不在已加载窗口里时，与起点有关的列显示 `—` 并标「起止未知」，不估算数值。滚到顶部会向 Session 请求更早的一页；视图按新增内容的高度补偿滚动位置，正在读的行不会跳走。 @@ -91,6 +91,7 @@ kind: "package-reference" - **浏览器证据是仅测试用的 overlay。** 包级测试覆盖折叠、汇总、格式化与 jsdom 表格；`apps/web/tests/client-ui-perf-panel.e2e.ts` 用仅测试用的 overlay 把插件挂到真实 Web 面上，并比对提交的 ARIA golden。由于还没有任何 shipped profile 挂载本包，暂无录制会话场景。 - **迟到重投不是轮次。** 压缩可能把被裁剪的工具结果按原来的轮次/步骤在之后重新投递；窗口里既没有加载到的起点、也没有问题或结束标记的轮次会被丢弃，不再显示成空的幽灵行。 - **计时是推导值。** 总用时取最后一步结束减第一步开始；排队和调度等待落在「其余」段里。总用时条把工具调用画成配对区间之并，并行调用只算一次；工具列仍显示逐次求和。 +- **提问图片走共用图片缓存。** 标签与预览用 `uiConversation.imageUrl` 解析持久附件，浏览器 URL 由缓存按 Session binding 持有并释放，视图不自己创建 URL。读取失败时显示本地化失败状态并可重试。触屏不打开悬停预览，点击标签会直接加载第一张图。 ### 开发备注 diff --git a/packages/experimental/client-ui-perf/package.json b/packages/experimental/client-ui-perf/package.json index cdffe7bc4f..20367be729 100644 --- a/packages/experimental/client-ui-perf/package.json +++ b/packages/experimental/client-ui-perf/package.json @@ -53,6 +53,7 @@ "@deepseek-ai/dsh-client-store": "workspace:*", "@deepseek-ai/dsh-client-test-runtime": "workspace:*", "@deepseek-ai/dsh-client-ui-conversation": "workspace:*", + "@deepseek-ai/dsh-client-ui-primitives": "workspace:*", "@deepseek-ai/dsh-client-ui-renderer": "workspace:*", "@deepseek-ai/dsh-client-ui-session": "workspace:*", "@deepseek-ai/dsh-client-ui-slots": "workspace:*", diff --git a/packages/experimental/client-ui-perf/src/client/PerfView.tsx b/packages/experimental/client-ui-perf/src/client/PerfView.tsx index 94e6dac380..13688677cb 100644 --- a/packages/experimental/client-ui-perf/src/client/PerfView.tsx +++ b/packages/experimental/client-ui-perf/src/client/PerfView.tsx @@ -4,10 +4,12 @@ */ import { - Fragment, useEffect, useLayoutEffect, useRef, useState, + Fragment, useCallback, useEffect, useLayoutEffect, useRef, useState, type ReactElement, type ReactNode, type UIEvent, } from 'react' +import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment' import type { ObservableSnapshot } from '@deepseek-ai/dsh-client-store' +import { HoverCard, ImageLightbox } from '@deepseek-ai/dsh-client-ui-primitives' import type { ConvViewProps } from '@deepseek-ai/dsh-client-ui-conversation/client' import type { InjectFace, PropsLocale } from '@deepseek-ai/dsh-client-ui-slots' import type { @@ -32,6 +34,8 @@ const COPY_FEEDBACK_MS = 1_200 const CODE_LINE_LIMIT = 400 /** Line count above which a diff falls back to whole-side removal and addition. */ const DIFF_LINE_LIMIT = 300 +/** Hover dwell before the prompt image preview opens. */ +const IMAGE_PREVIEW_DELAY_MS = 200 /** Numeric columns in table order. */ const COLUMNS = ['context', 'output', 'cache', 'ttft', 'model', 'tps', 'tool'] as const @@ -89,12 +93,33 @@ export interface PerfViewInjected { * @returns false when the snapshot produced no new page. */ loadOlder: () => Promise + /** + * Resolve one durable prompt image through the shared Conversation image cache. + * @param attachment - image reference the user message recorded. + * @returns browser URL owned by the cache for the Session binding's lifetime. + */ + loadImage: (attachment: ImageAttachmentRef) => Promise } /** Full props of the performance conversation View. */ export type PerfViewProps = ConvViewProps & InjectFace & PropsLocale<'perf'> +/** Resolve one durable prompt image to a browser URL. */ +type LoadImage = PerfViewInjected['loadImage'] + +/** Prompt image shown by the full-size lightbox. */ +interface LightboxImage { + readonly src: string + readonly alt: string +} + +/** Load state of one preview thumbnail. */ +type ImageLoad = + | { readonly status: 'loading' } + | { readonly status: 'failed' } + | { readonly status: 'ready'; readonly src: string } + /** Open/closed default for turns the user never toggled explicitly. */ type ViewMode = 'newest' | 'all' | 'none' @@ -195,6 +220,7 @@ interface CallGroup { interface TurnRowsOptions { readonly turn: PerfTurnMetrics readonly t: PerfTranslate + readonly loadImage: LoadImage readonly now: number readonly open: boolean readonly state: StepViewState @@ -267,6 +293,25 @@ interface PromptCellProps { readonly turn: PerfTurnMetrics readonly t: PerfTranslate readonly className: string | undefined + readonly loadImage: LoadImage +} + +/** Props of one preview thumbnail inside the image popover. */ +interface PromptImageThumbProps { + readonly attachment: ImageAttachmentRef + readonly index: number + readonly count: number + readonly loadImage: LoadImage + readonly t: PerfTranslate + readonly onOpen: (image: LightboxImage) => void +} + +/** Props of the image-count pill and its hover/focus preview. */ +interface ImagePromptPillProps { + readonly first: ImageAttachmentRef + readonly images: readonly ImageAttachmentRef[] + readonly loadImage: LoadImage + readonly t: PerfTranslate } /** Empty per-turn step state, shared so unexpanded turns allocate nothing. */ @@ -908,8 +953,146 @@ function TimeCell({ value, scale, gap, className, trackClass, now, t }: TimeCell ) } +/** Display name of one prompt image: its recorded name, otherwise its position. */ +function imageName(attachment: ImageAttachmentRef, index: number, count: number, t: PerfTranslate): string { + return attachment.name ?? t('imagePreview.alt', { index, count }) +} + +/** One prompt-image thumbnail: loads while the preview is open and opens the full image on click. */ +function PromptImageThumb({ + attachment, index, count, loadImage, t, onOpen, +}: PromptImageThumbProps): ReactElement { + const [image, setImage] = useState({ status: 'loading' }) + const [attempt, setAttempt] = useState(0) + useEffect(() => { + let live = true + void loadImage(attachment).then( + (src) => { if (live) setImage({ status: 'ready', src }) }, + () => { if (live) setImage({ status: 'failed' }) }, + ) + return () => { live = false } + }, [attachment, loadImage, attempt]) + if (image.status === 'ready') { + const name = imageName(attachment, index, count, t) + return ( + + ) + } + return ( + + {image.status === 'loading' ? t('imagePreview.loading') : t('imagePreview.failed')} + {image.status === 'failed' ? ( + + ) : null} + + ) +} + +/** + * Image-count pill for one turn's prompt: hovering or focusing it previews the + * images, and clicking it loads the first one full size. URLs come from the + * shared Conversation image cache, which owns their lifetime per Session. + */ +function ImagePromptPill({ first, images, loadImage, t }: ImagePromptPillProps): ReactElement { + const [lightbox, setLightbox] = useState(null) + const [status, setStatus] = useState<'idle' | 'loading' | 'failed'>('idle') + const count = images.length + const closeLightbox = useCallback(() => { setLightbox(null) }, []) + const label = status === 'idle' + ? t('turn.images', { count }) + : status === 'loading' ? t('imagePreview.loading') : t('imagePreview.failed') + const preview = ( +
+ {images.map((attachment, offset) => ( + + ))} +
+ ) + return ( + + { + // The row toggles its steps on click; the pill opens the image instead. + event.stopPropagation() + setStatus('loading') + void loadImage(first).then( + (src) => { + setStatus('idle') + setLightbox({ src, alt: imageName(first, 1, count, t) }) + }, + () => { setStatus('failed') }, + ) + }} + > + {label} + + )} + content={preview} + /> + {lightbox === null ? null : ( + { event.stopPropagation() }}> + + + )} + + ) +} + /** Prompt column: the turn's user question, clipped with the full text in the title. */ -function PromptCell({ turn, t, className }: PromptCellProps): ReactElement { +function PromptCell({ turn, t, className, loadImage }: PromptCellProps): ReactElement { + const images = turn.promptImages + const first = images[0] + const pill = first === undefined + ? null + : const source = turn.promptSource if (source !== null && source !== 'user') { // Automated turns explain what woke them instead of pretending to be a question. @@ -918,27 +1101,24 @@ function PromptCell({ turn, t, className }: PromptCellProps): ReactElement { return ( {label} + {pill} {text === '' ? null : {text}} ) } if (turn.prompt === null || turn.prompt === '') { // An image-only prompt still says how many images opened the turn. - if (turn.promptImages > 0) { - const label = t('turn.images', { count: turn.promptImages }) - return ( - - {label} - - ) - } - return + if (first === undefined) return + return ( + + {pill} + + ) } - const images = turn.promptImages > 0 ? ` · ${t('turn.images', { count: turn.promptImages })}` : '' - const text = turn.prompt + images return ( - - {text} + + {turn.prompt} + {pill} ) } @@ -1279,7 +1459,7 @@ function stepRows(options: StepRowsOptions): ReactNode[] { /** All rows of one turn: the clickable header, then the expanded body. */ function turnRows(options: TurnRowsOptions): ReactNode[] { - const { turn, t, now, open, state, turnScale, onToggle, onToggleDetail, onControl } = options + const { turn, t, loadImage, now, open, state, turnScale, onToggle, onToggleDetail, onControl } = options const cells = metricsCells(turn, t) const toggleLabel = open ? t('turn.collapse') : t('turn.expand') const rows: ReactNode[] = [ @@ -1315,7 +1495,7 @@ function turnRows(options: TurnRowsOptions): ReactNode[] { ) : null} {scopeMarkers(turn, t)} - + {turnDidCell(turn, t)} @@ -1366,7 +1546,7 @@ interface PendingPrepend { * @returns the scrolling table, its empty state, and the expand/collapse footer. */ export function PerfView(props: PerfViewProps): ReactElement { - const { usePerf, t, loadOlder } = props + const { usePerf, t, loadOlder, loadImage } = props const snapshot = usePerf(value => value) const [view, setView] = useState(createViewState) const [now, setNow] = useState(() => Date.now()) @@ -1533,6 +1713,7 @@ export function PerfView(props: PerfViewProps): ReactElement { {turns.flatMap(turn => turnRows({ turn, t, + loadImage, now, open: isTurnOpen(turn.turn), state: view.stepViews[turn.turn] ?? EMPTY_STEP_VIEW, diff --git a/packages/experimental/client-ui-perf/src/client/index.ts b/packages/experimental/client-ui-perf/src/client/index.ts index 37d842e1e5..d63bd71e02 100644 --- a/packages/experimental/client-ui-perf/src/client/index.ts +++ b/packages/experimental/client-ui-perf/src/client/index.ts @@ -4,6 +4,7 @@ */ import type { Context } from '@deepseek-ai/cordis' +import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment' import type { SessionBinding, SessionFace } from '@deepseek-ai/dsh-api-session-controller/client' import type { ObservableSnapshot } from '@deepseek-ai/dsh-client-store' import type { SessionId } from '@deepseek-ai/dsh-session/types' @@ -101,11 +102,14 @@ export function apply(ctx: Context): void { const grew = perf.getSnapshot() !== before if (grew) return true const snapshot = session.getSnapshot() - if (snapshot.hasMore === true) { + if (snapshot.hasMore) { throw new Error('client-ui-perf: history page did not advance') } return false }, + // The shared Conversation cache reads the durable bytes once per Session + // and owns the resulting browser URL until its binding is released. + loadImage: (attachment: ImageAttachmentRef) => ctx.uiConversation.imageUrl(sessionId, attachment), } }, }, PerfView)) diff --git a/packages/experimental/client-ui-perf/src/client/locales.ts b/packages/experimental/client-ui-perf/src/client/locales.ts index 6e07573dc6..69856be4e5 100644 --- a/packages/experimental/client-ui-perf/src/client/locales.ts +++ b/packages/experimental/client-ui-perf/src/client/locales.ts @@ -107,6 +107,14 @@ export const zh = { 'turn.stepCount': '{count} 步', 'turn.started': '这一轮开始于 {time}', 'turn.images': '{count} 张图', + 'imagePreview.hint': '悬停或聚焦预览这一轮的图片，点击查看大图', + 'imagePreview.dialog': '图片预览', + 'imagePreview.open': '查看第 {index} 张图（共 {count} 张）', + 'imagePreview.alt': '第 {index} 张图', + 'imagePreview.loading': '图片加载中…', + 'imagePreview.failed': '图片加载失败', + 'imagePreview.retry': '重试', + 'imagePreview.close': '关闭图片预览', 'steps.total': '共 {total} 步', 'steps.tail': '只看最新 {limit} 步', 'steps.all': '显示全部 {total} 步', @@ -232,6 +240,14 @@ export const en: Record = { 'turn.stepCount': '{count} steps', 'turn.started': 'this turn started at {time}', 'turn.images': '{count} images', + 'imagePreview.hint': 'Hover or focus to preview this turn\'s images; click to open the full image', + 'imagePreview.dialog': 'Image preview', + 'imagePreview.open': 'View image {index} of {count}', + 'imagePreview.alt': 'Image {index}', + 'imagePreview.loading': 'Loading image…', + 'imagePreview.failed': 'Image failed to load', + 'imagePreview.retry': 'Retry', + 'imagePreview.close': 'Close image preview', 'steps.total': '{total} steps', 'steps.tail': 'newest {limit} only', 'steps.all': 'Show all {total} steps', @@ -344,6 +360,14 @@ export const ja: Record = { 'turn.stepCount': '{count} ステップ', 'turn.started': 'このターンは {time} に開始', 'turn.images': '{count} 枚の画像', + 'imagePreview.hint': 'ホバーまたはフォーカスでこのターンの画像をプレビューし、クリックで原寸表示します', + 'imagePreview.dialog': '画像プレビュー', + 'imagePreview.open': '画像 {index} / {count} を表示', + 'imagePreview.alt': '画像 {index}', + 'imagePreview.loading': '画像を読み込み中…', + 'imagePreview.failed': '画像を読み込めませんでした', + 'imagePreview.retry': '再試行', + 'imagePreview.close': '画像プレビューを閉じる', 'steps.total': '全 {total} ステップ', 'steps.tail': '最新 {limit} ステップのみ', 'steps.all': '全 {total} ステップを表示', diff --git a/packages/experimental/client-ui-perf/src/client/perf-aggregate.ts b/packages/experimental/client-ui-perf/src/client/perf-aggregate.ts index 5a4075b071..650223732b 100644 --- a/packages/experimental/client-ui-perf/src/client/perf-aggregate.ts +++ b/packages/experimental/client-ui-perf/src/client/perf-aggregate.ts @@ -1,5 +1,6 @@ /** Turn aggregation over the step metrics a window holds. */ +import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment' import type { PerfCall, PerfContribution, PerfStepMetrics, PerfToolCount, PerfTurnMetrics, PerfViewNode, } from './perf-contract.ts' @@ -9,10 +10,13 @@ import { isNumber } from './perf-usage.ts' /** One Code-Mode root call's inner contribution, before merging into its step. */ type InnerContribution = Extract +/** Empty image list shared by turns whose trigger attached no image. */ +const NO_IMAGES: readonly ImageAttachmentRef[] = [] + /** The trigger that opened one turn, kept from the highest-priority source. */ interface TurnTrigger { readonly text: string - readonly images: number + readonly images: readonly ImageAttachmentRef[] readonly source: string } @@ -184,7 +188,7 @@ export function aggregateTurn(turn: number, steps: readonly PerfStepMetrics[]): aborted: steps.some(step => step.aborted), wall, prompt: null, - promptImages: 0, + promptImages: NO_IMAGES, promptSource: null, answer, } @@ -291,7 +295,7 @@ export function buildTurns(nodes: Iterable): PerfTurnMetrics[] { running: (index < order.length - 1 || at !== undefined) ? false : aggregate.running, interrupted: closedKind.get(turn) === 'interrupted' ? true : aggregate.interrupted, prompt: prompt === undefined || prompt.text === '' ? null : prompt.text, - promptImages: prompt === undefined ? 0 : prompt.images, + promptImages: prompt === undefined ? NO_IMAGES : prompt.images, promptSource: prompt === undefined ? null : prompt.source, }) }) diff --git a/packages/experimental/client-ui-perf/src/client/perf-contract.ts b/packages/experimental/client-ui-perf/src/client/perf-contract.ts index c80a9b913d..1559106436 100644 --- a/packages/experimental/client-ui-perf/src/client/perf-contract.ts +++ b/packages/experimental/client-ui-perf/src/client/perf-contract.ts @@ -1,5 +1,6 @@ /** Published performance model: per-step metrics, per-turn aggregates, and marker contributions. */ +import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment' import type { ConversationLocation, ConversationViewNode, } from '@deepseek-ai/dsh-client-ui-conversation/client' @@ -70,7 +71,8 @@ export interface PerfStepState { /** Turn-level marker kind; step states leave this null. */ readonly mark: 'prompt' | 'end' | null readonly prompt: string | null - readonly images: number + /** Durable image references the prompt attached, in message order. */ + readonly images: readonly ImageAttachmentRef[] readonly closed: boolean readonly closeKind: string | null /** Producer source kind of the trigger prompt Context. */ @@ -112,7 +114,7 @@ export interface PerfStepMetrics { readonly ttftPending?: number readonly mark?: 'prompt' | 'end' | null readonly prompt?: string | null - readonly images?: number + readonly images?: readonly ImageAttachmentRef[] readonly closed?: boolean readonly closeKind?: string | null } @@ -148,7 +150,8 @@ export interface PerfTurnMetrics { readonly aborted: boolean readonly wall: number | null readonly prompt: string | null - readonly promptImages: number + /** Durable image references the prompt attached, in message order. */ + readonly promptImages: readonly ImageAttachmentRef[] /** Producer source kind that opened the turn (`user`, `agent-message`, ...). */ readonly promptSource: string | null /** Last answer preview measured in the turn. */ @@ -169,7 +172,8 @@ export type PerfContribution = readonly kind: 'prompt' readonly turn: number readonly prompt: string | null - readonly images: number + /** Durable image references the prompt attached, in message order. */ + readonly images: readonly ImageAttachmentRef[] /** Producer source kind of the triggering user message. */ readonly source: string } diff --git a/packages/experimental/client-ui-perf/src/client/perf-fold.ts b/packages/experimental/client-ui-perf/src/client/perf-fold.ts index 88bcf3f077..d7e10c5574 100644 --- a/packages/experimental/client-ui-perf/src/client/perf-fold.ts +++ b/packages/experimental/client-ui-perf/src/client/perf-fold.ts @@ -1,6 +1,7 @@ /** Event folding and per-step metric derivation for the performance table. */ import type { SessionEventLike } from '@deepseek-ai/dsh-api-session-controller/client' +import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment' import type { AssistantMessage, ContentBlock, StreamChunk, UserMessage } from '@deepseek-ai/dsh-llm' import { assistantStreamFirstTokenTime } from '@deepseek-ai/dsh-llm/assistant-stream' // Type-only: the PTC event members this fold reads. @@ -302,7 +303,7 @@ export function newState( toolErrors: 0, mark, prompt: null, - images: 0, + images: [], closed: false, closeKind: null, trigger: null, @@ -340,14 +341,14 @@ function resultInfo(content: readonly ContentBlock[]): { /** * User question: only a human-typed `user` source is the text the user wrote. * @param message - one user-role message from the Session log. - * @returns the one-line question and the number of attached images. + * @returns the one-line question and the durable images it attached. */ -export function promptOf(message: UserMessage): { text: string | null; images: number } { +export function promptOf(message: UserMessage): { text: string | null; images: readonly ImageAttachmentRef[] } { const parts: string[] = [] - let images = 0 + const images: ImageAttachmentRef[] = [] for (const block of message.content) { if (block.type === 'text') parts.push(block.text) - if (block.type === 'image') images += 1 + if (block.type === 'image') images.push(block.attachment) } return { text: clip(parts.join(' '), 400), images } } diff --git a/packages/experimental/client-ui-perf/src/client/perf.module.css b/packages/experimental/client-ui-perf/src/client/perf.module.css index c603ee019f..a6fd5396f2 100644 --- a/packages/experimental/client-ui-perf/src/client/perf.module.css +++ b/packages/experimental/client-ui-perf/src/client/perf.module.css @@ -225,6 +225,109 @@ font: var(--dsw-font-xxxs-11); } +/* Image-count pill: the trigger pill's material on an interactive button. */ +.imagePill { + margin-right: 0; + background: transparent; + cursor: pointer; +} + +.promptText + .imagePill { + margin-left: 4px; +} + +.imagePill[data-state='failed'] { + color: var(--dsw-alias-state-error-primary); +} + +@media (hover: hover) { + .imagePill:hover { + background: var(--dsw-alias-bg-layer-2); + color: var(--dsw-alias-label-primary); + } +} + +.imagePill:focus-visible { + outline: 1px solid var(--dsw-alias-state-business-primary); + outline-offset: 1px; +} + +/* Prompt image preview inside the shared hover card: a small strip, or one large image. */ +.imagePreview { + display: flex; + flex-wrap: wrap; + gap: 6px; + max-width: 284px; +} + +.imageThumb { + box-sizing: border-box; + width: 76px; + height: 76px; + padding: 0; + border: 0.5px solid var(--dsw-alias-border-l2); + border-radius: 4px; + background: var(--dsw-alias-bg-layer-2); + cursor: pointer; + overflow: hidden; +} + +.imageThumb img { + display: block; + width: 100%; + height: 100%; + object-fit: cover; +} + +.imagePreview[data-single='true'] .imageThumb { + width: 100%; + height: 180px; +} + +.imagePreview[data-single='true'] .imageThumb img { + object-fit: contain; +} + +.imageThumb:focus-visible { + outline: 1px solid var(--dsw-alias-state-business-primary); + outline-offset: 1px; +} + +@media (hover: hover) { + .imageThumb:hover { + border-color: var(--dsw-alias-state-business-primary); + } +} + +.imageThumbStatus { + box-sizing: border-box; + display: inline-flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 4px; + width: 76px; + min-height: 76px; + padding: 4px; + border: 0.5px dashed var(--dsw-alias-border-l2); + border-radius: 4px; + color: var(--dsw-alias-label-secondary); + font: var(--dsw-font-xxxs-11); + text-align: center; +} + +.imagePreview[data-single='true'] .imageThumbStatus { + width: 100%; +} + +.imageThumbStatus[data-state='failed'] { + color: var(--dsw-alias-state-error-primary); +} + +.imageThumbStatus .chip { + margin-right: 0; +} + .answerPreview { color: var(--dsw-alias-label-tertiary); font-weight: 400; diff --git a/packages/experimental/client-ui-perf/tests/perf-fold.client.spec.ts b/packages/experimental/client-ui-perf/tests/perf-fold.client.spec.ts index 916a5f6fa6..2b90c93a7b 100644 --- a/packages/experimental/client-ui-perf/tests/perf-fold.client.spec.ts +++ b/packages/experimental/client-ui-perf/tests/perf-fold.client.spec.ts @@ -71,11 +71,11 @@ function textBlock(text: string): ContentBlock { return { type: 'text', text } } -function imageBlock(): ImageBlock { +function imageBlock(attachmentId = 'sha256:fixture'): ImageBlock { return { type: 'image', attachment: { - attachmentId: AttachmentId('sha256:fixture'), + attachmentId: AttachmentId(attachmentId), mediaType: 'image/png', bytes: 1, width: 1, @@ -692,13 +692,15 @@ describe('assistant timeline and usage', () => { }) describe('prompt helper', () => { - it('joins typed text blocks and counts images', () => { - const prompt = promptOf(userMessage([textBlock('explain'), textBlock('this'), imageBlock()])) + it('joins typed text blocks and keeps every attached image in order', () => { + const first = imageBlock() + const second = imageBlock('sha256:second') + const prompt = promptOf(userMessage([textBlock('explain'), textBlock('this'), first])) expect(prompt.text).toBe('explain this') - expect(prompt.images).toBe(1) - const imageOnly = promptOf(userMessage([imageBlock(), imageBlock()])) + expect(prompt.images).toEqual([first.attachment]) + const imageOnly = promptOf(userMessage([first, second])) expect(imageOnly.text).toBeNull() - expect(imageOnly.images).toBe(2) + expect(imageOnly.images).toEqual([first.attachment, second.attachment]) }) it('clips long questions and drops blank text', () => { @@ -1148,7 +1150,7 @@ describe('step metric derivation', () => { expect(metrics.cacheHit).toBeNull() expect(metrics.mark).toBeNull() expect(metrics.prompt).toBeNull() - expect(metrics.images).toBe(0) + expect(metrics.images).toEqual([]) expect(metrics.closed).toBe(false) expect(metrics.closeKind).toBeNull() }) @@ -1168,7 +1170,7 @@ describe('step metric derivation', () => { interrupted: true, mark: 'end', prompt: 'question', - images: 3, + images: [imageBlock().attachment, imageBlock('sha256:b').attachment, imageBlock('sha256:c').attachment], closed: true, closeKind: 'interrupted', })) @@ -1189,7 +1191,7 @@ describe('step metric derivation', () => { expect(metrics.toolPending).toBe(3) expect(metrics.mark).toBe('end') expect(metrics.prompt).toBe('question') - expect(metrics.images).toBe(3) + expect(metrics.images).toHaveLength(3) expect(metrics.closed).toBe(true) expect(metrics.closeKind).toBe('interrupted') }) diff --git a/packages/experimental/client-ui-perf/tests/perf-metrics.client.spec.ts b/packages/experimental/client-ui-perf/tests/perf-metrics.client.spec.ts index 240bec139f..cffa1a8e28 100644 --- a/packages/experimental/client-ui-perf/tests/perf-metrics.client.spec.ts +++ b/packages/experimental/client-ui-perf/tests/perf-metrics.client.spec.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest' +import { AttachmentId, type ImageAttachmentRef } from '@deepseek-ai/dsh-attachment' import { aggregateTurn, buildTurns } from '../src/client/perf-aggregate.ts' import type { PerfCall, PerfStepMetrics, PerfViewNode } from '../src/client/perf-contract.ts' import { CALL_LIMIT } from '../src/client/perf-fold.ts' @@ -72,7 +73,22 @@ function stepNode(value: PerfStepMetrics): PerfViewNode { } } -function promptNode(turn: number, prompt: string | null, images = 0, source = 'user'): PerfViewNode { +function imageRef(id: string): ImageAttachmentRef { + return { + attachmentId: AttachmentId(id), + mediaType: 'image/png', + bytes: 1, + width: 1, + height: 1, + } +} + +function promptNode( + turn: number, + prompt: string | null, + images: readonly ImageAttachmentRef[] = [], + source = 'user', +): PerfViewNode { return { key: `${turn}:prompt:${prompt ?? 'null'}`, kind: 'perf-prompt', @@ -142,7 +158,7 @@ describe('turn aggregation', () => { expect(turn.aborted).toBe(false) expect(turn.wall).toBeNull() expect(turn.prompt).toBeNull() - expect(turn.promptImages).toBe(0) + expect(turn.promptImages).toEqual([]) }) it('sums measured spans and reports partial coverage as a lower bound', () => { @@ -285,7 +301,7 @@ describe('turn aggregation', () => { it('describes the turn prompt as absent until the table attributes it', () => { const turn = aggregateTurn(2, [metricsOf()]) expect(turn.prompt).toBeNull() - expect(turn.promptImages).toBe(0) + expect(turn.promptImages).toEqual([]) expect(turn.steps).toHaveLength(1) }) }) @@ -299,13 +315,13 @@ describe('turn grouping', () => { stepNode(later), stepNode(one), stepNode(two), - promptNode(8, '问题', 2), + promptNode(8, '问题', [imageRef('sha256:a'), imageRef('sha256:b')]), endNode(9, 31_000, 'interrupted'), ]) expect(turns.map(turn => turn.turn)).toEqual([8, 9]) expect(turns[0]?.steps.map(step => step.step)).toEqual([1, 2]) expect(turns[0]?.prompt).toBe('问题') - expect(turns[0]?.promptImages).toBe(2) + expect(turns[0]?.promptImages).toEqual([imageRef('sha256:a'), imageRef('sha256:b')]) expect(turns[1]?.interrupted).toBe(true) }) @@ -324,7 +340,7 @@ describe('turn grouping', () => { ]) expect(turns).toHaveLength(1) expect(turns[0]?.prompt).toBeNull() - expect(turns[0]?.promptImages).toBe(0) + expect(turns[0]?.promptImages).toEqual([]) expect(turns[0]?.running).toBe(true) }) @@ -333,47 +349,47 @@ describe('turn grouping', () => { const turns = buildTurns([ stepNode(step), promptNode(1, null), - promptNode(1, 'first', 3), - promptNode(1, 'second', 4), + promptNode(1, 'first', [imageRef('sha256:a'), imageRef('sha256:b'), imageRef('sha256:c')]), + promptNode(1, 'second', [imageRef('sha256:d')]), ]) expect(turns[0]?.prompt).toBe('first') - expect(turns[0]?.promptImages).toBe(3) + expect(turns[0]?.promptImages).toHaveLength(3) }) it('classifies the turn trigger and prefers the human prompt', () => { const step = metricsOf({ turn: 1, step: 1, startTime: 100 }) const human = buildTurns([ stepNode(step), - promptNode(1, 'team message', 0, 'agent-message'), - promptNode(1, 'human question', 2, 'user'), + promptNode(1, 'team message', [], 'agent-message'), + promptNode(1, 'human question', [imageRef('sha256:a'), imageRef('sha256:b')], 'user'), ]) expect(human[0]?.prompt).toBe('human question') expect(human[0]?.promptSource).toBe('user') const team = buildTurns([ stepNode(metricsOf({ turn: 2, step: 1, startTime: 200 })), - promptNode(2, 'team message', 0, 'agent-message'), + promptNode(2, 'team message', [], 'agent-message'), ]) expect(team[0]?.prompt).toBe('team message') expect(team[0]?.promptSource).toBe('agent-message') const settled = buildTurns([ stepNode(metricsOf({ turn: 3, step: 1, startTime: 300 })), - promptNode(3, 'subagent done', 0, 'subagent-settled'), + promptNode(3, 'subagent done', [], 'subagent-settled'), ]) expect(settled[0]?.promptSource).toBe('subagent-settled') const goal = buildTurns([ stepNode(metricsOf({ turn: 4, step: 1, startTime: 400 })), - promptNode(4, 'goal round', 0, 'goal'), - promptNode(4, '', 0, 'goal'), + promptNode(4, 'goal round', [], 'goal'), + promptNode(4, '', [], 'goal'), ]) expect(goal[0]?.promptSource).toBe('goal') expect(goal[0]?.prompt).toBe('goal round') const context = buildTurns([ stepNode(metricsOf({ turn: 5, step: 1, startTime: 500 })), - promptNode(5, 'clock', 0, 'time-context'), + promptNode(5, 'clock', [], 'time-context'), ]) expect(context[0]?.promptSource).toBe('time-context') }) diff --git a/packages/experimental/client-ui-perf/tests/perf-plugin.client.spec.ts b/packages/experimental/client-ui-perf/tests/perf-plugin.client.spec.ts index 7b381a6c65..8dac650464 100644 --- a/packages/experimental/client-ui-perf/tests/perf-plugin.client.spec.ts +++ b/packages/experimental/client-ui-perf/tests/perf-plugin.client.spec.ts @@ -1,5 +1,6 @@ import { Context } from '@deepseek-ai/cordis' import { describe, expect, it, vi } from 'vitest' +import { AttachmentId, type ImageAttachmentRef } from '@deepseek-ai/dsh-attachment' import { LlmAttemptId } from '@deepseek-ai/dsh-llm/brand' import type { StreamChunk } from '@deepseek-ai/dsh-llm' import type { @@ -33,6 +34,15 @@ import { apply as hostApply } from '../src/index.ts' const SID = 'perf-session' as SessionId +/** One durable image the prompt fixtures attach. */ +const IMAGE: ImageAttachmentRef = { + attachmentId: AttachmentId('sha256:image'), + mediaType: 'image/png', + bytes: 12, + width: 2, + height: 2, +} + let seq = 0 /** @@ -115,6 +125,7 @@ interface PluginBench { readonly localeRegister: ReturnType readonly binding: ReturnType readonly loadOlder: ReturnType + readonly imageUrl: ReturnType readonly setOpenState: (state: 'cold' | 'open' | 'error') => void readonly settleOnNextSubscribe: () => void readonly setHasMore: (value: boolean) => void @@ -143,6 +154,7 @@ async function mountPlugin(): Promise { }), })) const loadOlder = vi.fn(async () => undefined) + const imageUrl = vi.fn(async () => 'blob:image') let openState: 'cold' | 'open' | 'error' = 'open' let hasMore = false let openListener: (() => void) | undefined @@ -165,6 +177,7 @@ async function mountPlugin(): Promise { ctx.provide('locale', { register: localeRegister, bind: () => makeTranslate(zh) } as never) ctx.provide('uiConversation', { binding, + imageUrl, events: { register: (definition: ConversationNodeDefinition) => { events.push(definition) @@ -193,6 +206,7 @@ async function mountPlugin(): Promise { localeRegister, binding, loadOlder, + imageUrl, setOpenState: (state: 'cold' | 'open' | 'error') => { openState = state openListener?.() @@ -391,9 +405,9 @@ describe('browser plugin registration', () => { bench.settleOnNextSubscribe() expect(await injected.loadOlder()).toBe(false) expect(bench.loadOlder).toHaveBeenCalledTimes(1) - // A no-progress page with older history left stays retryable. + // A no-progress page with older history left is surfaced, not hidden as exhausted. bench.setHasMore(true) - expect(await injected.loadOlder()).toBe(true) + await expect(injected.loadOlder()).rejects.toThrow(/did not advance/) await bench.fiber.dispose() }) @@ -420,6 +434,15 @@ describe('browser plugin registration', () => { await bench.fiber.dispose() }) + it('resolves prompt images through the shared Conversation image cache', async () => { + const bench = await mountPlugin() + const injectEntry = bench.entries()[0]!.inject as ((sessionId: SessionId) => PerfViewInjected) | undefined + const injected = injectEntry!(SID) + await expect(injected.loadImage(IMAGE)).resolves.toBe('blob:image') + expect(bench.imageUrl).toHaveBeenCalledWith(SID, IMAGE) + await bench.fiber.dispose() + }) + it('serves the empty snapshot until the target publishes turns', async () => { const bench = await mountPlugin() const injectEntry = bench.entries()[0]!.inject as ((sessionId: SessionId) => PerfViewInjected) | undefined @@ -435,11 +458,18 @@ describe('browser plugin registration', () => { bench.notify() expect(listener).toHaveBeenCalledTimes(1) unsubscribe() - expect(await injected.loadOlder()).toBe(false) - expect(bench.loadOlder).toHaveBeenCalledTimes(1) - // A no-progress page with older history left stays retryable. - bench.setHasMore(true) + // A page that grew the published snapshot reports progress. + bench.loadOlder.mockImplementationOnce(() => { + bench.setSnapshot({ turns: [], stepCount: 0 }) + }) expect(await injected.loadOlder()).toBe(true) + expect(bench.loadOlder).toHaveBeenCalledTimes(1) + // A page that leaves the snapshot unchanged and has no history left is exhausted. + expect(await injected.loadOlder()).toBe(false) + expect(bench.loadOlder).toHaveBeenCalledTimes(2) + // A no-progress page with older history left is surfaced, not hidden as exhausted. + bench.setHasMore(true) + await expect(injected.loadOlder()).rejects.toThrow(/did not advance/) // The source is cached per Session binding; a second injection reuses it. expect(injectEntry!(SID).hooks.perf).toBe(injected.hooks.perf) expect(() => injectEntry!('perf-missing' as SessionId)).toThrow(/unavailable/) @@ -485,7 +515,7 @@ describe('snapshot builder', () => { expect(builder.apply({ upserts: [], timeline: { turnOrder: [], turns: new Map() } })).toBe(first) const upserted = builder.apply({ upserts: [ - node('perf-prompt:1:0', { kind: 'prompt', turn: 1, prompt: '问', images: 2, source: 'user' }), + node('perf-prompt:1:0', { kind: 'prompt', turn: 1, prompt: '问', images: [IMAGE], source: 'user' }), node('perf-step:1:1', { kind: 'step', metrics: measuredStep(1, 1, 1_000, 1_800) }), node('perf-turn-end:1:100000', { kind: 'end', turn: 1, closeKind: 'completed', endTime: 1_800 }), ], @@ -510,7 +540,7 @@ describe('conversation Definitions', () => { const snapshot = assemble(definitions, [ entry(event('turn/start', { turn: 1 })), entry(event('user/message', { - role: 'user', id: 'u1', content: [{ type: 'text', text: '看看这个' }, { type: 'image' }], source: { kind: 'user' }, + role: 'user', id: 'u1', content: [{ type: 'text', text: '看看这个' }, { type: 'image', attachment: IMAGE }], source: { kind: 'user' }, })), entry(event('step/start', { turn: 1, step: 1 }, 2_000)), transient(1, 1, delta, 2_100), @@ -534,7 +564,7 @@ describe('conversation Definitions', () => { expect(snapshot?.stepCount).toBe(1) const turn = snapshot!.turns[0]! expect(turn.prompt).toBe('看看这个') - expect(turn.promptImages).toBe(1) + expect(turn.promptImages).toEqual([IMAGE]) expect(turn.interrupted).toBe(false) expect(turn.steps[0]?.startTime).toBe(2_000) expect(turn.steps[0]?.calls).toHaveLength(1) @@ -660,7 +690,7 @@ describe('conversation Definitions', () => { }) expect(prompt.match(runtime)).toEqual({ id: `${runtime.seq}`, role: 'start' }) const user = event('user/message', { - role: 'user', id: 'u6', content: [{ type: 'text', text: '继续' }, { type: 'image' }], source: { kind: 'user' }, + role: 'user', id: 'u6', content: [{ type: 'text', text: '继续' }, { type: 'image', attachment: IMAGE }], source: { kind: 'user' }, }) expect(prompt.match(user)).toEqual({ id: `${user.seq}`, role: 'start' }) const match = locatedStart(user, turnAt(9)) @@ -668,12 +698,12 @@ describe('conversation Definitions', () => { expect(state.turn).toBe(9) expect(state.mark).toBe('prompt') expect(state.prompt).toBe('继续') - expect(state.images).toBe(1) + expect(state.images).toEqual([IMAGE]) expect(state.trigger).toBe('user') const node = prompt.buildViewNode!({ ...contextOf(prompt, state, [match], match), }) as PerfViewNode - expect(node.data).toEqual({ kind: 'prompt', turn: 9, prompt: '继续', images: 1, source: 'user' }) + expect(node.data).toEqual({ kind: 'prompt', turn: 9, prompt: '继续', images: [IMAGE], source: 'user' }) const context: ConversationNodeContext & { readonly state: PerfStepState } = { ...contextOf(prompt, state, [match], match), state, diff --git a/packages/experimental/client-ui-perf/tests/perf-view.client.spec.tsx b/packages/experimental/client-ui-perf/tests/perf-view.client.spec.tsx index 9bd0b8cb49..01b2948741 100644 --- a/packages/experimental/client-ui-perf/tests/perf-view.client.spec.tsx +++ b/packages/experimental/client-ui-perf/tests/perf-view.client.spec.tsx @@ -11,6 +11,7 @@ import { createElement } from 'react' import { act, cleanup, fireEvent, render, screen, within } from '@testing-library/react' import { afterEach, describe, expect, it, vi } from 'vitest' +import { AttachmentId, type ImageAttachmentRef } from '@deepseek-ai/dsh-attachment' import { createSnapshotStore } from '@deepseek-ai/dsh-client-store' import { bindSnapshotSelector, makeTranslate } from '@deepseek-ai/dsh-client-test-runtime' import { PerfView } from '../src/client/PerfView.tsx' @@ -133,7 +134,7 @@ function turn(overrides: Partial = {}): PerfTurnMetrics { aborted: false, wall: 1_000, prompt: null, - promptImages: 0, + promptImages: [], promptSource: null, ...overrides, } @@ -158,14 +159,26 @@ function callSnapshot( /** Loader contract the view receives from the inject face. */ type LoadOlder = () => Promise +/** One durable prompt image reference. */ +function imageRef(id = 'sha256:image'): ImageAttachmentRef { + return { + attachmentId: AttachmentId(id), + mediaType: 'image/png', + bytes: 12, + width: 2, + height: 2, + } +} + /** Render the view with every prop it consumes. */ function mount( snapshot: PerfSnapshot, loadOlder: LoadOlder = async () => false, + loadImage: (attachment: ImageAttachmentRef) => Promise = async () => 'blob:image', ): ReturnType> { const store = createSnapshotStore(snapshot) const usePerf = bindSnapshotSelector(store) - render(createElement(PerfView, { usePerf, t, loadOlder } as never)) + render(createElement(PerfView, { usePerf, t, loadOlder, loadImage } as never)) return store } @@ -205,12 +218,15 @@ function stubInitialScroll(height: number): { top: number } { function deferred(): { readonly promise: Promise readonly resolve: (value: T) => void + readonly reject: (reason?: unknown) => void } { let settle: (value: T) => void = () => undefined - const promise = new Promise((resolve) => { + let fail: (reason?: unknown) => void = () => undefined + const promise = new Promise((resolve, reject) => { settle = resolve + fail = reject }) - return { promise, resolve: settle } + return { promise, resolve: settle, reject: fail } } /** Give the navigator a clipboard mock for copy-control cases. */ @@ -282,6 +298,7 @@ afterEach(() => { cleanup() vi.useRealTimers() vi.restoreAllMocks() + vi.unstubAllGlobals() Reflect.deleteProperty(navigator, 'clipboard') }) @@ -416,10 +433,11 @@ describe('older history', () => { expect(loadOlder).toHaveBeenCalledTimes(1) expect(screen.queryByRole('status')).toBeNull() expect(screen.queryByText(say('loadOlder.done'))).toBeNull() - expect(screen.getByRole('button', { name: say('loadOlder') })).toBeTruthy() + const retry = screen.getByRole('button', { name: say('loadOlder.failed') }) + expect(screen.queryByRole('button', { name: say('loadOlder') })).toBeNull() await act(async () => { - fireEvent.click(screen.getByRole('button', { name: say('loadOlder') })) + fireEvent.click(retry) }) expect(loadOlder).toHaveBeenCalledTimes(2) }) @@ -466,7 +484,7 @@ describe('turn rows', () => { mount({ turns: [turn({ prompt: 'why?', - promptImages: 2, + promptImages: [imageRef('sha256:a'), imageRef('sha256:b')], cacheHit: 0.2, cacheRead: 100, cacheWrite: 100, @@ -497,8 +515,9 @@ describe('turn rows', () => { expect(miss.getAttribute('title')).toBe(say('state.cacheMiss.title')) const cells = cellsOf(row) - expect(cells[1]?.textContent).toBe(`why? · ${say('turn.images', { count: 2 })}`) - expect(cells[1]?.getAttribute('title')).toBe(`why? · ${say('turn.images', { count: 2 })}`) + expect(cells[1]?.querySelector('[class*="promptText"]')?.textContent).toBe('why?') + expect(cells[1]?.textContent).toBe(`why?${say('turn.images', { count: 2 })}`) + expect(cells[1]?.getAttribute('title')).toBe('why?') expect(cells[2]?.textContent).toBe(say('did.answer')) expect(cells[3]?.textContent).toBe(formatTokensShort(1_000)) expect(cells[3]?.getAttribute('title')).toBeNull() @@ -599,18 +618,20 @@ describe('turn rows', () => { expect(did.getAttribute('title')).toBe(say('did.title', { calls: 3, tools: 'read ×1 · edit ×1 · write ×1' })) }) - it('renders a prompt without images and an empty prompt cell without one', () => { + it('renders a prompt without images and an image-only prompt as a pill', () => { mount({ turns: [ turn({ turn: 1, prompt: 'plain question' }), - turn({ turn: 2, prompt: '', promptImages: 1 }), + turn({ turn: 2, prompt: '', promptImages: [imageRef()] }), turn({ turn: 3, prompt: null }), ], stepCount: 3, }) expect(cellsOf(rowWithPrefix(say('turn.label', { turn: 1 })))[1]?.textContent).toBe('plain question') - expect(cellsOf(rowWithPrefix(say('turn.label', { turn: 2 })))[1]?.textContent).toBe('') - expect(cellsOf(rowWithPrefix(say('turn.label', { turn: 2 })))[1]?.getAttribute('title')).toBeNull() + const only = cellsOf(rowWithPrefix(say('turn.label', { turn: 2 })))[1] as HTMLTableCellElement + expect(within(only).getByRole('button', { name: say('turn.images', { count: 1 }) })).toBeTruthy() + expect(only.textContent).toBe(say('turn.images', { count: 1 })) + expect(only.getAttribute('title')).toBe(say('turn.images', { count: 1 })) expect(cellsOf(rowWithPrefix(say('turn.label', { turn: 3 })))[1]?.textContent).toBe('') }) @@ -1471,3 +1492,173 @@ describe('live clock', () => { expect(vi.getTimerCount()).toBe(0) }) }) + +describe('prompt images', () => { + /** jsdom reports no hover capability; the view asks the primitive, which reads matchMedia. */ + function stubHover(canHover = true): void { + vi.stubGlobal('matchMedia', (query: string) => ({ matches: canHover && query === '(any-hover: hover)' })) + } + + /** One image pill by its accessible count name. */ + function imagePill(count: number): HTMLButtonElement { + return button(say('turn.images', { count })) + } + + /** Hover the pill's wrapper long enough for the preview dwell. */ + function hoverPill(pill: HTMLButtonElement): void { + fireEvent.pointerEnter(pill.parentElement as HTMLElement) + act(() => { vi.advanceTimersByTime(200) }) + } + + it('previews a multi-image prompt on hover and opens a thumbnail full size', async () => { + vi.useFakeTimers() + stubHover() + const loadImage = vi.fn(async (attachment: ImageAttachmentRef) => `blob:${attachment.attachmentId}`) + mount(turnSnapshot([step()], { + prompt: 'compare', + promptImages: [{ ...imageRef('sha256:a'), name: 'left.png' }, imageRef('sha256:b')], + }), async () => false, loadImage) + + expect(imagePill(2)).toBeTruthy() + expect(cellsOf(rowWithPrefix(say('turn.label', { turn: 1 })))[1]?.textContent) + .toBe(`compare${say('turn.images', { count: 2 })}`) + + hoverPill(imagePill(2)) + await act(async () => {}) + expect(screen.getByRole('group', { name: say('imagePreview.dialog') })).toBeTruthy() + expect(loadImage).toHaveBeenCalledTimes(2) + expect(loadImage).toHaveBeenCalledWith({ ...imageRef('sha256:a'), name: 'left.png' }) + const thumbs = screen.getAllByRole('img') + expect(thumbs).toHaveLength(2) + expect(thumbs[0]?.getAttribute('alt')).toBe('left.png') + expect(thumbs[0]?.getAttribute('src')).toBe('blob:sha256:a') + expect(thumbs[1]?.getAttribute('alt')).toBe(say('imagePreview.alt', { index: 2, count: 2 })) + + fireEvent.click(button(say('imagePreview.open', { index: 2, count: 2 }))) + const dialog = screen.getByRole('dialog', { name: say('imagePreview.dialog') }) + expect(dialog.querySelector('img')?.getAttribute('src')).toBe('blob:sha256:b') + // A portalled image click must not collapse the row that owns the pill. + expect(stepRows()).toHaveLength(1) + + fireEvent.keyDown(window, { key: 'Escape' }) + expect(screen.queryByRole('dialog')).toBeNull() + expect(stepRows()).toHaveLength(1) + }) + + it('loads the first image from the pill, reports progress, and closes the lightbox', async () => { + const gate = deferred() + const loadImage = vi.fn(() => gate.promise) + mount(turnSnapshot([step()], { prompt: null, promptImages: [imageRef()] }), async () => false, loadImage) + + imagePill(1).focus() + fireEvent.click(imagePill(1)) + expect(button(say('imagePreview.loading'))).toBeTruthy() + expect(loadImage).toHaveBeenCalledWith(imageRef()) + + await act(async () => { gate.resolve('blob:full') }) + const dialog = screen.getByRole('dialog', { name: say('imagePreview.dialog') }) + expect(dialog.querySelector('img')?.getAttribute('src')).toBe('blob:full') + expect(dialog.querySelector('img')?.getAttribute('alt')).toBe(say('imagePreview.alt', { index: 1, count: 1 })) + + fireEvent.click(button(say('imagePreview.close'))) + expect(screen.queryByRole('dialog')).toBeNull() + // Closing restores focus to the pill instead of the row toggle. + expect(document.activeElement).toBe(imagePill(1)) + expect(stepRows()).toHaveLength(1) + }) + + it('reports a failed pill load and retries it', async () => { + const loadImage = vi.fn() + .mockRejectedValueOnce(new Error('unreadable attachment')) + .mockResolvedValueOnce('blob:later') + mount(turnSnapshot([], { prompt: 'see', promptImages: [imageRef()] }), async () => false, loadImage) + + fireEvent.click(imagePill(1)) + await act(async () => {}) + expect(loadImage).toHaveBeenCalledTimes(1) + const failed = button(say('imagePreview.failed')) + expect(failed.hasAttribute('data-state')).toBe(true) + expect(failed.getAttribute('data-state')).toBe('failed') + + fireEvent.click(failed) + await act(async () => {}) + expect(loadImage).toHaveBeenCalledTimes(2) + expect(screen.getByRole('dialog', { name: say('imagePreview.dialog') })).toBeTruthy() + }) + + it('retries one failed preview thumbnail without dropping the preview', async () => { + vi.useFakeTimers() + stubHover() + const loadImage = vi.fn() + .mockRejectedValueOnce(new Error('unreadable attachment')) + .mockResolvedValueOnce('blob:recovered') + mount(turnSnapshot([], { prompt: '', promptImages: [imageRef()] }), async () => false, loadImage) + + hoverPill(imagePill(1)) + await act(async () => {}) + expect(screen.getByText(say('imagePreview.failed'))).toBeTruthy() + fireEvent.click(button(say('imagePreview.retry'))) + await act(async () => {}) + expect(loadImage).toHaveBeenCalledTimes(2) + expect(screen.getByRole('img').getAttribute('src')).toBe('blob:recovered') + expect(screen.queryByText(say('imagePreview.failed'))).toBeNull() + }) + + it('reveals the preview for keyboard focus and ignores touch entry', async () => { + vi.useFakeTimers() + stubHover(false) + mount(turnSnapshot([], { prompt: 'keyboard', promptImages: [imageRef()] })) + + const pill = imagePill(1) + const touch = new Event('pointerover', { bubbles: true }) + Object.defineProperty(touch, 'pointerType', { value: 'touch' }) + fireEvent(pill.parentElement as HTMLElement, touch) + act(() => { vi.advanceTimersByTime(500) }) + expect(screen.queryByRole('group', { name: say('imagePreview.dialog') })).toBeNull() + + // jsdom does not model keyboard input modality for :focus-visible. + vi.spyOn(pill, 'matches').mockReturnValue(true) + fireEvent.focus(pill) + await act(async () => {}) + expect(screen.getByRole('group', { name: say('imagePreview.dialog') })).toBeTruthy() + expect(screen.getByRole('img').getAttribute('src')).toBe('blob:image') + + fireEvent.keyDown(window, { key: 'Escape' }) + expect(screen.queryByRole('group', { name: say('imagePreview.dialog') })).toBeNull() + }) + + it('keeps the trigger badge and the image pill together on an automated turn', () => { + mount(turnSnapshot([], { + prompt: 'team note', + promptSource: 'agent-message', + promptImages: [imageRef()], + })) + const cell = cellsOf(rowWithPrefix(say('turn.label', { turn: 1 })))[1] as HTMLTableCellElement + expect(within(cell).getByText(say('trigger.agent-message'))).toBeTruthy() + expect(within(cell).getByRole('button', { name: say('turn.images', { count: 1 }) })).toBeTruthy() + expect(cell.textContent).toContain('team note') + }) + + it('drops pending preview loads when the view unmounts', async () => { + vi.useFakeTimers() + stubHover() + const ready = deferred() + const failed = deferred() + const loadImage = vi.fn((attachment: ImageAttachmentRef) => ( + attachment.attachmentId === imageRef('sha256:a').attachmentId ? ready.promise : failed.promise + )) + mount(turnSnapshot([], { + prompt: 'closing', + promptImages: [imageRef('sha256:a'), imageRef('sha256:b')], + }), async () => false, loadImage) + hoverPill(imagePill(2)) + await act(async () => {}) + expect(loadImage).toHaveBeenCalledTimes(2) + cleanup() + await act(async () => { + ready.resolve('blob:late') + failed.reject(new Error('unreadable attachment')) + }) + expect(screen.queryByRole('img')).toBeNull() + }) +}) diff --git a/packages/experimental/client-ui-perf/tsconfig.json b/packages/experimental/client-ui-perf/tsconfig.json index 0c0328a646..0c17dbd5fe 100644 --- a/packages/experimental/client-ui-perf/tsconfig.json +++ b/packages/experimental/client-ui-perf/tsconfig.json @@ -8,9 +8,11 @@ "references": [ { "path": "../../../vendor/cordis" }, { "path": "../../api/session-controller/tsconfig.client.json" }, + { "path": "../../attachment/attachment" }, { "path": "../../client/locale/tsconfig.client.json" }, { "path": "../../client/store" }, { "path": "../../client/ui-conversation/tsconfig.client.json" }, + { "path": "../../client/ui-primitives" }, { "path": "../../client/ui-renderer" }, { "path": "../../client/ui-session" }, { "path": "../../client/ui-slots" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dfaef97b85..0916c0fd10 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8989,6 +8989,9 @@ importers: '@deepseek-ai/dsh-client-ui-conversation': specifier: workspace:* version: link:../../client/ui-conversation + '@deepseek-ai/dsh-client-ui-primitives': + specifier: workspace:* + version: link:../../client/ui-primitives '@deepseek-ai/dsh-client-ui-renderer': specifier: workspace:* version: link:../../client/ui-renderer