From 2917b99d5c21d4b714ac44619c6f2dd82345880d Mon Sep 17 00:00:00 2001 From: mektpoy Date: Wed, 21 Oct 2026 19:19:56 +0800 Subject: [PATCH] fix(experimental): attribute perf rows per Session and load history in place --- .../experimental/client-ui-perf/package.json | 7 +- .../client-ui-perf/src/client/index.ts | 31 +- .../src/client/perf-aggregate.ts | 88 +- .../src/client/perf-contract.ts | 18 + .../src/client/perf-definition.ts | 133 ++- .../client-ui-perf/src/client/perf-fold.ts | 264 ++--- .../tests/perf-fold.client.spec.ts | 1014 ++++++----------- .../tests/perf-metrics.client.spec.ts | 834 ++++++++------ .../tests/perf-plugin.client.spec.ts | 296 +++-- pnpm-lock.yaml | 9 + 10 files changed, 1341 insertions(+), 1353 deletions(-) diff --git a/packages/experimental/client-ui-perf/package.json b/packages/experimental/client-ui-perf/package.json index affedfc83c..6916ed648e 100644 --- a/packages/experimental/client-ui-perf/package.json +++ b/packages/experimental/client-ui-perf/package.json @@ -28,8 +28,10 @@ "dsh": { "client": { "inject": [ + "@deepseek-ai/dsh-api-session-controller", "@deepseek-ai/dsh-client-locale", - "@deepseek-ai/dsh-client-ui-conversation" + "@deepseek-ai/dsh-client-ui-conversation", + "@deepseek-ai/dsh-client-ui-renderer" ], "platform": "web" } @@ -45,6 +47,7 @@ "devDependencies": { "@deepseek-ai/cordis": "workspace:~", "@deepseek-ai/dsh-api-session-controller": "workspace:*", + "@deepseek-ai/dsh-attachment": "workspace:*", "@deepseek-ai/dsh-client-locale": "workspace:*", "@deepseek-ai/dsh-client-store": "workspace:*", "@deepseek-ai/dsh-client-test-runtime": "workspace:*", @@ -53,7 +56,9 @@ "@deepseek-ai/dsh-client-ui-session": "workspace:*", "@deepseek-ai/dsh-client-ui-slots": "workspace:*", "@deepseek-ai/dsh-llm": "workspace:*", + "@deepseek-ai/dsh-llm-retry": "workspace:*", "@deepseek-ai/dsh-session": "workspace:*", + "@deepseek-ai/dsh-tool-executor-ptc": "workspace:*", "@deepseek-ai/dsh-tools": "workspace:*", "@testing-library/react": "^16.1.0", "react": "^18.2.0", diff --git a/packages/experimental/client-ui-perf/src/client/index.ts b/packages/experimental/client-ui-perf/src/client/index.ts index fbd2264ac4..752d99c126 100644 --- a/packages/experimental/client-ui-perf/src/client/index.ts +++ b/packages/experimental/client-ui-perf/src/client/index.ts @@ -4,6 +4,7 @@ */ import type { Context } from '@deepseek-ai/cordis' +import type { SessionBinding } from '@deepseek-ai/dsh-api-session-controller/client' import type { ObservableSnapshot } from '@deepseek-ai/dsh-client-store' import type { SessionId } from '@deepseek-ai/dsh-session/types' // Type-only: pulls the locale plugin's Context merge (ctx.locale). @@ -23,8 +24,8 @@ import { PerfView, type PerfViewInjected } from './PerfView.tsx' export type { PerfKey } from './locales.ts' export type { PerfCall, PerfSnapshot, PerfStepMetrics, PerfTurnMetrics } from './perf-contract.ts' -/** Required services: the conversation slot registry, view registry, and locale. */ -export const inject = ['slots', 'uiConversation', 'locale'] +/** Required services: the conversation slot and view registries, Session paging, and the locale service. */ +export const inject = ['slots', 'sessions', 'uiConversation', 'locale'] /** Snapshot hook name; `InjectFace` turns it into the component's `usePerf` prop. */ const HOOK = 'perf' @@ -36,6 +37,8 @@ const HOOK = 'perf' * @param ctx - client root context. */ export function apply(ctx: Context): void { + // Per-plugin-instance source cache; unloading the plugin drops the map with it. + const sources = new WeakMap>() ctx.effect(() => ctx.locale.register(NS, { zh, en }), 'client-ui-perf: dictionaries') // Registration-time text (the tab label) reads through the bound translate as // a thunk, so it follows the active locale without re-registration. @@ -49,12 +52,28 @@ export function apply(ctx: Context): void { locale: NS, label: () => t('view.perf'), inject: (sessionId: SessionId): PerfViewInjected => { + const binding = ctx.sessions.binding(sessionId) + const session = binding?.session + if (binding === undefined || session === undefined) { + throw new Error(`client-ui-perf: session "${sessionId}" is unavailable`) + } const target = ctx.uiConversation.binding(sessionId).target('perf') - const perf: ObservableSnapshot = { - getSnapshot: () => target.getSnapshot() ?? EMPTY_PERF_SNAPSHOT, - subscribe: listener => target.subscribe(listener), + let perf = sources.get(binding) + if (perf === undefined) { + perf = { + getSnapshot: () => target.getSnapshot() ?? EMPTY_PERF_SNAPSHOT, + subscribe: listener => target.subscribe(listener), + } + sources.set(binding, perf) + } + return { + hooks: { [HOOK]: perf }, + loadOlder: async () => { + const before = perf.getSnapshot() + await session.loadOlder() + return perf.getSnapshot() !== before + }, } - return { hooks: { [HOOK]: perf } } }, }, PerfView)) } diff --git a/packages/experimental/client-ui-perf/src/client/perf-aggregate.ts b/packages/experimental/client-ui-perf/src/client/perf-aggregate.ts index 864687708d..163154447c 100644 --- a/packages/experimental/client-ui-perf/src/client/perf-aggregate.ts +++ b/packages/experimental/client-ui-perf/src/client/perf-aggregate.ts @@ -1,10 +1,14 @@ /** Turn aggregation over the step metrics a window holds. */ import type { - PerfStepMetrics, PerfToolCount, PerfTurnMetrics, PerfViewNode, + PerfCall, PerfContribution, PerfStepMetrics, PerfToolCount, PerfTurnMetrics, PerfViewNode, } from './perf-contract.ts' +import { CALL_LIMIT, isCodeMode, toolSpanUnion } from './perf-fold.ts' import { isNumber } from './perf-usage.ts' +/** One Code-Mode root call's inner contribution, before merging into its step. */ +type InnerContribution = Extract + /** Context measured last in the turn: steps without usage keep the previous value. */ function turnContext(steps: readonly PerfStepMetrics[]): number | null { let context: number | null = null @@ -12,6 +16,58 @@ function turnContext(steps: readonly PerfStepMetrics[]): number | null { return context } +/** Cache breakdown measured last in the turn, from the same step whose hit rate the row shows. */ +function turnCache(steps: readonly PerfStepMetrics[]): { read: number | null; write: number | null } { + let read: number | null = null + let write: number | null = null + for (const step of steps) { + if (!isNumber(step.cacheHit)) continue + read = step.cacheRead + write = step.cacheWrite + } + return { read, write } +} + +/** Tool tallies by descending count; ties keep call order. */ +function tallyCalls(calls: readonly PerfCall[]): PerfToolCount[] { + const counts = new Map() + for (const call of calls) { + if (call.name === null) continue + counts.set(call.name, (counts.get(call.name) ?? 0) + 1) + } + return [...counts.entries()] + .map(([name, count]) => ({ name, count })) + .sort((left, right) => right.count - left.count) +} + +/** + * Merge the Code-Mode dispatches of one step into its row. The runner call + * disappears in favor of the real tools, so the row shows what the code called + * and the characters those results pushed into the next prompt. + * @param step - step row folded from native events only. + * @param inner - the step's Code-Mode contributions. + * @returns the merged row, or the original when no dispatch was recorded. + */ +function mergeInnerCalls(step: PerfStepMetrics, inner: readonly InnerContribution[]): PerfStepMetrics { + if (inner.length === 0) return step + const innerCalls = inner.flatMap(contribution => contribution.calls) + const calls = [ + ...step.calls.filter(call => !isCodeMode(call.name)), + ...innerCalls, + ] + .sort((left, right) => left.seq - right.seq) + .slice(0, CALL_LIMIT) + const chars = calls.reduce((sum, call) => sum + (call.chars ?? 0), 0) + return { + ...step, + calls, + tools: tallyCalls(calls), + toolOutput: chars, + toolWall: toolSpanUnion(calls), + toolErrors: step.toolErrors + innerCalls.filter(call => call.error).length, + } +} + /** * Aggregate one turn's steps into the row values the table shows. * @param turn - turn number. @@ -30,6 +86,8 @@ export function aggregateTurn(turn: number, steps: readonly PerfStepMetrics[]): let toolSum = 0 let toolPairable = 0 let toolPending = 0 + let toolWallSum = 0 + let toolWallMeasured = false const tools = new Map() for (const step of steps) { for (const item of step.tools) tools.set(item.name, (tools.get(item.name) ?? 0) + item.count) @@ -46,6 +104,10 @@ export function aggregateTurn(turn: number, steps: readonly PerfStepMetrics[]): toolSum += step.tool toolPairable += 1 } + if (isNumber(step.toolWall)) { + toolWallSum += step.toolWall + toolWallMeasured = true + } toolPending += step.toolPending } const first = steps[0] @@ -68,6 +130,7 @@ export function aggregateTurn(turn: number, steps: readonly PerfStepMetrics[]): if (isNumber(step.toolOutput)) toolOutput += step.toolOutput if (isNumber(step.wall)) lastStepWall = step.wall } + const cache = turnCache(steps) const all = steps.length const toolCounts: PerfToolCount[] = [...tools.entries()] .map(([name, count]) => ({ name, count })) @@ -86,9 +149,12 @@ export function aggregateTurn(turn: number, steps: readonly PerfStepMetrics[]): tps: decodeSum > 0 && outputSum > 0 ? outputSum / (decodeSum / 1000) : null, output: outputUnknown === all ? null : outputSum, tool: toolPairable === 0 ? null : toolSum, + toolWall: toolWallMeasured ? toolWallSum : null, toolPending: toolPairable === 0 ? 0 : toolPending, tools: toolCounts, cacheHit, + cacheRead: cache.read, + cacheWrite: cache.write, retries, toolErrors, toolOutput, @@ -112,6 +178,8 @@ function isEmptyStep(step: PerfStepMetrics): boolean { /** * Group published nodes into ordered turns, closing each turn with its markers. + * Code-Mode contributions merge into the step that dispatched them, identified + * by the root call id the node carries. * @param nodes - every node the target builder currently holds. * @returns turn rows ordered oldest first. */ @@ -120,12 +188,14 @@ export function buildTurns(nodes: Iterable): PerfTurnMetrics[] { // turn/end marker: turn → end time and reason, so an interrupted tail closes. const closedAt = new Map() const closedKind = new Map() - // User question per turn. + // User question per turn; the first typed question wins over later retries. const prompts = new Map() + // Code-Mode dispatches per step, merged after the steps are ordered. + const inner = new Map() for (const node of nodes) { const data = node.data if (data.kind === 'prompt') { - if (data.prompt !== null && data.prompt !== '') { + if (data.prompt !== null && data.prompt !== '' && !prompts.has(data.turn)) { prompts.set(data.turn, { text: data.prompt, images: data.images }) } continue @@ -135,6 +205,13 @@ export function buildTurns(nodes: Iterable): PerfTurnMetrics[] { if (data.closeKind !== null) closedKind.set(data.turn, data.closeKind) continue } + if (data.kind === 'inner') { + const key = `${data.turn}:${data.step}` + const list = inner.get(key) + if (list === undefined) inner.set(key, [data]) + else list.push(data) + continue + } const metrics = data.metrics if (!isNumber(metrics.turn)) continue const bucket = grouped.get(metrics.turn) @@ -145,7 +222,7 @@ export function buildTurns(nodes: Iterable): PerfTurnMetrics[] { const order = [...grouped.keys()].sort((left, right) => left - right) order.forEach((turn, index) => { const bucket = grouped.get(turn) - /* v8 ignore next -- order is built from grouped's own keys, so the lookup always resolves. */ + /* v8 ignore next -- turn came from the map's own keys, so the bucket always exists. */ if (bucket === undefined) return const steps = bucket.slice().sort((left, right) => left.step - right.step) // A step followed by another step cannot still be running: a trimmed window @@ -170,7 +247,8 @@ export function buildTurns(nodes: Iterable): PerfTurnMetrics[] { aborted: true, } } - const aggregate = aggregateTurn(turn, steps) + const merged = steps.map(step => mergeInnerCalls(step, inner.get(`${turn}:${step.step}`) ?? [])) + const aggregate = aggregateTurn(turn, merged) const prompt = prompts.get(turn) turns.push({ ...aggregate, diff --git a/packages/experimental/client-ui-perf/src/client/perf-contract.ts b/packages/experimental/client-ui-perf/src/client/perf-contract.ts index 45e90159cc..5ff035060c 100644 --- a/packages/experimental/client-ui-perf/src/client/perf-contract.ts +++ b/packages/experimental/client-ui-perf/src/client/perf-contract.ts @@ -33,6 +33,10 @@ export interface PerfCall { readonly truncated: boolean /** Measured tool duration, or null while the result is missing. */ readonly ms: number | null + /** Wall-clock start of the call, for the non-overlapping tool-span union. */ + readonly startedAt: number | null + /** Wall-clock end of the call, present only after its result settled. */ + readonly endedAt: number | null } /** Tool call tally shown as one pill. */ @@ -84,6 +88,8 @@ export interface PerfStepMetrics { readonly tps: number | null readonly output: number | null readonly tool: number | null + /** Union of the paired calls' wall intervals; tool calls may run in parallel. */ + readonly toolWall: number | null readonly tools: readonly PerfToolCount[] readonly calls: readonly PerfCall[] readonly toolOutput: number | null @@ -119,9 +125,14 @@ export interface PerfTurnMetrics { readonly tps: number | null readonly output: number | null readonly tool: number | null + /** Union of the turn's step tool spans; steps never overlap. */ + readonly toolWall: number | null readonly toolPending: number readonly tools: readonly PerfToolCount[] readonly cacheHit: number | null + /** Cache breakdown of the last step that reported usage, for the turn tooltip. */ + readonly cacheRead: number | null + readonly cacheWrite: number | null readonly retries: number readonly toolErrors: number readonly toolOutput: number @@ -137,6 +148,13 @@ export interface PerfTurnMetrics { /** One published node: a step's metrics or a turn-level marker. */ export type PerfContribution = | { readonly kind: 'step'; readonly metrics: PerfStepMetrics } + | { + /** Inner tool calls one Code-Mode root call dispatched; merged into its step. */ + readonly kind: 'inner' + readonly turn: number + readonly step: number + readonly calls: readonly PerfCall[] + } | { readonly kind: 'prompt' readonly turn: number diff --git a/packages/experimental/client-ui-perf/src/client/perf-definition.ts b/packages/experimental/client-ui-perf/src/client/perf-definition.ts index 241a704d2e..b8eff4de39 100644 --- a/packages/experimental/client-ui-perf/src/client/perf-definition.ts +++ b/packages/experimental/client-ui-perf/src/client/perf-definition.ts @@ -1,6 +1,12 @@ /** * Conversation Definitions producing the performance target's nodes: one step - * context per `step/start`, one prompt marker per turn, one end marker per turn. + * context per `step/start`, one prompt context per user message, one end marker + * per turn, and one context per Code-Mode root call for its inner dispatches. + * + * Every callback is a pure function of the events it receives: turn and step + * identity comes from the engine's match location, never from module state, so + * one Definition set serves every Session binding and history paging + * (`replaceWindow`/`prepend`) cannot misattribute events. */ import type { Context } from '@deepseek-ai/cordis' @@ -9,7 +15,7 @@ import type { } from '@deepseek-ai/dsh-client-ui-conversation/client' import type { PerfStepState, PerfViewNode } from './perf-contract.ts' import { - currentTurnOf, fallbackState, foldEvent, newState, ptcStepOf, stepMetrics, + CALL_LIMIT, fallbackState, foldEvent, isCodeMode, newState, promptOf, stepMetrics, } from './perf-fold.ts' /** Target name shared by every Definition and the view builder. */ @@ -18,37 +24,52 @@ export const TARGET = 'perf' /** Definition kind of one measured step. */ const STEP_KIND = 'perf-step' -/** Definition kind of a turn's prompt marker. */ +/** Definition kind of one user prompt. */ const PROMPT_KIND = 'perf-prompt' /** Definition kind of a turn's end marker. */ const END_KIND = 'perf-turn-end' +/** Definition kind of one Code-Mode root call's inner dispatches. */ +const CODE_MODE_KIND = 'perf-code-mode' + /** Marker ids never collide with a step number. */ const PROMPT_STEP = 0 const END_STEP = 100_000 +/** Turn and step a match's Location resolves, or null when the window lost them. */ +function locationTurnStep( + location: ConversationLocation | undefined, +): { readonly turn: number; readonly step: number | null } | null { + if (location === undefined) return null + if (location.kind === 'turn') return { turn: location.turn.turn, step: null } + if (location.kind === 'step') return { turn: location.turn.turn, step: location.step.step } + return null +} + +/** Location a published node carries: the start match's, else the newest match's. */ +function nodeLocation(context: ConversationNodeContext): ConversationLocation { + return context.start?.location ?? context.matches.at(-1)?.location ?? { kind: 'unresolved' } +} + /** Wrap one contribution in the target's node envelope. */ function perfNode( context: ConversationNodeContext, anchorSeq: number, data: PerfViewNode['data'], ): PerfViewNode { - const location: ConversationLocation = context.start?.location - ?? context.matches.at(-1)?.location - ?? { kind: 'unresolved' } return { key: context.key, kind: context.kind, id: context.id, target: TARGET, anchorSeq, - location, + location: nodeLocation(context), data, } } -/** State a step context folds from the matches a trimmed window still holds. */ +/** State a step-like Context folds from the matches a trimmed window still holds. */ function stepState(context: ConversationNodeContext): PerfStepState | undefined { return context.state ?? fallbackState(context.matches) } @@ -69,11 +90,6 @@ const stepDefinition: ConversationNodeDefinition = { case 'tool/call': case 'tool/result': return { id: `${event.data.turn}:${event.data.step}`, role: 'update' } - case 'tool/ptc-dispatch': - case 'tool/ptc-dispatch-start': { - const id = ptcStepOf(event.data.rootCallId) - return id === null ? null : { id, role: 'update' } - } default: return null } @@ -102,38 +118,34 @@ const stepDefinition: ConversationNodeDefinition = { }, } -/** The user question of one turn; `user/message` carries no turn of its own. */ +/** The user question of one turn: one Context per human-typed `user/message`. */ const promptDefinition: ConversationNodeDefinition = { kind: PROMPT_KIND, target: TARGET, - match: (event) => { - if (event.type === 'turn/start') { - return { id: `${event.data.turn}:${PROMPT_STEP}`, role: 'start' } - } - if (event.type !== 'user/message') return null - const turn = currentTurnOf() - const source: unknown = event.data.source - if (turn === null || typeof source !== 'object' || source === null) return null - return (source as { kind?: unknown }).kind === 'user' - ? { id: `${turn}:${PROMPT_STEP}`, role: 'update' } - : null - }, + match: event => (event.type === 'user/message' && event.data.source.kind === 'user' + ? { id: `${event.seq}`, role: 'start' } + : null), start: (_context, match) => { - if (match.event.type !== 'turn/start') { - throw new Error('perf-prompt start requires turn/start') + if (match.event.type !== 'user/message') { + throw new Error('perf-prompt start requires user/message') + } + const located = locationTurnStep(match.location) + const prompt = promptOf(match.event.data) + return { + ...newState(located?.turn ?? 0, PROMPT_STEP, 'prompt'), + prompt: prompt.text, + images: prompt.images, } - /* v8 ignore next -- start only receives turn/start, which folding always handles. */ - return foldEvent(undefined, match.event) ?? newState(match.event.data.turn, PROMPT_STEP, 'prompt') }, - /* v8 ignore next -- the engine calls update only after start adopted a state, so folding never returns undefined here. */ - update: (context, match) => foldEvent(context.state, match.event) ?? context.state, - publication: match => (match.event.type === 'turn/start' ? 'none' : 'immediate'), + update: context => context.state, + publication: () => 'immediate', buildViewNode: (context) => { const state = context.state - if (state === undefined) return null + const located = locationTurnStep(nodeLocation(context)) + if (state === undefined || located === null) return null return perfNode(context, context.start?.event.seq ?? 0, { kind: 'prompt', - turn: state.turn, + turn: located.turn, prompt: state.prompt, images: state.images, }) @@ -151,14 +163,12 @@ const endDefinition: ConversationNodeDefinition = { if (match.event.type !== 'turn/end') { throw new Error('perf-turn-end start requires turn/end') } - const reason = match.event.data.reason - const closeKind = 'kind' in reason && typeof reason.kind === 'string' ? reason.kind : null return { ...newState(match.event.data.turn, END_STEP, 'end'), ended: true, endTime: match.event.time, closed: true, - closeKind, + closeKind: match.event.data.reason.kind, } }, update: context => context.state, @@ -175,7 +185,51 @@ const endDefinition: ConversationNodeDefinition = { } /** - * Register the three Definitions. Each registration rides the caller's fiber + * One Context per Code-Mode root call folds its inner dispatches; the builder + * merges them into the owning step by the root call id. The events carry no + * turn or step, so the Context takes both from the engine's match location. + */ +const codeModeDefinition: ConversationNodeDefinition = { + kind: CODE_MODE_KIND, + target: TARGET, + match: (event) => { + if (event.type !== 'tool/ptc-dispatch-start' && event.type !== 'tool/ptc-dispatch') return null + return { + id: `code:${event.data.rootCallId}`, + role: event.type === 'tool/ptc-dispatch-start' ? 'start' : 'update', + } + }, + start: (_context, match) => { + if (match.event.type !== 'tool/ptc-dispatch-start') { + throw new Error('perf-code-mode start requires tool/ptc-dispatch-start') + } + const located = locationTurnStep(match.location) + const base = located?.step === null || located === null + ? newState(0, 0) + : newState(located.turn, located.step) + /* v8 ignore next -- folding a defined state always returns a state. */ + return foldEvent(base, match.event) ?? base + }, + /* v8 ignore next -- the engine calls update only after start adopted a state, so folding never returns undefined here. */ + update: (context, match) => foldEvent(context.state, match.event) ?? context.state, + buildViewNode: (context) => { + const state = stepState(context) + const located = locationTurnStep(nodeLocation(context)) + if (state === undefined || located?.step === null || located === null) return null + const calls = stepMetrics(state).calls + .filter(call => !isCodeMode(call.name)) + .slice(0, CALL_LIMIT) + return perfNode(context, context.start?.event.seq ?? 0, { + kind: 'inner', + turn: located.turn, + step: located.step, + calls, + }) + }, +} + +/** + * Register the four Definitions. Each registration rides the caller's fiber * effect, so unloading the plugin removes the nodes with it. * @param ctx - client root context. */ @@ -183,4 +237,5 @@ export function registerPerfDefinitions(ctx: Context): void { ctx.uiConversation.events.register(stepDefinition) ctx.uiConversation.events.register(promptDefinition) ctx.uiConversation.events.register(endDefinition) + ctx.uiConversation.events.register(codeModeDefinition) } diff --git a/packages/experimental/client-ui-perf/src/client/perf-fold.ts b/packages/experimental/client-ui-perf/src/client/perf-fold.ts index 9d42463944..2bdc3d003c 100644 --- a/packages/experimental/client-ui-perf/src/client/perf-fold.ts +++ b/packages/experimental/client-ui-perf/src/client/perf-fold.ts @@ -1,15 +1,15 @@ /** Event folding and per-step metric derivation for the performance table. */ import type { SessionEventLike } from '@deepseek-ai/dsh-api-session-controller/client' -import type { StreamChunk } from '@deepseek-ai/dsh-llm' +import type { ContentBlock, StreamChunk, UserMessage } from '@deepseek-ai/dsh-llm' import { assistantStreamFirstTokenTime } from '@deepseek-ai/dsh-llm/assistant-stream' // Type-only: the PTC event members this fold reads. -import type {} from '@deepseek-ai/dsh-tools/types' +import type {} from '@deepseek-ai/dsh-tool-executor-ptc/types' import type { - PerfCall, PerfStepMetrics, PerfStepState, PerfToolCount, PerfUsage, + PerfCall, PerfStepMetrics, PerfStepState, PerfToolCount, } from './perf-contract.ts' import { - cacheHitRate, cacheReadTokens, cacheWriteTokens, contextTokens, isNumber, isUsage, + cacheHitRate, cacheReadTokens, cacheWriteTokens, contextTokens, isNumber, outputTokens, } from './perf-usage.ts' @@ -89,6 +89,11 @@ function countLines(text: string): number { return text.charCodeAt(text.length - 1) === 10 ? lines - 1 : lines } +/** Read a record entry only when the record owns it, so model ids never resolve inherited members. */ +function ownValue(record: Readonly> | null, key: string): T | undefined { + return record !== null && Object.hasOwn(record, key) ? record[key] : undefined +} + /** Longest common subsequence length, one rolling row; fine below 400 lines a side. */ function lcsLength(left: readonly string[], right: readonly string[]): number { const width = right.length + 1 @@ -96,9 +101,9 @@ function lcsLength(left: readonly string[], right: readonly string[]): number { let current = new Uint16Array(width) for (let row = 1; row <= left.length; row += 1) { for (let column = 1; column <= right.length; column += 1) { - /* v8 ignore next -- an in-bounds typed-array read is always a number; the fallback only satisfies the index type. */ + /* v8 ignore next -- an in-bounds typed-array read always yields a number; the fallback only satisfies the index type. */ const up = previous[column] ?? 0 - /* v8 ignore next -- an in-bounds typed-array read is always a number; the fallback only satisfies the index type. */ + /* v8 ignore next -- an in-bounds typed-array read always yields a number; the fallback only satisfies the index type. */ const leftCell = current[column - 1] ?? 0 /* v8 ignore next 3 -- the typed-array nullish fallback is unreachable; in-bounds reads never yield undefined. */ current[column] = left[row - 1] === right[column - 1] @@ -110,7 +115,7 @@ function lcsLength(left: readonly string[], right: readonly string[]): number { current = swap current.fill(0) } - /* v8 ignore next -- an in-bounds typed-array read is always a number; the fallback only satisfies the index type. */ + /* v8 ignore next -- an in-bounds typed-array read always yields a number; the fallback only satisfies the index type. */ return previous[right.length] ?? 0 } @@ -211,14 +216,24 @@ function toolHint(name: string, raw: unknown): string | null { } /** Code-Mode steps call this one tool; the real tools run inside it. */ -function isCodeMode(name: string | null): boolean { +export function isCodeMode(name: string | null): boolean { return name !== null && /^run_?code$/i.test(name) } -/** Tool name of one call; a nameless call counts but invents no name. */ -function calledTool(data: { readonly name?: unknown }): string | null { - const name = data.name - return typeof name === 'string' && name !== '' ? name : null +/** Arguments-derived PerfCall fields shared by a native call and one Code-Mode dispatch. */ +function callArguments( + name: string, + raw: unknown, +): Pick { + const delta = editDelta(name, raw) + return { + hint: toolHint(name, raw), + added: delta === null ? null : delta.added, + removed: delta === null ? null : delta.removed, + before: delta === null ? null : delta.before, + after: delta === null ? null : delta.after, + args: rawArgs(raw), + } } /** Whether one stream chunk counts as the first model token. */ @@ -234,49 +249,6 @@ function isTokenDelta(chunk: StreamChunk): boolean { } } -/** Root call id → step id, for inner Code-Mode calls that carry no turn/step. */ -const PTC_STEPS = new Map() -const PTC_STEPS_MAX = 4_000 - -/** Remember which step owns one Code-Mode root call, evicting the oldest entry. */ -function rememberPtcCall(callId: string, stepId: string): void { - PTC_STEPS.set(callId, stepId) - if (PTC_STEPS.size <= PTC_STEPS_MAX) return - const oldest = PTC_STEPS.keys().next().value - /* v8 ignore next -- the size check above guarantees a first key, so the guard never sees undefined. */ - if (oldest !== undefined) PTC_STEPS.delete(oldest) -} - -/** - * Step id owning one Code-Mode root call. - * @param rootCallId - root call id carried by a dispatch event. - * @returns the owning step id, or null when the call was not recorded. - */ -export function ptcStepOf(rootCallId: unknown): string | null { - return typeof rootCallId === 'string' ? PTC_STEPS.get(rootCallId) ?? null : null -} - -/** Clear the Code-Mode owner table; tests use it to isolate cases. */ -export function resetPtcSteps(): void { - PTC_STEPS.clear() -} - -/** Current turn number; `user/message` carries no turn and is attributed by order. */ -let currentTurn: number | null = null - -/** - * Turn number of the latest `turn/start`. - * @returns the current turn, or null before the window has one. - */ -export function currentTurnOf(): number | null { - return currentTurn -} - -/** Reset the current-turn marker; tests use it to isolate cases. */ -export function resetCurrentTurn(): void { - currentTurn = null -} - /** * Initial state of one step or one turn-level marker. * @param turn - turn number. @@ -318,38 +290,23 @@ export function newState( } } -/** Call id of a tool result lives on the message source, not on the result event. */ -function resultCallId(message: unknown): string | null { - if (message === null || typeof message !== 'object') return null - const source = (message as { source?: unknown }).source - if (source === null || typeof source !== 'object') return null - const callId = (source as { callId?: unknown }).callId - return typeof callId === 'string' ? callId : null -} - /** Result text: the text blocks of the tool message joined by newlines. */ -function resultText(message: unknown): string | null { - /* v8 ignore next -- callers reach this helper only after resolving a call id from an object message. */ - if (message === null || typeof message !== 'object') return null - const content = (message as { content?: unknown }).content - if (!Array.isArray(content)) return null +function resultText(content: readonly ContentBlock[]): string | null { const parts: string[] = [] - for (const item of content as unknown[]) { - if (item === null || typeof item !== 'object') continue - const block = item as { type?: unknown; text?: unknown } - if (block.type === 'text' && typeof block.text === 'string') parts.push(block.text) + for (const block of content) { + if (block.type === 'text') parts.push(block.text) } return parts.length === 0 ? null : parts.join('\n') } /** Size, line count, and bounded body of one tool result. */ -function resultInfo(message: unknown): { +function resultInfo(content: readonly ContentBlock[]): { chars: number lines: number text: string | null truncated: boolean } { - const text = resultText(message) + const text = resultText(content) if (text === null) return { chars: 0, lines: 0, text: null, truncated: false } let lines = 0 for (let index = 0; index < text.length; index += 1) { @@ -360,28 +317,12 @@ function resultInfo(message: unknown): { return { chars: text.length, lines, text: body.text, truncated: body.truncated } } -/** Whether one tool result reports an error. */ -function resultFailed(message: unknown): boolean { - return message !== null && typeof message === 'object' - && (message as { isError?: unknown }).isError === true -} - -/** - * User question: `user/message` also carries injected runtime context and time - * snapshots, so only `source.kind === 'user'` is the text the user typed. - */ -function promptOf(data: { - readonly content?: unknown - readonly source?: unknown -}): { text: string | null; images: number } { - const content = data.content - if (!Array.isArray(content)) return { text: null, images: 0 } +/** User question: only a human-typed `user` source is the text the user wrote. */ +export function promptOf(message: UserMessage): { text: string | null; images: number } { const parts: string[] = [] let images = 0 - for (const item of content as unknown[]) { - if (item === null || typeof item !== 'object') continue - const block = item as { type?: unknown; text?: unknown } - if (block.type === 'text' && typeof block.text === 'string') parts.push(block.text) + for (const block of message.content) { + if (block.type === 'text') parts.push(block.text) if (block.type === 'image') images += 1 } return { text: clip(parts.join(' '), 400), images } @@ -410,28 +351,11 @@ export function foldEvent( case 'step/start': return { ...newState(event.data.turn, event.data.step), startSeq: event.seq, startTime: event.time } - case 'turn/start': { - currentTurn = event.data.turn - return newState(event.data.turn, 0, 'prompt') - } - - case 'user/message': { - const next = state ?? newState(currentTurn ?? 0, 0) - const base: PerfStepState = { ...next, mark: 'prompt' } - const source: unknown = event.data.source - if (typeof source !== 'object' || source === null - || (source as { kind?: unknown }).kind !== 'user') return base - if (base.prompt !== null) return base - const prompt = promptOf(event.data) - return { ...base, prompt: prompt.text, images: prompt.images } - } - case 'assistant/live-chunk': { const next = state ?? newState(event.data.turn, event.data.step) const chunk = event.data.chunk if (chunk.type === 'usage') { - const usage: PerfUsage = chunk.usage - return isUsage(usage) ? { ...next, usage } : next + return { ...next, usage: chunk.usage } } if (next.firstTokenTime === null && isTokenDelta(chunk)) { return { ...next, firstTokenTime: event.time } @@ -447,11 +371,10 @@ export function foldEvent( if (isNumber(fromStream)) next = { ...next, firstTokenTime: fromStream } } if (event.type === 'assistant/attempt') return next - const usage = event.data.usage return { ...next, messageTime: next.messageTime ?? event.time, - usage: isUsage(usage) ? usage : next.usage, + usage: event.data.usage ?? next.usage, interrupted: event.data.interrupted === true, } } @@ -462,35 +385,30 @@ export function foldEvent( const pending: Record = { ...(next.pending ?? {}) } const known = Object.hasOwn(pending, callId) pending[callId] = event.time - const name = calledTool(event.data) + const name = event.data.name const calls: Record = { ...(next.calls ?? {}) } - if (isCodeMode(name)) rememberPtcCall(callId, `${event.data.turn}:${event.data.step}`) if (!Object.hasOwn(calls, callId)) { - const delta = name === null ? null : editDelta(name, event.data.arguments) calls[callId] = { callId, seq: event.seq, name, - hint: name === null ? null : toolHint(name, event.data.arguments), + ...callArguments(name, event.data.arguments), chars: null, lines: null, error: false, - added: delta === null ? null : delta.added, - removed: delta === null ? null : delta.removed, - before: delta === null ? null : delta.before, - after: delta === null ? null : delta.after, - args: rawArgs(event.data.arguments), inner: false, text: null, truncated: false, ms: null, + startedAt: event.time, + endedAt: null, } } return { ...next, pending, toolOpen: known ? next.toolOpen : next.toolOpen + 1, - tools: name === null ? next.tools : bumpTool(next.tools, name), + tools: bumpTool(next.tools, name), calls, } } @@ -500,10 +418,8 @@ export function foldEvent( /** * Inner Code-Mode call: the name and arguments are the real tool's, the - * content is the real result. It does not count toward tool time (the outer - * `run_code` call already covers it), but its own dispatch span is measured. - * The event carries no turn or step, so it only folds into a context the - * root-call map already resolved. + * content is the real result. It does not count toward the outer call's + * tool time; its own dispatch span is measured instead. */ case 'tool/ptc-dispatch-start': { if (state === undefined) return state @@ -514,30 +430,25 @@ export function foldEvent( case 'tool/ptc-dispatch': { const next = state if (next === undefined) return next - const name = calledTool(event.data) - if (name === null) return next - const info = resultInfo({ content: event.data.content }) - const delta = editDelta(name, event.data.arguments) + const name = event.data.name + const info = resultInfo(event.data.content) const callKey = event.data.subCallId - const startedAt = next.starts?.[callKey] + const startedAt = ownValue(next.starts, callKey) const calls: Record = { ...(next.calls ?? {}) } calls[callKey] = { callId: callKey, seq: event.seq, name, - hint: toolHint(name, event.data.arguments), + ...callArguments(name, event.data.arguments), chars: info.chars, lines: info.lines, text: info.text, truncated: info.truncated, error: event.data.isError, - added: delta === null ? null : delta.added, - removed: delta === null ? null : delta.removed, - before: delta === null ? null : delta.before, - after: delta === null ? null : delta.after, - args: rawArgs(event.data.arguments), inner: true, ms: isNumber(startedAt) ? Math.max(0, event.time - startedAt) : null, + startedAt: isNumber(startedAt) ? startedAt : null, + endedAt: event.time, } return { ...next, @@ -550,26 +461,27 @@ export function foldEvent( case 'tool/result': { const next = state ?? newState(event.data.turn, event.data.step) const message = event.data.message - const callId = resultCallId(message) - const failed = resultFailed(message) - const before = callId === null ? undefined : next.calls?.[callId] + const callId: string = message.source.callId + const failed = message.isError === true + const before = ownValue(next.calls, callId) const calls: Record = { ...(next.calls ?? {}) } - if (callId !== null && before !== undefined) { + if (before !== undefined) { calls[callId] = { ...before, - ...resultInfo(message), + ...resultInfo(message.content), error: failed || before.error, + /* v8 ignore next -- a native call record always carries the start time tool/call recorded. */ + ms: isNumber(before.startedAt) ? Math.max(0, event.time - before.startedAt) : before.ms, + endedAt: event.time, } } const withError = failed ? { ...next, toolErrors: next.toolErrors + 1 } : next - // Own-property check: the call id is model generated, so a name such as - // `constructor` must not resolve to an inherited function as a start time. - const dispatched = callId === null ? undefined : next.pending?.[callId] + const dispatched = ownValue(next.pending, callId) if (!isNumber(dispatched)) { return { ...withError, calls, toolMissing: next.toolMissing + 1 } } const pending: Record = {} - /* v8 ignore next -- a numeric dispatched time implies pending exists, so the empty fallback is unreachable. */ + /* v8 ignore next -- a numeric pending start implies pending exists; the empty fallback is unreachable. */ for (const [key, value] of Object.entries(next.pending ?? {})) { if (key !== callId) pending[key] = value } @@ -609,17 +521,6 @@ function callList(calls: Readonly> | null): PerfCall[] }) } -/** Tool result characters one step pushed into the next prompt. */ -function outputChars(calls: Readonly> | null, skipRunner: boolean): number { - if (calls === null) return 0 - let total = 0 - for (const call of Object.values(calls)) { - if (skipRunner && isCodeMode(call.name) && !call.inner) continue - if (isNumber(call.chars)) total += call.chars - } - return total -} - /** Tool tallies by descending count; ties keep first-call order. */ function sortedTools(tools: Readonly> | null): PerfToolCount[] { if (tools === null) return [] @@ -628,6 +529,37 @@ function sortedTools(tools: Readonly> | null): PerfToolCo .sort((left, right) => right.count - left.count) } +/** Wall span covered by calls that both started and settled; overlaps count once. */ +export function toolSpanUnion(calls: readonly PerfCall[]): number | null { + const intervals = calls + .filter((call): call is PerfCall & { startedAt: number; endedAt: number } => + isNumber(call.startedAt) && isNumber(call.endedAt)) + .map(call => [call.startedAt, call.endedAt] as const) + .sort((left, right) => left[0] - right[0]) + if (intervals.length === 0) return null + let total = 0 + let [start, end] = intervals[0] as readonly [number, number] + for (const [nextStart, nextEnd] of intervals.slice(1)) { + if (nextStart > end) { + total += end - start + start = nextStart + end = nextEnd + } else if (nextEnd > end) { + end = nextEnd + } + } + return total + end - start +} + +/** Tool result characters one step pushed into the next prompt. */ +function outputChars(calls: readonly PerfCall[]): number { + let total = 0 + for (const call of calls) { + if (isNumber(call.chars)) total += call.chars + } + return total +} + /** * Measured metrics of one step. * @param state - folded step state. @@ -646,10 +578,9 @@ export function stepMetrics(state: PerfStepState): PerfStepMetrics { ? output / (decode / 1000) : null const settled = isNumber(message) - const rawCalls = callList(state.calls) - const inner = rawCalls.some(call => call.inner) - const calls = inner ? rawCalls.filter(call => !isCodeMode(call.name)) : rawCalls - const tools = sortedTools(state.tools).filter(item => !(inner && isCodeMode(item.name))) + const calls = callList(state.calls) + // The union spans every paired call, not just the row's displayed prefix. + const allCalls = state.calls === null ? [] : Object.values(state.calls) return { turn: state.turn, step: state.step, @@ -662,9 +593,10 @@ export function stepMetrics(state: PerfStepState): PerfStepMetrics { tps, output, tool: state.toolPairs > 0 ? state.toolMs : null, - tools, + toolWall: toolSpanUnion(allCalls), + tools: sortedTools(state.tools), calls, - toolOutput: outputChars(state.calls, inner), + toolOutput: outputChars(calls), retries: state.retries, toolErrors: state.toolErrors, cacheRead: cacheReadTokens(state.usage), diff --git a/packages/experimental/client-ui-perf/tests/perf-fold.client.spec.ts b/packages/experimental/client-ui-perf/tests/perf-fold.client.spec.ts index af119edf12..b60eb46e97 100644 --- a/packages/experimental/client-ui-perf/tests/perf-fold.client.spec.ts +++ b/packages/experimental/client-ui-perf/tests/perf-fold.client.spec.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it } from 'vitest' import type { SessionEventLike } from '@deepseek-ai/dsh-api-session-controller/client' +import { AttachmentId } from '@deepseek-ai/dsh-attachment' import { LlmAttemptId, MessageId, @@ -7,34 +8,32 @@ import { type AssistantMessage, type AssistantStreamRecord, type ContentBlock, + type ImageBlock, type StreamChunk, type TokenUsage, type ToolResultMessage, type UserMessage, } from '@deepseek-ai/dsh-llm' +import { RetryId } from '@deepseek-ai/dsh-llm-retry' +import type {} from '@deepseek-ai/dsh-llm-retry/types' import { SessionSeq, type SessionEventMap, type SessionEventType } from '@deepseek-ai/dsh-session/types' -import type {} from '@deepseek-ai/dsh-tools/types' -import { aggregateTurn, buildTurns } from '../src/client/perf-aggregate.ts' +import type {} from '@deepseek-ai/dsh-tool-executor-ptc/types' import { - CALL_LIMIT, currentTurnOf, fallbackState, foldEvent, newState, ptcStepOf, - resetCurrentTurn, resetPtcSteps, stepMetrics, + CALL_LIMIT, fallbackState, foldEvent, isCodeMode, newState, promptOf, stepMetrics, toolSpanUnion, } from '../src/client/perf-fold.ts' import { DASH, barShare, formatChars, formatClock, formatDateTime, formatMs, formatPercent, formatTokens, formatTokensDelta, formatTokensShort, formatTps, } from '../src/client/perf-format.ts' -import type { - PerfCall, PerfStepMetrics, PerfStepState, PerfViewNode, -} from '../src/client/perf-contract.ts' +import type { PerfCall, PerfStepMetrics, PerfStepState } from '../src/client/perf-contract.ts' import { - cacheHitRate, cacheReadTokens, cacheWriteTokens, contextTokens, isCount, isNumber, - isUsage, outputTokens, + cacheHitRate, cacheReadTokens, cacheWriteTokens, contextTokens, isCount, isNumber, isUsage, outputTokens, } from '../src/client/perf-usage.ts' let seq = 0 let clock = 0 -/** One event envelope; running fixtures that need different times pass the time. */ +/** One event envelope; fixtures that need an exact time pass it. */ function emit(type: SessionEventType, data: object, time?: number): SessionEventLike { seq += 1 clock = time ?? clock + 10 @@ -50,18 +49,13 @@ function event( return emit(type, data, time) } -/** One event carrying a payload the fold must tolerate but the type map does not describe. */ -function unchecked(type: SessionEventType, data: object, time?: number): SessionEventLike { - return emit(type, data, time) -} - /** One client-only live chunk event. */ function liveChunk(turn: number, step: number, chunk: StreamChunk, time?: number): SessionEventLike { seq += 1 clock = time ?? clock + 10 return { type: 'assistant/live-chunk', - seq: SessionSeq(seq), + seq, time: clock, data: { attemptId: LlmAttemptId('attempt-1'), turn, step, chunk }, } @@ -77,6 +71,19 @@ function textBlock(text: string): ContentBlock { return { type: 'text', text } } +function imageBlock(): ImageBlock { + return { + type: 'image', + attachment: { + attachmentId: AttachmentId('sha256:fixture'), + mediaType: 'image/png', + bytes: 1, + width: 1, + height: 1, + }, + } +} + function userMessage( content: readonly ContentBlock[], source: UserMessage['source'] = { kind: 'user' }, @@ -142,6 +149,21 @@ function toolResult( }, time) } +function retry(turn: number, step: number, time: number): SessionEventLike { + return event('llm/retry', { + turn, + step, + retryId: RetryId('retry-1'), + provider: 'deepseek', + mode: 'normal', + policyKey: 'default', + retry: 1, + maxRetries: 3, + delayMs: 10, + failure: { message: 'rate limited', code: 'RATE_LIMITED' }, + }, time) +} + function ptcStart(root: string, sub: string, name: string, args: unknown, time: number): SessionEventLike { return event('tool/ptc-dispatch-start', { rootCallId: ToolCallId(root), @@ -213,36 +235,6 @@ function stateOf(overrides: Partial = {}): PerfStepState { return { ...newState(1, 1), ...overrides } } -function metricsOf(overrides: Partial = {}): PerfStepMetrics { - return { - turn: 1, - step: 1, - startTime: null, - endTime: null, - context: null, - ttft: null, - model: null, - decode: null, - tps: null, - output: null, - tool: null, - tools: [], - calls: [], - toolOutput: 0, - retries: 0, - toolErrors: 0, - cacheRead: null, - cacheWrite: null, - cacheHit: null, - wall: null, - toolPending: 0, - running: false, - interrupted: false, - aborted: false, - ...overrides, - } -} - function perfCall(callId: string, seqAt: number, overrides: Partial = {}): PerfCall { return { callId, @@ -261,6 +253,8 @@ function perfCall(callId: string, seqAt: number, overrides: Partial = text: null, truncated: false, ms: null, + startedAt: null, + endedAt: null, ...overrides, } } @@ -269,42 +263,6 @@ function callsRecord(calls: readonly PerfCall[]): Record { return Object.fromEntries(calls.map(call => [call.callId, call])) } -function stepNode(value: PerfStepMetrics): PerfViewNode { - return { - key: `${value.turn}:${value.step}`, - kind: 'perf-step', - id: `${value.turn}:${value.step}`, - target: 'perf', - anchorSeq: value.startTime ?? 0, - location: { kind: 'unresolved' }, - data: { kind: 'step', metrics: value }, - } -} - -function promptNode(turn: number, prompt: string | null, images = 0): PerfViewNode { - return { - key: `${turn}:prompt`, - kind: 'perf-prompt', - id: `${turn}:prompt`, - target: 'perf', - anchorSeq: 1, - location: { kind: 'unresolved' }, - data: { kind: 'prompt', turn, prompt, images }, - } -} - -function endNode(turn: number, endTime: number | null, closeKind: string | null): PerfViewNode { - return { - key: `${turn}:end`, - kind: 'perf-turn-end', - id: `${turn}:end`, - target: 'perf', - anchorSeq: 2, - location: { kind: 'unresolved' }, - data: { kind: 'end', turn, closeKind, endTime }, - } -} - /** Lines joined by newlines, for edit and write fixtures. */ function linesOf(count: number, prefix = 'line'): string { return Array.from({ length: count }, (_, index) => `${prefix}${index}`).join('\n') @@ -318,8 +276,6 @@ const usage: TokenUsage = { } beforeEach(() => { - resetPtcSteps() - resetCurrentTurn() seq = 0 clock = 0 }) @@ -442,213 +398,6 @@ describe('token accounting helpers', () => { }) }) -describe('prompt attribution', () => { - it('marks the turn-open state as the prompt row', () => { - expect(currentTurnOf()).toBeNull() - const turn = defined(foldEvent(undefined, event('turn/start', { turn: 3 }, 1_000))) - expect(currentTurnOf()).toBe(3) - expect(turn.turn).toBe(3) - expect(turn.step).toBe(0) - expect(turn.mark).toBe('prompt') - resetCurrentTurn() - expect(currentTurnOf()).toBeNull() - }) - - it('attributes a user question to the current turn by arrival order', () => { - const afterTurn = defined(foldEvent(undefined, event('turn/start', { turn: 3 }, 1_000))) - const state = defined(foldEvent(afterTurn, event('user/message', userMessage([ - textBlock('explain this'), textBlock('failure'), - ]), 1_010))) - expect(state.turn).toBe(3) - expect(state.prompt).toBe('explain this failure') - expect(state.images).toBe(0) - expect(state.mark).toBe('prompt') - }) - - it('falls back to turn zero when a question arrives before any turn marker', () => { - const state = defined(foldEvent(undefined, event('user/message', userMessage([ - textBlock('orphan question'), - ]), 1_000))) - expect(state.turn).toBe(0) - expect(state.step).toBe(0) - expect(state.prompt).toBe('orphan question') - }) - - it('keeps the first question of a turn', () => { - const first = defined(foldEvent(undefined, event('user/message', userMessage([ - textBlock('first'), - ], { kind: 'user' }), 1_000))) - const second = defined(foldEvent(first, event('user/message', userMessage([ - textBlock('second'), - ], { kind: 'user' }), 1_010))) - expect(second.prompt).toBe('first') - }) - - it('ignores injected messages but still marks the row as a prompt row', () => { - const state = defined(foldEvent(undefined, event('user/message', userMessage([ - textBlock(''), - ], { kind: 'model', provider: 'deepseek', model: 'deepseek-chat' }), 1_000))) - expect(state.mark).toBe('prompt') - expect(state.prompt).toBeNull() - }) - - it('tolerates a source or content the event map does not describe', () => { - const noSource = defined(foldEvent(undefined, unchecked('user/message', { - role: 'user', content: [textBlock('hidden')], source: null, - }, 1_000))) - expect(noSource.prompt).toBeNull() - const primitiveSource = defined(foldEvent(undefined, unchecked('user/message', { - role: 'user', content: [textBlock('hidden')], source: 'user', - }, 1_010))) - expect(primitiveSource.prompt).toBeNull() - const looseContent = defined(foldEvent(undefined, unchecked('user/message', { - role: 'user', content: 'not a list', source: { kind: 'user' }, - }, 1_020))) - expect(looseContent.prompt).toBeNull() - }) - - it('keeps only text blocks and counts images among the blocks', () => { - const state = defined(foldEvent(undefined, unchecked('user/message', { - role: 'user', - content: [null, 7, { type: 'text', text: 9 }, { type: 'text', text: 'kept' }, { type: 'image' }], - source: { kind: 'user' }, - }, 1_000))) - expect(state.prompt).toBe('kept') - expect(state.images).toBe(1) - }) - - it('clips long questions and drops blank ones', () => { - const long = defined(foldEvent(undefined, event('user/message', userMessage([ - textBlock('q'.repeat(500)), - ]), 1_000))) - expect(long.prompt).toHaveLength(400) - expect(long.prompt?.endsWith('…')).toBe(true) - const blank = defined(foldEvent(undefined, event('user/message', userMessage([ - textBlock(' '), - ]), 1_010))) - expect(blank.prompt).toBeNull() - }) - - it('keeps an image-only question with no text', () => { - const state = defined(foldEvent(undefined, unchecked('user/message', { - role: 'user', content: [{ type: 'image' }, { type: 'image' }], source: { kind: 'user' }, - }, 1_000))) - expect(state.prompt).toBeNull() - expect(state.images).toBe(2) - }) -}) - -describe('assistant timeline and usage', () => { - it('stamps the first visible live token and ignores empty or non-token chunks', () => { - const start = defined(foldEvent(undefined, stepStart(2, 1, 1_000))) - const empty = defined(foldEvent(start, liveChunk(2, 1, { type: 'text-delta', index: 0, text: '' }, 1_010))) - expect(empty.firstTokenTime).toBeNull() - const spaced = defined(foldEvent(empty, liveChunk(2, 1, { type: 'text-delta', index: 0, text: 'x' }, 1_020))) - expect(spaced.firstTokenTime).toBe(1_020) - const reasoning = defined(foldEvent(start, liveChunk(2, 1, { type: 'reasoning-delta', index: 0, text: 'why' }, 1_030))) - expect(reasoning.firstTokenTime).toBe(1_030) - const silentReasoning = defined(foldEvent(start, liveChunk(2, 1, { - type: 'reasoning-delta', index: 0, text: '', - }, 1_040))) - expect(silentReasoning.firstTokenTime).toBeNull() - const late = defined(foldEvent(spaced, liveChunk(2, 1, { type: 'text-delta', index: 0, text: 'y' }, 1_050))) - expect(late.firstTokenTime).toBe(1_020) - }) - - it('treats tool-call deltas as tokens only when they carry content', () => { - const start = defined(foldEvent(undefined, stepStart(2, 1, 1_000))) - const empty = defined(foldEvent(start, liveChunk(2, 1, { - type: 'tool-call-delta', index: 0, id: ToolCallId('c'), argumentsDelta: '', - }, 1_010))) - expect(empty.firstTokenTime).toBeNull() - const args = defined(foldEvent(start, liveChunk(2, 1, { - type: 'tool-call-delta', index: 0, id: ToolCallId('c'), argumentsDelta: '{"a":1}', - }, 1_020))) - expect(args.firstTokenTime).toBe(1_020) - const named = defined(foldEvent(start, liveChunk(2, 1, { - type: 'tool-call-delta', index: 0, id: ToolCallId('c'), name: 'read', argumentsDelta: '', - }, 1_030))) - expect(named.firstTokenTime).toBe(1_030) - const blockStart = defined(foldEvent(start, liveChunk(2, 1, { - type: 'block-start', index: 0, blockType: 'text', - }, 1_040))) - expect(blockStart.firstTokenTime).toBeNull() - }) - - it('adopts usage from a live chunk only when a count was reported', () => { - const start = defined(foldEvent(undefined, stepStart(2, 1, 1_000))) - const adopted = defined(foldEvent(start, liveChunk(2, 1, { type: 'usage', usage }, 1_010))) - expect(adopted.usage).toEqual(usage) - const rejected = defined(foldEvent(start, liveChunk(2, 1, { - type: 'usage', usage: { inputTokens: Number.NaN, outputTokens: -1 }, - }, 1_020))) - expect(rejected.usage).toBeNull() - const kept = defined(foldEvent(adopted, liveChunk(2, 1, { - type: 'usage', usage: { inputTokens: Number.NaN, outputTokens: -1 }, - }, 1_030))) - expect(kept.usage).toEqual(usage) - }) - - it('starts a live window without a step boundary', () => { - const state = defined(foldEvent(undefined, liveChunk(4, 2, { type: 'text-delta', index: 0, text: 'x' }, 2_000))) - expect(state.turn).toBe(4) - expect(state.step).toBe(2) - expect(state.firstTokenTime).toBe(2_000) - expect(state.startTime).toBeNull() - }) - - it('derives the first token from a durable attempt stream', () => { - const attempt = defined(foldEvent(undefined, event('assistant/attempt', { - turn: 2, step: 1, stream: streamAt(1_300), - }, 1_500))) - expect(attempt.firstTokenTime).toBe(1_300) - expect(attempt.messageTime).toBeNull() - const empty = defined(foldEvent(undefined, event('assistant/attempt', { - turn: 2, step: 1, stream: [], - }, 1_500))) - expect(empty.firstTokenTime).toBeNull() - const kept = defined(foldEvent(attempt, event('assistant/attempt', { - turn: 2, step: 1, stream: streamAt(1_400), - }, 1_600))) - expect(kept.firstTokenTime).toBe(1_300) - }) - - it('records the settled message time, usage, and interruption', () => { - const state = defined(foldEvent(undefined, event('assistant/message', { - turn: 2, - step: 1, - message: assistantMessage([textBlock('done')]), - stream: streamAt(1_300), - usage, - interrupted: true, - }, 1_500))) - expect(state.firstTokenTime).toBe(1_300) - expect(state.messageTime).toBe(1_500) - expect(state.usage).toEqual(usage) - expect(state.interrupted).toBe(true) - const repeated = defined(foldEvent(state, event('assistant/message', { - turn: 2, - step: 1, - message: assistantMessage([textBlock('again')]), - stream: [], - usage: { inputTokens: Number.NaN, outputTokens: -1 }, - }, 1_900))) - expect(repeated.messageTime).toBe(1_500) - expect(repeated.usage).toEqual(usage) - expect(repeated.interrupted).toBe(false) - }) - - it('leaves usage unset when the first message reports none', () => { - const state = defined(foldEvent(undefined, event('assistant/message', { - turn: 2, step: 1, message: assistantMessage(), stream: [], - }, 1_500))) - expect(state.usage).toBeNull() - expect(state.firstTokenTime).toBeNull() - expect(state.messageTime).toBe(1_500) - expect(state.interrupted).toBe(false) - }) -}) - describe('tool call hints', () => { it('hints at the target for every tool family', () => { const cases: readonly { name: string; args: string; hint: string | null }[] = [ @@ -763,6 +512,9 @@ describe('tool call arguments and text bounds', () => { expect(empty.removed).toBe(0) expect(empty.before).toBeNull() expect(empty.after).toBeNull() + const unnamed = listedCall(callMetrics('edit', JSON.stringify({ file_path: '/a/b' }))) + expect(unnamed.added).toBeNull() + expect(unnamed.removed).toBeNull() }) it('counts write content lines and bounds the kept body', () => { @@ -808,6 +560,138 @@ describe('tool call arguments and text bounds', () => { }) }) +describe('assistant timeline and usage', () => { + it('stamps the first visible live token and ignores empty or non-token chunks', () => { + const start = defined(foldEvent(undefined, stepStart(2, 1, 1_000))) + const empty = defined(foldEvent(start, liveChunk(2, 1, { type: 'text-delta', index: 0, text: '' }, 1_010))) + expect(empty.firstTokenTime).toBeNull() + const spaced = defined(foldEvent(empty, liveChunk(2, 1, { type: 'text-delta', index: 0, text: 'x' }, 1_020))) + expect(spaced.firstTokenTime).toBe(1_020) + const reasoning = defined(foldEvent(start, liveChunk(2, 1, { type: 'reasoning-delta', index: 0, text: 'why' }, 1_030))) + expect(reasoning.firstTokenTime).toBe(1_030) + const silentReasoning = defined(foldEvent(start, liveChunk(2, 1, { + type: 'reasoning-delta', index: 0, text: '', + }, 1_040))) + expect(silentReasoning.firstTokenTime).toBeNull() + const late = defined(foldEvent(spaced, liveChunk(2, 1, { type: 'text-delta', index: 0, text: 'y' }, 1_050))) + expect(late.firstTokenTime).toBe(1_020) + }) + + it('treats tool-call deltas as tokens only when they carry content', () => { + const start = defined(foldEvent(undefined, stepStart(2, 1, 1_000))) + const empty = defined(foldEvent(start, liveChunk(2, 1, { + type: 'tool-call-delta', index: 0, id: ToolCallId('c'), argumentsDelta: '', + }, 1_010))) + expect(empty.firstTokenTime).toBeNull() + const args = defined(foldEvent(start, liveChunk(2, 1, { + type: 'tool-call-delta', index: 0, id: ToolCallId('c'), argumentsDelta: '{"a":1}', + }, 1_020))) + expect(args.firstTokenTime).toBe(1_020) + const named = defined(foldEvent(start, liveChunk(2, 1, { + type: 'tool-call-delta', index: 0, id: ToolCallId('c'), name: 'read', argumentsDelta: '', + }, 1_030))) + expect(named.firstTokenTime).toBe(1_030) + const blockStart = defined(foldEvent(start, liveChunk(2, 1, { + type: 'block-start', index: 0, blockType: 'text', + }, 1_040))) + expect(blockStart.firstTokenTime).toBeNull() + }) + + it('adopts the usage a live chunk carries', () => { + const start = defined(foldEvent(undefined, stepStart(2, 1, 1_000))) + const adopted = defined(foldEvent(start, liveChunk(2, 1, { type: 'usage', usage }, 1_010))) + expect(adopted.usage).toEqual(usage) + const replaced = defined(foldEvent(adopted, liveChunk(2, 1, { + type: 'usage', usage: { inputTokens: 5, outputTokens: 6 }, + }, 1_020))) + expect(replaced.usage).toEqual({ inputTokens: 5, outputTokens: 6 }) + }) + + it('starts a live window without a step boundary', () => { + const state = defined(foldEvent(undefined, liveChunk(4, 2, { type: 'text-delta', index: 0, text: 'x' }, 2_000))) + expect(state.turn).toBe(4) + expect(state.step).toBe(2) + expect(state.firstTokenTime).toBe(2_000) + expect(state.startTime).toBeNull() + }) + + it('derives the first token from a durable attempt stream', () => { + const attempt = defined(foldEvent(undefined, event('assistant/attempt', { + turn: 2, step: 1, stream: streamAt(1_300), + }, 1_500))) + expect(attempt.firstTokenTime).toBe(1_300) + expect(attempt.messageTime).toBeNull() + const empty = defined(foldEvent(undefined, event('assistant/attempt', { + turn: 2, step: 1, stream: [], + }, 1_500))) + expect(empty.firstTokenTime).toBeNull() + const kept = defined(foldEvent(attempt, event('assistant/attempt', { + turn: 2, step: 1, stream: streamAt(1_400), + }, 1_600))) + expect(kept.firstTokenTime).toBe(1_300) + }) + + it('records the settled message time, usage, and interruption', () => { + const state = defined(foldEvent(undefined, event('assistant/message', { + turn: 2, + step: 1, + message: assistantMessage([textBlock('done')]), + stream: streamAt(1_300), + usage, + interrupted: true, + }, 1_500))) + expect(state.firstTokenTime).toBe(1_300) + expect(state.messageTime).toBe(1_500) + expect(state.usage).toEqual(usage) + expect(state.interrupted).toBe(true) + const repeated = defined(foldEvent(state, event('assistant/message', { + turn: 2, + step: 1, + message: assistantMessage([textBlock('again')]), + stream: [], + }, 1_900))) + expect(repeated.messageTime).toBe(1_500) + expect(repeated.usage).toEqual(usage) + expect(repeated.interrupted).toBe(false) + const refreshed = defined(foldEvent(repeated, event('assistant/message', { + turn: 2, + step: 1, + message: assistantMessage(), + stream: [], + usage: { inputTokens: 5, outputTokens: 6 }, + }, 2_100))) + expect(refreshed.usage).toEqual({ inputTokens: 5, outputTokens: 6 }) + }) + + it('leaves usage unset when the first message reports none', () => { + const state = defined(foldEvent(undefined, event('assistant/message', { + turn: 2, step: 1, message: assistantMessage(), stream: [], + }, 1_500))) + expect(state.usage).toBeNull() + expect(state.firstTokenTime).toBeNull() + expect(state.messageTime).toBe(1_500) + expect(state.interrupted).toBe(false) + }) +}) + +describe('prompt helper', () => { + it('joins typed text blocks and counts images', () => { + const prompt = promptOf(userMessage([textBlock('explain'), textBlock('this'), imageBlock()])) + expect(prompt.text).toBe('explain this') + expect(prompt.images).toBe(1) + const imageOnly = promptOf(userMessage([imageBlock(), imageBlock()])) + expect(imageOnly.text).toBeNull() + expect(imageOnly.images).toBe(2) + }) + + it('clips long questions and drops blank text', () => { + expect(promptOf(userMessage([textBlock('q'.repeat(500))])).text).toHaveLength(400) + expect(promptOf(userMessage([textBlock('q'.repeat(500))])).text?.endsWith('…')).toBe(true) + expect(promptOf(userMessage([textBlock(' ')])).text).toBeNull() + expect(promptOf(userMessage([textBlock(' a\n b ')])).text).toBe('a b') + }) +}) + describe('tool call folding', () => { it('records a call before any step boundary and tallies its tool', () => { const state = defined(foldEvent(undefined, toolCall(5, 2, 'c1', 'grep', '{"pattern":"x"}', 1_000))) @@ -827,19 +711,18 @@ describe('tool call folding', () => { expect(foldedState.pending).toEqual({ c1: 1_020, c2: 1_030 }) expect(foldedState.tools).toEqual({ read: 2, grep: 1 }) expect(recordedCall(foldedState, 'c1').name).toBe('read') + expect(recordedCall(foldedState, 'c1').seq).toBe(2) expect(stepMetrics(foldedState).tools).toEqual([{ name: 'read', count: 2 }, { name: 'grep', count: 1 }]) }) - it('keeps a nameless call without inventing a name or hint', () => { + it('marks the wall-clock start of a call and settles its end', () => { let state = foldEvent(undefined, stepStart(1, 1, 1_000)) - state = foldEvent(state, unchecked('tool/call', { - turn: 1, step: 1, callId: 'anonymous', arguments: '{"file_path":"/a/b"}', - }, 1_010)) - const foldedState = defined(state) - expect(foldedState.tools).toBeNull() - expect(foldedState.toolOpen).toBe(1) - expect(recordedCall(foldedState, 'anonymous').name).toBeNull() - expect(recordedCall(foldedState, 'anonymous').hint).toBeNull() + state = foldEvent(state, toolCall(1, 1, 'c1', 'read', '{}', 1_010)) + expect(recordedCall(defined(state), 'c1').startedAt).toBe(1_010) + expect(recordedCall(defined(state), 'c1').endedAt).toBeNull() + state = foldEvent(state, toolResult(1, 1, 'c1', [textBlock('x')], 1_210)) + expect(recordedCall(defined(state), 'c1').endedAt).toBe(1_210) + expect(stepMetrics(defined(state)).toolWall).toBe(200) }) }) @@ -861,7 +744,9 @@ describe('tool result pairing', () => { expect(call.text).toBe('one\ntwo') expect(call.truncated).toBe(false) expect(call.error).toBe(false) - expect(call.ms).toBeNull() + // The paired intervals give the per-call duration the accordion renders. + expect(call.ms).toBe(200) + expect(call.endedAt).toBe(1_210) }) it('joins text blocks and counts lines without a trailing empty line', () => { @@ -919,6 +804,17 @@ describe('tool result pairing', () => { expect(call.truncated).toBe(false) }) + it('ignores non-text content blocks in a tool result', () => { + const state = folded([ + stepStart(1, 1, 1_000), + toolCall(1, 1, 'c1', 'read', '{}', 1_010), + toolResult(1, 1, 'c1', [imageBlock(), textBlock('kept')], 1_110), + ]) + const call = recordedCall(state, 'c1') + expect(call.text).toBe('kept') + expect(call.chars).toBe(4) + }) + it('records a failed result and counts one tool error', () => { const state = folded([ stepStart(1, 1, 1_000), @@ -949,7 +845,7 @@ describe('tool result pairing', () => { expect(metrics.toolPending).toBe(2) }) - it('never resolves an inherited property as a pending call', () => { + it('never resolves an inherited property as a pending or recorded call', () => { const empty = folded([ stepStart(1, 1, 1_000), toolResult(1, 1, 'constructor', [textBlock('x')], 1_110), @@ -963,22 +859,7 @@ describe('tool result pairing', () => { ]) expect(populated.toolMissing).toBe(1) expect(populated.toolPairs).toBe(0) - }) - - it('tolerates a result message without a call id', () => { - const shapes: readonly object[] = [ - { turn: 1, step: 1, message: null }, - { turn: 1, step: 1, message: 'tool' }, - { turn: 1, step: 1, message: { role: 'tool', content: [] } }, - { turn: 1, step: 1, message: { role: 'tool', content: [], source: null } }, - { turn: 1, step: 1, message: { role: 'tool', content: [], source: { callId: 7 } } }, - { turn: 1, step: 1, message: { role: 'tool', content: 'text', source: { callId: 'c1' } } }, - ] - for (const data of shapes) { - let state = foldEvent(undefined, stepStart(1, 1, 1_000)) - state = foldEvent(state, unchecked('tool/result', data, 1_100)) - expect(defined(state).toolMissing, JSON.stringify(data)).toBe(1) - } + expect(recordedCall(populated, 'known').chars).toBeNull() }) it('folds a result that arrives without an open step', () => { @@ -988,53 +869,6 @@ describe('tool result pairing', () => { expect(state.toolMissing).toBe(1) }) - it('treats a result without a content list as empty', () => { - let state = foldEvent(undefined, stepStart(1, 1, 1_000)) - state = foldEvent(state, toolCall(1, 1, 'c1', 'read', '{}', 1_010)) - state = foldEvent(state, unchecked('tool/result', { - turn: 1, - step: 1, - message: { role: 'tool', content: 'text', source: { callId: 'c1' } }, - }, 1_100)) - const call = recordedCall(defined(state), 'c1') - expect(call.chars).toBe(0) - expect(call.lines).toBe(0) - expect(call.text).toBeNull() - }) - - it('skips content entries and blocks without usable text', () => { - const state = folded([ - stepStart(1, 1, 1_000), - toolCall(1, 1, 'c1', 'read', '{}', 1_010), - unchecked('tool/result', { - turn: 1, - step: 1, - message: { - role: 'tool', - content: [null, 'block', { type: 'text', text: 9 }, { type: 'text', text: 'kept' }], - source: { callId: 'c1' }, - }, - }, 1_110), - ]) - const call = recordedCall(state, 'c1') - expect(call.text).toBe('kept') - expect(call.chars).toBe(4) - }) - - it('treats a truthy non-boolean error flag as success', () => { - const state = folded([ - stepStart(1, 1, 1_000), - toolCall(1, 1, 'c1', 'read', '{}', 1_010), - unchecked('tool/result', { - turn: 1, - step: 1, - message: { role: 'tool', content: [textBlock('x')], source: { callId: 'c1' }, isError: 'yes' }, - }, 1_110), - ]) - expect(recordedCall(state, 'c1').error).toBe(false) - expect(stepMetrics(state).toolErrors).toBe(0) - }) - it('resolves pending calls one at a time and clamps a backwards duration', () => { let state = foldEvent(undefined, stepStart(1, 1, 1_000)) state = foldEvent(state, toolCall(1, 1, 'c1', 'read', '{}', 2_000)) @@ -1061,16 +895,6 @@ describe('Code-Mode dispatch', () => { ]) } - it('tracks the step that owns a root call', () => { - const state = codeStep() - expect(state.tools).toEqual({ run_code: 1 }) - expect(ptcStepOf('root')).toBe('2:1') - expect(ptcStepOf('missing')).toBeNull() - expect(ptcStepOf(42)).toBeNull() - resetPtcSteps() - expect(ptcStepOf('root')).toBeNull() - }) - it('records an inner call with its dispatch duration and result', () => { let state = codeStep() state = defined(foldEvent(state, ptcStart('root', 'sub-1', 'grep', '{"pattern":"x"}', 5_020))) @@ -1080,12 +904,29 @@ describe('Code-Mode dispatch', () => { expect(inner.name).toBe('grep') expect(inner.hint).toBe('x') expect(inner.ms).toBe(15) + expect(inner.startedAt).toBe(5_020) + expect(inner.endedAt).toBe(5_035) expect(inner.chars).toBe(3) expect(inner.text).toBe('hit') expect(inner.error).toBe(false) expect(defined(state).toolErrors).toBe(0) }) + it('keeps an inner call out of the outer tool time', () => { + const state = folded([ + stepStart(2, 1, 5_000), + toolCall(2, 1, 'root', 'run_code', '{}', 5_010), + ptcStart('root', 'sub-1', 'grep', '{}', 5_020), + ptcDispatch('root', 'sub-1', 'grep', '{}', [textBlock('hit')], 5_035), + ]) + const metrics = stepMetrics(state) + expect(state.toolPairs).toBe(0) + expect(metrics.tool).toBeNull() + expect(metrics.toolWall).toBe(15) + expect(metrics.calls.map(call => call.callId)).toEqual(['root', 'sub-1']) + expect(metrics.tools).toEqual([{ name: 'run_code', count: 1 }, { name: 'grep', count: 1 }]) + }) + it('records a dispatch inside a step with no root call', () => { const state = folded([ stepStart(2, 1, 5_000), @@ -1098,14 +939,19 @@ describe('Code-Mode dispatch', () => { it('leaves the duration unmeasured without a dispatch start', () => { let state = codeStep() state = defined(foldEvent(state, ptcDispatch('root', 'sub-1', 'read', '{}', [textBlock('x')], 5_035))) - expect(recordedCall(defined(state), 'sub-1').ms).toBeNull() + const inner = recordedCall(defined(state), 'sub-1') + expect(inner.ms).toBeNull() + expect(inner.startedAt).toBeNull() + expect(inner.endedAt).toBe(5_035) }) it('clamps a dispatch that settles before its start', () => { let state = codeStep() state = defined(foldEvent(state, ptcStart('root', 'sub-1', 'read', '{}', 5_050))) state = defined(foldEvent(state, ptcDispatch('root', 'sub-1', 'read', '{}', [textBlock('x')], 5_010))) - expect(recordedCall(defined(state), 'sub-1').ms).toBe(0) + const inner = recordedCall(defined(state), 'sub-1') + expect(inner.ms).toBe(0) + expect(inner.startedAt).toBe(5_050) }) it('counts a failed inner call as a tool error', () => { @@ -1145,8 +991,8 @@ describe('Code-Mode dispatch', () => { expect(inner.args).toBe('{"file_path":"/a/b.ts","content":"one\\ntwo"}') }) - it('accepts array, null, and absent arguments without a hint', () => { - const cases: readonly unknown[] = [[1, 2], null, undefined] + it('accepts array, null, absent, and scalar arguments without a hint', () => { + const cases: readonly unknown[] = [[1, 2], null, undefined, 42] for (const [index, args] of cases.entries()) { const sub = `sub-${index}` const state = folded([ @@ -1156,7 +1002,7 @@ describe('Code-Mode dispatch', () => { ]) const inner = recordedCall(state, sub) expect(inner.hint, String(index)).toBeNull() - expect(inner.args).toBe(args === undefined || args === null ? null : '[1,2]') + expect(inner.args, String(index)).toBe(args === undefined || args === null ? null : JSON.stringify(args)) expect(inner.added).toBeNull() } }) @@ -1174,35 +1020,19 @@ describe('Code-Mode dispatch', () => { expect(inner.hint).toBe('nested/file.ts') }) - it('ignores a dispatch with no tool name and one without a context', () => { - const state = codeStep() - const unnamed = foldEvent(state, unchecked('tool/ptc-dispatch', { - rootCallId: 'root', parentCallId: 'root', subCallId: 'sub-1', isError: false, content: [], - }, 5_040)) - expect(unnamed).toBe(state) - expect(foldEvent(undefined, ptcDispatch('root', 'sub-1', 'read', {}, [], 5_040))).toBeUndefined() + it('needs a folded step context for dispatch events', () => { expect(foldEvent(undefined, ptcStart('root', 'sub-1', 'read', {}, 5_040))).toBeUndefined() + expect(foldEvent(undefined, ptcDispatch('root', 'sub-1', 'read', {}, [], 5_040))).toBeUndefined() }) - it('remembers only code-mode root calls', () => { - let state = foldEvent(undefined, stepStart(3, 1, 6_000)) - for (const name of ['run_code', 'RunCode', 'runCode', 'run-code', 'grep']) { - state = foldEvent(state, toolCall(3, 1, `call-${name}`, name, '{}', 6_010)) - } - expect(ptcStepOf('call-run_code')).toBe('3:1') - expect(ptcStepOf('call-RunCode')).toBe('3:1') - expect(ptcStepOf('call-runCode')).toBe('3:1') - expect(ptcStepOf('call-run-code')).toBeNull() - expect(ptcStepOf('call-grep')).toBeNull() - }) - - it('evicts the oldest root call past the table cap', () => { - // Each call folds into a fresh state: only the root-call table grows here. - for (let index = 0; index <= 4_000; index += 1) { - foldEvent(newState(7, 1), toolCall(7, 1, `root-${index}`, 'run_code', '{}', 8_010 + index)) - } - expect(ptcStepOf('root-0')).toBeNull() - expect(ptcStepOf('root-4000')).toBe('7:1') + it('lists several inner calls in sequence order', () => { + const state = folded([ + stepStart(2, 1, 5_000), + toolCall(2, 1, 'root', 'run_code', '{}', 5_010), + ptcDispatch('root', 'sub-1', 'grep', '{}', [], 5_030), + ptcDispatch('root', 'sub-2', 'read', '{}', [], 5_040), + ]) + expect(stepMetrics(state).calls.map(call => call.callId)).toEqual(['root', 'sub-1', 'sub-2']) }) }) @@ -1229,21 +1059,25 @@ describe('step boundaries and the fold default', () => { it('counts retries from a retry event with or without an open step', () => { const fromStep = defined(foldEvent(undefined, stepStart(3, 1, 1_000))) - const once = defined(foldEvent(fromStep, unchecked('llm/retry', { turn: 3, step: 1 }, 1_010))) + const once = defined(foldEvent(fromStep, retry(3, 1, 1_010))) expect(once.retries).toBe(1) - const twice = defined(foldEvent(once, unchecked('llm/retry', { turn: 3, step: 1 }, 1_020))) + const twice = defined(foldEvent(once, retry(3, 1, 1_020))) expect(twice.retries).toBe(2) - const standalone = defined(foldEvent(undefined, unchecked('llm/retry', { turn: 3, step: 2 }, 1_030))) + const standalone = defined(foldEvent(undefined, retry(3, 2, 1_030))) expect(standalone.turn).toBe(3) expect(standalone.step).toBe(2) expect(standalone.retries).toBe(1) }) - it('leaves a state unchanged for events the table does not fold', () => { + it('leaves prompt folding and other markers to their definitions', () => { const state = defined(foldEvent(undefined, stepStart(1, 1, 1_000))) - const same = foldEvent(state, event('turn/end', { turn: 1, reason: { kind: 'completed' } }, 2_000)) - expect(same).toBe(state) - expect(foldEvent(undefined, event('turn/end', { turn: 1, reason: { kind: 'completed' } }, 2_000))) + const afterQuestion = foldEvent(state, event('user/message', userMessage([textBlock('question')]), 1_010)) + expect(afterQuestion).toBe(state) + const afterTurnStart = foldEvent(state, event('turn/start', { turn: 2 }, 1_020)) + expect(afterTurnStart).toBe(state) + const afterTurnEnd = foldEvent(state, event('turn/end', { turn: 1, reason: { kind: 'completed' } }, 1_030)) + expect(afterTurnEnd).toBe(state) + expect(foldEvent(undefined, event('user/message', userMessage([textBlock('orphan')]), 1_040))) .toBeUndefined() }) @@ -1263,6 +1097,7 @@ describe('step boundaries and the fold default', () => { const metrics = stepMetrics(state) expect(metrics.ttft).toBe(100) expect(metrics.tool).toBe(100) + expect(metrics.toolWall).toBe(100) expect(metrics.wall).toBe(400) expect(metrics.running).toBe(false) }) @@ -1282,6 +1117,7 @@ describe('step metric derivation', () => { expect(metrics.tps).toBeNull() expect(metrics.output).toBeNull() expect(metrics.tool).toBeNull() + expect(metrics.toolWall).toBeNull() expect(metrics.tools).toEqual([]) expect(metrics.calls).toEqual([]) expect(metrics.toolOutput).toBe(0) @@ -1290,6 +1126,9 @@ describe('step metric derivation', () => { expect(metrics.running).toBe(true) expect(metrics.aborted).toBe(false) expect(metrics.interrupted).toBe(false) + expect(metrics.cacheRead).toBeNull() + expect(metrics.cacheWrite).toBeNull() + expect(metrics.cacheHit).toBeNull() expect(metrics.mark).toBeNull() expect(metrics.prompt).toBeNull() expect(metrics.images).toBe(0) @@ -1338,6 +1177,13 @@ describe('step metric derivation', () => { expect(metrics.closeKind).toBe('interrupted') }) + it('reports a prompt marker through the same metric row', () => { + const metrics = stepMetrics(newState(1, 0, 'prompt')) + expect(metrics.turn).toBe(1) + expect(metrics.step).toBe(0) + expect(metrics.mark).toBe('prompt') + }) + it('requires a start for the first-token and model spans', () => { const noStart = stepMetrics(stateOf({ firstTokenTime: 1_100, messageTime: 1_500 })) expect(noStart.ttft).toBeNull() @@ -1404,6 +1250,13 @@ describe('step metric derivation', () => { expect(running.running).toBe(true) }) + it('reports tool time only when at least one call paired', () => { + const paired = stepMetrics(stateOf({ toolPairs: 1, toolMs: 200 })) + expect(paired.tool).toBe(200) + const unpaired = stepMetrics(stateOf({ toolPairs: 0, toolMs: 200 })) + expect(unpaired.tool).toBeNull() + }) + it('lists calls in call order and caps the list', () => { const calls = Array.from({ length: CALL_LIMIT + 2 }, (_, index) => perfCall( `call-${index}`, @@ -1446,7 +1299,7 @@ describe('step metric derivation', () => { expect(tied.tools).toEqual([{ name: 'read', count: 1 }, { name: 'grep', count: 1 }]) }) - it('hides the Code-Mode runner while keeping its inner and plain calls', () => { + it('keeps the Code-Mode runner until the builder merges inner dispatches', () => { const calls = callsRecord([ perfCall('run', 1, { name: 'run_code', chars: 1_000 }), perfCall('inner', 2, { name: 'read', chars: 5, inner: true }), @@ -1456,9 +1309,13 @@ describe('step metric derivation', () => { calls, tools: { run_code: 1, read: 1, write: 1 }, })) - expect(metrics.calls.map(call => call.callId)).toEqual(['inner', 'plain']) - expect(metrics.tools).toEqual([{ name: 'read', count: 1 }, { name: 'write', count: 1 }]) - expect(metrics.toolOutput).toBe(25) + expect(metrics.calls.map(call => call.callId)).toEqual(['run', 'inner', 'plain']) + expect(metrics.tools).toEqual([ + { name: 'run_code', count: 1 }, + { name: 'read', count: 1 }, + { name: 'write', count: 1 }, + ]) + expect(metrics.toolOutput).toBe(1_025) }) it('counts every result when no inner call is present', () => { @@ -1470,289 +1327,62 @@ describe('step metric derivation', () => { expect(metrics.toolOutput).toBe(12) expect(metrics.tools).toHaveLength(2) }) -}) -describe('turn aggregation', () => { - it('reports an empty turn without invented values', () => { - const turn = aggregateTurn(9, []) - expect(turn.turn).toBe(9) - expect(turn.steps).toEqual([]) - expect(turn.startTime).toBeNull() - expect(turn.context).toBeNull() - expect(turn.ttft).toBeNull() - expect(turn.ttftPending).toBe(0) - expect(turn.model).toBeNull() - expect(turn.modelPending).toBe(0) - expect(turn.decode).toBeNull() - expect(turn.tps).toBeNull() - expect(turn.output).toBeNull() - expect(turn.tool).toBeNull() - expect(turn.toolPending).toBe(0) - expect(turn.tools).toEqual([]) - expect(turn.cacheHit).toBeNull() - expect(turn.retries).toBe(0) - expect(turn.toolErrors).toBe(0) - expect(turn.toolOutput).toBe(0) - expect(turn.lastStepWall).toBeNull() - expect(turn.running).toBe(false) - expect(turn.interrupted).toBe(false) - expect(turn.aborted).toBe(false) - expect(turn.wall).toBeNull() - }) - - it('sums measured spans and reports partial coverage as a lower bound', () => { - const measured = metricsOf({ startTime: 1_000, ttft: 100, model: 400, decode: 300, output: 40, tool: 200 }) - const unknown = metricsOf({ startTime: 2_000, toolPending: 2 }) - const turn = aggregateTurn(3, [measured, unknown]) - expect(turn.ttft).toBe(100) - expect(turn.ttftPending).toBe(1) - expect(turn.model).toBe(400) - expect(turn.modelPending).toBe(1) - expect(turn.decode).toBe(300) - expect(turn.output).toBe(40) - expect(turn.tool).toBe(200) - expect(turn.toolPending).toBe(2) - expect(turn.startTime).toBe(1_000) - expect(turn.running).toBe(false) - }) - - it('reduces an entirely unmeasured turn to blanks', () => { - const turn = aggregateTurn(3, [metricsOf(), metricsOf({ step: 2 })]) - expect(turn.ttft).toBeNull() - expect(turn.ttftPending).toBe(0) - expect(turn.model).toBeNull() - expect(turn.modelPending).toBe(2) - expect(turn.decode).toBeNull() - expect(turn.output).toBeNull() - expect(turn.tool).toBeNull() - expect(turn.toolPending).toBe(0) - expect(turn.tps).toBeNull() - expect(turn.wall).toBeNull() - }) - - it('derives turn speed only when decode and output are both positive', () => { - const fast = aggregateTurn(1, [ - metricsOf({ step: 1, decode: 2_000, output: 50 }), - metricsOf({ step: 2, decode: 2_000, output: 50 }), + it('measures tool wall time as the union of overlapping call spans', () => { + const calls = callsRecord([ + perfCall('c1', 1, { startedAt: 1_000, endedAt: 1_200 }), + perfCall('c2', 2, { startedAt: 1_100, endedAt: 1_300 }), ]) - expect(fast.tps).toBeCloseTo(100 / 4) - const noDecode = aggregateTurn(1, [metricsOf({ decode: 0, output: 50 })]) - expect(noDecode.decode).toBe(0) - expect(noDecode.tps).toBeNull() - const noOutput = aggregateTurn(1, [metricsOf({ decode: 2_000, output: 0 })]) - expect(noOutput.tps).toBeNull() - }) - - it('keeps the last measured context and cache level of the turn', () => { - const turn = aggregateTurn(4, [ - metricsOf({ step: 1, context: 100, cacheHit: 0.2, wall: 10 }), - metricsOf({ step: 2, context: 200, cacheHit: 0.5, wall: 20 }), - metricsOf({ step: 3 }), - ]) - expect(turn.context).toBe(200) - expect(turn.cacheHit).toBe(0.5) - expect(turn.lastStepWall).toBe(20) - const heldBack = aggregateTurn(4, [ - metricsOf({ step: 1, context: 100, cacheHit: 0.2, wall: 10 }), - metricsOf({ step: 2, cacheHit: null, wall: null }), - ]) - expect(heldBack.context).toBe(100) - expect(heldBack.cacheHit).toBe(0.2) - expect(heldBack.lastStepWall).toBe(10) - }) - - it('sums retries, tool errors, and tool output', () => { - const turn = aggregateTurn(5, [ - metricsOf({ step: 1, retries: 1, toolErrors: 2, toolOutput: 10 }), - metricsOf({ step: 2, retries: 2, toolErrors: 3, toolOutput: 5 }), - ]) - expect(turn.retries).toBe(3) - expect(turn.toolErrors).toBe(5) - expect(turn.toolOutput).toBe(15) - const unrecorded = aggregateTurn(5, [ - metricsOf({ step: 1, toolOutput: null }), - metricsOf({ step: 2, toolOutput: 4 }), - ]) - expect(unrecorded.toolOutput).toBe(4) - }) - - it('merges tool tallies across steps', () => { - const turn = aggregateTurn(6, [ - metricsOf({ step: 1, tools: [{ name: 'read', count: 1 }] }), - metricsOf({ step: 2, tools: [{ name: 'read', count: 2 }, { name: 'grep', count: 1 }] }), - ]) - expect(turn.tools).toEqual([{ name: 'read', count: 3 }, { name: 'grep', count: 1 }]) - const tied = aggregateTurn(6, [ - metricsOf({ step: 1, tools: [{ name: 'read', count: 1 }, { name: 'grep', count: 1 }] }), - ]) - expect(tied.tools).toEqual([{ name: 'read', count: 1 }, { name: 'grep', count: 1 }]) - }) - - it('measures turn wall time only from a settled last step with a start', () => { - const first = metricsOf({ step: 1, startTime: 1_000 }) - const last = metricsOf({ step: 2, startTime: 2_000, endTime: 3_000 }) - expect(aggregateTurn(1, [first, last]).wall).toBe(2_000) - expect(aggregateTurn(1, [first, { ...last, running: true }]).wall).toBeNull() - expect(aggregateTurn(1, [{ ...first, startTime: null }, last]).wall).toBeNull() - expect(aggregateTurn(1, [first, { ...last, endTime: null }]).wall).toBeNull() - expect(aggregateTurn(1, [first, { ...last, endTime: 500 }]).wall).toBe(0) - }) - - it('reports last-step liveness and any interruption in the turn', () => { - const running = aggregateTurn(1, [ - metricsOf({ step: 1, running: false }), - metricsOf({ step: 2, running: true }), - ]) - expect(running.running).toBe(true) - const interrupted = aggregateTurn(2, [ - metricsOf({ step: 1, interrupted: true }), - metricsOf({ step: 2, aborted: true }), - ]) - expect(interrupted.interrupted).toBe(true) - expect(interrupted.aborted).toBe(true) - }) - - it('describes the turn prompt as absent until the table attributes it', () => { - const turn = aggregateTurn(2, [metricsOf()]) - expect(turn.prompt).toBeNull() - expect(turn.promptImages).toBe(0) - expect(turn.steps).toHaveLength(1) + const metrics = stepMetrics(stateOf({ calls, toolPairs: 2, toolMs: 400 })) + expect(metrics.tool).toBe(400) + expect(metrics.toolWall).toBe(300) }) }) -describe('turn grouping', () => { - it('groups steps into turns ordered oldest first with their markers', () => { - const one = metricsOf({ turn: 8, step: 2, startTime: 20_000 }) - const two = metricsOf({ turn: 8, step: 1, startTime: 19_000 }) - const later = metricsOf({ turn: 9, step: 1, startTime: 30_000 }) - const turns = buildTurns([ - stepNode(later), - stepNode(one), - stepNode(two), - promptNode(8, '问题', 2), - endNode(9, 31_000, 'interrupted'), - ]) - expect(turns.map(turn => turn.turn)).toEqual([8, 9]) - expect(turns[0]?.steps.map(step => step.step)).toEqual([1, 2]) - expect(turns[0]?.prompt).toBe('问题') - expect(turns[0]?.promptImages).toBe(2) - expect(turns[1]?.interrupted).toBe(true) +describe('tool span union', () => { + it('reports no span when no call settled with both ends', () => { + expect(toolSpanUnion([])).toBeNull() + expect(toolSpanUnion([ + perfCall('a', 1, { startedAt: 1_000 }), + perfCall('b', 2, { endedAt: 2_000 }), + ])).toBeNull() }) - it('returns no turn for markers without a step', () => { - expect(buildTurns([])).toEqual([]) - expect(buildTurns([promptNode(5, 'orphan'), endNode(5, 1_000, 'interrupted')])).toEqual([]) + it('adds disjoint spans and counts overlaps once', () => { + expect(toolSpanUnion([ + perfCall('a', 1, { startedAt: 1_000, endedAt: 1_200 }), + perfCall('b', 2, { startedAt: 1_500, endedAt: 1_700 }), + ])).toBe(400) + expect(toolSpanUnion([ + perfCall('a', 1, { startedAt: 1_000, endedAt: 1_500 }), + perfCall('b', 2, { startedAt: 1_200, endedAt: 1_300 }), + ])).toBe(500) + expect(toolSpanUnion([ + perfCall('a', 1, { startedAt: 1_000, endedAt: 1_200 }), + perfCall('b', 2, { startedAt: 1_100, endedAt: 1_400 }), + ])).toBe(400) }) - it('ignores prompt markers with no text and end markers with no detail', () => { - const step = metricsOf({ turn: 1, step: 1, startTime: 1_000, running: true }) - const turns = buildTurns([ - stepNode(step), - promptNode(1, null), - promptNode(1, ''), - endNode(1, Number.NaN, null), - ]) - expect(turns).toHaveLength(1) - expect(turns[0]?.prompt).toBeNull() - expect(turns[0]?.promptImages).toBe(0) - expect(turns[0]?.running).toBe(true) - }) - - it('skips a step whose turn number is not a number', () => { - const turns = buildTurns([ - stepNode(metricsOf({ turn: Number.NaN, step: 1 })), - stepNode(metricsOf({ turn: 2, step: 1 })), - ]) - expect(turns.map(turn => turn.turn)).toEqual([2]) - }) - - it('clears running before a later step without mutating the input', () => { - const first = metricsOf({ turn: 1, step: 1, running: true }) - const second = metricsOf({ turn: 1, step: 2, running: true }) - const third = metricsOf({ turn: 1, step: 3, running: true }) - const turns = buildTurns([stepNode(first), stepNode(second), stepNode(third)]) - expect(turns[0]?.steps.map(step => step.running)).toEqual([false, false, true]) - expect(first.running).toBe(true) - expect(turns[0]?.running).toBe(true) - }) - - it('keeps a running turn when a later turn follows', () => { - const turns = buildTurns([ - stepNode(metricsOf({ turn: 1, step: 1, running: true })), - stepNode(metricsOf({ turn: 2, step: 1, running: true })), - ]) - expect(turns[0]?.running).toBe(false) - expect(turns[1]?.running).toBe(true) - }) - - it('drops the empty step an interruption appends', () => { - const one = metricsOf({ turn: 11, step: 1, startTime: 50_000, context: 300 }) - const empty = metricsOf({ turn: 11, step: 2, wall: 0 }) - const turns = buildTurns([ - stepNode(one), - stepNode(empty), - endNode(11, 51_000, 'interrupted'), - ]) - expect(turns[0]?.steps).toHaveLength(1) - expect(turns[0]?.steps[0]?.wall).toBe(1_000) - expect(turns[0]?.steps[0]?.aborted).toBe(true) - }) - - it('keeps a tail step that still measured something', () => { - const variants: readonly PerfStepMetrics[] = [ - metricsOf({ turn: 12, step: 2, context: 5 }), - metricsOf({ turn: 12, step: 2, ttft: 5 }), - metricsOf({ turn: 12, step: 2, model: 5 }), - metricsOf({ turn: 12, step: 2, tool: 0 }), - metricsOf({ turn: 12, step: 2, toolPending: 1 }), - metricsOf({ turn: 12, step: 2, tools: [{ name: 'read', count: 1 }] }), - metricsOf({ turn: 12, step: 2, wall: 5 }), - ] - for (const tail of variants) { - const turns = buildTurns([ - stepNode(metricsOf({ turn: 12, step: 1, startTime: 1_000, context: 10 })), - stepNode(tail), - endNode(12, 2_000, 'interrupted'), - ]) - expect(turns[0]?.steps, JSON.stringify(tail)).toHaveLength(2) - } - }) - - it('closes an interrupted tail at the turn end time', () => { - const open = metricsOf({ turn: 13, step: 1, startTime: 1_000, running: true }) - const turns = buildTurns([stepNode(open), endNode(13, 4_000, 'interrupted')]) - const closed = turns[0]?.steps[0] - expect(closed?.endTime).toBe(4_000) - expect(closed?.wall).toBe(3_000) - expect(closed?.running).toBe(false) - expect(closed?.aborted).toBe(true) - expect(turns[0]?.wall).toBe(3_000) - expect(turns[0]?.running).toBe(false) - }) - - it('closes a tail with no start time without inventing a duration', () => { - const open = metricsOf({ turn: 14, step: 1, running: true }) - const turns = buildTurns([stepNode(open), endNode(14, 4_000, null)]) - expect(turns[0]?.steps[0]?.endTime).toBe(4_000) - expect(turns[0]?.steps[0]?.wall).toBeNull() - expect(turns[0]?.interrupted).toBe(false) - }) - - it('leaves a tail that already ended untouched', () => { - const settled = metricsOf({ turn: 15, step: 1, startTime: 1_000, endTime: 1_500 }) - const turns = buildTurns([stepNode(settled), endNode(15, 4_000, 'completed')]) - expect(turns[0]?.steps[0]?.endTime).toBe(1_500) - expect(turns[0]?.steps[0]?.wall).toBeNull() - expect(turns[0]?.wall).toBe(500) - }) - - it('prefers the interruption close kind over the step flags', () => { - const done = metricsOf({ turn: 16, step: 1, interrupted: true }) - const interrupted = buildTurns([stepNode(done), endNode(16, 1_000, 'interrupted')]) - expect(interrupted[0]?.interrupted).toBe(true) - const interruptedStep = buildTurns([stepNode(done), endNode(16, 1_000, 'completed')]) - expect(interruptedStep[0]?.interrupted).toBe(true) - const quiet = buildTurns([stepNode(metricsOf({ turn: 17, step: 1 }))]) - expect(quiet[0]?.interrupted).toBe(false) + it('sorts input spans and joins touching ones', () => { + expect(toolSpanUnion([ + perfCall('late', 1, { startedAt: 2_000, endedAt: 2_100 }), + perfCall('early', 2, { startedAt: 1_000, endedAt: 1_100 }), + ])).toBe(200) + expect(toolSpanUnion([ + perfCall('first', 1, { startedAt: 1_000, endedAt: 1_200 }), + perfCall('second', 2, { startedAt: 1_200, endedAt: 1_500 }), + ])).toBe(500) + }) +}) + +describe('code-mode name helper', () => { + it('recognizes the runner spellings and nothing else', () => { + expect(isCodeMode('run_code')).toBe(true) + expect(isCodeMode('RunCode')).toBe(true) + expect(isCodeMode('runCode')).toBe(true) + expect(isCodeMode('RUN_CODE')).toBe(true) + expect(isCodeMode('run-code')).toBe(false) + expect(isCodeMode('grep')).toBe(false) + expect(isCodeMode(null)).toBe(false) }) }) diff --git a/packages/experimental/client-ui-perf/tests/perf-metrics.client.spec.ts b/packages/experimental/client-ui-perf/tests/perf-metrics.client.spec.ts index d7bd5f6830..ffc6ab9092 100644 --- a/packages/experimental/client-ui-perf/tests/perf-metrics.client.spec.ts +++ b/packages/experimental/client-ui-perf/tests/perf-metrics.client.spec.ts @@ -1,381 +1,527 @@ import { describe, expect, it } from 'vitest' -import type { SessionEventLike } from '@deepseek-ai/dsh-api-session-controller/client' -import type { AssistantStreamRecord, StreamChunk } from '@deepseek-ai/dsh-llm' -import { SessionSeq } from '@deepseek-ai/dsh-session/types' -import type { SessionEventType } from '@deepseek-ai/dsh-session/types' import { aggregateTurn, buildTurns } from '../src/client/perf-aggregate.ts' -import { - currentTurnOf, foldEvent, newState, resetCurrentTurn, resetPtcSteps, stepMetrics, -} from '../src/client/perf-fold.ts' -import { - DASH, barShare, formatChars, formatClock, formatDateTime, formatMs, formatPercent, - formatTokens, formatTokensDelta, formatTokensShort, formatTps, -} from '../src/client/perf-format.ts' -import type { PerfStepMetrics, PerfStepState, PerfViewNode } from '../src/client/perf-contract.ts' -import { - cacheHitRate, contextTokens, isCount, isUsage, outputTokens, -} from '../src/client/perf-usage.ts' +import type { PerfCall, PerfStepMetrics, PerfViewNode } from '../src/client/perf-contract.ts' +import { CALL_LIMIT } from '../src/client/perf-fold.ts' -let seq = 0 - -/** One durable session event with a deterministic sequence and time. */ -function event(type: SessionEventType, data: object, time?: number): SessionEventLike { - const at = time ?? 1_000 + (seq += 1) * 10 - return { type, seq: SessionSeq(seq), time: at, data } as SessionEventLike -} - -/** Narrow a folded state the fixture guarantees exists. */ -function defined(value: T | undefined): T { - if (value === undefined) throw new Error('expected a folded state') - return value -} - -/** One client-only live chunk event. */ -function chunk(turn: number, step: number, value: StreamChunk, time?: number): SessionEventLike { - seq += 1 +/** One step row with every unmeasured column blank. */ +function metricsOf(overrides: Partial = {}): PerfStepMetrics { return { - type: 'assistant/live-chunk', - seq, - time: time ?? 1_000 + seq * 10, - data: { attemptId: 'attempt-1', turn, step, chunk: value }, - } as SessionEventLike + turn: 1, + step: 1, + startTime: null, + endTime: null, + context: null, + ttft: null, + model: null, + decode: null, + tps: null, + output: null, + tool: null, + toolWall: null, + tools: [], + calls: [], + toolOutput: 0, + retries: 0, + toolErrors: 0, + cacheRead: null, + cacheWrite: null, + cacheHit: null, + wall: null, + toolPending: 0, + running: false, + interrupted: false, + aborted: false, + ...overrides, + } } -/** A token delta chunk. */ -const delta: StreamChunk = { type: 'text-delta', index: 0, text: 'x' } - -/** Usage reported by one assistant message. */ -const usage = { - inputTokens: 100, - outputTokens: 40, - cacheReadTokens: 400, - cacheWriteTokens: 100, +function perfCall(callId: string, seqAt: number, overrides: Partial = {}): PerfCall { + return { + callId, + seq: seqAt, + name: 'read', + hint: null, + chars: null, + lines: null, + error: false, + added: null, + removed: null, + before: null, + after: null, + args: null, + inner: false, + text: null, + truncated: false, + ms: null, + startedAt: null, + endedAt: null, + ...overrides, + } } -/** A compact assistant stream recording the first token at `time`. */ -function stream(firstTokenAt: number): AssistantStreamRecord[] { - return [ - { type: 'chunk', time: firstTokenAt - 100, chunk: { type: 'block-start', index: 0, blockType: 'text' } }, - { type: 'chunk', time: firstTokenAt, chunk: delta }, - ] as AssistantStreamRecord[] +function stepNode(value: PerfStepMetrics): PerfViewNode { + return { + key: `${value.turn}:${value.step}`, + kind: 'perf-step', + id: `${value.turn}:${value.step}`, + target: 'perf', + anchorSeq: value.startTime ?? 0, + location: { kind: 'unresolved' }, + data: { kind: 'step', metrics: value }, + } } -/** Build a step state across a start, a stream, a message, one tool call, and an end. */ -function completedStep(turn: number, step: number, startTime = 1_000): ReturnType { - resetPtcSteps() - resetCurrentTurn() - let state = foldEvent(undefined, event('step/start', { turn, step }, startTime)) - state = foldEvent(state, chunk(turn, step, delta, startTime + 100)) - state = foldEvent(state, event('assistant/message', { - turn, - step, - message: { id: 'm', role: 'assistant', content: [], source: { provider: 'p', model: 'm' } }, - stream: stream(startTime + 300), - usage, - }, startTime + 400)) - state = foldEvent(state, event('tool/call', { - turn, step, callId: 'call-1', name: 'read', arguments: JSON.stringify({ file_path: '/a/b/c.ts' }), - }, startTime + 500)) - state = foldEvent(state, event('tool/result', { - turn, - step, - message: { role: 'tool', content: [{ type: 'text', text: 'one\ntwo' }], source: { callId: 'call-1' } }, - }, startTime + 700)) - state = foldEvent(state, event('step/end', { turn, step }, startTime + 800)) - return stepMetrics(defined(state)) +function promptNode(turn: number, prompt: string | null, images = 0): PerfViewNode { + return { + key: `${turn}:prompt:${prompt ?? 'null'}`, + kind: 'perf-prompt', + id: `${turn}:prompt`, + target: 'perf', + anchorSeq: 1, + location: { kind: 'unresolved' }, + data: { kind: 'prompt', turn, prompt, images }, + } } -describe('format helpers', () => { - it('formats durations at each scale boundary', () => { - expect(formatMs(null)).toBe(DASH) - expect(formatMs(999)).toBe('999 ms') - expect(formatMs(1_000)).toBe('1.00 s') - expect(formatMs(59_999)).toBe('60.00 s') - expect(formatMs(60_000)).toBe('1.0 min') - }) +function endNode(turn: number, endTime: number | null, closeKind: string | null): PerfViewNode { + return { + key: `${turn}:end`, + kind: 'perf-turn-end', + id: `${turn}:end`, + target: 'perf', + anchorSeq: 2, + location: { kind: 'unresolved' }, + data: { kind: 'end', turn, closeKind, endTime }, + } +} - it('keeps two decimals below one thousand tokens', () => { - expect(formatTokensShort(null)).toBe(DASH) - expect(formatTokensShort(30)).toBe('0.03K') - expect(formatTokensShort(1_500)).toBe('1.5K') - expect(formatTokensDelta(250)).toBe('+0.25K') - expect(formatTokensDelta(-730)).toBe('−0.73K') - expect(formatTokensDelta(null)).toBe('') - expect(formatTokens(1_234_567)).toBe('1,234,567') - }) - - it('formats characters, speed, and percent', () => { - expect(formatChars(null)).toBe(DASH) - expect(formatChars(0)).toBe(DASH) - expect(formatChars(12)).toBe('12') - expect(formatChars(1_234)).toBe('1.2K') - expect(formatTps(268.94)).toBe('268.9') - expect(formatTps(null)).toBe(DASH) - expect(formatPercent(0.962)).toBe('96.20%') - expect(formatPercent(null)).toBe(DASH) - }) - - it('formats the clock from the same instant it is given', () => { - const time = new Date(2026, 9, 20, 3, 4, 5).getTime() - expect(formatClock(time)).toBe('03:04:05') - expect(formatDateTime(time)).toBe('2026-10-20 03:04:05') - expect(formatClock(null)).toBe(DASH) - expect(formatDateTime(null)).toBe(DASH) - }) - - it('scales bars by square root of the group maximum', () => { - expect(barShare(10, 0)).toBe(1) - expect(barShare(0, 100)).toBe(0) - expect(barShare(100, 100)).toBe(1) - expect(barShare(25, 100)).toBe(0.5) - expect(barShare(200, 100)).toBe(1) - }) -}) - -describe('usage helpers', () => { - it('sums the prompt length and derives cache hit', () => { - expect(isUsage(null)).toBe(false) - expect(isUsage(usage)).toBe(true) - expect(isCount(-1)).toBe(false) - expect(contextTokens(usage)).toBe(600) - expect(contextTokens(null)).toBeNull() - expect(outputTokens(usage)).toBe(40) - expect(cacheHitRate(usage)).toBeCloseTo(400 / 600) - expect(cacheHitRate({})).toBeNull() - expect(cacheHitRate({ inputTokens: 0 })).toBeNull() - }) -}) - -describe('step folding and metrics', () => { - it('derives timeline, throughput, tool cost, and cache columns', () => { - const metrics = completedStep(1, 1) - expect(metrics.startTime).toBe(1_000) - expect(metrics.endTime).toBe(1_800) - expect(metrics.ttft).toBe(100) - expect(metrics.model).toBe(400) - expect(metrics.decode).toBe(300) - expect(metrics.tps).toBeCloseTo(40 / 0.3) - expect(metrics.tool).toBe(200) - expect(metrics.toolPending).toBe(0) - expect(metrics.wall).toBe(800) - expect(metrics.context).toBe(600) - expect(metrics.output).toBe(40) - expect(metrics.cacheHit).toBeCloseTo(400 / 600) - expect(metrics.running).toBe(false) - expect(metrics.aborted).toBe(false) - expect(metrics.tools).toEqual([{ name: 'read', count: 1 }]) - expect(metrics.toolOutput).toBe(7) - expect(metrics.calls[0]?.hint).toBe('b/c.ts') - }) - - it('keeps a running step measurable up to the last event', () => { - resetPtcSteps() - const start = foldEvent(undefined, event('step/start', { turn: 2, step: 1 }, 5_000)) - const metrics = stepMetrics(defined(start)) - expect(metrics.running).toBe(true) - expect(metrics.ttft).toBeNull() - expect(metrics.wall).toBeNull() - expect(metrics.context).toBeNull() - }) - - it('reports an unsettled step as aborted', () => { - resetPtcSteps() - let state = foldEvent(undefined, event('step/start', { turn: 2, step: 1 }, 5_000)) - state = foldEvent(state, chunk(2, 1, delta, 5_100)) - state = foldEvent(state, event('step/end', { turn: 2, step: 1 }, 5_200)) - expect(stepMetrics(defined(state)).aborted).toBe(true) - }) - - it('counts retries, tool errors, and unpaired results', () => { - resetPtcSteps() - let state: PerfStepState | undefined = newState(3, 1) - state = foldEvent(state, event('llm/retry', { - turn: 3, step: 1, failure: { message: 'rate limited' }, retry: 1, delayMs: 10, mode: 'normal', maxRetries: 3, - })) - state = foldEvent(state, event('tool/result', { - turn: 3, step: 1, message: { role: 'tool', content: [], isError: true, source: { callId: 'ghost' } }, - })) - const metrics = stepMetrics(defined(state)) - expect(metrics.retries).toBe(1) - expect(metrics.toolErrors).toBe(1) - expect(metrics.toolPending).toBe(1) - }) - - it('groups Code-Mode inner calls and hides the runner', () => { - resetPtcSteps() - let state = foldEvent(undefined, event('step/start', { turn: 4, step: 1 }, 7_000)) - state = foldEvent(state, event('tool/call', { - turn: 4, step: 1, callId: 'run-1', name: 'run_code', arguments: '{}', - }, 7_010)) - state = foldEvent(state, event('tool/ptc-dispatch-start', { - rootCallId: 'run-1', parentCallId: 'run-1', subCallId: 'sub-1', name: 'grep', arguments: '{"pattern":"x"}', - }, 7_020)) - state = foldEvent(state, event('tool/ptc-dispatch', { - rootCallId: 'run-1', parentCallId: 'run-1', subCallId: 'sub-1', name: 'grep', - arguments: '{"pattern":"x"}', isError: false, content: [{ type: 'text', text: 'hit' }], - }, 7_030)) - const metrics = stepMetrics(defined(state)) - expect(metrics.tools).toEqual([{ name: 'grep', count: 1 }]) - expect(metrics.calls).toHaveLength(1) - expect(metrics.calls[0]?.inner).toBe(true) - expect(metrics.calls[0]?.ms).toBe(10) - expect(metrics.toolOutput).toBe(3) - }) - - it('attributes inner calls through the root call map', () => { - resetPtcSteps() - let state = foldEvent(undefined, event('step/start', { turn: 5, step: 2 }, 7_000)) - state = foldEvent(state, event('tool/call', { turn: 5, step: 2, callId: 'root', name: 'run_code', arguments: '{}' })) - expect(currentTurnOf()).toBeNull() - state = foldEvent(state, event('tool/ptc-dispatch', { - rootCallId: 'root', parentCallId: 'root', subCallId: 's', name: 'read', - arguments: '{"file_path":"/x/y.js"}', isError: true, content: [], - })) - const metrics = stepMetrics(defined(state)) - expect(metrics.turn).toBe(5) - expect(metrics.step).toBe(2) - expect(metrics.toolErrors).toBe(1) - }) - - it('records the user question only from user-sourced messages', () => { - resetPtcSteps() - resetCurrentTurn() - let state = foldEvent(undefined, event('turn/start', { turn: 6 })) - expect(currentTurnOf()).toBe(6) - state = foldEvent(state, event('user/message', { - role: 'user', - id: 'u1', - content: [{ type: 'text', text: '帮我看看' }, { type: 'image' }], - source: { kind: 'user' }, - })) - expect(state!.prompt).toBe('帮我看看') - expect(state!.images).toBe(1) - // A second question never overwrites the first. - state = foldEvent(state, event('user/message', { - role: 'user', id: 'u2', content: [{ type: 'text', text: '再问一次' }], source: { kind: 'user' }, - })) - expect(state!.prompt).toBe('帮我看看') - // Injected context is not a question. - state = foldEvent(state, event('user/message', { - role: 'user', id: 'u3', content: [{ type: 'text', text: '' }], source: { kind: 'runtime-context' }, - })) - expect(state!.prompt).toBe('帮我看看') - }) -}) +function innerNode( + turn: number, + step: number, + calls: readonly PerfCall[], +): PerfViewNode { + return { + key: `${turn}:${step}:code:${calls.map(call => call.callId).join(',')}`, + kind: 'perf-code-mode', + id: `${turn}:${step}:code`, + target: 'perf', + anchorSeq: 3, + location: { kind: 'unresolved' }, + data: { kind: 'inner', turn, step, calls }, + } +} describe('turn aggregation', () => { - it('sums first-token waits and reports partial coverage', () => { - const one = completedStep(7, 1, 10_000) - const two = completedStep(7, 2, 11_000) - const partial: PerfStepMetrics = { ...one, ttft: null } - const turn = aggregateTurn(7, [one, two]) - expect(turn.ttft).toBe(200) + it('reports an empty turn without invented values', () => { + const turn = aggregateTurn(9, []) + expect(turn.turn).toBe(9) + expect(turn.steps).toEqual([]) + expect(turn.startTime).toBeNull() + expect(turn.context).toBeNull() + expect(turn.ttft).toBeNull() expect(turn.ttftPending).toBe(0) - expect(turn.model).toBe(800) - expect(turn.tool).toBe(400) - expect(turn.wall).toBe(two.endTime! - one.startTime!) - expect(turn.context).toBe(600) - const lower = aggregateTurn(7, [one, partial]) - expect(lower.ttft).toBe(100) - expect(lower.ttftPending).toBe(1) - expect(aggregateTurn(7, [{ ...one, ttft: null }]).ttft).toBeNull() + expect(turn.model).toBeNull() + expect(turn.modelPending).toBe(0) + expect(turn.decode).toBeNull() + expect(turn.tps).toBeNull() + expect(turn.output).toBeNull() + expect(turn.tool).toBeNull() + expect(turn.toolWall).toBeNull() + expect(turn.toolPending).toBe(0) + expect(turn.tools).toEqual([]) + expect(turn.cacheHit).toBeNull() + expect(turn.cacheRead).toBeNull() + expect(turn.cacheWrite).toBeNull() + expect(turn.retries).toBe(0) + expect(turn.toolErrors).toBe(0) + expect(turn.toolOutput).toBe(0) + expect(turn.lastStepWall).toBeNull() + expect(turn.running).toBe(false) + expect(turn.interrupted).toBe(false) + expect(turn.aborted).toBe(false) + expect(turn.wall).toBeNull() + expect(turn.prompt).toBeNull() + expect(turn.promptImages).toBe(0) + }) + + it('sums measured spans and reports partial coverage as a lower bound', () => { + const measured = metricsOf({ startTime: 1_000, ttft: 100, model: 400, decode: 300, output: 40, tool: 200 }) + const unknown = metricsOf({ startTime: 2_000, toolPending: 2 }) + const turn = aggregateTurn(3, [measured, unknown]) + expect(turn.ttft).toBe(100) + expect(turn.ttftPending).toBe(1) + expect(turn.model).toBe(400) + expect(turn.modelPending).toBe(1) + expect(turn.decode).toBe(300) + expect(turn.output).toBe(40) + expect(turn.tool).toBe(200) + expect(turn.toolPending).toBe(2) + expect(turn.startTime).toBe(1_000) + expect(turn.running).toBe(false) + }) + + it('reduces an entirely unmeasured turn to blanks', () => { + const turn = aggregateTurn(3, [metricsOf(), metricsOf({ step: 2 })]) + expect(turn.ttft).toBeNull() + expect(turn.ttftPending).toBe(0) + expect(turn.model).toBeNull() + expect(turn.modelPending).toBe(2) + expect(turn.decode).toBeNull() + expect(turn.output).toBeNull() + expect(turn.tool).toBeNull() + expect(turn.toolPending).toBe(0) + expect(turn.tps).toBeNull() + expect(turn.wall).toBeNull() + }) + + it('derives turn speed only when decode and output are both positive', () => { + const fast = aggregateTurn(1, [ + metricsOf({ step: 1, decode: 2_000, output: 50 }), + metricsOf({ step: 2, decode: 2_000, output: 50 }), + ]) + expect(fast.tps).toBeCloseTo(100 / 4) + const noDecode = aggregateTurn(1, [metricsOf({ decode: 0, output: 50 })]) + expect(noDecode.decode).toBe(0) + expect(noDecode.tps).toBeNull() + const noOutput = aggregateTurn(1, [metricsOf({ decode: 2_000, output: 0 })]) + expect(noOutput.tps).toBeNull() + }) + + it('keeps the last measured context and cache level of the turn', () => { + const turn = aggregateTurn(4, [ + metricsOf({ step: 1, context: 100, cacheHit: 0.2, cacheRead: 10, cacheWrite: 5, wall: 10 }), + metricsOf({ step: 2, context: 200, cacheHit: 0.5, cacheRead: 20, cacheWrite: 6, wall: 20 }), + metricsOf({ step: 3, context: 300, wall: 30 }), + ]) + expect(turn.context).toBe(300) + expect(turn.cacheHit).toBe(0.5) + expect(turn.cacheRead).toBe(20) + expect(turn.cacheWrite).toBe(6) + expect(turn.lastStepWall).toBe(30) + const heldBack = aggregateTurn(4, [ + metricsOf({ step: 1, context: 100, cacheHit: 0.2, cacheRead: 10, cacheWrite: 5, wall: 10 }), + metricsOf({ step: 2, cacheHit: null, cacheRead: 99, cacheWrite: 99, wall: null }), + ]) + expect(heldBack.context).toBe(100) + expect(heldBack.cacheHit).toBe(0.2) + expect(heldBack.cacheRead).toBe(10) + expect(heldBack.cacheWrite).toBe(5) + expect(heldBack.lastStepWall).toBe(10) + const bare = aggregateTurn(4, [metricsOf({ cacheHit: 0.4 })]) + expect(bare.cacheHit).toBe(0.4) + expect(bare.cacheRead).toBeNull() + expect(bare.cacheWrite).toBeNull() + }) + + it('sums the sequential step tool spans', () => { + const turn = aggregateTurn(5, [ + metricsOf({ step: 1, toolWall: 300, tool: 400 }), + metricsOf({ step: 2, toolWall: 200, tool: 250 }), + metricsOf({ step: 3 }), + ]) + expect(turn.toolWall).toBe(500) + expect(turn.tool).toBe(650) + expect(aggregateTurn(5, [metricsOf(), metricsOf({ step: 2 })]).toolWall).toBeNull() + }) + + it('never confuses the summed tool time with the wall union', () => { + const turn = aggregateTurn(6, [metricsOf({ step: 1, tool: 200, toolWall: 100 })]) + expect(turn.tool).toBe(200) + expect(turn.toolWall).toBe(100) + }) + + it('sums retries, tool errors, and tool output', () => { + const turn = aggregateTurn(5, [ + metricsOf({ step: 1, retries: 1, toolErrors: 2, toolOutput: 10 }), + metricsOf({ step: 2, retries: 2, toolErrors: 3, toolOutput: 5 }), + ]) + expect(turn.retries).toBe(3) + expect(turn.toolErrors).toBe(5) + expect(turn.toolOutput).toBe(15) + const unrecorded = aggregateTurn(5, [ + metricsOf({ step: 1, toolOutput: null }), + metricsOf({ step: 2, toolOutput: 4 }), + ]) + expect(unrecorded.toolOutput).toBe(4) + }) + + it('merges tool tallies across steps', () => { + const turn = aggregateTurn(6, [ + metricsOf({ step: 1, tools: [{ name: 'read', count: 1 }] }), + metricsOf({ step: 2, tools: [{ name: 'read', count: 2 }, { name: 'grep', count: 1 }] }), + ]) + expect(turn.tools).toEqual([{ name: 'read', count: 3 }, { name: 'grep', count: 1 }]) + const tied = aggregateTurn(6, [ + metricsOf({ step: 1, tools: [{ name: 'read', count: 1 }, { name: 'grep', count: 1 }] }), + ]) + expect(tied.tools).toEqual([{ name: 'read', count: 1 }, { name: 'grep', count: 1 }]) + }) + + it('measures turn wall time only from a settled last step with a start', () => { + const first = metricsOf({ step: 1, startTime: 1_000 }) + const last = metricsOf({ step: 2, startTime: 2_000, endTime: 3_000 }) + expect(aggregateTurn(1, [first, last]).wall).toBe(2_000) + expect(aggregateTurn(1, [first, { ...last, running: true }]).wall).toBeNull() + expect(aggregateTurn(1, [{ ...first, startTime: null }, last]).wall).toBeNull() + expect(aggregateTurn(1, [first, { ...last, endTime: null }]).wall).toBeNull() + expect(aggregateTurn(1, [first, { ...last, endTime: 500 }]).wall).toBe(0) + }) + + it('reports last-step liveness and any interruption in the turn', () => { + const running = aggregateTurn(1, [ + metricsOf({ step: 1, running: false }), + metricsOf({ step: 2, running: true }), + ]) + expect(running.running).toBe(true) + const interrupted = aggregateTurn(2, [ + metricsOf({ step: 1, interrupted: true }), + metricsOf({ step: 2, aborted: true }), + ]) + expect(interrupted.interrupted).toBe(true) + expect(interrupted.aborted).toBe(true) + }) + + it('describes the turn prompt as absent until the table attributes it', () => { + const turn = aggregateTurn(2, [metricsOf()]) + expect(turn.prompt).toBeNull() + expect(turn.promptImages).toBe(0) + expect(turn.steps).toHaveLength(1) }) }) describe('turn grouping', () => { - /** Wrap one step metrics value in the node envelope the builder receives. */ - function stepNode(metrics: PerfStepMetrics): PerfViewNode { - return { - key: `${metrics.turn}:${metrics.step}`, - kind: 'perf-step', - id: `${metrics.turn}:${metrics.step}`, - target: 'perf', - anchorSeq: metrics.startTime ?? 0, - location: { kind: 'unresolved' }, - data: { kind: 'step', metrics }, - } - } - - it('groups steps into ordered turns with the question and markers', () => { - const one = completedStep(8, 1, 20_000) - const two = completedStep(8, 2, 21_000) - const open: PerfStepMetrics = { - ...completedStep(9, 1, 30_000), - endTime: null, - wall: null, - running: true, - } - const nodes: PerfViewNode[] = [ - stepNode(two), + it('groups steps into turns ordered oldest first with their markers', () => { + const one = metricsOf({ turn: 8, step: 2, startTime: 20_000 }) + const two = metricsOf({ turn: 8, step: 1, startTime: 19_000 }) + const later = metricsOf({ turn: 9, step: 1, startTime: 30_000 }) + const turns = buildTurns([ + stepNode(later), stepNode(one), - stepNode(open), - { - key: '8:0', - kind: 'perf-prompt', - id: '8:0', - target: 'perf', - anchorSeq: 1, - location: { kind: 'unresolved' }, - data: { kind: 'prompt', turn: 8, prompt: '问题', images: 2 }, - }, - { - key: '9:100000', - kind: 'perf-turn-end', - id: '9:100000', - target: 'perf', - anchorSeq: 2, - location: { kind: 'unresolved' }, - data: { kind: 'end', turn: 9, closeKind: 'interrupted', endTime: 31_000 }, - }, - ] - const turns = buildTurns(nodes) + stepNode(two), + promptNode(8, '问题', 2), + endNode(9, 31_000, 'interrupted'), + ]) expect(turns.map(turn => turn.turn)).toEqual([8, 9]) expect(turns[0]?.steps.map(step => step.step)).toEqual([1, 2]) expect(turns[0]?.prompt).toBe('问题') expect(turns[0]?.promptImages).toBe(2) - // The interrupted turn closes its last step at the turn-end time. expect(turns[1]?.interrupted).toBe(true) - expect(turns[1]?.steps[0]?.endTime).toBe(31_000) - expect(turns[1]?.steps[0]?.wall).toBe(1_000) - expect(turns[1]?.steps[0]?.running).toBe(false) - expect(turns[1]?.steps[0]?.aborted).toBe(true) }) - it('never leaves a step running when a later step exists', () => { - const running = { ...completedStep(10, 1, 40_000), running: true } - const next = completedStep(10, 2, 41_000) - const turns = buildTurns([stepNode(running), stepNode(next)]) - expect(turns[0]?.steps[0]?.running).toBe(false) - expect(turns[0]?.steps[1]?.running).toBe(false) + it('returns no turn for markers without a step', () => { + expect(buildTurns([])).toEqual([]) + expect(buildTurns([promptNode(5, 'orphan'), endNode(5, 1_000, 'interrupted')])).toEqual([]) + }) + + it('ignores blank prompt markers and end markers without detail', () => { + const step = metricsOf({ turn: 1, step: 1, startTime: 1_000, running: true }) + const turns = buildTurns([ + stepNode(step), + promptNode(1, null), + promptNode(1, ''), + endNode(1, Number.NaN, null), + ]) + expect(turns).toHaveLength(1) + expect(turns[0]?.prompt).toBeNull() + expect(turns[0]?.promptImages).toBe(0) + expect(turns[0]?.running).toBe(true) + }) + + it('keeps the first non-blank prompt of a turn', () => { + const step = metricsOf({ turn: 1, step: 1 }) + const turns = buildTurns([ + stepNode(step), + promptNode(1, null), + promptNode(1, 'first', 3), + promptNode(1, 'second', 4), + ]) + expect(turns[0]?.prompt).toBe('first') + expect(turns[0]?.promptImages).toBe(3) + }) + + it('skips a step whose turn number is not a number', () => { + const turns = buildTurns([ + stepNode(metricsOf({ turn: Number.NaN, step: 1 })), + stepNode(metricsOf({ turn: 2, step: 1 })), + ]) + expect(turns.map(turn => turn.turn)).toEqual([2]) + }) + + it('clears running before a later step without mutating the input', () => { + const first = metricsOf({ turn: 1, step: 1, running: true }) + const second = metricsOf({ turn: 1, step: 2, running: true }) + const third = metricsOf({ turn: 1, step: 3, running: true }) + const turns = buildTurns([stepNode(first), stepNode(second), stepNode(third)]) + expect(turns[0]?.steps.map(step => step.running)).toEqual([false, false, true]) + expect(first.running).toBe(true) + expect(turns[0]?.running).toBe(true) + }) + + it('clears running when a later turn follows', () => { + const turns = buildTurns([ + stepNode(metricsOf({ turn: 1, step: 1, running: true })), + stepNode(metricsOf({ turn: 2, step: 1, running: true })), + ]) + expect(turns[0]?.running).toBe(false) + expect(turns[1]?.running).toBe(true) }) it('drops the empty step an interruption appends', () => { - const one = completedStep(11, 1, 50_000) - const empty = { - ...one, - step: 2, - context: null, - ttft: null, - model: null, - tool: null, - toolPending: 0, - tools: [], - wall: 0, + const one = metricsOf({ turn: 11, step: 1, startTime: 50_000, context: 300 }) + for (const wall of [0, null]) { + const empty = metricsOf({ turn: 11, step: 2, wall }) + const turns = buildTurns([ + stepNode(one), + stepNode(empty), + endNode(11, 51_000, 'interrupted'), + ]) + expect(turns[0]?.steps, String(wall)).toHaveLength(1) + expect(turns[0]?.steps[0]?.wall).toBe(1_000) + expect(turns[0]?.steps[0]?.aborted).toBe(true) } + }) + + it('keeps a tail step that still measured something', () => { + const variants: readonly PerfStepMetrics[] = [ + metricsOf({ turn: 12, step: 2, context: 5 }), + metricsOf({ turn: 12, step: 2, ttft: 5 }), + metricsOf({ turn: 12, step: 2, model: 5 }), + metricsOf({ turn: 12, step: 2, tool: 0 }), + metricsOf({ turn: 12, step: 2, toolPending: 1 }), + metricsOf({ turn: 12, step: 2, tools: [{ name: 'read', count: 1 }] }), + metricsOf({ turn: 12, step: 2, wall: 5 }), + ] + for (const tail of variants) { + const turns = buildTurns([ + stepNode(metricsOf({ turn: 12, step: 1, startTime: 1_000, context: 10 })), + stepNode(tail), + endNode(12, 2_000, 'interrupted'), + ]) + expect(turns[0]?.steps, JSON.stringify(tail)).toHaveLength(2) + } + }) + + it('closes an interrupted tail at the turn end time', () => { + const open = metricsOf({ turn: 13, step: 1, startTime: 1_000, running: true }) + const turns = buildTurns([stepNode(open), endNode(13, 4_000, 'interrupted')]) + const closed = turns[0]?.steps[0] + expect(closed?.endTime).toBe(4_000) + expect(closed?.wall).toBe(3_000) + expect(closed?.running).toBe(false) + expect(closed?.aborted).toBe(true) + expect(turns[0]?.wall).toBe(3_000) + expect(turns[0]?.running).toBe(false) + }) + + it('closes a tail with no start time without inventing a duration', () => { + const open = metricsOf({ turn: 14, step: 1, running: true }) + const turns = buildTurns([stepNode(open), endNode(14, 4_000, null)]) + expect(turns[0]?.steps[0]?.endTime).toBe(4_000) + expect(turns[0]?.steps[0]?.wall).toBeNull() + expect(turns[0]?.interrupted).toBe(false) + }) + + it('leaves a tail that already ended untouched', () => { + const settled = metricsOf({ turn: 15, step: 1, startTime: 1_000, endTime: 1_500 }) + const turns = buildTurns([stepNode(settled), endNode(15, 4_000, 'completed')]) + expect(turns[0]?.steps[0]?.endTime).toBe(1_500) + expect(turns[0]?.steps[0]?.wall).toBeNull() + expect(turns[0]?.wall).toBe(500) + }) + + it('prefers the interruption close kind over the step flags', () => { + const done = metricsOf({ turn: 16, step: 1, interrupted: true }) + const interrupted = buildTurns([stepNode(done), endNode(16, 1_000, 'interrupted')]) + expect(interrupted[0]?.interrupted).toBe(true) + const unrelated = buildTurns([stepNode(done), endNode(16, 1_000, 'completed')]) + expect(unrelated[0]?.interrupted).toBe(true) + const quiet = buildTurns([stepNode(metricsOf({ turn: 17, step: 1 }))]) + expect(quiet[0]?.interrupted).toBe(false) + }) +}) + +describe('Code-Mode contribution merging', () => { + it('replaces the runner with the real tools and recomputes the totals', () => { + const runner = perfCall('root', 1, { name: 'run_code', chars: 500 }) + const step = metricsOf({ + turn: 2, + step: 1, + calls: [runner], + tools: [{ name: 'run_code', count: 1 }], + toolOutput: 500, + toolWall: 400, + toolErrors: 1, + tool: 200, + }) + const inner = innerNode(2, 1, [ + perfCall('sub-1', 2, { + name: 'grep', chars: 20, inner: true, startedAt: 1_100, endedAt: 1_150, + }), + perfCall('sub-2', 3, { + name: 'read', chars: 5, inner: true, error: true, startedAt: 1_200, endedAt: 1_300, + }), + perfCall('sub-3', 4, { name: null, inner: true, chars: null }), + ]) + const turns = buildTurns([stepNode(step), inner]) + const merged = turns[0]?.steps[0] + expect(merged?.calls.map(call => call.callId)).toEqual(['sub-1', 'sub-2', 'sub-3']) + expect(merged?.tools).toEqual([{ name: 'grep', count: 1 }, { name: 'read', count: 1 }]) + expect(merged?.toolOutput).toBe(25) + expect(merged?.toolWall).toBe(150) + expect(merged?.toolErrors).toBe(2) + expect(merged?.tool).toBe(200) + expect(turns[0]?.tools).toEqual([{ name: 'grep', count: 1 }, { name: 'read', count: 1 }]) + expect(turns[0]?.toolErrors).toBe(2) + expect(turns[0]?.toolWall).toBe(150) + }) + + it('merges every root contribution of one step in sequence order', () => { + const step = metricsOf({ turn: 3, step: 1 }) const turns = buildTurns([ - stepNode(one), - stepNode(empty), - { - key: '11:100000', - kind: 'perf-turn-end', - id: '11:100000', - target: 'perf', - anchorSeq: 2, - location: { kind: 'unresolved' }, - data: { kind: 'end', turn: 11, closeKind: 'interrupted', endTime: 51_000 }, - }, + stepNode(step), + innerNode(3, 1, [perfCall('late', 5, { name: 'write', inner: true, chars: 2 })]), + innerNode(3, 1, [perfCall('early', 2, { name: 'grep', inner: true, chars: 3 })]), ]) + expect(turns[0]?.steps[0]?.calls.map(call => call.callId)).toEqual(['early', 'late']) + expect(turns[0]?.steps[0]?.toolOutput).toBe(5) + }) + + it('caps the merged call list at the table limit', () => { + const step = metricsOf({ turn: 4, step: 1 }) + const calls = Array.from({ length: CALL_LIMIT + 3 }, (_, index) => perfCall( + `inner-${index}`, + index + 1, + { inner: true, name: 'read', chars: index + 1 }, + )) + const turns = buildTurns([stepNode(step), innerNode(4, 1, calls)]) + expect(turns[0]?.steps[0]?.calls).toHaveLength(CALL_LIMIT) + expect(turns[0]?.steps[0]?.calls[0]?.callId).toBe('inner-0') + expect(turns[0]?.steps[0]?.calls.at(-1)?.callId).toBe(`inner-${CALL_LIMIT - 1}`) + expect(turns[0]?.steps[0]?.toolOutput).toBe( + Array.from({ length: CALL_LIMIT }, (_, index) => index + 1).reduce((sum, value) => sum + value, 0), + ) + }) + + it('keeps the runner when no inner contribution arrived', () => { + const runner = perfCall('root', 1, { name: 'run_code', chars: 500 }) + const turns = buildTurns([stepNode(metricsOf({ + turn: 5, + step: 1, + calls: [runner], + tools: [{ name: 'run_code', count: 1 }], + toolOutput: 500, + }))]) + expect(turns[0]?.steps[0]?.calls.map(call => call.callId)).toEqual(['root']) + expect(turns[0]?.tools).toEqual([{ name: 'run_code', count: 1 }]) + }) + + it('ignores inner contributions for steps the window no longer holds', () => { + const step = metricsOf({ turn: 6, step: 1, toolOutput: 7 }) + const turns = buildTurns([stepNode(step), innerNode(6, 2, [perfCall('sub', 2, { + name: 'grep', chars: 9, inner: true, + })])]) expect(turns[0]?.steps).toHaveLength(1) - expect(turns[0]?.steps[0]?.wall).not.toBeNull() + expect(turns[0]?.steps[0]?.calls).toEqual([]) + expect(turns[0]?.toolOutput).toBe(7) }) }) diff --git a/packages/experimental/client-ui-perf/tests/perf-plugin.client.spec.ts b/packages/experimental/client-ui-perf/tests/perf-plugin.client.spec.ts index e67f15733c..4125f05bee 100644 --- a/packages/experimental/client-ui-perf/tests/perf-plugin.client.spec.ts +++ b/packages/experimental/client-ui-perf/tests/perf-plugin.client.spec.ts @@ -22,7 +22,6 @@ import { en, NS, zh } from '../src/client/locales.ts' import type { PerfCall, PerfContribution, PerfSnapshot, PerfStepMetrics, PerfStepState, PerfViewNode, } from '../src/client/perf-contract.ts' -import { resetCurrentTurn, resetPtcSteps } from '../src/client/perf-fold.ts' import { TARGET } from '../src/client/perf-definition.ts' import { PerfView } from '../src/client/PerfView.tsx' import { @@ -115,6 +114,7 @@ interface PluginBench { readonly views: readonly ConversationViewDefinition[] readonly localeRegister: ReturnType readonly binding: ReturnType + readonly loadOlder: ReturnType readonly entries: () => readonly StoredEntry[] readonly setSnapshot: (next: PerfSnapshot | undefined) => void readonly notify: () => void @@ -139,6 +139,9 @@ async function mountPlugin(): Promise { }, }), })) + const loadOlder = vi.fn(async () => undefined) + const sessionBinding = { session: { loadOlder } } + ctx.provide('sessions', { binding: (id: SessionId) => (id === SID ? sessionBinding : undefined) } as never) ctx.provide('locale', { register: localeRegister, bind: () => makeTranslate(zh) } as never) ctx.provide('uiConversation', { binding, @@ -169,6 +172,7 @@ async function mountPlugin(): Promise { views, localeRegister, binding, + loadOlder, entries: () => ctx.slots.entries('conversation.view'), setSnapshot: (next) => { snapshot = next }, notify: () => { listener?.() }, @@ -196,8 +200,6 @@ function assemble( /** The Definitions captured by one plugin registration. */ async function registeredDefinitions(): Promise { - resetCurrentTurn() - resetPtcSteps() const bench = await mountPlugin() const definitions = bench.events await bench.fiber.dispose() @@ -223,6 +225,8 @@ function measuredStep(turn: number, step: number, startTime: number, endTime: nu text: 'one\ntwo', truncated: false, ms: 200, + startedAt: startTime + 50, + endedAt: endTime - 10, } return { turn, @@ -236,6 +240,7 @@ function measuredStep(turn: number, step: number, startTime: number, endTime: nu tps: 40 / 0.3, output: 40, tool: 200, + toolWall: 200, tools: [{ name: 'read', count: 1 }], calls: [call], toolOutput: 7, @@ -296,13 +301,47 @@ function contextOf( } } +/** Location data store the hand-built Locations never let a Definition read. */ +const LOCATION_DATA = { + get: () => undefined, + source: () => ({ getSnapshot: () => undefined, subscribe: () => () => undefined }), +} as never + +/** Fabricated Turn Location carrying only the number a Definition reads. */ +function turnAt(turn: number): ConversationLocation { + return { + kind: 'turn', + turn: { turn, start: undefined, end: undefined, status: 'open', steps: [], data: LOCATION_DATA }, + } +} + +/** Fabricated Step Location carrying only the numbers a Definition reads. */ +function stepAt(turn: number, step: number): ConversationLocation { + return { + kind: 'step', + turn: { turn, start: undefined, end: undefined, status: 'open', steps: [], data: LOCATION_DATA }, + step: { turn, step, start: undefined, end: undefined, status: 'unknown', data: LOCATION_DATA }, + } +} + +/** A start-role match placed at a fabricated Location. */ +function locatedStart(event: SessionEventLike, location: ConversationLocation): ConversationStartMatch { + return { event, role: 'start', location } +} + +/** An update-role match placed at a fabricated Location. */ +function locatedUpdate(event: SessionEventLike, location: ConversationLocation): ConversationMatch { + return { event, role: 'update', location } +} + describe('browser plugin registration', () => { it('registers the performance tab, dictionaries, Definitions, and view target', async () => { const bench = await mountPlugin() expect(bench.localeRegister).toHaveBeenCalledWith(NS, { zh, en }) expect(bench.events.map(definition => definition.kind)).toEqual([ - 'perf-step', 'perf-prompt', 'perf-turn-end', + 'perf-step', 'perf-prompt', 'perf-turn-end', 'perf-code-mode', ]) + expect(clientInject).toEqual(['slots', 'sessions', 'uiConversation', 'locale']) expect(bench.views).toEqual([perfViewDefinition]) expect(bench.entries()).toHaveLength(1) const registered = bench.entries()[0]! @@ -332,6 +371,11 @@ describe('browser plugin registration', () => { bench.notify() expect(listener).toHaveBeenCalledTimes(1) unsubscribe() + expect(await injected.loadOlder()).toBe(false) + expect(bench.loadOlder).toHaveBeenCalledTimes(1) + // The source is cached per Session binding; a second injection reuses it. + expect(injectEntry!(SID).hooks.perf).toBe(injected.hooks.perf) + expect(() => injectEntry!('perf-missing' as SessionId)).toThrow(/unavailable/) await bench.fiber.dispose() }) }) @@ -347,7 +391,8 @@ describe('locale dictionaries', () => { expect(Object.keys(en).sort()).toEqual(Object.keys(zh).sort()) expect(Object.values(en).every(value => value !== '')).toBe(true) expect(en['view.perf']).toBe('Performance') - expect(en['page.info']).toContain('{page}') + expect(en['summary']).toContain('{turns}') + expect(en['loadOlder']).toBe('Load earlier') }) }) @@ -445,17 +490,6 @@ describe('conversation Definitions', () => { expect(turn.steps[0]?.running).toBe(false) }) - it('closes a turn whose reason carries no kind', async () => { - const definitions = await registeredDefinitions() - const snapshot = assemble(definitions, [ - entry(event('turn/start', { turn: 3 })), - entry(event('step/start', { turn: 3, step: 1 }, 6_000)), - entry(event('step/end', { turn: 3, step: 1 }, 6_200)), - entry(event('turn/end', { turn: 3, reason: {} }, 6_200)), - ]) - expect(snapshot!.turns[0]?.interrupted).toBe(false) - }) - it('folds a trimmed window from the update events it still holds', async () => { const definitions = await registeredDefinitions() const snapshot = assemble(definitions, [ @@ -529,48 +563,151 @@ describe('conversation Definitions', () => { expect(snapshot!.turns[0]!.steps[0]!.ttft).toBe(300) }) - it('reports a prompt Definition only after a turn start', async () => { + it('publishes a prompt node from a human-typed user message location', async () => { + const definitions = await registeredDefinitions() + const prompt = definitions[1] as ConversationNodeDefinition + expect(prompt.match(event('turn/start', { turn: 9 }))).toBeNull() + expect(prompt.match(event('user/message', { + role: 'user', id: 'u4', content: [{ type: 'text', text: '' }], source: { kind: 'runtime-context' }, + }))).toBeNull() + const user = event('user/message', { + role: 'user', id: 'u6', content: [{ type: 'text', text: '继续' }, { type: 'image' }], source: { kind: 'user' }, + }) + expect(prompt.match(user)).toEqual({ id: `${user.seq}`, role: 'start' }) + const match = locatedStart(user, turnAt(9)) + const state = prompt.start(contextOf(prompt, undefined), match, reader) + expect(state.turn).toBe(9) + expect(state.mark).toBe('prompt') + expect(state.prompt).toBe('继续') + expect(state.images).toBe(1) + const node = prompt.buildViewNode!({ + ...contextOf(prompt, state, [match], match), + }) as PerfViewNode + expect(node.data).toEqual({ kind: 'prompt', turn: 9, prompt: '继续', images: 1 }) + const context: ConversationNodeContext & { readonly state: PerfStepState } = { + ...contextOf(prompt, state, [match], match), + state, + } + expect(prompt.update(context, match)).toBe(state) + }) + + it('takes a prompt turn from a step location and anchors without a start', async () => { const definitions = await registeredDefinitions() const prompt = definitions[1] as ConversationNodeDefinition const user = event('user/message', { - role: 'user', id: 'u3', content: [{ type: 'text', text: '无轮次' }], source: { kind: 'user' }, + role: 'user', id: 'u7', content: [{ type: 'text', text: '问题' }], source: { kind: 'user' }, }) - expect(prompt.match(user)).toBeNull() - resetCurrentTurn() - expect(prompt.match(event('turn/start', { turn: 9 }))).toEqual({ id: '9:0', role: 'start' }) - const state = prompt.start( - contextOf(prompt, undefined), - startMatch(event('turn/start', { turn: 9 })), - reader, - ) - expect(state.turn).toBe(9) - expect(state.mark).toBe('prompt') + const located = locatedStart(user, stepAt(4, 1)) + const state = prompt.start(contextOf(prompt, undefined), located, reader) + expect(state.turn).toBe(4) + const update = locatedUpdate(user, stepAt(4, 1)) + const node = prompt.buildViewNode!({ + ...contextOf(prompt, state, [update]), + }) as PerfViewNode + expect(node.anchorSeq).toBe(0) + expect(node.data).toMatchObject({ kind: 'prompt', turn: 4 }) + // A message the window cannot place publishes no row. + expect(prompt.buildViewNode!({ ...contextOf(prompt, state) })).toBeNull() }) - it('ignores user messages that are not typed prompts', async () => { + it('folds Code-Mode dispatches into one node per root call', async () => { const definitions = await registeredDefinitions() - const prompt = definitions[1] as ConversationNodeDefinition - const turnStart = event('turn/start', { turn: 10 }) - const promptState = prompt.start( - contextOf(prompt, undefined), - startMatch(turnStart), + const codeMode = definitions[3] as ConversationNodeDefinition + const startEvent = event('tool/ptc-dispatch-start', { + rootCallId: 'root-1', parentCallId: 'root-1', subCallId: 'sub-1', name: 'grep', arguments: '{"pattern":"x"}', + }, 8_200) + const dispatch = event('tool/ptc-dispatch', { + rootCallId: 'root-1', parentCallId: 'root-1', subCallId: 'sub-1', name: 'grep', + arguments: '{"pattern":"x"}', isError: false, content: [{ type: 'text', text: 'hit' }], + }, 8_300) + expect(codeMode.match(startEvent)).toEqual({ id: 'code:root-1', role: 'start' }) + expect(codeMode.match(dispatch)).toEqual({ id: 'code:root-1', role: 'update' }) + expect(codeMode.match(event('step/start', { turn: 1, step: 1 }))).toBeNull() + const start = locatedStart(startEvent, stepAt(5, 1)) + const update = locatedUpdate(dispatch, stepAt(5, 1)) + const state = codeMode.start(contextOf(codeMode, undefined), start, reader) + expect(state.starts).toEqual({ 'sub-1': 8_200 }) + const context: ConversationNodeContext & { readonly state: PerfStepState } = { + ...contextOf(codeMode, state, [start, update], start), + state, + } + const folded = codeMode.update(context, update) + expect(folded.calls?.['sub-1']?.name).toBe('grep') + const node = codeMode.buildViewNode!({ + ...contextOf(codeMode, folded, [start, update], start), + }) as PerfViewNode + expect(node.data).toMatchObject({ kind: 'inner', turn: 5, step: 1 }) + const inner = node.data + if (inner.kind !== 'inner') throw new Error('expected an inner contribution') + expect(inner.calls.map(call => call.name)).toEqual(['grep']) + expect(inner.calls[0]?.ms).toBe(100) + // Without a step location the contribution has no row to merge into. + expect(codeMode.buildViewNode!({ + ...contextOf(codeMode, folded, [start, update], locatedStart(startEvent, turnAt(5))), + })).toBeNull() + }) + + it('publishes an inner node only when the window still knows its step', async () => { + const definitions = await registeredDefinitions() + const codeMode = definitions[3] as ConversationNodeDefinition + const startEvent = event('tool/ptc-dispatch-start', { + rootCallId: 'root-2', parentCallId: 'root-2', subCallId: 'sub-2', name: 'read', arguments: '{}', + }, 6_200) + const dispatch = event('tool/ptc-dispatch', { + rootCallId: 'root-2', parentCallId: 'root-2', subCallId: 'sub-2', name: 'read', + arguments: '{}', isError: false, content: [], + }, 6_300) + const start = locatedStart(startEvent, stepAt(6, 1)) + const update = locatedUpdate(dispatch, stepAt(6, 1)) + const state = codeMode.start(contextOf(codeMode, undefined), start, reader) + const folded = codeMode.update({ + ...contextOf(codeMode, state, [start, update], start), + state, + }, update) + // Without a start match the node anchors at zero and takes the last location. + const noStart = codeMode.buildViewNode!({ + ...contextOf(codeMode, folded, [update]), + }) as PerfViewNode + expect(noStart.anchorSeq).toBe(0) + expect(noStart.data).toMatchObject({ kind: 'inner', turn: 6, step: 1 }) + // A turn-only location, or no location evidence, publishes no row. + expect(codeMode.buildViewNode!({ + ...contextOf(codeMode, folded, [locatedUpdate(dispatch, turnAt(6))]), + })).toBeNull() + expect(codeMode.buildViewNode!({ ...contextOf(codeMode, folded) })).toBeNull() + // The node still merges even when the dispatch lost its own start. + const lostStart = { ...folded, calls: { 'sub-2': { ...folded.calls!['sub-2']!, ms: null, startedAt: null } } } + const node = codeMode.buildViewNode!({ + ...contextOf(codeMode, lostStart, [update]), + }) as PerfViewNode + expect(node.data).toMatchObject({ kind: 'inner', turn: 6, step: 1 }) + }) + + it('keeps a Code-Mode start unplaced when the window has no step location', async () => { + const definitions = await registeredDefinitions() + const codeMode = definitions[3] as ConversationNodeDefinition + const startEvent = event('tool/ptc-dispatch-start', { + rootCallId: 'root-3', parentCallId: 'root-3', subCallId: 'sub-3', name: 'grep', arguments: '{}', + }, 7_200) + const turnOnly = codeMode.start( + contextOf(codeMode, undefined), + locatedStart(startEvent, turnAt(7)), reader, ) - const runtime = event('user/message', { - role: 'user', id: 'u4', content: [{ type: 'text', text: '' }], source: { kind: 'runtime-context' }, - }) - expect(prompt.match(runtime)).toBeNull() - expect(prompt.match(event('user/message', { role: 'user', id: 'u5', content: [] }))).toBeNull() - expect(prompt.match(event('step/start', { turn: 10, step: 1 }))).toBeNull() - const typed = event('user/message', { - role: 'user', id: 'u6', content: [{ type: 'text', text: '继续' }], source: { kind: 'user' }, - }) - expect(prompt.match(typed)).toEqual({ id: '10:0', role: 'update' }) - const context: ConversationNodeContext & { readonly state: PerfStepState } = { - ...contextOf(prompt, promptState, [updateMatch(typed)]), - state: promptState, - } - expect(prompt.update(context, updateMatch(typed)).prompt).toBe('继续') + expect(turnOnly.turn).toBe(0) + expect(turnOnly.step).toBe(0) + const unlocated = codeMode.start( + contextOf(codeMode, undefined), + { ...startMatch(startEvent), location: undefined as never }, + reader, + ) + expect(unlocated.turn).toBe(0) + // An update that arrives before any start keeps the absent state. + const orphan = event('tool/ptc-dispatch', { + rootCallId: 'root-4', parentCallId: 'root-4', subCallId: 'sub-4', name: 'read', + arguments: '{}', isError: false, content: [], + }, 7_300) + expect(codeMode.update(contextOf(codeMode, undefined) as never, updateMatch(orphan))).toBeUndefined() }) it('rejects a start callback handed the wrong lifecycle event', async () => { @@ -600,10 +737,10 @@ describe('conversation Definitions', () => { expect(step.publication!(updateMatch(event('assistant/attempt', { turn: 1, step: 1 })))).toBe('none') expect(step.publication!(updateMatch(event('step/end', { turn: 1, step: 1 })))).toBe('immediate') const prompt = definitions[1] as ConversationNodeDefinition - expect(prompt.publication!(startMatch(event('turn/start', { turn: 1 })))).toBe('none') - expect(prompt.publication!(updateMatch(event('user/message', { + expect(prompt.publication!(startMatch(event('user/message', { role: 'user', id: 'u', content: [], source: { kind: 'user' }, })))).toBe('immediate') + expect(Object.hasOwn(definitions[3]!, 'publication')).toBe(false) }) it('anchors and locates a step node from its folded state', async () => { @@ -639,33 +776,15 @@ describe('conversation Definitions', () => { const contribution = withStart.data if (contribution.kind !== 'step') throw new Error('expected a step contribution') expect(contribution.metrics.context).toBe(600) + expect(contribution.metrics.toolWall).toBeNull() // Without a start match, the node anchors at 0 and takes the last match location. - const stepLocation: ConversationLocation = { - kind: 'step', - turn: { - turn: 11, - start: undefined, - end: undefined, - status: 'open', - steps: [], - data: { get: () => undefined, source: () => ({ getSnapshot: () => undefined, subscribe: () => () => {} }) }, - }, - step: { - turn: 11, - step: 2, - start: undefined, - end: undefined, - status: 'unknown', - data: { get: () => undefined, source: () => ({ getSnapshot: () => undefined, subscribe: () => () => {} }) }, - }, - } const trimmed = step.buildViewNode!({ ...contextOf(step, - { ...folded, startSeq: null }, [{ ...update, location: stepLocation }]), + { ...folded, startSeq: null }, [{ ...update, location: stepAt(11, 2) }]), }) as PerfViewNode expect(trimmed.anchorSeq).toBe(0) - expect(trimmed.location).toEqual(stepLocation) + expect(trimmed.location).toEqual(stepAt(11, 2)) const unresolved = step.buildViewNode!({ ...contextOf(step, { ...folded, startSeq: null }, []), }) as PerfViewNode @@ -682,33 +801,20 @@ describe('conversation Definitions', () => { [event('assistant/message', { turn: 12, step: 3 }), '12:3'], [event('llm/retry', { turn: 12, step: 3 }), '12:3'], [event('tool/call', { turn: 12, step: 3, callId: 'c', name: 'read', arguments: '{}' }), '12:3'], - [event('tool/result', { turn: 12, step: 3, message: { role: 'tool', content: [] } }), '12:3'], + [event('tool/result', { + turn: 12, step: 3, message: { role: 'tool', content: [], source: { callId: 'c' }, toolCallId: 'c' }, + }), '12:3'], [transient(12, 3, delta).event, '12:3'], ] for (const [fixture, id] of cases) { expect(step.match(fixture)?.id).toBe(id) } expect(step.match(event('developer/message', { turn: 12, step: 3 }))).toBeNull() - // A dispatch with no recorded root call owns no step. - expect(step.match(event('tool/ptc-dispatch', { - rootCallId: 'missing', parentCallId: 'missing', subCallId: 's', - name: 'grep', arguments: '{}', isError: false, content: [], - }))).toBeNull() + expect(step.match(event('user/message', { role: 'user', id: 'u', content: [], source: { kind: 'user' } }))).toBeNull() }) it('builds prompt and end markers from their folded states', async () => { const definitions = await registeredDefinitions() - const prompt = definitions[1] as ConversationNodeDefinition - const promptStart = startMatch(event('turn/start', { turn: 13 })) - const promptState = prompt.start( - contextOf(prompt, undefined), - promptStart, - reader, - ) - const promptNode = prompt.buildViewNode!({ - ...contextOf(prompt, promptState, [], promptStart), - }) as PerfViewNode - expect(promptNode.data).toMatchObject({ kind: 'prompt', turn: 13 }) const end = definitions[2] as ConversationNodeDefinition const endStart = startMatch(event('turn/end', { turn: 13, reason: { kind: 'interrupted' } }, 13_500)) const ended = end.start( @@ -725,18 +831,8 @@ describe('conversation Definitions', () => { expect(end.match(event('step/start', { turn: 13, step: 1 }))).toBeNull() }) - it('anchors marker nodes at zero without a recorded start', async () => { + it('keeps the end state and anchors it at zero without a recorded start', async () => { const definitions = await registeredDefinitions() - const prompt = definitions[1] as ConversationNodeDefinition - const promptState = prompt.start( - contextOf(prompt, undefined), - startMatch(event('turn/start', { turn: 14 })), - reader, - ) - const promptNode = prompt.buildViewNode!({ - ...contextOf(prompt, promptState), - }) as PerfViewNode - expect(promptNode.anchorSeq).toBe(0) const end = definitions[2] as ConversationNodeDefinition const endState = end.start( contextOf(end, undefined), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3cff6c298c..dfaef97b85 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8974,6 +8974,9 @@ importers: '@deepseek-ai/dsh-api-session-controller': specifier: workspace:* version: link:../../api/session-controller + '@deepseek-ai/dsh-attachment': + specifier: workspace:* + version: link:../../attachment/attachment '@deepseek-ai/dsh-client-locale': specifier: workspace:* version: link:../../client/locale @@ -8998,9 +9001,15 @@ importers: '@deepseek-ai/dsh-llm': specifier: workspace:* version: link:../../llm/llm + '@deepseek-ai/dsh-llm-retry': + specifier: workspace:* + version: link:../../llm/llm-retry '@deepseek-ai/dsh-session': specifier: workspace:* version: link:../../core/session + '@deepseek-ai/dsh-tool-executor-ptc': + specifier: workspace:* + version: link:../../core/tool-executor-ptc '@deepseek-ai/dsh-tools': specifier: workspace:* version: link:../../core/tools