From 28c26472932d9444cbf92d61261a75e141170e90 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Tue, 18 Aug 2026 11:56:18 +0800 Subject: [PATCH] fix(llm-pi-ai): address image offload review --- ...8-18-request-image-payload-bound.i18n.yaml | 4 +- .../2026-08-18-request-image-payload-bound.md | 6 +- ...26-08-18-request-image-payload-bound.zh.md | 6 +- examples/acp-agent/tests/acp.snapshot.ts | 97 ++++++++++++++++++- .../tests/fixtures/image-offload.cordis.yml | 41 ++++++++ packages/llm/llm-pi-ai/README.i18n.yaml | 4 +- packages/llm/llm-pi-ai/README.md | 2 +- packages/llm/llm-pi-ai/README.zh.md | 2 +- packages/llm/llm-pi-ai/src/config.ts | 8 +- packages/llm/llm-pi-ai/src/context.ts | 72 +++++++++----- packages/llm/llm-pi-ai/src/stream.ts | 2 +- packages/llm/llm-pi-ai/tests/context.spec.ts | 24 +++++ packages/llm/llm-pi-ai/tests/convert.spec.ts | 4 + 13 files changed, 229 insertions(+), 43 deletions(-) create mode 100644 examples/acp-agent/tests/fixtures/image-offload.cordis.yml diff --git a/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.i18n.yaml index 9483543cb8..cd601d6c9f 100644 --- a/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.i18n.yaml +++ b/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.md -2026-08-18-request-image-payload-bound.md: 3554aa9489c9c5fbf616eb0f875a7a235715d0ed -2026-08-18-request-image-payload-bound.zh.md: c0a9643d74177b6becbcd61f42a4f5e6accca324 +2026-08-18-request-image-payload-bound.md: caeb584b230740d0daff0603cec8b1638a9066f2 +2026-08-18-request-image-payload-bound.zh.md: 0c4de2a63055fcf743b71b4143bf3315deeb970b diff --git a/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.md b/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.md index 3554aa9489..caeb584b23 100644 --- a/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.md +++ b/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.md @@ -10,11 +10,11 @@ Every image in session history is base64-inlined into every model request by the ## Decision -The pi-ai provider profile carries `maxRequestImageBytes` (default `DEFAULT_MAX_REQUEST_IMAGE_BYTES = 24MiB`, a positive integer, per route, changeable from cordis.yml and the `llm-pi-ai` settings section). At request conversion, `toPiContext` sums the base64 length of every image in history (derived from `ImageAttachmentRef.bytes` without reading data) and, while the sum exceeds the bound, replaces the oldest images with a fixed model-facing placeholder telling the model the image was omitted and can be re-read. History order is oldest-first, so the most recent images always survive; offloaded images are never read from the attachment store. `classifyPiAiError` now classifies 413 and request-body-cap wording as `INVALID_REQUEST` (resending the same body cannot succeed). The default stays under the smallest documented provider request-body cap (30MiB) with headroom for text and JSON structure; deployments behind stricter gateways lower the value per route. +The pi-ai provider profile carries `maxRequestImageBytes` (default `DEFAULT_MAX_REQUEST_IMAGE_BYTES = 24MiB`, a positive integer, per route, changeable from cordis.yml and the `llm-pi-ai` settings section). At request conversion, `toPiContext` sums the base64 length of every image in history (derived from `ImageAttachmentRef.bytes` without reading data) and, while the sum exceeds the bound, replaces the oldest images with a fixed model-facing placeholder. The placeholder tells the model to read the file again when a path is available or ask the user to attach the image again. The most recent images are omitted last; an image larger than the bound is itself omitted. Offload locations use message and nested block indexes rather than object identity, so replaying the same JSON log produces the same request. Offloaded images are never read from the attachment store. `classifyPiAiError` classifies 413 and specific request-body-cap wording as `INVALID_REQUEST` (resending the same body cannot succeed). The default stays under the smallest documented provider request-body cap (30MiB) with headroom for text and JSON structure; deployments behind stricter gateways lower the value per route. ## Offload is conversion, not history -The placeholder is model-visible but not logged as a session event. It stays within the model-visible ⟺ logged invariant the same way the adapter's other serialization does (`(no output)` fallbacks, text-only folding): the offload set is a pure function of the logged history and the route configuration, so the exact request remains reconstructable from the session log plus the composition. A logged elision event becomes necessary only when offload decisions gain non-deterministic inputs (for example live gateway feedback), which belongs to the deferred capability-metadata design. +The placeholder is model-visible but not logged as a session event. It stays within the model-visible ⟺ logged invariant the same way the adapter's other serialization does (`(no output)` fallbacks, text-only folding): the offload locations are a pure function of the logged history and the route configuration, so the exact request remains reconstructable from the session log plus the composition. A logged elision event becomes necessary only when offload decisions gain non-deterministic inputs (for example live gateway feedback), which belongs to the deferred capability-metadata design. ## Alternatives considered @@ -29,7 +29,7 @@ The placeholder is model-visible but not logged as a session event. It stays wit ## Consequences -- An image-heavy long session keeps completing requests; the model loses the oldest images but is told so and can re-read them, and the newest images always survive. +- An image-heavy long session keeps completing requests. The oldest images are omitted first; the most recent image is omitted only when it cannot fit within the bound. - Crossing the bound rewrites an early message, so the provider prompt-cache prefix ends at the newly offloaded image until the offloaded prefix stabilizes. - The bound counts base64 image payload only; deployments must keep it below their gateway's request-body cap with headroom, and the shipped default cannot know a private gateway's cap. - Route capability metadata driving admission and assembly together (image count, per-image size, request size, provider token formulas) remains deferred design work tracked outside this fix. diff --git a/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.zh.md b/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.zh.md index c0a9643d74..0c4de2a630 100644 --- a/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.zh.md +++ b/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.zh.md @@ -10,11 +10,11 @@ pi-ai 适配器把会话历史中的每张图片 base64 内联进每一个模型 ## Decision -pi-ai provider profile 增加 `maxRequestImageBytes`（默认 `DEFAULT_MAX_REQUEST_IMAGE_BYTES = 24MiB`，正整数，按路由生效，可从 cordis.yml 与 `llm-pi-ai` settings 段修改）。请求转换时，`toPiContext` 由 `ImageAttachmentRef.bytes` 推算每张历史图片的 base64 长度（无需读取数据）求和，总和超过上限时从最老的图片开始替换为一段固定的模型可见占位文本，告知模型该图片已省略、可重新读取。历史顺序即最老在前，最新的图片总是保留；被 offload 的图片不会从附件存储读取。`classifyPiAiError` 现在把 413 与请求体上限类措辞归类为 `INVALID_REQUEST`（原样重发不可能成功）。默认值低于已知最小的提供方请求体上限（30MiB）并为文本与 JSON 结构留出余量；网关更严格的部署按路由调低。 +pi-ai provider profile 增加 `maxRequestImageBytes`（默认 `DEFAULT_MAX_REQUEST_IMAGE_BYTES = 24MiB`，正整数，按路由生效，可从 cordis.yml 与 `llm-pi-ai` settings 段修改）。请求转换时，`toPiContext` 由 `ImageAttachmentRef.bytes` 推算每张历史图片的 base64 长度（无需读取数据）求和，总和超过上限时从最老的图片开始替换为一段固定的模型可见占位文本。占位文本要求模型在有路径时重新读取文件，否则请用户重新附上图片。越新的图片越晚被省略；单张图片本身超过上限时也会被省略。offload 位置用消息与嵌套块的索引表示，不依赖对象身份，因此重放同一份 JSON 日志会产生相同请求。被 offload 的图片不会从附件存储读取。`classifyPiAiError` 把 413 与明确的请求体上限措辞归类为 `INVALID_REQUEST`（原样重发不可能成功）。默认值低于已知最小的提供方请求体上限（30MiB）并为文本与 JSON 结构留出余量；网关更严格的部署按路由调低。 ## offload 是转换而非历史 -占位文本模型可见，但不记录为会话事件。它与适配器的其他序列化（`(no output)` 回退、纯文本折叠）以同样的方式满足「模型可见 ⟺ 已记录」不变量：offload 集合是已记录历史与路由配置的纯函数，确切请求仍可由会话日志加组合配置重建。只有当 offload 决策引入非确定性输入（例如网关的实时反馈）时才需要记录省略事件，那属于暂缓的能力元数据设计。 +占位文本模型可见，但不记录为会话事件。它与适配器的其他序列化（`(no output)` 回退、纯文本折叠）以同样的方式满足「模型可见 ⟺ 已记录」不变量：offload 位置是已记录历史与路由配置的纯函数，确切请求仍可由会话日志加组合配置重建。只有当 offload 决策引入非确定性输入（例如网关的实时反馈）时才需要记录省略事件，那属于暂缓的能力元数据设计。 ## Alternatives considered @@ -29,7 +29,7 @@ pi-ai provider profile 增加 `maxRequestImageBytes`（默认 `DEFAULT_MAX_REQUE ## Consequences -- 图片较多的长会话持续可用；模型失去最老的图片但被明确告知、可重新读取，最新图片总是保留。 +- 图片较多的长会话持续可用。最老的图片优先省略；仅当最新图片本身无法装进上限时才会省略它。 - 越过上限会改写较早的一条消息，提供方 prompt cache 前缀在新被 offload 的图片处截止，直到被 offload 的前缀稳定。 - 上限只统计 base64 图片载荷；部署必须让它低于自家网关的请求体上限并留出余量，发行默认值无法预知私有网关的上限。 - 由路由能力元数据同时驱动准入与组装（图片数量、单图大小、请求大小、提供方 token 公式）的设计仍为暂缓工作，在本修复之外跟踪。 diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index c5bf81fc1f..e617206dcb 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -1,11 +1,19 @@ import { fileURLToPath } from 'node:url' import { readFileSync } from 'node:fs' import { spawnSync } from 'node:child_process' +import { createServer } from 'node:http' +import type { IncomingMessage, ServerResponse } from 'node:http' import { mkdir, utimes, writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' import { homedir } from 'node:os' import { expect, it } from 'vitest' -import { defineAcpSnapshotSuite, type Scenario, type SnapshotSuiteOptions } from '@deepseek-ai/dsh-acp-snapshot' +import { + defineAcpSnapshotSuite, + runScenario, + type InputScript, + type Scenario, + type SnapshotSuiteOptions, +} from '@deepseek-ai/dsh-acp-snapshot' import { resolvePwshPath } from '@deepseek-ai/dsh-pwsh-local' import { decodeStorageRecord } from '@deepseek-ai/dsh-session' @@ -40,6 +48,7 @@ const ADVANCED_CONFIG = fileURLToPath(new URL('../advanced.cordis.yml', import.m const FS_CONFIG = fileURLToPath(new URL('../fs.cordis.yml', import.meta.url)) const SESSION_QUERY_CONFIG = fileURLToPath(new URL('../session-query.cordis.yml', import.meta.url)) const IMAGE_CONFIG = fileURLToPath(new URL('../image.cordis.yml', import.meta.url)) +const IMAGE_OFFLOAD_CONFIG = fileURLToPath(new URL('./fixtures/image-offload.cordis.yml', import.meta.url)) const IMAGE_TEXT_ROUTE_CONFIG = fileURLToPath(new URL('../image-text-route.cordis.yml', import.meta.url)) const PTY_CONFIG = fileURLToPath(new URL('../pty.cordis.yml', import.meta.url)) const DEPTH_TWO_CONFIG = fileURLToPath(new URL('../depth-two.cordis.yml', import.meta.url)) @@ -655,6 +664,92 @@ defineAcpSnapshotSuite({ hasPwsh, }) +it('pins pi-ai image offload in the request sent by the assembled app', async () => { + const requests: Record[] = [] + const server = createServer((request: IncomingMessage, response: ServerResponse) => { + let body = '' + request.setEncoding('utf8') + request.on('data', (chunk: string) => { body += chunk }) + request.on('end', () => { + requests.push(JSON.parse(body) as Record) + response.writeHead(200, { 'content-type': 'text/event-stream' }) + response.end([ + 'data: {"choices":[{"delta":{"role":"assistant","content":""},"index":0,"finish_reason":null}]}', + 'data: {"choices":[{"delta":{"content":"DONE"},"index":0,"finish_reason":null}]}', + 'data: {"choices":[{"delta":{},"index":0,"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":1}}', + 'data: [DONE]', + '', + ].join('\n\n')) + }) + }) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + if (address === null || typeof address === 'string') throw new Error('image-offload snapshot server has no port') + + const image = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC' + const input: InputScript = { + steps: [ + { op: 'initialize' }, + { op: 'newSession' }, + { + op: 'promptContent', + content: [ + { type: 'text', text: 'Compare the older image ' }, + { type: 'image', data: image, mimeType: 'image/png' }, + { type: 'text', text: ' with the newer image ' }, + { type: 'image', data: image, mimeType: 'image/png' }, + { type: 'text', text: ', then reply with DONE.' }, + ], + }, + ], + } + + try { + const result = await runScenario(input, { + agent: AGENT, + mode: 'record', + configPath: IMAGE_OFFLOAD_CONFIG, + fixtureFile: join(SNAPSHOTS_DIR, 'image-offload-request', 'session.jsonl'), + env: { + DSH_SNAPSHOT_API_KEY: 'snapshot-key', + DSH_SNAPSHOT_BASE_URL: `http://127.0.0.1:${address.port}/v1`, + }, + }) + expect(result.stderr).toBe('') + expect(requests).toHaveLength(1) + const messages = requests[0]?.messages as { content?: unknown }[] | undefined + const offloaded = messages?.find(message => JSON.stringify(message.content).includes('[image omitted')) + expect(offloaded?.content).toMatchInlineSnapshot(` + [ + { + "text": "Compare the older image ", + "type": "text", + }, + { + "text": "[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]", + "type": "text", + }, + { + "text": " with the newer image ", + "type": "text", + }, + { + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC", + }, + "type": "image_url", + }, + { + "text": ", then reply with DONE.", + "type": "text", + }, + ] + `) + } finally { + await new Promise(resolve => server.close(() => { resolve() })) + } +}, 45_000) + it('packed ACP fixture retains every chunk row kind without changing the logical session', () => { const source = fixtureRecords(PACKED_CHUNKS_SOURCE) const packed = fixtureRecords('packed-chunks') diff --git a/examples/acp-agent/tests/fixtures/image-offload.cordis.yml b/examples/acp-agent/tests/fixtures/image-offload.cordis.yml new file mode 100644 index 0000000000..6007c5d44d --- /dev/null +++ b/examples/acp-agent/tests/fixtures/image-offload.cordis.yml @@ -0,0 +1,41 @@ +# Keyless assembled-request snapshot for pi-ai image offload. The local +# provider endpoint is supplied by the snapshot test; the real attachment +# store and ACP bridge carry two uploaded images into one model request. +- id: base + name: '@deepseek-ai/cordis-plugin-include' + config: + path: ../../cordis.yml + patches: + - id: llm-deepseek + name: '@deepseek-ai/dsh-llm-deepseek' + disabled: true + - id: acp-agent + name: '@deepseek-ai/dsh-acp-demo' + config: + provider: snapshot + model: vision + persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions' + persistenceCompression: none + workspaceContext: + maxBytes: 65536 + persona: | + You are a coding assistant powered by the {{model}} model. Your working directory is {{cwd}}. + + Keep answers brief and factual. + - insert: + - id: attachment-local + name: '@deepseek-ai/dsh-attachment-local' + - id: llm-pi-ai + name: '@deepseek-ai/dsh-llm-pi-ai' + config: + providers: + snapshot: + apiKeyEnv: DSH_SNAPSHOT_API_KEY + api: openai-completions + baseURL: !!js process.env.DSH_SNAPSHOT_BASE_URL + maxRequestImageBytes: 92 + models: + - id: vision + contextWindow: 32768 + maxTokens: 1024 + input: [text, image] diff --git a/packages/llm/llm-pi-ai/README.i18n.yaml b/packages/llm/llm-pi-ai/README.i18n.yaml index b094c19e5e..384b66871e 100644 --- a/packages/llm/llm-pi-ai/README.i18n.yaml +++ b/packages/llm/llm-pi-ai/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/llm/llm-pi-ai/README.md -README.md: ce4a1daa0d6df949ca6d82c16981086a61d31908 -README.zh.md: f29e36fb8a1302e683f66bad9c3f76bbf71930d5 +README.md: aa37ef7f06f54f18dedcdd732eb1eba60da0a960 +README.zh.md: 53b90f9314ee0b12a01584eaf73ca868c1b2153d diff --git a/packages/llm/llm-pi-ai/README.md b/packages/llm/llm-pi-ai/README.md index ce4a1daa0d..aa37ef7f06 100644 --- a/packages/llm/llm-pi-ai/README.md +++ b/packages/llm/llm-pi-ai/README.md @@ -163,7 +163,7 @@ pi-ai installs several provider SDKs and lazy-loads the one selected by the cata #### What the model sees -The selected catalog model receives `GenerateOptions.system`, history, tools, and sampling fields supported by pi-ai's common streaming API. This package adds no prompt prose, with one exception: when a request's accumulated base64 image payload exceeds the route's `maxRequestImageBytes`, each offloaded image (oldest first) is replaced by a fixed placeholder text telling the model the image was omitted and can be re-read. Provider-native replay metadata is restored only when the adapter validates it for the historical content. +The selected catalog model receives `GenerateOptions.system`, history, tools, and sampling fields supported by pi-ai's common streaming API. This package adds no prompt prose, with one exception: when a request's accumulated base64 image payload exceeds the route's `maxRequestImageBytes`, each offloaded image (oldest first) is replaced by fixed text. The text tells the model to read the file again when a path is available or ask the user to attach the image again. Provider-native replay metadata is restored only when the adapter validates it for the historical content. #### Token effect diff --git a/packages/llm/llm-pi-ai/README.zh.md b/packages/llm/llm-pi-ai/README.zh.md index f29e36fb8a..53b90f9314 100644 --- a/packages/llm/llm-pi-ai/README.zh.md +++ b/packages/llm/llm-pi-ai/README.zh.md @@ -164,7 +164,7 @@ pi-ai 会安装多个提供方 SDK，并延迟加载 catalog 模型所选的 SDK #### 模型看到的内容 -所选 catalog 模型会收到 `GenerateOptions.system`、历史、工具，以及 pi-ai 通用流式 API 支持的采样字段。本包不添加提示词文本，仅有一个例外：请求累积的 base64 图片载荷超过路由的 `maxRequestImageBytes` 时，被 offload 的图片（从最老开始）会被替换为一段固定占位文本，告知模型该图片已省略、可重新读取。只有当适配器验证提供方原生回放元数据与历史内容匹配时，才会恢复这些元数据。 +所选 catalog 模型会收到 `GenerateOptions.system`、历史、工具，以及 pi-ai 通用流式 API 支持的采样字段。本包不添加提示词文本，仅有一个例外：请求累积的 base64 图片载荷超过路由的 `maxRequestImageBytes` 时，被 offload 的图片（从最老开始）会被替换为一段固定文本。该文本要求模型在有路径时重新读取文件，否则请用户重新附上图片。只有当适配器验证提供方原生回放元数据与历史内容匹配时，才会恢复这些元数据。 #### Token 影响 diff --git a/packages/llm/llm-pi-ai/src/config.ts b/packages/llm/llm-pi-ai/src/config.ts index 9d4682ebe0..c39bcddda2 100644 --- a/packages/llm/llm-pi-ai/src/config.ts +++ b/packages/llm/llm-pi-ai/src/config.ts @@ -344,15 +344,15 @@ export function resolveProfiles( `llm-pi-ai: provider "${provider}" streamIdleTimeoutMs must be a positive finite number no greater than ${MAX_TIMER_DELAY_MS}`, ) } + const maxRequestImageBytes = source.maxRequestImageBytes ?? DEFAULT_MAX_REQUEST_IMAGE_BYTES + if (!Number.isInteger(maxRequestImageBytes) || maxRequestImageBytes <= 0) { + throw new Error(`llm-pi-ai: provider "${provider}" maxRequestImageBytes must be a positive integer`) + } // Detached from the configuration object because pi-ai types `Model.input` // mutable. The schema's explicit default covers an absent key, so an empty // list here is always one someone typed — and unlike an entry's, nothing // below it can answer instead — so it is refused rather than read as "no // answer". - const maxRequestImageBytes = source.maxRequestImageBytes ?? DEFAULT_MAX_REQUEST_IMAGE_BYTES - if (!Number.isInteger(maxRequestImageBytes) || maxRequestImageBytes <= 0) { - throw new Error(`llm-pi-ai: provider "${provider}" maxRequestImageBytes must be a positive integer`) - } const defaultInput = [...source.defaultInput ?? DEFAULT_INPUT] if (defaultInput.length === 0) { throw new Error(`llm-pi-ai: provider "${provider}" defaultInput must name at least one modality`) diff --git a/packages/llm/llm-pi-ai/src/context.ts b/packages/llm/llm-pi-ai/src/context.ts index 0dc8d891e3..5a2d330b7d 100644 --- a/packages/llm/llm-pi-ai/src/context.ts +++ b/packages/llm/llm-pi-ai/src/context.ts @@ -28,7 +28,7 @@ function toolResultText(blocks: readonly ContentBlock[]): string { /** Model-facing stand-in for an image dropped to fit the request bound. */ export const OFFLOADED_IMAGE_TEXT - = '[image omitted to keep the request within the provider size limit; older images are omitted first. Re-read the source file if this image is still needed.]' + = '[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]' /** Base64 length of `bytes` raw bytes (4 output characters per 3 input bytes, padded). */ function base64Length(bytes: number): number { @@ -37,49 +37,68 @@ function base64Length(bytes: number): number { /** * Select the images a request must drop to fit the per-request payload bound. - * History order is oldest-first, so dropping from the front keeps the most - * recent images. The result is keyed by block identity: the same conversion - * pass walks the same block objects it was derived from. + * History order is oldest-first, so the most recent images are omitted last. + * A single image larger than the bound is itself omitted. Locations use + * message and nested block indexes so JSON replay cannot change the result by + * splitting or preserving shared object identities. * @param messages - complete request history, oldest first. * @param maxRequestImageBytes - bound on total base64-encoded image payload; undefined leaves every image in place. - * @returns the image blocks the conversion replaces with {@link OFFLOADED_IMAGE_TEXT}. + * @returns the image locations the conversion replaces with {@link OFFLOADED_IMAGE_TEXT}. */ function offloadedImages( messages: readonly Message[], maxRequestImageBytes: number | undefined, -): ReadonlySet { - const offloaded = new Set() +): ReadonlySet { + const offloaded = new Set() if (maxRequestImageBytes === undefined) return offloaded - const images: { block: ContentBlock; base64Bytes: number }[] = [] - const collect = (blocks: readonly ContentBlock[]): void => { - for (const block of blocks) { - if (block.type === 'image') images.push({ block, base64Bytes: base64Length(block.attachment.bytes) }) - else if (block.type === 'tool-result') collect(block.content) + const images: { location: string; base64Bytes: number }[] = [] + const collect = (messageIndex: number, blocks: readonly ContentBlock[], prefix: readonly number[] = []): void => { + for (const [blockIndex, block] of blocks.entries()) { + const path = [...prefix, blockIndex] + if (block.type === 'image') { + images.push({ + location: `${messageIndex}:${path.join('.')}`, + base64Bytes: base64Length(block.attachment.bytes), + }) + } else if (block.type === 'tool-result') { + collect(messageIndex, block.content, path) + } } } - for (const message of messages) collect(message.content) + for (const [messageIndex, message] of messages.entries()) collect(messageIndex, message.content) let total = images.reduce((sum, image) => sum + image.base64Bytes, 0) for (const image of images) { if (total <= maxRequestImageBytes) break - offloaded.add(image.block) + offloaded.add(image.location) total -= image.base64Bytes } return offloaded } +interface LocatedContentBlock { + readonly block: ContentBlock + readonly path: readonly number[] +} + +/** Attach stable nested indexes to blocks from one message. */ +function locatedBlocks(blocks: readonly ContentBlock[], prefix: readonly number[] = []): LocatedContentBlock[] { + return blocks.map((block, index) => ({ block, path: [...prefix, index] })) +} + async function userContent( - blocks: readonly ContentBlock[], + blocks: readonly LocatedContentBlock[], attachments: AttachmentStore, - offloaded: ReadonlySet, + offloaded: ReadonlySet, + messageIndex: number, ): Promise { const content: (TextContent | ImageContent)[] = [] - for (const block of blocks) { + for (const { block, path } of blocks) { switch (block.type) { case 'text': if (block.text.length > 0) content.push({ type: 'text', text: block.text }) break case 'image': { - if (offloaded.has(block)) { + if (offloaded.has(`${messageIndex}:${path.join('.')}`)) { content.push({ type: 'text', text: OFFLOADED_IMAGE_TEXT }) break } @@ -93,7 +112,7 @@ async function userContent( } case 'tool-result': { - const nested = await userContent(block.content, attachments, offloaded) + const nested = await userContent(locatedBlocks(block.content, path), attachments, offloaded, messageIndex) if (typeof nested === 'string') { if (nested.length > 0) content.push({ type: 'text', text: nested }) } else { @@ -219,7 +238,7 @@ async function toPiContextWithImages( const toolNames = new Map() const messages: PiMessage[] = [] - for (const message of options.messages) { + for (const [messageIndex, message] of options.messages.entries()) { if (message.role === 'system') { if (contentHasImage(message.content)) { throw new LlmError('pi-ai cannot represent an image in an in-history system message', 'UNSUPPORTED_CONTENT') @@ -239,14 +258,17 @@ async function toPiContextWithImages( continue } // user role: text + tool results (each result becomes its own message). - const regular = message.content.filter(block => block.type !== 'tool-result') - const content = await userContent(regular, attachments, offloaded) - const results = message.content.filter(block => block.type === 'tool-result') + const located = locatedBlocks(message.content) + const regular = located.filter(({ block }) => block.type !== 'tool-result') + const content = await userContent(regular, attachments, offloaded, messageIndex) + const results = located.filter((entry): entry is LocatedContentBlock & { block: Extract } => ( + entry.block.type === 'tool-result' + )) if (content.length > 0 || results.length === 0) { messages.push({ role: 'user', content, timestamp: 0 }) } - for (const result of results) { - const resultContent = await userContent(result.content, attachments, offloaded) + for (const { block: result, path } of results) { + const resultContent = await userContent(locatedBlocks(result.content, path), attachments, offloaded, messageIndex) messages.push({ role: 'toolResult', toolCallId: result.toolCallId, diff --git a/packages/llm/llm-pi-ai/src/stream.ts b/packages/llm/llm-pi-ai/src/stream.ts index c455d5464c..d45df45a39 100644 --- a/packages/llm/llm-pi-ai/src/stream.ts +++ b/packages/llm/llm-pi-ai/src/stream.ts @@ -42,7 +42,7 @@ function classifyPiAiError(message: string): string { if (/\b429\b|rate.?limit/i.test(message)) return 'RATE_LIMIT' // A rejected request body (gateway or provider size cap): resending the // same request cannot succeed, so it is invalid, not transient. - if (/\b413\b|length limit exceeded|payload too large|request body too large/i.test(message)) return 'INVALID_REQUEST' + if (/\b413\b|failed to buffer the request body:\s*length limit exceeded|payload too large|request body too large/i.test(message)) return 'INVALID_REQUEST' if (/\b400\b|invalid.?request/i.test(message)) return 'INVALID_REQUEST' if (/\b5\d\d\b/.test(message)) return 'SERVER' if (/\btime(?:d)?\s*out\b|timeout/i.test(message)) return 'TIMEOUT' diff --git a/packages/llm/llm-pi-ai/tests/context.spec.ts b/packages/llm/llm-pi-ai/tests/context.spec.ts index 01e72fca20..be41a12b07 100644 --- a/packages/llm/llm-pi-ai/tests/context.spec.ts +++ b/packages/llm/llm-pi-ai/tests/context.spec.ts @@ -202,6 +202,30 @@ describe('pi-ai request context conversion', () => { expect(readImage).not.toHaveBeenCalled() }) + it('offloads repeated image-block occurrences by position rather than shared object identity', async () => { + const sized: ImageAttachmentRef = { ...ref, bytes: 3 } + const shared: ContentBlock = { type: 'image', attachment: sized } + const readImage = vi.fn(() => Promise.resolve({ ref: sized, data: Uint8Array.of(1, 2, 3) })) + const store = { readImage } as unknown as AttachmentStore + const aliased = await toPiContext(request([user([shared, shared])]), store, undefined, 4) + const replayed = await toPiContext(request([user([ + { type: 'image', attachment: { ...sized } }, + { type: 'image', attachment: { ...sized } }, + ])]), store, undefined, 4) + + const expected = [{ + role: 'user', + content: [ + { type: 'text', text: OFFLOADED_IMAGE_TEXT }, + { type: 'image', data: 'AQID', mimeType: 'image/png' }, + ], + timestamp: 0, + }] + expect(aliased.messages).toEqual(expected) + expect(replayed.messages).toEqual(expected) + expect(readImage).toHaveBeenCalledTimes(2) + }) + it('keeps empty text-only users while separating result-only messages', () => { const callId = CallId('unknown-call') expect(toPiContext(request([ diff --git a/packages/llm/llm-pi-ai/tests/convert.spec.ts b/packages/llm/llm-pi-ai/tests/convert.spec.ts index 016ee2ef24..2a3b41b0c4 100644 --- a/packages/llm/llm-pi-ai/tests/convert.spec.ts +++ b/packages/llm/llm-pi-ai/tests/convert.spec.ts @@ -785,6 +785,10 @@ describe('mapStopReason / mapUsage', () => { stopReason: 'error', errorMessage: 'Failed to buffer the request body: length limit exceeded', }))).toMatchObject({ kind: 'error', failure: { code: 'INVALID_REQUEST' } }) + expect(mapStopReason(assistant({ + stopReason: 'error', + errorMessage: 'vector length limit exceeded', + }))).toMatchObject({ kind: 'error', failure: { code: 'PI_AI_ERROR' } }) }) it.each([