Merge pull request #6108 from deepseek-harness/fix/empty-response-text
fix(llm): 保留跨协议完整历史并过滤 DeepSeek 请求空文本
This commit is contained in:
commit
a29f5ee1e3
15 files changed
+325
-14
No files matched your search
@@ -24,8 +24,8 @@
|
||||
en: cfc2de7ef728dbf9
|
||||
zh: 2ce1a406061a71aa
|
||||
/deepseek-ai-dsh-llm-deepseek/use-this-package/endpoint-and-wire-format:
|
||||
en: fce88d2ebad6296c
|
||||
zh: 0c351f26b472d521
|
||||
en: 522e74b770e9d4ca
|
||||
zh: a8b261b41677bfe7
|
||||
/deepseek-ai-dsh-llm-deepseek/use-this-package/account-credentials:
|
||||
en: 3853ba1a1ffbb46c
|
||||
zh: 97203e80ad933e59
|
||||
|
||||
@@ -82,7 +82,7 @@ When [proactive compaction](../../compaction/compaction-basic/README.md#use-this
|
||||
|
||||
The official root is `https://api.deepseek.com/anthropic`. An explicit `baseURL` or `$DEEPSEEK_BASE_URL` supplies a Messages-compatible root. Model and Files requests append `/v1/messages` and `/v1/files`, except that an exact final `/v1` segment is reused. Trailing slashes do not change these results. A base URL must use HTTP(S) without credentials, query, or fragment.
|
||||
|
||||
Messages sends text, thinking, tool calls, and tool results as content blocks, reasoning effort as `output_config.effort`, and images as Files references or inline base64. Models declaring `systemPromptUpdate: in-history` retain the initial top-level system and send new system snapshots after their corresponding user/tool-result turn; undeclared models use the latest snapshot as the top-level system. Replay metadata preserves the model and thinking signatures. Invalid replay metadata emits a warning and omits signatures while retaining text and tool history. Model entries may declare `toolUpdate: addition-only` or `in-history`; the default `deepseek-flash` entry declares `addition-only`. Projected developer tool updates become system-role `tool_addition` and `tool_removal` blocks referencing declared names, and deferred declarations carry `defer_loading`. Requests containing those blocks send the `mid-conversation-tool-changes-2026-07-01` beta header.
|
||||
Messages sends text, thinking, tool calls, and tool results as content blocks, reasoning effort as `output_config.effort`, and images as Files references or inline base64. Models declaring `systemPromptUpdate: in-history` retain the initial top-level system and send new system snapshots after their corresponding user/tool-result turn; undeclared models use the latest snapshot as the top-level system. Replay metadata preserves the model and thinking signatures. Invalid replay metadata emits a warning and omits signatures while retaining text and tool history. Model entries may declare `toolUpdate: addition-only` or `in-history`; the default `deepseek-flash` entry declares `addition-only`. Projected developer tool updates become system-role `tool_addition` and `tool_removal` blocks referencing declared names, and deferred declarations carry `defer_loading`. Requests containing those blocks send the `mid-conversation-tool-changes-2026-07-01` beta header. Messages omits empty assistant text blocks and skips assistant messages left without content, including saved history from other protocols. The stored Session content remains intact.
|
||||
|
||||
### Account credentials
|
||||
|
||||
|
||||
@@ -82,7 +82,7 @@ kind: "package-reference"
|
||||
|
||||
官方根地址为 `https://api.deepseek.com/anthropic`。显式 `baseURL` 或 `$DEEPSEEK_BASE_URL` 提供兼容 Messages 的根地址。模型与 Files 请求分别追加 `/v1/messages` 和 `/v1/files`,但末尾严格匹配的 `/v1` 路径段会直接复用。末尾斜线不改变这些结果。基址必须使用 HTTP(S),且不含凭据、查询或片段。
|
||||
|
||||
Messages 以内容块发送文本、思考、工具调用和工具结果,以 `output_config.effort` 发送推理强度,并以 Files 引用或内联 base64 发送图片。声明 `systemPromptUpdate: in-history` 的模型保留初始顶层 system,在对应 user/tool-result 轮次之后发送新的 system 快照;未声明能力时,使用最新快照作为顶层 system。回放元数据保留模型与思考签名。无效的回放元数据产生警告并省略签名,不丢弃文本或工具历史。 模型条目可声明 `toolUpdate: addition-only` 或 `in-history`;默认 `deepseek-flash` 条目声明 `addition-only`。投影后的 developer 工具更新转换为 system 角色的 `tool_addition` 和 `tool_removal` 块,引用已声明名称,延迟声明携带 `defer_loading`。包含这些块的请求发送 `mid-conversation-tool-changes-2026-07-01` beta 请求头。
|
||||
Messages 以内容块发送文本、思考、工具调用和工具结果,以 `output_config.effort` 发送推理强度,并以 Files 引用或内联 base64 发送图片。声明 `systemPromptUpdate: in-history` 的模型保留初始顶层 system,在对应 user/tool-result 轮次之后发送新的 system 快照;未声明能力时,使用最新快照作为顶层 system。回放元数据保留模型与思考签名。无效的回放元数据产生警告并省略签名,不丢弃文本或工具历史。 模型条目可声明 `toolUpdate: addition-only` 或 `in-history`;默认 `deepseek-flash` 条目声明 `addition-only`。投影后的 developer 工具更新转换为 system 角色的 `tool_addition` 和 `tool_removal` 块,引用已声明名称,延迟声明携带 `defer_loading`。包含这些块的请求发送 `mid-conversation-tool-changes-2026-07-01` beta 请求头。 Messages 会省略 assistant 的空文本块,并跳过转换后没有内容的 assistant 消息,包括其他协议保存的历史。已存储的 Session 内容保持原样。
|
||||
|
||||
### 账号凭据
|
||||
|
||||
|
||||
@@ -25,9 +25,9 @@ function toolInput(raw: string): Record<string, unknown> {
|
||||
|
||||
function assistant(message: Message, model: string, onReplayDegrade?: (reason: string) => void): WireBlock[] {
|
||||
const replay = readReplay(message, model, onReplayDegrade)
|
||||
return message.content.map((block, index): WireBlock => {
|
||||
return message.content.flatMap((block, index): WireBlock | WireBlock[] => {
|
||||
switch (block.type) {
|
||||
case 'text': return { type: 'text', text: block.text }
|
||||
case 'text': return block.text.length === 0 ? [] : { type: 'text', text: block.text }
|
||||
case 'reasoning': return {
|
||||
type: 'thinking', thinking: block.text,
|
||||
...replay?.[index]?.signature === undefined ? {} : { signature: replay[index].signature },
|
||||
@@ -40,7 +40,8 @@ function assistant(message: Message, model: string, onReplayDegrade?: (reason: s
|
||||
|
||||
/** Serialize one complete request using already prepared image bytes.
|
||||
* User and tool-result content omits reasoning and tool-call blocks.
|
||||
* Empty user messages are skipped; empty tool results retain their call ids.
|
||||
* Empty assistant text blocks and empty user/assistant messages are skipped;
|
||||
* empty tool results retain their call ids.
|
||||
* Developer messages, already projected for this route by `LlmRuntime`, become
|
||||
* system-role updates after the preceding user turn, with tool changes as
|
||||
* `tool_addition` and `tool_removal` references to the declared tool.
|
||||
@@ -114,13 +115,13 @@ export function serialize(
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (message.role === 'assistant') flushSystemUpdates()
|
||||
const content: WireBlock[] = message.role === 'assistant'
|
||||
? assistant(message, options.model, onReplayDegrade)
|
||||
: message.role === 'tool'
|
||||
? [{ type: 'tool_result', tool_use_id: message.toolCallId, content: input(message.content), ...message.isError === undefined ? {} : { is_error: message.isError } }]
|
||||
: message.content.flatMap((block): WireBlock[] => input([block]))
|
||||
if (message.role === 'user' && content.length === 0) continue
|
||||
if ((message.role === 'user' || message.role === 'assistant') && content.length === 0) continue
|
||||
if (message.role === 'assistant') flushSystemUpdates()
|
||||
const wireRole = message.role === 'tool' ? 'user' : message.role
|
||||
const previous = messages.at(-1)
|
||||
if (previous?.role === wireRole) previous.content.push(...content)
|
||||
|
||||
@@ -239,6 +239,24 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('DeepSeek Messages real API', ()
|
||||
expect(third.assembler.finish.kind).toBe('stop')
|
||||
})
|
||||
|
||||
it('continues saved Responses tool history with an empty trailing text item', async () => {
|
||||
const ctx = await boot()
|
||||
const calls = [0, 1, 2].map(index => ({
|
||||
type: 'tool-call' as const, id: ToolCallId(`call_${index}|item_${index}`), name: 'lookup_value', arguments: '{"key":"secret"}',
|
||||
}))
|
||||
const history = [
|
||||
user('Look up the secret value.'),
|
||||
createAssistantMessage({ source: { provider: 'foreign', model: 'claude' }, content: [...calls, { type: 'text', text: '' }] }),
|
||||
...calls.map(call => createToolResultMessage({ callId: call.id, content: [{ type: 'text', text: 'EMPTY_TEXT_RECOVERED_731' }], isError: false })),
|
||||
user('Do not call tools. Reply with exactly EMPTY_TEXT_RECOVERED_731.'),
|
||||
]
|
||||
const saved = JSON.stringify(history)
|
||||
const response = await assemble(ctx.llm.stream(options({ model: 'deepseek-flash', messages: history, tools: [tool], reasoningEffort: ReasoningEffortId('off') })))
|
||||
expect(response.assembler.finish.kind).toBe('stop')
|
||||
expect(response.message.content.filter(block => block.type === 'text').map(block => block.text).join('')).toContain('EMPTY_TEXT_RECOVERED_731')
|
||||
expect(JSON.stringify(history)).toBe(saved)
|
||||
})
|
||||
|
||||
it('continues persisted foreign history containing malformed tool arguments', async () => {
|
||||
const ctx = await boot()
|
||||
const callId = ToolCallId('historical_lookup')
|
||||
|
||||
+4
-1
@@ -9,7 +9,10 @@ export const inject = ['llm']
|
||||
export function apply(ctx: Context): void {
|
||||
const connection = resolveAdapterOptions({})
|
||||
ctx.on('llm/stream', (options, next) => {
|
||||
serialize(options, connection, options.messages, new Map(), () => undefined)
|
||||
const request = serialize(options, connection, options.messages, new Map(), () => undefined)
|
||||
if (request.messages.some(message => message.content.some(block => block.type === 'text' && block.text.length === 0))) {
|
||||
throw new Error('Messages history contains an empty text block')
|
||||
}
|
||||
return next()
|
||||
})
|
||||
}
|
||||
@@ -23,6 +23,40 @@ const capable = resolveAdapterOptions({ models: [{ id: MODEL, systemPromptUpdate
|
||||
const nativeBody = (messages: Message[]) => serialize(options({ messages }), capable, messages, new Map(), () => undefined)
|
||||
|
||||
describe('Messages request conversion', () => {
|
||||
it('omits saved empty assistant text without changing tool ids or thinking signatures', () => {
|
||||
const id = 'toolu_example|toolu_example'
|
||||
const content: ContentBlock[] = [
|
||||
{ type: 'text', text: '' }, { type: 'reasoning', text: 'reason' }, call(id), { type: 'text', text: '' },
|
||||
]
|
||||
const message = createAssistantMessage({ content, source: {
|
||||
provider: 'deepseek-official', model: MODEL,
|
||||
replayState: replayState(MODEL, [{ type: 'text' }, { type: 'reasoning', signature: 'signed' }, { type: 'tool-call' }, { type: 'text' }]),
|
||||
} })
|
||||
const request = body([user(), assistant([{ type: 'text', text: '' }]), message, result(id)])
|
||||
expect(request.messages).toEqual([
|
||||
{ role: 'user', content: [{ type: 'text', text: 'hello' }] },
|
||||
{ role: 'assistant', content: [
|
||||
{ type: 'thinking', thinking: 'reason', signature: 'signed' },
|
||||
{ type: 'tool_use', id, name: 'read', input: { path: 'a' } },
|
||||
] },
|
||||
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: id, content: [{ type: 'text', text: 'result' }], is_error: false }] },
|
||||
])
|
||||
expect(message.content).toEqual(content)
|
||||
expect(body([assistant([{ type: 'text', text: ' ' }])]).messages[0]?.content).toEqual([{ type: 'text', text: ' ' }])
|
||||
})
|
||||
|
||||
it('does not flush a system update at an omitted empty assistant message', () => {
|
||||
const request = nativeBody([
|
||||
user(), createSystemMessage('updated'), assistant([{ type: 'text', text: '' }]),
|
||||
user(), assistant([{ type: 'text', text: 'answer' }]),
|
||||
])
|
||||
expect(request.messages).toEqual([
|
||||
{ role: 'user', content: [{ type: 'text', text: 'hello' }, { type: 'text', text: 'hello' }] },
|
||||
{ role: 'system', content: [{ type: 'text', text: 'updated' }] },
|
||||
{ role: 'assistant', content: [{ type: 'text', text: 'answer' }] },
|
||||
])
|
||||
})
|
||||
|
||||
it('rejects unknown plugin content without interpreting its payload', () => {
|
||||
expect(() => body([createUserMessage({ source: { kind: 'user' }, content: [
|
||||
{ type: 'plugin:text', text: 'opaque' } as never,
|
||||
|
||||
@@ -54,8 +54,8 @@
|
||||
en: e9aa65cd3417fc05
|
||||
zh: 4568471e712b055e
|
||||
/deepseek-ai-dsh-llm-pi-ai/understand-the-implementation/replay-and-vocabulary:
|
||||
en: 36e0c59a0f9f5980
|
||||
zh: 90a62c4a9cd23429
|
||||
en: 73242ca9b8a98c76
|
||||
zh: 4ab98526ae70bd23
|
||||
/deepseek-ai-dsh-llm-pi-ai/further-exploration:
|
||||
en: a2a2ccdeae59d408
|
||||
zh: c47ee091aa90ad91
|
||||
|
||||
@@ -163,7 +163,7 @@ The plugin declares every installed catalog provider it can authenticate in the
|
||||
|
||||
### Replay and vocabulary
|
||||
|
||||
Successful assistant responses store a versioned, lossless-JSON replay state beside the provider and model that produced them — response-level facts plus one per-block entry per streamed block. At request time, `LlmRuntime` passes replay state only when the same adapter instance owns both routes; the adapter validates it and restores native response ids, provider signatures, and optional `providerThinkingLevel` effort metadata, keeping absent effort metadata absent. Replay validates the requested model identity against the assistant source and uses it to retain same-model signatures; provider-reported response models remain diagnostic metadata. Absent or unusable replay state degrades to provider-neutral content while preserving the assistant source’s required provider and model. pi-ai tool-call arguments are parsed objects, so the adapter parses input and re-stringifies output to the harness raw-JSON convention; pi-ai in-stream error events map to terminal `finish` chunks.
|
||||
Successful assistant responses store a versioned, lossless-JSON replay state beside the provider and model that produced them — response-level facts plus one per-block entry per streamed block. At request time, `LlmRuntime` passes replay state only when the same adapter instance owns both routes; the adapter validates it and restores native response ids, provider signatures, and optional `providerThinkingLevel` effort metadata, keeping absent effort metadata absent. Replay validates the requested model identity against the assistant source and uses it to retain same-model signatures; provider-reported response models remain diagnostic metadata. Absent or unusable replay state degrades to provider-neutral content while preserving the assistant source’s required provider and model. pi-ai tool-call arguments are parsed objects, so the adapter parses input and re-stringifies output to the harness raw-JSON convention; pi-ai in-stream error events map to terminal `finish` chunks. Protocol conversion operates on request-local copies. Switching from A to B and back to A preserves the stored content, including empty text blocks, tool ids, original block order, and native replay metadata; unsupported signatures are omitted only from the foreign request.
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
@@ -163,7 +163,7 @@ Config 更新严格验证发生变化的 provider。初始加载将已存储的
|
||||
|
||||
### 回放与词汇
|
||||
|
||||
成功 assistant 响应会存储带版本的、无损 JSON 回放状态,与产生它们的提供方和模型放在一起——响应级事实加每个流式块一条逐块条目。请求时,`LlmRuntime` 仅当同一适配器实例拥有两条路由时才传递回放状态;适配器校验它并恢复原生响应 id、提供方签名与可选的 `providerThinkingLevel` effort 元数据,缺失的 effort 元数据仍保持缺失。回放会对照 assistant 来源校验请求模型身份,并用它保留同模型签名;提供方报告的响应模型仅作为诊断元数据。回放状态缺失或无法使用时会降级为提供方无关内容,并保留 assistant 来源中必需的提供方和模型。pi-ai 工具调用参数是解析后的对象,因此适配器解析输入并重新字符串化输出,以符合 harness 原始 JSON 约定;pi-ai 流内错误事件映射为终止 `finish` 分片。
|
||||
成功 assistant 响应会存储带版本的、无损 JSON 回放状态,与产生它们的提供方和模型放在一起——响应级事实加每个流式块一条逐块条目。请求时,`LlmRuntime` 仅当同一适配器实例拥有两条路由时才传递回放状态;适配器校验它并恢复原生响应 id、提供方签名与可选的 `providerThinkingLevel` effort 元数据,缺失的 effort 元数据仍保持缺失。回放会对照 assistant 来源校验请求模型身份,并用它保留同模型签名;提供方报告的响应模型仅作为诊断元数据。回放状态缺失或无法使用时会降级为提供方无关内容,并保留 assistant 来源中必需的提供方和模型。pi-ai 工具调用参数是解析后的对象,因此适配器解析输入并重新字符串化输出,以符合 harness 原始 JSON 约定;pi-ai 流内错误事件映射为终止 `finish` 分片。 协议转换仅作用于请求副本。从 A 切换到 B 再切回 A 时,存储的内容保持完整,包括空文本块、工具 id、原始块顺序和原生回放元数据;不受支持的签名仅从发给其他协议的请求中省略。
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
/** Cross-protocol requests must not overwrite the history needed by its original provider. */
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { Context } from '@deepseek-ai/cordis'
|
||||
import LlmRuntime, { createToolResultMessage, createUserMessage } from '@deepseek-ai/dsh-llm'
|
||||
import type { Message } from '@deepseek-ai/dsh-llm'
|
||||
import * as LlmPiAi from '@deepseek-ai/dsh-llm-pi-ai'
|
||||
import * as DeepSeek from '@deepseek-ai/dsh-llm-deepseek-api-key'
|
||||
import { assemble } from './assemble.ts'
|
||||
import { anthropicFrame, closeMockServers, mockServer } from './mock-server.ts'
|
||||
|
||||
type Api = 'anthropic-messages' | 'openai-responses' | 'openai-completions'
|
||||
const apis: Api[] = ['anthropic-messages', 'openai-responses', 'openai-completions']
|
||||
const contexts: Context[] = []
|
||||
afterEach(async () => {
|
||||
await Promise.all(contexts.splice(0).map(ctx => ctx.fiber.dispose()))
|
||||
await closeMockServers()
|
||||
vi.unstubAllEnvs()
|
||||
})
|
||||
|
||||
function responsesFrames(tools: boolean): string[] {
|
||||
const items = tools ? [
|
||||
{ type: 'reasoning', id: 'rs_original', encrypted_content: 'encrypted-reasoning', summary: [{ type: 'summary_text', text: 'source reasoning' }] },
|
||||
{ type: 'message', id: 'msg_original', role: 'assistant', content: [{ type: 'output_text', text: 'source text', annotations: [] }] },
|
||||
...[0, 1].map(index => ({ type: 'function_call', id: `fc_${index}`, call_id: `call_${index}`, name: 'lookup', arguments: JSON.stringify({ key: index }) })),
|
||||
{ type: 'message', id: 'msg_empty', role: 'assistant', content: [{ type: 'output_text', text: '', annotations: [] }] },
|
||||
] : [{ type: 'message', id: 'msg_reply', role: 'assistant', content: [{ type: 'output_text', text: 'reply', annotations: [] }] }]
|
||||
return [
|
||||
JSON.stringify({ type: 'response.created', response: { id: 'resp_original' } }),
|
||||
...items.flatMap((item, output_index) => [
|
||||
JSON.stringify({ type: 'response.output_item.added', output_index, item }),
|
||||
JSON.stringify({ type: 'response.output_item.done', output_index, item }),
|
||||
]),
|
||||
JSON.stringify({ type: 'response.completed', response: {
|
||||
id: 'resp_original', status: 'completed', output: items, usage: { input_tokens: 3, output_tokens: 1, total_tokens: 4 },
|
||||
} }),
|
||||
]
|
||||
}
|
||||
|
||||
function anthropicFrames(tools: boolean): ReturnType<typeof anthropicFrame>[] {
|
||||
const blocks = tools ? [
|
||||
{ type: 'thinking', thinking: 'source reasoning', signature: 'signed-reasoning' },
|
||||
{ type: 'redacted_thinking', data: 'encrypted-redacted-reasoning' },
|
||||
{ type: 'text', text: 'source text' },
|
||||
...[0, 1].map(index => ({ type: 'tool_use', id: `toolu_${index}`, name: 'lookup', input: { key: index } })),
|
||||
{ type: 'text', text: '' },
|
||||
] : [{ type: 'text', text: 'reply' }]
|
||||
return [
|
||||
anthropicFrame({ type: 'message_start', message: {
|
||||
id: 'msg_original', type: 'message', role: 'assistant', model: 'model', content: [],
|
||||
stop_reason: null, stop_sequence: null, usage: { input_tokens: 3, output_tokens: 1 },
|
||||
} }),
|
||||
...blocks.flatMap((content_block, index) => [
|
||||
anthropicFrame({ type: 'content_block_start', index, content_block }),
|
||||
anthropicFrame({ type: 'content_block_stop', index }),
|
||||
]),
|
||||
anthropicFrame({ type: 'message_delta', delta: { stop_reason: tools ? 'tool_use' : 'end_turn', stop_sequence: null }, usage: { output_tokens: 1 } }),
|
||||
anthropicFrame({ type: 'message_stop' }),
|
||||
]
|
||||
}
|
||||
|
||||
function frames(api: Api, tools = false): (string | ReturnType<typeof anthropicFrame>)[] {
|
||||
if (api === 'openai-responses') return responsesFrames(tools)
|
||||
if (api === 'anthropic-messages') return anthropicFrames(tools)
|
||||
return [
|
||||
JSON.stringify({ choices: [{ index: 0, delta: {
|
||||
role: 'assistant', content: tools ? 'source text' : 'reply',
|
||||
...tools ? {
|
||||
reasoning_content: 'source reasoning',
|
||||
tool_calls: [0, 1].map(index => ({ index, id: `call_${index}`, type: 'function', function: { name: 'lookup', arguments: JSON.stringify({ key: index }) } })),
|
||||
} : {},
|
||||
}, finish_reason: null }] }),
|
||||
JSON.stringify({ choices: [{ index: 0, delta: {}, finish_reason: tools ? 'tool_calls' : 'stop' }], usage: { prompt_tokens: 3, completion_tokens: 1 } }),
|
||||
'[DONE]',
|
||||
]
|
||||
}
|
||||
|
||||
function wireHistory(request: unknown, api: Api): unknown[] {
|
||||
expect(request).toBeTypeOf('object')
|
||||
const history = (request as Record<string, unknown>)[api === 'openai-responses' ? 'input' : 'messages']
|
||||
if (!Array.isArray(history)) throw new Error('Expected a wire message array')
|
||||
return history
|
||||
}
|
||||
|
||||
function expectToolPairs(request: unknown, api: Api): void {
|
||||
const messages = wireHistory(request, api) as Record<string, unknown>[]
|
||||
const calls: unknown[] = []
|
||||
const results: unknown[] = []
|
||||
for (const message of messages) {
|
||||
if (api === 'openai-responses') {
|
||||
if (message['type'] === 'function_call') calls.push(message['call_id'])
|
||||
if (message['type'] === 'function_call_output') results.push(message['call_id'])
|
||||
} else if (api === 'openai-completions') {
|
||||
for (const call of (message['tool_calls'] ?? []) as Record<string, unknown>[]) calls.push(call['id'])
|
||||
if (message['role'] === 'tool') results.push(message['tool_call_id'])
|
||||
} else if (Array.isArray(message['content'])) {
|
||||
for (const block of message['content'] as Record<string, unknown>[]) {
|
||||
if (block['type'] === 'tool_use') calls.push(block['id'])
|
||||
if (block['type'] === 'tool_result') results.push(block['tool_use_id'])
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(calls).toHaveLength(2)
|
||||
expect(calls.every(id => typeof id === 'string' && id.length > 0)).toBe(true)
|
||||
expect(new Set(calls).size).toBe(2)
|
||||
expect(results).toEqual(calls)
|
||||
}
|
||||
|
||||
const tool = { name: 'lookup', description: 'Look up a value.', parameters: { type: 'object', properties: { key: { type: 'number' } }, required: ['key'] } }
|
||||
const pairs: { a: Api; b: Api | 'deepseek-messages' }[] = [
|
||||
...apis.flatMap(a => apis.filter(b => b !== a).map(b => ({ a, b }))),
|
||||
...apis.map(a => ({ a, b: 'deepseek-messages' as const })),
|
||||
]
|
||||
|
||||
describe('protocol history round trips', () => {
|
||||
it.each(pairs)('$a → $b → $a preserves saved content and restores native replay', async ({ a, b }) => {
|
||||
vi.stubEnv('PROTOCOL_ROUNDTRIP_KEY', 'test-key')
|
||||
const source = await mockServer([{ events: frames(a, true) }, { events: frames(a) }, { events: frames(a) }])
|
||||
const targetApi = b === 'deepseek-messages' ? 'anthropic-messages' : b
|
||||
const targetProvider = b === 'deepseek-messages' ? 'deepseek-official' : 'route-b'
|
||||
const target = await mockServer([{ events: frames(targetApi) }])
|
||||
const ctx = new Context()
|
||||
contexts.push(ctx)
|
||||
await ctx.plugin(LlmRuntime)
|
||||
await ctx.plugin(LlmPiAi, { providers: {
|
||||
'route-a': { api: a, baseURL: source.url, apiKeyEnv: 'PROTOCOL_ROUNDTRIP_KEY', cacheRetention: 'none', models: [{ id: 'model' }] },
|
||||
...b === 'deepseek-messages' ? {} : {
|
||||
'route-b': { api: b, baseURL: target.url, apiKeyEnv: 'PROTOCOL_ROUNDTRIP_KEY', cacheRetention: 'none', models: [{ id: 'model' }] },
|
||||
},
|
||||
} })
|
||||
if (b === 'deepseek-messages') await ctx.plugin(DeepSeek, {
|
||||
baseURL: target.url, apiKeyEnv: 'PROTOCOL_ROUNDTRIP_KEY', models: [{ id: 'model' }], thinking: 'disabled',
|
||||
})
|
||||
let history: Message[] = [createUserMessage({ source: { kind: 'user' }, content: [{ type: 'text', text: 'Look up both values.' }] })]
|
||||
const original = await assemble(ctx, { provider: 'route-a', model: 'model', messages: history, tools: [tool] })
|
||||
expect(original.finish).toEqual({ kind: 'tool-calls' })
|
||||
const calls = original.message.content.filter(block => block.type === 'tool-call')
|
||||
expect(calls).toHaveLength(2)
|
||||
if (a !== 'openai-completions') expect(original.message.content.at(-1)).toEqual({ type: 'text', text: '' })
|
||||
expect(original.message.source).toMatchObject({ replayState: { response: { api: a, provider: 'route-a', model: 'model' } } })
|
||||
if (a === 'anthropic-messages') {
|
||||
const blocks: unknown = expect.arrayContaining([
|
||||
{ type: 'reasoning', thinkingSignature: 'signed-reasoning' },
|
||||
{ type: 'reasoning', thinkingSignature: 'encrypted-redacted-reasoning', redacted: true },
|
||||
])
|
||||
expect(original.message.source).toMatchObject({ replayState: { blocks } })
|
||||
} else if (a === 'openai-responses') {
|
||||
const thinkingSignature: unknown = expect.stringContaining('encrypted-reasoning')
|
||||
const blocks: unknown = expect.arrayContaining([
|
||||
{ type: 'reasoning', thinkingSignature },
|
||||
{ type: 'text', textSignature: JSON.stringify({ v: 1, id: 'msg_empty' }) },
|
||||
])
|
||||
expect(original.message.source).toMatchObject({ replayState: { response: { responseId: 'resp_original' }, blocks } })
|
||||
}
|
||||
history.push(original.message, ...calls.map(call => createToolResultMessage({
|
||||
callId: call.id, content: [{ type: 'text', text: 'value' }], isError: false,
|
||||
})))
|
||||
const saved = JSON.stringify(history)
|
||||
history = JSON.parse(saved) as Message[]
|
||||
|
||||
// Capture A's request before any foreign conversion, then use the same saved history for B.
|
||||
expect((await assemble(ctx, { provider: 'route-a', model: 'model', messages: history, tools: [tool] })).finish).toEqual({ kind: 'stop' })
|
||||
const baseline = wireHistory(source.requests[1], a)
|
||||
expectToolPairs(source.requests[1], a)
|
||||
if (a === 'anthropic-messages') {
|
||||
expect(JSON.stringify(baseline)).toContain('signed-reasoning')
|
||||
expect(JSON.stringify(baseline)).toContain('encrypted-redacted-reasoning')
|
||||
} else if (a === 'openai-responses') {
|
||||
expect(JSON.stringify(baseline)).toContain('encrypted-reasoning')
|
||||
expect(JSON.stringify(baseline)).toContain('fc_0')
|
||||
}
|
||||
const foreign = await assemble(ctx, { provider: targetProvider, model: 'model', messages: history, tools: [tool] })
|
||||
expect(foreign.finish).toEqual({ kind: 'stop' })
|
||||
expect(JSON.stringify(history)).toBe(saved)
|
||||
expectToolPairs(target.requests[0], targetApi)
|
||||
expect(JSON.stringify(target.requests[0])).not.toContain('encrypted-reasoning')
|
||||
expect(JSON.stringify(target.requests[0])).not.toContain('encrypted-redacted-reasoning')
|
||||
expect(JSON.stringify(target.requests[0])).not.toContain('signed-reasoning')
|
||||
history.push(foreign.message, createUserMessage({ source: { kind: 'user' }, content: [{ type: 'text', text: 'Continue with the original model.' }] }))
|
||||
history = JSON.parse(JSON.stringify(history)) as Message[]
|
||||
expect(history.slice(0, -2)).toEqual(JSON.parse(saved))
|
||||
expect((await assemble(ctx, { provider: 'route-a', model: 'model', messages: history, tools: [tool] })).finish).toEqual({ kind: 'stop' })
|
||||
expect(wireHistory(source.requests[2], a).slice(0, baseline.length)).toEqual(baseline)
|
||||
expect(history.slice(0, -2)).toEqual(JSON.parse(saved))
|
||||
expect(source.requests).toHaveLength(3)
|
||||
expect(target.requests).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
@@ -3,6 +3,8 @@ import type { AssistantMessageEvent, Model } from '@earendil-works/pi-ai'
|
||||
import { stream as streamAnthropic } from '@earendil-works/pi-ai/api/anthropic-messages'
|
||||
import { stream as streamCompletions } from '@earendil-works/pi-ai/api/openai-completions'
|
||||
import { stream as streamResponses } from '@earendil-works/pi-ai/api/openai-responses'
|
||||
import { BlockAssembler } from '@deepseek-ai/dsh-llm'
|
||||
import { toStreamChunks } from '../src/stream.ts'
|
||||
import { normalizeContext } from '@earendil-works/pi-ai/utils/transcript'
|
||||
import { anthropicFrame as frame, anthropicMessageStart, closeMockServers, mockServer } from './mock-server.ts'
|
||||
|
||||
@@ -40,6 +42,35 @@ async function collect(events: AsyncIterable<AssistantMessageEvent>): Promise<{
|
||||
const context = normalizeContext({ messages: [{ role: 'user', content: 'hi', timestamp: 0 }] })
|
||||
|
||||
describe('streamed tool-call arguments (patched pi-ai)', () => {
|
||||
it('preserves an empty Responses message after parallel tool calls with its replay metadata', async () => {
|
||||
const calls = [0, 1, 2].map(index => ({ type: 'function_call', call_id: `call_${index}`, id: `item_${index}`, name: 'read', arguments: '{}' }))
|
||||
const empty = { type: 'message', id: 'message_empty', role: 'assistant', content: [{ type: 'output_text', text: '', annotations: [] }] }
|
||||
const server = await mockServer([{ events: [
|
||||
'{"type":"response.created","response":{"id":"response_empty_text"}}',
|
||||
...[...calls, empty].flatMap((item, output_index) => [
|
||||
JSON.stringify({ type: 'response.output_item.added', output_index, item }),
|
||||
JSON.stringify({ type: 'response.output_item.done', output_index, item }),
|
||||
]),
|
||||
JSON.stringify({ type: 'response.completed', response: {
|
||||
id: 'response_empty_text', status: 'completed', output: [...calls, empty], usage: { input_tokens: 3, output_tokens: 1, total_tokens: 4 },
|
||||
} }),
|
||||
] }])
|
||||
const assembler = new BlockAssembler()
|
||||
for await (const chunk of toStreamChunks(streamResponses(model('openai-responses', server.url), context, { apiKey: 'test-key' }))) {
|
||||
assembler.push(chunk)
|
||||
}
|
||||
expect(assembler.blocks()).toEqual([
|
||||
...calls.map(call => ({ type: 'tool-call', id: `${call.call_id}|${call.id}`, name: 'read', arguments: '{}' })),
|
||||
{ type: 'text', text: '' },
|
||||
])
|
||||
expect(assembler.replayState?.blocks).toEqual([
|
||||
...calls.map(() => ({ type: 'tool-call' })),
|
||||
{ type: 'text', textSignature: JSON.stringify({ v: 1, id: 'message_empty' }) },
|
||||
])
|
||||
expect(assembler.finish).toEqual({ kind: 'tool-calls' })
|
||||
expect(server.requests).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('openai-completions parses arguments once, at the end of the call', async () => {
|
||||
const server = await mockServer([{ events: [
|
||||
JSON.stringify({ choices: [{
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
- insert:
|
||||
- id: messages-history-snapshot
|
||||
name: '../../../packages/llm/llm-deepseek/tests/fixtures/history.ts'
|
||||
@@ -0,0 +1,24 @@
|
||||
{"type":"session","version":4,"id":"{{session:1}}","createdAt":1783352044766,"cwd":"{{cwd}}","isSeeded":false,"delegationDepth":0}
|
||||
{"type":"permission/preset","data":{"preset":"danger-full-access"}}
|
||||
{"type":"sandbox/mode","data":{"mode":"danger-full-access"}}
|
||||
{"type":"approval/policy","data":{"policy":"never"}}
|
||||
{"type":"agent/inbox/spliced","data":{"target":"next-turn","start":0,"inserted":[{"content":[{"type":"text","text":"Use the bash tool to run exactly: echo SNAPSHOT_OK. Then reply with the single word DONE and stop."}],"source":{"kind":"user"},"role":"user","id":"{{message:1}}"}]}}
|
||||
{"type":"turn/start","data":{"turn":1}}
|
||||
{"type":"agent/inbox/spliced","data":{"target":"next-turn","start":0,"removedCount":1,"inserted":[]}}
|
||||
{"type":"step/start","data":{"turn":1,"step":1}}
|
||||
{"type":"system/message","data":{"turn":1,"step":1,"message":{"role":"system","content":[{"type":"text","text":"{{system}}"}],"source":{"kind":"system-prompt"},"id":"{{message:2}}"}},"surfaceOp":"append"}
|
||||
{"type":"user/message","data":{"content":[{"type":"text","text":"Use the bash tool to run exactly: echo SNAPSHOT_OK. Then reply with the single word DONE and stop."}],"source":{"kind":"user"},"role":"user","id":"{{message:1}}"},"surfaceOp":"append"}
|
||||
{"type":"user/message","data":{"content":[{"type":"text","text":"Current runtime context. This snapshot supersedes earlier runtime-context snapshots.\n\nCurrent working directory: \"{{cwd}}\".\n\nCurrent DSH file policy: danger-full-access. The DSH file sandbox does not restrict file modifications by available operations.\n\nApproval prompts are disabled in this session: actions that require approval are rejected automatically — do not request sandbox escalation (do not set `sandbox_permissions`)."}],"source":{"kind":"runtime-context","form":"snapshot","sections":[{"name":"working-directory:current","text":"Current working directory: \"{{cwd}}\"."},{"name":"sandbox:policy","text":"Current DSH file policy: danger-full-access. The DSH file sandbox does not restrict file modifications by available operations."},{"name":"approval:policy","text":"Approval prompts are disabled in this session: actions that require approval are rejected automatically — do not request sandbox escalation (do not set `sandbox_permissions`)."}]},"role":"user","id":"{{message:3}}"},"surfaceOp":"append"}
|
||||
{"type":"request/header","data":{"header":{"config":{"provider":"deepseek-official","model":"deepseek-v4-flash"},"tools":"{{tools}}"},"reason":"initial"}}
|
||||
{"type":"request/context","data":{"provider":"deepseek-official","model":"deepseek-v4-flash"}}
|
||||
{"type":"session/title","data":{"title":"Use the bash tool to","messageSeqs":[8],"source":{"kind":"fallback"}}}
|
||||
{"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"reasoning","text":"The user wants me to run a specific bash command and then reply with DONE."},{"type":"tool-call","id":"call_00_Rn2Mz1y8uZN62ukEXiNO2077","name":"bash","arguments":"{\"command\": \"echo SNAPSHOT_OK\", \"description\": \"Run echo SNAPSHOT_OK\"}"},{"type":"text","text":""}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash"},"id":"{{message:4}}"},"usage":{"inputTokens":2879,"outputTokens":89,"cacheReadTokens":0,"reasoningTokens":17},"stream":[{"type":"chunk","time":1791427900979,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}},{"type":"reasoning-chunks","time0":1791427900979,"index":0,"dt":[0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"texts":["The"," user"," wants"," me"," to"," run"," a"," specific"," bash"," command"," and"," then"," reply"," with"," D","ONE","."]},{"type":"chunk","time":1791427900980,"chunk":{"type":"block-start","index":1,"blockType":"tool-call"}},{"type":"tool-call-chunks","time0":1791427900980,"index":1,"dt":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"id":"call_00_Rn2Mz1y8uZN62ukEXiNO2077","name":"bash","args":["","{","\"","command","\"",": ","\"","echo"," S","NA","PS","H","OT","_OK","\"",", ","\"","description","\"",": ","\"","Run"," echo"," S","NA","PS","H","OT","_OK","\"","}"]},{"type":"chunk","time":1791427900980,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user wants me to run a specific bash command and then reply with DONE."}}},{"type":"chunk","time":1791427900980,"chunk":{"type":"block-end","index":1,"block":{"type":"tool-call","id":"call_00_Rn2Mz1y8uZN62ukEXiNO2077","name":"bash","arguments":"{\"command\": \"echo SNAPSHOT_OK\", \"description\": \"Run echo SNAPSHOT_OK\"}"}}},{"type":"chunk","time":1791427900980,"chunk":{"type":"block-start","index":2,"blockType":"text"}},{"type":"chunk","time":1791427900980,"chunk":{"type":"block-end","index":2,"block":{"type":"text","text":""}}},{"type":"chunk","time":1791427900980,"chunk":{"type":"usage","usage":{"inputTokens":2879,"outputTokens":89,"cacheReadTokens":0,"reasoningTokens":17}}},{"type":"chunk","time":1791427900980,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
|
||||
{"type":"tool/call","data":{"turn":1,"step":1,"callId":"call_00_Rn2Mz1y8uZN62ukEXiNO2077","name":"bash","arguments":"{\"command\": \"echo SNAPSHOT_OK\", \"description\": \"Run echo SNAPSHOT_OK\"}"}}
|
||||
{"type":"job/registered","data":{"jobId":"{{id:1}}","owner":"{{session:1}}","kind":"bash","label":"echo SNAPSHOT_OK","startedAt":1791427900990,"toolCall":{"callId":"call_00_Rn2Mz1y8uZN62ukEXiNO2077","rootCallId":"call_00_Rn2Mz1y8uZN62ukEXiNO2077"}}}
|
||||
{"type":"job/settled","data":{"jobId":"{{id:1}}","owner":"{{session:1}}","kind":"bash","label":"echo SNAPSHOT_OK","startedAt":1791427900990,"toolCall":{"callId":"call_00_Rn2Mz1y8uZN62ukEXiNO2077","rootCallId":"call_00_Rn2Mz1y8uZN62ukEXiNO2077"},"finishedAt":1791427900991,"status":"completed","cause":"producer","producerReported":true,"detail":"exit code: 0"}}
|
||||
{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"call_00_Rn2Mz1y8uZN62ukEXiNO2077"},"toolCallId":"call_00_Rn2Mz1y8uZN62ukEXiNO2077","content":[{"type":"text","text":"SNAPSHOT_OK\n"}],"isError":false,"id":"{{message:5}}"},"meta":{"cwd":"{{cwd}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"}
|
||||
{"type":"step/end","data":{"turn":1,"step":1}}
|
||||
{"type":"step/start","data":{"turn":1,"step":2}}
|
||||
{"type":"assistant/message","data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"reasoning","text":"The command executed successfully and printed SNAPSHOT_OK. Now I need to reply with the single word DONE."},{"type":"text","text":"DONE"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash"},"id":"{{message:6}}"},"usage":{"inputTokens":170,"outputTokens":28,"cacheReadTokens":2816,"reasoningTokens":25},"stream":[{"type":"chunk","time":1791427901005,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}},{"type":"reasoning-chunks","time0":1791427901005,"index":0,"dt":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"texts":["The"," command"," executed"," successfully"," and"," printed"," S","NA","PS","H","OT","_OK","."," Now"," I"," need"," to"," reply"," with"," the"," single"," word"," D","ONE","."]},{"type":"chunk","time":1791427901005,"chunk":{"type":"block-start","index":1,"blockType":"text"}},{"type":"text-chunks","time0":1791427901005,"index":1,"dt":[0],"texts":["D","ONE"]},{"type":"chunk","time":1791427901005,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The command executed successfully and printed SNAPSHOT_OK. Now I need to reply with the single word DONE."}}},{"type":"chunk","time":1791427901005,"chunk":{"type":"block-end","index":1,"block":{"type":"text","text":"DONE"}}},{"type":"chunk","time":1791427901005,"chunk":{"type":"usage","usage":{"inputTokens":170,"outputTokens":28,"cacheReadTokens":2816,"reasoningTokens":25}}},{"type":"chunk","time":1791427901005,"chunk":{"type":"finish","reason":{"kind":"stop"}}}]},"surfaceOp":"append"}
|
||||
{"type":"step/end","data":{"turn":1,"step":2}}
|
||||
{"type":"turn/end","data":{"turn":1,"reason":{"kind":"completed"}}}
|
||||
@@ -0,0 +1,10 @@
|
||||
version: 1
|
||||
scenario: deepseek-messages-empty-text-history
|
||||
profile: headless
|
||||
composition: default
|
||||
recording: authored
|
||||
header:
|
||||
class: deepseek-messages-empty-text-history
|
||||
pin: true
|
||||
systemPromptSource: text-turn
|
||||
toolSchemasSource: text-turn
|
||||
Reference in new issue
Block a user