From 523e741a31cfc365218c3b48d20bdf2e9c19de76 Mon Sep 17 00:00:00 2001 From: Yixiang Tu Date: Fri, 23 Oct 2026 11:05:32 +0800 Subject: [PATCH] test(llm-deepseek): cover all-deferred requests and separate tool updates --- .../llm/llm-deepseek/tests/adapter.e2e.ts | 26 ++++++++----- .../tests/fixtures/all-deferred-tools.ts | 38 +++++++++++++++++++ .../cordis.snapshot.yml | 18 +++++++++ .../deepseek-all-deferred-tools/cordis.yml | 3 ++ .../session.v4.jsonl | 17 +++++++++ .../deepseek-all-deferred-tools/snapshot.yml | 8 ++++ .../system-prompt.expected.md | 14 +++++++ .../tool-schemas.expected.json | 14 +++++++ 8 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 packages/llm/llm-deepseek/tests/fixtures/all-deferred-tools.ts create mode 100644 snapshots/session/deepseek-all-deferred-tools/cordis.snapshot.yml create mode 100644 snapshots/session/deepseek-all-deferred-tools/cordis.yml create mode 100644 snapshots/session/deepseek-all-deferred-tools/session.v4.jsonl create mode 100644 snapshots/session/deepseek-all-deferred-tools/snapshot.yml create mode 100644 snapshots/session/deepseek-all-deferred-tools/system-prompt.expected.md create mode 100644 snapshots/session/deepseek-all-deferred-tools/tool-schemas.expected.json diff --git a/packages/llm/llm-deepseek/tests/adapter.e2e.ts b/packages/llm/llm-deepseek/tests/adapter.e2e.ts index 6c5264c2a8..c437d810fd 100644 --- a/packages/llm/llm-deepseek/tests/adapter.e2e.ts +++ b/packages/llm/llm-deepseek/tests/adapter.e2e.ts @@ -1,6 +1,6 @@ import * as Protocol from '@deepseek-ai/dsh-llm-deepseek' /** - * Real Messages round trips use the official root and require credentials. + * Real Messages round trips require credentials; tool replacement also accepts DEEPSEEK_BASE_URL. * System-update checks additionally require DEEPSEEK_IN_HISTORY_MODEL. */ import { randomUUID } from 'node:crypto' @@ -33,7 +33,7 @@ afterEach(async () => { vi.unstubAllEnvs() vi.unstubAllGlobals() }) -async function boot(models?: Messages.Options['models']) { +async function boot(models?: Messages.Options['models'], baseURL = Protocol.PUBLIC_BASE_URL) { const home = await mkdtemp(join(tmpdir(), 'dsh-messages-e2e-')) cleanups.push(() => rm(home, { recursive: true, force: true })) vi.stubEnv('DSH_HOME', home) @@ -41,7 +41,7 @@ async function boot(models?: Messages.Options['models']) { cleanups.push(() => ctx.fiber.dispose()) await ctx.plugin(LlmRuntime) await ctx.plugin(Messages, { - baseURL: Protocol.PUBLIC_BASE_URL, + baseURL, maxTokens: 4096, ...models === undefined ? {} : { models }, }) @@ -50,17 +50,23 @@ async function boot(models?: Messages.Options['models']) { const tool = { name: 'lookup_value', description: 'Read the requested value. Always call this tool to obtain a value.', parameters: { type: 'object', properties: { key: { type: 'string' } }, required: ['key'] } } describe.skipIf(!process.env.DEEPSEEK_API_KEY)('DeepSeek Messages real API', () => { - it('calls a replacement tool after every baseline tool has been removed', async () => { + it.each([false, true])('calls a replacement tool after every baseline tool has been removed, separate updates=%s', async (separate) => { const model = 'deepseek-flash' - const ctx = await boot([{ id: model, toolUpdate: 'addition-only' }]) - const update = createDeveloperMessage({ source: { kind: 'tool-registry' }, content: [ - { type: 'tool-removal', toolName: 'old_lookup' }, { type: 'tool-addition', toolName: tool.name }, - ] }) + const ctx = await boot([{ id: model, toolUpdate: 'addition-only' }], process.env.DEEPSEEK_BASE_URL ?? Protocol.PUBLIC_BASE_URL) + const updates = [createDeveloperMessage({ source: { kind: 'tool-registry' }, content: [ + { type: 'tool-removal', toolName: 'old_lookup' }, + ...separate ? [] : [{ type: 'tool-addition' as const, toolName: tool.name }], + ] })] + if (separate) updates.push(createDeveloperMessage({ source: { kind: 'tool-registry' }, content: [ + { type: 'tool-addition', toolName: tool.name }, + ] })) const response = await assemble(ctx.llm.stream(options({ model, reasoningEffort: ReasoningEffortId('high'), - messages: [user('Call lookup_value with key "answer" to obtain the value. Do not guess.'), update], + messages: [user('Call lookup_value with key "answer" to obtain the value. Do not guess.'), ...updates], tools: [tool], - toolHistory: { tools: [{ ...tool, name: 'old_lookup' }], updates: [{ messageId: update.id, additions: [tool] }] }, + toolHistory: { tools: [{ ...tool, name: 'old_lookup' }], updates: updates.map(update => ({ + messageId: update.id, additions: update.content.some(block => block.type === 'tool-addition') ? [tool] : [], + })) }, })), model) expect(response.assembler.finish).toEqual({ kind: 'tool-calls' }) expect(response.message.content.filter(block => block.type === 'tool-call').map(call => call.name)).toEqual([tool.name]) diff --git a/packages/llm/llm-deepseek/tests/fixtures/all-deferred-tools.ts b/packages/llm/llm-deepseek/tests/fixtures/all-deferred-tools.ts new file mode 100644 index 0000000000..969347e651 --- /dev/null +++ b/packages/llm/llm-deepseek/tests/fixtures/all-deferred-tools.ts @@ -0,0 +1,38 @@ +/** Replay explicitly deferred declarations without activating them. */ +import assert from 'node:assert/strict' +import type { Context } from '@deepseek-ai/cordis' +import type {} from '@deepseek-ai/dsh-agent' +import { projectToolUpdates } from '@deepseek-ai/dsh-llm' +import { defineContentToolFixture } from '@deepseek-ai/dsh-tool-testkit' +import { resolveAdapterOptions } from '../../src/config.ts' +import { serialize } from '../../src/serialize.ts' + +export const name = 'messages-all-deferred-tools-snapshot' +export const inject = ['tools', 'llm'] + +/** + * Keep one real tool deferred and inspect the derived Messages request. + * @param ctx - Snapshot composition context. + */ +export function apply(ctx: Context): void { + ctx.on('agent/created', ({ agent }) => { + agent.ctx.tools.restrict({ allow: [] }) + agent.ctx.tools.register(defineContentToolFixture({ + name: 'snapshot_pending', description: 'Wait for activation.', parameters: {}, deferLoading: true, + async execute() { assert.fail('An unactivated tool must not be called') }, + })) + }) + const connection = resolveAdapterOptions({}) + ctx.on('llm/stream', (options, next) => { + const projected = projectToolUpdates(options.messages, options.tools, 'addition-only', options.toolHistory) + assert.ok(projected.tools) + assert.deepEqual(projected.tools, options.tools) + const request = serialize({ ...options, tools: [...projected.tools] }, connection, projected.messages, new Map(), () => undefined) + assert.deepEqual(request.tools, [ + { name: 'DeferredToolPlaceholder', description: 'Reserved placeholder that keeps deferred tool loading active; never call this tool.', input_schema: { type: 'object', properties: {} } }, + { name: 'snapshot_pending', description: 'Wait for activation.', input_schema: { type: 'object', properties: {} }, defer_loading: true }, + ]) + assert.deepEqual(request.messages.filter(message => message.role === 'system'), []) + return next() + }) +} diff --git a/snapshots/session/deepseek-all-deferred-tools/cordis.snapshot.yml b/snapshots/session/deepseek-all-deferred-tools/cordis.snapshot.yml new file mode 100644 index 0000000000..2f1d87deef --- /dev/null +++ b/snapshots/session/deepseek-all-deferred-tools/cordis.snapshot.yml @@ -0,0 +1,18 @@ +- id: llm-deepseek + disabled: true + +- insert: + - id: llm-replay + name: '@deepseek-ai/dsh-llm-replay' + config: + providers: + - id: deepseek-official + name: DeepSeek + models: + - id: deepseek-v4-flash + toolUpdate: addition-only + defaultMaxTokens: 256000 + reasoningEfforts: [off, low, high, max] + defaultReasoningEffort: high + - id: messages-all-deferred-tools-snapshot + name: '../../../packages/llm/llm-deepseek/tests/fixtures/all-deferred-tools.ts' diff --git a/snapshots/session/deepseek-all-deferred-tools/cordis.yml b/snapshots/session/deepseek-all-deferred-tools/cordis.yml new file mode 100644 index 0000000000..9bbd9f0e59 --- /dev/null +++ b/snapshots/session/deepseek-all-deferred-tools/cordis.yml @@ -0,0 +1,3 @@ +- insert: + - id: messages-all-deferred-tools-snapshot + name: '../../../packages/llm/llm-deepseek/tests/fixtures/all-deferred-tools.ts' diff --git a/snapshots/session/deepseek-all-deferred-tools/session.v4.jsonl b/snapshots/session/deepseek-all-deferred-tools/session.v4.jsonl new file mode 100644 index 0000000000..c7f585e660 --- /dev/null +++ b/snapshots/session/deepseek-all-deferred-tools/session.v4.jsonl @@ -0,0 +1,17 @@ +{"type":"session","version":4,"id":"{{session:1}}","createdAt":0,"cwd":"{{cwd}}","isSeeded":false,"delegationDepth":0} +{"type":"permission/preset","data":{"preset":"danger-full-access"}} +{"type":"sandbox/mode","data":{"mode":"danger-full-access"}} +{"type":"approval/policy","data":{"policy":"never"}} +{"type":"agent/inbox/spliced","data":{"target":"next-turn","start":0,"inserted":[{"content":[{"type":"text","text":"Reply DONE without calling any tools."}],"source":{"kind":"user"},"role":"user","id":"{{message:1}}"}]}} +{"type":"turn/start","data":{"turn":1}} +{"type":"agent/inbox/spliced","data":{"target":"next-turn","start":0,"removedCount":1,"inserted":[]}} +{"type":"step/start","data":{"turn":1,"step":1}} +{"type":"system/message","data":{"turn":1,"step":1,"message":{"role":"system","content":[{"type":"text","text":"{{system}}"}],"source":{"kind":"system-prompt"},"id":"{{message:2}}"}},"surfaceOp":"append"} +{"type":"user/message","data":{"content":[{"type":"text","text":"Reply DONE without calling any tools."}],"source":{"kind":"user"},"role":"user","id":"{{message:1}}"},"surfaceOp":"append"} +{"type":"user/message","data":{"content":[{"type":"text","text":"Current runtime context. This snapshot supersedes earlier runtime-context snapshots.\n\nCurrent working directory: \"{{cwd}}\".\n\nCurrent DSH file policy: danger-full-access. The DSH file sandbox does not restrict file modifications by available operations.\n\nApproval prompts are disabled in this session: actions that require approval are rejected automatically — do not request sandbox escalation (do not set `sandbox_permissions`)."}],"source":{"kind":"runtime-context","form":"snapshot","sections":[{"name":"working-directory:current","text":"Current working directory: \"{{cwd}}\"."},{"name":"sandbox:policy","text":"Current DSH file policy: danger-full-access. The DSH file sandbox does not restrict file modifications by available operations."},{"name":"approval:policy","text":"Approval prompts are disabled in this session: actions that require approval are rejected automatically — do not request sandbox escalation (do not set `sandbox_permissions`)."}]},"role":"user","id":"{{message:3}}"},"surfaceOp":"append"} +{"type":"request/header","data":{"header":{"config":{"provider":"deepseek-official","model":"deepseek-v4-flash","maxTokens":256000,"reasoningEffort":"high"},"adapterDefaults":{"reasoningEffort":true,"maxTokens":true},"tools":"{{tools}}"},"reason":"initial"}} +{"type":"request/context","data":{"provider":"deepseek-official","model":"deepseek-v4-flash"}} +{"type":"session/title","data":{"title":"Reply DONE without calling any","messageSeqs":[8],"source":{"kind":"fallback"}}} +{"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"text","text":"DONE"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash"},"id":"{{message:4}}"},"stream":[{"type":"chunk","time":1792661643324,"chunk":{"type":"block-start","index":0,"blockType":"text"}},{"type":"text-chunks","time0":1792661643324,"index":0,"dt":[],"texts":["DONE"]},{"type":"chunk","time":1792661643324,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"DONE"}}},{"type":"chunk","time":1792661643324,"chunk":{"type":"finish","reason":{"kind":"stop"}}}]},"surfaceOp":"append"} +{"type":"step/end","data":{"turn":1,"step":1}} +{"type":"turn/end","data":{"turn":1,"reason":{"kind":"completed"}}} diff --git a/snapshots/session/deepseek-all-deferred-tools/snapshot.yml b/snapshots/session/deepseek-all-deferred-tools/snapshot.yml new file mode 100644 index 0000000000..9473be6a93 --- /dev/null +++ b/snapshots/session/deepseek-all-deferred-tools/snapshot.yml @@ -0,0 +1,8 @@ +version: 1 +scenario: deepseek-all-deferred-tools +profile: headless +composition: deepseek-all-deferred-tools +recording: authored +header: + class: deepseek-all-deferred-tools + pin: true diff --git a/snapshots/session/deepseek-all-deferred-tools/system-prompt.expected.md b/snapshots/session/deepseek-all-deferred-tools/system-prompt.expected.md new file mode 100644 index 0000000000..7c0cf23533 --- /dev/null +++ b/snapshots/session/deepseek-all-deferred-tools/system-prompt.expected.md @@ -0,0 +1,14 @@ +You are an AI agent powered by DeepSeek Harness. + +You are a coding assistant powered by the deepseek-v4-flash model. Your bash tool runs under a file sandbox — a `[sandbox: file access denied …]` result is policy, not a command bug. + +Verify your work by running the code or tests. Keep answers brief and factual. + + +Check the [exit code: N] marker on every bash result; investigate failures before moving on. + +Track every background job id you start. You are notified in-session when a job finishes — do not busy-poll or sleep on one; keep working on independent steps and do not duplicate a running job's work. Before giving a final answer, collect every still-relevant job with job_output (set wait: true only when you are genuinely blocked on it), and job_kill jobs that stopped mattering. + +create_goal may infer goal intent from a direct human request in any language. After session resume or fork, an active goal is disarmed: when a human asks to continue or resume in any wording or language, use update_goal action resume to rearm it. Mark complete only when the objective is actually achieved. Mark blocked only after the same blocking condition persists for at least 3 consecutive goal rounds, and report that concrete condition in blocked_reason; difficulty, uncertainty, or useful remaining work is not blocked. + +Use the workflow tool ONLY when the user explicitly asks for a workflow or for large multi-agent orchestration: you write a JavaScript script (the tool description documents the exact format) that fans work out across many subagents with phases and structured results. For one or two delegations, prefer plain subagent calls. diff --git a/snapshots/session/deepseek-all-deferred-tools/tool-schemas.expected.json b/snapshots/session/deepseek-all-deferred-tools/tool-schemas.expected.json new file mode 100644 index 0000000000..1c2c684312 --- /dev/null +++ b/snapshots/session/deepseek-all-deferred-tools/tool-schemas.expected.json @@ -0,0 +1,14 @@ +{ + "initial": [ + { + "name": "snapshot_pending", + "description": "Wait for activation.", + "parameters": { + "type": "object", + "properties": {} + }, + "deferLoading": true + } + ], + "changes": [] +}