_KermanandTianyi Cui
f4a32dbd0a
Represent tool results as tool-role messages and migrate released wrappers without losing outer own JSON fields, call identity, error state, or writer field order. Refuse nested or wrapper-only information that has no lossless current representation before successor publication. Validate current event relationships directly instead of projecting V4 data back into V3. Keep lifecycle, PTC, title/command, compaction, system-field and retired-syntax checks on catalog restoration and ordinary JSONL opens. The Worker JSON equality helper must inspect own keys so it cannot admit PTC arguments that Node rejects. Keep the source role change, required native admission and consumer fixture updates in one complete review layer. Explain shared-type changes in the persistence record and the native reader rationale in its existing note. Use checked branded fixture identities and discriminant narrowing instead of unknown double casts; fixture values and assertions remain intact. This recomposes the original tool-role work by _Kerman with the independently validated migration and admission repairs by Tianyi Cui. Frozen generations and snapshot input version selection are retained. Co-authored-by: Tianyi Cui <[email protected]>
89 lines
3.9 KiB
TypeScript
89 lines
3.9 KiB
TypeScript
/** End-to-end SDK continuation through built dsh sdk-minimal with an explicitly mounted file editor. */
|
|
|
|
import { mkdir, writeFile } from 'node:fs/promises'
|
|
import { join } from 'node:path'
|
|
import { performance } from 'node:perf_hooks'
|
|
import { DeepSeekHarness } from '@deepseek-ai/dsh-sdk-client'
|
|
import { assertBuiltBenchmarkRuntime } from '../support/built-worker.ts'
|
|
import { PARENT_ID, resultText, WORKLOAD } from './workload.ts'
|
|
|
|
/** Parent-observed wall time, including profile launch and SDK shutdown. */
|
|
export interface ProfileReport {
|
|
readonly totalMs: number
|
|
readonly bootMs: number
|
|
readonly turnsMs: number
|
|
readonly closeMs: number
|
|
readonly requests: number
|
|
readonly toolCalls: number
|
|
}
|
|
|
|
async function run(root: string): Promise<ProfileReport> {
|
|
const home = join(root, 'home')
|
|
const cwd = join(root, 'workspace')
|
|
await mkdir(cwd, { recursive: true })
|
|
await mkdir(home, { recursive: true })
|
|
await writeFile(join(cwd, 'synthetic.txt'), resultText(0))
|
|
const patch = join(root, 'profile.patch.yml')
|
|
await writeFile(patch, [
|
|
'- id: llm-deepseek', ' disabled: true',
|
|
'- id: sessions', ' config:', ' root: ' + JSON.stringify(join(root, 'profile-sessions')), ' compression: zstd',
|
|
'- insert:',
|
|
' - id: fs-local', " name: '@deepseek-ai/dsh-fs-local'",
|
|
' - id: str-replace-editor', " name: '@deepseek-ai/dsh-tool-str-replace-editor'",
|
|
' - id: benchmark-model', ' name: ' + JSON.stringify(join(import.meta.dirname, 'profile-adapter.js')),
|
|
'',
|
|
].join('\n'))
|
|
const env: NodeJS.ProcessEnv = {
|
|
PATH: process.env.PATH, HOME: home, USERPROFILE: home,
|
|
DSH_AGENTS_HOME: join(home, 'agents'),
|
|
}
|
|
const harness = new DeepSeekHarness({
|
|
dshBin: join(import.meta.dirname, '..', '..', '..', 'apps', 'cli', 'lib', 'bin.js'),
|
|
profile: 'sdk-minimal', dshHome: home, processCwd: cwd, cwd,
|
|
provider: 'bench', model: 'bench', patches: [patch], env,
|
|
initializeTimeoutMs: 15_000, requestTimeoutMs: 15_000,
|
|
})
|
|
let closing: Promise<void> | undefined
|
|
const close = (): Promise<void> => closing ??= harness.close()
|
|
let expired = false
|
|
const deadline = setTimeout(() => {
|
|
expired = true
|
|
// The awaited finally close below reports shutdown failures; this only requests cancellation.
|
|
void close().catch(() => undefined)
|
|
}, 40_000)
|
|
let requests = 0
|
|
let toolCalls = 0
|
|
const start = performance.now()
|
|
try {
|
|
await harness.start()
|
|
const booted = performance.now()
|
|
for (let turn = 0; turn < WORKLOAD.profileTurns; turn++) {
|
|
const result = await harness.run('Read the synthetic file ' + String(turn), { sessionId: PARENT_ID })
|
|
requests += result.events.filter(event => event.type === 'assistant/message').length
|
|
for (const event of result.events) {
|
|
if (event.type !== 'tool/result') continue
|
|
const message = event.data.message
|
|
if (message.isError || !message.content.some(block => block.type === 'text' && block.text.includes('export const synthetic = 42;'))) {
|
|
throw new Error('profile benchmark did not read the synthetic file')
|
|
}
|
|
toolCalls++
|
|
}
|
|
}
|
|
const turnsDone = performance.now()
|
|
await close()
|
|
const end = performance.now()
|
|
if (expired || requests !== WORKLOAD.profileTurns * 2 || toolCalls !== WORKLOAD.profileTurns * WORKLOAD.toolsPerLiveTurn) {
|
|
throw new Error('profile benchmark did not finish every model request and real tool call')
|
|
}
|
|
return { totalMs: end - start, bootMs: booted - start, turnsMs: turnsDone - booted, closeMs: end - turnsDone, requests, toolCalls }
|
|
} finally {
|
|
clearTimeout(deadline)
|
|
await close()
|
|
}
|
|
}
|
|
|
|
assertBuiltBenchmarkRuntime(import.meta.url, { '@deepseek-ai/dsh-sdk-client': import.meta.resolve('@deepseek-ai/dsh-sdk-client') })
|
|
const [root] = process.argv.slice(2)
|
|
if (root === undefined) throw new Error('usage: profile-continuation.worker.js <root>')
|
|
process.stdout.write(JSON.stringify(await run(root)) + '\n')
|