test(snapshot): stabilize minimal compaction estimates
This commit is contained in:
1 parent
b560423995
commit
2de77d0339
7 files changed
+30
-15
No files matched your search
@@ -211,7 +211,7 @@ describe('minimal manual compaction', () => {
|
||||
expect(prompts).toHaveLength(2)
|
||||
const [initial, continuation] = prompts
|
||||
if (initial === undefined || continuation === undefined) throw new Error('minimal compaction fixture requires two user prompts')
|
||||
scaffold = await launchWebScaffold({ replayFixture: fixture, compareReplaySession: true })
|
||||
scaffold = await launchWebScaffold({ replayFixture: fixture, compareReplaySession: true, workspacePathLength: 192 })
|
||||
scaffold.ctx.on('llm/stream', async function* (options, next) {
|
||||
requests.push(options)
|
||||
yield* next()
|
||||
|
||||
@@ -310,6 +310,8 @@ export interface LaunchOptions {
|
||||
openInAppEnvironment?: LaunchEnvironmentSnapshot
|
||||
/** Compare the replayed root Session; `read-only` also forbids refresh writes to a borrowed fixture. */
|
||||
compareReplaySession?: boolean | 'read-only'
|
||||
/** Fixed JSON-quoted workspace path length for compaction estimates; omitted values use the ordinary prefix. */
|
||||
workspacePathLength?: number
|
||||
/**
|
||||
* Optional product overlay applied after the shipped Web surface and before
|
||||
* the scaffold's hermetic test patches, matching the launcher's `--patch`
|
||||
@@ -502,7 +504,15 @@ export async function launchWebScaffold(options: LaunchOptions = {}): Promise<We
|
||||
process.env.DEEPSEEK_API_KEY = originalDeepSeekCredential
|
||||
}
|
||||
}
|
||||
const workspaceCwd = await realpath(await mkdtemp(join(tmpdir(), 'dsh-web-e2e-ws-')))
|
||||
const workspaceParent = await realpath(tmpdir())
|
||||
let workspacePrefix = 'dsh-web-e2e-ws-'
|
||||
if (options.workspacePathLength !== undefined) {
|
||||
// mkdtemp appends six characters; JSON quoting is part of the runtime-context text.
|
||||
const paddingLength = options.workspacePathLength - JSON.stringify(join(workspaceParent, `${workspacePrefix}XXXXXX`)).length
|
||||
if (paddingLength < 0) throw new Error('compaction snapshot temporary root exceeds its fixed cwd text length')
|
||||
workspacePrefix += '-'.repeat(paddingLength)
|
||||
}
|
||||
const workspaceCwd = await realpath(await mkdtemp(join(workspaceParent, workspacePrefix)))
|
||||
// Isolated harness home: the settings/credentials rows resolve $DSH_HOME
|
||||
// paths at load, and an in-process boot must NEVER touch the developer's
|
||||
// real ~/.dsh document or credential file.
|
||||
|
||||
@@ -782,10 +782,7 @@ describe('registration surface', () => {
|
||||
expect((await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model'))).isError).toBe(true)
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ config: { read: false, write: false, edit: false }, tools: ['read_image'] },
|
||||
{ config: {}, tools: ['read', 'read_image', 'write', 'edit'] },
|
||||
])('withdraws and restores enabled image registration with the attachment store: $config', async ({ config, tools }) => {
|
||||
it('withdraws and restores image-only registration with the attachment store', async () => {
|
||||
const ctx = new Context()
|
||||
try {
|
||||
provideWorkingDirectoryFixture(ctx)
|
||||
@@ -793,13 +790,13 @@ describe('registration surface', () => {
|
||||
await ctx.plugin(ToolRuntime)
|
||||
await ctx.plugin(LocalFileSystem, { cwd: dir })
|
||||
const attachment = await ctx.plugin(LocalAttachmentStore, { dshHome: home })
|
||||
const plugin = await ctx.plugin(ToolFs, config)
|
||||
const plugin = await ctx.plugin(ToolFs, { read: false, write: false, edit: false })
|
||||
const names = () => ctx.tools.schemas().map(tool => tool.name).sort()
|
||||
expect(names()).toEqual([...tools].sort())
|
||||
expect(names()).toEqual(['read_image'])
|
||||
await attachment.dispose()
|
||||
expect(names()).toEqual(tools.filter(name => name !== 'read_image').sort())
|
||||
expect(names()).toEqual([])
|
||||
await ctx.plugin(LocalAttachmentStore, { dshHome: home })
|
||||
expect(names()).toEqual([...tools].sort())
|
||||
expect(names()).toEqual(['read_image'])
|
||||
await plugin.dispose()
|
||||
expect(names()).toEqual([])
|
||||
} finally {
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
{"method":"session.event","params":{"sessionId":"{{sessionId}}","event":{"type":"tool/result","seq":12,"time":0,"data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"minimal-image-read"},"toolCallId":"minimal-image-read","content":[{"type":"text","text":"<path>{{cwd}}/red.png</path>\n<type>image</type>\n<content>\nimage/png image, 1x1 px, 69 bytes\n</content>"},{"type":"image","attachment":{"attachmentId":"sha256:b1ff9c8ea3a780bad09b346c423d2d0e46815926879b18e841d928376a946640","mediaType":"image/png","bytes":69,"width":1,"height":1,"name":"red.png"}}],"isError":false,"id":"{{sessionId}}"},"meta":{"path":"{{cwd}}/red.png"}},"sourceEventSeqs":[11],"surfaceOp":"append"}}}
|
||||
{"method":"session.event","params":{"sessionId":"{{sessionId}}","event":{"type":"step/end","seq":13,"time":0,"data":{"turn":1,"step":1}}}}
|
||||
{"method":"session.event","params":{"sessionId":"{{sessionId}}","event":{"type":"compaction/start","seq":14,"time":0,"data":{"compactionId":"{{sessionId}}","turn":1}}}}
|
||||
{"method":"session.event","params":{"sessionId":"{{sessionId}}","event":{"type":"compaction/summary","seq":15,"time":0,"data":{"compactionId":"{{sessionId}}","summary":[{"type":"text","text":"Read red.png and report MINIMAL_IMAGE_OK."}],"rawOutput":[{"type":"text","text":"Read red.png and report MINIMAL_IMAGE_OK."}],"llmStreamCall":true,"shadowedRange":{"start":5,"end":6},"shadowedSeqs":[5,6],"shadowedTokenCount":2923,"provider":"deepseek-official","model":"deepseek-flash","maxTokens":256}}}}
|
||||
{"method":"session.event","params":{"sessionId":"{{sessionId}}","event":{"type":"compaction/summary","seq":15,"time":0,"data":{"compactionId":"{{sessionId}}","summary":[{"type":"text","text":"Read red.png and report MINIMAL_IMAGE_OK."}],"rawOutput":[{"type":"text","text":"Read red.png and report MINIMAL_IMAGE_OK."}],"llmStreamCall":true,"shadowedRange":{"start":5,"end":6},"shadowedSeqs":[5,6],"shadowedTokenCount":2945,"provider":"deepseek-official","model":"deepseek-flash","maxTokens":256}}}}
|
||||
{"method":"session.event","params":{"sessionId":"{{sessionId}}","event":{"type":"user/message","seq":16,"time":0,"data":{"content":[{"type":"text","text":"This is an automatically generated checkpoint condensing an earlier span of the conversation to free up context. Treat the captured context as established background and build on it without restating it. Continue the task directly from the messages that follow, without acknowledging this checkpoint.\n\n<compacted-summary>"},{"type":"text","text":"Read red.png and report MINIMAL_IMAGE_OK."},{"type":"text","text":"</compacted-summary>"}],"source":{"kind":"compact-checkpoint","compactionId":"{{sessionId}}"},"role":"user","id":"{{sessionId}}"},"sourceEventSeqs":[14,15,5,6],"surfaceOp":{"op":"replace","startSeq":5,"endSeq":6}}}}
|
||||
{"method":"session.event","params":{"sessionId":"{{sessionId}}","event":{"type":"compaction/end","seq":17,"time":0,"data":{"compactionId":"{{sessionId}}","turn":1}}}}
|
||||
{"method":"session.event","params":{"sessionId":"{{sessionId}}","event":{"type":"step/start","seq":18,"time":0,"data":{"turn":1,"step":2}}}}
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"minimal-image-read"},"toolCallId":"minimal-image-read","content":[{"type":"text","text":"<path>{{cwd}}/red.png</path>\n<type>image</type>\n<content>\nimage/png image, 1x1 px, 69 bytes\n</content>"},{"type":"image","attachment":{"attachmentId":"sha256:b1ff9c8ea3a780bad09b346c423d2d0e46815926879b18e841d928376a946640","mediaType":"image/png","bytes":69,"width":1,"height":1,"name":"red.png"}}],"isError":false,"id":"{{message:5}}"},"meta":{"path":"{{cwd}}/red.png"}},"sourceEventSeqs":[11],"surfaceOp":"append"}
|
||||
{"type":"step/end","data":{"turn":1,"step":1}}
|
||||
{"type":"compaction/start","data":{"compactionId":"{{id:1}}","turn":1}}
|
||||
{"type":"compaction/summary","data":{"compactionId":"{{id:1}}","summary":[{"type":"text","text":"Read red.png and report MINIMAL_IMAGE_OK."}],"rawOutput":[{"type":"text","text":"Read red.png and report MINIMAL_IMAGE_OK."}],"llmStreamCall":true,"shadowedRange":{"start":5,"end":6},"shadowedSeqs":[5,6],"shadowedTokenCount":2923,"provider":"deepseek-official","model":"deepseek-flash","maxTokens":256}}
|
||||
{"type":"compaction/summary","data":{"compactionId":"{{id:1}}","summary":[{"type":"text","text":"Read red.png and report MINIMAL_IMAGE_OK."}],"rawOutput":[{"type":"text","text":"Read red.png and report MINIMAL_IMAGE_OK."}],"llmStreamCall":true,"shadowedRange":{"start":5,"end":6},"shadowedSeqs":[5,6],"shadowedTokenCount":2945,"provider":"deepseek-official","model":"deepseek-flash","maxTokens":256}}
|
||||
{"type":"user/message","data":{"content":[{"type":"text","text":"This is an automatically generated checkpoint condensing an earlier span of the conversation to free up context. Treat the captured context as established background and build on it without restating it. Continue the task directly from the messages that follow, without acknowledging this checkpoint.\n\n<compacted-summary>"},{"type":"text","text":"Read red.png and report MINIMAL_IMAGE_OK."},{"type":"text","text":"</compacted-summary>"}],"source":{"kind":"compact-checkpoint","compactionId":"{{id:1}}"},"role":"user","id":"{{message:6}}"},"sourceEventSeqs":[14,15,5,6],"surfaceOp":{"op":"replace","startSeq":5,"endSeq":6}}
|
||||
{"type":"compaction/end","data":{"compactionId":"{{id:1}}","turn":1}}
|
||||
{"type":"step/start","data":{"turn":1,"step":2}}
|
||||
|
||||
@@ -572,7 +572,15 @@ async function runScenario(scenario: CorpusScenario): Promise<{
|
||||
finalWorkspace: WorkspaceSnapshotEntry[]
|
||||
cwd: string
|
||||
}> {
|
||||
const cwd = await realpath(await mkdtemp(join(tmpdir(), `sdk-snapshot-${scenario.name}-`)))
|
||||
const workspaceParent = await realpath(tmpdir())
|
||||
let workspacePrefix = `sdk-snapshot-${scenario.name}-`
|
||||
if (scenario.name === 'minimal-image-compaction') {
|
||||
// mkdtemp appends six characters; JSON quoting is part of the runtime-context text.
|
||||
const paddingLength = 192 - JSON.stringify(join(workspaceParent, `${workspacePrefix}XXXXXX`)).length
|
||||
if (paddingLength < 0) throw new Error('compaction snapshot temporary root exceeds its fixed cwd text length')
|
||||
workspacePrefix += '-'.repeat(paddingLength)
|
||||
}
|
||||
const cwd = await realpath(await mkdtemp(join(workspaceParent, workspacePrefix)))
|
||||
const dshHome = join(cwd, '.dsh')
|
||||
const sessionsRoot = join(dshHome, 'sessions')
|
||||
const replayFixtures = recording ? [] : await hydrateReplayFixtures(scenario, cwd)
|
||||
|
||||
@@ -17,10 +17,10 @@
|
||||
{"type":"turn/end","data":{"turn":1,"reason":{"kind":"completed"}}}
|
||||
{"type":"command/run","data":{"commandId":"{{command:1}}","name":"compact","args":"","source":{"kind":"user"}}}
|
||||
{"type":"compaction/start","data":{"compactionId":"{{id:1}}","sourceCommandId":"{{command:1}}","turn":null}}
|
||||
{"type":"compaction/summary","data":{"compactionId":"{{id:1}}","sourceCommandId":"{{command:1}}","summary":[{"type":"text","text":"Implement minimal image reading through the existing filesystem tool, preserving permissions, model capability checks, and standard mode. Enable automatic and manual compaction."}],"rawOutput":[{"type":"text","text":"Implement minimal image reading through the existing filesystem tool, preserving permissions, model capability checks, and standard mode. Enable automatic and manual compaction."}],"llmStreamCall":true,"shadowedRange":{"start":8,"end":9},"shadowedSeqs":[8,9],"shadowedTokenCount":326,"provider":"deepseek-official","model":"deepseek-v4-flash","maxTokens":65536}}
|
||||
{"type":"compaction/summary","data":{"compactionId":"{{id:1}}","sourceCommandId":"{{command:1}}","summary":[{"type":"text","text":"Implement minimal image reading through the existing filesystem tool, preserving permissions, model capability checks, and standard mode. Enable automatic and manual compaction."}],"rawOutput":[{"type":"text","text":"Implement minimal image reading through the existing filesystem tool, preserving permissions, model capability checks, and standard mode. Enable automatic and manual compaction."}],"llmStreamCall":true,"shadowedRange":{"start":8,"end":9},"shadowedSeqs":[8,9],"shadowedTokenCount":354,"provider":"deepseek-official","model":"deepseek-v4-flash","maxTokens":65536}}
|
||||
{"type":"user/message","data":{"content":[{"type":"text","text":"This is an automatically generated checkpoint condensing an earlier span of the conversation to free up context. Treat the captured context as established background and build on it without restating it. Continue the task directly from the messages that follow, without acknowledging this checkpoint.\n\n<compacted-summary>"},{"type":"text","text":"Implement minimal image reading through the existing filesystem tool, preserving permissions, model capability checks, and standard mode. Enable automatic and manual compaction."},{"type":"text","text":"</compacted-summary>"}],"source":{"kind":"compact-checkpoint","compactionId":"{{id:1}}","sourceCommandId":"{{command:1}}"},"role":"user","id":"{{message:5}}"},"sourceEventSeqs":[17,18,8,9],"surfaceOp":{"op":"replace","startSeq":8,"endSeq":9}}
|
||||
{"type":"compaction/end","data":{"compactionId":"{{id:1}}","sourceCommandId":"{{command:1}}","turn":null}}
|
||||
{"type":"command/done","data":{"commandId":"{{command:1}}","kind":"success","text":"Compacted 2 history items (~326 tokens).","sourceEventSeq":18}}
|
||||
{"type":"command/done","data":{"commandId":"{{command:1}}","kind":"success","text":"Compacted 2 history items (~354 tokens).","sourceEventSeq":18}}
|
||||
{"type":"agent/inbox/spliced","data":{"target":"next-turn","start":0,"inserted":[{"content":[{"type":"text","text":"Continue using the project requirements. Reply exactly MINIMAL_COMPACT_OK and stop."}],"source":{"kind":"user"},"role":"user","id":"{{message:6}}"}]}}
|
||||
{"type":"turn/start","data":{"turn":2}}
|
||||
{"type":"agent/inbox/spliced","data":{"target":"next-turn","start":0,"removedCount":1,"inserted":[]}}
|
||||
|
||||
Reference in new issue
Block a user