test(ci): control Python stray UTF-8 fragment delivery
This commit is contained in:
turtle1999 committed
1 parent
96ee6e49f9
commit
40e39f4673
2 files changed
+31
-100
No files matched your search
@@ -1494,106 +1494,6 @@ describe('PythonCodeRuntime — programs and bindings', () => {
|
||||
await fiber.dispose()
|
||||
})
|
||||
|
||||
it('bounds an illegal-UTF-8 native residual by its U+FFFD-decoded cost', async () => {
|
||||
// Every 0xFF byte is illegal in any UTF-8 sequence, so `toString('utf8')`
|
||||
// renders each as U+FFFD (3 serialized bytes). `accrueStrayCost` must charge
|
||||
// that 3, not the raw 1: otherwise the newline-free residual grows to a full
|
||||
// budget's worth of RAW bytes before flushing — a ~3x undercount that near a
|
||||
// large maxLogBytes retains hundreds of MiB then expands toward a ~1 GiB peak
|
||||
// in flushStray's concat + toString. Paced single-byte writes (each its own
|
||||
// `data` chunk, like the sealing case) expose the sub-chunk accrual: charged
|
||||
// at 3 the residual crosses a 3072-byte budget after ~1024 bytes and flushes;
|
||||
// charged at 1 it would need ~3072 bytes, so the peak residual triples. The
|
||||
// largest merged buffer is the discriminator.
|
||||
const realConcat = Buffer.concat.bind(Buffer)
|
||||
let maxConcat = 0
|
||||
Buffer.concat = (list: readonly Uint8Array[], total?: number): Buffer<ArrayBuffer> => {
|
||||
const merged = realConcat(list, total)
|
||||
if (merged.length > maxConcat) maxConcat = merged.length
|
||||
return merged
|
||||
}
|
||||
let result: CodeRunResult
|
||||
try {
|
||||
const { runtime } = await setup({ maxLogBytes: 3072, maxWallMs: 30_000 })
|
||||
result = await runtime.run({
|
||||
program: [
|
||||
'import os, time',
|
||||
// One byte per chunk on every host: a plain yield lets a loaded
|
||||
// reader coalesce, and the coalesced chunk is what the bound below
|
||||
// measures. The payload stays above the 2048 discriminator, so a
|
||||
// raw-byte undercount still flushes the whole residual at EOF.
|
||||
'for _ in range(3200):',
|
||||
' os.write(1, b"\\xff")',
|
||||
' time.sleep(0.001)',
|
||||
'return None',
|
||||
].join('\n'),
|
||||
bindings: [],
|
||||
})
|
||||
} finally {
|
||||
Buffer.concat = realConcat
|
||||
}
|
||||
expect(result.error).toBeUndefined()
|
||||
expect(result.logs.at(-1)).toBe(logTruncationMarker(3072))
|
||||
// Charged at 3, the residual flushes around 1024 raw bytes; the largest
|
||||
// merged buffer stays well under 2048. A raw-byte undercount would let it
|
||||
// reach ~3072 before flushing, so 2048 discriminates.
|
||||
expect(maxConcat).toBeLessThan(2048)
|
||||
// The paced payload costs ~3.2s deterministically, which is above the
|
||||
// 5000ms default the local unit entry grants, so the case carries its own
|
||||
// bound instead of relying on the lane to widen it.
|
||||
}, 20_000)
|
||||
|
||||
it('charges a structurally-valid but illegal UTF-8 sequence its U+FFFD-decoded cost', async () => {
|
||||
// A CESU-8 lone surrogate `ED A0 80` is structurally well-formed (a 3-byte
|
||||
// lead plus two 0x80–0xBF continuations) but ILLEGAL: `toString('utf8')`
|
||||
// renders each of the three bytes as its own U+FFFD (serialized cost 9), not
|
||||
// one width-3 character. The newline-free flush trigger weighs the residual
|
||||
// through `accrueStrayCost`, which must validate each lead's
|
||||
// first-continuation range (ED excludes A0–BF) and charge the true 9 — else a
|
||||
// CESU flood undercounts 3x and the residual grows toward a full budget's raw
|
||||
// bytes before flushing, the same peak-memory vector as the 0xFF case. The
|
||||
// bytes are written one at a time (each its own `data` chunk, no pipe
|
||||
// coalescing) and `Buffer.concat` is wrapped to measure the peak residual.
|
||||
const realConcat = Buffer.concat.bind(Buffer)
|
||||
let maxConcat = 0
|
||||
Buffer.concat = (list: readonly Uint8Array[], total?: number): Buffer<ArrayBuffer> => {
|
||||
const merged = realConcat(list, total)
|
||||
if (merged.length > maxConcat) maxConcat = merged.length
|
||||
return merged
|
||||
}
|
||||
let result: CodeRunResult
|
||||
try {
|
||||
const { runtime } = await setup({ maxLogBytes: 3072, maxWallMs: 30_000 })
|
||||
result = await runtime.run({
|
||||
program: [
|
||||
'import os, time',
|
||||
'seq = (0xed, 0xa0, 0x80)',
|
||||
// 1100 sequences are 3300 raw bytes, past the 3072-byte budget a
|
||||
// raw-byte undercount reaches, so the undercount flushes above the
|
||||
// 2048 discriminator instead of only at EOF.
|
||||
'for _ in range(1100):',
|
||||
' for b in seq:',
|
||||
' os.write(1, bytes((b,)))',
|
||||
' time.sleep(0.001)',
|
||||
'return None',
|
||||
].join('\n'),
|
||||
bindings: [],
|
||||
})
|
||||
} finally {
|
||||
Buffer.concat = realConcat
|
||||
}
|
||||
expect(result.error).toBeUndefined()
|
||||
expect(result.logs.at(-1)).toBe(logTruncationMarker(3072))
|
||||
// Each 3-byte sequence costs 9 (three U+FFFD), so single-byte-paced the
|
||||
// residual crosses the 3072 budget after ~342 raw bytes and flushes; the
|
||||
// largest merged buffer stays well under 2048. Charging the structural width
|
||||
// 3 would need ~1024 raw bytes, tripling the peak past 2048.
|
||||
expect(maxConcat).toBeLessThan(2048)
|
||||
// The paced payload costs ~3.3s deterministically, which is above the
|
||||
// 5000ms default the local unit entry grants, so the case carries its own
|
||||
// bound instead of relying on the lane to widen it.
|
||||
}, 20_000)
|
||||
|
||||
it('charges a lone surrogate its full six escaped bytes, not three', async () => {
|
||||
// A forged `log` frame carrying `\ud800` escapes materializes lone
|
||||
// surrogates after JSON.parse. `Buffer.byteLength` of U+FFFD is 3, but
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Context } from '@deepseek-ai/cordis'
|
||||
import { expect, it, vi } from 'vitest'
|
||||
import { logTruncationMarker } from '../src/protocol.ts'
|
||||
|
||||
// Keep the interpreter and pipe lifecycle real; only OS-dependent read sizes
|
||||
// change. Each byte reaches the runtime as its own data event.
|
||||
@@ -54,3 +55,33 @@ it('seals stray fragments without recopying the sealed prefix', async () => {
|
||||
await fiber.dispose()
|
||||
}
|
||||
}, 40_000)
|
||||
|
||||
it.each([
|
||||
{ name: 'illegal UTF-8 bytes', payload: 'b"\\xff" * 3200' },
|
||||
{ name: 'CESU-8 lone surrogates', payload: 'b"\\xed\\xa0\\x80" * 1100' },
|
||||
])('bounds $name by their U+FFFD-decoded cost', async ({ payload }) => {
|
||||
const ctx = new Context()
|
||||
const fiber = await ctx.plugin(PythonCodeRuntime, { maxLogBytes: 3072, maxWallMs: 30_000 })
|
||||
const realConcat = Buffer.concat.bind(Buffer)
|
||||
let maxConcat = 0
|
||||
const concat = vi.spyOn(Buffer, 'concat').mockImplementation((list, total) => {
|
||||
const merged = realConcat(list, total)
|
||||
maxConcat = Math.max(maxConcat, merged.length)
|
||||
return merged
|
||||
})
|
||||
try {
|
||||
const result = await ctx.codeRuntime.run({
|
||||
program: `import os\nos.write(1, ${payload})\nreturn None`,
|
||||
bindings: [],
|
||||
})
|
||||
expect(result.error).toBeUndefined()
|
||||
expect(result.logs.at(-1)).toBe(logTruncationMarker(3072))
|
||||
// Each raw byte decodes to U+FFFD (three UTF-8 bytes), so a 3072-byte
|
||||
// budget flushes near 1024 raw bytes. Charging raw or structural widths
|
||||
// instead retains over 2048 bytes before flushing these payloads.
|
||||
expect(maxConcat).toBeLessThan(2048)
|
||||
} finally {
|
||||
concat.mockRestore()
|
||||
await fiber.dispose()
|
||||
}
|
||||
}, 20_000)
|
||||
Reference in new issue
Block a user