Merge pull request #5929 from deepseek-harness/fix/sensevoice-vad-onset-clipping

fix(speech-to-text-sensevoice): restore the onset a cold VAD drops

docs/config-catalog.md

@@ -1218,6 +1218,8 @@ export interface Config {
   minSpeechSeconds: number
   /** Silence separating two speech segments. */
   minSilenceSeconds: number
+  /** Recording audio restored ahead of the first VAD segment, which cold start reports late; zero disables it. */
+  vadOnsetPaddingSeconds: number
   /** Maximum decoded WAV bytes accepted by the private worker. */
   maxAudioBytes: number
   /** Deadline for runtime preparation and cold model loading. */

docs/config-catalog.zh.md

@@ -1220,6 +1220,8 @@ export interface Config {
   minSpeechSeconds: number
   /** Silence separating two speech segments. */
   minSilenceSeconds: number
+  /** Recording audio restored ahead of the first VAD segment, which cold start reports late; zero disables it. */
+  vadOnsetPaddingSeconds: number
   /** Maximum decoded WAV bytes accepted by the private worker. */
   maxAudioBytes: number
   /** Deadline for runtime preparation and cold model loading. */

packages/experimental/speech-to-text-sensevoice/README.i18n.yaml

@@ -18,8 +18,8 @@
   en: 5205fc2456b2c2c2
   zh: ed9dbcb71d48eb98
 /deepseek-ai-dsh-experimental-speech-to-text-sensevoice/understand-the-implementation:
-  en: 73f576b08a6f2639
-  zh: e7189948a7f2c3dd
+  en: d014ffd7d400cb04
+  zh: bc1a23e8bb3bdcca
 /deepseek-ai-dsh-experimental-speech-to-text-sensevoice/further-exploration:
   en: 5c64426add9d79d6
   zh: 4e06aa2ead8d2ccd

packages/experimental/speech-to-text-sensevoice/README.md

@@ -43,7 +43,7 @@ Every activation checks the selected model, tokens and VAD on disk. Complete ver
 <details>
 <summary>Maintainer details — click to expand</summary>
 
-One managed Node subprocess serializes inference. Recordings submitted during wake-up join the bounded queue. At execution, the worker requires ready or standby resources. Transcription never starts preparation downloads. `threads`, `segmentSeconds`, `vadThreshold`, `minSpeechSeconds` and `minSilenceSeconds` tune CPU inference and segmentation; `maxPending` bounds accepted work. Preparation and inference have separate deadlines. Invalid language or WAV input is rejected before native inference and retains the loaded worker; inference failures, malformed responses and transport failures reclaim it. Cancellation terminates an active worker and joins its process range; cancelled queued recordings never run. `idleTimeoutMs` releases the worker after inactivity, with zero retaining it until disposal. Models remain cached. Authenticated loopback requests keep audio in memory. Electron workers run in Node mode and copy VAD buffers to satisfy the V8 memory cage.
+One managed Node subprocess serializes inference. Recordings submitted during wake-up join the bounded queue. At execution, the worker requires ready or standby resources. Transcription never starts preparation downloads. `threads`, `segmentSeconds`, `vadThreshold`, `minSpeechSeconds`, `minSilenceSeconds` and `vadOnsetPaddingSeconds` tune CPU inference and segmentation, the last restoring the onset a cold detector reports late; `maxPending` bounds accepted work. Preparation and inference have separate deadlines. Invalid language or WAV input is rejected before native inference and retains the loaded worker; inference failures, malformed responses and transport failures reclaim it. Cancellation terminates an active worker and joins its process range; cancelled queued recordings never run. `idleTimeoutMs` releases the worker after inactivity, with zero retaining it until disposal. Models remain cached. Authenticated loopback requests keep audio in memory. Electron workers run in Node mode and copy VAD buffers to satisfy the V8 memory cage.
 
 The Host owns preparation across page and Session changes: check resources, download recognition weights and tokens, download VAD, verify files, and load the worker. Explicit model paths omit their download steps. Downloads report bytes; other steps report elapsed time. An idle worker retains prepared paths and needs only model loading on wake-up. Cancellation after a preparation result is published waits for cleanup without replacing that result.
 

packages/experimental/speech-to-text-sensevoice/README.zh.md

@@ -43,7 +43,7 @@ kind: "package-reference"
 <details>
 <summary>维护者信息 — 点击展开</summary>
 
-一个托管 Node 子进程串行执行推理。唤醒期间提交的录音进入有界队列。工作进程在执行时要求资源处于就绪或待唤醒状态。转写不会启动准备下载。`threads`、`segmentSeconds`、`vadThreshold`、`minSpeechSeconds` 和 `minSilenceSeconds` 调节 CPU 推理与分段；`maxPending` 限制已接收任务数。准备和推理分别有截止时间。无效语言或 WAV 输入在原生推理前被拒绝，保留已加载的工作进程；推理失败、响应格式错误和传输失败会回收进程。取消会终止运行中的工作进程并等待其进程范围退出；已取消的排队录音不会执行。`idleTimeoutMs` 在空闲后释放进程，零表示保留至插件释放。模型继续保留在缓存中。带认证的环回请求仅在内存中保存音频。Electron 工作进程使用 Node 模式，并复制 VAD 缓冲区以兼容 V8 内存笼。
+一个托管 Node 子进程串行执行推理。唤醒期间提交的录音进入有界队列。工作进程在执行时要求资源处于就绪或待唤醒状态。转写不会启动准备下载。`threads`、`segmentSeconds`、`vadThreshold`、`minSpeechSeconds`、`minSilenceSeconds` 和 `vadOnsetPaddingSeconds` 调节 CPU 推理与分段，其中最后一项回填冷启动检测器迟报的起音；`maxPending` 限制已接收任务数。准备和推理分别有截止时间。无效语言或 WAV 输入在原生推理前被拒绝，保留已加载的工作进程；推理失败、响应格式错误和传输失败会回收进程。取消会终止运行中的工作进程并等待其进程范围退出；已取消的排队录音不会执行。`idleTimeoutMs` 在空闲后释放进程，零表示保留至插件释放。模型继续保留在缓存中。带认证的环回请求仅在内存中保存音频。Electron 工作进程使用 Node 模式，并复制 VAD 缓冲区以兼容 V8 内存笼。
 
 Host 在页面和 Session 变化期间拥有准备任务：检查资源、下载识别权重与词表、下载 VAD、校验文件、加载工作进程。显式模型路径会省略对应下载步骤。下载报告字节，其他步骤报告等待时间。空闲进程保留已准备路径，唤醒时只需加载模型。准备结果发布后再取消，只等待任务收尾，不会覆盖该结果。
 

packages/experimental/speech-to-text-sensevoice/src/config.ts

@@ -30,6 +30,8 @@ export interface Config {
   minSpeechSeconds: number
   /** Silence separating two speech segments. */
   minSilenceSeconds: number
+  /** Recording audio restored ahead of the first VAD segment, which cold start reports late; zero disables it. */
+  vadOnsetPaddingSeconds: number
   /** Maximum decoded WAV bytes accepted by the private worker. */
   maxAudioBytes: number
   /** Deadline for runtime preparation and cold model loading. */
@@ -66,6 +68,7 @@ export const Config: z<Partial<Config>, Config> = z.object({
   vadThreshold: z.number().min(0).max(1).default(0.5),
   minSpeechSeconds: z.number().min(0).default(0.25),
   minSilenceSeconds: z.number().min(0.01).default(0.5),
+  vadOnsetPaddingSeconds: z.number().min(0).max(1).default(0.3),
   maxAudioBytes: z.natural().min(46).default(4 * 1024 * 1024),
   prepareTimeoutMs: z.natural().min(1).max(MAX_TIMER_DELAY_MS).default(3_600_000),
   inferenceTimeoutMs: z.natural().min(1).max(MAX_TIMER_DELAY_MS).default(120_000),

packages/experimental/speech-to-text-sensevoice/src/inference.ts

@@ -21,7 +21,7 @@ interface Recognizer {
 interface Detector {
   acceptWaveform(samples: Float32Array): void
   isEmpty(): boolean
-  front(externalBuffer: false): { samples: Float32Array }
+  front(externalBuffer: false): { start: number; samples: Float32Array }
   pop(): void
   reset(): void
   flush(): void
@@ -58,12 +58,27 @@ export function createTranscriber(config: InferenceConfig): (audio: Uint8Array,
     recognizer.setConfig(nativeConfig)
     detector.reset()
     const started = performance.now(), texts: string[] = []
+    // Silero cannot back-date the first segment of a cold detector, so it reports a start index
+    // several frames after speech actually began; restore that dropped onset from the recording.
+    const onsetPadding = Math.round(config.vadOnsetPaddingSeconds * 16000)
+    let isFirstSegment = true
     const drain = (): void => {
       while (!detector.isEmpty()) {
         // Electron's V8 memory cage requires copied native buffers, including VAD output.
         const segment = detector.front(false)
+        let speech = segment.samples
+        if (isFirstSegment) {
+          isFirstSegment = false
+          const from = Math.max(0, segment.start - onsetPadding)
+          if (from < segment.start) {
+            const restored = new Float32Array(segment.start - from + speech.length)
+            restored.set(samples.subarray(from, segment.start))
+            restored.set(speech, segment.start - from)
+            speech = restored
+          }
+        }
         const stream = recognizer.createStream()
-        stream.acceptWaveform({ sampleRate: 16000, samples: segment.samples })
+        stream.acceptWaveform({ sampleRate: 16000, samples: speech })
         recognizer.decode(stream)
         texts.push(recognizer.getResult(stream).text.trim())
         detector.pop()

packages/experimental/speech-to-text-sensevoice/tests/inference.spec.ts

@@ -4,13 +4,18 @@ import { Config } from '../src/config.ts'
 import { createTranscriber } from '../src/inference.ts'
 
 const native = vi.hoisted(() => ({ speech: true, texts: ['第一句。', '第二句。'], configs: [] as object[],
-  front: vi.fn(), resets: vi.fn(), accepted: [] as number[], windows: [] as number[] }))
+  front: vi.fn(), resets: vi.fn(), accepted: [] as number[], windows: [] as number[],
+  starts: [] as number[], heads: [] as number[] }))
 vi.mock('node:module', () => {
   return { createRequire: () => () => ({
     OfflineRecognizer: class {
       constructor(config: object) { native.configs.push(structuredClone(config)) }
       setConfig(config: object) { native.configs.push(structuredClone(config)) }
-      createStream() { return { acceptWaveform: ({ samples }: { samples: Float32Array }) => { native.accepted.push(samples.length) } } }
+      createStream() {
+        return { acceptWaveform: ({ samples }: { samples: Float32Array }) => {
+          native.accepted.push(samples.length); native.heads.push(samples[0] ?? 0)
+        } }
+      }
       decode() {}
       getResult() { return { text: native.texts.shift() ?? '' } }
     },
@@ -19,18 +24,22 @@ vi.mock('node:module', () => {
       constructor(config: object) { native.configs.push(structuredClone(config)) }
       acceptWaveform(samples: Float32Array) { native.windows.push(samples.length); this.ready = native.speech }
       isEmpty() { return !this.ready }
-      front(externalBuffer: false) { native.front(externalBuffer); return { samples: new Float32Array(512) } }
+      front(externalBuffer: false) {
+        native.front(externalBuffer)
+        return { start: native.starts.shift() ?? 0, samples: new Float32Array(512).fill(0.5) }
+      }
       pop() { this.ready = false }
       reset() { native.resets(); this.ready = false }
       flush() {}
     },
   }) }
 })
-function wave(samples = 1024): Uint8Array {
+function wave(samples = 1024, first = 0): Uint8Array {
   const b = Buffer.alloc(44 + samples * 2)
   b.write('RIFF'); b.writeUInt32LE(b.length - 8, 4); b.write('WAVEfmt ', 8); b.writeUInt32LE(16, 16)
   b.writeUInt16LE(1, 20); b.writeUInt16LE(1, 22); b.writeUInt32LE(16000, 24); b.writeUInt32LE(32000, 28)
   b.writeUInt16LE(2, 32); b.writeUInt16LE(16, 34); b.write('data', 36); b.writeUInt32LE(samples * 2, 40)
+  for (let i = 0; i < samples; i++) b.writeInt16LE(first + i, 44 + i * 2)
   return b
 }
 it('loads one model pair, carries language changes, and copies VAD buffers for Electron', () => {
@@ -53,3 +62,24 @@ it('rejects unsupported language, malformed audio and recordings above the worke
   expect(() => transcribe(new Uint8Array(46), 'zh')).toThrow('WAV')
   expect(() => transcribe(wave(), 'zh')).toThrow('byte limit')
 })
+it('restores the onset a cold VAD reports late, for the first segment only and within the window', () => {
+  const transcribe = createTranscriber(Object.assign(Config({ dataRoot: '/cache' }), { model: '/model', tokens: '/tokens', vad: '/vad' }))
+  const run = (start: number, samples: number, first = 0): void => {
+    native.speech = true; native.accepted.length = 0; native.heads.length = 0
+    native.starts = [start]; native.texts = ['第一句。', '第二句。']
+    transcribe(wave(samples, first), 'zh')
+  }
+  run(900, 1024, 1000)
+  expect(native.accepted).toEqual([900 + 512, 512])
+  expect(native.heads[0]).toBeCloseTo(1000 / 32768, 6)
+  expect(native.heads[1]).toBeCloseTo(0.5, 6)
+  run(6000, 8192)
+  expect(native.accepted[0]).toBe(0.3 * 16000 + 512)
+  run(0, 1024)
+  expect(native.accepted).toEqual([512, 512])
+  const off = createTranscriber(Object.assign(Config({ dataRoot: '/cache', vadOnsetPaddingSeconds: 0 }), { model: '/model', tokens: '/tokens', vad: '/vad' }))
+  native.speech = true; native.accepted.length = 0
+  native.starts = [900]; native.texts = ['第一句。', '第二句。']
+  off(wave(1024, 1000), 'zh')
+  expect(native.accepted).toEqual([512, 512])
+})