From 47ae64ee68b416cc0aa78891892ae9e9be6590c5 Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Thu, 17 Sep 2026 12:39:53 +0800 Subject: [PATCH 01/12] feat(voice-input): add optional local SenseVoice dictation --- ...6-09-16-experimental-voice-input.i18n.yaml | 6 + .../2026-09-16-experimental-voice-input.md | 35 +++ .../2026-09-16-experimental-voice-input.zh.md | 35 +++ THIRD_PARTY_NOTICES.md | 1 + apps/cli/package.json | 3 +- apps/desktop/README.i18n.yaml | 4 +- apps/desktop/README.md | 2 + apps/desktop/README.zh.md | 2 + apps/desktop/electron-builder.config.d.mts | 1 + apps/desktop/electron-builder.config.mjs | 1 + apps/desktop/src/main.ts | 2 + apps/desktop/src/microphone-permissions.ts | 31 ++ apps/desktop/tests/macos-signature.spec.ts | 1 + apps/desktop/tests/main-startup.spec.ts | 3 +- .../tests/microphone-permissions.spec.ts | 51 ++++ apps/web/package.json | 4 +- apps/web/tests/scaffold.ts | 6 +- apps/web/tests/voice-input.e2e.ts | 82 +++++ apps/web/tsconfig.json | 1 + docs/capability-seams.i18n.yaml | 4 +- docs/capability-seams.md | 11 + docs/capability-seams.zh.md | 11 + docs/config-catalog.i18n.yaml | 4 +- docs/config-catalog.md | 90 ++++++ docs/config-catalog.zh.md | 90 ++++++ docs/subsystems/README.i18n.yaml | 4 +- docs/subsystems/README.md | 1 + docs/subsystems/README.zh.md | 1 + docs/subsystems/slots.i18n.yaml | 4 +- docs/subsystems/slots.md | 2 + docs/subsystems/slots.zh.md | 2 + docs/subsystems/voice-input.i18n.yaml | 6 + docs/subsystems/voice-input.md | 158 ++++++++++ docs/subsystems/voice-input.zh.md | 158 ++++++++++ packages/boot/app-boot/src/profile.ts | 1 + .../tests/message-image.client.spec.tsx | 2 + .../ui-chat/tests/chat-view.client.spec.tsx | 2 + .../client/ui-conversation/README.i18n.yaml | 4 +- packages/client/ui-conversation/README.md | 4 + packages/client/ui-conversation/README.zh.md | 4 + .../ui-conversation/src/client/apply.ts | 1 + .../src/client/contract/input.ts | 9 + .../src/client/contract/slots.ts | 10 +- .../src/client/input/editor/runtime.ts | 13 + .../src/client/input/facade.ts | 6 + .../src/client/skeleton/InputBar.module.css | 7 + .../src/client/skeleton/InputBar.tsx | 21 +- .../tests/async-insertion.client.spec.ts | 38 +++ .../tests/input-bar.client.spec.tsx | 23 +- .../client/ui-plugin-manager/README.i18n.yaml | 4 +- packages/client/ui-plugin-manager/README.md | 2 + .../client/ui-plugin-manager/README.zh.md | 2 + .../src/client/PluginManagerPage.tsx | 8 +- .../ui-plugin-manager/src/client/index.ts | 3 +- .../ui-plugin-manager/src/client/locales.ts | 4 + .../src/client/presentation.ts | 1 + .../src/client/slot-contract.ts | 8 + .../tests/components.client.spec.tsx | 11 + .../client/ui-primitives/README.i18n.yaml | 4 +- packages/client/ui-primitives/README.md | 2 +- packages/client/ui-primitives/README.zh.md | 2 +- .../ui-primitives/src/StateDot.module.css | 5 + .../client/ui-primitives/src/StateDot.tsx | 9 +- .../client/ui-primitives/src/icons/index.tsx | 7 + .../tests/state-dot.client.spec.tsx | 8 + .../ui-trajectory/tests/views.client.spec.tsx | 4 + .../tests/plan-review-panel.client.spec.tsx | 2 + .../user-questions-composer.client.spec.tsx | 2 + .../tests/workflow-run.client.spec.tsx | 2 + packages/experimental/README.i18n.yaml | 4 +- packages/experimental/README.md | 5 + packages/experimental/README.zh.md | 5 + .../api-speech-to-text/README.i18n.yaml | 6 + .../experimental/api-speech-to-text/README.md | 78 +++++ .../api-speech-to-text/README.zh.md | 78 +++++ .../api-speech-to-text/package.json | 60 ++++ .../api-speech-to-text/src/index.ts | 110 +++++++ .../api-speech-to-text/src/types.ts | 25 ++ .../tests/controller.spec.ts | 62 ++++ .../api-speech-to-text/tests/wave.spec.ts | 28 ++ .../api-speech-to-text/tsconfig.json | 24 ++ .../api-speech-to-text/tsdown.config.ts | 3 + .../client-ui-voice-input/README.i18n.yaml | 6 + .../client-ui-voice-input/README.md | 76 +++++ .../client-ui-voice-input/README.zh.md | 76 +++++ .../client-ui-voice-input/package.json | 97 ++++++ .../src/client/PreparationCard.tsx | 142 +++++++++ .../src/client/VoiceInput.module.css | 36 +++ .../src/client/VoiceInput.tsx | 151 ++++++++++ .../src/client/Waveform.tsx | 33 +++ .../client-ui-voice-input/src/client/audio.ts | 137 +++++++++ .../client-ui-voice-input/src/client/index.ts | 15 + .../src/client/locales.ts | 160 ++++++++++ .../client-ui-voice-input/src/client/mount.ts | 61 ++++ .../src/client/readiness.ts | 40 +++ .../src/css-modules.d.ts | 5 + .../client-ui-voice-input/src/index.ts | 3 + .../tests/audio.client.spec.ts | 128 ++++++++ .../tests/mount.client.spec.ts | 87 ++++++ .../tests/preparation.client.spec.tsx | 134 +++++++++ .../tests/readiness.client.spec.ts | 59 ++++ .../tests/voice-input.client.spec.tsx | 202 +++++++++++++ .../tests/waveform.client.spec.tsx | 25 ++ .../client-ui-voice-input/tsconfig.json | 60 ++++ .../client-ui-voice-input/tsdown.config.ts | 3 + .../README.i18n.yaml | 6 + .../speech-to-text-sensevoice/README.md | 78 +++++ .../speech-to-text-sensevoice/README.zh.md | 78 +++++ .../speech-to-text-sensevoice/package.json | 53 ++++ .../runtime/assets.json | 28 ++ .../speech-to-text-sensevoice/src/config.ts | 71 +++++ .../speech-to-text-sensevoice/src/index.ts | 43 +++ .../src/inference.ts | 81 +++++ .../src/process-server.ts | 51 ++++ .../src/recognizer.ts | 280 ++++++++++++++++++ .../speech-to-text-sensevoice/src/runtime.ts | 126 ++++++++ .../speech-to-text-sensevoice/src/worker.ts | 13 + .../tests/inference.spec.ts | 55 ++++ .../tests/installer.spec.ts | 82 +++++ .../tests/local.e2e.ts | 31 ++ .../tests/process-server.spec.ts | 30 ++ .../tests/protocol.spec.ts | 77 +++++ .../tests/provider.spec.ts | 37 +++ .../tests/runtime.spec.ts | 62 ++++ .../tests/worker-source.fixture.ts | 2 + .../tests/worker.fixture.mjs | 22 ++ .../tests/worker.spec.ts | 269 +++++++++++++++++ .../speech-to-text-sensevoice/tsconfig.json | 27 ++ .../tsdown.config.ts | 3 + .../speech-to-text/README.i18n.yaml | 6 + .../experimental/speech-to-text/README.md | 80 +++++ .../experimental/speech-to-text/README.zh.md | 80 +++++ .../experimental/speech-to-text/package.json | 50 ++++ .../experimental/speech-to-text/src/index.ts | 204 +++++++++++++ .../experimental/speech-to-text/src/types.ts | 115 +++++++ .../experimental/speech-to-text/src/wave.ts | 23 ++ .../speech-to-text/tests/service.spec.ts | 152 ++++++++++ .../experimental/speech-to-text/tsconfig.json | 24 ++ .../speech-to-text/tsdown.config.ts | 3 + .../voice-input-bundle/README.i18n.yaml | 6 + .../experimental/voice-input-bundle/README.md | 76 +++++ .../voice-input-bundle/README.zh.md | 76 +++++ .../voice-input-bundle/cordis.patch.yml | 11 + .../voice-input-bundle/package.json | 50 ++++ .../voice-input-bundle/src/index.ts | 2 + .../voice-input-bundle/tsconfig.json | 15 + .../voice-input-bundle/tsdown.config.ts | 3 + .../src/client/slot-catalog.ts | 103 ++++++- .../extensions/tool-cordis/src/api-catalog.ts | 174 +++++++++++ pnpm-lock.yaml | 236 ++++++++++++++- scripts/check-workspace-constraints.spec.ts | 10 + scripts/check-workspace-constraints.ts | 1 + scripts/gen-cordis-catalog.ts | 20 ++ scripts/gen-doc-graphs.ts | 16 + .../verify-package-readme-model-experience.ts | 5 + .../web/voice-input/recording.expected.md | 8 + snapshots/web/voice-input/snapshot.yml | 9 + snapshots/web/voice-input/ui.expected.md | 22 ++ tsconfig.base.json | 10 + tsconfig.client.json | 2 + tsconfig.host.json | 6 + 161 files changed, 6420 insertions(+), 59 deletions(-) create mode 100644 .agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.i18n.yaml create mode 100644 .agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.md create mode 100644 .agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.zh.md create mode 100644 apps/desktop/src/microphone-permissions.ts create mode 100644 apps/desktop/tests/microphone-permissions.spec.ts create mode 100644 apps/web/tests/voice-input.e2e.ts create mode 100644 docs/subsystems/voice-input.i18n.yaml create mode 100644 docs/subsystems/voice-input.md create mode 100644 docs/subsystems/voice-input.zh.md create mode 100644 packages/client/ui-conversation/tests/async-insertion.client.spec.ts create mode 100644 packages/experimental/api-speech-to-text/README.i18n.yaml create mode 100644 packages/experimental/api-speech-to-text/README.md create mode 100644 packages/experimental/api-speech-to-text/README.zh.md create mode 100644 packages/experimental/api-speech-to-text/package.json create mode 100644 packages/experimental/api-speech-to-text/src/index.ts create mode 100644 packages/experimental/api-speech-to-text/src/types.ts create mode 100644 packages/experimental/api-speech-to-text/tests/controller.spec.ts create mode 100644 packages/experimental/api-speech-to-text/tests/wave.spec.ts create mode 100644 packages/experimental/api-speech-to-text/tsconfig.json create mode 100644 packages/experimental/api-speech-to-text/tsdown.config.ts create mode 100644 packages/experimental/client-ui-voice-input/README.i18n.yaml create mode 100644 packages/experimental/client-ui-voice-input/README.md create mode 100644 packages/experimental/client-ui-voice-input/README.zh.md create mode 100644 packages/experimental/client-ui-voice-input/package.json create mode 100644 packages/experimental/client-ui-voice-input/src/client/PreparationCard.tsx create mode 100644 packages/experimental/client-ui-voice-input/src/client/VoiceInput.module.css create mode 100644 packages/experimental/client-ui-voice-input/src/client/VoiceInput.tsx create mode 100644 packages/experimental/client-ui-voice-input/src/client/Waveform.tsx create mode 100644 packages/experimental/client-ui-voice-input/src/client/audio.ts create mode 100644 packages/experimental/client-ui-voice-input/src/client/index.ts create mode 100644 packages/experimental/client-ui-voice-input/src/client/locales.ts create mode 100644 packages/experimental/client-ui-voice-input/src/client/mount.ts create mode 100644 packages/experimental/client-ui-voice-input/src/client/readiness.ts create mode 100644 packages/experimental/client-ui-voice-input/src/css-modules.d.ts create mode 100644 packages/experimental/client-ui-voice-input/src/index.ts create mode 100644 packages/experimental/client-ui-voice-input/tests/audio.client.spec.ts create mode 100644 packages/experimental/client-ui-voice-input/tests/mount.client.spec.ts create mode 100644 packages/experimental/client-ui-voice-input/tests/preparation.client.spec.tsx create mode 100644 packages/experimental/client-ui-voice-input/tests/readiness.client.spec.ts create mode 100644 packages/experimental/client-ui-voice-input/tests/voice-input.client.spec.tsx create mode 100644 packages/experimental/client-ui-voice-input/tests/waveform.client.spec.tsx create mode 100644 packages/experimental/client-ui-voice-input/tsconfig.json create mode 100644 packages/experimental/client-ui-voice-input/tsdown.config.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/README.i18n.yaml create mode 100644 packages/experimental/speech-to-text-sensevoice/README.md create mode 100644 packages/experimental/speech-to-text-sensevoice/README.zh.md create mode 100644 packages/experimental/speech-to-text-sensevoice/package.json create mode 100644 packages/experimental/speech-to-text-sensevoice/runtime/assets.json create mode 100644 packages/experimental/speech-to-text-sensevoice/src/config.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/src/index.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/src/inference.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/src/process-server.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/src/recognizer.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/src/runtime.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/src/worker.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/tests/inference.spec.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/tests/installer.spec.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/tests/local.e2e.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/tests/process-server.spec.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/tests/protocol.spec.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/tests/provider.spec.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/tests/runtime.spec.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/tests/worker-source.fixture.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/tests/worker.fixture.mjs create mode 100644 packages/experimental/speech-to-text-sensevoice/tests/worker.spec.ts create mode 100644 packages/experimental/speech-to-text-sensevoice/tsconfig.json create mode 100644 packages/experimental/speech-to-text-sensevoice/tsdown.config.ts create mode 100644 packages/experimental/speech-to-text/README.i18n.yaml create mode 100644 packages/experimental/speech-to-text/README.md create mode 100644 packages/experimental/speech-to-text/README.zh.md create mode 100644 packages/experimental/speech-to-text/package.json create mode 100644 packages/experimental/speech-to-text/src/index.ts create mode 100644 packages/experimental/speech-to-text/src/types.ts create mode 100644 packages/experimental/speech-to-text/src/wave.ts create mode 100644 packages/experimental/speech-to-text/tests/service.spec.ts create mode 100644 packages/experimental/speech-to-text/tsconfig.json create mode 100644 packages/experimental/speech-to-text/tsdown.config.ts create mode 100644 packages/experimental/voice-input-bundle/README.i18n.yaml create mode 100644 packages/experimental/voice-input-bundle/README.md create mode 100644 packages/experimental/voice-input-bundle/README.zh.md create mode 100644 packages/experimental/voice-input-bundle/cordis.patch.yml create mode 100644 packages/experimental/voice-input-bundle/package.json create mode 100644 packages/experimental/voice-input-bundle/src/index.ts create mode 100644 packages/experimental/voice-input-bundle/tsconfig.json create mode 100644 packages/experimental/voice-input-bundle/tsdown.config.ts create mode 100644 snapshots/web/voice-input/recording.expected.md create mode 100644 snapshots/web/voice-input/snapshot.yml create mode 100644 snapshots/web/voice-input/ui.expected.md diff --git a/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.i18n.yaml b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.i18n.yaml new file mode 100644 index 0000000000..ec0c24bf9c --- /dev/null +++ b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.md +2026-09-16-experimental-voice-input.md: f385f4dc2d872f467aace6a07b09a11acdf8ab9f +2026-09-16-experimental-voice-input.zh.md: 15e55efe33905487f660f1373bc9c425378d9e46 diff --git a/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.md b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.md new file mode 100644 index 0000000000..f385f4dc2d --- /dev/null +++ b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.md @@ -0,0 +1,35 @@ +# Agent Note: Experimental voice input with explicit recognition providers + +Status: implemented + +English | [中文](2026-09-16-experimental-voice-input.zh.md) + +## Problem + +Dictation must preserve user control over when a coding task starts. Local recognition brings model files, a native runtime and CPU work; merely installing or enabling a UI contribution must not allocate these resources. Later cloud providers need a selection mechanism that does not silently change where recordings are processed. + +## Decision + +Five public packages under `packages/experimental/` compose one optional, default-disabled voice input bundle. Service Definition, SenseVoice Provider and Remote Consumer are separate because provider deployment and browser transport evolve independently. The browser UI dynamically mounts its generated Remote contribution. Stable API Remotes do not acquire an experimental dependency. + +Provider ids select exact registrations. Resolution captures an instance, and deregistration closes admission, cancels and joins accepted work. There is no provider fallback: a local failure cannot authorize uploading speech to a cloud service. A new cloud provider supplies its own credentials and explicit processing location through the same registry. + +Recordings and transcripts remain transient until the user submits ordinary text. Recognition does not start an agent turn or create an audio Session event. The input facade accepts asynchronous plain text against a captured draft revision, preserving reference chips and undo history. Changed drafts retain the recognized text for explicit insertion instead of replacing later edits. Session changes and plugin disposal invalidate late results and release recording resources. + +SenseVoice ONNX and Silero VAD run in one managed CPU subprocess. The native sherpa-onnx package includes ONNX Runtime; explicit preparation only downloads and verifies models; subsequent calls reuse the warm model until the configured idle deadline. Disabling the provider joins its work and process exit while retaining reusable disk assets. Native browser recording and resampling avoid an additional audio conversion executable. The authenticated private loopback protocol carries complete PCM recordings, allowing cancellation to terminate the worker without sharing a long-running inference with another request. + +The Host owns preparation independently of the view that started it. An ordered step list survives view remounts and records completion only as execution advances. Recognition weights, VAD and final verification have separate operations; downloads use pinned byte counts, while other work reports elapsed time. Package status uses `plugins.bundle.status` and details use the upstream `plugins.bundle.config`, both keyed by package name. A single composer activity Slot expands the voice controls without replacing the draft editor. Shared primitives provide buttons, disclosure and status markers; only measured waveform drawing belongs to the voice UI. Recordings start on click and transcribe on Stop, with feedback inside the activity. Persistent preferences remain in plugin settings. + +## Alternatives considered + +Always loading the model on plugin activation would make a UI toggle incur substantial startup time and memory even without dictation. Python/FunASR/PyTorch adds interpreter provisioning, dependency installation and disk cost; completed ONNX artifacts and platform-native packages serve CPU dictation. End users never export models. Browser-native speech recognition does not provide a uniform local-processing guarantee. Streaming voice interaction would require partial-result revision, turn arbitration and bidirectional audio semantics that complete-recording dictation does not need. + +## Consequences + +First use needs download time and disk space. Local means the Host machine, including a remote Host. A cancelled active inference loses its warm worker and the next request reloads it. Optional preparation paths support existing offline environments without silently changing defaults. + +Verification covers exact-provider routing, registration teardown, bounded audio parsing, microphone release, stale-draft and Session-switch handling, worker queue/deadline recovery, and verified download publication. Real local inference uses the same managed worker as the product. Browser composition verifies optional enablement and draft insertion; ordinary Session replay verifies that only the submitted text reaches the model. + +Real Node 24 and Electron 44 children complete FP32 and INT8 inference. Electron copies VAD output to avoid external-buffer restrictions. Three runs of the 4.2-second public Chinese sample agree; this short sample is not a complete accuracy evaluation. Desktop microphone requests are restricted to audio from the primary application window, with macOS authorization handled by the operating system. Windows, Linux and signed installers still require platform acceptance. + +INT8 is the default to reduce download and storage cost; `precision: fp32` selects the reference weights. Public English and concatenated bilingual samples show word differences between precisions, so quantization is not treated as accuracy-equivalent. A roughly 101-second repeated-speech sample completes in both runtimes; its observed post-request RSS is not a peak-memory bound. diff --git a/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.zh.md b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.zh.md new file mode 100644 index 0000000000..15e55efe33 --- /dev/null +++ b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.zh.md @@ -0,0 +1,35 @@ +# Agent Note:使用显式识别 Provider 的实验性语音输入 + +Status: implemented + +[English](2026-09-16-experimental-voice-input.md) | 中文 + +## 问题 + +听写必须保留用户对编码任务启动时机的控制。本地识别带来模型文件、原生运行时和 CPU 工作;仅安装或启用 UI 贡献不应分配这些资源。之后的云端 Provider 需要明确的选择机制,不能悄悄改变录音的处理位置。 + +## 决策 + +`packages/experimental/` 下的五个公开包组合成一个默认禁用的可选语音输入 Bundle。服务定义、SenseVoice Provider 和 Remote 消费者分开,因为 Provider 部署与浏览器传输独立演化。浏览器 UI 动态挂载其生成的 Remote 贡献。稳定 API Remotes 不引入实验性依赖。 + +Provider id 选择确切的注册。解析捕获实例,注销先拒绝新任务,再取消并等待已接收的任务。不存在 Provider 回退:本地失败不能授权将语音上传至云端服务。新增云端 Provider 通过同一注册表提供自己的凭据配置与显式处理位置。 + +录音与转写在用户提交普通文字前都是临时数据。识别不启动智能体轮次,也不创建音频 Session 事件。输入门面按照捕获的草稿版本接收异步纯文本,保留引用芯片与撤销历史。草稿变化时保留识别文字,等待显式插入,而非替换之后的编辑。切换 Session 和释放插件会使迟到结果失效并释放录音资源。 + +SenseVoice ONNX 与 Silero VAD 在一个托管 CPU 子进程中运行。原生 sherpa-onnx 包包含 ONNX Runtime;显式准备只下载并校验模型;后续调用复用预热模型,直到配置的空闲截止时间。禁用 Provider 等待其任务和进程退出,同时保留可复用的磁盘资源。浏览器原生录音与重采样避免增加音频转换可执行文件。带认证的私有环回协议传输完整 PCM 录音,取消时可以终止工作进程,而不与另一请求共享长时间推理。 + +Host 独立于发起视图拥有准备任务。有序步骤列表在视图重新挂载后保留,仅随着执行推进记录完成。识别模型、VAD 与最终校验分别准备;下载使用固定字节数,其他工作报告等待时间。包状态使用 `plugins.bundle.status`,详情使用上游 `plugins.bundle.config`,均以包名为 key。单个输入框活动 Slot 展开语音控件,同时保留草稿编辑器。按钮、折叠与状态标记复用共享原语,语音 UI 只负责实测波形绘制。点击开始录音,停止后转写,反馈留在活动栏内。持久化偏好仍在插件设置中。 + +## 考虑的替代方案 + +在插件启用时加载模型,会让一个 UI 开关在尚未听写时就产生大量启动时间和内存占用。Python/FunASR/PyTorch 增加解释器、依赖安装与磁盘成本;ONNX 成品和平台原生包满足 CPU 听写。用户端不执行模型导出。浏览器原生语音识别不能提供一致的本地处理保证。流式语音交互需要部分结果修订、轮次仲裁与双向音频语义,而完整录音听写无需这些机制。 + +## 影响与验证 + +首次使用需要下载时间和磁盘空间。本地指 Host 所在机器,包括远程 Host。取消运行中的推理会丢失预热进程,下次请求重新加载。可选准备路径支持已有离线环境,不会悄悄改变默认行为。 + +验证覆盖精确 Provider 路由、注册释放、有界音频解析、麦克风释放、过期草稿和 Session 切换处理、工作进程队列与截止时间恢复,以及经过校验的下载发布。真实本地推理使用与产品相同的托管工作进程。浏览器组合验证可选启用与草稿插入;普通 Session 回放验证只有提交的文字进入模型。 + +Node 24 与 Electron 44 的真实子进程均完成 FP32 和 INT8 识别。Electron 的 VAD 输出使用复制缓冲区,避免 external buffer 限制。4.2 秒公开中文样本的三次识别结果一致;该短样本不代表完整准确率评测。桌面麦克风请求限制为主应用窗口的音频请求,macOS 权限由系统处理。Windows、Linux 真实机器及签名后的安装包仍需平台验收。 + +INT8 以降低下载和存储成本为默认值,`precision: fp32` 选择参考权重。公开英文与拼接的中英双语样本显示两种精度存在用词差异,因此不将量化视为准确率等效。约 101 秒的重复语音样本在两种运行时均完成识别;请求后的 RSS 观测不构成峰值内存上限。 diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index d4838dfe32..3b1f408f0f 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -109,6 +109,7 @@ External packages installed for runtime use or distributed inside the prebuilt b | [`resolve.exports`](https://github.com/lukeed/resolve.exports) | MIT | | [`semver`](https://github.com/npm/node-semver) | ISC | | [`sharp`](https://github.com/lovell/sharp) | Apache-2.0 | +| [`sherpa-onnx-node`](https://github.com/csukuangfj/sherpa-onnx) | Apache-2.0 | | [`shiki`](https://github.com/shikijs/shiki) | MIT | | [`simple-icons`](https://github.com/simple-icons/simple-icons) | CC0-1.0 | | [`supports-color`](https://github.com/chalk/supports-color) | MIT | diff --git a/apps/cli/package.json b/apps/cli/package.json index 5204f55a29..1945ba8424 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -105,7 +105,8 @@ "@deepseek-ai/dsh-http-proxy": "workspace:^", "@deepseek-ai/dsh-mcp-resources": "workspace:^", "@deepseek-ai/dsh-plugin-manager": "workspace:^", - "@deepseek-ai/dsh-atomic-write": "workspace:^" + "@deepseek-ai/dsh-atomic-write": "workspace:^", + "@deepseek-ai/dsh-experimental-voice-input-bundle": "workspace:^" }, "devDependencies": { "@agentclientprotocol/sdk": "1.4.0", diff --git a/apps/desktop/README.i18n.yaml b/apps/desktop/README.i18n.yaml index de2415f74b..6755dce11c 100644 --- a/apps/desktop/README.i18n.yaml +++ b/apps/desktop/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write apps/desktop/README.md -README.md: 393f41f82ed20fbc71b1659cd2417a8d7f9b5013 -README.zh.md: e5bdb032b47795dc28848561f08f0464f83e49d3 +README.md: 179992c17a3d93dd0bcb65567dd4bb6748306460 +README.zh.md: 0357908160f88566bff9d760905b4522d9a73d05 diff --git a/apps/desktop/README.md b/apps/desktop/README.md index 393f41f82e..179992c17a 100644 --- a/apps/desktop/README.md +++ b/apps/desktop/README.md @@ -8,6 +8,8 @@ Desktop’s local native directory flow opens an Electron folder dialog attached Creator and the Web Plugin Manager use Desktop’s bundled pnpm under Electron Node mode without requiring pnpm on PATH. The private Node launcher environment applies only to package operations. +Desktop microphone access is restricted to audio requests from the primary `dsh-app://app` frame. macOS uses system microphone authorization and a packaged usage description. Native speech libraries remain outside ASAR and use the existing runtime signing path; recognition runs in a managed Node-mode child. + ## Key technical decisions The original artwork lives in `resources/icon.png` and `resources/icon.svg`; platform adaptations retain the whale and gradients in `resources/icon-windows.*` and `resources/icon-macos.*`. Export each platform SVG as a transparent 1024×1024 PNG. Electron-builder generates the multi-size ICO for the Windows application, installer, and uninstaller ([Windows icon requirements](https://learn.microsoft.com/en-us/windows/apps/design/iconography/app-icon-construction)). The installation pages use matching artwork in both themes; the uninstaller's welcome and finish pages share `installer/assets/uninstaller-sidebar.png`, converted to a 164×314 BMP during preparation. diff --git a/apps/desktop/README.zh.md b/apps/desktop/README.zh.md index e5bdb032b4..0357908160 100644 --- a/apps/desktop/README.zh.md +++ b/apps/desktop/README.zh.md @@ -8,6 +8,8 @@ Desktop 的本地原生目录流程打开绑定应用窗口的 Electron 文件 Creator 和 Web Plugin Manager 在 Electron Node 模式下使用 Desktop 内置 pnpm,无需 PATH 中存在 pnpm。私有 Node 启动器环境仅应用于包操作。 +桌面麦克风访问仅允许主 `dsh-app://app` 页面发起的音频请求。macOS 使用系统麦克风授权与随包用途说明。原生语音库保留在 ASAR 外并使用既有运行时签名路径;识别在托管的 Node 模式子进程中执行。 + ## 关键技术决策 设计师原稿位于 `resources/icon.png` 和 `resources/icon.svg`;平台适配保留鲸鱼与渐变,分别位于 `resources/icon-windows.*` 和 `resources/icon-macos.*`。将各平台 SVG 导出为透明的 1024×1024 PNG。electron-builder 为 Windows 应用、安装程序和卸载程序生成多尺寸 ICO([Windows 图标要求](https://learn.microsoft.com/en-us/windows/apps/design/iconography/app-icon-construction))。安装页面在两种主题下使用匹配的图案;卸载程序的欢迎和完成页共用 `installer/assets/uninstaller-sidebar.png`,准备阶段将其转换为 164×314 BMP。 diff --git a/apps/desktop/electron-builder.config.d.mts b/apps/desktop/electron-builder.config.d.mts index 3aefaec12f..01dcbd0e99 100644 --- a/apps/desktop/electron-builder.config.d.mts +++ b/apps/desktop/electron-builder.config.d.mts @@ -15,6 +15,7 @@ export interface DesktopElectronBuilderConfig { readonly asarUnpack: readonly string[] readonly extraResources: readonly [{ readonly from: string, readonly to: 'runtime' }] readonly mac: { + readonly extendInfo: { readonly NSMicrophoneUsageDescription: string } readonly identity: string | undefined readonly forceCodeSigning: boolean readonly notarize: boolean diff --git a/apps/desktop/electron-builder.config.mjs b/apps/desktop/electron-builder.config.mjs index ca846818b6..98d41b64a9 100644 --- a/apps/desktop/electron-builder.config.mjs +++ b/apps/desktop/electron-builder.config.mjs @@ -102,6 +102,7 @@ export function createElectronBuilderConfig( identity: macOSSigning?.signingIdentity, forceCodeSigning: true, hardenedRuntime: true, + extendInfo: { NSMicrophoneUsageDescription: 'DeepSeek Harness uses your microphone to transcribe speech into message drafts.' }, // ASAR-unpacked native runtime files are pre-signed; PAK resources are sealed by their enclosing bundle. signIgnore: ['/Contents/Resources/app\\.asar\\.unpacked/dsh(?:/|$)', '/Contents/Resources/runtime/primary-runtime(?:/|$)', '\\.pak$'], notarize: true, diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index 4eb65c0521..e82289da82 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -20,6 +20,7 @@ import { resolveDesktopPaths } from './paths.ts' import { DesktopProjectManager, type DesktopProjectHooks } from './project-manager.ts' import { DesktopHostProcess } from './host-process.ts' import { installDesktopDirectoryPicker } from './directory-picker.ts' +import { installMicrophonePermissions } from './microphone-permissions.ts' import { DesktopBackendController } from './backend-controller.ts' import { DESKTOP_IPC, SCHEME, assertDesktopSender, type DesktopUpdateState } from './ipc.ts' import { formatDesktopMessage, resolveDesktopLocale } from './locale.ts' @@ -289,6 +290,7 @@ async function main(): Promise { }) installDesktopDirectoryPicker(() => mainWindow) + installMicrophonePermissions(session.defaultSession, () => mainWindow?.webContents) ipcMain.handle(DESKTOP_IPC.boot, async (event) => { assertDesktopSender(event, ['app']) diff --git a/apps/desktop/src/microphone-permissions.ts b/apps/desktop/src/microphone-permissions.ts new file mode 100644 index 0000000000..ba644ac80f --- /dev/null +++ b/apps/desktop/src/microphone-permissions.ts @@ -0,0 +1,31 @@ +/** Microphone access belongs to the primary application frame and the operating system. */ +import { systemPreferences, type Session, type WebContents } from 'electron' + +function applicationFrame(url: string): boolean { + try { + const parsed = new URL(url) + return parsed.protocol === 'dsh-app:' && parsed.hostname === 'app' + } catch (_error) { return false /* Invalid frame URLs cannot request microphone access. */ } +} + +/** + * Handle microphone requests from the owned application window; other permissions retain Electron's defaults. + * @param session - application's browser session. + * @param primary - current primary window contents, absent while no window is open. + */ +export function installMicrophonePermissions(session: Session, primary: () => WebContents | undefined): void { + session.setPermissionCheckHandler((contents, permission, origin, details) => { + if (permission !== 'media') return true + return contents != null && contents === primary() && details.isMainFrame + && applicationFrame(origin) && details.mediaType === 'audio' + && (process.platform !== 'darwin' || systemPreferences.getMediaAccessStatus('microphone') === 'granted') + }) + session.setPermissionRequestHandler((contents, permission, callback, details) => { + if (permission !== 'media') { callback(true); return } + const allowed = contents === primary() && details.isMainFrame && applicationFrame(details.requestingUrl) + && 'mediaTypes' in details && details.mediaTypes.length === 1 && details.mediaTypes[0] === 'audio' + if (!allowed) { callback(false); return } + if (process.platform !== 'darwin') { callback(true); return } + void systemPreferences.askForMediaAccess('microphone').then(callback, () => { callback(false) }) + }) +} diff --git a/apps/desktop/tests/macos-signature.spec.ts b/apps/desktop/tests/macos-signature.spec.ts index 3286f23e89..514e150ab5 100644 --- a/apps/desktop/tests/macos-signature.spec.ts +++ b/apps/desktop/tests/macos-signature.spec.ts @@ -40,6 +40,7 @@ describe('desktop macOS release signature', () => { const { createElectronBuilderConfig } = await import('../electron-builder.config.mjs') const config = createElectronBuilderConfig(RELEASE_ENVIRONMENT, 'darwin', 'arm64') expect(portablePath(config.directories.output)).toContain('/.desktop-build/targets/mac-arm64/artifacts') + expect(config.mac.extendInfo.NSMicrophoneUsageDescription).toContain('microphone') expect(config.extraResources).toHaveLength(1) expect(config.extraResources[0]?.to).toBe('runtime') expect(portablePath(config.extraResources[0]?.from ?? '')).toContain('/.desktop-build/targets/mac-arm64/runtime') diff --git a/apps/desktop/tests/main-startup.spec.ts b/apps/desktop/tests/main-startup.spec.ts index 4856b0a5cb..96e1e1d5fb 100644 --- a/apps/desktop/tests/main-startup.spec.ts +++ b/apps/desktop/tests/main-startup.spec.ts @@ -135,7 +135,8 @@ vi.mock('electron', () => ({ handle: (channel: string, handler: (event: { senderFrame: { url: string } }) => unknown) => { if (harness.handlers.has(channel)) throw new Error(`duplicate IPC handler ${channel}`); harness.handlers.set(channel, handler) }, }, Menu: harness.menu, - session: { defaultSession: { webRequest: { onBeforeSendHeaders: harness.socketHeaders } } }, + session: { defaultSession: { setPermissionCheckHandler: vi.fn(), setPermissionRequestHandler: vi.fn(), + webRequest: { onBeforeSendHeaders: harness.socketHeaders } } }, protocol: { registerSchemesAsPrivileged: vi.fn(), handle: vi.fn() }, })) vi.mock('../src/paths.ts', () => ({ resolveDesktopPaths: () => ({ profile: 'desktop-test-profile' }) })) diff --git a/apps/desktop/tests/microphone-permissions.spec.ts b/apps/desktop/tests/microphone-permissions.spec.ts new file mode 100644 index 0000000000..9cdd416481 --- /dev/null +++ b/apps/desktop/tests/microphone-permissions.spec.ts @@ -0,0 +1,51 @@ +/** Desktop media permissions exclude external pages, subframes and camera requests. */ +import type { Session, WebContents } from 'electron' +import { afterEach, expect, it, vi } from 'vitest' +import { installMicrophonePermissions } from '../src/microphone-permissions.ts' + +const access = vi.hoisted(() => ({ getMediaAccessStatus: vi.fn(() => 'granted'), askForMediaAccess: vi.fn(async () => true) })) +vi.mock('electron', () => ({ systemPreferences: access })) +afterEach(() => { vi.unstubAllGlobals(); vi.clearAllMocks() }) +function fixture(platform: NodeJS.Platform = 'darwin') { + vi.stubGlobal('process', { ...process, platform }) + const setPermissionCheckHandler = vi.fn() + const setPermissionRequestHandler = vi.fn() + const primary = {} as WebContents + installMicrophonePermissions({ setPermissionCheckHandler, setPermissionRequestHandler } as unknown as Session, () => primary) + return { primary, check: setPermissionCheckHandler.mock.calls[0]![0]!, request: setPermissionRequestHandler.mock.calls[0]![0]! } +} +it('allows only audio checks from the owned primary application frame', () => { + const f = fixture(), details = { isMainFrame: true, mediaType: 'audio' as const } + expect(f.check(f.primary, 'media', 'dsh-app://app', details)).toBe(true) + expect(f.check(null, 'media', 'dsh-app://app', details)).toBe(false) + expect(f.check({} as WebContents, 'media', 'dsh-app://app', details)).toBe(false) + expect(f.check(f.primary, 'media', 'https://example.com', details)).toBe(false) + expect(f.check(f.primary, 'media', 'invalid', details)).toBe(false) + expect(f.check(f.primary, 'media', 'dsh-app://app', { ...details, isMainFrame: false })).toBe(false) + expect(f.check(f.primary, 'media', 'dsh-app://app', { ...details, mediaType: 'video' })).toBe(false) + access.getMediaAccessStatus.mockReturnValueOnce('denied') + expect(f.check(f.primary, 'media', 'dsh-app://app', details)).toBe(false) + expect(f.check(f.primary, 'clipboard-sanitized-write', 'dsh-app://app', details)).toBe(true) +}) +it('requests macOS microphone access and reports operating-system rejection', async () => { + const f = fixture(), done = vi.fn(), details = { isMainFrame: true, requestingUrl: 'dsh-app://app/', mediaTypes: ['audio'] } + f.request(f.primary, 'media', done, details) + await vi.waitFor(() => { expect(done).toHaveBeenLastCalledWith(true) }) + expect(access.askForMediaAccess).toHaveBeenCalledWith('microphone') + access.askForMediaAccess.mockRejectedValueOnce(new Error('permission denied')) + f.request(f.primary, 'media', done, details) + await vi.waitFor(() => { expect(done).toHaveBeenLastCalledWith(false) }) + for (const denied of [{ ...details, mediaTypes: ['audio', 'video'] }, { ...details, mediaTypes: [] }, + { ...details, isMainFrame: false }, { isMainFrame: true, requestingUrl: 'dsh-app://app/' }]) { + f.request(f.primary, 'media', done, denied); expect(done).toHaveBeenLastCalledWith(false) + } + f.request(f.primary, 'clipboard-sanitized-write', done, details) + expect(done).toHaveBeenLastCalledWith(true) +}) +it.each(['win32', 'linux'] as const)('leaves system authorization to Chromium on %s', (platform) => { + const f = fixture(platform), done = vi.fn() + expect(f.check(f.primary, 'media', 'dsh-app://app', { isMainFrame: true, mediaType: 'audio' })).toBe(true) + f.request(f.primary, 'media', done, { isMainFrame: true, requestingUrl: 'dsh-app://app/', mediaTypes: ['audio'] }) + expect(done).toHaveBeenCalledWith(true) + expect(access.askForMediaAccess).not.toHaveBeenCalled() +}) diff --git a/apps/web/package.json b/apps/web/package.json index acdb74244a..a4858c3199 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -60,6 +60,8 @@ "@deepseek-ai/dsh-launch-environment": "workspace:^", "@deepseek-ai/dsh-workspace-changes": "workspace:^", "@deepseek-ai/dsh-subprocess-local": "workspace:^", - "@deepseek-ai/dsh-experimental-auto-review": "workspace:^" + "@deepseek-ai/dsh-experimental-auto-review": "workspace:^", + "@deepseek-ai/dsh-experimental-speech-to-text": "workspace:^", + "@deepseek-ai/dsh-experimental-voice-input-bundle": "workspace:^" } } diff --git a/apps/web/tests/scaffold.ts b/apps/web/tests/scaffold.ts index f807eb83c6..8a83895643 100644 --- a/apps/web/tests/scaffold.ts +++ b/apps/web/tests/scaffold.ts @@ -305,8 +305,8 @@ export interface LaunchOptions { profileResolutionMode?: Extract /** Enable the real Open In rows with deterministic launch-environment facts. */ openInAppEnvironment?: LaunchEnvironmentSnapshot - /** Compare the replayed root session with `replayFixture`; defaults on for a manifest-owned canonical recording. */ - compareReplaySession?: boolean + /** Compare the replayed root Session; `read-only` also forbids refresh writes to a borrowed fixture. */ + compareReplaySession?: boolean | 'read-only' /** * Optional product overlay applied after the shipped Web surface and before * the scaffold's hermetic test patches, matching the launcher's `--patch` @@ -920,7 +920,7 @@ export async function launchWebScaffold(options: LaunchOptions = {}): Promise { + const scratch = await mkdtemp(join(tmpdir(), 'dsh-voice-browser-')) + const resources: { scaffold?: WebScaffold; browser?: Browser } = {} + onTestFinished(async () => { + try { await resources.browser?.close() } finally { + try { await resources.scaffold?.close() } finally { await rm(scratch, { recursive: true, force: true }) } + } + }) + const overlay = join(scratch, 'voice.patch.yml') + await writeFile(overlay, '- id: speech-to-text-sensevoice\n disabled: true\n') + const prompt = fixtureUserPrompts(await readFile(fixture, 'utf8'))[0]! + const prefix = 'Use the bash tool to ' + const scaffold = await launchWebScaffold({ profile: { packages: [{ dir: bundle, enabled: true }] }, + extraOverlayPath: overlay, replayFixture: fixture, compareReplaySession: 'read-only', + }) + resources.scaffold = scaffold + const recognize = vi.fn(async (input: SpeechInput) => { + expect(Buffer.from(input.audio).subarray(0, 4).toString()).toBe('RIFF') + expect(input.audio.byteLength).toBeGreaterThan(44) + return { text: prompt.slice(prefix.length), audioSeconds: 1, inferenceSeconds: 0.1 } + }) + await scaffold.ctx.plugin({ inject: ['speechToText'], apply(ctx) { + ctx.effect(() => ctx.speechToText.register({ + info: { id: 'sensevoice-local' as SpeechProviderId, name: 'Recorded recognizer', location: 'host-local' }, + transcribe: recognize, + })) + } }) + let messages = 0 + scaffold.ctx.on('session/event', (_session, event) => { + if (event.type === 'user/message' && event.data.source.kind === 'user') messages++ + }) + const browser = await chromium.launch({ args: ['--use-fake-ui-for-media-stream', '--use-fake-device-for-media-stream'] }) + resources.browser = browser + const page = await newEnglishPage(browser), tripwire = watchConsole(page) + await page.goto(scaffold.authenticatedUrl) + await connectFreshWorkspace(page, scaffold.workspaceCwd) + const input = page.locator('[data-composer-input]'), mic = page.getByRole('button', { name: 'Start recording', exact: true }) + await mic.waitFor() + const micBox = await mic.boundingBox(), modelBox = await page.getByRole('button', { name: /^Select model, current/ }).boundingBox() + const sendBox = await page.getByRole('button', { name: 'Send message', exact: true }).boundingBox() + expect(micBox!.x).toBeGreaterThan(modelBox!.x) + expect(micBox!.x).toBeLessThan(sendBox!.x) + await input.fill(prefix) + await mic.click() + await page.getByRole('button', { name: 'Stop and transcribe', exact: true }).waitFor() + await compareOrRefreshGolden(recordingExpected, + await captureStableAria(page, '[data-composer-card]', scaffold.workspaceCwd), webSnapshotMode()) + expect(await page.getByRole('dialog').count()).toBe(0) + // Controlled fake microphone audio occupies this recording interval; it is not a readiness wait. + await page.waitForTimeout(300) + await page.getByRole('button', { name: 'Stop and transcribe', exact: true }).click() + await expect.poll(() => input.innerText()).toBe(prompt) + expect(recognize).toHaveBeenCalledOnce() + expect(messages).toBe(0) + await compareOrRefreshGolden(expected, await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd), webSnapshotMode()) + const settled = scaffold.whenTurnSettled() + await input.press('Enter') + await settled + await page.getByText('DONE', { exact: true }).waitFor() + expect(messages).toBe(1) + expect(tripwire.pageErrors).toEqual([]) +}, 120_000) diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json index 83b574f9bf..880d27e739 100644 --- a/apps/web/tsconfig.json +++ b/apps/web/tsconfig.json @@ -25,6 +25,7 @@ "tests/default-product-isolation.e2e.ts", "tests/diff-context.e2e.ts", "tests/scaffold.ts", + "tests/voice-input.e2e.ts", "tests/auto-review-fixture.ts", "tests/scaffold-generation.spec.ts", "tests/scaffold-hermetic.e2e.ts", diff --git a/docs/capability-seams.i18n.yaml b/docs/capability-seams.i18n.yaml index 7157fab563..f59bbdc570 100644 --- a/docs/capability-seams.i18n.yaml +++ b/docs/capability-seams.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/capability-seams.md -capability-seams.md: df39022715cf1000daca9fdeaf668e9cedf88bb1 -capability-seams.zh.md: fc9c8fd46375d77fb42fcc6cf0aef160527d100e +capability-seams.md: c97c2d97b9555f74bbb0151a46116da177eb5a8b +capability-seams.zh.md: c825f666b0466d52c82caab5212e88e5af1d2f9a diff --git a/docs/capability-seams.md b/docs/capability-seams.md index df39022715..c97c2d97b9 100644 --- a/docs/capability-seams.md +++ b/docs/capability-seams.md @@ -57,6 +57,8 @@ flowchart LR pkg_subagent_in_process_driver["subagent-in-process-driver"] pkg_invariants["invariants"] pkg_message_feedback["message-feedback"] + pkg_experimental_api_speech_to_text["experimental-api-speech-to-text"] + svc_speechController["ctx.speechController
Experimental transcription Remote"] svc_sessionController["ctx.sessionController
Host Session Remote controller"] svc_sessionFileReferences["ctx.sessionFileReferences
Session-addressed file-reference Remote adapter"] svc_sessionSkillCatalog["ctx.sessionSkillCatalog
Session-addressed skill Remote adapter"] @@ -207,6 +209,9 @@ flowchart LR pkg_subagent_dsh_sdk["subagent-dsh-sdk"] pkg_tool_subagent_control["tool-subagent-control"] pkg_tool_ralph["tool-ralph"] + pkg_experimental_speech_to_text["experimental-speech-to-text"] + svc_speechToText["ctx.speechToText
Experimental speech recognition providers"] + pkg_experimental_speech_to_text_sensevoice["experimental-speech-to-text-sensevoice"] pkg_experimental_agent_team["experimental-agent-team"] svc_agentTeams["ctx.agentTeams
Agent Teams coordination domain"] pkg_experimental_tool_agent_team["experimental-tool-agent-team"] @@ -284,12 +289,15 @@ flowchart LR pkg_credentials_local --> svc_credentials pkg_deepseek_llm_api_extensions --> svc_deepseekLlmApiExtensions pkg_experimental_agent_team --> svc_agentTeams + pkg_experimental_api_speech_to_text --> svc_speechController pkg_experimental_browser_use_chrome_devtools_mcp --> svc_browserUse pkg_experimental_browser_use_playwright_mcp --> svc_browserUse pkg_experimental_browser_use_stagehand_native --> svc_browserUse pkg_experimental_computer_use_cua_driver_mcp --> svc_computerUse pkg_experimental_computer_use_cua_driver_native --> svc_computerUse pkg_experimental_ptc_runtime_python --> svc_ptcRuntime + pkg_experimental_speech_to_text --> svc_speechToText + pkg_experimental_speech_to_text_sensevoice --> svc_speechToText pkg_file_reference --> svc_fileReferences pkg_file_reference_local --> svc_fileReferences pkg_fs --> svc_fs @@ -475,6 +483,7 @@ flowchart LR svc_shellEnv --> pkg_tool_bash svc_shellEnv --> pkg_tool_pwsh svc_skills --> pkg_tool_skill + svc_speechToText --> pkg_experimental_api_speech_to_text svc_spillStore --> pkg_spill_policy svc_ssh --> pkg_fs_ssh svc_ssh --> pkg_sandbox_ssh @@ -540,6 +549,7 @@ flowchart LR | `ctx.tokenMeter` | `core` | [`token-meter`](../packages/llm/token-meter) | - | [`compaction-basic`](../packages/compaction/compaction-basic) | - | Owns isolated per-session replay folds; pressure consumers share immutable revisioned measurements. | | `ctx.toolResultPruner` | `core` | [`compaction-tool-result-pruner`](../packages/compaction/compaction-tool-result-pruner) | - | [`compaction-basic`](../packages/compaction/compaction-basic) | - | Rewrites oversized current tool results through replayable single-node surface replacements before summary compaction. | | `ctx.sessions` | `core` | [`session`](../packages/core/session) | - | [`agent-loop`](../packages/core/agent-loop), [`agent`](../packages/core/agent), [`session-persistence`](../packages/session/session-persistence), [`session-query`](../packages/session-query/session-query), [`session-query-sqlite`](../packages/session-query/session-query-sqlite), [`subagent-in-process-driver`](../packages/subagent/subagent-in-process-driver), [`invariants`](../packages/runtime-diagnostics/invariants), [`message-feedback`](../packages/feedback/message-feedback) | - | Owns append-only Session instances and emits the durable session event feed. | +| `ctx.speechController` | `core` | [`experimental-api-speech-to-text`](../packages/experimental/api-speech-to-text) | - | - | - | Validates bounded browser audio before provider dispatch. | | `ctx.sessionController` | `core` | [`api-session-controller`](../packages/api/session-controller) | - | - | - | Owns Session commands, cold reads, durable-event following, live control state, model catalogs, workspace opening, and Agent activation policy. | | `ctx.sessionFileReferences` | `core` | [`api-session-controller`](../packages/api/session-controller) | - | - | - | Delegates file-reference discovery through the Session Controller's established Agent lookup policy. | | `ctx.sessionSkillCatalog` | `core` | [`api-session-controller`](../packages/api/session-controller) | - | - | - | Lists the Session composition's user-invocable skills without activating a cold Agent. | @@ -594,6 +604,7 @@ flowchart LR | `ctx.fs` | `seam` | [`fs`](../packages/fs/fs) | [`fs-local`](../packages/fs/fs-local), [`fs-sandbox`](../packages/fs/fs-sandbox), [`fs-ssh`](../packages/ssh/fs-ssh) | [`tool-fs`](../packages/fs/tool-fs) | [`fs-observation-policy`](../packages/fs/fs-observation-policy) | tool-fs executes read/write/edit through ctx.fs; fs-sandbox fences mutations by the shared sandbox mode; fs-observation-policy contributes observed-state checks through the fs/* event gate. | | `ctx.compaction` | `seam` | [`compaction`](../packages/compaction/compaction) | [`compaction-basic`](../packages/compaction/compaction-basic) | [`compaction-basic`](../packages/compaction/compaction-basic) | - | The basic backend consumes post-step pressure and request-error recovery events; there is no model-facing compact tool. | | `ctx.subagents` | `seam` | [`subagent`](../packages/subagent/subagent) | [`subagent-spawn-in-process`](../packages/subagent/subagent-spawn-in-process), [`subagent-fork-in-process`](../packages/subagent/subagent-fork-in-process), [`subagent-acp`](../packages/subagent/subagent-acp), [`subagent-codex`](../packages/subagent/subagent-codex), [`subagent-claude-code`](../packages/subagent/subagent-claude-code), [`subagent-dsh-sdk`](../packages/subagent/subagent-dsh-sdk) | [`tool-subagent`](../packages/subagent/tool-subagent), [`tool-subagent-control`](../packages/subagent/tool-subagent-control), [`tool-ralph`](../packages/workflow/tool-ralph) | - | Providers implement transports; the service also owns optional Activation-based continuation orchestration, tool-subagent selects one-shot or continuable delegation, tool-subagent-control delivers follow-ups, and tool-ralph requires one fresh structured-output route. | +| `ctx.speechToText` | `seam` | [`experimental-speech-to-text`](../packages/experimental/speech-to-text) | [`experimental-speech-to-text-sensevoice`](../packages/experimental/speech-to-text-sensevoice) | [`experimental-api-speech-to-text`](../packages/experimental/api-speech-to-text) | - | Routes explicit recognizers; the browser uses the authenticated Remote and keeps transcripts in the draft until submission. | | `ctx.agentTeams` | `core` | [`experimental-agent-team`](../packages/experimental/agent-team) | - | [`experimental-tool-agent-team`](../packages/experimental/tool-agent-team), [`experimental-client-ui-agent-team`](../packages/experimental/client-ui-agent-team) | - | Owns the implicit-root roster, durable peer mailbox, shared task DAG, continuable-child lifecycle, and generated Team Remote methods; tool-agent-team contributes model controls and client-ui-agent-team mounts the browser contribution. | | `ctx.inspector` | `core` | `inspector` | - | - | - | Owns the Worker-hosted CDP target and the transport-independent Host and Client observation and Cordis-tree query API. | | `ctx.jobs` | `seam` | [`jobs`](../packages/jobs/jobs) | [`jobs-local`](../packages/jobs/jobs-local) | [`tool-bash`](../packages/shell/tool-bash), [`tool-terminal`](../packages/terminal/tool-terminal), [`tool-subagent`](../packages/subagent/tool-subagent), [`tool-jobs`](../packages/jobs/tool-jobs) | - | Producers (background bash, PTY sends, and subagent delegations) register running work; tool-jobs is the model-facing controller that reads, lists, and kills it; jobs-local is the process-local registry. | diff --git a/docs/capability-seams.zh.md b/docs/capability-seams.zh.md index fc9c8fd463..c825f666b0 100644 --- a/docs/capability-seams.zh.md +++ b/docs/capability-seams.zh.md @@ -59,6 +59,8 @@ flowchart LR pkg_subagent_in_process_driver["subagent-in-process-driver"] pkg_invariants["invariants"] pkg_message_feedback["message-feedback"] + pkg_experimental_api_speech_to_text["experimental-api-speech-to-text"] + svc_speechController["ctx.speechController
Experimental transcription Remote"] svc_sessionController["ctx.sessionController
Host Session Remote controller"] svc_sessionFileReferences["ctx.sessionFileReferences
Session-addressed file-reference Remote adapter"] svc_sessionSkillCatalog["ctx.sessionSkillCatalog
Session-addressed skill Remote adapter"] @@ -209,6 +211,9 @@ flowchart LR pkg_subagent_dsh_sdk["subagent-dsh-sdk"] pkg_tool_subagent_control["tool-subagent-control"] pkg_tool_ralph["tool-ralph"] + pkg_experimental_speech_to_text["experimental-speech-to-text"] + svc_speechToText["ctx.speechToText
Experimental speech recognition providers"] + pkg_experimental_speech_to_text_sensevoice["experimental-speech-to-text-sensevoice"] pkg_experimental_agent_team["experimental-agent-team"] svc_agentTeams["ctx.agentTeams
Agent Teams coordination domain"] pkg_experimental_tool_agent_team["experimental-tool-agent-team"] @@ -286,12 +291,15 @@ flowchart LR pkg_credentials_local --> svc_credentials pkg_deepseek_llm_api_extensions --> svc_deepseekLlmApiExtensions pkg_experimental_agent_team --> svc_agentTeams + pkg_experimental_api_speech_to_text --> svc_speechController pkg_experimental_browser_use_chrome_devtools_mcp --> svc_browserUse pkg_experimental_browser_use_playwright_mcp --> svc_browserUse pkg_experimental_browser_use_stagehand_native --> svc_browserUse pkg_experimental_computer_use_cua_driver_mcp --> svc_computerUse pkg_experimental_computer_use_cua_driver_native --> svc_computerUse pkg_experimental_ptc_runtime_python --> svc_ptcRuntime + pkg_experimental_speech_to_text --> svc_speechToText + pkg_experimental_speech_to_text_sensevoice --> svc_speechToText pkg_file_reference --> svc_fileReferences pkg_file_reference_local --> svc_fileReferences pkg_fs --> svc_fs @@ -477,6 +485,7 @@ flowchart LR svc_shellEnv --> pkg_tool_bash svc_shellEnv --> pkg_tool_pwsh svc_skills --> pkg_tool_skill + svc_speechToText --> pkg_experimental_api_speech_to_text svc_spillStore --> pkg_spill_policy svc_ssh --> pkg_fs_ssh svc_ssh --> pkg_sandbox_ssh @@ -542,6 +551,7 @@ flowchart LR | `ctx.tokenMeter` | `core` | [`token-meter`](../packages/llm/token-meter) | - | [`compaction-basic`](../packages/compaction/compaction-basic) | - | 拥有按会话隔离的回放折叠区;压力消费方共享不可变且带修订版本的测量结果。 | | `ctx.toolResultPruner` | `core` | [`compaction-tool-result-pruner`](../packages/compaction/compaction-tool-result-pruner) | - | [`compaction-basic`](../packages/compaction/compaction-basic) | - | 在摘要压缩前,通过可回放的单节点表层替换来改写过大的当前工具结果。 | | `ctx.sessions` | `core` | [`session`](../packages/core/session) | - | [`agent-loop`](../packages/core/agent-loop), [`agent`](../packages/core/agent), [`session-persistence`](../packages/session/session-persistence), [`session-query`](../packages/session-query/session-query), [`session-query-sqlite`](../packages/session-query/session-query-sqlite), [`subagent-in-process-driver`](../packages/subagent/subagent-in-process-driver), [`invariants`](../packages/runtime-diagnostics/invariants), [`message-feedback`](../packages/feedback/message-feedback) | - | 拥有仅追加的 Session 实例,并发出持久的会话事件流。 | +| `ctx.speechController` | `core` | [`experimental-api-speech-to-text`](../packages/experimental/api-speech-to-text) | - | - | - | 在 Provider 调用前校验受限的浏览器音频。 | | `ctx.sessionController` | `core` | [`api-session-controller`](../packages/api/session-controller) | - | - | - | 负责 Session 命令、冷读取、持久事件跟随、实时控制状态、模型目录、workspace 打开与 Agent 激活策略。 | | `ctx.sessionFileReferences` | `core` | [`api-session-controller`](../packages/api/session-controller) | - | - | - | 通过 Session Controller 的既有 Agent lookup 策略委托文件引用发现。 | | `ctx.sessionSkillCatalog` | `core` | [`api-session-controller`](../packages/api/session-controller) | - | - | - | 在不激活冷 Agent 的前提下列出 Session 组合中允许用户调用的 skill。 | @@ -596,6 +606,7 @@ flowchart LR | `ctx.fs` | `seam` | [`fs`](../packages/fs/fs) | [`fs-local`](../packages/fs/fs-local), [`fs-sandbox`](../packages/fs/fs-sandbox), [`fs-ssh`](../packages/ssh/fs-ssh) | [`tool-fs`](../packages/fs/tool-fs) | [`fs-observation-policy`](../packages/fs/fs-observation-policy) | tool-fs 通过 ctx.fs 执行读取/写入/编辑;fs-sandbox 按共享沙箱模式限制变更;fs-observation-policy 通过 fs/* 事件门禁贡献基于观测状态的检查。 | | `ctx.compaction` | `seam` | [`compaction`](../packages/compaction/compaction) | [`compaction-basic`](../packages/compaction/compaction-basic) | [`compaction-basic`](../packages/compaction/compaction-basic) | - | 基础后端消费步骤后的压力事件和请求错误恢复事件;不存在面向模型的压缩工具。 | | `ctx.subagents` | `seam` | [`subagent`](../packages/subagent/subagent) | [`subagent-spawn-in-process`](../packages/subagent/subagent-spawn-in-process), [`subagent-fork-in-process`](../packages/subagent/subagent-fork-in-process), [`subagent-acp`](../packages/subagent/subagent-acp), [`subagent-codex`](../packages/subagent/subagent-codex), [`subagent-claude-code`](../packages/subagent/subagent-claude-code), [`subagent-dsh-sdk`](../packages/subagent/subagent-dsh-sdk) | [`tool-subagent`](../packages/subagent/tool-subagent), [`tool-subagent-control`](../packages/subagent/tool-subagent-control), [`tool-ralph`](../packages/workflow/tool-ralph) | - | 提供方实现传输;该服务还负责可选的、基于 Activation 的延续编排,tool-subagent 选择一次性或可延续委派,tool-subagent-control 传递后续消息,而 tool-ralph 要求一条全新的结构化输出路由。 | +| `ctx.speechToText` | `seam` | [`experimental-speech-to-text`](../packages/experimental/speech-to-text) | [`experimental-speech-to-text-sensevoice`](../packages/experimental/speech-to-text-sensevoice) | [`experimental-api-speech-to-text`](../packages/experimental/api-speech-to-text) | - | 路由显式选择的识别器;浏览器使用带认证的 Remote,并在提交前将转写保留在草稿中。 | | `ctx.agentTeams` | `core` | [`experimental-agent-team`](../packages/experimental/agent-team) | - | [`experimental-tool-agent-team`](../packages/experimental/tool-agent-team), [`experimental-client-ui-agent-team`](../packages/experimental/client-ui-agent-team) | - | 负责隐式 Root roster、持久 peer mailbox、共享任务 DAG、continuable child 生命周期与生成式 Team Remote method;tool-agent-team 提供模型控制工具,client-ui-agent-team 挂载浏览器 contribution。 | | `ctx.inspector` | `core` | `inspector` | - | - | - | 负责 Worker 托管的 CDP target,以及独立于传输的 Host 和 Client observation 与 Cordis tree query API。 | | `ctx.jobs` | `seam` | [`jobs`](../packages/jobs/jobs) | [`jobs-local`](../packages/jobs/jobs-local) | [`tool-bash`](../packages/shell/tool-bash), [`tool-terminal`](../packages/terminal/tool-terminal), [`tool-subagent`](../packages/subagent/tool-subagent), [`tool-jobs`](../packages/jobs/tool-jobs) | - | 生产方(后台 bash、PTY 发送和 subagent 委派)登记正在运行的工作;tool-jobs 是面向模型的控制器,用于读取、列出和终止这些工作;jobs-local 是进程本地注册表。 | diff --git a/docs/config-catalog.i18n.yaml b/docs/config-catalog.i18n.yaml index c049eab040..f576e2aea7 100644 --- a/docs/config-catalog.i18n.yaml +++ b/docs/config-catalog.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/config-catalog.md -config-catalog.md: c3e84ba47ad687915ae1c641e8600bd43a2102b6 -config-catalog.zh.md: b8b86030d96ebd8ea49cc6e545007a0c319720f3 +config-catalog.md: fe01cf82f37e94bb6d2709e8991652e252ebc196 +config-catalog.zh.md: 02d732455384e90c24a00869f2cde6d5d3a066ee diff --git a/docs/config-catalog.md b/docs/config-catalog.md index c3e84ba47a..fe01cf82f3 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -572,6 +572,24 @@ export interface Config { Source: [`packages/experimental/agent-team/src/types.ts:130`](../packages/experimental/agent-team/src/types.ts) + + +## `@deepseek-ai/dsh-experimental-api-speech-to-text` + +Requires: `speechToText` · `typert` + +```ts config-catalog +/** Limits applied before decoding or calling a provider. */ +export interface Config { + /** Maximum decoded WAV bytes per request. */ + maxAudioBytes: number + /** Maximum PCM recording duration in seconds. */ + maxDurationSeconds: number +} +``` + +Source: [`packages/experimental/api-speech-to-text/src/index.ts:20`](../packages/experimental/api-speech-to-text/src/index.ts) + ## `@deepseek-ai/dsh-experimental-browser-use-chrome-devtools-mcp` @@ -802,6 +820,76 @@ export interface Config { Source: [`packages/experimental/ptc-runtime-python/src/index.ts:42`](../packages/experimental/ptc-runtime-python/src/index.ts) + + +## `@deepseek-ai/dsh-experimental-speech-to-text` + +```ts config-catalog +/** Composition defaults resolved before a transcription starts. */ +export interface Config { + /** Registered provider selected when the caller omits an id. */ + defaultProvider: string + /** Provider language hint selected when the caller omits one. */ + language: string +} +``` + +Source: [`packages/experimental/speech-to-text/src/index.ts:18`](../packages/experimental/speech-to-text/src/index.ts) + + + +## `@deepseek-ai/dsh-experimental-speech-to-text-sensevoice` + +Requires: `speechToText` · `subprocess` + +```ts config-catalog +/** Local runtime, inference, and retention settings. */ +export interface Config { + /** Unique registration id; consumers select this exact id. */ + providerId: string + /** Absolute directory for verified ONNX models. */ + dataRoot: string + /** Existing directory containing the selected ONNX model and tokens.txt; omission downloads verified files. */ + modelDirectory?: string | undefined + /** Existing Silero VAD ONNX file; omission downloads the verified model. */ + vadModelPath?: string | undefined + /** Weight precision; INT8 minimizes first-use download and model storage. */ + precision: 'int8' | 'fp32' + /** Hugging Face-compatible origin for pinned model URLs, including private mirrors. */ + modelOrigin: string + /** CPU intra-operation thread count. */ + threads: number + /** Maximum speech segment length passed to the recognizer. */ + segmentSeconds: number + /** Silero speech probability threshold. */ + vadThreshold: number + /** Minimum speech duration retained by VAD. */ + minSpeechSeconds: number + /** Silence separating two speech segments. */ + minSilenceSeconds: number + /** Maximum decoded WAV bytes accepted by the private worker. */ + maxAudioBytes: number + /** Deadline for runtime preparation and cold model loading. */ + prepareTimeoutMs: number + /** Deadline for one inference after the worker is ready. */ + inferenceTimeoutMs: number + /** Idle period before stopping the worker; zero keeps it warm. */ + idleTimeoutMs: number + /** Maximum accepted running and waiting transcriptions. */ + maxPending: number + /** Managed process termination grace period. */ + graceMs: number + /** Maximum retained worker diagnostic bytes. */ + maxLogBytes: number + /** Maximum transcript response bytes. */ + maxResponseBytes: number + /** Minimum interval between intermediate download progress notifications. */ + progressIntervalMs: number +} +``` + +Source: [`packages/experimental/speech-to-text-sensevoice/src/config.ts:6`](../packages/experimental/speech-to-text-sensevoice/src/config.ts) + ## `@deepseek-ai/dsh-experimental-tool-agent-team` @@ -3761,6 +3849,7 @@ These load from a `cordis.yml` entry with no `config:` block; they declare no co - `@deepseek-ai/dsh-deepseek-llm-api-extensions` ([`packages/llm/deepseek-llm-api-extensions/src/index.ts`](../packages/llm/deepseek-llm-api-extensions/src/index.ts)) - `@deepseek-ai/dsh-experimental-auto-review` — requires `llm` · `permissionPresets` · `sessions` · `tools` ([`packages/experimental/auto-review/src/index.ts`](../packages/experimental/auto-review/src/index.ts)) - `@deepseek-ai/dsh-experimental-client-ui-agent-team` ([`packages/experimental/client-ui-agent-team/src/index.ts`](../packages/experimental/client-ui-agent-team/src/index.ts)) +- `@deepseek-ai/dsh-experimental-client-ui-voice-input` ([`packages/experimental/client-ui-voice-input/src/index.ts`](../packages/experimental/client-ui-voice-input/src/index.ts)) - `@deepseek-ai/dsh-experimental-computer-use-cua-driver-native` — requires `computerUse` · `tools` · `systemPrompt` ([`packages/experimental/computer-use-cua-driver-native/src/index.ts`](../packages/experimental/computer-use-cua-driver-native/src/index.ts)) - `@deepseek-ai/dsh-fs-observation-policy` ([`packages/fs/fs-observation-policy/src/index.ts`](../packages/fs/fs-observation-policy/src/index.ts)) - `@deepseek-ai/dsh-fs-ssh` — requires `ssh` · `sandboxPolicy` ([`packages/ssh/fs-ssh/src/index.ts`](../packages/ssh/fs-ssh/src/index.ts)) @@ -3834,6 +3923,7 @@ Imported as libraries by other packages; a `cordis.yml` cannot load them. - `@deepseek-ai/dsh-experimental-agent-team-profile` ([`packages/experimental/agent-team-profile/src/index.ts`](../packages/experimental/agent-team-profile/src/index.ts)) - `@deepseek-ai/dsh-experimental-agent-team-web-profile` ([`packages/experimental/agent-team-web-profile/src/index.ts`](../packages/experimental/agent-team-web-profile/src/index.ts)) - `@deepseek-ai/dsh-experimental-browser-use-runtime` ([`packages/experimental/browser-use-runtime/src/index.ts`](../packages/experimental/browser-use-runtime/src/index.ts)) +- `@deepseek-ai/dsh-experimental-voice-input-bundle` ([`packages/experimental/voice-input-bundle/src/index.ts`](../packages/experimental/voice-input-bundle/src/index.ts)) - `@deepseek-ai/dsh-experimental-webworker-packer` ([`packages/experimental/webworker-packer/src/index.ts`](../packages/experimental/webworker-packer/src/index.ts)) - `@deepseek-ai/dsh-experimental-webworker-runtime` ([`packages/experimental/webworker-runtime/src/index.ts`](../packages/experimental/webworker-runtime/src/index.ts)) - `@deepseek-ai/dsh-home-paths` ([`packages/util/home-paths/src/index.ts`](../packages/util/home-paths/src/index.ts)) diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index b8b86030d9..02d7324553 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -574,6 +574,24 @@ export interface Config { 来源:[`packages/experimental/agent-team/src/types.ts:130`](../packages/experimental/agent-team/src/types.ts) + + +## `@deepseek-ai/dsh-experimental-api-speech-to-text` + +需要: `speechToText` · `typert` + +```ts config-catalog +/** Limits applied before decoding or calling a provider. */ +export interface Config { + /** Maximum decoded WAV bytes per request. */ + maxAudioBytes: number + /** Maximum PCM recording duration in seconds. */ + maxDurationSeconds: number +} +``` + +来源: [`packages/experimental/api-speech-to-text/src/index.ts:20`](../packages/experimental/api-speech-to-text/src/index.ts) + ## `@deepseek-ai/dsh-experimental-browser-use-chrome-devtools-mcp` @@ -804,6 +822,76 @@ export interface Config { 来源:[`packages/experimental/ptc-runtime-python/src/index.ts:42`](../packages/experimental/ptc-runtime-python/src/index.ts) + + +## `@deepseek-ai/dsh-experimental-speech-to-text` + +```ts config-catalog +/** Composition defaults resolved before a transcription starts. */ +export interface Config { + /** Registered provider selected when the caller omits an id. */ + defaultProvider: string + /** Provider language hint selected when the caller omits one. */ + language: string +} +``` + +来源: [`packages/experimental/speech-to-text/src/index.ts:18`](../packages/experimental/speech-to-text/src/index.ts) + + + +## `@deepseek-ai/dsh-experimental-speech-to-text-sensevoice` + +需要: `speechToText` · `subprocess` + +```ts config-catalog +/** Local runtime, inference, and retention settings. */ +export interface Config { + /** Unique registration id; consumers select this exact id. */ + providerId: string + /** Absolute directory for verified ONNX models. */ + dataRoot: string + /** Existing directory containing the selected ONNX model and tokens.txt; omission downloads verified files. */ + modelDirectory?: string | undefined + /** Existing Silero VAD ONNX file; omission downloads the verified model. */ + vadModelPath?: string | undefined + /** Weight precision; INT8 minimizes first-use download and model storage. */ + precision: 'int8' | 'fp32' + /** Hugging Face-compatible origin for pinned model URLs, including private mirrors. */ + modelOrigin: string + /** CPU intra-operation thread count. */ + threads: number + /** Maximum speech segment length passed to the recognizer. */ + segmentSeconds: number + /** Silero speech probability threshold. */ + vadThreshold: number + /** Minimum speech duration retained by VAD. */ + minSpeechSeconds: number + /** Silence separating two speech segments. */ + minSilenceSeconds: number + /** Maximum decoded WAV bytes accepted by the private worker. */ + maxAudioBytes: number + /** Deadline for runtime preparation and cold model loading. */ + prepareTimeoutMs: number + /** Deadline for one inference after the worker is ready. */ + inferenceTimeoutMs: number + /** Idle period before stopping the worker; zero keeps it warm. */ + idleTimeoutMs: number + /** Maximum accepted running and waiting transcriptions. */ + maxPending: number + /** Managed process termination grace period. */ + graceMs: number + /** Maximum retained worker diagnostic bytes. */ + maxLogBytes: number + /** Maximum transcript response bytes. */ + maxResponseBytes: number + /** Minimum interval between intermediate download progress notifications. */ + progressIntervalMs: number +} +``` + +来源: [`packages/experimental/speech-to-text-sensevoice/src/config.ts:6`](../packages/experimental/speech-to-text-sensevoice/src/config.ts) + ## `@deepseek-ai/dsh-experimental-tool-agent-team` @@ -3763,6 +3851,7 @@ export interface Config { - `@deepseek-ai/dsh-deepseek-llm-api-extensions`([`packages/llm/deepseek-llm-api-extensions/src/index.ts`](../packages/llm/deepseek-llm-api-extensions/src/index.ts)) - `@deepseek-ai/dsh-experimental-auto-review` — 需要 `llm` · `permissionPresets` · `sessions` · `tools`([`packages/experimental/auto-review/src/index.ts`](../packages/experimental/auto-review/src/index.ts)) - `@deepseek-ai/dsh-experimental-client-ui-agent-team`([`packages/experimental/client-ui-agent-team/src/index.ts`](../packages/experimental/client-ui-agent-team/src/index.ts)) +- `@deepseek-ai/dsh-experimental-client-ui-voice-input` ([`packages/experimental/client-ui-voice-input/src/index.ts`](../packages/experimental/client-ui-voice-input/src/index.ts)) - `@deepseek-ai/dsh-experimental-computer-use-cua-driver-native` — requires `computerUse` · `tools` · `systemPrompt` ([`packages/experimental/computer-use-cua-driver-native/src/index.ts`](../packages/experimental/computer-use-cua-driver-native/src/index.ts)) - `@deepseek-ai/dsh-fs-observation-policy`([`packages/fs/fs-observation-policy/src/index.ts`](../packages/fs/fs-observation-policy/src/index.ts)) - `@deepseek-ai/dsh-fs-ssh` — 需要 `ssh` · `sandboxPolicy`([`packages/ssh/fs-ssh/src/index.ts`](../packages/ssh/fs-ssh/src/index.ts)) @@ -3835,6 +3924,7 @@ export interface Config { - `@deepseek-ai/dsh-experimental-agent-team-profile`([`packages/experimental/agent-team-profile/src/index.ts`](../packages/experimental/agent-team-profile/src/index.ts)) - `@deepseek-ai/dsh-experimental-agent-team-web-profile`([`packages/experimental/agent-team-web-profile/src/index.ts`](../packages/experimental/agent-team-web-profile/src/index.ts)) - `@deepseek-ai/dsh-experimental-browser-use-runtime` ([`packages/experimental/browser-use-runtime/src/index.ts`](../packages/experimental/browser-use-runtime/src/index.ts)) +- `@deepseek-ai/dsh-experimental-voice-input-bundle`([`packages/experimental/voice-input-bundle/src/index.ts`](../packages/experimental/voice-input-bundle/src/index.ts)) - `@deepseek-ai/dsh-experimental-webworker-packer`([`packages/experimental/webworker-packer/src/index.ts`](../packages/experimental/webworker-packer/src/index.ts)) - `@deepseek-ai/dsh-experimental-webworker-runtime`([`packages/experimental/webworker-runtime/src/index.ts`](../packages/experimental/webworker-runtime/src/index.ts)) - `@deepseek-ai/dsh-home-paths`([`packages/util/home-paths/src/index.ts`](../packages/util/home-paths/src/index.ts)) diff --git a/docs/subsystems/README.i18n.yaml b/docs/subsystems/README.i18n.yaml index 92e23f39f1..212654acc1 100644 --- a/docs/subsystems/README.i18n.yaml +++ b/docs/subsystems/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/subsystems/README.md -README.md: b4d6aa226979525ef658f9cf1d33d5ca3e59dcf6 -README.zh.md: 9134fcf37789911aec05dcdc9e9c3dc036a33873 +README.md: 34522c050a224fa5dae6ef78a324125adef59f72 +README.zh.md: d5a132f203cb16810a55c3b699a7a2fb2f37d217 diff --git a/docs/subsystems/README.md b/docs/subsystems/README.md index b4d6aa2269..34522c050a 100644 --- a/docs/subsystems/README.md +++ b/docs/subsystems/README.md @@ -45,6 +45,7 @@ One page per subsystem of the DeepSeek Harness: what it is, the data structures | [skills.md](skills.md) | the skill service: discovery priority, `SkillSummary`/`SkillDefinition`, session-prefix catalog, model-facing `skill` loading | | [compaction.md](compaction.md) | the compaction seam: the `compaction/*` session events, `CompactionResult`, the `CompactionEngine` interface | | [subagent.md](subagent.md) | the subagent seam: the named-provider registry, `SubagentStartRequest`/`Result`/`Run`, the start-time-vs-runtime capability split | +| [voice-input.md](voice-input.md) | experimental named recognizers, transient audio and revision-guarded draft insertion | | [agent-team.md](agent-team.md) | Agent Teams: implicit Lead identity, named continuable teammates, durable peer mailbox, and shared task DAG | | [web.md](web.md) | the web access seam: `WebSearchRequest`/`Result`, `WebFetchRequest`/`Result`, `WebFetchBody`, provider availability, `WebError` | | [spill.md](spill.md) | the spill storage seam: `SaveTextSpill`, `SpillOwner`/`SpillSource`, `SpillRef`, the branded `SpillLocator` | diff --git a/docs/subsystems/README.zh.md b/docs/subsystems/README.zh.md index 9134fcf377..d5a132f203 100644 --- a/docs/subsystems/README.zh.md +++ b/docs/subsystems/README.zh.md @@ -45,6 +45,7 @@ | [skills.md](skills.zh.md) | skill(技能)服务:发现优先级、`SkillSummary`/`SkillDefinition`、会话前缀目录、面向模型的 `skill` 加载 | | [compaction.md](compaction.zh.md) | 压缩(compaction)seam:`compaction/*` 会话事件、`CompactionResult`、`CompactionEngine` 接口 | | [subagent.md](subagent.zh.md) | subagent seam:命名提供方注册表、`SubagentStartRequest`/`Result`/`Run`、启动时与运行时能力拆分 | +| [voice-input.md](voice-input.zh.md) | 实验性具名语音识别、临时音频和受版本保护的草稿插入 | | [agent-team.md](agent-team.zh.md) | Agent Teams:隐式 Lead 身份、具名 continuable teammate、持久 peer mailbox 与共享任务 DAG | | [web.md](web.zh.md) | Web 访问 seam:`WebSearchRequest`/`Result`、`WebFetchRequest`/`Result`、`WebFetchBody`、提供方可用性、`WebError` | | [spill.md](spill.zh.md) | spill 存储 seam:`SaveTextSpill`、`SpillOwner`/`SpillSource`、`SpillRef`、品牌类型 `SpillLocator` | diff --git a/docs/subsystems/slots.i18n.yaml b/docs/subsystems/slots.i18n.yaml index d25c088405..30df105998 100644 --- a/docs/subsystems/slots.i18n.yaml +++ b/docs/subsystems/slots.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/subsystems/slots.md -slots.md: 177bbc5b8f56b7276f271a2ddb7ac71ac8888363 -slots.zh.md: c746ac88704c2e22e5c50ef82a893fde9f091668 +slots.md: ff1d69c08f1ff5093135d4a979eb86ae352147a4 +slots.zh.md: fd9ad5bc8c49febe9e39ba6ab19abd31a4a7d697 diff --git a/docs/subsystems/slots.md b/docs/subsystems/slots.md index 177bbc5b8f..ff1d69c08f 100644 --- a/docs/subsystems/slots.md +++ b/docs/subsystems/slots.md @@ -6,6 +6,8 @@ Slots are the Web Client's typed React composition system. [`dsh-client-ui-slots This page documents slot ownership, component inputs, extension APIs, and the shipped hierarchy. The surrounding boot, Remote, Client model, and Conversation paths are in [Web Client architecture](web-client.md). +`plugins.bundle.status` is keyed by npm package name and renders short status before the bundle switch; `plugins.bundle.config` supplies bundle detail configuration. `conversation.input.activity` supplies one action between the model selector and Send, with toolbar expansion released on unmount. + ## Declaration and lifecycle `SlotMap` is the compile-time registry. A package declaration-merges the key, cardinality, scope, owner props, keyed props, and optional slot-level inject face. The runtime declaration is the matching `children` entry on the component that owns the render location. diff --git a/docs/subsystems/slots.zh.md b/docs/subsystems/slots.zh.md index c746ac8870..fd9ad5bc8c 100644 --- a/docs/subsystems/slots.zh.md +++ b/docs/subsystems/slots.zh.md @@ -6,6 +6,8 @@ Slots 是 Web Client 的类型化 React 组合系统。[`dsh-client-ui-slots`](. 本文记录 slot 的所有权、组件输入、扩展 API 与当前层级。外围的启动、Remote、Client model 与 Conversation 数据通路见 [Web Client 架构](web-client.zh.md)。 +`plugins.bundle.status` 以 npm 包名为 key,在 Bundle 卡片开关前显示简短状态;`plugins.bundle.config` 提供 Bundle 详情配置。`conversation.input.activity` 提供模型选择器与发送按钮之间的单个活动控件,可展开工具栏并在卸载时释放展开状态。 + ## 声明与生命周期 `SlotMap` 是编译期注册表。包通过声明合并写入 key、cardinality(基数)、scope、owner props、keyed props 与可选的 slot 级 inject face。运行时声明则是拥有该渲染位置的组件在 `children` 中给出的对应条目。 diff --git a/docs/subsystems/voice-input.i18n.yaml b/docs/subsystems/voice-input.i18n.yaml new file mode 100644 index 0000000000..bde639522c --- /dev/null +++ b/docs/subsystems/voice-input.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write docs/subsystems/voice-input.md +voice-input.md: d72bf01a372413fbc33a93a8a3a77f5152a3807b +voice-input.zh.md: c45275516166820f9dfac2d7d96c3a43b70d72d8 diff --git a/docs/subsystems/voice-input.md b/docs/subsystems/voice-input.md new file mode 100644 index 0000000000..d72bf01a37 --- /dev/null +++ b/docs/subsystems/voice-input.md @@ -0,0 +1,158 @@ +# Voice input + +English | [中文](voice-input.zh.md) + +Experimental speech recognition has three roles: the [Service Definition](../../packages/experimental/speech-to-text/README.md) routes named providers, the [SenseVoice provider](../../packages/experimental/speech-to-text-sensevoice/README.md) owns local inference, and the [Remote consumer](../../packages/experimental/api-speech-to-text/README.md) serves the browser. The [optional bundle](../../packages/experimental/voice-input-bundle/README.md) composes them with the microphone UI. + +## Provider selection + +`SpeechProviderId` brands the registration identity. `SpeechProviderInfo` carries a display name and `host-local` or `cloud` processing location. `SpeechRequest` contains WAV bytes and optional provider/language selection; `resolve()` produces `SpeechSpec` with one captured provider instance. A missing provider fails; replacement or withdrawal invalidates resolved work. Audio never falls back to a different provider. + +`SpeechProvider.transcribe()` accepts one complete `SpeechInput` and an `AbortSignal`. `Transcript` returns plain `text`, `audioSeconds` and `inferenceSeconds`. Providers honor cancellation and settle their work before deregistration completes. The local provider serializes calls, bounds its queue and manages its worker lifetime independently of Sessions. + +## Browser and Host ownership + +The browser owns microphone tracks and the unsent draft. `TranscriptionRequest` sends canonical base64 PCM16 WAV through the authenticated Remote. `SpeechCatalog` advertises available providers, the default selection and recording byte/duration limits. The API validates this process input before recognition. + +The input facade captures a revision-bearing selection before recording. `InputActions.insertText()` inserts one undoable plain-text edit only while the selection revision is current and submission permits editing. A rejected insertion leaves the transcript available for explicit insertion. Switching Sessions or disposing the plugin invalidates late results. Recognition itself writes no Session event; ordinary user submission owns the final model-visible text. + +## Preparation and settings + +The Host provider owns one preparation task across page and Session changes. The Client shares one `follow()` subscription across the composer, plugin status, details. Optional `SpeechSetupEstimate` metadata supplies provider-specific planning hints, separate from measured progress. `SpeechPreparationStepKind` identifies ordered resource operations; `SpeechPreparationStep` records each status and start time. The complete `SpeechPreparationState` retains these steps after cancellation or failure. The collapsed UI shows the current operation; expanding lists all steps, with byte progress or elapsed time only for the running step. Closing an observer never cancels preparation. Verified files survive retries and idle worker reclamation. + +The microphone occupies `conversation.input.activity`, between the model selector and Send. Clicking starts capture and expands the toolbar; Stop transcribes and inserts into the draft. The activity preserves the editor and submit action, owns local feedback, and releases expansion on unmount. Cancel, Escape or hiding the page discards capture. Waveform history displays measured microphone amplitude. Bundle details store recognition preferences, and `plugins.bundle.status` displays short readiness beside the enable switch. + +## Design rationale + +The [voice input decision](../../.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.md) explains transient audio, explicit provider selection and lazy local preparation. + + + + + +## Cordis API + +Generated from source by `scripts/gen-cordis-catalog.ts` (verified fresh by `pnpm run verify-cordis-catalog` in doc-sync; regenerate with `pnpm run gen-cordis-catalog`) — the language sides differ only in locale-specific paired document paths. Signature blocks use a `ts cordis-catalog` fence and keep the original source JSDoc; dispatch modes are defined in the [primer](../cordis-primer.md#dispatch-modes), and the framework-inherited `ctx` API lives in [cordis-api/inherited.md](../cordis-api/inherited.md). + + + +### `ctx.speechController` — `SpeechController` + +Speech calls never activate or submit to an Agent. + +```ts cordis-catalog +/** + * Read provider choices without preparing a recognizer. + * @returns available providers, resolved default, and recording limits. + */ +@Remote catalog(): SpeechCatalog + +/** + * Follow provider readiness independently of Session and preparation lifetimes. + * @param signal - Client observation lifetime. + * @returns initial and subsequent complete readiness snapshots. + */ +@Remote({ mode: 'stream' }) async *follow(signal: AbortSignal): AsyncIterable + +/** + * Persist the user's recognition preferences. + * @param patch - changed preference fields. + * @returns after preferences are saved. + */ +@Remote configure(patch: SpeechSelectionPatch): Promise + +/** + * Start or join one Host-owned preparation task. + * @param providerId - selected recognizer. + */ +@Remote prepare(providerId: SpeechProviderId): void + +/** + * Explicitly cancel resource preparation. + * @param providerId - selected recognizer. + * @returns after the preparation task settles. + */ +@Remote cancelPreparation(providerId: SpeechProviderId): Promise + +/** + * Validate and transcribe one recording through the explicit provider selection. + * @param request - canonical WAV encoded as base64, provider id and language hint. + * @param signal - Client cancellation or Remote contribution disposal. + * @returns final transcript without adding a Session event. + */ +@Remote async transcribe(request: TranscriptionRequest, signal: AbortSignal): Promise +``` + +Source: [`packages/experimental/api-speech-to-text/src/index.ts`](../../packages/experimental/api-speech-to-text/src/index.ts) + + + +### `ctx.speechToText` — `SpeechToText` + +Registry shared by all transcription consumers in one Host composition. + +```ts cordis-catalog +/** + * Register one recognizer; duplicate ids fail without replacing the original. + * @param provider - recognizer owned by the contributing fiber. + * @returns idempotent disposer which rejects admission, cancels, and joins accepted work. + */ +register(provider: SpeechProvider): () => Promise + +/** + * Read the current recognizer roster. + * @returns available provider facts in registration order. + */ +listProviders(): readonly SpeechProviderInfo[] + +/** + * Observe complete readiness snapshots; a slow reader coalesces intermediate progress. + * @param caller - observer lifetime, independent of any preparation task. + * @returns an initial snapshot followed by the latest provider states. + */ +async *follow(caller: AbortSignal): AsyncIterable + +/** + * Read provider readiness and current user preferences together. + * @returns one detached complete observation. + */ +snapshot(): SpeechSnapshot + +/** + * Persist changed preference fields in the ordinary user-settings document. + * @param patch - explicit provider or language changes. + * @returns after persistence and the resolved preference update. + */ +async configure(patch: SpeechSelectionPatch): Promise + +/** + * Start or join provider-owned preparation. + * @param id - exact registered provider identity. + */ +prepare(id: SpeechProviderId): void + +/** + * Explicitly cancel provider preparation without tying it to a browser connection. + * @param id - exact registered provider identity. + * @returns after the preparation task settles. + */ +async cancelPreparation(id: SpeechProviderId): Promise + +/** + * Apply composition defaults and capture the selected provider. Missing providers fail explicitly. + * @param request - complete recording and optional selection. + * @returns provider-pinned input for transcribe(). + */ +resolve(request: SpeechRequest): SpeechSpec + +/** + * Execute exactly the resolved provider; no fallback sends audio elsewhere. + * @param spec - resolved input; a withdrawn or replaced registration is rejected. + * @param signal - caller cancellation. + * @returns final transcript after provider settlement. + */ +async transcribe(spec: SpeechSpec, signal: AbortSignal): Promise +``` + +Source: [`packages/experimental/speech-to-text/src/index.ts`](../../packages/experimental/speech-to-text/src/index.ts) + diff --git a/docs/subsystems/voice-input.zh.md b/docs/subsystems/voice-input.zh.md new file mode 100644 index 0000000000..c452755161 --- /dev/null +++ b/docs/subsystems/voice-input.zh.md @@ -0,0 +1,158 @@ +# 语音输入 + +[English](voice-input.md) | 中文 + +实验性语音识别包含三个角色:[服务定义](../../packages/experimental/speech-to-text/README.zh.md)路由具名 Provider,[SenseVoice Provider](../../packages/experimental/speech-to-text-sensevoice/README.zh.md)拥有本地推理,[Remote 消费者](../../packages/experimental/api-speech-to-text/README.zh.md)服务浏览器。[可选 Bundle](../../packages/experimental/voice-input-bundle/README.zh.md)将它们与麦克风 UI 组合。 + +## Provider 选择 + +`SpeechProviderId` 为注册标识添加类型品牌。`SpeechProviderInfo` 携带显示名称及 `host-local` 或 `cloud` 处理位置。`SpeechRequest` 包含 WAV 字节和可选的 Provider、语言选择;`resolve()` 生成捕获一个 Provider 实例的 `SpeechSpec`。缺失的 Provider 会报错;替换或撤销会使已解析的任务失效。音频不会回退到其他 Provider。 + +`SpeechProvider.transcribe()` 接收完整的 `SpeechInput` 与 `AbortSignal`。`Transcript` 返回纯文本 `text`、`audioSeconds` 和 `inferenceSeconds`。Provider 响应取消,并在注销完成前结束其任务。本地 Provider 串行执行调用、限制队列,并独立于 Session 管理工作进程生命周期。 + +## 浏览器与 Host 所有权 + +浏览器拥有麦克风轨道与未发送的草稿。`TranscriptionRequest` 通过带认证的 Remote 发送规范 base64 PCM16 WAV。`SpeechCatalog` 公布可用 Provider、默认选择及录音字节和时长限制。API 在识别前校验这些进程输入。 + +输入门面在录音前捕获带版本的选区。`InputActions.insertText()` 仅在选区版本仍有效且提交状态允许编辑时,插入一次可撤销的纯文本编辑。插入被拒绝时,转写文字保留以供显式插入。切换 Session 或释放插件使迟到结果失效。识别本身不写入 Session 事件;普通用户提交拥有最终的模型可见文字。 + +## 准备与设置 + +Host Provider 在页面和 Session 变化期间拥有同一个准备任务。Client 在输入框、插件状态、详情之间共享一份 `follow()` 订阅。可选 `SpeechSetupEstimate` 元数据提供各 Provider 的资源预期,与实测进度分开。`SpeechPreparationStepKind` 标识有序资源操作;`SpeechPreparationStep` 记录各步骤状态与开始时间。完整 `SpeechPreparationState` 在取消或失败后保留这些步骤。折叠 UI 显示当前操作,展开后列出所有步骤,仅进行中步骤显示字节进度或等待时间。关闭观察者不会取消准备。已校验文件在重试及空闲进程回收后继续复用。 + +麦克风占用模型选择器与发送按钮之间的 `conversation.input.activity`。点击开始录音并展开工具栏;停止后转写并插入草稿。活动栏保留编辑器与提交按钮,拥有局部反馈,并在卸载时释放展开状态。取消、Escape 或隐藏页面会丢弃录音。波形历史展示实测麦克风音量。Bundle 详情保存识别偏好,`plugins.bundle.status` 在启用开关旁显示简短准备状态。 + +## 设计依据 + +[语音输入决策](../../.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.zh.md)解释临时音频、显式 Provider 选择和延迟准备本地运行时。 + + + + + +## Cordis API + +Generated from source by `scripts/gen-cordis-catalog.ts` (verified fresh by `pnpm run verify-cordis-catalog` in doc-sync; regenerate with `pnpm run gen-cordis-catalog`) — the language sides differ only in locale-specific paired document paths. Signature blocks use a `ts cordis-catalog` fence and keep the original source JSDoc; dispatch modes are defined in the [primer](../cordis-primer.zh.md#dispatch-modes), and the framework-inherited `ctx` API lives in [cordis-api/inherited.md](../cordis-api/inherited.md). + + + +### `ctx.speechController` — `SpeechController` + +Speech calls never activate or submit to an Agent. + +```ts cordis-catalog +/** + * Read provider choices without preparing a recognizer. + * @returns available providers, resolved default, and recording limits. + */ +@Remote catalog(): SpeechCatalog + +/** + * Follow provider readiness independently of Session and preparation lifetimes. + * @param signal - Client observation lifetime. + * @returns initial and subsequent complete readiness snapshots. + */ +@Remote({ mode: 'stream' }) async *follow(signal: AbortSignal): AsyncIterable + +/** + * Persist the user's recognition preferences. + * @param patch - changed preference fields. + * @returns after preferences are saved. + */ +@Remote configure(patch: SpeechSelectionPatch): Promise + +/** + * Start or join one Host-owned preparation task. + * @param providerId - selected recognizer. + */ +@Remote prepare(providerId: SpeechProviderId): void + +/** + * Explicitly cancel resource preparation. + * @param providerId - selected recognizer. + * @returns after the preparation task settles. + */ +@Remote cancelPreparation(providerId: SpeechProviderId): Promise + +/** + * Validate and transcribe one recording through the explicit provider selection. + * @param request - canonical WAV encoded as base64, provider id and language hint. + * @param signal - Client cancellation or Remote contribution disposal. + * @returns final transcript without adding a Session event. + */ +@Remote async transcribe(request: TranscriptionRequest, signal: AbortSignal): Promise +``` + +Source: [`packages/experimental/api-speech-to-text/src/index.ts`](../../packages/experimental/api-speech-to-text/src/index.ts) + + + +### `ctx.speechToText` — `SpeechToText` + +Registry shared by all transcription consumers in one Host composition. + +```ts cordis-catalog +/** + * Register one recognizer; duplicate ids fail without replacing the original. + * @param provider - recognizer owned by the contributing fiber. + * @returns idempotent disposer which rejects admission, cancels, and joins accepted work. + */ +register(provider: SpeechProvider): () => Promise + +/** + * Read the current recognizer roster. + * @returns available provider facts in registration order. + */ +listProviders(): readonly SpeechProviderInfo[] + +/** + * Observe complete readiness snapshots; a slow reader coalesces intermediate progress. + * @param caller - observer lifetime, independent of any preparation task. + * @returns an initial snapshot followed by the latest provider states. + */ +async *follow(caller: AbortSignal): AsyncIterable + +/** + * Read provider readiness and current user preferences together. + * @returns one detached complete observation. + */ +snapshot(): SpeechSnapshot + +/** + * Persist changed preference fields in the ordinary user-settings document. + * @param patch - explicit provider or language changes. + * @returns after persistence and the resolved preference update. + */ +async configure(patch: SpeechSelectionPatch): Promise + +/** + * Start or join provider-owned preparation. + * @param id - exact registered provider identity. + */ +prepare(id: SpeechProviderId): void + +/** + * Explicitly cancel provider preparation without tying it to a browser connection. + * @param id - exact registered provider identity. + * @returns after the preparation task settles. + */ +async cancelPreparation(id: SpeechProviderId): Promise + +/** + * Apply composition defaults and capture the selected provider. Missing providers fail explicitly. + * @param request - complete recording and optional selection. + * @returns provider-pinned input for transcribe(). + */ +resolve(request: SpeechRequest): SpeechSpec + +/** + * Execute exactly the resolved provider; no fallback sends audio elsewhere. + * @param spec - resolved input; a withdrawn or replaced registration is rejected. + * @param signal - caller cancellation. + * @returns final transcript after provider settlement. + */ +async transcribe(spec: SpeechSpec, signal: AbortSignal): Promise +``` + +Source: [`packages/experimental/speech-to-text/src/index.ts`](../../packages/experimental/speech-to-text/src/index.ts) + diff --git a/packages/boot/app-boot/src/profile.ts b/packages/boot/app-boot/src/profile.ts index d187f3c10a..71a8da853c 100644 --- a/packages/boot/app-boot/src/profile.ts +++ b/packages/boot/app-boot/src/profile.ts @@ -165,6 +165,7 @@ export const DEFAULT_PROFILE_BUNDLES: readonly string[] = ['@deepseek-ai/dsh-bas * manager ([rationale](../../../../.agents/notes/implemented/process/2026-09-15-shipped-optional-bundles.md)). */ export const OPTIONAL_BUNDLES: readonly string[] = [ + '@deepseek-ai/dsh-experimental-voice-input-bundle', '@deepseek-ai/dsh-experimental-agent-team-profile', '@deepseek-ai/dsh-experimental-agent-team-web-profile', ] diff --git a/packages/client/ui-attachment/tests/message-image.client.spec.tsx b/packages/client/ui-attachment/tests/message-image.client.spec.tsx index 3d3711a77a..e195dd6533 100644 --- a/packages/client/ui-attachment/tests/message-image.client.spec.tsx +++ b/packages/client/ui-attachment/tests/message-image.client.spec.tsx @@ -282,6 +282,8 @@ describe('ImageGallery', () => { useTrajectory, useInput, inputActions: { + captureInsertion: () => ({ start: 0, end: 0, draftRev: 0 }), + insertText: () => false, setDraft: vi.fn(), addAttachments: vi.fn(() => true), removeAttachment: vi.fn(), diff --git a/packages/client/ui-chat/tests/chat-view.client.spec.tsx b/packages/client/ui-chat/tests/chat-view.client.spec.tsx index 8a63203668..546c13c7ff 100644 --- a/packages/client/ui-chat/tests/chat-view.client.spec.tsx +++ b/packages/client/ui-chat/tests/chat-view.client.spec.tsx @@ -392,6 +392,8 @@ function makeHarness( }, useInput: (() => { throw new Error('unused') }), inputActions: { + captureInsertion: () => ({ start: 0, end: 0, draftRev: 0 }), + insertText: () => false, setDraft: () => {}, addAttachments: () => true, removeAttachment: () => {}, diff --git a/packages/client/ui-conversation/README.i18n.yaml b/packages/client/ui-conversation/README.i18n.yaml index 6f234d0b92..1217ad6dd9 100644 --- a/packages/client/ui-conversation/README.i18n.yaml +++ b/packages/client/ui-conversation/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/client/ui-conversation/README.md -README.md: 749c2d0d99ce7e5afe2f9a03b1451d62a17206e7 -README.zh.md: 98d7d9672922ca68ea4df5499f8789e8c2fde162 +README.md: 83fcfbb527e631e25bddb532fa60d4def0ef33f0 +README.zh.md: 80f7df87e3277729e37f573371cf3d2b21dff721 diff --git a/packages/client/ui-conversation/README.md b/packages/client/ui-conversation/README.md index 749c2d0d99..83fcfbb527 100644 --- a/packages/client/ui-conversation/README.md +++ b/packages/client/ui-conversation/README.md @@ -114,6 +114,10 @@ try { The selector must be a pure function of the owner currency. Its non-null return is delivered to the component as `matched`; `PropsRuntime<'conversation.composer'>` supplies the standard Session and global props. Chain order remains ascending `priority`, then registration order, and the first non-null selector wins. The shell keeps the default composer mounted beneath a takeover. Request state, listeners, response encoding, and any request-specific child slots belong to the business package; they are not carried by `SessionSnapshot` or declared by this core package. +`InputActions.captureInsertion()` captures the draft selection and revision; `insertText(text, span)` inserts one undoable plain-text edit only while that revision is current and the editor permits editing. Asynchronous consumers retain rejected results for user action. + +`conversation.input.activity` hosts one control between the model selector and Send. Its `onActiveChange` callback expands that control across the toolbar and hides ordinary accessory controls while preserving the editor and submit action. The occupant releases expansion on unmount and owns any activity-specific feedback. + ## Model Experience diff --git a/packages/client/ui-conversation/README.zh.md b/packages/client/ui-conversation/README.zh.md index 98d7d96729..80f7df87e3 100644 --- a/packages/client/ui-conversation/README.zh.md +++ b/packages/client/ui-conversation/README.zh.md @@ -114,6 +114,10 @@ try { selector 必须是 owner currency 的纯函数。非 null 返回值作为 `matched` 传给组件;`PropsRuntime<'conversation.composer'>` 提供标准 Session 与 global props。Chain 顺序仍按 `priority` 升序,再按注册顺序;首个返回非 null 的 selector 获选。Shell 会在 takeover 下保持默认 composer 挂载。Request 状态、listener、response encoding 和任何 request-specific child slot 都属于业务 package,不进入 `SessionSnapshot`,也不由 core 包声明。 +`InputActions.captureInsertion()` 捕获草稿选区与版本;`insertText(text, span)` 仅在版本未变且编辑器允许编辑时,插入一次可撤销的纯文本编辑。异步消费者在插入被拒绝后负责保留结果,等待用户操作。 + +`conversation.input.activity` 在模型选择器与发送按钮之间承载一个控件。其 `onActiveChange` 回调将控件展开至整条工具栏并隐藏普通辅助控件,同时保留编辑器和提交按钮。占用者在卸载时释放展开状态,并拥有活动专属反馈。 + ## 模型体验 diff --git a/packages/client/ui-conversation/src/client/apply.ts b/packages/client/ui-conversation/src/client/apply.ts index 1402639dac..27e96749fd 100644 --- a/packages/client/ui-conversation/src/client/apply.ts +++ b/packages/client/ui-conversation/src/client/apply.ts @@ -323,6 +323,7 @@ export function apply(ctx: Context, config: Config = Config({})): void { 'conversation.input.plan': { kind: 'single', scope: 'session' }, 'conversation.input.right': { kind: 'list', scope: 'session' }, 'conversation.input.model': { kind: 'single', scope: 'session' }, + 'conversation.input.activity': { kind: 'single', scope: 'session' }, 'conversation.composer.dock': { kind: 'list', scope: 'session' }, }, inject: (sessionId: SessionId | undefined): ComposerBarInjected => { diff --git a/packages/client/ui-conversation/src/client/contract/input.ts b/packages/client/ui-conversation/src/client/contract/input.ts index b4cd6b9bdf..4036736374 100644 --- a/packages/client/ui-conversation/src/client/contract/input.ts +++ b/packages/client/ui-conversation/src/client/contract/input.ts @@ -220,6 +220,15 @@ export interface SessionInputResolver { * paste/…) stay InputBar-private and never ride this face. */ export interface InputActions { + /** @returns a revision-guarded copy of the current editor selection. */ + captureInsertion(): TokenSpan + /** + * Insert asynchronous text without replacing subsequent edits or reference chips. + * @param text - plain text to insert in one undo step. + * @param span - captured selection and draft revision. + * @returns false when the draft changed or submission locked the editor. + */ + insertText(text: string, span: TokenSpan): boolean /** Replace the whole draft (persisted-draft seed and programmatic writes). */ setDraft(text: string): void /** Append ordered browser-owned attachment ids; busy admission phases refuse. */ diff --git a/packages/client/ui-conversation/src/client/contract/slots.ts b/packages/client/ui-conversation/src/client/contract/slots.ts index cccd6a0fe6..b1ef334cb5 100644 --- a/packages/client/ui-conversation/src/client/contract/slots.ts +++ b/packages/client/ui-conversation/src/client/contract/slots.ts @@ -183,6 +183,8 @@ declare module '@deepseek-ai/dsh-client-ui-slots' { 'conversation.input.left': { kind: 'list'; scope: 'session' } /** Compact controls before the composer submit action. */ 'conversation.input.right': { kind: 'list'; scope: 'session' } + /** Compact action after the model selector; it can expand across the toolbar while retaining the editor and submit action. */ + 'conversation.input.activity': { kind: 'single'; scope: 'session'; owner: InputActivityOwnerProps } /** Resident composer body, including the no-Session inert state. */ 'conversation.composer.bar': { kind: 'single'; scope: 'session-maybe'; owner: ComposerBarOwnerProps } /** Optional draft-attachment rail and drop target. */ @@ -351,6 +353,12 @@ export interface InputControlOwnerProps { locked: boolean } +/** A toolbar activity hides ordinary accessory controls while expanded; its occupant must release expansion on unmount. */ +export interface InputActivityOwnerProps extends InputControlOwnerProps { + /** @param active - whether the occupant needs the toolbar width before the submit action. */ + onActiveChange: (active: boolean) => void +} + /** Full props of the resident composer bar. */ export type ComposerBarProps = PropsRuntime<'conversation.composer.bar'> @@ -358,7 +366,7 @@ export type ComposerBarProps = | 'conversation.input.attachments' | 'conversation.input.overlay' | 'conversation.input.permission' | 'conversation.input.left' | 'conversation.input.plan' - | 'conversation.input.right' | 'conversation.input.model' + | 'conversation.input.right' | 'conversation.input.model' | 'conversation.input.activity' | 'conversation.composer.dock' > & InjectFace diff --git a/packages/client/ui-conversation/src/client/input/editor/runtime.ts b/packages/client/ui-conversation/src/client/input/editor/runtime.ts index 2054624342..ca6047c4ec 100644 --- a/packages/client/ui-conversation/src/client/input/editor/runtime.ts +++ b/packages/client/ui-conversation/src/client/input/editor/runtime.ts @@ -210,6 +210,19 @@ export class DraftEditorRuntime { return applied } + /** + * Insert an asynchronous text result as one independent undo operation. + * @param span - owner-validated insertion range. + * @param text - text sanitized with the same rules as paste. + * @returns whether the range mapped and the edit applied. + */ + insertAsyncText(span: DetectSpan, text: string): boolean { + let applied = false + const clean = text.replace(REFERENCE_PLACEHOLDER_RE, '') + this.applyEdit(() => { applied = $replaceDetectSpanWithText(span, clean) }, PASTE_TAG) + return applied + } + /** * Insert a reference chip with the existing trailing-space rule. * @param span - detect-coordinate range. diff --git a/packages/client/ui-conversation/src/client/input/facade.ts b/packages/client/ui-conversation/src/client/input/facade.ts index 4f4d0aa300..8931d1c4d0 100644 --- a/packages/client/ui-conversation/src/client/input/facade.ts +++ b/packages/client/ui-conversation/src/client/input/facade.ts @@ -117,6 +117,12 @@ export class SessionInputShell implements SessionInput { } /** The public provide-channel action face (one stable identity per session). */ readonly actions: InputActions = { + captureInsertion: () => ({ ...this.caretSpan(), draftRev: this.rev }), + insertText: (text, span) => { + if (this.snapshot.phase === 'adjudicating' || this.snapshot.phase === 'submitting' || this.disposed) return false + if (span.draftRev !== this.rev) return false + return this.draftEditor.insertAsyncText(span, text) + }, setDraft: (text) => { this.setDraft(text) }, addAttachments: ids => this.addAttachments(ids), removeAttachment: (id) => { this.removeAttachment(id) }, diff --git a/packages/client/ui-conversation/src/client/skeleton/InputBar.module.css b/packages/client/ui-conversation/src/client/skeleton/InputBar.module.css index 6369e5728d..4e6764a14a 100644 --- a/packages/client/ui-conversation/src/client/skeleton/InputBar.module.css +++ b/packages/client/ui-conversation/src/client/skeleton/InputBar.module.css @@ -392,3 +392,10 @@ font-size: 12px; cursor: pointer; } + +.standardControls { display: flex; align-items: center; gap: 12px; min-width: 0; } +.tools[hidden], .standardControls[hidden] { display: none; } +.trailingActive { flex: 1; margin-left: 0; } +.activity { display: flex; align-items: center; } +.activity:empty { display: none; } +.activityExpanded { flex: 1; min-width: 0; } diff --git a/packages/client/ui-conversation/src/client/skeleton/InputBar.tsx b/packages/client/ui-conversation/src/client/skeleton/InputBar.tsx index 283f50c456..9349a58b48 100644 --- a/packages/client/ui-conversation/src/client/skeleton/InputBar.tsx +++ b/packages/client/ui-conversation/src/client/skeleton/InputBar.tsx @@ -55,6 +55,8 @@ export const InputBar = memo(function InputBar({ const busyEnter = useBusyEnter(s => s) void useLexicon // hook seat stays bound by the inject compartment; text-ref decoration rides the shell's editor transforms const commandMenuOpen = useMenuLauncher(source => source === 'command') + const [activity, setActivity] = useState(false) + useEffect(() => { setActivity(false) }, [sessionId]) const promptError = useSession(s => s.promptError) ?? null const running = useSession(s => s.running) ?? false const subagent = useSession(s => s.subagent) ?? null @@ -406,7 +408,7 @@ export const InputBar = memo(function InputBar({ showPlaceholder={draft === '' && attachments.length === 0 && !claimActive} />
-
+ {input === undefined || sessionId === undefined ? null :
{renderSlot('conversation.input.activity', { locked, onActiveChange: setActivity })}
} +
{interruptible && ( , + }) + const meter = view.getByRole('button', { name: '上下文已用 25%' }) + fireEvent.click(view.getByRole('button', { name: 'microphone' })) + expect(view.getByRole('button', { name: '上下文已用 25%' })).toBe(meter) + fireEvent.click(meter) + expect(view.getByRole('dialog', { name: '上下文已用' })).toBeTruthy() + expect(view.getByRole('button', { name: '发送消息' })).toBeTruthy() +}) diff --git a/packages/client/ui-plugin-manager/README.i18n.yaml b/packages/client/ui-plugin-manager/README.i18n.yaml index b7ec25c601..379cfb3c3b 100644 --- a/packages/client/ui-plugin-manager/README.i18n.yaml +++ b/packages/client/ui-plugin-manager/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/client/ui-plugin-manager/README.md -README.md: f50289840f9dda3bb34252480803cb3b0a3b680c -README.zh.md: 4e812870571d33c6b65d3c04a7f000f7754df97f +README.md: 1be8aaf8aed93dcfef527f373300f5cfb49c9bb6 +README.zh.md: eac6512b91916302e0d265045365d0ad8e890c82 diff --git a/packages/client/ui-plugin-manager/README.md b/packages/client/ui-plugin-manager/README.md index f50289840f..1be8aaf8ae 100644 --- a/packages/client/ui-plugin-manager/README.md +++ b/packages/client/ui-plugin-manager/README.md @@ -79,7 +79,7 @@ The browser plugin registers the `plugins` sidebar entry and its `main` panel th The page's `main` registration declares `plugins.item`, `plugins.bundle.config`, and `plugins.row.config` as its children, so the slots exist while the page does and a registrant's `ctx.slots.inject` waits for them. `configLedgerSource` projects the three ledgers into one observable — the official items in ledger order with their labels resolved in the active locale, and the bundle and row keys — cached until a ledger or the locale moves; the page binds it as `useConfigLedger` beside the store and never names a configurable plugin itself. Which page is open is page-local state: the cards, a bundle, an official plugin, or a row of a bundle. A registration lives with the browser half that made it. `dsh-client-modules` attaches a package's browser half to the Loader row whose specifier is the bare package name, so every page a bundle registers, for itself or for any of its rows, goes away when that row is switched off; a sub-plugin whose page must outlive the other rows ships as its own package. -`plugins.bundle.status` is keyed by npm package name and renders short status before the bundle switch; `plugins.bundle.config` supplies bundle detail configuration. +`plugins.bundle.status` is keyed by npm package name and renders short status before the bundle switch; `plugins.bundle.config` supplies bundle detail configuration. `plugins.bundle.activation` offers bundle-owned guidance after explicit enablement from the list, with callbacks to dismiss it or open the bundle details. It does not appear merely because an enabled bundle was listed. diff --git a/packages/client/ui-plugin-manager/README.zh.md b/packages/client/ui-plugin-manager/README.zh.md index 4e81287057..eac6512b91 100644 --- a/packages/client/ui-plugin-manager/README.zh.md +++ b/packages/client/ui-plugin-manager/README.zh.md @@ -79,7 +79,7 @@ ctx.slots.inject('plugins.row.config', () => ctx.slots.register({ 页面的 `main` 注册把 `plugins.item`、`plugins.bundle.config` 与 `plugins.row.config` 声明为子 slot,因此它们与页面同生,注册方的 `ctx.slots.inject` 会等到它们出现。`configLedgerSource` 把三份账本投影成一个可观察对象——按账本顺序排列、标签按当前语言解析的官方条目,以及组合包与行的键——在账本或语言变化前保持缓存;页面把它作为 `useConfigLedger` 绑在 store 旁边,自身从不点名任何可配置插件。打开的是哪一页是页面本地状态:卡片、某个组合包、某个官方插件,或组合包的某一行。注册与做出它的浏览器半侧同生共死。`dsh-client-modules` 只把一个包的浏览器半侧挂在说明符恰为包名的那一行 Loader 行上,所以组合包为自己或任一行注册的页面,都会在那一行被关闭时一起消失;需要在其他行关闭时仍保留页面的子插件,应作为独立的包发布。 -`plugins.bundle.status` 以 npm 包名为 key,在 Bundle 卡片开关前显示简短状态;`plugins.bundle.config` 提供 Bundle 详情配置。 +`plugins.bundle.status` 以 npm 包名为 key,在 Bundle 卡片开关前显示简短状态;`plugins.bundle.config` 提供 Bundle 详情配置。`plugins.bundle.activation` 在用户从列表显式启用后提供 Bundle 自有引导,并传入关闭引导和打开详情的回调。仅列出已启用的 Bundle 不会触发引导。 diff --git a/packages/client/ui-plugin-manager/src/client/PluginManagerPage.tsx b/packages/client/ui-plugin-manager/src/client/PluginManagerPage.tsx index 1e063fe84b..3a94095741 100644 --- a/packages/client/ui-plugin-manager/src/client/PluginManagerPage.tsx +++ b/packages/client/ui-plugin-manager/src/client/PluginManagerPage.tsx @@ -33,7 +33,7 @@ import css from './PluginManagerPage.module.css' export type PluginManagerPageProps = PropsRuntime<'main'> & PropsLocale<'pluginManager'> - & PropsRenderSlots<'plugins.item' | 'plugins.bundle.config' | 'plugins.row.config' | 'plugins.bundle.status'> + & PropsRenderSlots<'plugins.item' | 'plugins.bundle.config' | 'plugins.row.config' | 'plugins.bundle.status' | 'plugins.bundle.activation'> & InjectFace /** The page's slot renderer, narrowed to the configuration slots. */ @@ -834,6 +834,7 @@ export function PluginManagerPage(props: PluginManagerPageProps): ReactNode { const ledger = props.useConfigLedger(snapshot => snapshot) // What is open; a package that leaves the list (uninstalled) drops back to the cards. const [view, setView] = useState({ kind: 'list' }) + const [activation, setActivation] = useState(null) useEffect(() => { ensure() }, [ensure]) // A package an install just enabled: scroll it into view and mark it for a moment. const { highlight, clearHighlight } = { highlight: state.highlight, clearHighlight: props.clearHighlight } @@ -857,6 +858,7 @@ export function PluginManagerPage(props: PluginManagerPageProps): ReactNode { const openItem = view.kind === 'item' ? ledger.items.find(item => item.id === view.id) : undefined const openRow = view.kind === 'row' && openPkg !== undefined ? openPkg.rows.find(row => row.rowId === view.rowId) : undefined const showsCards = openPkg === undefined && openItem === undefined + const activated = listed.find(pkg => pkg.name === activation && pkg.enabled && !state.busy.includes(pkg.name)) const setRowEnabled = (row: PackageRow, enabled: boolean): void => { /* v8 ignore next -- a row without a live entry has its switch disabled */ if (row.entryId !== undefined) props.setRowEnabled(row.entryId, enabled) @@ -873,8 +875,8 @@ export function PluginManagerPage(props: PluginManagerPageProps): ReactNode { busy={state.busy.includes(pkg.name)} highlighted={state.highlight === pkg.name} statusContent={renderSlot('plugins.bundle.status', { packageName: pkg.name, enabled: pkg.enabled }, { entryKey: pkg.name })} - onOpen={() => { setView({ kind: 'package', name: pkg.name }) }} - onSetEnabled={(enabled) => { props.setEnabled(pkg.name, enabled) }} + onOpen={() => { setActivation(null); setView({ kind: 'package', name: pkg.name }) }} + onSetEnabled={(enabled) => { setActivation(enabled ? pkg.name : null); props.setEnabled(pkg.name, enabled) }} /> ) // The Official group: the bundles the installation ships, then the plugins that registered their configuration. @@ -977,6 +979,12 @@ export function PluginManagerPage(props: PluginManagerPageProps): ReactNode { ) : null} + {showsCards && activated !== undefined && !state.install.open + ? renderSlot('plugins.bundle.activation', { + packageName: activated.name, + onDismiss: () => { setActivation(null) }, + onOpenDetails: () => { setActivation(null); setView({ kind: 'package', name: activated.name }) }, + }, { entryKey: activated.name }) : null} { setActivation(state.install.installed); props.enableInstalled() }} onApproveBuilds={props.approveBuildsAndRetry} /> {state.confirm === null diff --git a/packages/client/ui-plugin-manager/src/client/index.ts b/packages/client/ui-plugin-manager/src/client/index.ts index f5d9061f3c..102a73a3d7 100644 --- a/packages/client/ui-plugin-manager/src/client/index.ts +++ b/packages/client/ui-plugin-manager/src/client/index.ts @@ -31,7 +31,7 @@ export type { PluginManagerPageProps } from './PluginManagerPage.tsx' export type { ConfigLedger, OfficialItem } from './config-ledger.ts' export type { PluginManagerFace } from './manager-store.ts' export type { PluginManagerLocaleKey } from './locales.ts' -export type { PluginConfigViewProps, PluginPackageOwnerProps } from './slot-contract.ts' +export type { PluginConfigViewProps, PluginPackageOwnerProps, PluginActivationOwnerProps } from './slot-contract.ts' declare module '@deepseek-ai/dsh-client-ui-slots' { interface LocaleNamespaceMap { @@ -88,6 +88,7 @@ export function apply(ctx: ClientContext): void { children: { 'plugins.item': { kind: 'list', scope: 'root' }, 'plugins.bundle.status': { kind: 'keyed', scope: 'root' }, + 'plugins.bundle.activation': { kind: 'keyed', scope: 'root' }, 'plugins.bundle.config': { kind: 'keyed', scope: 'root' }, 'plugins.row.config': { kind: 'keyed', scope: 'root' }, }, diff --git a/packages/client/ui-plugin-manager/src/client/slot-contract.ts b/packages/client/ui-plugin-manager/src/client/slot-contract.ts index fb3d4bfe6e..9fd4fec14b 100644 --- a/packages/client/ui-plugin-manager/src/client/slot-contract.ts +++ b/packages/client/ui-plugin-manager/src/client/slot-contract.ts @@ -24,10 +24,21 @@ export interface PluginPackageOwnerProps { readonly enabled: boolean } +/** One user-requested bundle activation and navigation to its configuration page. */ +export interface PluginActivationOwnerProps { + readonly packageName: string + /** Dismiss guidance for this activation. */ + readonly onDismiss: () => void + /** Dismiss guidance and open this bundle's detail page. */ + readonly onOpenDetails: () => void +} + declare module '@deepseek-ai/dsh-client-ui-slots' { interface SlotMap { /** Short readiness beside the bundle switch, keyed by npm package name. */ 'plugins.bundle.status': { kind: 'keyed'; scope: 'root'; owner: PluginPackageOwnerProps } + /** Optional guidance after the user enables a bundle from the list, keyed by npm package name. */ + 'plugins.bundle.activation': { kind: 'keyed'; scope: 'root'; owner: PluginActivationOwnerProps } /** * One official plugin the Plugins page lists in its Official group after * the official bundles: `label` is the card's title and `order` its place. diff --git a/packages/client/ui-plugin-manager/tests/components.client.spec.tsx b/packages/client/ui-plugin-manager/tests/components.client.spec.tsx index 3adce8a6ed..12f1019827 100644 --- a/packages/client/ui-plugin-manager/tests/components.client.spec.tsx +++ b/packages/client/ui-plugin-manager/tests/components.client.spec.tsx @@ -10,6 +10,7 @@ import type { PluginManagerPageProps } from '../src/client/PluginManagerPage.tsx import type { ConfigLedger } from '../src/client/config-ledger.ts' import { rowKey, type InstallState, type PackageRow, type PackageView, type PluginManagerState } from '../src/client/manager-store.ts' import { en, zh, type PluginManagerLocaleKey } from '../src/client/locales.ts' +import type { PluginActivationOwnerProps } from '../src/client/slot-contract.ts' afterEach(cleanup) @@ -53,7 +54,7 @@ const READY: PluginManagerState = { } /** Configuration entries a test supplies: what each slot cell renders, by `:` and the view asked for. */ -type SlotBodies = Record ReactNode> +type SlotBodies = Record ReactNode> const NO_CONFIG: ConfigLedger = { items: [], bundles: new Set(), rows: new Set() } @@ -86,7 +87,7 @@ function renderTab(state: Partial = {}, config: Partial - bodies[`${name}:${opts.only ?? opts.entryKey ?? ''}`]?.(owner.view) ?? null, + bodies[`${name}:${opts.only ?? opts.entryKey ?? ''}`]?.(owner.view, owner) ?? null, } as unknown as PluginManagerPageProps const { rerender } = render() return { @@ -711,3 +712,41 @@ it('places bundle readiness immediately before its enable switch', () => { expect(status.parentElement).toBe(toggle.parentElement) expect(status.compareDocumentPosition(toggle) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy() }) + +it('offers bundle-owned guidance only after explicit enablement and navigates to its detail page', () => { + const name = 'dsh-better-sidebar' + const { set, actions } = renderTab({ packages: [pkg({ enabled: false })] }, { bundles: new Set([name]) }, { + [`plugins.bundle.activation:${name}`]: (_view, owner) => , + [`plugins.bundle.config:${name}`]: () =>
Bundle setup
, + }) + expect(screen.queryByText('Go to setup')).toBeNull() + set({ packages: [pkg()] }) + expect(screen.queryByText('Go to setup')).toBeNull() + set({ packages: [pkg({ enabled: false })] }) + fireEvent.click(screen.getByRole('switch')) + expect(actions.setEnabled).toHaveBeenCalledWith(name, true) + expect(screen.queryByText('Go to setup')).toBeNull() + set({ packages: [pkg()], busy: [name] }) + expect(screen.queryByText('Go to setup')).toBeNull() + set({ busy: [] }) + fireEvent.click(screen.getByText('Go to setup')) + expect(screen.getByText('Bundle setup')).toBeTruthy() + expect(screen.queryByText('Go to setup')).toBeNull() +}) + +it('dismisses activation guidance until the user enables the bundle again', () => { + const name = 'dsh-better-sidebar' + const { set } = renderTab({ packages: [pkg({ enabled: false })] }, {}, { + [`plugins.bundle.activation:${name}`]: (_view, owner) => , + }) + fireEvent.click(screen.getByRole('switch')) + set({ packages: [pkg()] }) + fireEvent.click(screen.getByText('Later')) + set({ packages: [pkg()] }) + expect(screen.queryByText('Later')).toBeNull() + fireEvent.click(screen.getByRole('switch')) + set({ packages: [pkg({ enabled: false })] }) + fireEvent.click(screen.getByRole('switch')) + set({ packages: [pkg()] }) + expect(screen.getByText('Later')).toBeTruthy() +}) diff --git a/packages/experimental/client-ui-voice-input/README.i18n.yaml b/packages/experimental/client-ui-voice-input/README.i18n.yaml index f6ffe92804..ff5188a628 100644 --- a/packages/experimental/client-ui-voice-input/README.i18n.yaml +++ b/packages/experimental/client-ui-voice-input/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/experimental/client-ui-voice-input/README.md -README.md: b8e4ef32158b577dade8665362e30138644eca42 -README.zh.md: aedb37794db837f5b3a4a5ae971b4e21a01066ff +README.md: f6a7c2ba0e8567205ddd093b85915a487c3da898 +README.zh.md: 42657a6f0d73d61caefb12c83e7d73d7408d52cb diff --git a/packages/experimental/client-ui-voice-input/README.md b/packages/experimental/client-ui-voice-input/README.md index b8e4ef3215..f6a7c2ba0e 100644 --- a/packages/experimental/client-ui-voice-input/README.md +++ b/packages/experimental/client-ui-voice-input/README.md @@ -25,7 +25,7 @@ This optional browser plugin adds a microphone between the model selector and Se ## Use this package -Enable the [voice input bundle](../voice-input-bundle/README.md) from Plugins, open its details and choose Download and prepare. Before preparation, local providers can show their estimated disk, memory and setup-time requirements. The preparation summary starts collapsed; expand it to inspect completed, current and pending steps. Once ready, click the microphone, allow access and click Stop to transcribe. Cancel or Escape discards capture. Recognition feedback stays inside the toolbar. Provider and language preferences are saved from bundle details through the Settings service. Browser microphone access requires HTTPS or loopback and operating-system permission. +Enable the [voice input bundle](../voice-input-bundle/README.md) from Plugins. After cache inspection, missing models trigger a setup prompt with Go to setup and Later actions. Go to setup opens bundle details; Download and prepare starts installation there. Intact cached models need no prompt or repeated installation. Before preparation, local providers can show their estimated disk, memory and setup-time requirements. The preparation summary starts collapsed; expand it to inspect completed, current and pending steps. Once ready, click the microphone, allow access and click Stop to transcribe. Cancel or Escape discards capture. Recognition feedback stays inside the toolbar. Provider and language preferences are saved from bundle details through the Settings service. Browser microphone access requires HTTPS or loopback and operating-system permission. ----- diff --git a/packages/experimental/client-ui-voice-input/README.zh.md b/packages/experimental/client-ui-voice-input/README.zh.md index aedb37794d..42657a6f0d 100644 --- a/packages/experimental/client-ui-voice-input/README.zh.md +++ b/packages/experimental/client-ui-voice-input/README.zh.md @@ -25,7 +25,7 @@ kind: "package-reference" ## 使用此包 -在插件管理页启用[语音输入 Bundle](../voice-input-bundle/README.zh.md),打开详情并点击“下载并准备”。准备前,本地 Provider 可展示磁盘、内存与安装时间的估算。准备摘要默认折叠;展开后可查看已完成、当前和未开始的步骤。就绪后点击麦克风,允许访问,再点击停止以转写。取消或 Escape 丢弃录音。识别反馈留在工具栏内部。Provider 和语言偏好在 Bundle 详情中通过 Settings 服务保存。浏览器麦克风访问需要 HTTPS 或环回地址,以及操作系统权限。 +在插件管理页启用[语音输入 Bundle](../voice-input-bundle/README.zh.md)。缓存检查后,缺少模型时弹出安装引导,可选择“前往安装”或“稍后”。“前往安装”打开 Bundle 详情,再点击“下载并准备”才开始安装。缓存完整时不弹引导,也无需重复安装。准备前,本地 Provider 可展示磁盘、内存与安装时间的估算。准备摘要默认折叠;展开后可查看已完成、当前和未开始的步骤。就绪后点击麦克风,允许访问,再点击停止以转写。取消或 Escape 丢弃录音。识别反馈留在工具栏内部。Provider 和语言偏好在 Bundle 详情中通过 Settings 服务保存。浏览器麦克风访问需要 HTTPS 或环回地址,以及操作系统权限。 ----- diff --git a/packages/experimental/client-ui-voice-input/src/client/VoiceInput.tsx b/packages/experimental/client-ui-voice-input/src/client/VoiceInput.tsx index 4de6cd70df..4d6057be8b 100644 --- a/packages/experimental/client-ui-voice-input/src/client/VoiceInput.tsx +++ b/packages/experimental/client-ui-voice-input/src/client/VoiceInput.tsx @@ -129,7 +129,7 @@ export function VoiceInput({ sessionId, inputActions, locked, onActiveChange, } } if (!expanded) return return
+ + }> +

{t('setupPrompt.body')}

+ +} diff --git a/packages/experimental/client-ui-voice-input/src/client/locales.ts b/packages/experimental/client-ui-voice-input/src/client/locales.ts index 785ab3c278..918652da79 100644 --- a/packages/experimental/client-ui-voice-input/src/client/locales.ts +++ b/packages/experimental/client-ui-voice-input/src/client/locales.ts @@ -5,6 +5,11 @@ export const NS = 'voice-input' /** Chinese dictionary and key source. */ export const zh = { + dictate: '听写', + 'setupPrompt.title': '使用语音输入前需要安装', + 'setupPrompt.body': '语音输入插件已开启。首次使用需要在本机下载并准备识别模型,请前往插件详情页查看空间、内存和时间说明,再开始安装。', + 'setupPrompt.later': '稍后', + 'setupPrompt.open': '前往安装', 'setup.local': '将在运行 DSH 的机器上下载本地模型,无需安装 Python 或编译工具。', 'setup.disk': '硬盘空间', 'setup.diskValue': '建议预留约 {gb} GB,包含模型、运行时和下载缓存', @@ -81,6 +86,11 @@ export type VoiceKey = keyof typeof zh /** English dictionary with the same complete key set. */ export const en: Record = { + dictate: 'Dictate', + 'setupPrompt.title': 'Set up voice input before recording', + 'setupPrompt.body': 'Voice input is enabled. First use requires downloading and preparing local recognition models. Open the plugin details to review disk space, memory and time estimates before starting setup.', + 'setupPrompt.later': 'Later', + 'setupPrompt.open': 'Go to setup', 'setup.local': 'Local models will be downloaded to the machine running DSH. No Python or compiler is required.', 'setup.disk': 'Disk space', 'setup.diskValue': 'Allow about {gb} GB for models, runtime, and download caches', diff --git a/packages/experimental/client-ui-voice-input/src/client/mount.ts b/packages/experimental/client-ui-voice-input/src/client/mount.ts index 7e617324d9..3af8f6d451 100644 --- a/packages/experimental/client-ui-voice-input/src/client/mount.ts +++ b/packages/experimental/client-ui-voice-input/src/client/mount.ts @@ -12,6 +12,7 @@ import { en, NS, zh } from './locales.ts' import type {} from '@deepseek-ai/dsh-client-ui-plugin-manager/client' import { observeReadiness } from './readiness.ts' import { VoicePreparation, VoiceStatus } from './PreparationCard.tsx' +import { VoiceSetupPrompt } from './VoiceSetupPrompt.tsx' export const inject = ['remote', 'slots', 'locale'] @@ -44,6 +45,9 @@ function registerUi(ctx: Context): void { ctx.slots.inject('plugins.bundle.config', () => ctx.slots.register({ name: 'plugins.bundle.config', key: '@deepseek-ai/dsh-experimental-voice-input-bundle', locale: NS, inject: () => actions, }, VoicePreparation)) + ctx.slots.inject('plugins.bundle.activation', () => ctx.slots.register({ name: 'plugins.bundle.activation', + key: '@deepseek-ai/dsh-experimental-voice-input-bundle', locale: NS, inject: () => actions, + }, VoiceSetupPrompt)) } diff --git a/packages/experimental/client-ui-voice-input/tests/mount.client.spec.ts b/packages/experimental/client-ui-voice-input/tests/mount.client.spec.ts index 4660be0cca..a11f6d7eac 100644 --- a/packages/experimental/client-ui-voice-input/tests/mount.client.spec.ts +++ b/packages/experimental/client-ui-voice-input/tests/mount.client.spec.ts @@ -31,6 +31,7 @@ async function fixture(fail = false) { ctx.slots.register({ name: 'root', children: { 'conversation.input.activity': { kind: 'single', scope: 'session' }, 'plugins.bundle.status': { kind: 'keyed', scope: 'root' }, + 'plugins.bundle.activation': { kind: 'keyed', scope: 'root' }, 'plugins.bundle.config': { kind: 'keyed', scope: 'root' }, } } as never, () => null) @@ -54,7 +55,7 @@ it('withdraws its Remote, localized slot and microphone captures on disposal', a await actions.configure({ language: 'zh' }) await actions.prepare('local' as SpeechProviderId) await actions.cancelPreparation('local' as SpeechProviderId) - for (const slot of ['plugins.bundle.status', 'plugins.bundle.config'] as const) { + for (const slot of ['plugins.bundle.status', 'plugins.bundle.config', 'plugins.bundle.activation'] as const) { const item = b.ctx.slots.entries(slot)[0]! expect(item.locale).toBe('voice-input') const injected = (item.inject as unknown as () => VoiceInputInjected & { compact?: boolean })() @@ -74,6 +75,7 @@ it('withdraws its Remote, localized slot and microphone captures on disposal', a await fiber.dispose() expect(dispose).toHaveBeenCalledOnce() expect(b.ctx.slots.entries('conversation.input.activity')).toHaveLength(0) + expect(b.ctx.slots.entries('plugins.bundle.activation')).toHaveLength(0) expect(b.unmount).toHaveBeenCalledOnce() } finally { await b.ctx.fiber.dispose() } }) diff --git a/packages/experimental/client-ui-voice-input/tests/setup-prompt.client.spec.tsx b/packages/experimental/client-ui-voice-input/tests/setup-prompt.client.spec.tsx new file mode 100644 index 0000000000..cb66648dfe --- /dev/null +++ b/packages/experimental/client-ui-voice-input/tests/setup-prompt.client.spec.tsx @@ -0,0 +1,61 @@ +// @vitest-environment jsdom +/** Activation guidance waits for cache inspection and leaves installation to the detail page. */ +import { act, cleanup, fireEvent, render, screen } from '@testing-library/react' +import { bindSnapshotSelector, makeTranslate } from '@deepseek-ai/dsh-client-test-runtime' +import { createSnapshotStore } from '@deepseek-ai/dsh-client-store' +import type { SpeechPreparationState, SpeechProviderId } from '@deepseek-ai/dsh-experimental-speech-to-text/types' +import { afterEach, expect, it, vi } from 'vitest' +import { VoiceSetupPrompt, type VoiceSetupPromptProps } from '../src/client/VoiceSetupPrompt.tsx' +import type { SpeechReadiness } from '../src/client/readiness.ts' +import { zh } from '../src/client/locales.ts' + +afterEach(cleanup) + +function fixture() { + const state = createSnapshotStore({ catalog: null, connected: false, error: null }) + const onDismiss = vi.fn(), onOpenDetails = vi.fn(), providerId = 'local' as SpeechProviderId + const props = { packageName: 'voice-bundle', t: makeTranslate(zh), onDismiss, onOpenDetails, + useSpeechReadiness: bindSnapshotSelector(state) } as VoiceSetupPromptProps + render() + return { onDismiss, onOpenDetails, publish(preparation: SpeechPreparationState, location: 'host-local' | 'cloud' = 'host-local') { + act(() => { state.set({ connected: true, error: null, catalog: { + selection: { providerId, language: 'auto' }, maxAudioBytes: 1000, maxDurationSeconds: 120, + providers: [{ id: providerId, name: 'SenseVoice', location, preparation }], + } }) }) + } } +} + +it('waits for inspection, then offers the installation page without starting preparation', () => { + const b = fixture() + expect(screen.queryByText(zh['setupPrompt.title'])).toBeNull() + b.publish({ phase: 'checking', step: 'check', startedAt: 0 }) + expect(screen.queryByText(zh['setupPrompt.title'])).toBeNull() + expect(b.onDismiss).not.toHaveBeenCalled() + b.publish({ phase: 'unprepared' }) + expect(screen.getByText(zh['setupPrompt.body'])).toBeTruthy() + fireEvent.click(screen.getByRole('button', { name: zh['setupPrompt.open'] })) + expect(b.onOpenDetails).toHaveBeenCalledOnce() + expect(b.onDismiss).not.toHaveBeenCalled() +}) + +it('allows postponing installation while leaving the plugin enabled', () => { + const b = fixture() + b.publish({ phase: 'unprepared' }) + fireEvent.click(screen.getByRole('button', { name: zh['setupPrompt.later'] })) + expect(b.onDismiss).toHaveBeenCalledOnce() + expect(b.onOpenDetails).not.toHaveBeenCalled() +}) + +it.each(['standby', 'ready', 'failed'] as const)('does not offer installation for %s resources', (phase) => { + const b = fixture() + b.publish(phase === 'failed' ? { phase, message: 'cache unreadable' } : { phase }) + expect(screen.queryByText(zh['setupPrompt.title'])).toBeNull() + expect(b.onDismiss).toHaveBeenCalledOnce() +}) + +it('does not describe cloud-provider setup as a local model installation', () => { + const b = fixture() + b.publish({ phase: 'unprepared' }, 'cloud') + expect(screen.queryByText(zh['setupPrompt.title'])).toBeNull() + expect(b.onDismiss).toHaveBeenCalledOnce() +}) diff --git a/packages/experimental/client-ui-voice-input/tests/voice-input.client.spec.tsx b/packages/experimental/client-ui-voice-input/tests/voice-input.client.spec.tsx index d1ae8fe14a..9bb28bcc89 100644 --- a/packages/experimental/client-ui-voice-input/tests/voice-input.client.spec.tsx +++ b/packages/experimental/client-ui-voice-input/tests/voice-input.client.spec.tsx @@ -43,6 +43,7 @@ function stop(): void { fireEvent.click(screen.getByRole('button', { name: zh.st it('clicks to record, shows measured audio, and stops to insert without sending or a popup', async () => { const b = fixture() + expect(screen.getByRole('button', { name: zh.start }).getAttribute('title')).toBe('听写') fireEvent.mouseDown(screen.getByRole('button', { name: zh.start })) await start() expect(b.props.onActiveChange).toHaveBeenLastCalledWith(true) diff --git a/packages/experimental/speech-to-text-sensevoice/README.i18n.yaml b/packages/experimental/speech-to-text-sensevoice/README.i18n.yaml index 8f15679ab0..740410587c 100644 --- a/packages/experimental/speech-to-text-sensevoice/README.i18n.yaml +++ b/packages/experimental/speech-to-text-sensevoice/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/experimental/speech-to-text-sensevoice/README.md -README.md: f8fe45e2208aa0235811182173abfe1418ab28b3 -README.zh.md: d1775798847fb2ee4be8fc92c71f1c3f600facd9 +README.md: 62912955502c16504838a91e2dc866d31d7170ff +README.zh.md: 9251ac78107d7003d7aa47a8af5616685592d684 diff --git a/packages/experimental/speech-to-text-sensevoice/README.md b/packages/experimental/speech-to-text-sensevoice/README.md index f8fe45e220..6291295550 100644 --- a/packages/experimental/speech-to-text-sensevoice/README.md +++ b/packages/experimental/speech-to-text-sensevoice/README.md @@ -9,7 +9,7 @@ English | [中文](README.zh.md) ## Summary -This provider recognizes speech with SenseVoiceSmall ONNX and Silero VAD on the Host CPU. The platform-specific sherpa-onnx Node package includes ONNX Runtime; users need no Python, compiler or model conversion. Activation registers the provider without loading models or downloading assets. +This provider recognizes speech with SenseVoiceSmall ONNX and Silero VAD on the Host CPU. The platform-specific sherpa-onnx Node package includes ONNX Runtime; users need no Python, compiler or model conversion. Activation checks cached resources without loading models or downloading assets. ## Table of Contents @@ -27,6 +27,8 @@ This provider recognizes speech with SenseVoiceSmall ONNX and Silero VAD on the The [bundle](../voice-input-bundle/README.md) supplies an absolute `dataRoot` under the DSH home. Preparation downloads revision-pinned model files, verifies their sizes and SHA-256 hashes, then loads them. `precision` defaults to `int8`; `fp32` selects the larger reference weights. `modelDirectory` supplies an existing absolute directory containing the selected ONNX file and `tokens.txt`; `vadModelPath` selects an existing Silero ONNX file. `modelOrigin` selects a Hugging Face-compatible download origin while preserving pinned paths and checksums. Verified completed files remain reusable after cancellation or failure. +Every activation checks the selected model, tokens and VAD on disk. Complete verified caches restore readiness immediately after inspection; the first recording wakes the worker. Missing files or mismatched managed checksums require explicit preparation, while unreadable paths report an error. Explicit deployment paths are checked for accessibility; their contents remain the deployer's responsibility. Disabling the plugin or restarting the Host does not require downloading intact caches again. + ----- diff --git a/packages/experimental/speech-to-text-sensevoice/README.zh.md b/packages/experimental/speech-to-text-sensevoice/README.zh.md index d177579884..9251ac7810 100644 --- a/packages/experimental/speech-to-text-sensevoice/README.zh.md +++ b/packages/experimental/speech-to-text-sensevoice/README.zh.md @@ -9,7 +9,7 @@ kind: "package-reference" ## 概述 -此 Provider 在 Host CPU 上使用 SenseVoiceSmall ONNX 与 Silero VAD 识别语音。各平台的 sherpa-onnx Node 包包含 ONNX Runtime;用户无需 Python、编译器或模型转换。启用只注册 Provider,不加载模型或下载资源。 +此 Provider 在 Host CPU 上使用 SenseVoiceSmall ONNX 与 Silero VAD 识别语音。各平台的 sherpa-onnx Node 包包含 ONNX Runtime;用户无需 Python、编译器或模型转换。启用时检查缓存资源,不加载模型或下载资源。 ## 目录 @@ -27,6 +27,8 @@ kind: "package-reference" [Bundle](../voice-input-bundle/README.zh.md) 在 DSH 主目录下提供绝对 `dataRoot`。准备会下载固定修订的模型文件,校验大小与 SHA-256,再加载模型。`precision` 默认为 `int8`,`fp32` 选择较大的参考权重。`modelDirectory` 提供包含所选 ONNX 文件和 `tokens.txt` 的已有绝对目录,`vadModelPath` 选择已有 Silero ONNX 文件。`modelOrigin` 选择兼容 Hugging Face 的下载源,固定路径和校验值保持不变。取消或失败后复用已完成并通过校验的文件。 +每次启用都会检查磁盘上的所选模型、词表与 VAD。缓存完整且校验通过后立即恢复就绪,首次录音再唤醒工作进程。文件缺失或托管文件校验不符时需要显式准备;路径不可读时报告错误。显式部署路径只检查可访问性,文件内容由部署者负责。禁用插件或重启 Host 后,无需重新下载完好的缓存。 + ----- diff --git a/packages/experimental/speech-to-text-sensevoice/src/index.ts b/packages/experimental/speech-to-text-sensevoice/src/index.ts index 1bc4d70bf7..5fd467f9f1 100644 --- a/packages/experimental/speech-to-text-sensevoice/src/index.ts +++ b/packages/experimental/speech-to-text-sensevoice/src/index.ts @@ -12,7 +12,7 @@ export const name = 'experimental-speech-to-text-sensevoice' export const inject = ['speechToText', 'subprocess'] /** - * Register the local recognizer under its configured provider id. + * Register the local recognizer and inspect disk caches without downloading or loading models. * @param ctx - Host registry and subprocess owner. * @param config - validated runtime configuration. */ @@ -34,6 +34,7 @@ export function apply(ctx: Context, config: Config): void { return await worker.transcribe(input, signal) }, }) + worker.inspect() return async () => { const removing = unregister() await worker.dispose() diff --git a/packages/experimental/speech-to-text-sensevoice/src/recognizer.ts b/packages/experimental/speech-to-text-sensevoice/src/recognizer.ts index 69cfd2ca56..797c86ea0e 100644 --- a/packages/experimental/speech-to-text-sensevoice/src/recognizer.ts +++ b/packages/experimental/speech-to-text-sensevoice/src/recognizer.ts @@ -7,7 +7,7 @@ import type { SpeechInput, SpeechPreparationState, SpeechPreparationStep, Speech import { deadline } from '@deepseek-ai/dsh-timeout' import { z } from 'zod' import type { Config } from './config.ts' -import { prepareRuntime, type RuntimePaths } from './runtime.ts' +import { inspectRuntime, prepareRuntime, type RuntimePaths } from './runtime.ts' const transcriptSchema = z.object({ text: z.string(), audioSeconds: z.number().nonnegative(), inferenceSeconds: z.number().nonnegative(), @@ -101,7 +101,6 @@ export class SenseVoiceWorker { private readonly lifetime = new AbortController() private idle: ReturnType | undefined private runtime: RuntimePaths | undefined - private loaded = false private state: SpeechPreparationState & { readonly steps: readonly SpeechPreparationStep[] } private readonly listeners = new Set<() => void>() private lastProgressAt = 0 @@ -133,7 +132,7 @@ export class SenseVoiceWorker { private publish(state: SpeechPreparationState): void { const previous = this.state - const steps = previous.steps.map((item): SpeechPreparationStep => { + const steps = state.steps ?? previous.steps.map((item): SpeechPreparationStep => { if (state.phase === 'ready') return { ...item, status: 'complete' } if (state.step === 'check' && item.kind !== 'check') return { kind: item.kind, status: 'pending' } if (state.step === item.kind) return item.status === 'running' ? item @@ -151,15 +150,34 @@ export class SenseVoiceWorker { for (const listener of this.listeners) listener() } + /** Inspect disk caches on activation; valid resources enter standby without starting a worker. */ + inspect(): void { + this.runPreparation(async (signal) => { + using inspection = deadline(signal, this.config.prepareTimeoutMs, 'SPEECH_PREPARE_TIMEOUT') + this.publish({ phase: 'checking', step: 'check', startedAt: Date.now() }) + const runtime = await inspectRuntime(this.config, inspection.signal) + inspection.signal.throwIfAborted() + this.runtime = runtime + this.publish({ phase: runtime ? 'standby' : 'unprepared', steps: this.state.steps.map(({ kind }) => ({ kind, + status: kind === 'check' || runtime && kind !== 'load' ? 'complete' : 'pending', + })) }) + }) + } + /** Start one Host-owned preparation task; repeated callers join it. */ prepare(): void { + this.runPreparation(async (signal) => { await this.start(signal) }) + } + + private runPreparation(run: (signal: AbortSignal) => Promise): void { if (this.preparing || this.worker) return this.lifetime.signal.throwIfAborted() const abort = new AbortController() const task = { abort, settled: Promise.resolve() } this.preparing = task - task.settled = this.enqueue(async (signal) => { await this.start(signal) }, abort.signal) + task.settled = this.enqueue(run, abort.signal) .catch((error: unknown) => { + if (this.lifetime.signal.aborted) return this.publish(abort.signal.aborted ? { phase: 'cancelled' } : { phase: 'failed', message: error instanceof Error ? error.message : String(error) }) }).finally(() => { this.preparing = undefined }) @@ -215,7 +233,7 @@ export class SenseVoiceWorker { const runtime = this.runtime ?? await prepareRuntime(this.ctx, this.config, setup.signal, (state) => { this.publish(state) }) this.runtime = runtime setup.signal.throwIfAborted() - this.publish({ phase: this.loaded ? 'waking' : 'loading', step: 'load', startedAt: Date.now() }) + this.publish({ phase: cached ? 'waking' : 'loading', step: 'load', startedAt: Date.now() }) await mkdir(this.config.dataRoot, { recursive: true }) const token = randomBytes(32).toString('hex') const handle = this.ctx.subprocess.spawn({ @@ -228,7 +246,6 @@ export class SenseVoiceWorker { const port = await readReady(handle, this.config.maxLogBytes, setup.signal) const worker = { handle, url: `http://127.0.0.1:${port}`, token, closed: false } this.worker = worker - this.loaded = true this.publish({ phase: 'ready' }) const exited = (): void => { if (!worker.closed) { worker.closed = true; this.publish({ phase: 'standby' }) } @@ -272,6 +289,7 @@ export class SenseVoiceWorker { /** Stop the local recognizer. @returns after admission closes, queued jobs settle, and the managed worker exits. */ async dispose(): Promise { + this.listeners.clear() this.lifetime.abort(new Error('SenseVoice provider disposed')) clearTimeout(this.idle) try { await this.tail } finally { await this.stop() } diff --git a/packages/experimental/speech-to-text-sensevoice/src/runtime.ts b/packages/experimental/speech-to-text-sensevoice/src/runtime.ts index 2290b3658b..a2ef43a17d 100644 --- a/packages/experimental/speech-to-text-sensevoice/src/runtime.ts +++ b/packages/experimental/speech-to-text-sensevoice/src/runtime.ts @@ -1,7 +1,7 @@ /** Verified runtime assets and managed command execution for local transcription. */ import { createHash, randomUUID } from 'node:crypto' import { createReadStream, readFileSync } from 'node:fs' -import { access, mkdir, rename, rm } from 'node:fs/promises' +import { access, mkdir, rename, rm, stat } from 'node:fs/promises' import { join } from 'node:path' import { Transform } from 'node:stream' import { pipeline } from 'node:stream/promises' @@ -34,6 +34,62 @@ export interface RuntimePaths { readonly worker: string } +async function matchesAsset(path: string, asset: Asset, signal: AbortSignal): Promise { + signal.throwIfAborted() + try { + const info = await stat(path) + if (!info.isFile()) throw new Error(`Speech asset is not a regular file: ${path}`) + if (info.size !== asset.bytes) return false + const digest = createHash('sha256') + for await (const chunk of createReadStream(path, { signal })) digest.update(chunk as Buffer) + return digest.digest('hex') === asset.sha256 + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error + return false + } +} + +function resolveRuntime(config: Config) { + const supported = ['darwin-arm64', 'darwin-x64', 'linux-arm64', 'linux-x64', 'win32-x64'] + if (!supported.includes(`${process.platform}-${process.arch}`)) throw new Error(`Local speech is unavailable for ${process.platform}-${process.arch}`) + const lock = JSON.parse(readFileSync(new URL('../runtime/assets.json', import.meta.url), 'utf8')) as RuntimeLock + const modelRoot = config.modelDirectory ?? join(config.dataRoot, 'models', 'sensevoice-onnx') + const paths: RuntimePaths = { + model: join(modelRoot, lock.models[config.precision].name), tokens: join(modelRoot, lock.tokens.name), + vad: config.vadModelPath ?? join(config.dataRoot, 'models', 'silero', lock.vad.name), + /* v8 ignore next -- built worker resolution is exercised by real Node and Electron process smokes */ + worker: fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './worker.ts' : './worker.js', import.meta.url)), + } + return { lock, modelRoot, paths } +} + +async function verifyRuntime(config: Config, paths: RuntimePaths, lock: RuntimeLock, signal: AbortSignal): Promise { + signal.throwIfAborted() + try { await Promise.all([paths.model, paths.tokens, paths.vad].map(path => access(path))) } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error + return false + } + const verified = [ + ...config.modelDirectory === undefined ? [[paths.model, lock.models[config.precision]], [paths.tokens, lock.tokens]] as const : [], + ...config.vadModelPath === undefined ? [[paths.vad, lock.vad]] as const : [], + ] + for (const [path, asset] of verified) if (!await matchesAsset(path, asset, signal)) return false + signal.throwIfAborted() + return true +} + +/** + * Inspect existing models without downloading, writing files or starting the worker. + * @param config - model paths and selected precision. + * @param signal - provider cancellation or inspection deadline. + * @returns cached paths when all files exist and managed assets match their pinned size and hash; otherwise undefined. + */ +export async function inspectRuntime(config: Config, signal: AbortSignal): Promise { + const { lock, paths } = resolveRuntime(config) + return await verifyRuntime(config, paths, lock, signal) ? paths : undefined +} + /** * Download into a unique partial file, verify, then publish it atomically. * @param asset - pinned release identity. @@ -47,13 +103,7 @@ export async function downloadAsset(asset: Asset, root: string, signal: AbortSig signal.throwIfAborted() await mkdir(root, { recursive: true }) const destination = join(root, asset.name) - try { - const digest = createHash('sha256') - for await (const chunk of createReadStream(destination, { signal })) digest.update(chunk as Buffer) - if (digest.digest('hex') === asset.sha256) return destination - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error - } + if (await matchesAsset(destination, asset, signal)) return destination const partial = `${destination}.${randomUUID()}.part` try { const response = await fetch(asset.url, { signal }) @@ -89,12 +139,7 @@ export async function downloadAsset(asset: Asset, root: string, signal: AbortSig */ export async function prepareRuntime(_ctx: Context, config: Config, signal: AbortSignal, report: (state: SpeechPreparationState) => void = () => {}): Promise { - const supported = ['darwin-arm64', 'darwin-x64', 'linux-arm64', 'linux-x64', 'win32-x64'] - if (!supported.includes(`${process.platform}-${process.arch}`)) throw new Error(`Local speech is unavailable for ${process.platform}-${process.arch}`) - const lock = JSON.parse(readFileSync(new URL('../runtime/assets.json', import.meta.url), 'utf8')) as RuntimeLock - const modelRoot = config.modelDirectory ?? join(config.dataRoot, 'models', 'sensevoice-onnx') - const model = join(modelRoot, lock.models[config.precision].name), tokens = join(modelRoot, lock.tokens.name) - const vad = config.vadModelPath ?? join(config.dataRoot, 'models', 'silero', lock.vad.name) + const { lock, modelRoot, paths } = resolveRuntime(config) const download = async (asset: Asset, root: string, step: 'model' | 'vad'): Promise => { const url = new URL(asset.url) const pinned = { ...asset, url: `${config.modelOrigin.replace(/\/$/, '')}${url.pathname}` } @@ -110,17 +155,6 @@ export async function prepareRuntime(_ctx: Context, config: Config, signal: Abor await download(lock.vad, join(config.dataRoot, 'models', 'silero'), 'vad') } report({ phase: 'checking', step: 'verify', startedAt: Date.now() }) - await Promise.all([model, tokens, vad].map(path => access(path))) - const verified = [ - ...config.modelDirectory === undefined ? [[model, lock.models[config.precision]], [tokens, lock.tokens]] as const : [], - ...config.vadModelPath === undefined ? [[vad, lock.vad]] as const : [], - ] - for (const [path, asset] of verified) { - const digest = createHash('sha256') - for await (const chunk of createReadStream(path, { signal })) digest.update(chunk as Buffer) - if (digest.digest('hex') !== asset.sha256) throw new Error(`Speech model verification failed: ${asset.name}`) - } - signal.throwIfAborted() - /* v8 ignore next -- built worker resolution is exercised by real Node and Electron process smokes */ - return { model, tokens, vad, worker: fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './worker.ts' : './worker.js', import.meta.url)) } + if (!await verifyRuntime(config, paths, lock, signal)) throw new Error('Speech model verification failed: missing or corrupted model files') + return paths } diff --git a/packages/experimental/speech-to-text-sensevoice/tests/installer.spec.ts b/packages/experimental/speech-to-text-sensevoice/tests/installer.spec.ts index e229a96f4a..e1030c7322 100644 --- a/packages/experimental/speech-to-text-sensevoice/tests/installer.spec.ts +++ b/packages/experimental/speech-to-text-sensevoice/tests/installer.spec.ts @@ -1,19 +1,26 @@ /** ONNX preparation reuses verified files and honors explicit offline deployments. */ import { writeFileSync } from 'node:fs' import { createHash } from 'node:crypto' -import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { access, mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { Context } from '@deepseek-ai/cordis' +import LocalSubprocess from '@deepseek-ai/dsh-subprocess-local' +import SpeechToText from '@deepseek-ai/dsh-experimental-speech-to-text' import { afterEach, expect, it, vi } from 'vitest' import type { SpeechPreparationState } from '@deepseek-ai/dsh-experimental-speech-to-text/types' -import { prepareRuntime, type Asset } from '../src/runtime.ts' +import { inspectRuntime, prepareRuntime, type Asset } from '../src/runtime.ts' import { Config } from '../src/config.ts' +import * as Provider from '../src/index.ts' const bytes = Buffer.from('pinned ONNX fixture') const asset = (name: string): Asset => ({ name, url: `https://huggingface.co/owner/model/resolve/revision/${name}`, bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') }) const fake = vi.hoisted(() => ({ lock: {} })) +vi.mock('node:fs/promises', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, access: vi.fn(actual.access) } +}) vi.mock('node:fs', async (importOriginal) => { const actual = await importOriginal() return { ...actual, readFileSync: (...args: Parameters) => { @@ -23,9 +30,65 @@ vi.mock('node:fs', async (importOriginal) => { }) const roots: string[] = [] afterEach(async () => { + vi.restoreAllMocks() vi.unstubAllGlobals() for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }) }) + +it('restores verified cached models on every provider activation without downloads or a worker', async () => { + const { root, fetcher } = await fixture(), config = Config({ dataRoot: root }) + await prepareRuntime(new Context(), config, new AbortController().signal) + fetcher.mockClear() + const ctx = new Context() + try { + await ctx.plugin(LocalSubprocess) + await ctx.plugin(SpeechToText) + const spawn = vi.spyOn(ctx.get('subprocess')!, 'spawn'), speech = ctx.get('speechToText')! + for (let activation = 0; activation < 2; activation++) { + const fiber = ctx.plugin(Provider, config) + await fiber + await vi.waitFor(() => { expect(speech.snapshot().providers[0]?.preparation.phase).toBe('standby') }) + expect(speech.snapshot().providers[0]?.preparation.steps?.map(({ kind, status }) => [kind, status])).toEqual([ + ['check', 'complete'], ['model', 'complete'], ['vad', 'complete'], ['verify', 'complete'], ['load', 'pending'], + ]) + expect(fetcher).not.toHaveBeenCalled() + expect(spawn).not.toHaveBeenCalled() + await fiber.dispose() + expect(speech.snapshot().providers).toEqual([]) + } + } finally { await ctx.fiber.dispose() } +}) + +it.each(['model', 'tokens', 'vad'] as const)('requires preparation for a missing or corrupted cached %s', async (file) => { + const { root, fetcher } = await fixture(), config = Config({ dataRoot: root }), signal = new AbortController().signal + const paths = await prepareRuntime(new Context(), config, signal) + fetcher.mockClear() + expect(await inspectRuntime(config, signal)).toEqual(paths) + await writeFile(paths[file], Buffer.alloc(bytes.length)) + expect(await inspectRuntime(config, signal)).toBeUndefined() + await rm(paths[file]) + expect(await inspectRuntime(config, signal)).toBeUndefined() + expect(fetcher).not.toHaveBeenCalled() +}) + +it('inspects an empty cache without creating it and respects precision and offline paths', async () => { + const { root, fetcher } = await fixture(), signal = new AbortController().signal + const missingRoot = join(root, 'absent') + expect(await inspectRuntime(Config({ dataRoot: missingRoot }), signal)).toBeUndefined() + await expect(readFile(missingRoot)).rejects.toThrow() + expect(fetcher).not.toHaveBeenCalled() + const config = Config({ dataRoot: root }), paths = await prepareRuntime(new Context(), config, signal) + expect(await inspectRuntime(Config(Object.assign({}, config, { precision: 'fp32' })), signal)).toBeUndefined() + expect(await inspectRuntime(Config(Object.assign({}, config, { modelDirectory: join(root, 'models', 'sensevoice-onnx'), vadModelPath: paths.vad })), signal)).toEqual(paths) +}) + +it('reports filesystem access errors instead of treating unreadable caches as absent', async () => { + const { root, fetcher } = await fixture() + const failure = Object.assign(new Error('cache is not readable'), { code: 'EACCES' }) + vi.mocked(access).mockRejectedValueOnce(failure) + await expect(inspectRuntime(Config({ dataRoot: root }), new AbortController().signal)).rejects.toBe(failure) + expect(fetcher).not.toHaveBeenCalled() +}) async function fixture() { const root = await mkdtemp(join(tmpdir(), 'dsh-speech-onnx-')); roots.push(root) fake.lock = { models: { int8: asset('model.int8.onnx'), fp32: asset('model.onnx') }, tokens: asset('tokens.txt'), vad: asset('silero_vad.onnx') } diff --git a/packages/experimental/speech-to-text-sensevoice/tests/worker.spec.ts b/packages/experimental/speech-to-text-sensevoice/tests/worker.spec.ts index c00df0a59d..09af51a3c7 100644 --- a/packages/experimental/speech-to-text-sensevoice/tests/worker.spec.ts +++ b/packages/experimental/speech-to-text-sensevoice/tests/worker.spec.ts @@ -9,9 +9,9 @@ import LocalSubprocess from '@deepseek-ai/dsh-subprocess-local' import { afterEach, expect, it, vi } from 'vitest' import { Config } from '../src/config.ts' import { SenseVoiceWorker } from '../src/recognizer.ts' -import { prepareRuntime } from '../src/runtime.ts' +import { inspectRuntime, prepareRuntime } from '../src/runtime.ts' -vi.mock('../src/runtime.ts', () => ({ prepareRuntime: vi.fn() })) +vi.mock('../src/runtime.ts', () => ({ inspectRuntime: vi.fn(), prepareRuntime: vi.fn() })) const cleanup: Array<() => Promise> = [] afterEach(async () => { for (const close of cleanup.splice(0).reverse()) await close(); vi.clearAllMocks() }) @@ -51,6 +51,63 @@ it('keeps one worker warm across recordings and joins its exit on disposal', asy await expect(worker.transcribe({ audio, language: 'en' }, signal())).rejects.toThrow('disposed') }) +it('wakes verified caches on the first recording without preparing them again', async () => { + const { worker, spawn, root } = await fixture() + const paths = { tokens: root, model: root, vad: root, worker: fileURLToPath(new URL('./worker.fixture.mjs', import.meta.url)) } + vi.mocked(inspectRuntime).mockResolvedValueOnce(paths) + const phases: string[] = [] + worker.subscribe(() => { phases.push(worker.snapshot().phase) }) + worker.inspect(); worker.inspect() + await vi.waitFor(() => { expect(worker.snapshot().phase).toBe('standby') }) + expect(inspectRuntime).toHaveBeenCalledOnce() + expect(spawn).not.toHaveBeenCalled() + expect((await worker.transcribe({ audio, language: 'zh' }, signal())).text).toBe('zh') + expect(phases).toContain('waking') + expect(phases).not.toContain('loading') + expect(prepareRuntime).not.toHaveBeenCalled() + expect(spawn).toHaveBeenCalledOnce() +}) + +it('keeps absent caches unprepared and reports unreadable caches as retryable failures', async () => { + const { worker, spawn } = await fixture() + vi.mocked(inspectRuntime).mockResolvedValueOnce(undefined) + worker.inspect() + await vi.waitFor(() => { expect(worker.snapshot().steps?.[0]?.status).toBe('complete') }) + expect(worker.snapshot().phase).toBe('unprepared') + expect(spawn).not.toHaveBeenCalled() + expect(prepareRuntime).not.toHaveBeenCalled() + vi.mocked(inspectRuntime).mockRejectedValueOnce(new Error('cache unreadable')) + worker.inspect() + await vi.waitFor(() => { expect(worker.snapshot()).toMatchObject({ phase: 'failed', message: 'cache unreadable' }) }) + worker.prepare() + await vi.waitFor(() => { expect(worker.snapshot().phase).toBe('ready') }, { timeout: 10000 }) + expect(prepareRuntime).toHaveBeenCalledOnce() +}) + +it('cancels cache inspection and joins it without late notifications when the provider unloads', async () => { + const { worker } = await fixture(), entered = Promise.withResolvers(), released = Promise.withResolvers() + vi.mocked(inspectRuntime).mockImplementationOnce(async (_config, signal) => { + entered.resolve(undefined) + await released.promise + signal.throwIfAborted() + return undefined + }) + const changed = vi.fn() + worker.subscribe(changed) + worker.inspect(); await entered.promise + const cancelled = worker.cancel() + expect(worker.snapshot().phase).toBe('cancelling') + let settled = false + const disposed = worker.dispose().then(() => { settled = true }) + const notifications = changed.mock.calls.length + await Promise.resolve() + expect(settled).toBe(false) + released.resolve(undefined) + await cancelled; await disposed + expect(changed).toHaveBeenCalledTimes(notifications) + expect(prepareRuntime).not.toHaveBeenCalled() +}) + it('cancels active inference before the next recording can acquire a fresh worker', async () => { const { worker, root, spawn } = await fixture() const cancel = new AbortController() diff --git a/packages/extensions/cordis-client-runner/src/client/slot-catalog.ts b/packages/extensions/cordis-client-runner/src/client/slot-catalog.ts index f08e818bf6..e5f8c3224b 100644 --- a/packages/extensions/cordis-client-runner/src/client/slot-catalog.ts +++ b/packages/extensions/cordis-client-runner/src/client/slot-catalog.ts @@ -1582,6 +1582,44 @@ export const CLIENT_SLOT_API: readonly ClientSlotEntry[] = [ example: 'return {\n inject: [\'slots\'],\n apply(ctx) {\n ctx.slots.inject(\'main.conversation\', () => ctx.slots.register(\n { name: \'main.conversation\' },\n () => React.createElement(\'div\', null, \'hello\'),\n ))\n },\n}', source: 'packages/client/ui-conversation/src/client/contract/slots.ts:120', }, + { + key: 'plugins.bundle.activation', + kind: 'keyed', + scope: 'root', + summary: 'Optional guidance after the user enables a bundle from the list, keyed by npm package name.', + doc: 'Optional guidance after the user enables a bundle from the list, keyed by npm package name.', + registerOptions: [ + { + name: 'key', + requirement: 'required', + type: 'string', + doc: 'Your cell key: the entry renders where the owner dispatches this exact key. Registering an already-occupied key replaces that occupant.', + }, + ], + ownerProps: [ + '/** One user-requested bundle activation and navigation to its configuration page. */\nexport interface PluginActivationOwnerProps {\n readonly packageName: string\n /** Dismiss guidance for this activation. */\n readonly onDismiss: () => void\n /** Dismiss guidance and open this bundle\'s detail page. */\n readonly onOpenDetails: () => void\n}', + ], + ownerPropsReferences: [], + standardProps: [ + 'useResource: UseResource', + 'useWorkspaces: SnapshotSelectorHook', + 'usePanelInfo: UsePanelInfo', + 'useSessions: UseSessions', + 'useSessionStatus: UseSessionStatus', + 'useSessionRetainInfo: UseSessionRetainInfo', + 'useWorkspaces: SnapshotSelectorHook', + ], + keyDomain: 'open: any string the owner dispatches (no compile-time key set), already taken: @deepseek-ai/dsh-experimental-voice-input-bundle', + hookContext: '', + slotInject: '', + declaredBy: 'an entry in \'main\' (client-ui-plugin-manager), so it exists while that entry is mounted', + occupants: [ + 'experimental-client-ui-voice-input VoiceSetupPrompt key \'@deepseek-ai/dsh-experimental-voice-input-bundle\'', + ], + replaceRisk: 'shadows-shipped-ui', + example: 'return {\n inject: [\'slots\'],\n apply(ctx) {\n ctx.slots.inject(\'plugins.bundle.activation\', () => ctx.slots.register(\n { name: \'plugins.bundle.activation\', key: \'\' },\n () => React.createElement(\'div\', null, \'hello\'),\n ))\n },\n}', + source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:41', + }, { key: 'plugins.bundle.config', kind: 'keyed', @@ -1618,7 +1656,7 @@ export const CLIENT_SLOT_API: readonly ClientSlotEntry[] = [ ], replaceRisk: 'shadows-shipped-ui', example: 'return {\n inject: [\'slots\'],\n apply(ctx) {\n ctx.slots.inject(\'plugins.bundle.config\', () => ctx.slots.register(\n { name: \'plugins.bundle.config\', key: \'\' },\n () => React.createElement(\'div\', null, \'hello\'),\n ))\n },\n}', - source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:46', + source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:57', }, { key: 'plugins.bundle.status', @@ -1656,7 +1694,7 @@ export const CLIENT_SLOT_API: readonly ClientSlotEntry[] = [ ], replaceRisk: 'shadows-shipped-ui', example: 'return {\n inject: [\'slots\'],\n apply(ctx) {\n ctx.slots.inject(\'plugins.bundle.status\', () => ctx.slots.register(\n { name: \'plugins.bundle.status\', key: \'\' },\n () => React.createElement(\'div\', null, \'hello\'),\n ))\n },\n}', - source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:30', + source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:39', }, { key: 'plugins.item', @@ -1709,7 +1747,7 @@ export const CLIENT_SLOT_API: readonly ClientSlotEntry[] = [ ], replaceRisk: 'none', example: 'return {\n inject: [\'slots\'],\n apply(ctx) {\n ctx.slots.inject(\'plugins.item\', () => ctx.slots.register(\n { name: \'plugins.item\', id: \'my-entry\', order: 100, label: \'My entry\' },\n () => React.createElement(\'div\', null, \'hello\'),\n ))\n },\n}', - source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:40', + source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:51', }, { key: 'plugins.row.config', @@ -1745,7 +1783,7 @@ export const CLIENT_SLOT_API: readonly ClientSlotEntry[] = [ occupants: [], replaceRisk: 'none', example: 'return {\n inject: [\'slots\'],\n apply(ctx) {\n ctx.slots.inject(\'plugins.row.config\', () => ctx.slots.register(\n { name: \'plugins.row.config\', key: \'\' },\n () => React.createElement(\'div\', null, \'hello\'),\n ))\n },\n}', - source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:53', + source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:64', }, { key: 'rightbar', From 9d0f475a47d3c0b22a54d357924c953651670267 Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Thu, 17 Sep 2026 13:33:38 +0800 Subject: [PATCH 03/12] fix(voice-input): keep preparation status in plugin details --- ...6-09-16-experimental-voice-input.i18n.yaml | 4 +- .../2026-09-16-experimental-voice-input.md | 2 +- .../2026-09-16-experimental-voice-input.zh.md | 2 +- apps/web/tests/voice-setup.e2e.ts | 2 + apps/web/tsconfig.json | 1 + docs/subsystems/slots.i18n.yaml | 4 +- docs/subsystems/slots.md | 2 +- docs/subsystems/slots.zh.md | 2 +- docs/subsystems/voice-input.i18n.yaml | 4 +- docs/subsystems/voice-input.md | 2 +- docs/subsystems/voice-input.zh.md | 2 +- .../client/ui-plugin-manager/README.i18n.yaml | 4 +- packages/client/ui-plugin-manager/README.md | 2 +- .../client/ui-plugin-manager/README.zh.md | 2 +- .../src/client/PluginManagerPage.tsx | 8 ++-- .../ui-plugin-manager/src/client/index.ts | 3 +- .../src/client/slot-contract.ts | 8 ---- .../tests/components.client.spec.tsx | 10 ---- .../client-ui-voice-input/README.i18n.yaml | 4 +- .../client-ui-voice-input/README.md | 2 +- .../client-ui-voice-input/README.zh.md | 2 +- .../src/client/PreparationCard.tsx | 11 ----- .../src/client/VoiceInput.module.css | 1 - .../src/client/locales.ts | 20 -------- .../client-ui-voice-input/src/client/mount.ts | 5 +- .../tests/mount.client.spec.ts | 3 +- .../tests/preparation.client.spec.tsx | 8 +--- .../src/client/slot-catalog.ts | 46 ++----------------- tsconfig.host.json | 1 + 29 files changed, 36 insertions(+), 131 deletions(-) diff --git a/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.i18n.yaml b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.i18n.yaml index 83b32e2b63..8e6b585de4 100644 --- a/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.md -2026-09-16-experimental-voice-input.md: b63d70a190d962b523fde9b76423d0da9ecb45fb -2026-09-16-experimental-voice-input.zh.md: 9a21aaf1a622bcad3f47b8b0f95bb5b18023941f +2026-09-16-experimental-voice-input.md: 8b646e917965175f099c37224261ca05c7c1adfb +2026-09-16-experimental-voice-input.zh.md: 15ffc46d8e964c0448e8bd1c7395a5ad464b2b2c diff --git a/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.md b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.md index b63d70a190..8b646e9179 100644 --- a/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.md +++ b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.md @@ -18,7 +18,7 @@ Recordings and transcripts remain transient until the user submits ordinary text SenseVoice ONNX and Silero VAD run in one managed CPU subprocess. The native sherpa-onnx package includes ONNX Runtime; explicit preparation only downloads and verifies models; subsequent calls reuse the warm model until the configured idle deadline. Every activation verifies cached assets and restores complete caches to standby without starting the worker. Readiness therefore follows disk contents rather than the lifetime of a Provider instance. Inspection uses the same owned task as preparation, so cancellation and disposal join it. Disabling the provider joins its work and process exit while retaining reusable disk assets. Native browser recording and resampling avoid an additional audio conversion executable. The authenticated private loopback protocol carries complete PCM recordings, allowing cancellation to terminate the worker without sharing a long-running inference with another request. -The Host owns preparation independently of the view that started it. An ordered step list survives view remounts and records completion only as execution advances. Recognition weights, VAD and final verification have separate operations; downloads use pinned byte counts, while other work reports elapsed time. Package status uses `plugins.bundle.status` and details use the upstream `plugins.bundle.config`, both keyed by package name. The generic `plugins.bundle.activation` slot carries explicit enablement intent and detail navigation; the voice occupant waits for Host cache inspection and offers installation guidance only for unprepared local models. Enabling an already prepared provider does not prompt or download. A single composer activity Slot expands the voice controls without replacing the draft editor or context-usage control. Shared primitives provide buttons, disclosure and status markers; only measured waveform drawing belongs to the voice UI. Recordings start on click and transcribe on Stop, with feedback inside the activity. Persistent preferences remain in plugin settings. +The Host owns preparation independently of the view that started it. An ordered step list survives view remounts and records completion only as execution advances. Recognition weights, VAD and final verification have separate operations; downloads use pinned byte counts, while other work reports elapsed time. Preparation status and progress stay in bundle details through `plugins.bundle.config`, keyed by package name. The generic `plugins.bundle.activation` slot carries explicit enablement intent and detail navigation; the voice occupant waits for Host cache inspection and offers installation guidance only for unprepared local models. Enabling an already prepared provider does not prompt or download. A single composer activity Slot expands the voice controls without replacing the draft editor or context-usage control. Shared primitives provide buttons, disclosure and status markers; only measured waveform drawing belongs to the voice UI. Recordings start on click and transcribe on Stop, with feedback inside the activity. Persistent preferences remain in plugin settings. ## Alternatives considered diff --git a/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.zh.md b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.zh.md index 9a21aaf1a6..15ffc46d8e 100644 --- a/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.zh.md +++ b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.zh.md @@ -18,7 +18,7 @@ Provider id 选择确切的注册。解析捕获实例,注销先拒绝新任 SenseVoice ONNX 与 Silero VAD 在一个托管 CPU 子进程中运行。原生 sherpa-onnx 包包含 ONNX Runtime;显式准备只下载并校验模型;后续调用复用预热模型,直到配置的空闲截止时间。每次启用都会校验缓存资源,完整缓存恢复为待唤醒状态,不启动工作进程。就绪状态因此取决于磁盘内容,而非 Provider 实例的存活时间。检查复用准备任务的生命周期,取消与释放都会等待检查结束。禁用 Provider 等待其任务和进程退出,同时保留可复用的磁盘资源。浏览器原生录音与重采样避免增加音频转换可执行文件。带认证的私有环回协议传输完整 PCM 录音,取消时可以终止工作进程,而不与另一请求共享长时间推理。 -Host 独立于发起视图拥有准备任务。有序步骤列表在视图重新挂载后保留,仅随着执行推进记录完成。识别模型、VAD 与最终校验分别准备;下载使用固定字节数,其他工作报告等待时间。包状态使用 `plugins.bundle.status`,详情使用上游 `plugins.bundle.config`,均以包名为 key。通用 `plugins.bundle.activation` Slot 传递显式启用意图和详情导航;语音贡献等待 Host 缓存检查,只为尚未准备的本地模型提供安装引导。启用已准备的 Provider 不弹窗也不下载。单个输入框活动 Slot 展开语音控件,同时保留草稿编辑器和独立的上下文用量控件。按钮、折叠与状态标记复用共享原语,语音 UI 只负责实测波形绘制。点击开始录音,停止后转写,反馈留在活动栏内。持久化偏好仍在插件设置中。 +Host 独立于发起视图拥有准备任务。有序步骤列表在视图重新挂载后保留,仅随着执行推进记录完成。识别模型、VAD 与最终校验分别准备;下载使用固定字节数,其他工作报告等待时间。准备状态和进度通过以包名为 key 的 `plugins.bundle.config` 显示在详情页。通用 `plugins.bundle.activation` Slot 传递显式启用意图和详情导航;语音贡献等待 Host 缓存检查,只为尚未准备的本地模型提供安装引导。启用已准备的 Provider 不弹窗也不下载。单个输入框活动 Slot 展开语音控件,同时保留草稿编辑器和独立的上下文用量控件。按钮、折叠与状态标记复用共享原语,语音 UI 只负责实测波形绘制。点击开始录音,停止后转写,反馈留在活动栏内。持久化偏好仍在插件设置中。 ## 考虑的替代方案 diff --git a/apps/web/tests/voice-setup.e2e.ts b/apps/web/tests/voice-setup.e2e.ts index d3767fa7d3..93eb86b5c5 100644 --- a/apps/web/tests/voice-setup.e2e.ts +++ b/apps/web/tests/voice-setup.e2e.ts @@ -31,6 +31,8 @@ it('guides a newly enabled voice plugin to installation and lets the user postpo await dialog.getByRole('button', { name: 'Later', exact: true }).click() await dialog.waitFor({ state: 'hidden' }) expect(await toggle.getAttribute('aria-checked')).toBe('true') + const card = page.locator('[data-plugin-package="@deepseek-ai/dsh-experimental-voice-input-bundle"]') + expect(await card.getByRole('status').count()).toBe(0) await toggle.click() await expect.poll(() => toggle.getAttribute('aria-checked')).toBe('false') await toggle.click() diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json index 880d27e739..1f7b31cc86 100644 --- a/apps/web/tsconfig.json +++ b/apps/web/tsconfig.json @@ -26,6 +26,7 @@ "tests/diff-context.e2e.ts", "tests/scaffold.ts", "tests/voice-input.e2e.ts", + "tests/voice-setup.e2e.ts", "tests/auto-review-fixture.ts", "tests/scaffold-generation.spec.ts", "tests/scaffold-hermetic.e2e.ts", diff --git a/docs/subsystems/slots.i18n.yaml b/docs/subsystems/slots.i18n.yaml index 8c608cc994..00b9312824 100644 --- a/docs/subsystems/slots.i18n.yaml +++ b/docs/subsystems/slots.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/subsystems/slots.md -slots.md: eb3fed3df595b7cd0b777109ed67979d14d579c5 -slots.zh.md: a02d292b2c6cd268a768e0b3e9254f2e0f36a1e4 +slots.md: e4cc9ac9977d680fdd7235b01089eb6b0ba31caf +slots.zh.md: 00f1772010d878083ca354ced1ef808454bea094 diff --git a/docs/subsystems/slots.md b/docs/subsystems/slots.md index eb3fed3df5..e4cc9ac997 100644 --- a/docs/subsystems/slots.md +++ b/docs/subsystems/slots.md @@ -6,7 +6,7 @@ Slots are the Web Client's typed React composition system. [`dsh-client-ui-slots This page documents slot ownership, component inputs, extension APIs, and the shipped hierarchy. The surrounding boot, Remote, Client model, and Conversation paths are in [Web Client architecture](web-client.md). -`plugins.bundle.status` is keyed by npm package name and renders short status before the bundle switch; `plugins.bundle.config` supplies bundle detail configuration. `plugins.bundle.activation` renders optional guidance after user-requested enablement, with owner callbacks to dismiss or open that bundle’s details. `conversation.input.activity` supplies one action between the model selector and Send, with toolbar expansion released on unmount. +`plugins.bundle.config` supplies bundle detail configuration, keyed by npm package name. `plugins.bundle.activation` renders optional guidance after user-requested enablement, with owner callbacks to dismiss or open that bundle’s details. `conversation.input.activity` supplies one action between the model selector and Send, with toolbar expansion released on unmount. ## Declaration and lifecycle diff --git a/docs/subsystems/slots.zh.md b/docs/subsystems/slots.zh.md index a02d292b2c..00f1772010 100644 --- a/docs/subsystems/slots.zh.md +++ b/docs/subsystems/slots.zh.md @@ -6,7 +6,7 @@ Slots 是 Web Client 的类型化 React 组合系统。[`dsh-client-ui-slots`](. 本文记录 slot 的所有权、组件输入、扩展 API 与当前层级。外围的启动、Remote、Client model 与 Conversation 数据通路见 [Web Client 架构](web-client.zh.md)。 -`plugins.bundle.status` 以 npm 包名为 key,在 Bundle 卡片开关前显示简短状态;`plugins.bundle.config` 提供 Bundle 详情配置。`plugins.bundle.activation` 在用户显式启用后渲染可选引导,由拥有者提供关闭引导和打开该 Bundle 详情的回调。`conversation.input.activity` 提供模型选择器与发送按钮之间的单个活动控件,可展开工具栏并在卸载时释放展开状态。 +`plugins.bundle.config` 以 npm 包名为 key,提供 Bundle 详情配置。`plugins.bundle.activation` 在用户显式启用后渲染可选引导,由拥有者提供关闭引导和打开该 Bundle 详情的回调。`conversation.input.activity` 提供模型选择器与发送按钮之间的单个活动控件,可展开工具栏并在卸载时释放展开状态。 ## 声明与生命周期 diff --git a/docs/subsystems/voice-input.i18n.yaml b/docs/subsystems/voice-input.i18n.yaml index bde639522c..7e94efdd15 100644 --- a/docs/subsystems/voice-input.i18n.yaml +++ b/docs/subsystems/voice-input.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/subsystems/voice-input.md -voice-input.md: d72bf01a372413fbc33a93a8a3a77f5152a3807b -voice-input.zh.md: c45275516166820f9dfac2d7d96c3a43b70d72d8 +voice-input.md: c1507d27705131243890d7441c8a945ca63ba419 +voice-input.zh.md: 9300f4cf110a748ec54bf809526fac2f2ea35eaa diff --git a/docs/subsystems/voice-input.md b/docs/subsystems/voice-input.md index d72bf01a37..c1507d2770 100644 --- a/docs/subsystems/voice-input.md +++ b/docs/subsystems/voice-input.md @@ -20,7 +20,7 @@ The input facade captures a revision-bearing selection before recording. `InputA The Host provider owns one preparation task across page and Session changes. The Client shares one `follow()` subscription across the composer, plugin status, details. Optional `SpeechSetupEstimate` metadata supplies provider-specific planning hints, separate from measured progress. `SpeechPreparationStepKind` identifies ordered resource operations; `SpeechPreparationStep` records each status and start time. The complete `SpeechPreparationState` retains these steps after cancellation or failure. The collapsed UI shows the current operation; expanding lists all steps, with byte progress or elapsed time only for the running step. Closing an observer never cancels preparation. Verified files survive retries and idle worker reclamation. -The microphone occupies `conversation.input.activity`, between the model selector and Send. Clicking starts capture and expands the toolbar; Stop transcribes and inserts into the draft. The activity preserves the editor and submit action, owns local feedback, and releases expansion on unmount. Cancel, Escape or hiding the page discards capture. Waveform history displays measured microphone amplitude. Bundle details store recognition preferences, and `plugins.bundle.status` displays short readiness beside the enable switch. +The microphone occupies `conversation.input.activity`, between the model selector and Send. Clicking starts capture and expands the toolbar; Stop transcribes and inserts into the draft. The activity preserves the editor and submit action, owns local feedback, and releases expansion on unmount. Cancel, Escape or hiding the page discards capture. Waveform history displays measured microphone amplitude. Bundle details contain recognition preferences, preparation state and progress. Explicit enablement uses `plugins.bundle.activation` to guide users with missing models to setup; the list shows only the bundle description and switch. ## Design rationale diff --git a/docs/subsystems/voice-input.zh.md b/docs/subsystems/voice-input.zh.md index c452755161..9300f4cf11 100644 --- a/docs/subsystems/voice-input.zh.md +++ b/docs/subsystems/voice-input.zh.md @@ -20,7 +20,7 @@ Host Provider 在页面和 Session 变化期间拥有同一个准备任务。Client 在输入框、插件状态、详情之间共享一份 `follow()` 订阅。可选 `SpeechSetupEstimate` 元数据提供各 Provider 的资源预期,与实测进度分开。`SpeechPreparationStepKind` 标识有序资源操作;`SpeechPreparationStep` 记录各步骤状态与开始时间。完整 `SpeechPreparationState` 在取消或失败后保留这些步骤。折叠 UI 显示当前操作,展开后列出所有步骤,仅进行中步骤显示字节进度或等待时间。关闭观察者不会取消准备。已校验文件在重试及空闲进程回收后继续复用。 -麦克风占用模型选择器与发送按钮之间的 `conversation.input.activity`。点击开始录音并展开工具栏;停止后转写并插入草稿。活动栏保留编辑器与提交按钮,拥有局部反馈,并在卸载时释放展开状态。取消、Escape 或隐藏页面会丢弃录音。波形历史展示实测麦克风音量。Bundle 详情保存识别偏好,`plugins.bundle.status` 在启用开关旁显示简短准备状态。 +麦克风占用模型选择器与发送按钮之间的 `conversation.input.activity`。点击开始录音并展开工具栏;停止后转写并插入草稿。活动栏保留编辑器与提交按钮,拥有局部反馈,并在卸载时释放展开状态。取消、Escape 或隐藏页面会丢弃录音。波形历史展示实测麦克风音量。Bundle 详情包含识别偏好、准备状态和进度。显式启用时通过 `plugins.bundle.activation` 引导缺少模型的用户前往安装;列表只显示 Bundle 描述和开关。 ## 设计依据 diff --git a/packages/client/ui-plugin-manager/README.i18n.yaml b/packages/client/ui-plugin-manager/README.i18n.yaml index 379cfb3c3b..b5171caf3e 100644 --- a/packages/client/ui-plugin-manager/README.i18n.yaml +++ b/packages/client/ui-plugin-manager/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/client/ui-plugin-manager/README.md -README.md: 1be8aaf8aed93dcfef527f373300f5cfb49c9bb6 -README.zh.md: eac6512b91916302e0d265045365d0ad8e890c82 +README.md: 34eca80392ad66f0e1a170cc5455f81d4650f5d3 +README.zh.md: 14d1af9b240c23b780c1afd58fbdc2abda43f5b9 diff --git a/packages/client/ui-plugin-manager/README.md b/packages/client/ui-plugin-manager/README.md index 1be8aaf8ae..34eca80392 100644 --- a/packages/client/ui-plugin-manager/README.md +++ b/packages/client/ui-plugin-manager/README.md @@ -79,7 +79,7 @@ The browser plugin registers the `plugins` sidebar entry and its `main` panel th The page's `main` registration declares `plugins.item`, `plugins.bundle.config`, and `plugins.row.config` as its children, so the slots exist while the page does and a registrant's `ctx.slots.inject` waits for them. `configLedgerSource` projects the three ledgers into one observable — the official items in ledger order with their labels resolved in the active locale, and the bundle and row keys — cached until a ledger or the locale moves; the page binds it as `useConfigLedger` beside the store and never names a configurable plugin itself. Which page is open is page-local state: the cards, a bundle, an official plugin, or a row of a bundle. A registration lives with the browser half that made it. `dsh-client-modules` attaches a package's browser half to the Loader row whose specifier is the bare package name, so every page a bundle registers, for itself or for any of its rows, goes away when that row is switched off; a sub-plugin whose page must outlive the other rows ships as its own package. -`plugins.bundle.status` is keyed by npm package name and renders short status before the bundle switch; `plugins.bundle.config` supplies bundle detail configuration. `plugins.bundle.activation` offers bundle-owned guidance after explicit enablement from the list, with callbacks to dismiss it or open the bundle details. It does not appear merely because an enabled bundle was listed. +`plugins.bundle.config` supplies bundle detail configuration, keyed by npm package name. `plugins.bundle.activation` offers bundle-owned guidance after explicit enablement from the list, with callbacks to dismiss it or open the bundle details. It does not appear merely because an enabled bundle was listed. diff --git a/packages/client/ui-plugin-manager/README.zh.md b/packages/client/ui-plugin-manager/README.zh.md index eac6512b91..14d1af9b24 100644 --- a/packages/client/ui-plugin-manager/README.zh.md +++ b/packages/client/ui-plugin-manager/README.zh.md @@ -79,7 +79,7 @@ ctx.slots.inject('plugins.row.config', () => ctx.slots.register({ 页面的 `main` 注册把 `plugins.item`、`plugins.bundle.config` 与 `plugins.row.config` 声明为子 slot,因此它们与页面同生,注册方的 `ctx.slots.inject` 会等到它们出现。`configLedgerSource` 把三份账本投影成一个可观察对象——按账本顺序排列、标签按当前语言解析的官方条目,以及组合包与行的键——在账本或语言变化前保持缓存;页面把它作为 `useConfigLedger` 绑在 store 旁边,自身从不点名任何可配置插件。打开的是哪一页是页面本地状态:卡片、某个组合包、某个官方插件,或组合包的某一行。注册与做出它的浏览器半侧同生共死。`dsh-client-modules` 只把一个包的浏览器半侧挂在说明符恰为包名的那一行 Loader 行上,所以组合包为自己或任一行注册的页面,都会在那一行被关闭时一起消失;需要在其他行关闭时仍保留页面的子插件,应作为独立的包发布。 -`plugins.bundle.status` 以 npm 包名为 key,在 Bundle 卡片开关前显示简短状态;`plugins.bundle.config` 提供 Bundle 详情配置。`plugins.bundle.activation` 在用户从列表显式启用后提供 Bundle 自有引导,并传入关闭引导和打开详情的回调。仅列出已启用的 Bundle 不会触发引导。 +`plugins.bundle.config` 以 npm 包名为 key,提供 Bundle 详情配置。`plugins.bundle.activation` 在用户从列表显式启用后提供 Bundle 自有引导,并传入关闭引导和打开详情的回调。仅列出已启用的 Bundle 不会触发引导。 diff --git a/packages/client/ui-plugin-manager/src/client/PluginManagerPage.tsx b/packages/client/ui-plugin-manager/src/client/PluginManagerPage.tsx index 3a94095741..d507993bcf 100644 --- a/packages/client/ui-plugin-manager/src/client/PluginManagerPage.tsx +++ b/packages/client/ui-plugin-manager/src/client/PluginManagerPage.tsx @@ -33,7 +33,7 @@ import css from './PluginManagerPage.module.css' export type PluginManagerPageProps = PropsRuntime<'main'> & PropsLocale<'pluginManager'> - & PropsRenderSlots<'plugins.item' | 'plugins.bundle.config' | 'plugins.row.config' | 'plugins.bundle.status' | 'plugins.bundle.activation'> + & PropsRenderSlots<'plugins.item' | 'plugins.bundle.config' | 'plugins.row.config' | 'plugins.bundle.activation'> & InjectFace /** The page's slot renderer, narrowed to the configuration slots. */ @@ -282,8 +282,7 @@ function DetailTop({ crumbLabel, crumbText, onBack, icon, actions }: { } /** One package as a card that opens its page: its name, its one-liner, its tags, and its bundle switch. */ -function PackageCard({ pkg, t, busy, highlighted, onOpen, onSetEnabled, statusContent }: { - readonly statusContent: ReactNode +function PackageCard({ pkg, t, busy, highlighted, onOpen, onSetEnabled }: { readonly pkg: PackageView readonly t: Translate readonly busy: boolean @@ -311,7 +310,7 @@ function PackageCard({ pkg, t, busy, highlighted, onOpen, onSetEnabled, statusCo )} description={description} - end={<>{statusContent}} + end={} /> ) @@ -874,7 +873,6 @@ export function PluginManagerPage(props: PluginManagerPageProps): ReactNode { t={t} busy={state.busy.includes(pkg.name)} highlighted={state.highlight === pkg.name} - statusContent={renderSlot('plugins.bundle.status', { packageName: pkg.name, enabled: pkg.enabled }, { entryKey: pkg.name })} onOpen={() => { setActivation(null); setView({ kind: 'package', name: pkg.name }) }} onSetEnabled={(enabled) => { setActivation(enabled ? pkg.name : null); props.setEnabled(pkg.name, enabled) }} /> diff --git a/packages/client/ui-plugin-manager/src/client/index.ts b/packages/client/ui-plugin-manager/src/client/index.ts index 102a73a3d7..f11d187fa0 100644 --- a/packages/client/ui-plugin-manager/src/client/index.ts +++ b/packages/client/ui-plugin-manager/src/client/index.ts @@ -31,7 +31,7 @@ export type { PluginManagerPageProps } from './PluginManagerPage.tsx' export type { ConfigLedger, OfficialItem } from './config-ledger.ts' export type { PluginManagerFace } from './manager-store.ts' export type { PluginManagerLocaleKey } from './locales.ts' -export type { PluginConfigViewProps, PluginPackageOwnerProps, PluginActivationOwnerProps } from './slot-contract.ts' +export type { PluginConfigViewProps, PluginActivationOwnerProps } from './slot-contract.ts' declare module '@deepseek-ai/dsh-client-ui-slots' { interface LocaleNamespaceMap { @@ -87,7 +87,6 @@ export function apply(ctx: ClientContext): void { inject: () => controller.inject(configLedger), children: { 'plugins.item': { kind: 'list', scope: 'root' }, - 'plugins.bundle.status': { kind: 'keyed', scope: 'root' }, 'plugins.bundle.activation': { kind: 'keyed', scope: 'root' }, 'plugins.bundle.config': { kind: 'keyed', scope: 'root' }, 'plugins.row.config': { kind: 'keyed', scope: 'root' }, diff --git a/packages/client/ui-plugin-manager/src/client/slot-contract.ts b/packages/client/ui-plugin-manager/src/client/slot-contract.ts index 9fd4fec14b..5548602791 100644 --- a/packages/client/ui-plugin-manager/src/client/slot-contract.ts +++ b/packages/client/ui-plugin-manager/src/client/slot-contract.ts @@ -18,12 +18,6 @@ export interface PluginConfigViewProps { readonly view: 'summary' | 'page' } -/** Package identity and activation state supplied to its compact status view. */ -export interface PluginPackageOwnerProps { - readonly packageName: string - readonly enabled: boolean -} - /** One user-requested bundle activation and navigation to its configuration page. */ export interface PluginActivationOwnerProps { readonly packageName: string @@ -35,8 +29,6 @@ export interface PluginActivationOwnerProps { declare module '@deepseek-ai/dsh-client-ui-slots' { interface SlotMap { - /** Short readiness beside the bundle switch, keyed by npm package name. */ - 'plugins.bundle.status': { kind: 'keyed'; scope: 'root'; owner: PluginPackageOwnerProps } /** Optional guidance after the user enables a bundle from the list, keyed by npm package name. */ 'plugins.bundle.activation': { kind: 'keyed'; scope: 'root'; owner: PluginActivationOwnerProps } /** diff --git a/packages/client/ui-plugin-manager/tests/components.client.spec.tsx b/packages/client/ui-plugin-manager/tests/components.client.spec.tsx index 12f1019827..246f942839 100644 --- a/packages/client/ui-plugin-manager/tests/components.client.spec.tsx +++ b/packages/client/ui-plugin-manager/tests/components.client.spec.tsx @@ -703,16 +703,6 @@ describe('PluginManagerPage', () => { }) }) -it('places bundle readiness immediately before its enable switch', () => { - const name = '@deepseek-ai/dsh-experimental-voice-input-bundle' - renderTab({ packages: [pkg({ name })] }, {}, { - [`plugins.bundle.status:${name}`]: () => Ready for dictation, - }) - const status = screen.getByText('Ready for dictation'), toggle = screen.getByRole('switch') - expect(status.parentElement).toBe(toggle.parentElement) - expect(status.compareDocumentPosition(toggle) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy() -}) - it('offers bundle-owned guidance only after explicit enablement and navigates to its detail page', () => { const name = 'dsh-better-sidebar' const { set, actions } = renderTab({ packages: [pkg({ enabled: false })] }, { bundles: new Set([name]) }, { diff --git a/packages/experimental/client-ui-voice-input/README.i18n.yaml b/packages/experimental/client-ui-voice-input/README.i18n.yaml index ff5188a628..2666ead465 100644 --- a/packages/experimental/client-ui-voice-input/README.i18n.yaml +++ b/packages/experimental/client-ui-voice-input/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/experimental/client-ui-voice-input/README.md -README.md: f6a7c2ba0e8567205ddd093b85915a487c3da898 -README.zh.md: 42657a6f0d73d61caefb12c83e7d73d7408d52cb +README.md: 539de47b0e1abe400b359f2ea4c2d55ede7554c0 +README.zh.md: 1c15160f333838de61ab25c64817e1a63e7cf5ce diff --git a/packages/experimental/client-ui-voice-input/README.md b/packages/experimental/client-ui-voice-input/README.md index f6a7c2ba0e..539de47b0e 100644 --- a/packages/experimental/client-ui-voice-input/README.md +++ b/packages/experimental/client-ui-voice-input/README.md @@ -35,7 +35,7 @@ Enable the [voice input bundle](../voice-input-bundle/README.md) from Plugins. A
Maintainer details — click to expand -Native MediaRecorder captures audio and Web Audio converts it to the Host PCM format after flushing the final recorder chunks. The original editor selection carries a draft revision: changed drafts retain the transcript for explicit insertion or discard beside the microphone. Session changes, hiding the page during capture and plugin disposal invalidate late results and release tracks. One plugin-owned readiness subscription serves the composer, package cards, details. These views use the existing Slot lifecycle and own no preparation tasks. No runtime invariant companion is published because readiness comes from one Host subscription and recording state belongs to one capture operation. +Native MediaRecorder captures audio and Web Audio converts it to the Host PCM format after flushing the final recorder chunks. The original editor selection carries a draft revision: changed drafts retain the transcript for explicit insertion or discard beside the microphone. Session changes, hiding the page during capture and plugin disposal invalidate late results and release tracks. One plugin-owned readiness subscription serves the composer, installation prompt and details. These views use the existing Slot lifecycle and own no preparation tasks. No runtime invariant companion is published because readiness comes from one Host subscription and recording state belongs to one capture operation.
diff --git a/packages/experimental/client-ui-voice-input/README.zh.md b/packages/experimental/client-ui-voice-input/README.zh.md index 42657a6f0d..1c15160f33 100644 --- a/packages/experimental/client-ui-voice-input/README.zh.md +++ b/packages/experimental/client-ui-voice-input/README.zh.md @@ -35,7 +35,7 @@ kind: "package-reference"
维护者信息 — 点击展开 -原生 MediaRecorder 捕获音频,等待最后一块数据后由 Web Audio 转换为 Host 的 PCM 格式。原编辑器选区携带草稿版本:草稿变化时,转写文字保留在麦克风旁,供显式插入或丢弃。切换 Session、录音期间隐藏页面和释放插件会使迟到结果失效并释放轨道。插件拥有的一份准备状态订阅服务于输入框、包卡片、详情。这些视图使用既有 Slot 生命周期,不拥有准备任务。不发布运行时不变量伴随模块,因为准备状态来自一份 Host 订阅,录音状态属于一个捕获操作。 +原生 MediaRecorder 捕获音频,等待最后一块数据后由 Web Audio 转换为 Host 的 PCM 格式。原编辑器选区携带草稿版本:草稿变化时,转写文字保留在麦克风旁,供显式插入或丢弃。切换 Session、录音期间隐藏页面和释放插件会使迟到结果失效并释放轨道。插件拥有的一份准备状态订阅服务于输入框、安装引导和详情。这些视图使用既有 Slot 生命周期,不拥有准备任务。不发布运行时不变量伴随模块,因为准备状态来自一份 Host 订阅,录音状态属于一个捕获操作。
diff --git a/packages/experimental/client-ui-voice-input/src/client/PreparationCard.tsx b/packages/experimental/client-ui-voice-input/src/client/PreparationCard.tsx index d8639c6e9c..28d74ad143 100644 --- a/packages/experimental/client-ui-voice-input/src/client/PreparationCard.tsx +++ b/packages/experimental/client-ui-voice-input/src/client/PreparationCard.tsx @@ -99,17 +99,6 @@ export function PreparationCard({ provider, connected, prepare, cancelPreparatio } -/** Short selected-provider readiness for the plugin card's switch-adjacent status Slot. */ -export function VoiceStatus({ useSpeechReadiness, t }: InjectFace & PropsLocale) { - const readiness = useSpeechReadiness(value => value) - const provider = readiness.catalog?.providers.find(item => item.id === readiness.catalog?.selection.providerId) - const state = provider?.preparation - const text = !readiness.connected || !state ? t('short.connecting') : t(`short.${state.phase}`) - return - {text} - -} - /** Recognition preferences and preparation cards shared by plugin details and Settings. */ export function VoicePreparation({ useSpeechReadiness, ...props }: InjectFace & PropsLocale) { const readiness = useSpeechReadiness(value => value), catalog = readiness.catalog diff --git a/packages/experimental/client-ui-voice-input/src/client/VoiceInput.module.css b/packages/experimental/client-ui-voice-input/src/client/VoiceInput.module.css index 4b4c9e3403..4881303583 100644 --- a/packages/experimental/client-ui-voice-input/src/client/VoiceInput.module.css +++ b/packages/experimental/client-ui-voice-input/src/client/VoiceInput.module.css @@ -21,7 +21,6 @@ .stepBody { flex: 1; min-width: 0; } .stepProgress { display: flex; flex-direction: column; gap: 6px; margin-top: 6px; } .stepProgress small { font-size: 11px; color: var(--dsw-alias-label-tertiary); overflow-wrap: anywhere; } -.packageStatus { display: inline-flex; align-items: center; gap: 6px; font-size: 12px; color: var(--dsw-alias-label-secondary); white-space: nowrap; } .preferences { display: grid; gap: 12px; font-size: 13px; margin: 18px 0; } .preferences label { display: grid; grid-template-columns: 100px minmax(120px, 280px); align-items: center; gap: 12px; } .preferences select { border: 1px solid var(--dsw-alias-border-l1); border-radius: 6px; padding: 6px 10px; background: transparent; color: inherit; font: inherit; } diff --git a/packages/experimental/client-ui-voice-input/src/client/locales.ts b/packages/experimental/client-ui-voice-input/src/client/locales.ts index 918652da79..9b9c33cb99 100644 --- a/packages/experimental/client-ui-voice-input/src/client/locales.ts +++ b/packages/experimental/client-ui-voice-input/src/client/locales.ts @@ -28,18 +28,8 @@ export const zh = { 'stepStatus.complete': '已完成', 'stepStatus.failed': '失败', 'stepStatus.cancelled': '已取消', - 'short.connecting': '连接中', - 'short.unprepared': '需要安装', - 'short.checking': '检查中', 'short.downloading': '下载中', - 'short.installing': '安装中', - 'short.loading': '加载中', - 'short.waking': '唤醒中', - 'short.cancelling': '取消中', - 'short.cancelled': '需要安装', 'short.failed': '准备失败', - 'short.ready': '已就绪', - 'short.standby': '已就绪', retryRecording: '重新录音', preparationSteps: '准备步骤', 'prepare': '下载并准备', @@ -109,18 +99,8 @@ export const en: Record = { 'stepStatus.complete': 'Completed', 'stepStatus.failed': 'Failed', 'stepStatus.cancelled': 'Cancelled', - 'short.connecting': 'Connecting', - 'short.unprepared': 'Setup required', - 'short.checking': 'Checking', 'short.downloading': 'Downloading', - 'short.installing': 'Installing', - 'short.loading': 'Loading', - 'short.waking': 'Waking', - 'short.cancelling': 'Cancelling', - 'short.cancelled': 'Setup required', 'short.failed': 'Setup failed', - 'short.ready': 'Ready', - 'short.standby': 'Ready', retryRecording: 'Record again', preparationSteps: 'Preparation steps', 'prepare': 'Download and prepare', diff --git a/packages/experimental/client-ui-voice-input/src/client/mount.ts b/packages/experimental/client-ui-voice-input/src/client/mount.ts index 3af8f6d451..cff12b511b 100644 --- a/packages/experimental/client-ui-voice-input/src/client/mount.ts +++ b/packages/experimental/client-ui-voice-input/src/client/mount.ts @@ -11,7 +11,7 @@ import { Recording } from './audio.ts' import { en, NS, zh } from './locales.ts' import type {} from '@deepseek-ai/dsh-client-ui-plugin-manager/client' import { observeReadiness } from './readiness.ts' -import { VoicePreparation, VoiceStatus } from './PreparationCard.tsx' +import { VoicePreparation } from './PreparationCard.tsx' import { VoiceSetupPrompt } from './VoiceSetupPrompt.tsx' export const inject = ['remote', 'slots', 'locale'] @@ -39,9 +39,6 @@ function registerUi(ctx: Context): void { ctx.slots.inject('conversation.input.activity', () => ctx.slots.register({ name: 'conversation.input.activity', locale: NS, inject: () => actions, }, VoiceInput)) - ctx.slots.inject('plugins.bundle.status', () => ctx.slots.register({ name: 'plugins.bundle.status', - key: '@deepseek-ai/dsh-experimental-voice-input-bundle', locale: NS, inject: () => actions, - }, VoiceStatus)) ctx.slots.inject('plugins.bundle.config', () => ctx.slots.register({ name: 'plugins.bundle.config', key: '@deepseek-ai/dsh-experimental-voice-input-bundle', locale: NS, inject: () => actions, }, VoicePreparation)) diff --git a/packages/experimental/client-ui-voice-input/tests/mount.client.spec.ts b/packages/experimental/client-ui-voice-input/tests/mount.client.spec.ts index a11f6d7eac..2f5833d882 100644 --- a/packages/experimental/client-ui-voice-input/tests/mount.client.spec.ts +++ b/packages/experimental/client-ui-voice-input/tests/mount.client.spec.ts @@ -30,7 +30,6 @@ async function fixture(fail = false) { await ctx.plugin(SlotRegistry) ctx.slots.register({ name: 'root', children: { 'conversation.input.activity': { kind: 'single', scope: 'session' }, - 'plugins.bundle.status': { kind: 'keyed', scope: 'root' }, 'plugins.bundle.activation': { kind: 'keyed', scope: 'root' }, 'plugins.bundle.config': { kind: 'keyed', scope: 'root' }, } } as never, @@ -55,7 +54,7 @@ it('withdraws its Remote, localized slot and microphone captures on disposal', a await actions.configure({ language: 'zh' }) await actions.prepare('local' as SpeechProviderId) await actions.cancelPreparation('local' as SpeechProviderId) - for (const slot of ['plugins.bundle.status', 'plugins.bundle.config', 'plugins.bundle.activation'] as const) { + for (const slot of ['plugins.bundle.config', 'plugins.bundle.activation'] as const) { const item = b.ctx.slots.entries(slot)[0]! expect(item.locale).toBe('voice-input') const injected = (item.inject as unknown as () => VoiceInputInjected & { compact?: boolean })() diff --git a/packages/experimental/client-ui-voice-input/tests/preparation.client.spec.tsx b/packages/experimental/client-ui-voice-input/tests/preparation.client.spec.tsx index 8315a9a4c7..ba8bb03680 100644 --- a/packages/experimental/client-ui-voice-input/tests/preparation.client.spec.tsx +++ b/packages/experimental/client-ui-voice-input/tests/preparation.client.spec.tsx @@ -6,7 +6,7 @@ import { createSnapshotStore } from '@deepseek-ai/dsh-client-store' import { zh as commonZh } from '@deepseek-ai/dsh-client-locale/src/locales/zh.ts' import type { SpeechPreparationState, SpeechPreparationStep, SpeechProviderId } from '@deepseek-ai/dsh-experimental-speech-to-text/types' import { afterEach, expect, it, vi } from 'vitest' -import { PreparationCard, VoiceStatus, VoicePreparation } from '../src/client/PreparationCard.tsx' +import { PreparationCard, VoicePreparation } from '../src/client/PreparationCard.tsx' import type { VoiceInputProps } from '../src/client/VoiceInput.tsx' import type { SpeechReadiness } from '../src/client/readiness.ts' import { zh } from '../src/client/locales.ts' @@ -84,7 +84,7 @@ it('shows waking without an explicit preparation cancellation control', () => { expect(screen.getByText('已等待 5 秒')).toBeTruthy() expect(screen.queryByRole('button', { name: zh.cancelPrepare })).toBeNull() }) -it('persists settings and renders a short selected-provider status independently of the detail card', async () => { +it('persists settings and reports disconnection in the detail card', async () => { const store = createSnapshotStore({ connected: true, error: null, catalog: { selection: { providerId: id, language: 'auto' }, maxAudioBytes: 100, maxDurationSeconds: 120, providers: [{ id, name: 'SenseVoiceSmall', location: 'host-local', preparation: { phase: 'ready' } }, @@ -105,11 +105,7 @@ it('persists settings and renders a short selected-provider status independently configure.mockRejectedValueOnce('settings are locked') fireEvent.change(screen.getByLabelText(zh.language), { target: { value: 'ja' } }) await screen.findByText('语音识别失败:settings are locked') - view.rerender() - expect(screen.getByRole('status').textContent).toBe(zh['short.ready']) - expect(screen.queryByRole('combobox')).toBeNull() act(() => { store.set({ catalog: null, connected: false, error: 'disconnected' }) }) - expect(screen.getByRole('status').textContent).toBe(zh['short.connecting']) view.rerender() expect(screen.getByText(zh.loading)).toBeTruthy() expect(screen.getByRole('alert').textContent).toContain('disconnected') diff --git a/packages/extensions/cordis-client-runner/src/client/slot-catalog.ts b/packages/extensions/cordis-client-runner/src/client/slot-catalog.ts index e5f8c3224b..8e2d4903f5 100644 --- a/packages/extensions/cordis-client-runner/src/client/slot-catalog.ts +++ b/packages/extensions/cordis-client-runner/src/client/slot-catalog.ts @@ -1618,7 +1618,7 @@ export const CLIENT_SLOT_API: readonly ClientSlotEntry[] = [ ], replaceRisk: 'shadows-shipped-ui', example: 'return {\n inject: [\'slots\'],\n apply(ctx) {\n ctx.slots.inject(\'plugins.bundle.activation\', () => ctx.slots.register(\n { name: \'plugins.bundle.activation\', key: \'\' },\n () => React.createElement(\'div\', null, \'hello\'),\n ))\n },\n}', - source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:41', + source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:33', }, { key: 'plugins.bundle.config', @@ -1656,45 +1656,7 @@ export const CLIENT_SLOT_API: readonly ClientSlotEntry[] = [ ], replaceRisk: 'shadows-shipped-ui', example: 'return {\n inject: [\'slots\'],\n apply(ctx) {\n ctx.slots.inject(\'plugins.bundle.config\', () => ctx.slots.register(\n { name: \'plugins.bundle.config\', key: \'\' },\n () => React.createElement(\'div\', null, \'hello\'),\n ))\n },\n}', - source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:57', - }, - { - key: 'plugins.bundle.status', - kind: 'keyed', - scope: 'root', - summary: 'Short readiness beside the bundle switch, keyed by npm package name.', - doc: 'Short readiness beside the bundle switch, keyed by npm package name.', - registerOptions: [ - { - name: 'key', - requirement: 'required', - type: 'string', - doc: 'Your cell key: the entry renders where the owner dispatches this exact key. Registering an already-occupied key replaces that occupant.', - }, - ], - ownerProps: [ - '/** Package identity and activation state supplied to its compact status view. */\nexport interface PluginPackageOwnerProps {\n readonly packageName: string\n readonly enabled: boolean\n}', - ], - ownerPropsReferences: [], - standardProps: [ - 'useResource: UseResource', - 'useWorkspaces: SnapshotSelectorHook', - 'usePanelInfo: UsePanelInfo', - 'useSessions: UseSessions', - 'useSessionStatus: UseSessionStatus', - 'useSessionRetainInfo: UseSessionRetainInfo', - 'useWorkspaces: SnapshotSelectorHook', - ], - keyDomain: 'open: any string the owner dispatches (no compile-time key set), already taken: @deepseek-ai/dsh-experimental-voice-input-bundle', - hookContext: '', - slotInject: '', - declaredBy: 'an entry in \'main\' (client-ui-plugin-manager), so it exists while that entry is mounted', - occupants: [ - 'experimental-client-ui-voice-input VoiceStatus key \'@deepseek-ai/dsh-experimental-voice-input-bundle\'', - ], - replaceRisk: 'shadows-shipped-ui', - example: 'return {\n inject: [\'slots\'],\n apply(ctx) {\n ctx.slots.inject(\'plugins.bundle.status\', () => ctx.slots.register(\n { name: \'plugins.bundle.status\', key: \'\' },\n () => React.createElement(\'div\', null, \'hello\'),\n ))\n },\n}', - source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:39', + source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:49', }, { key: 'plugins.item', @@ -1747,7 +1709,7 @@ export const CLIENT_SLOT_API: readonly ClientSlotEntry[] = [ ], replaceRisk: 'none', example: 'return {\n inject: [\'slots\'],\n apply(ctx) {\n ctx.slots.inject(\'plugins.item\', () => ctx.slots.register(\n { name: \'plugins.item\', id: \'my-entry\', order: 100, label: \'My entry\' },\n () => React.createElement(\'div\', null, \'hello\'),\n ))\n },\n}', - source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:51', + source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:43', }, { key: 'plugins.row.config', @@ -1783,7 +1745,7 @@ export const CLIENT_SLOT_API: readonly ClientSlotEntry[] = [ occupants: [], replaceRisk: 'none', example: 'return {\n inject: [\'slots\'],\n apply(ctx) {\n ctx.slots.inject(\'plugins.row.config\', () => ctx.slots.register(\n { name: \'plugins.row.config\', key: \'\' },\n () => React.createElement(\'div\', null, \'hello\'),\n ))\n },\n}', - source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:64', + source: 'packages/client/ui-plugin-manager/src/client/slot-contract.ts:56', }, { key: 'rightbar', diff --git a/tsconfig.host.json b/tsconfig.host.json index a434e5a361..f5e24d790b 100644 --- a/tsconfig.host.json +++ b/tsconfig.host.json @@ -13,6 +13,7 @@ "apps/web/tests/diff-context.e2e.ts", "apps/web/tests/scaffold.ts", "apps/web/tests/voice-input.e2e.ts", + "apps/web/tests/voice-setup.e2e.ts", "apps/web/tests/auto-review-fixture.ts", "apps/web/tests/auto-review-denial.e2e.ts", "apps/web/tests/scaffold-generation.spec.ts", From 018c0e515658d831877895ffd7c3338a12750252 Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Thu, 17 Sep 2026 13:57:51 +0800 Subject: [PATCH 04/12] fix(voice-input): align dictation toolbar controls --- ...026-09-16-experimental-voice-input.i18n.yaml | 4 ++-- .../2026-09-16-experimental-voice-input.md | 2 +- .../2026-09-16-experimental-voice-input.zh.md | 2 +- apps/web/tests/voice-input.e2e.ts | 17 +++++++++++++---- .../client/ui-conversation/README.i18n.yaml | 4 ++-- packages/client/ui-conversation/README.md | 2 +- packages/client/ui-conversation/README.zh.md | 2 +- .../src/client/skeleton/InputBar.module.css | 2 +- .../src/client/skeleton/InputBar.tsx | 2 +- .../tests/input-bar.client.spec.tsx | 14 +++++++++++--- .../src/client/VoiceInput.module.css | 3 ++- .../src/client/VoiceInput.tsx | 6 +++--- 12 files changed, 39 insertions(+), 21 deletions(-) diff --git a/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.i18n.yaml b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.i18n.yaml index 8e6b585de4..84bd998a51 100644 --- a/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.md -2026-09-16-experimental-voice-input.md: 8b646e917965175f099c37224261ca05c7c1adfb -2026-09-16-experimental-voice-input.zh.md: 15ffc46d8e964c0448e8bd1c7395a5ad464b2b2c +2026-09-16-experimental-voice-input.md: 60a871cc6ba7e604896ce4e339bcfbe45d174f9b +2026-09-16-experimental-voice-input.zh.md: 76d14511d1b014bb6d40cc4bf3e9e6ed19efc3f6 diff --git a/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.md b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.md index 8b646e9179..60a871cc6b 100644 --- a/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.md +++ b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.md @@ -18,7 +18,7 @@ Recordings and transcripts remain transient until the user submits ordinary text SenseVoice ONNX and Silero VAD run in one managed CPU subprocess. The native sherpa-onnx package includes ONNX Runtime; explicit preparation only downloads and verifies models; subsequent calls reuse the warm model until the configured idle deadline. Every activation verifies cached assets and restores complete caches to standby without starting the worker. Readiness therefore follows disk contents rather than the lifetime of a Provider instance. Inspection uses the same owned task as preparation, so cancellation and disposal join it. Disabling the provider joins its work and process exit while retaining reusable disk assets. Native browser recording and resampling avoid an additional audio conversion executable. The authenticated private loopback protocol carries complete PCM recordings, allowing cancellation to terminate the worker without sharing a long-running inference with another request. -The Host owns preparation independently of the view that started it. An ordered step list survives view remounts and records completion only as execution advances. Recognition weights, VAD and final verification have separate operations; downloads use pinned byte counts, while other work reports elapsed time. Preparation status and progress stay in bundle details through `plugins.bundle.config`, keyed by package name. The generic `plugins.bundle.activation` slot carries explicit enablement intent and detail navigation; the voice occupant waits for Host cache inspection and offers installation guidance only for unprepared local models. Enabling an already prepared provider does not prompt or download. A single composer activity Slot expands the voice controls without replacing the draft editor or context-usage control. Shared primitives provide buttons, disclosure and status markers; only measured waveform drawing belongs to the voice UI. Recordings start on click and transcribe on Stop, with feedback inside the activity. Persistent preferences remain in plugin settings. +The Host owns preparation independently of the view that started it. An ordered step list survives view remounts and records completion only as execution advances. Recognition weights, VAD and final verification have separate operations; downloads use pinned byte counts, while other work reports elapsed time. Preparation status and progress stay in bundle details through `plugins.bundle.config`, keyed by package name. The generic `plugins.bundle.activation` slot carries explicit enablement intent and detail navigation; the voice occupant waits for Host cache inspection and offers installation guidance only for unprepared local models. Enabling an already prepared provider does not prompt or download. A single composer activity Slot expands the voice controls while preserving the draft editor and submit action. The context meter precedes the idle microphone; recording and transcription hide the meter along with ordinary toolbar controls until the activity closes. Shared primitives provide buttons, disclosure and status markers; only measured waveform drawing belongs to the voice UI. Recordings start on click and transcribe on Stop, with feedback inside the activity. Persistent preferences remain in plugin settings. ## Alternatives considered diff --git a/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.zh.md b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.zh.md index 15ffc46d8e..76d14511d1 100644 --- a/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.zh.md +++ b/.agents/notes/implemented/architecture/2026-09-16-experimental-voice-input.zh.md @@ -18,7 +18,7 @@ Provider id 选择确切的注册。解析捕获实例,注销先拒绝新任 SenseVoice ONNX 与 Silero VAD 在一个托管 CPU 子进程中运行。原生 sherpa-onnx 包包含 ONNX Runtime;显式准备只下载并校验模型;后续调用复用预热模型,直到配置的空闲截止时间。每次启用都会校验缓存资源,完整缓存恢复为待唤醒状态,不启动工作进程。就绪状态因此取决于磁盘内容,而非 Provider 实例的存活时间。检查复用准备任务的生命周期,取消与释放都会等待检查结束。禁用 Provider 等待其任务和进程退出,同时保留可复用的磁盘资源。浏览器原生录音与重采样避免增加音频转换可执行文件。带认证的私有环回协议传输完整 PCM 录音,取消时可以终止工作进程,而不与另一请求共享长时间推理。 -Host 独立于发起视图拥有准备任务。有序步骤列表在视图重新挂载后保留,仅随着执行推进记录完成。识别模型、VAD 与最终校验分别准备;下载使用固定字节数,其他工作报告等待时间。准备状态和进度通过以包名为 key 的 `plugins.bundle.config` 显示在详情页。通用 `plugins.bundle.activation` Slot 传递显式启用意图和详情导航;语音贡献等待 Host 缓存检查,只为尚未准备的本地模型提供安装引导。启用已准备的 Provider 不弹窗也不下载。单个输入框活动 Slot 展开语音控件,同时保留草稿编辑器和独立的上下文用量控件。按钮、折叠与状态标记复用共享原语,语音 UI 只负责实测波形绘制。点击开始录音,停止后转写,反馈留在活动栏内。持久化偏好仍在插件设置中。 +Host 独立于发起视图拥有准备任务。有序步骤列表在视图重新挂载后保留,仅随着执行推进记录完成。识别模型、VAD 与最终校验分别准备;下载使用固定字节数,其他工作报告等待时间。准备状态和进度通过以包名为 key 的 `plugins.bundle.config` 显示在详情页。通用 `plugins.bundle.activation` Slot 传递显式启用意图和详情导航;语音贡献等待 Host 缓存检查,只为尚未准备的本地模型提供安装引导。启用已准备的 Provider 不弹窗也不下载。单个输入框活动 Slot 展开语音控件,同时保留草稿编辑器和提交按钮。空闲时上下文用量按钮位于麦克风之前;录音和转写期间,用量按钮与普通工具栏控件一起隐藏,关闭活动后恢复。按钮、折叠与状态标记复用共享原语,语音 UI 只负责实测波形绘制。点击开始录音,停止后转写,反馈留在活动栏内。持久化偏好仍在插件设置中。 ## 考虑的替代方案 diff --git a/apps/web/tests/voice-input.e2e.ts b/apps/web/tests/voice-input.e2e.ts index b1b1f665b8..34b0332908 100644 --- a/apps/web/tests/voice-input.e2e.ts +++ b/apps/web/tests/voice-input.e2e.ts @@ -88,20 +88,29 @@ it.skipIf(webSnapshotMode() === 'record')('records from cached standby and submi await page.setViewportSize({ width, height: 900 }) await expect.poll(async () => { const micRect = await mic.boundingBox(), meterRect = await meter.boundingBox() - return micRect !== null && meterRect !== null && micRect.x + micRect.width <= meterRect.x + return micRect !== null && meterRect !== null && meterRect.x + meterRect.width <= micRect.x }).toBe(true) await meter.click() await page.getByRole('dialog', { name: 'of context used', exact: true }).waitFor() await page.keyboard.press('Escape') } expect(await mic.getAttribute('title')).toBe('Dictate') + const controlBackground = await page.getByRole('button', { name: 'Add files or run commands', exact: true }) + .evaluate(element => getComputedStyle(element).backgroundColor) await mic.click() - await page.getByRole('button', { name: 'Stop and transcribe', exact: true }).waitFor() - expect(await meter.isVisible()).toBe(true) + const stop = page.getByRole('button', { name: 'Stop and transcribe', exact: true }) + await stop.waitFor() + await input.hover() + expect(await meter.isVisible()).toBe(false) + expect(await stop.evaluate(element => getComputedStyle(element).backgroundColor)).toBe(controlBackground) + expect(await page.getByRole('button', { name: 'Cancel', exact: true }) + .evaluate(element => getComputedStyle(element).backgroundColor)).toBe(controlBackground) + await page.keyboard.press('Escape') + await mic.waitFor() + await meter.waitFor() await meter.click() await page.getByRole('dialog', { name: 'of context used', exact: true }).waitFor() await page.keyboard.press('Escape') - await mic.waitFor() expect(messages).toBe(1) expect(tripwire.pageErrors).toEqual([]) }, 120_000) diff --git a/packages/client/ui-conversation/README.i18n.yaml b/packages/client/ui-conversation/README.i18n.yaml index 73e1bacf58..e192087d0f 100644 --- a/packages/client/ui-conversation/README.i18n.yaml +++ b/packages/client/ui-conversation/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/client/ui-conversation/README.md -README.md: 87a939fb42b4dc451fb66dbfe1a58262614e5b70 -README.zh.md: 12d5c955f3b8f86d7ea3ccaf3681ef95595486e9 +README.md: 548ffd3c49e6ac9bf9769c7cf252439387e9de5b +README.zh.md: becdb0da3b832458bfc9154cccf0efa265e46a65 diff --git a/packages/client/ui-conversation/README.md b/packages/client/ui-conversation/README.md index 87a939fb42..548ffd3c49 100644 --- a/packages/client/ui-conversation/README.md +++ b/packages/client/ui-conversation/README.md @@ -116,7 +116,7 @@ The selector must be a pure function of the owner currency. Its non-null return `InputActions.captureInsertion()` captures the draft selection and revision; `insertText(text, span)` inserts one undoable plain-text edit only while that revision is current and the editor permits editing. Asynchronous consumers retain rejected results for user action. -`conversation.input.activity` hosts one control between the model selector and Send. Its `onActiveChange` callback expands that control across the toolbar and hides ordinary accessory controls while preserving the editor, context-usage control and submit action. The context meter keeps its own space between the activity and Send. The occupant releases expansion on unmount and owns any activity-specific feedback. +`conversation.input.activity` hosts one control between the context meter and Send, following the model selector. Its `onActiveChange` callback expands that control across the toolbar and hides ordinary accessory controls and the context meter while preserving the editor and submit action. Closing the activity restores those controls. The occupant releases expansion on unmount and owns any activity-specific feedback. ## Model Experience diff --git a/packages/client/ui-conversation/README.zh.md b/packages/client/ui-conversation/README.zh.md index 12d5c955f3..becdb0da3b 100644 --- a/packages/client/ui-conversation/README.zh.md +++ b/packages/client/ui-conversation/README.zh.md @@ -116,7 +116,7 @@ selector 必须是 owner currency 的纯函数。非 null 返回值作为 `match `InputActions.captureInsertion()` 捕获草稿选区与版本;`insertText(text, span)` 仅在版本未变且编辑器允许编辑时,插入一次可撤销的纯文本编辑。异步消费者在插入被拒绝后负责保留结果,等待用户操作。 -`conversation.input.activity` 在模型选择器与发送按钮之间承载一个控件。其 `onActiveChange` 回调将控件展开至整条工具栏并隐藏普通辅助控件,同时保留编辑器、上下文用量控件和提交按钮。上下文用量按钮在活动控件与发送按钮之间保留独立位置。占用者在卸载时释放展开状态,并拥有活动专属反馈。 +`conversation.input.activity` 在上下文用量与发送按钮之间承载一个控件,位于模型选择器之后。其 `onActiveChange` 回调将控件展开至整条工具栏并隐藏普通辅助控件和上下文用量按钮,同时保留编辑器与提交按钮。关闭活动后恢复这些控件。占用者在卸载时释放展开状态,并拥有活动专属反馈。 ## 模型体验 diff --git a/packages/client/ui-conversation/src/client/skeleton/InputBar.module.css b/packages/client/ui-conversation/src/client/skeleton/InputBar.module.css index 2513e2f231..ff1bcea3e7 100644 --- a/packages/client/ui-conversation/src/client/skeleton/InputBar.module.css +++ b/packages/client/ui-conversation/src/client/skeleton/InputBar.module.css @@ -394,7 +394,7 @@ } .standardControls { display: flex; align-items: center; gap: 12px; min-width: 0; } -.tools[hidden], .standardControls[hidden] { display: none; } +.tools[hidden], .standardControls[hidden], .contextControl[hidden] { display: none; } .trailingActive { flex: 1; margin-left: 0; } .activity, .contextControl { display: flex; align-items: center; flex: none; } .activity:empty, .contextControl:empty { display: none; } diff --git a/packages/client/ui-conversation/src/client/skeleton/InputBar.tsx b/packages/client/ui-conversation/src/client/skeleton/InputBar.tsx index 4f64fbacb1..0220623290 100644 --- a/packages/client/ui-conversation/src/client/skeleton/InputBar.tsx +++ b/packages/client/ui-conversation/src/client/skeleton/InputBar.tsx @@ -446,10 +446,10 @@ export const InputBar = memo(function InputBar({ : renderSlot('conversation.input.right', {})} {sessionId === undefined ? null : renderSlot('conversation.input.model', { locked: modelSeatLocked })}
+ {input === undefined || sessionId === undefined ? null :
{renderSlot('conversation.input.activity', { locked, onActiveChange: setActivity })}
} -
{interruptible && ( , + activityEntry: owner => <> + + + , }) const meter = view.getByRole('button', { name: '上下文已用 25%' }) - fireEvent.click(view.getByRole('button', { name: 'microphone' })) + const microphone = view.getByRole('button', { name: 'microphone' }) + expect(meter.compareDocumentPosition(microphone) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy() + fireEvent.click(microphone) + expect(view.queryByRole('button', { name: '上下文已用 25%' })).toBeNull() + expect(view.getByRole('button', { name: '发送消息' })).toBeTruthy() + fireEvent.click(view.getByRole('button', { name: 'close activity' })) expect(view.getByRole('button', { name: '上下文已用 25%' })).toBe(meter) fireEvent.click(meter) expect(view.getByRole('dialog', { name: '上下文已用' })).toBeTruthy() diff --git a/packages/experimental/client-ui-voice-input/src/client/VoiceInput.module.css b/packages/experimental/client-ui-voice-input/src/client/VoiceInput.module.css index 4881303583..28b75820e0 100644 --- a/packages/experimental/client-ui-voice-input/src/client/VoiceInput.module.css +++ b/packages/experimental/client-ui-voice-input/src/client/VoiceInput.module.css @@ -1,6 +1,7 @@ .trigger { width: 28px; padding: 0; flex: none; } .captureRow { display: flex; align-items: center; gap: 12px; width: 100%; min-width: 0; min-height: 34px; } -.roundButton { width: 32px; height: 32px; padding: 0; border-radius: 50%; corner-shape: round; flex: none; } +.roundButton { width: 32px; height: 32px; padding: 0; border-radius: 50%; corner-shape: round; flex: none; background: var(--dsw-specific-selector); } +.roundButton:hover:not(:disabled) { background: var(--dsw-alias-interactive-bg-hover-solid); } .waveform { display: block; flex: 1; width: 0; height: 34px; min-width: 24px; color: var(--dsw-alias-label-secondary); } .activityMessage { display: flex; align-items: center; gap: 8px; flex: 1; min-width: 0; font-size: 12px; color: var(--dsw-alias-label-secondary); overflow: hidden; white-space: nowrap; text-overflow: ellipsis; } .inlineAction { flex: none; } diff --git a/packages/experimental/client-ui-voice-input/src/client/VoiceInput.tsx b/packages/experimental/client-ui-voice-input/src/client/VoiceInput.tsx index 4d6057be8b..6d50aefcce 100644 --- a/packages/experimental/client-ui-voice-input/src/client/VoiceInput.tsx +++ b/packages/experimental/client-ui-voice-input/src/client/VoiceInput.tsx @@ -132,20 +132,20 @@ export function VoiceInput({ sessionId, inputActions, locked, onActiveChange, aria-label={t('start')} title={t(usable ? 'dictate' : 'prepareRequired')} onMouseDown={(event) =>{ event.preventDefault() }} onClick={() => { void start() }}> return
- {phase === 'recording' ? : {(phase === 'requesting' || phase === 'transcribing') && } {phase === 'feedback' ? message : t(phase === 'requesting' ? 'requesting' : provider?.preparation.phase === 'waking' ? 'wakingShort' : 'transcribingShort')}} - {phase === 'recording' && } {phase === 'feedback' && (pending ? - : )}
} From 7e12403d8439bd25dcd7b68003f74d74e3eccc4a Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Thu, 17 Sep 2026 14:11:11 +0800 Subject: [PATCH 05/12] fix(voice-input): reduce recording action icon sizes --- .../client-ui-voice-input/src/client/VoiceInput.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/experimental/client-ui-voice-input/src/client/VoiceInput.tsx b/packages/experimental/client-ui-voice-input/src/client/VoiceInput.tsx index 6d50aefcce..977810e3ad 100644 --- a/packages/experimental/client-ui-voice-input/src/client/VoiceInput.tsx +++ b/packages/experimental/client-ui-voice-input/src/client/VoiceInput.tsx @@ -133,14 +133,14 @@ export function VoiceInput({ sessionId, inputActions, locked, onActiveChange, onMouseDown={(event) =>{ event.preventDefault() }} onClick={() => { void start() }}> return
+ onClick={cancel}> {phase === 'recording' ? : {(phase === 'requesting' || phase === 'transcribing') && } {phase === 'feedback' ? message : t(phase === 'requesting' ? 'requesting' : provider?.preparation.phase === 'waking' ? 'wakingShort' : 'transcribingShort')}} {phase === 'recording' && } + onClick={() => { void finish() }}>} {phase === 'feedback' && (pending ? + if (!expanded) return + + return
diff --git a/packages/experimental/client-ui-voice-input/src/client/locales.ts b/packages/experimental/client-ui-voice-input/src/client/locales.ts index 09a7a91dd7..5289da2d3c 100644 --- a/packages/experimental/client-ui-voice-input/src/client/locales.ts +++ b/packages/experimental/client-ui-voice-input/src/client/locales.ts @@ -43,7 +43,6 @@ export const zh = { 'reconnecting': '正在连接语音服务…', 'preparation.unprepared': '首次使用需要下载本地识别模型。', 'preparation.checking': '正在检查本地资源…', - 'preparation.installing': '正在下载并安装语音运行依赖…', 'preparation.loading': '正在加载 {name}…', 'preparation.waking': '正在唤醒本地语音…', 'preparation.ready': '本地语音已就绪。', @@ -51,25 +50,22 @@ export const zh = { 'preparation.cancelled': '准备已取消,已完成的下载会保留。', 'preparation.cancelling': '正在取消准备…', - settingsDescription: '选择识别服务与语言,准备本地模型。', cloudReady: '云端语音识别已就绪。', - hold: '按住说话', release: '松开结束录音', prepareRequired: '请先在语音插件设置中准备识别服务', - permissionRetry: '允许麦克风访问后,请再按住按钮说话。', discard: '丢弃识别文字', + prepareRequired: '请先在语音插件设置中准备识别服务', + discard: '丢弃识别文字', transcribingShort: '识别中…', wakingShort: '唤醒中…', - trigger: '语音输入', close: '关闭语音输入', provider: '识别服务', language: '识别语言', + provider: '识别服务', language: '识别语言', auto: '自动识别', zh: '中文', en: '英语', yue: '粤语', ja: '日语', ko: '韩语', start: '开始录音', stop: '停止并识别', cancel: '取消', insert: '插入文字', loading: '正在读取识别服务…', requesting: '请允许使用麦克风…', recording: '正在录音…', interrupted: '录音中断,请重试。', - transcribing: '正在识别语音…', local: '音频在运行 DSH 的机器上识别。', cloud: '音频将发送至所选云端服务。', - idle: '停止录音后,识别文字会插入草稿,由你确认发送。', empty: '未识别到语音', cancelled: '已取消语音输入。', conflict: '草稿已被修改。识别文字已保留,可在当前光标位置插入。', - pending: '待插入的识别文字', failed: '语音识别失败:{message}', + failed: '语音识别失败:{message}', unavailable: '当前浏览器不支持录音,请使用支持麦克风的浏览器。', permission: '麦克风权限未开启,请在浏览器和系统设置中允许访问。', - inserted: '识别文字已插入草稿。', tooLarge: '录音超过服务限制,请缩短录音后重试。', + tooLarge: '录音超过服务限制,请缩短录音后重试。', } as const /** Keys accepted by the voice input translator. */ @@ -115,7 +111,6 @@ export const en: Record = { 'reconnecting': 'Connecting to speech recognition…', 'preparation.unprepared': 'First use downloads the local recognition models.', 'preparation.checking': 'Checking local resources…', - 'preparation.installing': 'Downloading and installing speech dependencies…', 'preparation.loading': 'Loading {name}…', 'preparation.waking': 'Waking local speech recognition…', 'preparation.ready': 'Local speech recognition is ready.', @@ -123,25 +118,22 @@ export const en: Record = { 'preparation.cancelled': 'Preparation cancelled. Completed downloads are retained.', 'preparation.cancelling': 'Cancelling preparation…', - settingsDescription: 'Choose the recognizer and language, and prepare local models.', cloudReady: 'Cloud speech recognition is ready.', - hold: 'Hold to speak', release: 'Release to transcribe', prepareRequired: 'Prepare recognition in the voice plugin settings first', - permissionRetry: 'After allowing microphone access, hold the button again to speak.', discard: 'Discard transcript', + prepareRequired: 'Prepare recognition in the voice plugin settings first', + discard: 'Discard transcript', transcribingShort: 'Transcribing…', wakingShort: 'Waking…', - trigger: 'Voice input', close: 'Close voice input', provider: 'Recognition service', language: 'Recognition language', + provider: 'Recognition service', language: 'Recognition language', auto: 'Detect automatically', zh: 'Chinese', en: 'English', yue: 'Cantonese', ja: 'Japanese', ko: 'Korean', start: 'Start recording', stop: 'Stop and transcribe', cancel: 'Cancel', insert: 'Insert text', loading: 'Loading recognition services…', requesting: 'Allow microphone access to continue…', recording: 'Recording…', interrupted: 'Recording was interrupted. Please try again.', - transcribing: 'Transcribing speech…', local: 'Audio is recognized on the machine running DSH.', cloud: 'Audio will be sent to the selected cloud service.', - idle: 'Stop recording to insert the transcript into your draft. Review it before sending.', empty: 'No speech recognized', cancelled: 'Voice input cancelled.', conflict: 'Your draft changed. The transcript is preserved and can be inserted at the current cursor.', - pending: 'Transcript awaiting insertion', failed: 'Speech recognition failed: {message}', + failed: 'Speech recognition failed: {message}', unavailable: 'This browser cannot record audio. Use a browser with microphone support.', permission: 'Microphone access is disabled. Allow it in browser and system settings.', - inserted: 'Transcript inserted into the draft.', tooLarge: 'The recording exceeds the service limit. Try a shorter recording.', + tooLarge: 'The recording exceeds the service limit. Try a shorter recording.', } declare module '@deepseek-ai/dsh-client-ui-slots' { diff --git a/packages/experimental/client-ui-voice-input/tests/preparation.client.spec.tsx b/packages/experimental/client-ui-voice-input/tests/preparation.client.spec.tsx index ba8bb03680..dea23e9c01 100644 --- a/packages/experimental/client-ui-voice-input/tests/preparation.client.spec.tsx +++ b/packages/experimental/client-ui-voice-input/tests/preparation.client.spec.tsx @@ -1,6 +1,6 @@ // @vitest-environment jsdom /** Collapsed progress summaries and Host-owned steps survive view remounts. */ -import { act, cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { act, cleanup, fireEvent, render, screen, waitFor, within } from '@testing-library/react' import { bindSnapshotSelector, makeTranslate } from '@deepseek-ai/dsh-client-test-runtime' import { createSnapshotStore } from '@deepseek-ai/dsh-client-store' import { zh as commonZh } from '@deepseek-ai/dsh-client-locale/src/locales/zh.ts' @@ -19,13 +19,13 @@ const steps: SpeechPreparationStep[] = [ ] function fixture(preparation: SpeechPreparationState, connected = true) { const prepare = vi.fn(async (_id: SpeechProviderId) => {}), cancelPreparation = vi.fn(async (_id: SpeechProviderId) => {}) - const props = { provider: { id, name: 'SenseVoiceSmall', location: 'host-local' as const, preparation }, + const props = { provider: { id, name: 'SenseVoiceSmall', location: 'host-local' as const, languages: ['auto', 'zh', 'en', 'ja'], preparation }, connected, prepare, cancelPreparation, t } return { ...render(), prepare, cancelPreparation, props } } it('collapses by default, expands all steps, and shows details only for the active step', async () => { vi.useFakeTimers(); vi.setSystemTime(5000) - const b = fixture({ phase: 'installing', step: 'verify', startedAt: 0, steps }) + const b = fixture({ phase: 'checking', step: 'verify', startedAt: 0, steps }) expect(screen.queryByRole('list')).toBeNull() expect(screen.getByText('已等待 5 秒')).toBeTruthy() expect(screen.queryByRole('progressbar')).toBeNull() @@ -37,7 +37,7 @@ it('collapses by default, expands all steps, and shows details only for the acti await act(async () => { await vi.advanceTimersByTimeAsync(1000) }) expect(screen.getByText('已等待 6 秒')).toBeTruthy() b.unmount() - fixture({ phase: 'installing', step: 'verify', startedAt: 0, steps }) + fixture({ phase: 'checking', step: 'verify', startedAt: 0, steps }) expect(screen.getByText('已等待 6 秒')).toBeTruthy() expect(screen.queryByRole('list')).toBeNull() }) @@ -87,8 +87,8 @@ it('shows waking without an explicit preparation cancellation control', () => { it('persists settings and reports disconnection in the detail card', async () => { const store = createSnapshotStore({ connected: true, error: null, catalog: { selection: { providerId: id, language: 'auto' }, maxAudioBytes: 100, maxDurationSeconds: 120, - providers: [{ id, name: 'SenseVoiceSmall', location: 'host-local', preparation: { phase: 'ready' } }, - { id: 'cloud' as SpeechProviderId, name: 'Cloud', location: 'cloud', preparation: { phase: 'ready' } }], + providers: [{ id, name: 'SenseVoiceSmall', location: 'host-local', languages: ['auto', 'zh', 'en', 'ja'], preparation: { phase: 'ready' } }, + { id: 'cloud' as SpeechProviderId, name: 'Cloud', location: 'cloud', languages: ['auto', 'en', 'zh', 'ja', 'fr'], preparation: { phase: 'ready' } }], } }) const configure = vi.fn(async () => {}) const props = { useSpeechReadiness: bindSnapshotSelector(store), configure, t, @@ -99,6 +99,9 @@ it('persists settings and reports disconnection in the detail card', async () => const current = store.getSnapshot() act(() => { store.set({ ...current, catalog: { ...current.catalog!, selection: { providerId: 'cloud' as SpeechProviderId, language: 'en' } } }) }) expect(screen.getByText(zh.cloud)).toBeTruthy() + expect(within(screen.getByLabelText(zh.language)).getAllByRole('option').map(item => item.value)) + .toEqual(['auto', 'en', 'zh', 'ja', 'fr']) + expect(within(screen.getByLabelText(zh.language)).getByRole('option', { name: 'fr' })).toBeTruthy() configure.mockRejectedValueOnce(new Error('settings are read-only')) fireEvent.change(screen.getByLabelText(zh.language), { target: { value: 'zh' } }) await screen.findByText('语音识别失败:settings are read-only') @@ -125,6 +128,6 @@ it('shows provider-specific installation estimates before preparation and remove expect(screen.getByText(/建议预留约 5 GB/)).toBeTruthy() expect(screen.getByText(/模型加载后约 2 GB/)).toBeTruthy() expect(screen.getByText(/参考 5–30 分钟/)).toBeTruthy() - b.rerender() + b.rerender() expect(screen.queryByText(zh['setup.local'])).toBeNull() }) diff --git a/packages/experimental/client-ui-voice-input/tests/setup-prompt.client.spec.tsx b/packages/experimental/client-ui-voice-input/tests/setup-prompt.client.spec.tsx index cb66648dfe..16fe9f538a 100644 --- a/packages/experimental/client-ui-voice-input/tests/setup-prompt.client.spec.tsx +++ b/packages/experimental/client-ui-voice-input/tests/setup-prompt.client.spec.tsx @@ -20,7 +20,7 @@ function fixture() { return { onDismiss, onOpenDetails, publish(preparation: SpeechPreparationState, location: 'host-local' | 'cloud' = 'host-local') { act(() => { state.set({ connected: true, error: null, catalog: { selection: { providerId, language: 'auto' }, maxAudioBytes: 1000, maxDurationSeconds: 120, - providers: [{ id: providerId, name: 'SenseVoice', location, preparation }], + providers: [{ id: providerId, name: 'SenseVoice', location, languages: ['auto'], preparation }], } }) }) } } } diff --git a/packages/experimental/client-ui-voice-input/tests/voice-input.client.spec.tsx b/packages/experimental/client-ui-voice-input/tests/voice-input.client.spec.tsx index 3364e71a91..698c3d85ee 100644 --- a/packages/experimental/client-ui-voice-input/tests/voice-input.client.spec.tsx +++ b/packages/experimental/client-ui-voice-input/tests/voice-input.client.spec.tsx @@ -24,7 +24,7 @@ function fixture(recording?: Recording) { const inputActions = { notify: vi.fn(), captureInsertion: vi.fn(() => ({ start: 3, end: 3, draftRev: 1 })), insertText: vi.fn(() => true), setDraft: vi.fn(), addAttachments: vi.fn(() => true), removeAttachment: vi.fn(), pruneAttachments: vi.fn(), submit: vi.fn() } const readiness = createSnapshotStore({ connected: true, error: null, catalog: { - providers: [{ id, name: 'SenseVoiceSmall', location: 'host-local', preparation: { phase: 'ready' } }], + providers: [{ id, name: 'SenseVoiceSmall', location: 'host-local', languages: ['auto', 'zh', 'en', 'ja'], preparation: { phase: 'ready' } }], selection: { providerId: id, language: 'auto' }, maxAudioBytes: 100, maxDurationSeconds: 120, } }) const transcribe = vi.fn<(request: unknown, signal: AbortSignal) => Promise>>( @@ -44,7 +44,8 @@ function stop(): void { fireEvent.click(screen.getByRole('button', { name: zh.st it('clicks to record, shows measured audio, and stops to insert without sending or a popup', async () => { const b = fixture() - expect(screen.getByRole('button', { name: zh.start }).getAttribute('title')).toBe('听写') + fireEvent.mouseEnter(screen.getByRole('button', { name: zh.start }).parentElement!) + expect(screen.getByRole('tooltip').textContent).toBe('听写') fireEvent.mouseDown(screen.getByRole('button', { name: zh.start })) await start() expect(b.props.onActiveChange).toHaveBeenLastCalledWith(true) @@ -258,3 +259,45 @@ it('allows recording while verified local resources are waking', async () => { stop() await waitFor(() => { expect(b.inputActions.insertText).toHaveBeenCalledWith(transcript.text, expect.anything()) }) }) + +it('keeps the first recording after the permission prompt causes window blur', async () => { + const b = fixture(), permission = Promise.withResolvers() + b.capture.start.mockReturnValueOnce(permission.promise) + fireEvent.click(screen.getByRole('button', { name: zh.start })) + fireEvent.blur(window) + expect(screen.getByRole('status').textContent).toBe(zh.requesting) + expect(b.capture.dispose).not.toHaveBeenCalled() + permission.resolve(undefined) + await screen.findByRole('button', { name: zh.stop }) + stop() + await waitFor(() => { expect(b.inputActions.insertText).toHaveBeenCalledOnce() }) +}) + +it.each(['escape', 'hidden', 'session'])('discards pending permission after %s', async (action) => { + const b = fixture(), permission = Promise.withResolvers() + b.capture.start.mockReturnValueOnce(permission.promise) + fireEvent.click(screen.getByRole('button', { name: zh.start })) + if (action === 'escape') fireEvent.keyDown(document, { key: 'Escape' }) + if (action === 'session') b.view.rerender() + if (action === 'hidden') { + Object.defineProperty(document, 'hidden', { configurable: true, value: true }) + try { fireEvent(document, new Event('visibilitychange')) } finally { delete (document as { hidden?: boolean }).hidden } + } + permission.resolve(undefined) + await act(async () => {}) + expect(b.capture.dispose).toHaveBeenCalledOnce() + expect(screen.getByRole('button', { name: zh.start })).toBeTruthy() + expect(screen.queryByRole('button', { name: zh.stop })).toBeNull() + expect(b.transcribe).not.toHaveBeenCalled() +}) + +it('explains preparation on hover even when the microphone is disabled', () => { + const b = fixture(), state = b.readiness.getSnapshot() + act(() => { b.readiness.set({ ...state, catalog: { ...state.catalog!, providers: [ + { ...state.catalog!.providers[0]!, preparation: { phase: 'unprepared' } }, + ] } }) }) + const mic = screen.getByRole('button', { name: zh.start }) + expect(mic.disabled).toBe(true) + fireEvent.mouseEnter(mic.parentElement!) + expect(screen.getByRole('tooltip').textContent).toBe(zh.prepareRequired) +}) diff --git a/packages/experimental/speech-to-text-sensevoice/README.i18n.yaml b/packages/experimental/speech-to-text-sensevoice/README.i18n.yaml index 751ef66b7c..df7b1b5d04 100644 --- a/packages/experimental/speech-to-text-sensevoice/README.i18n.yaml +++ b/packages/experimental/speech-to-text-sensevoice/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/experimental/speech-to-text-sensevoice/README.md -README.md: d8d26118ee5ae6632db0c59e67e3f405d89e85e1 -README.zh.md: c6d992c2828d624d00cefc2a62cc0043edf7cada +README.md: ea751ca1e080ac4805a4cdff58e47dfb9b1dc75d +README.zh.md: fc142d1f8bd0f46d93904a29915d985ed9eb2ba0 diff --git a/packages/experimental/speech-to-text-sensevoice/README.md b/packages/experimental/speech-to-text-sensevoice/README.md index d8d26118ee..ea751ca1e0 100644 --- a/packages/experimental/speech-to-text-sensevoice/README.md +++ b/packages/experimental/speech-to-text-sensevoice/README.md @@ -37,9 +37,9 @@ Every activation checks the selected model, tokens and VAD on disk. Complete ver
Maintainer details — click to expand -One managed Node subprocess serializes inference. Recordings submitted during wake-up join the bounded queue. At execution, the worker requires ready or standby resources. Transcription never starts preparation downloads. `threads`, `segmentSeconds`, `vadThreshold`, `minSpeechSeconds` and `minSilenceSeconds` tune CPU inference and segmentation; `maxPending` bounds accepted work. Preparation and inference have separate deadlines. Cancellation terminates an active worker and joins its process range; cancelled queued recordings never run. `idleTimeoutMs` releases the worker after inactivity, with zero retaining it until disposal. Models remain cached. Authenticated loopback requests keep audio in memory. Electron workers run in Node mode and copy VAD buffers to satisfy the V8 memory cage. No runtime invariant companion is published because `dsh-subprocess` owns process-range observations and this provider has no independent projection to reconcile. +One managed Node subprocess serializes inference. Recordings submitted during wake-up join the bounded queue. At execution, the worker requires ready or standby resources. Transcription never starts preparation downloads. `threads`, `segmentSeconds`, `vadThreshold`, `minSpeechSeconds` and `minSilenceSeconds` tune CPU inference and segmentation; `maxPending` bounds accepted work. Preparation and inference have separate deadlines. Invalid language or WAV input is rejected before native inference and retains the loaded worker; inference failures, malformed responses and transport failures reclaim it. Cancellation terminates an active worker and joins its process range; cancelled queued recordings never run. `idleTimeoutMs` releases the worker after inactivity, with zero retaining it until disposal. Models remain cached. Authenticated loopback requests keep audio in memory. Electron workers run in Node mode and copy VAD buffers to satisfy the V8 memory cage. No runtime invariant companion is published because `dsh-subprocess` owns process-range observations and this provider has no independent projection to reconcile. -The Host owns preparation across page and Session changes: check resources, download recognition weights and tokens, download VAD, verify files, and load the worker. Explicit model paths omit their download steps. Downloads report bytes; other steps report elapsed time. An idle worker retains prepared paths and needs only model loading on wake-up. +The Host owns preparation across page and Session changes: check resources, download recognition weights and tokens, download VAD, verify files, and load the worker. Explicit model paths omit their download steps. Downloads report bytes; other steps report elapsed time. An idle worker retains prepared paths and needs only model loading on wake-up. Cancellation after a preparation result is published waits for cleanup without replacing that result.
diff --git a/packages/experimental/speech-to-text-sensevoice/README.zh.md b/packages/experimental/speech-to-text-sensevoice/README.zh.md index c6d992c282..fc142d1f8b 100644 --- a/packages/experimental/speech-to-text-sensevoice/README.zh.md +++ b/packages/experimental/speech-to-text-sensevoice/README.zh.md @@ -37,9 +37,9 @@ kind: "package-reference"
维护者信息 — 点击展开 -一个托管 Node 子进程串行执行推理。唤醒期间提交的录音进入有界队列。工作进程在执行时要求资源处于就绪或待唤醒状态。转写不会启动准备下载。`threads`、`segmentSeconds`、`vadThreshold`、`minSpeechSeconds` 和 `minSilenceSeconds` 调节 CPU 推理与分段;`maxPending` 限制已接收任务数。准备和推理分别有截止时间。取消会终止运行中的工作进程并等待其进程范围退出;已取消的排队录音不会执行。`idleTimeoutMs` 在空闲后释放进程,零表示保留至插件释放。模型继续保留在缓存中。带认证的环回请求仅在内存中保存音频。Electron 工作进程使用 Node 模式,并复制 VAD 缓冲区以兼容 V8 内存笼。`dsh-subprocess` 拥有进程范围观测;本包没有需要协调的独立投影,因此不发布不变量伴随模块。 +一个托管 Node 子进程串行执行推理。唤醒期间提交的录音进入有界队列。工作进程在执行时要求资源处于就绪或待唤醒状态。转写不会启动准备下载。`threads`、`segmentSeconds`、`vadThreshold`、`minSpeechSeconds` 和 `minSilenceSeconds` 调节 CPU 推理与分段;`maxPending` 限制已接收任务数。准备和推理分别有截止时间。无效语言或 WAV 输入在原生推理前被拒绝,保留已加载的工作进程;推理失败、响应格式错误和传输失败会回收进程。取消会终止运行中的工作进程并等待其进程范围退出;已取消的排队录音不会执行。`idleTimeoutMs` 在空闲后释放进程,零表示保留至插件释放。模型继续保留在缓存中。带认证的环回请求仅在内存中保存音频。Electron 工作进程使用 Node 模式,并复制 VAD 缓冲区以兼容 V8 内存笼。`dsh-subprocess` 拥有进程范围观测;本包没有需要协调的独立投影,因此不发布不变量伴随模块。 -Host 在页面和 Session 变化期间拥有准备任务:检查资源、下载识别权重与词表、下载 VAD、校验文件、加载工作进程。显式模型路径会省略对应下载步骤。下载报告字节,其他步骤报告等待时间。空闲进程保留已准备路径,唤醒时只需加载模型。 +Host 在页面和 Session 变化期间拥有准备任务:检查资源、下载识别权重与词表、下载 VAD、校验文件、加载工作进程。显式模型路径会省略对应下载步骤。下载报告字节,其他步骤报告等待时间。空闲进程保留已准备路径,唤醒时只需加载模型。准备结果发布后再取消,只等待任务收尾,不会覆盖该结果。
diff --git a/packages/experimental/speech-to-text-sensevoice/src/index.ts b/packages/experimental/speech-to-text-sensevoice/src/index.ts index c0363d9def..987f0443ed 100644 --- a/packages/experimental/speech-to-text-sensevoice/src/index.ts +++ b/packages/experimental/speech-to-text-sensevoice/src/index.ts @@ -6,6 +6,7 @@ import type {} from '@deepseek-ai/dsh-subprocess' import type { SpeechProviderId } from '@deepseek-ai/dsh-experimental-speech-to-text/types' import { Config } from './config.ts' import { SenseVoiceWorker } from './recognizer.ts' +import { languages } from './input.ts' export { Config } from './config.ts' export const name = 'experimental-speech-to-text-sensevoice' @@ -22,10 +23,11 @@ export function apply(ctx: Context, config: Config): void { } new URL(config.modelOrigin) const worker = new SenseVoiceWorker(ctx, config) + const estimatedBytes = config.precision === 'int8' ? 1_000_000_000 : 2_000_000_000 ctx.effect(() => { const unregister = ctx.speechToText.register({ - info: { id: config.providerId as SpeechProviderId, name: `SenseVoiceSmall (${config.precision.toUpperCase()})`, location: 'host-local', - setupEstimate: { recommendedDiskBytes: config.precision === 'int8' ? 1_000_000_000 : 2_000_000_000, expectedMemoryBytes: config.precision === 'int8' ? 1_000_000_000 : 2_000_000_000, + info: { id: config.providerId as SpeechProviderId, name: `SenseVoiceSmall (${config.precision.toUpperCase()})`, location: 'host-local', languages, + setupEstimate: { recommendedDiskBytes: estimatedBytes, expectedMemoryBytes: estimatedBytes, minimumMinutes: 1, maximumMinutes: 10 } }, preparation: worker, transcribe: async (input, signal) => await worker.transcribe(input, signal), diff --git a/packages/experimental/speech-to-text-sensevoice/src/inference.ts b/packages/experimental/speech-to-text-sensevoice/src/inference.ts index 0b6859d6e2..ce24200843 100644 --- a/packages/experimental/speech-to-text-sensevoice/src/inference.ts +++ b/packages/experimental/speech-to-text-sensevoice/src/inference.ts @@ -1,8 +1,8 @@ /** CPU SenseVoice inference and Silero segmentation, confined to the recognition process. */ import { createRequire } from 'node:module' import type { Transcript } from '@deepseek-ai/dsh-experimental-speech-to-text/types' -import { validateWave } from '@deepseek-ai/dsh-experimental-speech-to-text/wave' import type { Config } from './config.ts' +import { validateInput } from './input.ts' /** Verified files and validated inference settings sent by the Host. */ export interface InferenceConfig extends Config { @@ -51,9 +51,7 @@ export function createTranscriber(config: InferenceConfig): (audio: Uint8Array, sampleRate: 16000, numThreads: config.threads, provider: 'cpu', debug: 0, }, config.segmentSeconds + config.minSilenceSeconds + 1) return (audio, language) => { - if (!['auto', 'zh', 'en', 'yue', 'ja', 'ko'].includes(language)) throw new Error('Unsupported SenseVoice language') - if (audio.byteLength > config.maxAudioBytes) throw new Error('Speech audio exceeds the worker byte limit') - const audioSeconds = validateWave(audio, config.maxAudioBytes / 32000) + const audioSeconds = validateInput(audio, language, config.maxAudioBytes) const pcm = new DataView(audio.buffer, audio.byteOffset + 44, audio.byteLength - 44) const samples = Float32Array.from({ length: pcm.byteLength / 2 }, (_, i) => pcm.getInt16(i * 2, true) / 32768) nativeConfig.modelConfig.senseVoice.language = language diff --git a/packages/experimental/speech-to-text-sensevoice/src/input.ts b/packages/experimental/speech-to-text-sensevoice/src/input.ts new file mode 100644 index 0000000000..613c19e941 --- /dev/null +++ b/packages/experimental/speech-to-text-sensevoice/src/input.ts @@ -0,0 +1,22 @@ +/** SenseVoice language support and input failures that leave native inference untouched. */ +import { validateWave } from '@deepseek-ai/dsh-experimental-speech-to-text/wave' + +/** Language hints accepted by both provider metadata and native inference. */ +export const languages: readonly string[] = ['auto', 'zh', 'en', 'yue', 'ja', 'ko'] + +/** A request rejected before native inference; the loaded worker remains reusable. */ +export class SpeechInputError extends Error {} + +/** + * Validate a recording before touching the native recognizer. + * @param audio - untrusted WAV request bytes. + * @param language - requested SenseVoice language hint. + * @param maxAudioBytes - configured worker byte limit. + * @returns validated audio duration in seconds; invalid inputs throw SpeechInputError. + */ +export function validateInput(audio: Uint8Array, language: string, maxAudioBytes: number): number { + if (!languages.includes(language)) throw new SpeechInputError('Unsupported SenseVoice language') + if (audio.byteLength > maxAudioBytes) throw new SpeechInputError('Speech audio exceeds the worker byte limit') + try { return validateWave(audio, maxAudioBytes / 32000) } + catch (error) { throw new SpeechInputError('Invalid speech WAV', { cause: error }) } +} diff --git a/packages/experimental/speech-to-text-sensevoice/src/process-server.ts b/packages/experimental/speech-to-text-sensevoice/src/process-server.ts index 099254d44d..fe24cd8a5e 100644 --- a/packages/experimental/speech-to-text-sensevoice/src/process-server.ts +++ b/packages/experimental/speech-to-text-sensevoice/src/process-server.ts @@ -2,6 +2,7 @@ import { createServer, type Server } from 'node:http' import { timingSafeEqual } from 'node:crypto' import type { Transcript } from '@deepseek-ai/dsh-experimental-speech-to-text/types' +import { SpeechInputError } from './input.ts' /** * Bind an ephemeral loopback listener; model loading completes before readiness is published. @@ -27,7 +28,7 @@ export async function startRecognitionServer(token: string, maxAudioBytes: numbe } const length = Number(request.headers['content-length']) if (!Number.isSafeInteger(length) || length < 46 || length > maxAudioBytes) { - request.resume(); reply(413, { error: 'Invalid speech audio size' }); return + request.resume(); reply(413, { error: 'Invalid speech audio size', code: 'invalid-input' }); return } void (async () => { try { @@ -38,7 +39,10 @@ export async function startRecognitionServer(token: string, maxAudioBytes: numbe } reply(200, transcribe(Buffer.concat(chunks), url.searchParams.get('language') ?? 'auto')) } catch (error) { - reply(400, { error: error instanceof Error ? error.message : String(error) }) + reply(error instanceof SpeechInputError ? 400 : 500, { + error: error instanceof Error ? error.message : String(error), + ...error instanceof SpeechInputError ? { code: 'invalid-input' } : {}, + }) } })() }) diff --git a/packages/experimental/speech-to-text-sensevoice/src/recognizer.ts b/packages/experimental/speech-to-text-sensevoice/src/recognizer.ts index db4d3c759a..d0a2c0e58b 100644 --- a/packages/experimental/speech-to-text-sensevoice/src/recognizer.ts +++ b/packages/experimental/speech-to-text-sensevoice/src/recognizer.ts @@ -7,6 +7,7 @@ import type { SpeechInput, SpeechPreparationState, SpeechPreparationStep, Speech import { deadline } from '@deepseek-ai/dsh-timeout' import { z } from 'zod' import type { Config } from './config.ts' +import { SpeechInputError } from './input.ts' import { inspectRuntime, prepareRuntime, type RuntimePaths } from './runtime.ts' const transcriptSchema = z.object({ @@ -66,7 +67,7 @@ export async function readReady(handle: SubprocessHandle, limit: number, signal: * Decode a bounded worker HTTP response; malformed worker output fails the request. * @param response - private authenticated worker response. * @param limit - maximum bytes retained before JSON parsing. - * @returns validated final transcript. + * @returns validated final transcript; marked input rejections throw SpeechInputError. */ export async function readTranscript(response: Response, limit: number): Promise { if (!response.body) throw new Error('Speech worker returned no response') @@ -87,7 +88,10 @@ export async function readTranscript(response: Response, limit: number): Promise } const value: unknown = JSON.parse(Buffer.concat(chunks).toString('utf8')) if (!response.ok) { - const failure = z.object({ error: z.string() }).parse(value) + const failure = z.object({ error: z.string(), code: z.literal('invalid-input').optional() }).parse(value) + if ((response.status === 400 || response.status === 413) && failure.code === 'invalid-input') { + throw new SpeechInputError(failure.error) + } throw new Error(failure.error) } return transcriptSchema.parse(value) @@ -104,7 +108,7 @@ export class SenseVoiceWorker { private state: SpeechPreparationState & { readonly steps: readonly SpeechPreparationStep[] } private readonly listeners = new Set<() => void>() private lastProgressAt = 0 - private preparing: { abort: AbortController; settled: Promise } | undefined + private preparing: { abort: AbortController; settled: Promise; completed: boolean } | undefined constructor(private readonly ctx: Context, private readonly config: Config) { const kinds: SpeechPreparationStepKind[] = ['check'] @@ -131,6 +135,9 @@ export class SenseVoiceWorker { } private publish(state: SpeechPreparationState): void { + if (this.preparing && ['ready', 'standby', 'unprepared', 'failed', 'cancelled'].includes(state.phase)) { + this.preparing.completed = true + } const previous = this.state const steps = state.steps ?? previous.steps.map((item): SpeechPreparationStep => { if (state.phase === 'ready') return { ...item, status: 'complete' } @@ -173,7 +180,7 @@ export class SenseVoiceWorker { if (this.preparing || this.worker) return this.lifetime.signal.throwIfAborted() const abort = new AbortController() - const task = { abort, settled: Promise.resolve() } + const task = { abort, settled: Promise.resolve(), completed: false } this.preparing = task task.settled = this.enqueue(run, abort.signal) .catch((error: unknown) => { @@ -183,12 +190,14 @@ export class SenseVoiceWorker { }).finally(() => { this.preparing = undefined }) } - /** Cancel only preparation. @returns after its queued or active work settles. */ + /** Cancel unfinished preparation; completed readiness is retained. @returns after its queued or active work settles. */ async cancel(): Promise { const task = this.preparing if (!task) return - this.publish({ phase: 'cancelling', startedAt: Date.now() }) - task.abort.abort(new Error('Speech preparation cancelled')) + if (!task.completed) { + this.publish({ phase: 'cancelling', startedAt: Date.now() }) + task.abort.abort(new Error('Speech preparation cancelled')) + } await task.settled } @@ -270,10 +279,14 @@ export class SenseVoiceWorker { method: 'POST', headers: { authorization: `Bearer ${worker.token}`, 'content-type': 'audio/wav' }, body: Buffer.from(input.audio), signal: call.signal, }) - const result = await readTranscript(response, this.config.maxResponseBytes) + const result = await readTranscript(response, this.config.maxResponseBytes).catch((error: unknown) => { + call.signal.throwIfAborted() + throw error + }) call.signal.throwIfAborted() return result } catch (error) { + if (error instanceof SpeechInputError) throw error await this.stop() this.publish({ phase: 'standby' }) throw error diff --git a/packages/experimental/speech-to-text-sensevoice/tests/installer.spec.ts b/packages/experimental/speech-to-text-sensevoice/tests/installer.spec.ts index e1030c7322..0e3b532f70 100644 --- a/packages/experimental/speech-to-text-sensevoice/tests/installer.spec.ts +++ b/packages/experimental/speech-to-text-sensevoice/tests/installer.spec.ts @@ -42,7 +42,7 @@ it('restores verified cached models on every provider activation without downloa const ctx = new Context() try { await ctx.plugin(LocalSubprocess) - await ctx.plugin(SpeechToText) + await ctx.plugin(SpeechToText, { defaultProvider: 'sensevoice-local', language: 'auto' }) const spawn = vi.spyOn(ctx.get('subprocess')!, 'spawn'), speech = ctx.get('speechToText')! for (let activation = 0; activation < 2; activation++) { const fiber = ctx.plugin(Provider, config) diff --git a/packages/experimental/speech-to-text-sensevoice/tests/local.e2e.ts b/packages/experimental/speech-to-text-sensevoice/tests/local.e2e.ts index 44e10ab9a9..e6e16e22b3 100644 --- a/packages/experimental/speech-to-text-sensevoice/tests/local.e2e.ts +++ b/packages/experimental/speech-to-text-sensevoice/tests/local.e2e.ts @@ -11,6 +11,7 @@ const { DSH_SPEECH_E2E_ROOT: dataRoot, DSH_SPEECH_E2E_AUDIO: audioPath } = proce it.skipIf(!dataRoot || !audioPath)('prepares a real local recognizer, transcribes speech, and releases it', { timeout: 3_660_000, retry: 0 }, async () => { const ctx = new Context() await ctx.plugin(LocalSubprocess) + const spawn = vi.spyOn(ctx.get('subprocess')!, 'spawn') const worker = new SenseVoiceWorker(ctx, Config({ dataRoot: dataRoot!, modelDirectory: process.env.DSH_SPEECH_E2E_MODEL, vadModelPath: process.env.DSH_SPEECH_E2E_VAD, @@ -25,9 +26,13 @@ it.skipIf(!dataRoot || !audioPath)('prepares a real local recognizer, transcribe expect(cold.text.length).toBeGreaterThan(0) expect(warm.text).toBe(cold.text) expect(warm.audioSeconds).toBeGreaterThan(0) + await expect(worker.transcribe({ ...input, language: 'fr' }, new AbortController().signal)).rejects.toThrow('Unsupported SenseVoice language') + expect((await worker.transcribe(input, new AbortController().signal)).text).toBe(warm.text) + expect(spawn).toHaveBeenCalledOnce() console.info('SenseVoice inference', { cold, warm }) } finally { await worker.dispose() await ctx.fiber.dispose() + spawn.mockRestore() } }) diff --git a/packages/experimental/speech-to-text-sensevoice/tests/process-server.spec.ts b/packages/experimental/speech-to-text-sensevoice/tests/process-server.spec.ts index 6ad0186f72..23d78df2a0 100644 --- a/packages/experimental/speech-to-text-sensevoice/tests/process-server.spec.ts +++ b/packages/experimental/speech-to-text-sensevoice/tests/process-server.spec.ts @@ -1,5 +1,6 @@ /** The private recognizer admits only authenticated bounded audio requests. */ import { expect, it, onTestFinished, vi } from 'vitest' +import { SpeechInputError } from '../src/input.ts' import { startRecognitionServer } from '../src/process-server.ts' it('serves transcripts on an ephemeral loopback port and rejects unauthorized or invalid requests', async () => { @@ -19,12 +20,17 @@ it('serves transcripts on an ephemeral loopback port and rejects unauthorized or const good = await fetch(`${base}/transcribe?language=zh`, { method: 'POST', headers, body }) expect(await good.json()).toEqual({ text: 'hello', audioSeconds: 1, inferenceSeconds: 0.1 }) expect(transcribe).toHaveBeenLastCalledWith(body, 'zh') - transcribe.mockImplementationOnce(() => { throw new Error('bad audio') }) + transcribe.mockImplementationOnce(() => { throw new SpeechInputError('bad audio') }) const bad = await fetch(`${base}/transcribe`, { method: 'POST', headers, body }) expect(bad.status).toBe(400) - expect(await bad.json()).toEqual({ error: 'bad audio' }) + expect(await bad.json()).toEqual({ error: 'bad audio', code: 'invalid-input' }) expect(transcribe).toHaveBeenLastCalledWith(body, 'auto') transcribe.mockImplementationOnce(() => { throw 'native failure' }) const failure = await fetch(`${base}/transcribe`, { method: 'POST', headers, body }) + expect(failure.status).toBe(500) expect(await failure.json()).toEqual({ error: 'native failure' }) + transcribe.mockImplementationOnce(() => { throw new Error('native decode failed') }) + const nativeFailure = await fetch(`${base}/transcribe`, { method: 'POST', headers, body }) + expect(nativeFailure.status).toBe(500) + expect(await nativeFailure.json()).toEqual({ error: 'native decode failed' }) }) diff --git a/packages/experimental/speech-to-text-sensevoice/tests/protocol.spec.ts b/packages/experimental/speech-to-text-sensevoice/tests/protocol.spec.ts index aeb30e4b6f..17363d2915 100644 --- a/packages/experimental/speech-to-text-sensevoice/tests/protocol.spec.ts +++ b/packages/experimental/speech-to-text-sensevoice/tests/protocol.spec.ts @@ -2,6 +2,7 @@ import { PassThrough } from 'node:stream' import type { SubprocessHandle } from '@deepseek-ai/dsh-subprocess' import { expect, it } from 'vitest' +import { SpeechInputError } from '../src/input.ts' import { readReady, readTranscript } from '../src/recognizer.ts' function child() { @@ -75,3 +76,9 @@ it('contains stdout failures and primitive cancellation reasons while waiting fo await expect(pending).rejects.toThrow('Speech operation cancelled') cancelled.done.resolve({ exitCode: 0, signal: null }); cancelled.stdout.destroy() }) + +it.each([400, 413, 500])('classifies marked input failures only on input status %s', async (status) => { + const error = await readTranscript(Response.json({ error: 'rejected', code: 'invalid-input' }, { status }), 100).catch((value: unknown) => value) + expect(error).toBeInstanceOf(Error) + expect(error instanceof SpeechInputError).toBe(status !== 500) +}) diff --git a/packages/experimental/speech-to-text-sensevoice/tests/provider-queue.spec.ts b/packages/experimental/speech-to-text-sensevoice/tests/provider-queue.spec.ts index 06dd34c12a..47f77250fc 100644 --- a/packages/experimental/speech-to-text-sensevoice/tests/provider-queue.spec.ts +++ b/packages/experimental/speech-to-text-sensevoice/tests/provider-queue.spec.ts @@ -29,7 +29,7 @@ it.each([false, true])('queues recordings during startup and honors waiting canc await writeFile(worker, `await fetch('http://127.0.0.1:${address.port}/');\nawait import(${JSON.stringify(fixtureUrl)});\n`) vi.mocked(inspectRuntime).mockResolvedValue({ tokens: root, model: root, vad: root, worker }) await ctx.plugin(LocalSubprocess) - await ctx.plugin(SpeechToText) + await ctx.plugin(SpeechToText, { defaultProvider: 'sensevoice-local', language: 'auto' }) const spawn = vi.spyOn(ctx.subprocess, 'spawn') const enqueue = vi.spyOn(SenseVoiceWorker.prototype, 'transcribe') await ctx.plugin(Provider, { dataRoot: root, idleTimeoutMs: 0, maxPending: 2 }) diff --git a/packages/experimental/speech-to-text-sensevoice/tests/provider.spec.ts b/packages/experimental/speech-to-text-sensevoice/tests/provider.spec.ts index a650d3d122..a2fef1fd18 100644 --- a/packages/experimental/speech-to-text-sensevoice/tests/provider.spec.ts +++ b/packages/experimental/speech-to-text-sensevoice/tests/provider.spec.ts @@ -12,13 +12,17 @@ it.each(['int8', 'fp32'] as const)('registers %s lazily and joins inference befo const ctx = new Context() try { await ctx.plugin(LocalSubprocess) - await ctx.plugin(SpeechToText) + await ctx.plugin(SpeechToText, { defaultProvider: 'sensevoice-local', language: 'auto' }) const transcribe = vi.spyOn(SenseVoiceWorker.prototype, 'transcribe') const fiber = ctx.plugin(Provider, { dataRoot: process.cwd(), precision }) await fiber expect(transcribe).not.toHaveBeenCalled() const speech = ctx.get('speechToText')! - expect(speech.listProviders()).toMatchObject([{ id: 'sensevoice-local', location: 'host-local' }]) + expect(speech.listProviders()).toMatchObject([{ id: 'sensevoice-local', location: 'host-local', languages: ['auto', 'zh', 'en', 'yue', 'ja', 'ko'] }]) + expect(speech.listProviders()[0]?.setupEstimate).toEqual({ + recommendedDiskBytes: precision === 'int8' ? 1e9 : 2e9, expectedMemoryBytes: precision === 'int8' ? 1e9 : 2e9, + minimumMinutes: 1, maximumMinutes: 10, + }) await expect(speech.transcribe(speech.resolve({ audio: new Uint8Array() }), new AbortController().signal)).rejects.toThrow('Prepare') transcribe.mockResolvedValue({ text: 'hello', audioSeconds: 1, inferenceSeconds: 0.1 }) expect((await speech.transcribe(speech.resolve({ audio: new Uint8Array() }), new AbortController().signal)).text).toBe('hello') diff --git a/packages/experimental/speech-to-text-sensevoice/tests/worker.fixture.mjs b/packages/experimental/speech-to-text-sensevoice/tests/worker.fixture.mjs index ee54787927..e461185866 100644 --- a/packages/experimental/speech-to-text-sensevoice/tests/worker.fixture.mjs +++ b/packages/experimental/speech-to-text-sensevoice/tests/worker.fixture.mjs @@ -14,6 +14,7 @@ const server = createServer((request, response) => { request.on('end', () => { if (language === 'hold') return if (language === 'crash') { process.exit(1); return } + if (language === 'invalid-input') { response.writeHead(400).end(JSON.stringify({ error: 'invalid input', code: 'invalid-input' })); return } if (language === 'error') { response.writeHead(400).end(JSON.stringify({ error: 'provider failed' })); return } response.setHeader('content-type', 'application/json') response.end(JSON.stringify({ text: language, audioSeconds: 1, inferenceSeconds: 0.1 })) diff --git a/packages/experimental/speech-to-text-sensevoice/tests/worker.spec.ts b/packages/experimental/speech-to-text-sensevoice/tests/worker.spec.ts index 3cc12d1198..730256df43 100644 --- a/packages/experimental/speech-to-text-sensevoice/tests/worker.spec.ts +++ b/packages/experimental/speech-to-text-sensevoice/tests/worker.spec.ts @@ -299,9 +299,10 @@ it('retains a worker whose idle cleanup cannot observe exit until that range is it('retains completed preparation steps when cancellation settles the active step', async () => { const { worker } = await fixture(), entered = Promise.withResolvers() vi.mocked(prepareRuntime).mockImplementationOnce(async (_ctx, _config, signal, report) => { - for (const step of ['model', 'vad', 'verify'] as const) report!({ phase: 'installing', step, startedAt: Date.now() }) + for (const step of ['model', 'vad'] as const) report!({ phase: 'downloading', step, resource: step, completedBytes: 1, totalBytes: 1 }) + report!({ phase: 'checking', step: 'verify', startedAt: Date.now() }) const before = worker.snapshot().steps?.find(step => step.kind === 'verify')?.startedAt - report!({ phase: 'installing', step: 'verify', startedAt: Date.now() }) + report!({ phase: 'checking', step: 'verify', startedAt: Date.now() }) expect(worker.snapshot().steps?.find(step => step.kind === 'verify')?.startedAt).toBe(before) entered.resolve(undefined) return await new Promise((_resolve, reject) => { @@ -371,3 +372,54 @@ it('rechecks readiness after a queued cache inspection without implicitly downlo expect(prepareRuntime).not.toHaveBeenCalled() expect(spawn).not.toHaveBeenCalled() }) + +it.each(['ready', 'standby', 'unprepared', 'failed'] as const)('joins cancellation at committed %s without replacing its result', async (phase) => { + const { worker, root, spawn } = await fixture() + const cancellation = Promise.withResolvers>() + const unsubscribe = worker.subscribe(() => { + if (worker.snapshot().phase === phase) cancellation.resolve(worker.cancel()) + }) + try { + if (phase === 'ready') worker.prepare() + else { + if (phase === 'failed') vi.mocked(inspectRuntime).mockRejectedValueOnce(new Error('cache unavailable')) + else vi.mocked(inspectRuntime).mockResolvedValueOnce(phase === 'standby' + ? { tokens: root, model: root, vad: root, worker: fileURLToPath(new URL('./worker.fixture.mjs', import.meta.url)) } + : undefined) + worker.inspect() + } + await cancellation.promise + expect(worker.snapshot().phase).toBe(phase) + if (phase === 'ready' || phase === 'standby') { + expect((await worker.transcribe({ audio, language: 'zh' }, signal())).text).toBe('zh') + expect(spawn).toHaveBeenCalledOnce() + } else { + await expect(worker.transcribe({ audio, language: 'zh' }, signal())).rejects.toThrow('Prepare') + expect(spawn).not.toHaveBeenCalled() + await prepare(worker) + expect(worker.snapshot().phase).toBe('ready') + } + } finally { unsubscribe() } +}) + +it('cancels a queued preparation before any resources are acquired', async () => { + const { worker, spawn } = await fixture() + worker.prepare() + await worker.cancel() + expect(worker.snapshot().phase).toBe('cancelled') + expect(prepareRuntime).not.toHaveBeenCalled() + expect(spawn).not.toHaveBeenCalled() +}) + +it('reuses a loaded worker only after explicitly classified input rejection', async () => { + const { worker, spawn } = await fixture() + await prepare(worker) + await expect(worker.transcribe({ audio, language: 'invalid-input' }, signal())).rejects.toThrow('invalid input') + expect(worker.snapshot().phase).toBe('ready') + expect((await worker.transcribe({ audio, language: 'en' }, signal())).text).toBe('en') + expect(spawn).toHaveBeenCalledOnce() + await expect(worker.transcribe({ audio, language: 'error' }, signal())).rejects.toThrow('provider failed') + expect(worker.snapshot().phase).toBe('standby') + expect((await worker.transcribe({ audio, language: 'zh' }, signal())).text).toBe('zh') + expect(spawn).toHaveBeenCalledTimes(2) +}) diff --git a/packages/experimental/speech-to-text/README.i18n.yaml b/packages/experimental/speech-to-text/README.i18n.yaml index 93ead52b7d..6d5068c1ff 100644 --- a/packages/experimental/speech-to-text/README.i18n.yaml +++ b/packages/experimental/speech-to-text/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/experimental/speech-to-text/README.md -README.md: b64af2d04e7a7f5842fa9ce64f00e3927ff9be0a -README.zh.md: 7f93de89835c1e44739f12bad4ac8b5518851415 +README.md: 6b2481590c2e1ee7c02401ca0838ab62d4a64b86 +README.zh.md: 46436194dd2350f85fc652b31377fad2f0d79348 diff --git a/packages/experimental/speech-to-text/README.md b/packages/experimental/speech-to-text/README.md index b64af2d04e..6b2481590c 100644 --- a/packages/experimental/speech-to-text/README.md +++ b/packages/experimental/speech-to-text/README.md @@ -25,7 +25,7 @@ This Service Definition selects named speech recognizers through `ctx.speechToTe ## Use this package -Load through the [voice input bundle](../voice-input-bundle/README.md), or compose the service with a provider and consumer. `defaultProvider` selects an exact registered id; `language` supplies the omitted language hint. A missing or duplicate provider fails explicitly. +Load through the [voice input bundle](../voice-input-bundle/README.md), or compose the service with a provider and consumer. `defaultProvider` is required and selects an exact registered id; the bundle supplies `sensevoice-local`. `language` supplies the omitted language hint. A missing or duplicate provider fails explicitly. ----- @@ -37,7 +37,7 @@ Load through the [voice input bundle](../voice-input-bundle/README.md), or compo `resolve()` captures the provider instance, recording and language. `transcribe()` rejects a withdrawn or replaced registration. A registration disposer closes admission, aborts accepted requests and joins provider settlement; providers must honor cancellation. No fallback selects a different recognizer or uploads audio. No runtime invariant companion is published because the registry is the sole source of provider and preparation observations. -When `settings` is present, the `voice-input` namespace persists provider and language choices; composition defaults apply before an override is saved. `configure()` updates only supplied fields. Provider-owned preparation is observed through complete `SpeechSnapshot` values; closing an observer never cancels preparation. +When `settings` is present, the `voice-input` namespace persists provider and language choices; composition defaults apply before an override is saved. `configure()` updates only supplied fields and rejects a language unsupported by the selected provider before saving. Providers advertise accepted language hints through `languages`; `resolve()` validates the selected hint before transcription. Provider-owned preparation is observed through complete `SpeechSnapshot` values; closing an observer never cancels preparation. The `./wave` helper validates canonical 16 kHz mono PCM16 WAV for the Remote consumer and the native recognition process. Both reject inconsistent headers and lengths before decoding samples. diff --git a/packages/experimental/speech-to-text/README.zh.md b/packages/experimental/speech-to-text/README.zh.md index 7f93de8983..46436194dd 100644 --- a/packages/experimental/speech-to-text/README.zh.md +++ b/packages/experimental/speech-to-text/README.zh.md @@ -25,7 +25,7 @@ kind: "package-reference" ## 使用此包 -通过[语音输入 Bundle](../voice-input-bundle/README.zh.md)加载,或组合此服务、Provider 与消费者。`defaultProvider` 选择确切的注册 id;`language` 提供省略的语言提示。缺失或重复的 Provider 会明确报错。 +通过[语音输入 Bundle](../voice-input-bundle/README.zh.md)加载,或组合此服务、Provider 与消费者。`defaultProvider` 为必填项,选择确切的注册 id;Bundle 提供 `sensevoice-local`。`language` 提供省略的语言提示。缺失或重复的 Provider 会明确报错。 ----- @@ -37,7 +37,7 @@ kind: "package-reference" `resolve()` 捕获 Provider 实例、录音和语言。`transcribe()` 拒绝已撤销或替换的注册。注册的清理函数先拒绝新请求,再取消已接收的请求并等待 Provider 完成;Provider 必须响应取消。服务不会回退到其他识别器或上传音频。不发布运行时不变量伴随模块,因为注册表是 Provider 与准备状态观测的唯一来源。 -存在 `settings` 时,`voice-input` 命名空间持久化 Provider 与语言选择;保存覆盖值之前使用组合配置默认值。`configure()` 只更新传入的字段。观察者通过完整 `SpeechSnapshot` 接收 Provider 拥有的准备状态;关闭观察者不会取消准备。 +存在 `settings` 时,`voice-input` 命名空间持久化 Provider 与语言选择;保存覆盖值之前使用组合配置默认值。`configure()` 只更新传入的字段,并在保存前拒绝所选 Provider 不支持的语言。Provider 通过 `languages` 公布支持的语言提示;`resolve()` 在转写前校验所选提示。观察者通过完整 `SpeechSnapshot` 接收 Provider 拥有的准备状态;关闭观察者不会取消准备。 `./wave` 辅助函数为 Remote 消费者与原生识别进程校验规范的 16 kHz 单声道 PCM16 WAV。两者均在解码采样前拒绝不一致的头部和长度。 diff --git a/packages/experimental/speech-to-text/src/index.ts b/packages/experimental/speech-to-text/src/index.ts index 875a8a8d42..d265d73b2d 100644 --- a/packages/experimental/speech-to-text/src/index.ts +++ b/packages/experimental/speech-to-text/src/index.ts @@ -32,7 +32,7 @@ interface Registration { /** Registry shared by all transcription consumers in one Host composition. */ export default class SpeechToText extends Service { static Config: z = z.object({ - defaultProvider: z.string().min(1).default('sensevoice-local'), + defaultProvider: z.string().min(1).required(), language: z.string().min(1).default('auto'), }) @@ -130,19 +130,28 @@ export default class SpeechToText extends Service { } /** - * Persist changed preference fields in the ordinary user-settings document. + * Persist changed preference fields; the resulting language must be accepted by the selected provider. * @param patch - explicit provider or language changes. * @returns after persistence and the resolved preference update. */ async configure(patch: SpeechSelectionPatch): Promise { if (!this.preferences) throw new Error('Speech preferences require the user-settings service') - if (patch.providerId !== undefined && !this.providers.has(patch.providerId)) { - throw new Error(`Speech provider is unavailable: ${patch.providerId}`) - } + const current = this.preferences.get() + const id = patch.providerId ?? current.defaultProvider as SpeechProviderId + this.selectedProvider(id, patch.language ?? current.language) await this.preferences.update({ ...patch.providerId === undefined ? {} : { defaultProvider: patch.providerId }, ...patch.language === undefined ? {} : { language: patch.language } }) } + private selectedProvider(id: SpeechProviderId, language: string): SpeechProvider { + const registration = this.providers.get(id) + if (!registration) throw new Error(`Speech provider is unavailable: ${id}`) + if (!registration.provider.info.languages.includes(language)) { + throw new Error(`Speech provider ${id} does not support language: ${language}`) + } + return registration.provider + } + /** * Start or join provider-owned preparation. * @param id - exact registered provider identity. @@ -165,16 +174,15 @@ export default class SpeechToText extends Service { } /** - * Apply composition defaults and capture the selected provider. Missing providers fail explicitly. + * Apply composition defaults and capture the selected provider. Missing providers and unsupported languages fail explicitly. * @param request - complete recording and optional selection. * @returns provider-pinned input for transcribe(). */ resolve(request: SpeechRequest): SpeechSpec { const config = this.preferences?.get() ?? this.config const id = request.providerId ?? config.defaultProvider as SpeechProviderId - const registration = this.providers.get(id) - if (!registration) throw new Error(`Speech provider is unavailable: ${id}`) - return { provider: registration.provider, audio: request.audio, language: request.language ?? config.language } + const language = request.language ?? config.language + return { provider: this.selectedProvider(id, language), audio: request.audio, language } } /** diff --git a/packages/experimental/speech-to-text/src/types.ts b/packages/experimental/speech-to-text/src/types.ts index d96bcca650..47a246a461 100644 --- a/packages/experimental/speech-to-text/src/types.ts +++ b/packages/experimental/speech-to-text/src/types.ts @@ -17,6 +17,8 @@ export interface SpeechProviderInfo { readonly id: SpeechProviderId readonly name: string readonly location: 'host-local' | 'cloud' + /** Accepted language hints, including automatic detection when supported. */ + readonly languages: readonly string[] readonly setupEstimate?: SpeechSetupEstimate } @@ -34,7 +36,7 @@ export interface SpeechPreparationStep { export type SpeechPreparationState = ( | { readonly phase: 'unprepared' | 'ready' | 'standby' | 'cancelled' } | { readonly phase: 'downloading'; readonly resource: string; readonly completedBytes: number; readonly totalBytes?: number } - | { readonly phase: 'checking' | 'installing' | 'loading' | 'waking' | 'cancelling'; readonly startedAt: number } + | { readonly phase: 'checking' | 'loading' | 'waking' | 'cancelling'; readonly startedAt: number } | { readonly phase: 'failed'; readonly message: string } ) & { readonly step?: SpeechPreparationStepKind diff --git a/packages/experimental/speech-to-text/tests/service.spec.ts b/packages/experimental/speech-to-text/tests/service.spec.ts index fa8fe2a5d9..6ab724a9a5 100644 --- a/packages/experimental/speech-to-text/tests/service.spec.ts +++ b/packages/experimental/speech-to-text/tests/service.spec.ts @@ -1,5 +1,6 @@ /** Provider routing, cancellation, and fiber-owned registration lifetimes. */ import { Context } from '@deepseek-ai/cordis' +import z from '@deepseek-ai/schemastery' import { describe, expect, it, vi } from 'vitest' import SpeechToText from '../src/index.ts' import type { SpeechPreparationState, SpeechProvider, SpeechProviderId, Transcript } from '../src/types.ts' @@ -9,11 +10,11 @@ const result: Transcript = { text: 'hello', audioSeconds: 1, inferenceSeconds: 0 const audio = new Uint8Array([1, 2]) const input = new AbortController().signal function provider(id: string, transcribe: SpeechProvider['transcribe'] = async () => result): SpeechProvider { - return { info: { id: id as SpeechProviderId, name: id, location: 'host-local' }, transcribe } + return { info: { id: id as SpeechProviderId, name: id, location: 'host-local', languages: ['auto', 'zh', 'en', 'ja'] }, transcribe } } it('shares Host preparation across observers and does not cancel work when observation ends', async () => { - const ctx = new Context(), base = ctx.plugin(SpeechToText) + const ctx = new Context(), base = ctx.plugin(SpeechToText, { defaultProvider: 'sensevoice-local', language: 'auto' }) await base try { const service = ctx.get('speechToText')!, id = 'local' as SpeechProviderId @@ -46,7 +47,7 @@ it('persists explicit provider and language changes and publishes them with read const ctx = new Context() const settings = ctx.plugin(MemorySettings) await settings - const base = ctx.plugin(SpeechToText) + const base = ctx.plugin(SpeechToText, { defaultProvider: 'sensevoice-local', language: 'auto' }) await base try { const speech = ctx.get('speechToText')! @@ -65,7 +66,7 @@ it('persists explicit provider and language changes and publishes them with read }) it('ends waiting observers and rejects registration after service disposal', async () => { - const ctx = new Context(), base = ctx.plugin(SpeechToText) + const ctx = new Context(), base = ctx.plugin(SpeechToText, { defaultProvider: 'sensevoice-local', language: 'auto' }) await base const service = ctx.get('speechToText')!, stream = service.follow(new AbortController().signal)[Symbol.asyncIterator]() await stream.next() @@ -138,7 +139,7 @@ describe('speech providers', () => { it('removes a registration with its contributing fiber', async () => { const ctx = new Context() - const base = ctx.plugin(SpeechToText) + const base = ctx.plugin(SpeechToText, { defaultProvider: 'sensevoice-local', language: 'auto' }) await base const fiber = ctx.inject(['speechToText'], (owner) => { owner.effect(() => owner.speechToText.register(provider('sensevoice-local'))) @@ -150,3 +151,44 @@ describe('speech providers', () => { await base.dispose() }) }) + +it('rejects unsupported language selections before persistence or transcription', async () => { + const ctx = new Context() + try { + await ctx.plugin(MemorySettings) + await ctx.plugin(SpeechToText, { defaultProvider: 'local', language: 'auto' }) + const speech = ctx.get('speechToText')!, transcribe = vi.fn(async () => result) + speech.register(provider('local', transcribe)) + const cloud = provider('cloud', transcribe) + speech.register({ ...cloud, info: { ...cloud.info, languages: ['fr'] } }) + await expect(speech.configure({ language: 'fr' })).rejects.toThrow('does not support language') + await expect(speech.configure({ providerId: cloud.info.id })).rejects.toThrow('does not support language') + expect((ctx.get('settings') as MemorySettings).doc['voice-input']).toBeUndefined() + expect(() => speech.resolve({ audio, language: 'fr' })).toThrow('does not support language') + expect(transcribe).not.toHaveBeenCalled() + await speech.configure({ providerId: cloud.info.id, language: 'fr' }) + expect(speech.resolve({ audio })).toMatchObject({ provider: { info: { id: 'cloud', languages: ['fr'] } }, language: 'fr' }) + } finally { await ctx.fiber.dispose() } +}) + +it('requires the composition to choose its default provider', () => { + expect(() => z.resolve({}, SpeechToText.Config, {})).toThrow() + expect(SpeechToText.Config({ defaultProvider: 'custom', language: 'auto' })).toEqual({ defaultProvider: 'custom', language: 'auto' }) +}) + +it('ends observers without fallback settings when the service unloads with Settings mounted', async () => { + const ctx = new Context() + try { + await ctx.plugin(MemorySettings) + const base = ctx.plugin(SpeechToText, { defaultProvider: 'local', language: 'auto' }) + await base + const speech = ctx.get('speechToText')! + speech.register(provider('local')) + await speech.configure({ language: 'zh' }) + const stream = speech.follow(new AbortController().signal)[Symbol.asyncIterator]() + expect((await stream.next()).value).toMatchObject({ selection: { language: 'zh' } }) + const waiting = stream.next() + await base.dispose() + expect(await waiting).toEqual({ done: true, value: undefined }) + } finally { await ctx.fiber.dispose() } +}) diff --git a/packages/experimental/voice-input-bundle/README.i18n.yaml b/packages/experimental/voice-input-bundle/README.i18n.yaml index 50860ef622..a97ade03fb 100644 --- a/packages/experimental/voice-input-bundle/README.i18n.yaml +++ b/packages/experimental/voice-input-bundle/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/experimental/voice-input-bundle/README.md -README.md: 31033a61aa474cb038d856fc013a314a237cba8a -README.zh.md: 869e5e2b97f40d740070c2c117c50a16f1d1a362 +README.md: 13984083581a6c574ab74cf5ef6361fcd8254119 +README.zh.md: 013ad4a0cf6edb27753fb4c1df3fb6952ea070f4 diff --git a/packages/experimental/voice-input-bundle/README.md b/packages/experimental/voice-input-bundle/README.md index 31033a61aa..1398408358 100644 --- a/packages/experimental/voice-input-bundle/README.md +++ b/packages/experimental/voice-input-bundle/README.md @@ -35,7 +35,7 @@ Open Plugins in the Web sidebar and enable Voice Input. If models need preparati
Maintainer details — click to expand -The static `cordis.patch.yml` adds the four voice rows and supplies the provider cache directory with `dshHomePath`. Optional-bundle installation makes the package available to management without selecting it in default profiles. The browser contribution owns its generated Remote mount; stable API Remotes do not import experimental code. No runtime invariant companion is published because this configuration-only package has no independently mutable runtime state. +The static `cordis.patch.yml` adds the four voice rows, selects `sensevoice-local` as the default recognizer and supplies the provider cache directory with `dshHomePath`. Optional-bundle installation makes the package available to management without selecting it in default profiles. The browser contribution owns its generated Remote mount; stable API Remotes do not import experimental code. No runtime invariant companion is published because this configuration-only package has no independently mutable runtime state.
diff --git a/packages/experimental/voice-input-bundle/README.zh.md b/packages/experimental/voice-input-bundle/README.zh.md index 869e5e2b97..013ad4a0cf 100644 --- a/packages/experimental/voice-input-bundle/README.zh.md +++ b/packages/experimental/voice-input-bundle/README.zh.md @@ -35,7 +35,7 @@ kind: "package-bundle"
维护者信息 — 点击展开 -静态 `cordis.patch.yml` 添加四个语音条目,并通过 `dshHomePath` 提供 Provider 缓存目录。可选 Bundle 安装使插件管理器能够发现此包,但不会在默认配置中选择它。浏览器贡献拥有其生成 Remote 的挂载;稳定 API Remotes 不导入实验性代码。此纯配置包没有独立可变的运行时状态,因此不发布不变量伴随模块。 +静态 `cordis.patch.yml` 添加四个语音条目,选择 `sensevoice-local` 作为默认识别器,并通过 `dshHomePath` 提供 Provider 缓存目录。可选 Bundle 安装使插件管理器能够发现此包,但不会在默认配置中选择它。浏览器贡献拥有其生成 Remote 的挂载;稳定 API Remotes 不导入实验性代码。此纯配置包没有独立可变的运行时状态,因此不发布不变量伴随模块。
diff --git a/packages/experimental/voice-input-bundle/cordis.patch.yml b/packages/experimental/voice-input-bundle/cordis.patch.yml index 8456e58334..255734df80 100644 --- a/packages/experimental/voice-input-bundle/cordis.patch.yml +++ b/packages/experimental/voice-input-bundle/cordis.patch.yml @@ -1,6 +1,8 @@ - insert: - id: speech-to-text name: '@deepseek-ai/dsh-experimental-speech-to-text' + config: + defaultProvider: sensevoice-local - id: speech-to-text-sensevoice name: '@deepseek-ai/dsh-experimental-speech-to-text-sensevoice' config: diff --git a/packages/extensions/tool-cordis/src/api-catalog.ts b/packages/extensions/tool-cordis/src/api-catalog.ts index 295ea148ce..33fada01d0 100644 --- a/packages/extensions/tool-cordis/src/api-catalog.ts +++ b/packages/extensions/tool-cordis/src/api-catalog.ts @@ -2549,7 +2549,7 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ }, { signature: 'async configure(patch: SpeechSelectionPatch): Promise', - description: 'Persist changed preference fields in the ordinary user-settings document.', + description: 'Persist changed preference fields; the resulting language must be accepted by the selected provider.', parameters: [{ name: 'patch', description: 'explicit provider or language changes.' }], returns: 'after persistence and the resolved preference update.', }, @@ -2566,7 +2566,7 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ }, { signature: 'resolve(request: SpeechRequest): SpeechSpec', - description: 'Apply composition defaults and capture the selected provider. Missing providers fail explicitly.', + description: 'Apply composition defaults and capture the selected provider. Missing providers and unsupported languages fail explicitly.', parameters: [{ name: 'request', description: 'complete recording and optional selection.' }], returns: 'provider-pinned input for transcribe().', }, @@ -6558,7 +6558,7 @@ export const TYPE_API: readonly TypeApiEntry[] = [ }, { name: 'SpeechPreparationState', - declaration: 'export type SpeechPreparationState = ({\n readonly phase: \'unprepared\' | \'ready\' | \'standby\' | \'cancelled\';\n} | {\n readonly phase: \'downloading\';\n readonly resource: string;\n readonly completedBytes: number;\n readonly totalBytes?: number;\n} | {\n readonly phase: \'checking\' | \'installing\' | \'loading\' | \'waking\' | \'cancelling\';\n readonly startedAt: number;\n} | {\n readonly phase: \'failed\';\n readonly message: string;\n}) & {\n readonly step?: SpeechPreparationStepKind;\n readonly steps?: readonly SpeechPreparationStep[];\n};', + declaration: 'export type SpeechPreparationState = ({\n readonly phase: \'unprepared\' | \'ready\' | \'standby\' | \'cancelled\';\n} | {\n readonly phase: \'downloading\';\n readonly resource: string;\n readonly completedBytes: number;\n readonly totalBytes?: number;\n} | {\n readonly phase: \'checking\' | \'loading\' | \'waking\' | \'cancelling\';\n readonly startedAt: number;\n} | {\n readonly phase: \'failed\';\n readonly message: string;\n}) & {\n readonly step?: SpeechPreparationStepKind;\n readonly steps?: readonly SpeechPreparationStep[];\n};', }, { name: 'SpeechPreparationStep', @@ -6578,7 +6578,7 @@ export const TYPE_API: readonly TypeApiEntry[] = [ }, { name: 'SpeechProviderInfo', - declaration: 'export interface SpeechProviderInfo {\n readonly id: SpeechProviderId;\n readonly name: string;\n readonly location: \'host-local\' | \'cloud\';\n readonly setupEstimate?: SpeechSetupEstimate;\n}', + declaration: 'export interface SpeechProviderInfo {\n readonly id: SpeechProviderId;\n readonly name: string;\n readonly location: \'host-local\' | \'cloud\';\n readonly languages: readonly string[];\n readonly setupEstimate?: SpeechSetupEstimate;\n}', }, { name: 'SpeechProviderView',