diff --git a/.agents/notes/implemented/process/2026-10-05-pr-artifact-typechecks.i18n.yaml b/.agents/notes/implemented/process/2026-10-05-pr-artifact-typechecks.i18n.yaml new file mode 100644 index 0000000000..79f3339e71 --- /dev/null +++ b/.agents/notes/implemented/process/2026-10-05-pr-artifact-typechecks.i18n.yaml @@ -0,0 +1,19 @@ +# Bilingual-pair consistency record for 2026-10-05-pr-artifact-typechecks.md (docs/i18n/README.md): per heading +# section, a hash of its English and Chinese blocks outside code blocks and generated regions. +# After editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/implemented/process/2026-10-05-pr-artifact-typechecks.md +/agent-note-pr-artifact-builds-and-aggregate-typescript-checks: + en: a25e7cdcad03934f + zh: 48e31429730e0885 +/agent-note-pr-artifact-builds-and-aggregate-typescript-checks/problem: + en: e5c984fa32cc5e69 + zh: 25d5742b2106d830 +/agent-note-pr-artifact-builds-and-aggregate-typescript-checks/decision: + en: 6e84bceaba4573ca + zh: 25a37c147a675643 +/agent-note-pr-artifact-builds-and-aggregate-typescript-checks/alternatives-considered: + en: f61da066f9ddcd62 + zh: 816fd6e0063017cd +/agent-note-pr-artifact-builds-and-aggregate-typescript-checks/consequences: + en: 46c247ab91126d52 + zh: f1be7561ff402bcc diff --git a/.agents/notes/implemented/process/2026-10-05-pr-artifact-typechecks.md b/.agents/notes/implemented/process/2026-10-05-pr-artifact-typechecks.md new file mode 100644 index 0000000000..7577a7c06e --- /dev/null +++ b/.agents/notes/implemented/process/2026-10-05-pr-artifact-typechecks.md @@ -0,0 +1,45 @@ +# Agent Note: PR artifact builds and aggregate TypeScript checks + +Status: implemented + +English | [中文](2026-10-05-pr-artifact-typechecks.zh.md) + +## Problem + +PR jobs need the same compiled packages for benchmarks, compatibility, packaging, and runtime checks. Building the repository-wide test and script programs in every artifact job repeats expensive diagnostics. In the allowed #3661 baseline, benchmark build preparation takes about 304 seconds before 432 seconds of serial measurements. A cold macOS arm64 Host compiler profile on Node 24.19.0 takes 84.01 seconds, including 55.92 seconds in the aggregate program; its 282 emitted projects take 24.33 seconds combined. Increasing concurrency consumes more shared-runner resources. + +## Decision + +Required Linux consumer and native Windows build jobs retain normal builds, including both complete aggregate programs. Public `build` and `typecheck` keep the same checks. [Node version ownership](2026-07-06-node-engine-floor.md) and [native Windows ownership](2026-08-08-native-windows-pull-request-ci.md) remain in force. + +Checked PR artifact builds compile every existing project reference from each compiler face with one serial `tsc -b` process, then run the ordinary Typert, bundling, Desktop, and Web stages. The helper reads the existing Host and Client aggregates rather than maintaining another package list. It omits only each root's additional test and script program; package diagnostics, emitted outputs, and Host-before-Client ordering remain required. Non-PR release and real-API workflow builds retain their normal checks. + +Only the CI benchmark helper selects `tsc -b --noCheck` for package emission. Benchmarks consume JavaScript and Typert runtime metadata; their declaration and build-information files are disposable and never published. Typert analyzes source types independently, while required complete builds own semantic diagnostics. Syntax and configuration failures still reject benchmark preparation. TypeScript records pending checks so a later ordinary incremental build diagnoses the same source without deleting caches or forcing a rebuild. Public benchmark scripts and every release, Python, preview, compatibility, and normal artifact builder keep their checked defaults. + +The Python executable builder selects artifact compilation with `--artifacts-only` after verifying its runtime closure. Invalid closure inputs fail before compilation, deployment, or packing; the default complete build and prebuilt `--skip-build` mode keep their existing stages. + +CI invokes the internal build helpers directly and keeps the root package manifest unchanged. Root manifest edits activate the separate native-addon platform matrix even when only build aliases change; avoiding that trigger preserves the existing workflow set and concurrent-job ceiling without weakening native validation. + +Job inventory, runner selectors, and benchmark sample counts stay fixed; gate and worker ceilings do not increase. The three compatibility cases run one job at a time; the two-target Python runtime matrix starts after the keyless Python SDK job succeeds. Four Linux lanes, the Python SDK, three Windows lanes, and one compatibility job permit at most nine worker jobs initially. Once the SDK job finishes, the remaining seven independent lanes, one compatibility job, and two runtime targets permit at most ten. The runtime plan and wheel stages precede those targets and each use one job. This permits runtime validation to overlap the Node 22 build within the original ten-job ceiling. Coverage partitions reuse compact file-cost metadata within their platform and runner pool. Parallel release packing uses pnpm's existing bounded scheduler; serial publication keeps family order. + +Each Node version runs the same seven source compatibility specs in one serial Vitest invocation. Both file parallelism and the CLI worker limit are fixed to one, and the gate sets `VITEST_MAX_WORKERS=1` because that environment variable overrides Vitest's CLI limit. Files retain fork isolation; the build-backed Node 22 lazy-search smoke keeps its separate build dependency. Three paired local samples preserve all 253 assertion outcomes and reduce startup-only median time from 11.52 to 8.69 seconds, without increasing owned descendant-process peak. + +## Alternatives considered + +**Raise job or worker concurrency.** Rejected because the shared runner must keep its current resource ceiling. + +**Remove aggregate checks from every build.** Rejected because package projects do not typecheck all repository tests and scripts; required Linux and Windows jobs must retain those diagnostics. + +**Cache compiled outputs across revisions.** Deferred because cache correctness and invalidation add obligations that removing duplicate aggregate work does not need. + +## Consequences + +The cold Host reference build takes 23.49 seconds with 1.91 GiB peak RSS, compared with 84.01 seconds and 3.87 GiB for the complete compiler pass. All 5,932 emitted files match by path and SHA-256. These are single local compiler samples, excluding bundling and tests; current-head CI owns the PR latency result. The benchmark scenarios and their budgets remain unchanged. + +Three alternating cold preparation pairs on Apple M4 Pro / Node 24.19.0 compare checked reference compilation with benchmark-only emission: 61.16, 82.53, and 57.80 seconds versus 55.53, 50.82, and 50.75 seconds. Median preparation falls from 61.16 to 50.82 seconds (16.9%); this includes native support, libraries, benchmark workers, and Web assets, excluding dependency installation and measurements. Every pair has the same 11,894 output paths, including 3,181 byte-identical JavaScript files and unchanged source maps. The 356 build-information files differ, and nine declarations only reorder union members; those files are not published from the benchmark lane. Library-build median user CPU falls from 76.45 to 64.23 seconds, while median maximum RSS stays near 2.75 GiB. Required CI determines the complete PR timing. + +Parallel packing relies on the pinned pnpm CLI to honor exact recursive filters, the pack destination, the workspace concurrency limit, and each package's lifecycle hooks. A pnpm upgrade must pass the real-CLI checks in `scripts/release/pack.spec.ts` before release packing adopts it. + +Coverage weights sum Vitest's distinct environment, preparation, setup, collection, and execution costs. CI's platform/environment/pool cache prefixes exclude the previous `coverage-times-` archives. Persistent local checkouts can retain wall-clock weights for files not measured during rollout; removing `.coverage-times.json` resets that history. These advisory weights affect partition assignment, while the complete inventory and merged coverage thresholds remain required. + +Checked artifact builds deliberately do not diagnose aggregate-only test or script errors. Required normal builds reject those errors; fixture checks also reject package errors on both paths and compare emitted JavaScript, declarations, maps, and build information. Removing or changing a required full build requires transferring both aggregate checks to another blocking owner first. Master serial references remain complete under their [existing policy](2026-07-21-serial-cross-platform-ci-reference.md). diff --git a/.agents/notes/implemented/process/2026-10-05-pr-artifact-typechecks.zh.md b/.agents/notes/implemented/process/2026-10-05-pr-artifact-typechecks.zh.md new file mode 100644 index 0000000000..c0b3c1bcf4 --- /dev/null +++ b/.agents/notes/implemented/process/2026-10-05-pr-artifact-typechecks.zh.md @@ -0,0 +1,45 @@ +# Agent Note: PR 产物构建与 aggregate TypeScript 检查 + +Status: implemented + +[English](2026-10-05-pr-artifact-typechecks.md) | 中文 + +## Problem + +PR job 需要相同的编译包来运行 benchmark、兼容性、打包和运行时检查。每个产物 job 都构建仓库级测试和脚本 program,会重复执行昂贵的诊断。在允许使用的 #3661 baseline 中,benchmark 在 432 秒的串行测量前花费约 304 秒准备构建。在 macOS arm64、Node 24.19.0 上,一次冷启动 Host 编译器 profile 耗时 84.01 秒,其中 aggregate program 占 55.92 秒;282 个有 emit 的 project 合计耗时 24.33 秒。提高并发会消耗更多共享运行器资源。 + +## Decision + +必需的 Linux consumer 与原生 Windows build job 保留普通构建,包括两个完整 aggregate program。公开的 `build` 和 `typecheck` 保留相同检查。[Node 版本职责](2026-07-06-node-engine-floor.zh.md)与[原生 Windows 职责](2026-08-08-native-windows-pull-request-ci.zh.md)仍然有效。 + +带检查的 PR 产物构建通过一个串行 `tsc -b` 进程编译各 compiler face 的全部现有 project reference,然后运行普通 Typert、bundling、Desktop 和 Web 阶段。helper 读取现有 Host 与 Client aggregate,而不维护另一份包清单。它只省略各 root 附加的测试和脚本 program;包诊断、emit 输出及 Host 先于 Client 的顺序仍然必需。非 PR 的 release 和真实 API 工作流构建保留普通检查。 + +只有 CI benchmark helper 选择 `tsc -b --noCheck` 来 emit 包。benchmark 消费 JavaScript 和 Typert 运行时元数据;其声明与 build-information 文件是临时产物,不会发布。Typert 独立分析源码类型,语义诊断由必需的完整构建负责。语法与配置错误仍会拒绝 benchmark 准备。TypeScript 记录待检查状态,使后续普通增量构建无需删除缓存或强制重建,就能诊断相同源码。公开 benchmark script,以及所有 release、Python、preview、compatibility 和普通产物 builder,均保留带检查的默认行为。 + +Python executable builder 在验证 runtime closure 后,通过 `--artifacts-only` 选择产物编译。无效 closure 在编译、部署或打包前失败;默认完整构建和复用预构建产物的 `--skip-build` 模式保留原有阶段。 + +CI 直接调用内部 build helper,并保持 root package manifest 不变。即使只修改 build alias,root manifest 改动也会激活独立的 native-addon 平台矩阵;避免这一触发条件,可以在不削弱 native 验证的情况下保留现有 workflow 集合与并发 job 上限。 + +job 清单、runner 选择和 benchmark sample 数量保持固定;gate 与 worker 上限不提高。三个 compatibility case 每次运行一个 job;双目标 Python runtime 矩阵在 keyless Python SDK job 成功后启动。四个 Linux lane、Python SDK、三个 Windows lane 和一个 compatibility job,初始最多占用九个 worker job。SDK job 完成后,剩余七个独立 lane、一个 compatibility job 和两个 runtime target,最多占用十个 job。runtime plan 与 wheel 阶段先于这些 target,各占一个 job。这允许 runtime 验证与 Node 22 构建重叠,同时保持原有十个 job 的上限。覆盖率分区在各自的平台和 runner pool 内复用精简的文件成本元数据。并行 release 打包使用 pnpm 现有的有界调度器;串行发布保持 family 顺序。 + +每个 Node 版本在一次串行 Vitest 调用中运行相同的七个 source compatibility spec。文件并行和 CLI worker 上限均固定为一,gate 还设置 `VITEST_MAX_WORKERS=1`,因为该环境变量会覆盖 Vitest 的 CLI 上限。文件保持 fork 隔离;依赖构建的 Node 22 lazy-search smoke 保留独立的 build 依赖。三对本地样本保持全部 253 个 assertion outcome,将启动开销比较的 median 从 11.52 秒降至 8.69 秒,且不增加所拥有 descendant process 的峰值。 + +## Alternatives considered + +**提高 job 或 worker 并发。** 否决,因为共享运行器必须保持现有资源上限。 + +**从所有构建中移除 aggregate 检查。** 否决,因为包 project 不会检查所有仓库测试和脚本的类型;必需的 Linux 与 Windows job 必须保留这些诊断。 + +**跨 revision 缓存编译输出。** 暂缓,因为缓存正确性与失效规则增加了额外义务,而移除重复 aggregate 工作无需承担这些义务。 + +## Consequences + +冷启动 Host reference 构建耗时 23.49 秒、峰值 RSS 为 1.91 GiB;完整编译器阶段为 84.01 秒和 3.87 GiB。全部 5,932 个 emit 文件的路径与 SHA-256 相同。这些是单次本地编译器样本,不含 bundling 和测试;PR 延迟结果以当前 head 的 CI 为准。benchmark 场景与预算保持不变。 + +在 Apple M4 Pro / Node 24.19.0 上,三对交替冷启动准备构建比较带检查的 reference 编译与仅用于 benchmark 的 emit:前者为 61.16、82.53、57.80 秒,后者为 55.53、50.82、50.75 秒。准备阶段中位数从 61.16 降至 50.82 秒(16.9%);范围包括 native support、library、benchmark worker 和 Web 产物,不含依赖安装与测量。每对都包含相同的 11,894 个输出路径,其中 3,181 个 JavaScript 文件逐字节相同,source map 不变。356 个 build-information 文件不同,九个声明仅调整 union 成员顺序;benchmark lane 不会发布这些文件。library 构建的 user CPU 中位数从 76.45 降至 64.23 秒,maximum RSS 中位数保持在约 2.75 GiB。完整 PR 耗时以必需 CI 为准。 + +并行打包依赖锁定的 pnpm CLI 正确处理精确 recursive filter、pack destination、workspace 并发上限及各包的 lifecycle hook。升级 pnpm 时,必须先通过 `scripts/release/pack.spec.ts` 中的真实 CLI 检查,再让 release 打包采用该版本。 + +覆盖率权重累加 Vitest 互不重叠的 environment、preparation、setup、collection 和 execution 成本。CI 的平台/environment/pool cache prefix 不匹配原来的 `coverage-times-` archive。持久化本地 checkout 在切换期间可能为未测量的文件保留墙钟权重;删除 `.coverage-times.json` 可重置该历史。这些参考权重影响分区分配,完整清单与合并后的覆盖率阈值仍然必需。 + +带检查的产物构建有意不诊断仅属于 aggregate 的测试或脚本错误。必需的普通构建拒绝这些错误;fixture 检查还验证两个路径都会拒绝包错误,并比较 emit 的 JavaScript、声明、map 和 build 信息。删除或改变必需的完整构建前,必须先把两个 aggregate 检查转交另一处阻塞型 owner。master 串行 reference 按[现有政策](2026-07-21-serial-cross-platform-ci-reference.zh.md)保持完整。 diff --git a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.i18n.yaml b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.i18n.yaml index 4bf28f6452..d03a5ff8c5 100644 --- a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.i18n.yaml @@ -9,8 +9,8 @@ en: 00ba1f66195351e2 zh: a738b81adceda351 /agent-note-required-ci-performance-gate-for-opening-large-sessions/decision: - en: 0a483af531d0dc46 - zh: 466a489dcb53dc60 + en: 4f94b61c48cceaf4 + zh: dbb51bfe1fb0f681 /agent-note-required-ci-performance-gate-for-opening-large-sessions/calibration-evidence: en: f1001a139470f6b7 zh: b7b46f96b84b1271 diff --git a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.md b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.md index dd1d0d876e..926203ea5b 100644 --- a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.md +++ b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.md @@ -12,7 +12,7 @@ Measuring only `SessionPersistence.open()` does not stably describe the result f ## Decision -Linux pull requests run a required `node 24 / benchmarks` job that executes `pnpm run check:ci:bench` → `pnpm run test:bench`. The private `@deepseek-ai/dsh-benchmarks` workspace owns benchmark-only dependencies. The command first builds workspace libraries and dedicated workers under `benchmarks/.dsh-build/`, then invokes `vitest.bench.config.ts`. The [standard hosted runner reference](../../../../.github/workflows/ci.yml) owns runner selection and the outer job timeout. The job runs the benchmark lane alone; Vitest runs one file at a time and only prepares input, starts measurement children, aggregates results, and enforces budgets. Every timed Node CPU path executes compiled JavaScript under plain Node with `NODE_OPTIONS` removed and no TypeScript loader; bare workspace imports therefore resolve from `benchmarks/node_modules` through package exports to built `lib/` entries. +Linux pull requests run a required `node 24 / benchmarks` job that executes `pnpm run check:ci:bench` → `scripts/run-ci-bench.ts`. Its disposable library build omits repeated TypeScript diagnostics under the [PR artifact build policy](../process/2026-10-05-pr-artifact-typechecks.md); required full builds retain them. The private `@deepseek-ai/dsh-benchmarks` workspace owns benchmark-only dependencies. The command first builds workspace libraries and dedicated workers under `benchmarks/.dsh-build/`, then invokes `vitest.bench.config.ts`. The [standard hosted runner reference](../../../../.github/workflows/ci.yml) owns runner selection and the outer job timeout. The job runs the benchmark lane alone; Vitest runs one file at a time and only prepares input, starts measurement children, aggregates results, and enforces budgets. Every timed Node CPU path executes compiled JavaScript under plain Node with `NODE_OPTIONS` removed and no TypeScript loader; bare workspace imports therefore resolve from `benchmarks/node_modules` through package exports to built `lib/` entries. Required performance gates live under top-level `benchmarks/`, grouped by measured user path rather than package ownership. Host files use `*.bench.ts`, Client-face files use `*.bench.client.ts`, and scenario-specific workers and fixtures stay beside their benchmark without a benchmark suffix. Package-local `.perf.ts` files remain non-gating diagnostics; `scripts/` owns orchestration rather than benchmark cases. diff --git a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.zh.md b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.zh.md index 226d3a709f..e8a64dca20 100644 --- a/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.zh.md +++ b/.agents/notes/implemented/testing/2026-09-04-session-open-performance-gate.zh.md @@ -12,7 +12,7 @@ Session format v2 的推出改变了两条成本随模型输出增长的路径 ## 决定 -Linux pull request 运行必需的 `node 24 / benchmarks` job,执行 `pnpm run check:ci:bench` → `pnpm run test:bench`。私有 `@deepseek-ai/dsh-benchmarks` workspace 拥有 benchmark 专属依赖。该命令先构建 workspace library 和 `benchmarks/.dsh-build/` 下的专用 worker,再调用 `vitest.bench.config.ts`。[标准托管运行器说明](../../../../.github/workflows/ci.yml)拥有运行器选择及外层 job 超时。该 job 单独运行 benchmark lane;Vitest 逐文件运行,只负责准备输入、启动测量子进程、汇总结果和执行预算断言。每条被计时的 Node CPU 路径都以纯 Node 执行编译后的 JavaScript,并移除 `NODE_OPTIONS` 且不加载 TypeScript runtime;workspace 裸导入因此从 `benchmarks/node_modules` 通过 package exports 解析到构建后的 `lib/` 入口。 +Linux pull request 运行必需的 `node 24 / benchmarks` job,执行 `pnpm run check:ci:bench` → `scripts/run-ci-bench.ts`。其临时 library 构建按 [PR 产物构建政策](../process/2026-10-05-pr-artifact-typechecks.zh.md)省略重复的 TypeScript 诊断;必需的完整构建保留这些诊断。私有 `@deepseek-ai/dsh-benchmarks` workspace 拥有 benchmark 专属依赖。该命令先构建 workspace library 和 `benchmarks/.dsh-build/` 下的专用 worker,再调用 `vitest.bench.config.ts`。[标准托管运行器说明](../../../../.github/workflows/ci.yml)拥有运行器选择及外层 job 超时。该 job 单独运行 benchmark lane;Vitest 逐文件运行,只负责准备输入、启动测量子进程、汇总结果和执行预算断言。每条被计时的 Node CPU 路径都以纯 Node 执行编译后的 JavaScript,并移除 `NODE_OPTIONS` 且不加载 TypeScript runtime;workspace 裸导入因此从 `benchmarks/node_modules` 通过 package exports 解析到构建后的 `lib/` 入口。 必需性能 gate 位于顶层 `benchmarks/`,按被测用户路径而非 package 归属组织。Host 文件使用 `*.bench.ts`,Client 面文件使用 `*.bench.client.ts`,场景专属 worker 与 fixture 留在对应 benchmark 旁且不带 benchmark 后缀。包内 `.perf.ts` 文件仍是非门禁诊断;`scripts/` 负责编排而不承载 benchmark case。 diff --git a/.github/workflows/build-exe-for-python-sdk.yml b/.github/workflows/build-exe-for-python-sdk.yml index e53d524ead..4dfad5492a 100644 --- a/.github/workflows/build-exe-for-python-sdk.yml +++ b/.github/workflows/build-exe-for-python-sdk.yml @@ -259,7 +259,9 @@ jobs: - name: Build single-exe env: DSH_BUILD_CLIENT_PROFILE: official - run: pnpm exec tsx scripts/build-exe-for-python-sdk.ts --targets=${{ matrix.target }} + run: >- + pnpm exec tsx scripts/build-exe-for-python-sdk.ts --targets=${{ matrix.target }} + ${{ inputs.ci && github.event_name == 'pull_request' && '--artifacts-only' || '' }} - name: Resolve platform outputs (POSIX) id: runtime-posix diff --git a/.github/workflows/build-preview-cloudflare.yml b/.github/workflows/build-preview-cloudflare.yml index 80caebdb76..1b5ee1bde5 100644 --- a/.github/workflows/build-preview-cloudflare.yml +++ b/.github/workflows/build-preview-cloudflare.yml @@ -69,9 +69,9 @@ jobs: # apps/web consumes workspace packages as built lib products, and # build:preview packs the image through the packer's installed bin - # (lib/bin.js), so neither half exists before the full build runs. + # (lib/bin.js), so neither half exists before the artifact build runs. - name: Build workspace - run: pnpm run build + run: pnpm run build --artifacts-only - name: Build the preview page and pack the VFS image env: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fdcec2540d..61b65fd893 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,6 +141,14 @@ jobs: with: persist-credentials: false + - name: Restore coverage duration history + uses: actions/cache/restore@v4 + with: + path: .coverage-times.json + key: coverage-times-Linux-${{ runner.environment }}-${{ vars.DSH_CI_FAILOVER_LINUX || 'enterprise' }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + coverage-times-Linux-${{ runner.environment }}-${{ vars.DSH_CI_FAILOVER_LINUX || 'enterprise' }}- + # Redirect the Node compile cache (enabled by pnpm and TypeScript) off # the root partition's /tmp before the first pnpm call in this lane. - name: Redirect Node compile cache to runner temp @@ -188,6 +196,13 @@ jobs: - name: Run exhaustive coverage run: pnpm run check:ci:coverage + - name: Save coverage duration history + if: ${{ !cancelled() }} + uses: actions/cache/save@v4 + with: + path: .coverage-times.json + key: coverage-times-Linux-${{ runner.environment }}-${{ vars.DSH_CI_FAILOVER_LINUX || 'enterprise' }}-${{ github.run_id }}-${{ github.run_attempt }} + node-24-bench: if: github.event_name == 'pull_request' runs-on: ubuntu-24.04 @@ -383,6 +398,7 @@ jobs: DSH_GATE_FAIL_FAST: '1' strategy: fail-fast: false + max-parallel: 1 matrix: include: - node: '22.19' @@ -483,6 +499,7 @@ jobs: # Linux ARM64 and both macOS architectures run in ci-master.yml. python-runtime: if: github.event_name == 'pull_request' + needs: python-sdk name: python runtime / release-shaped matrix uses: ./.github/workflows/build-exe-for-python-sdk.yml with: @@ -592,9 +609,9 @@ jobs: uses: actions/cache/restore@v4 with: path: .coverage-times.json - key: coverage-times-${{ github.run_id }} + key: coverage-times-Windows-${{ runner.environment }}-${{ vars.DSH_CI_FAILOVER_WINDOWS || 'enterprise' }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | - coverage-times- + coverage-times-Windows-${{ runner.environment }}-${{ vars.DSH_CI_FAILOVER_WINDOWS || 'enterprise' }}- - name: Enable Developer Mode (symlink support) shell: pwsh run: >- @@ -635,7 +652,7 @@ jobs: uses: actions/cache/save@v4 with: path: .coverage-times.json - key: coverage-times-${{ github.run_id }} + key: coverage-times-Windows-${{ runner.environment }}-${{ vars.DSH_CI_FAILOVER_WINDOWS || 'enterprise' }}-${{ github.run_id }}-${{ github.run_attempt }} windows-native-tests: if: github.event_name == 'pull_request' diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8b3bfc1400..6fb50bde2d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -107,7 +107,7 @@ jobs: # the built artifact under plain Node, resolving plugins through real package # exports — the shape a real consumer runs. That requires a prior build. - name: Build (lib for the e2e example bins) - run: pnpm run build:official + run: ${{ github.event_name == 'pull_request' && 'pnpm run build --artifacts-only --profile official' || 'pnpm run build:official' }} - name: Prepare Office runtime run: pnpm run prepare:primary-runtime --target linux-x64 --output .cache/e2e-office diff --git a/.github/workflows/release-vendor.yml b/.github/workflows/release-vendor.yml index 75fb865767..bf0bdf082e 100644 --- a/.github/workflows/release-vendor.yml +++ b/.github/workflows/release-vendor.yml @@ -89,7 +89,7 @@ jobs: # The vendored packages publish their own sources and build outputs; the # host build produces what their manifests select. - name: Build - run: pnpm run build:lib:host + run: ${{ github.event_name == 'pull_request' && 'pnpm exec tsx scripts/compile-referenced-projects.ts host-libraries' || 'pnpm run build:lib:host' }} # Concurrency here is rehearsal-only: the credentialed publish workflows # invoke release:pack without the flag and keep the strictly serial path. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 633186c8f1..ba151e22e9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -146,7 +146,7 @@ jobs: run: pnpm run release:verify --family dsh - name: Build - run: pnpm run build:official + run: ${{ github.event_name == 'pull_request' && 'pnpm run build --artifacts-only --profile official' || 'pnpm run build:official' }} # Concurrency here is rehearsal-only: the credentialed publish workflows # invoke release:pack without the flag and keep the strictly serial path. diff --git a/docs/development.i18n.yaml b/docs/development.i18n.yaml index 904fdd17cb..e3adf54b3e 100644 --- a/docs/development.i18n.yaml +++ b/docs/development.i18n.yaml @@ -21,8 +21,8 @@ en: 3221d1076554001f zh: 29f166be6a5c04d9 /development-guide/contributor-reference/typescript-project-layout: - en: 42c03e89df4379e3 - zh: 9df43176aa64d047 + en: e3327c5492a593cb + zh: d43a5edcb11523ae /development-guide/contributor-reference/environment-variables: en: dba2c72ee6ffa930 zh: b65c27fbb9129ec8 @@ -30,8 +30,8 @@ en: dc98eda4f4021feb zh: b08766708803cf10 /development-guide/contributor-reference/ci-gates: - en: 8fa1aca14c548969 - zh: a2bd1199fa7f7a8b + en: b64a914e41b1d285 + zh: 6c031b507c3c0e84 /development-guide/contributor-reference/daily-commands: en: 08aa6ec1cdabfeb7 zh: dcce5ae9d3c829d0 diff --git a/docs/development.md b/docs/development.md index d2f3ff337f..734ac9aea6 100644 --- a/docs/development.md +++ b/docs/development.md @@ -85,6 +85,8 @@ tsdown --env.DSH_BUILD_FACE client pnpm run build:web ``` +`pnpm run build --artifacts-only` emits the same project and application artifacts while checking and compiling only the existing package references of each compiler face. Required Linux and Windows normal builds and public `typecheck` also check the repository-wide test and script programs. CI benchmark preparation additionally uses `--noCheck` for its disposable library outputs, relying on those required builds for diagnostics; other artifact builders keep package checking ([rationale](../.agents/notes/implemented/process/2026-10-05-pr-artifact-typechecks.md)). + Both tsdown passes match `vendor/*`, `packages/*/*`, and `apps/cli`; the Host pass also matches `apps/desktop-host`. They neither scan build artifacts to discover Client packages nor maintain a Host/Client package filter list. Package-local tsdown configs select entries for the current phase through `DSH_BUILD_FACE`: an ordinary Client plugin produces both its Node loader and browser bundle during the Client phase; `api-remotes` uses `hostPhase: true` to produce its Host entry early and only its browser bundle during the Client phase. Tsdown consumes only the JavaScript emitted to `lib/types` by the preceding tsc phase. Tsdown builds the matched workspace members concurrently, so `apps/desktop`, whose main bundle inlines workspace devDependencies from their `lib/` output, bundles in its own step after the Host pass ([Desktop README](../apps/desktop/README.md#bundled-workspace-dependencies)). Typert runs only during Host tsdown, seeded by `tsconfig.host.json`. It analyzes Host types and generates both Host reflection artifacts and the Host-for-Client Remote projection; Client tsdown does not start Typert. Consequently, `pnpm run typecheck` runs the complete Host lib phase before Client tsc, while `pnpm run build` continues through Client tsdown and the Web build. @@ -132,7 +134,7 @@ Contributors can opt into the comprehensive local gate set with `pnpm run check: ### CI gates -The keyless [CI workflow](../.github/workflows/ci.yml) groups independent gates into broad lanes and runs a smaller compatibility signal across supported Node versions. Artifact consumers wait for one build within their lane. Required benchmarks run separately on standard GitHub-hosted Linux; the [benchmark runner reference](../benchmarks/AGENTS.md) owns routing and the job timeout. The separate real-API workflow runs `pnpm run test:e2e` with its configured worker bound. See [scripts/run-gates.ts](../scripts/run-gates.ts) and the workflow files for the current gate and job inventory. +The keyless [CI workflow](../.github/workflows/ci.yml) groups independent gates into broad lanes and runs a smaller compatibility signal across supported Node versions. Artifact consumers wait for one build within their lane. Coverage partitions reuse file timings scoped to their platform and runner pool to balance work within the configured worker limit. Required benchmarks run separately on standard GitHub-hosted Linux; the [benchmark runner reference](../benchmarks/AGENTS.md) owns routing and the job timeout. The separate real-API workflow runs `pnpm run test:e2e` with its configured worker bound. See [scripts/run-gates.ts](../scripts/run-gates.ts) and the workflow files for the current gate and job inventory. The credential-free dsh dependency-layout and dsh/vendor pack rehearsals use the existing Linux self-hosted pool only when `DSH_CI_FAILOVER_LINUX=selfhosted` and the event is a trusted master push or same-repository, non-fork, non-Dependabot pull request. All other cases, including manual dispatch, use `ubuntu-24.04`; manual publication stays hosted. See the [release rehearsal runner reference](../.agents/notes/implemented/process/2026-07-26-ci-failover-runbook.md) for persistent-store isolation and fallback limits. diff --git a/docs/development.zh.md b/docs/development.zh.md index 0ad92e883d..1bf9403a7f 100644 --- a/docs/development.zh.md +++ b/docs/development.zh.md @@ -89,6 +89,8 @@ tsdown --env.DSH_BUILD_FACE client pnpm run build:web ``` +`pnpm run build --artifacts-only` 产出相同的 project 和应用产物,但只检查并编译各 compiler face 的现有包 reference。必需的 Linux 与 Windows 普通构建以及公开 `typecheck` 还检查仓库级测试和脚本 program。CI benchmark 准备阶段还对其临时 library 输出使用 `--noCheck`,由这些必需构建负责诊断;其他产物 builder 保留包检查([理由](../.agents/notes/implemented/process/2026-10-05-pr-artifact-typechecks.zh.md))。 + 两次 tsdown 都匹配 `vendor/*`、`packages/*/*` 与 `apps/cli`,Host 阶段另外匹配 `apps/desktop-host`;两者都不扫描构建产物来发现 Client 包,也不维护 Host/Client 包过滤表。包内 tsdown 配置根据 `DSH_BUILD_FACE` 决定当前阶段的入口:普通 Client 插件在 Client 阶段同时生成 Node loader 与 browser bundle;`api-remotes` 通过 `hostPhase: true` 提前生成 Host 入口,再在 Client 阶段只生成 browser bundle。tsdown 只消费 `lib/types` 中由前置 tsc 发射的 JavaScript。tsdown 并发构建匹配到的 workspace 成员;`apps/desktop` 的主进程 bundle 需要从工作区 devDependencies 的 `lib/` 产物内联这些包,因此在 Host 阶段之后单独一步打 bundle([Desktop README](../apps/desktop/README.zh.md#bundled-workspace-dependencies))。 Typert 只在 Host tsdown 中以 `tsconfig.host.json` 为种子运行。它分析 Host 类型并生成 Host 反射产物及 Host-for-Client Remote 投影;Client tsdown 不启动 Typert。`pnpm run typecheck` 因此先执行完整 Host lib 阶段,再运行 Client tsc;`pnpm run build` 继续执行 Client tsdown 和 Web 构建。 @@ -136,7 +138,7 @@ vendor manifest 守卫检查 `vendor/*/src` 下的改动是否连同对应的 `v ### CI 门禁 -keyless [CI 工作流](../.github/workflows/ci.yml) 将独立门禁分组到若干宽粒度 lane,并在受支持的 Node 版本上运行一组较小的兼容性检查。产物消费方在各自 lane 内等待一次 build。必需 benchmark 在标准 GitHub 托管 Linux 上独立运行;[benchmark 运行器说明](../benchmarks/AGENTS.md)拥有路由及 job 超时。单独的真实 API 工作流按其配置的 worker 上限运行 `pnpm run test:e2e`。当前门禁和 job 清单以 [scripts/run-gates.ts](../scripts/run-gates.ts) 和工作流文件为准。 +keyless [CI 工作流](../.github/workflows/ci.yml) 将独立门禁分组到若干宽粒度 lane,并在受支持的 Node 版本上运行一组较小的兼容性检查。产物消费方在各自 lane 内等待一次 build。覆盖率分区复用按平台和运行器池隔离的文件耗时,在配置的 worker 上限内均衡工作量。必需 benchmark 在标准 GitHub 托管 Linux 上独立运行;[benchmark 运行器说明](../benchmarks/AGENTS.md)拥有路由及 job 超时。单独的真实 API 工作流按其配置的 worker 上限运行 `pnpm run test:e2e`。当前门禁和 job 清单以 [scripts/run-gates.ts](../scripts/run-gates.ts) 和工作流文件为准。 不带凭据的 dsh 依赖布局检查与 dsh/vendor 打包演练仅在 `DSH_CI_FAILOVER_LINUX=selfhosted`,且事件为受信任的 master 推送或同仓库、非 fork、非 Dependabot 拉取请求时使用现有 Linux 自托管池。其余情况(包括手动触发)均使用 `ubuntu-24.04`;手动发布仍使用托管运行器。持久化存储隔离与回退限制见[发布演练运行器说明](../.agents/notes/implemented/process/2026-07-26-ci-failover-runbook.zh.md)。 diff --git a/packages/typert/generator/src/tsdown-plugin.ts b/packages/typert/generator/src/tsdown-plugin.ts index dedfd366e9..6e9ec559c2 100644 --- a/packages/typert/generator/src/tsdown-plugin.ts +++ b/packages/typert/generator/src/tsdown-plugin.ts @@ -22,10 +22,10 @@ interface TypertPlugin { const DECORATOR_SYNTAX = /^\s*@[A-Za-z_$][\w$]*/m -// This plugin consumes tsc-emitted `lib/types` output, so every project it -// would re-diagnose has already passed the workspace tsc build in the same -// orchestration; the generator skips its per-package diagnostic pass here. -const TSC_VERIFIED_INPUT = { checkDiagnostics: false } as const +// Full builds and typecheck own TypeScript diagnostics. Benchmark-only emission +// defers them to the required full-build CI jobs; Typert generates reflection +// without repeating those diagnostics. +const TSC_OWNED_DIAGNOSTICS = { checkDiagnostics: false } as const /** Generation scope selected by a tsdown build phase. */ export interface TypertPluginOptions { @@ -83,7 +83,7 @@ export function typertPlugin(pluginOptions: TypertPluginOptions = {}): TypertPlu if (manifest.name === undefined || !hasTypertExport(manifest.exports)) return let artifacts = artifactsByRoot.get(root) if (artifacts === undefined) { - const generator = new WorkspaceTypertGenerator(root, TSC_VERIFIED_INPUT) + const generator = new WorkspaceTypertGenerator(root, TSC_OWNED_DIAGNOSTICS) artifacts = pluginOptions.faces === undefined ? generator.generate() : generator.generate(undefined, pluginOptions.faces) @@ -94,7 +94,7 @@ export function typertPlugin(pluginOptions: TypertPluginOptions = {}): TypertPlu } function emitWorkspace(root: string, faces: readonly TypertFace[] | undefined): void { - const generator = new WorkspaceTypertGenerator(root, TSC_VERIFIED_INPUT) + const generator = new WorkspaceTypertGenerator(root, TSC_OWNED_DIAGNOSTICS) const packages = generator.discover(faces) .filter(candidate => hasTypertExport(readManifest(join(root, candidate.root)).exports)) .map(candidate => candidate.package) diff --git a/scripts/build-exe-for-python-sdk.spec.ts b/scripts/build-exe-for-python-sdk.spec.ts index f2f563fd61..962c2ab50e 100644 --- a/scripts/build-exe-for-python-sdk.spec.ts +++ b/scripts/build-exe-for-python-sdk.spec.ts @@ -23,6 +23,78 @@ function run(env: NodeJS.ProcessEnv, ...args: string[]) { } describe('Python runtime executable builder CLI', () => { + it.each([ + { flags: [], command: 'run build' }, + { flags: ['--artifacts-only'], command: 'run build --artifacts-only' }, + ])('validates the closure before $command and deploys only after building', ({ flags, command }) => { + const result = run( + { npm_execpath: 'C:\\tools\\pnpm.cjs' }, + '--dry-run', + '--targets=node24-macos-arm64', + ...flags, + ) + + expect(result.error).toBeUndefined() + expect(result.signal).toBeNull() + expect(result.status, result.stderr).toBe(0) + const commands = result.stdout.split('\n').filter(line => line.includes('C:\\tools\\pnpm.cjs ')) + expect(commands[0]).toMatch(/ run verify-runtime-closure$/) + expect(commands[1]?.endsWith(' ' + command)).toBe(true) + expect(commands[2]).toContain(' --filter dsh-python-runtime-closure deploy ') + }) + + it('rejects simultaneous build modes before invoking pnpm', () => { + const setup = mkdtempSync(join(tmpdir(), 'dsh-python-build-mode-')) + temporaryDirectories.push(setup) + const entrypoint = join(setup, 'pnpm.cjs') + const commands = join(setup, 'commands.jsonl') + writeFileSync(entrypoint, [ + "require('node:fs').appendFileSync(process.env.DSH_PYTHON_BUILD_COMMANDS, JSON.stringify(process.argv.slice(2)) + '\\n')", + 'process.exit(31)', + '', + ].join('\n')) + + const result = run( + { npm_execpath: entrypoint, DSH_PYTHON_BUILD_COMMANDS: commands }, + '--skip-build', + '--artifacts-only', + '--targets=node24-macos-arm64', + ) + + expect(result.error).toBeUndefined() + expect(result.signal).toBeNull() + expect(result.status).not.toBe(0) + expect(result.stderr).toContain('--skip-build and --artifacts-only cannot be combined') + expect(result.stdout).toBe('') + expect(existsSync(commands)).toBe(false) + }) + + it('stops before artifact building or deployment when the closure check fails', () => { + const setup = mkdtempSync(join(tmpdir(), 'dsh-python-closure-')) + temporaryDirectories.push(setup) + const entrypoint = join(setup, 'pnpm.cjs') + const commands = join(setup, 'commands.jsonl') + writeFileSync(entrypoint, [ + "require('node:fs').appendFileSync(process.env.DSH_PYTHON_BUILD_COMMANDS, JSON.stringify(process.argv.slice(2)) + '\\n')", + "if (process.argv.slice(2).join(' ') === 'run verify-runtime-closure') process.exit(31)", + 'process.exit(32)', + '', + ].join('\n')) + + const result = run( + { npm_execpath: entrypoint, DSH_PYTHON_BUILD_COMMANDS: commands }, + '--artifacts-only', + '--targets=node24-macos-arm64', + ) + + expect(result.error).toBeUndefined() + expect(result.signal).toBeNull() + expect(result.status).not.toBe(0) + expect(result.stderr).toContain('runtime dependency closure failed (exit code 31)') + expect(readFileSync(commands, 'utf8').trim().split('\n')) + .toEqual([JSON.stringify(['run', 'verify-runtime-closure'])]) + }) + it('keeps the single-file dispatcher on the Python packaging surface', () => { const bootstrapPath = resolve(root, 'python/sdk-runtime/runtime-bootstrap.mjs') const bootstrap = readFileSync(bootstrapPath, 'utf8') @@ -59,6 +131,7 @@ describe('Python runtime executable builder CLI', () => { expect(result.status).toBe(0) expect(result.stdout).toContain(`${process.execPath} C:\\tools\\pnpm.cjs run verify-runtime-closure`) + expect(result.stdout).not.toMatch(/\[dry-run\] .* run build(?: --artifacts-only)?$/m) expect(result.stdout).toContain(`${process.execPath} C:\\tools\\pnpm.cjs --filter dsh-python-runtime-closure deploy`) const deploy = result.stdout.split('\n').find(line => line.includes(' --filter dsh-python-runtime-closure deploy')) expect(deploy).toContain('--prod --config.allow-unused-patches=true') diff --git a/scripts/build-exe-for-python-sdk.ts b/scripts/build-exe-for-python-sdk.ts index e2000c960b..adb63352af 100644 --- a/scripts/build-exe-for-python-sdk.ts +++ b/scripts/build-exe-for-python-sdk.ts @@ -156,8 +156,10 @@ class BuildCli { private constructor( /** Build targets; defaults to the host platform only. */ readonly targets: readonly Target[], - /** Skip step 1 (`pnpm run build`); lib/ artifacts must already exist. */ + /** Skip the package build; lib/ artifacts must already exist. */ readonly skipBuild: boolean, + /** Emit package and Web artifacts without repository test and script typechecks. */ + readonly artifactsOnly: boolean, /** Print every command and config patch instead of executing. */ readonly dryRun: boolean, ) {} @@ -181,6 +183,9 @@ class BuildCli { console.log(BuildCli.usage()) process.exit(0) } + if (values['skip-build'] && values['artifacts-only']) { + throw new Error('build-exe-for-python-sdk: --skip-build and --artifacts-only cannot be combined.') + } const targets = values.targets === undefined ? [Target.host()] : values.targets.split(',').map(part => part.trim()).filter(part => part !== '').map(spec => Target.parse(spec)) @@ -193,7 +198,7 @@ class BuildCli { } seen.add(key) } - return new BuildCli(targets, values['skip-build'], values['dry-run']) + return new BuildCli(targets, values['skip-build'], values['artifacts-only'], values['dry-run']) } private static parseRaw(argv: string[]) { @@ -202,6 +207,7 @@ class BuildCli { options: { 'targets': { type: 'string' }, 'skip-build': { type: 'boolean', default: false }, + 'artifacts-only': { type: 'boolean', default: false }, 'dry-run': { type: 'boolean', default: false }, 'help': { type: 'boolean', default: false }, }, @@ -215,6 +221,7 @@ class BuildCli { ' --targets= pkg targets, e.g. node24-linux-x64,node24-linux-arm64,node24-macos-arm64,node24-macos-x64,node24-win-x64.', ' Default: the host platform only (on node24).', ' --skip-build skip `pnpm run build` (lib/ artifacts must already exist).', + ' --artifacts-only omit repository test and script typechecks from the artifact build.', ' --dry-run print every command and config patch without executing.', ' --help print this help.', '', @@ -281,7 +288,8 @@ class SingleExeBuild { console.log('build-exe-for-python-sdk: skipping pnpm run build (--skip-build)') return } - await this.runPnpm('build', ['run', 'build']) + const args = this.cli.artifactsOnly ? ['run', 'build', '--artifacts-only'] : ['run', 'build'] + await this.runPnpm('build', args) } /** Clear and deploy the runtime closure into the node carrier. */ diff --git a/scripts/build.spec.ts b/scripts/build.spec.ts index 07f5f8b98f..ecb6420061 100644 --- a/scripts/build.spec.ts +++ b/scripts/build.spec.ts @@ -15,4 +15,14 @@ describe('repository build CLI', () => { expect(result.status).not.toBe(0) expect(result.stderr).toContain('build: Node.js TypeScript type stripping is unavailable') }) + + it('rejects a value for the artifacts-only flag before any build step', () => { + const result = spawnSync( + process.execPath, + ['--import', 'tsx/esm', resolve(root, 'scripts/build.ts'), '--artifacts-only=yes'], + { cwd: root, encoding: 'utf8' }, + ) + expect(result.status).not.toBe(0) + expect(result.stderr).toContain('does not take an argument') + }) }) diff --git a/scripts/build.ts b/scripts/build.ts index a7b06bda65..8cd6cfff8f 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -1,4 +1,4 @@ -/** Run the complete repository build and bind its client artifacts to their public environment. */ +/** Build repository artifacts and bind client outputs to their public environment. */ import { spawnSync } from 'node:child_process' import { rmSync } from 'node:fs' @@ -14,9 +14,9 @@ import { } from './client-build-environment.ts' import { pnpmInvocation } from './pnpm-invocation.ts' -/** Run one package script through the package manager that invoked this build. */ -function runScript(script: string, environment: NodeJS.ProcessEnv): void { - const invocation = pnpmInvocation(['run', script], environment) +/** Run one package-manager command with the selected public build environment. */ +function runCommand(args: readonly string[], environment: NodeJS.ProcessEnv, label: string): void { + const invocation = pnpmInvocation(args, environment) const result = spawnSync(invocation.command, invocation.args, { cwd: resolve(import.meta.dirname, '..'), env: environment, @@ -24,18 +24,26 @@ function runScript(script: string, environment: NodeJS.ProcessEnv): void { }) if (result.error !== undefined) throw result.error if (result.status !== 0) { - throw new Error(`build: ${script} exited with ${String(result.status ?? result.signal)}`) + throw new Error(`build: ${label} exited with ${String(result.status ?? result.signal)}`) } } -/** Run the full build selected by `--profile` or `DSH_BUILD_CLIENT_PROFILE`. */ +/** Run one package script through the package manager that invoked this build. */ +function runScript(script: string, environment: NodeJS.ProcessEnv): void { + runCommand(['run', script], environment, script) +} + +/** Build client artifacts and optionally omit the repository test and script typechecks. */ function main(): void { // tsdown.config.ts loads only through Node type stripping (`--config-loader native`); this names the cause before tsdown fails. if (!process.features.typescript) { throw new Error('build: Node.js TypeScript type stripping is unavailable in this Node.js process; remove --no-experimental-strip-types from NODE_OPTIONS or use a Node.js build with TypeScript support') } const { values } = parseArgs({ - options: { profile: { type: 'string' } }, + options: { + profile: { type: 'string' }, + 'artifacts-only': { type: 'boolean' }, + }, allowPositionals: false, }) const root = resolve(import.meta.dirname, '..') @@ -46,7 +54,11 @@ function main(): void { rmSync(resolve(root, CLIENT_BUILD_RECORD_PATH), { force: true }) runScript('build:native-system', buildEnvironment) - runScript('build:lib', buildEnvironment) + if (values['artifacts-only']) { + runCommand(['exec', 'tsx', 'scripts/compile-referenced-projects.ts', 'libraries'], buildEnvironment, 'artifact libraries') + } else { + runScript('build:lib', buildEnvironment) + } runScript('build:web', buildEnvironment) const record = writeClientBuildRecord(root, clientEnvironment) console.log( diff --git a/scripts/ci-workflow.spec.ts b/scripts/ci-workflow.spec.ts index dd52510c86..1af8850669 100644 --- a/scripts/ci-workflow.spec.ts +++ b/scripts/ci-workflow.spec.ts @@ -14,6 +14,131 @@ const runnerPrivatePnpmDestination = /^\$\{\{ runner\.temp \}\}\/setup-pnpm-\$\{ const nativeWindowsPnpmDestination = '${{ runner.temp }}/setup-pnpm-js-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}' describe('CI workflow', () => { + it('keeps at most ten worker jobs active while runtime targets follow the SDK suite', () => { + const ci = loadWorkflow('.github/workflows/ci.yml') + const compatibility = workflowJob(ci, 'node-compat') + const runtime = workflowJob(ci, 'python-runtime') + if (!isRecord(ci.jobs) || !isRecord(compatibility.strategy) || !isRecord(runtime.with)) { + throw new TypeError('CI must define jobs, compatibility scheduling, and runtime inputs') + } + const compatibilityLimit = compatibility.strategy['max-parallel'] + const targets = runtime.with.targets + if (typeof compatibilityLimit !== 'number' || typeof targets !== 'string') { + throw new TypeError('CI requires a numeric compatibility limit and explicit runtime targets') + } + expect(Number.isInteger(compatibilityLimit)).toBe(true) + expect(compatibilityLimit).toBeGreaterThan(0) + expect(compatibility.strategy).toMatchObject({ + 'fail-fast': false, + matrix: { include: [{ node: '22.19' }, { node: '24.9' }, { node: 26 }] }, + }) + expect(workflowJob(ci, 'all-checks-passed').needs).toContain('node-compat') + const ordinaryJobs = Object.entries(ci.jobs) + .filter(([name]) => !['node-compat', 'python-runtime', 'all-checks-passed'].includes(name)) + for (const [name, job] of ordinaryJobs) { + if (!isRecord(job)) throw new TypeError(`${name} must define a job`) + expect(job).not.toHaveProperty('strategy') + expect(job).not.toHaveProperty('uses') + } + expect(ordinaryJobs.map(([name]) => name)).toContain('python-sdk') + expect(runtime.needs).toBe('python-sdk') + expect(runtime.strategy).toBeUndefined() + expect(runtime.with).toMatchObject({ targets: 'node24-linux-x64,node24-win-x64' }) + expect(runtime.uses).toBe('./.github/workflows/build-exe-for-python-sdk.yml') + + const builder = loadWorkflow('.github/workflows/build-exe-for-python-sdk.yml') + if (!isRecord(builder.jobs)) throw new TypeError('Python runtime builder must define jobs') + expect(Object.keys(builder.jobs).sort()).toEqual(['build', 'plan', 'sdk-wheel']) + const plan = workflowJob(builder, 'plan') + const wheel = workflowJob(builder, 'sdk-wheel') + const build = workflowJob(builder, 'build') + expect(plan.needs).toBeUndefined() + expect(plan.strategy).toBeUndefined() + expect(plan.uses).toBeUndefined() + expect(wheel.needs).toBe('plan') + expect(wheel.strategy).toBeUndefined() + expect(wheel.uses).toBeUndefined() + expect(build.needs).toEqual(['plan', 'sdk-wheel']) + expect(build.uses).toBeUndefined() + if (!isRecord(build.strategy)) throw new TypeError('Python runtime build must define its target matrix') + expect(build.strategy.matrix).toEqual({ include: '${{ fromJSON(needs.plan.outputs.matrix) }}' }) + + const beforeSdk = ordinaryJobs.length + compatibilityLimit + const afterSdk = ordinaryJobs.length - 1 + compatibilityLimit + Math.max(1, targets.split(',').length) + expect(beforeSdk).toBeLessThanOrEqual(10) + expect(afterSdk).toBeLessThanOrEqual(10) + }) + + it('retains aggregate diagnostics in required builds and protected publication', () => { + const ci = loadWorkflow('.github/workflows/ci.yml') + for (const [name, command] of [ + ['node-24-consumers', 'pnpm run check:ci:consumers'], + ['windows-build', 'pnpm run check:ci:windows-blocking'], + ] as const) { + const job = workflowJob(ci, name) + expect(job.steps).toContainEqual(expect.objectContaining({ run: command })) + } + for (const [file, command] of [ + ['release-publish.yml', 'pnpm run build:official'], + ['release-vendor-publish.yml', 'pnpm run build:lib:host'], + ] as const) { + expect(workflowJob(loadWorkflow('.github/workflows/' + file), 'pack').steps) + .toContainEqual(expect.objectContaining({ name: 'Build', run: command })) + } + const scripts = JSON.parse(readFileSync(resolve(root, 'package.json'), 'utf8')) as { scripts: Record } + expect(scripts.scripts['build:lib:host']).toContain('tsc -b tsconfig.host.json') + expect(scripts.scripts['build:lib:client']).toContain('tsc -b tsconfig.client.json') + expect(scripts.scripts.typecheck).toBe('npm run build:lib:host && npm run typecheck:contracts-ready') + }) + + it('selects artifact builds only for PR release and API rehearsals', () => { + const official = "${{ github.event_name == 'pull_request' && 'pnpm run build --artifacts-only --profile official' || 'pnpm run build:official' }}" + const vendor = "${{ github.event_name == 'pull_request' && 'pnpm exec tsx scripts/compile-referenced-projects.ts host-libraries' || 'pnpm run build:lib:host' }}" + for (const [file, jobName, expected] of [ + ['release.yml', 'pack', official], + ['release-vendor.yml', 'pack', vendor], + ['e2e.yml', 'e2e', official], + ] as const) { + const job = workflowJob(loadWorkflow('.github/workflows/' + file), jobName) + if (!Array.isArray(job.steps)) throw new TypeError(`${jobName} must define steps`) + const step = job.steps.filter(isRecord).find(candidate => typeof candidate.run === 'string' + && (candidate.run.includes('--artifacts-only') || candidate.run.includes('host-libraries'))) + expect(step?.run).toBe(expected) + for (const event of ['pull_request', 'push', 'workflow_dispatch']) { + const expression = expected.slice('${{ '.length, -' }}'.length) + const selected: unknown = runInNewContext(expression, { github: { event_name: event } }, { timeout: 1000 }) + expect(selected).toBe(event === 'pull_request' + ? file === 'release-vendor.yml' + ? 'pnpm exec tsx scripts/compile-referenced-projects.ts host-libraries' + : 'pnpm run build --artifacts-only --profile official' + : file === 'release-vendor.yml' ? 'pnpm run build:lib:host' : 'pnpm run build:official') + } + } + }) + + it('selects PR artifact compilation inside the Python executable builder', () => { + const job = workflowJob(loadWorkflow('.github/workflows/build-exe-for-python-sdk.yml'), 'build') + if (!Array.isArray(job.steps)) throw new TypeError('Python runtime builder must define steps') + const steps = job.steps.filter(isRecord) + const execute = steps.find(step => step.name === 'Build single-exe') + expect(steps.some(step => step.run === 'pnpm run build --artifacts-only')).toBe(false) + expect(execute).toMatchObject({ + env: { DSH_BUILD_CLIENT_PROFILE: 'official' }, + }) + if (typeof execute?.run !== 'string') throw new TypeError('Python executable builder must define a command') + const expression = [...execute.run.matchAll(/\$\{\{ ([^{}]+) \}\}/g)] + .map(match => match[1]).find(value => value?.startsWith('inputs.ci')) + expect(expression).toBe("inputs.ci && github.event_name == 'pull_request' && '--artifacts-only' || ''") + if (expression === undefined) throw new TypeError('Python executable builder must select its build mode') + for (const ci of [false, true]) { + for (const event of ['pull_request', 'push', 'workflow_dispatch']) { + const context = { inputs: { ci }, github: { event_name: event } } + const selected: unknown = runInNewContext(expression, context, { timeout: 1000 }) + expect(selected).toBe(ci && event === 'pull_request' ? '--artifacts-only' : '') + } + } + }) + it('prepares confinement before Node compatibility smokes', () => { const job = workflowJob(loadWorkflow('.github/workflows/ci.yml'), 'node-compat') if (!Array.isArray(job.steps)) throw new TypeError('Node compatibility job must define steps') @@ -62,6 +187,34 @@ describe('CI workflow', () => { expect(wine.steps).toContainEqual(expect.objectContaining({ name: 'Shut down wineserver', if: 'always()' })) }) + it.each([ + ['node-24-coverage', 'Linux', 'LINUX', 'pnpm run check:ci:coverage'], + ['windows-coverage', 'Windows', 'WINDOWS', 'pnpm run check:ci:coverage'], + ])('reuses %s coverage timings only within its platform and runner pool', (jobName, platform, variable, command) => { + const job = workflowJob(loadWorkflow('.github/workflows/ci.yml'), jobName) + if (!Array.isArray(job.steps)) throw new TypeError(`${jobName} must define steps`) + const steps = job.steps.filter(isRecord) + const checkout = steps.findIndex(step => step.uses === 'actions/checkout@v6') + const restore = steps.findIndex(step => step.name === 'Restore coverage duration history') + const coverage = steps.findIndex(step => step.run === command) + const save = steps.findIndex(step => step.name === 'Save coverage duration history') + const prefix = `coverage-times-${platform}-\${{ runner.environment }}-\${{ vars.DSH_CI_FAILOVER_${variable} || 'enterprise' }}-` + const key = prefix + '${{ github.run_id }}-${{ github.run_attempt }}' + expect(checkout).toBeGreaterThanOrEqual(0) + expect(restore).toBeGreaterThan(checkout) + expect(coverage).toBeGreaterThan(restore) + expect(save).toBeGreaterThan(coverage) + expect(steps[restore]).toMatchObject({ + uses: 'actions/cache/restore@v4', + with: { path: '.coverage-times.json', key, 'restore-keys': `${prefix}\n` }, + }) + expect(steps[save]).toMatchObject({ + uses: 'actions/cache/save@v4', + if: '${{ !cancelled() }}', + with: { path: '.coverage-times.json', key }, + }) + }) + it('isolates every pnpm action setup destination per runner', () => { const files = ['.github/workflows/ci.yml', '.github/workflows/ci-master.yml'] const setups: Array<{ jobName: string; step: unknown }> = [] diff --git a/scripts/compile-referenced-projects.spec.ts b/scripts/compile-referenced-projects.spec.ts new file mode 100644 index 0000000000..daf20ef3da --- /dev/null +++ b/scripts/compile-referenced-projects.spec.ts @@ -0,0 +1,310 @@ +import { spawnSync } from 'node:child_process' +import { globSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { describe, expect, it, onTestFinished, vi } from 'vitest' +import { buildLibraryArtifacts, compileReferencedProjects } from './compile-referenced-projects.ts' +import { runCiBench } from './run-ci-bench.ts' +import { removeFixtureSafely } from './test-fixture-cleanup.ts' + +const compiler = fileURLToPath(import.meta.resolve('typescript/bin/tsc')) + +function fixture(): { root: string; leaf: string } { + const root = mkdtempSync(join(tmpdir(), 'dsh-referenced-projects-')) + onTestFinished(() => { removeFixtureSafely(root) }) + const leaf = join(root, 'leaf package') + mkdirSync(join(leaf, 'src'), { recursive: true }) + writeFileSync(join(root, 'package.json'), JSON.stringify({ type: 'module' })) + writeFileSync(join(leaf, 'src/index.ts'), 'export const value: number = 42\n') + const compilerOptions = { + target: 'ES2024', + module: 'NodeNext', + moduleResolution: 'NodeNext', + lib: ['ES2024'], + types: [], + composite: true, + strict: true, + } + writeFileSync(join(leaf, 'tsconfig.json'), JSON.stringify({ + compilerOptions: { + ...compilerOptions, + rootDir: 'src', + outDir: 'lib/types', + declaration: true, + declarationMap: true, + sourceMap: true, + }, + include: ['src'], + })) + writeFileSync(join(root, 'aggregate.ts'), 'import { value } from "./leaf package/src/index.js"\nexport const aggregate: number = value\n') + for (const face of ['host', 'client']) { + writeFileSync(join(root, `tsconfig.${face}.json`), JSON.stringify({ + compilerOptions: { ...compilerOptions, noEmit: true }, + include: ['aggregate.ts'], + references: [{ path: './leaf package' }], + })) + } + return { root, leaf } +} + +function artifacts(leaf: string): Record { + const output = join(leaf, 'lib') + return Object.fromEntries(globSync('**/*', { cwd: output }) + .filter(path => statSync(join(output, path)).isFile()) + .sort() + .map(path => [path.replaceAll('\\', '/'), readFileSync(join(output, path)).toString('base64')])) +} + +function recordedCommands(root: string, failHostBundle = false): () => { args: string[]; title: string }[] { + const log = join(root, 'commands.jsonl') + const entrypoint = join(root, 'pnpm', 'bin', 'pnpm.cjs') + mkdirSync(join(root, 'pnpm', 'bin'), { recursive: true }) + writeFileSync(entrypoint, [ + 'const { appendFileSync } = require("node:fs")', + 'const args = process.argv.slice(2)', + `appendFileSync(${JSON.stringify(log)}, JSON.stringify({ args, title: process.env.DSH_CLIENT_TITLE }) + "\\n")`, + ...failHostBundle ? ['if (args.includes("host")) process.exit(7)'] : [], + ].join('\n') + '\n') + vi.stubEnv('npm_execpath', entrypoint) + vi.stubEnv('DSH_CLIENT_TITLE', 'Inherited Build Title') + onTestFinished(() => { vi.unstubAllEnvs() }) + return () => readFileSync(log, 'utf8').trim().split('\n') + .map(line => JSON.parse(line) as { args: string[]; title: string }) +} + +describe('internal artifact build order', () => { + it('resolves pnpm for a direct CLI without npm_execpath and preserves its environment', () => { + const { root } = fixture() + const marker = join(root, 'direct-cli-title.txt') + const environment: NodeJS.ProcessEnv = { ...process.env, DSH_CLIENT_TITLE: 'Direct CLI Title' } + delete environment.npm_execpath + const helper = pathToFileURL(resolve(import.meta.dirname, 'compile-referenced-projects.ts')).href + const code = `import { runPnpmCommand } from ${JSON.stringify(helper)}; runPnpmCommand(${JSON.stringify(root)}, ${JSON.stringify([ + 'exec', process.execPath, '-e', + `require('node:fs').writeFileSync(${JSON.stringify(marker)}, process.env.DSH_CLIENT_TITLE)`, + ])})` + const result = spawnSync(process.execPath, [ + '--import', import.meta.resolve('tsx/esm'), + '--input-type=module', '-e', code, + ], { cwd: root, env: environment, encoding: 'utf8' }) + + expect(result.status, result.stdout + result.stderr).toBe(0) + expect(readFileSync(marker, 'utf8')).toBe('Direct CLI Title') + }) + + it.each([true, false])('preserves bundler order and the public environment with hostOnly=%s', (hostOnly) => { + const { root } = fixture() + const commands = recordedCommands(root) + + buildLibraryArtifacts(root, hostOnly) + + const expected = [ + ['exec', 'tsdown', '--config-loader', 'native', '--env.DSH_BUILD_FACE', 'host'], + ['--filter', '@deepseek-ai/dsh-desktop', 'run', 'bundle'], + ...hostOnly ? [] : [['exec', 'tsdown', '--config-loader', 'native', '--env.DSH_BUILD_FACE', 'client']], + ] + expect(commands()).toEqual(expected.map(args => ({ args, title: 'Inherited Build Title' }))) + }) + + it('stops after a failed Host bundler before Desktop and Client builds', () => { + const { root } = fixture() + const commands = recordedCommands(root, true) + + expect(() => { buildLibraryArtifacts(root, false) }).toThrow('exited with 7') + expect(commands()).toHaveLength(1) + }) + + it('defers benchmark diagnostics while normal artifact builds reject the same source', () => { + const { root, leaf } = fixture() + const commands = recordedCommands(root) + writeFileSync(join(leaf, 'src/index.ts'), 'export const value: number = "invalid"\n') + + runCiBench(root) + + expect(commands()).toEqual([ + ['run', 'build:native-system'], + ['exec', 'tsdown', '--config-loader', 'native', '--env.DSH_BUILD_FACE', 'host'], + ['--filter', '@deepseek-ai/dsh-desktop', 'run', 'bundle'], + ['exec', 'tsdown', '--config-loader', 'native', '--env.DSH_BUILD_FACE', 'client'], + ['exec', 'tsdown', '--config-loader', 'native', '--config', 'benchmarks/tsdown.config.ts'], + ['run', 'build:web'], + ['run', 'test:bench:built'], + ].map(args => ({ args, title: 'Inherited Build Title' }))) + + expect(() => { buildLibraryArtifacts(root, false) }).toThrow('host compiler exited with') + expect(commands()).toHaveLength(7) + }) +}) + +describe('package project compilation', () => { + it('records Windows-style glob keys without changing artifact bytes', () => { + const { leaf } = fixture() + const output = join(leaf, 'lib') + mkdirSync(join(output, 'types'), { recursive: true }) + const bytes = Buffer.from([0, 255, 13, 10]) + writeFileSync(join(output, 'types\\index.js'), bytes) + + expect(artifacts(leaf)).toEqual({ 'types/index.js': bytes.toString('base64') }) + }) + + it.each(['host', 'client'] as const)('preserves every %s package artifact byte', (face) => { + const { root, leaf } = fixture() + const complete = spawnSync(process.execPath, [compiler, '-b', `tsconfig.${face}.json`], { + cwd: root, + encoding: 'utf8', + }) + expect(complete.status, complete.stdout + complete.stderr).toBe(0) + const expected = artifacts(leaf) + expect(Object.keys(expected)).toEqual(expect.arrayContaining([ + 'types/index.js', 'types/index.js.map', 'types/index.d.ts', 'types/index.d.ts.map', + ])) + rmSync(join(leaf, 'lib'), { recursive: true }) + + compileReferencedProjects(root, face) + + expect(artifacts(leaf)).toEqual(expected) + }) + + it.each(['host', 'client'] as const)('preserves %s runtime output during benchmark emission', (face) => { + const { root, leaf } = fixture() + writeFileSync(join(leaf, 'src/support.ts'), [ + 'export interface Payload { label: string }', + 'export const enum Offset { Baseline = 40 }', + '', + ].join('\n')) + writeFileSync(join(leaf, 'src/index.ts'), [ + 'import { Offset, type Payload } from "./support.js"', + 'export const value: number = Offset.Baseline + 2', + 'export function describe(payload: Payload): string { return `${payload.label}: ${value}` }', + '', + ].join('\n')) + compileReferencedProjects(root, face) + const runtimeArtifacts = () => Object.fromEntries(Object.entries(artifacts(leaf)) + .filter(([path]) => /\.js(?:\.map)?$/.test(path))) + const expected = runtimeArtifacts() + expect(Object.keys(expected)).toEqual([ + 'types/index.js', 'types/index.js.map', 'types/support.js', 'types/support.js.map', + ]) + rmSync(join(leaf, 'lib'), { recursive: true }) + + compileReferencedProjects(root, face, 'benchmark-emit') + + expect(runtimeArtifacts()).toEqual(expected) + const entrypoint = pathToFileURL(join(leaf, 'lib/types/index.js')).href + const result = spawnSync(process.execPath, [ + '--input-type=module', '-e', + `import { describe } from ${JSON.stringify(entrypoint)}; console.log(describe({ label: 'answer' }))`, + ], { cwd: root, encoding: 'utf8' }) + expect(result.error).toBeUndefined() + expect(result.signal).toBeNull() + expect(result.status, result.stderr).toBe(0) + expect(result.stdout.trim()).toBe('answer: 42') + }) + + it.each(['host', 'client'] as const)('rechecks %s semantic errors after benchmark emission without forcing a rebuild', (face) => { + const { root, leaf } = fixture() + writeFileSync(join(leaf, 'src/index.ts'), 'export const value: number = "invalid"\n') + + compileReferencedProjects(root, face, 'benchmark-emit') + + expect(Object.keys(artifacts(leaf)).some(path => path.endsWith('.tsbuildinfo'))).toBe(true) + const complete = spawnSync(process.execPath, [compiler, '-b', `tsconfig.${face}.json`], { + cwd: root, + encoding: 'utf8', + }) + expect(complete.error).toBeUndefined() + expect(complete.signal).toBeNull() + expect(complete.status).not.toBe(0) + expect(complete.stdout).toContain('src/index.ts(1,14): error TS2322') + }) + + it('rejects package syntax errors during benchmark emission', () => { + const { root, leaf } = fixture() + writeFileSync(join(leaf, 'src/index.ts'), 'export const value = ;\n') + + expect(() => { compileReferencedProjects(root, 'host', 'benchmark-emit') }) + .toThrow('host compiler exited with') + }) + + it('keeps aggregate-only errors in the complete compiler check', () => { + const { root } = fixture() + writeFileSync(join(root, 'aggregate.ts'), 'export const aggregate: number = "invalid"\n') + + expect(() => { compileReferencedProjects(root, 'host') }).not.toThrow() + const complete = spawnSync(process.execPath, [compiler, '-b', 'tsconfig.host.json'], { + cwd: root, + encoding: 'utf8', + }) + expect(complete.status).not.toBe(0) + expect(complete.stdout).toContain('aggregate.ts(1,14): error TS2322') + }) + + it('rejects package source errors', () => { + const { root, leaf } = fixture() + writeFileSync(join(leaf, 'src/index.ts'), 'export const value: number = "invalid"\n') + + expect(() => { compileReferencedProjects(root, 'host') }).toThrow('host compiler exited with') + }) + + it.each([undefined, []])('rejects an aggregate with missing or empty references: %j', (references) => { + const { root } = fixture() + writeFileSync(join(root, 'tsconfig.host.json'), JSON.stringify({ files: [], references })) + + expect(() => { compileReferencedProjects(root, 'host') }).toThrow('requires non-empty project references') + }) + + it('rejects a missing referenced project', () => { + const { root } = fixture() + writeFileSync(join(root, 'tsconfig.host.json'), JSON.stringify({ + compilerOptions: { noEmit: true }, + files: [], + references: [{ path: './absent' }], + })) + + expect(() => { compileReferencedProjects(root, 'host') }).toThrow('host compiler exited with') + }) + + it.each([undefined, false])('rejects an aggregate capable of emitting files: noEmit=%s', (noEmit) => { + const { root } = fixture() + writeFileSync(join(root, 'tsconfig.host.json'), JSON.stringify({ + compilerOptions: { noEmit }, + include: ['aggregate.ts'], + references: [{ path: './leaf package' }], + })) + + expect(() => { compileReferencedProjects(root, 'host') }).toThrow( + `${join(root, 'tsconfig.host.json')} requires noEmit: true`, + ) + }) + + it('rejects a missing aggregate config', () => { + const { root } = fixture() + rmSync(join(root, 'tsconfig.host.json')) + + expect(() => { compileReferencedProjects(root, 'host') }).toThrow('Cannot read file') + }) + + it('rejects invalid compiler options', () => { + const { root } = fixture() + writeFileSync(join(root, 'tsconfig.host.json'), JSON.stringify({ + compilerOptions: { target: 'invalid' }, + files: [], + references: [{ path: './leaf package' }], + })) + + expect(() => { compileReferencedProjects(root, 'host') }).toThrow("Argument for '--target' option must be") + }) + + it('rejects an unsupported build mode before building', () => { + const repository = resolve(import.meta.dirname, '..') + const result = spawnSync(process.execPath, [ + '--import', 'tsx/esm', + resolve(repository, 'scripts/compile-referenced-projects.ts'), + 'unsupported', + ], { cwd: repository, encoding: 'utf8' }) + + expect(result.status).not.toBe(0) + expect(result.stderr).toContain('expected exactly one mode: host, client, host-libraries, or libraries') + }) +}) diff --git a/scripts/compile-referenced-projects.ts b/scripts/compile-referenced-projects.ts new file mode 100644 index 0000000000..ac2aece229 --- /dev/null +++ b/scripts/compile-referenced-projects.ts @@ -0,0 +1,111 @@ +/** Compile and bundle package projects without repeating aggregate test and script typechecks. */ + +import { spawnSync } from 'node:child_process' +import { relative, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { parseArgs } from 'node:util' +import ts from 'typescript' +import { pnpmCommand } from './release/process.ts' + +type CompilationMode = 'checked' | 'benchmark-emit' + +/** + * Run one package-manager command serially with the inherited build environment. + * @param root - Repository root supplying the command's working directory. + * @param args - Package-manager arguments, without shell quoting. + * @throws If the command cannot start or exits unsuccessfully. + */ +export function runPnpmCommand(root: string, args: readonly string[]): void { + const [command, ...prefix] = pnpmCommand() + const result = spawnSync(command, [...prefix, ...args], { cwd: root, stdio: 'inherit' }) + if (result.error !== undefined) throw result.error + if (result.status !== 0) { + throw new Error(`compile-referenced-projects: pnpm ${args.join(' ')} exited with ${String(result.status ?? result.signal)}`) + } +} + +/** + * Compile one aggregate's project references with the serial TypeScript builder. + * @param root - Repository root containing the aggregate tsconfig. + * @param face - Independent compiler face whose package projects are emitted. + * @param mode - Check package types by default; benchmark emission defers diagnostics to required full builds. + * @throws If the aggregate emits files, its configuration is invalid, or package compilation fails. + */ +export function compileReferencedProjects( + root: string, + face: 'host' | 'client', + mode: CompilationMode = 'checked', +): void { + const configPath = resolve(root, `tsconfig.${face}.json`) + const config = ts.getParsedCommandLineOfConfigFile(configPath, {}, { + ...ts.sys, + onUnRecoverableConfigFileDiagnostic(diagnostic) { + throw new Error(ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n')) + }, + }) + if (config === undefined) throw new Error(`compile-referenced-projects: cannot read ${configPath}`) + const references = config.projectReferences + if (references === undefined || references.length === 0) { + throw new Error(`compile-referenced-projects: ${configPath} requires non-empty project references`) + } + if (config.errors.length > 0) { + throw new Error(config.errors + .map(diagnostic => ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n')) + .join('\n')) + } + if (config.options.noEmit !== true) { + throw new Error(`compile-referenced-projects: ${configPath} requires noEmit: true to exclude its aggregate program`) + } + const compiler = fileURLToPath(import.meta.resolve('typescript/bin/tsc')) + const result = spawnSync(process.execPath, [ + ...face === 'host' ? ['--max-old-space-size=4096'] : [], + compiler, + '-b', + ...mode === 'benchmark-emit' ? ['--noCheck'] : [], + // Relative arguments keep the complete workspace graph below Windows' command-line limit. + ...references.map(reference => relative(root, reference.path)), + ], { cwd: root, stdio: 'inherit' }) + if (result.error !== undefined) throw result.error + if (result.status !== 0) { + throw new Error(`compile-referenced-projects: ${face} compiler exited with ${String(result.status ?? result.signal)}`) + } +} + +/** + * Emit package libraries in Host, Desktop, then Client dependency order. + * @param root - Repository root containing the compiler and bundle configurations. + * @param hostOnly - Omit the Client compiler and bundler after the Host artifacts are ready. + * @param mode - Check package types by default; benchmark emission defers diagnostics to required full builds. + * @throws If any compiler or bundler fails. + */ +export function buildLibraryArtifacts(root: string, hostOnly: boolean, mode: CompilationMode = 'checked'): void { + compileReferencedProjects(root, 'host', mode) + runPnpmCommand(root, ['exec', 'tsdown', '--config-loader', 'native', '--env.DSH_BUILD_FACE', 'host']) + runPnpmCommand(root, ['--filter', '@deepseek-ai/dsh-desktop', 'run', 'bundle']) + if (hostOnly) return + compileReferencedProjects(root, 'client', mode) + runPnpmCommand(root, ['exec', 'tsdown', '--config-loader', 'native', '--env.DSH_BUILD_FACE', 'client']) +} + +if (import.meta.main) { + const { positionals } = parseArgs({ allowPositionals: true }) + const [mode] = positionals + if (positionals.length !== 1) { + throw new Error('compile-referenced-projects: expected exactly one mode: host, client, host-libraries, or libraries') + } + const root = resolve(import.meta.dirname, '..') + switch (mode) { + case 'host': + case 'client': + compileReferencedProjects(root, mode) + break + case 'host-libraries': + buildLibraryArtifacts(root, true) + break + case 'libraries': + buildLibraryArtifacts(root, false) + break + default: + throw new Error('compile-referenced-projects: expected exactly one mode: host, client, host-libraries, or libraries') + } +} diff --git a/scripts/coverage-file-times.spec.ts b/scripts/coverage-file-times.spec.ts new file mode 100644 index 0000000000..be412f49a0 --- /dev/null +++ b/scripts/coverage-file-times.spec.ts @@ -0,0 +1,67 @@ +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { ModuleDiagnostic, ResolvedConfig } from 'vitest/node' +import { afterEach, describe, expect, it } from 'vitest' +import CoverageFileTimesReporter from './coverage-file-times.ts' + +const roots: string[] = [] +afterEach(async () => { + for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }) +}) + +async function temporaryRoot(): Promise { + const root = await mkdtemp(join(tmpdir(), 'dsh-coverage-file-times-')) + roots.push(root) + return root +} + +describe('coverage file timing report', () => { + it('includes import and hook costs once without serializing coverage or test objects', async () => { + const root = await temporaryRoot() + const reporter = new CoverageFileTimesReporter() + reporter.onInit({ config: { root, outputFile: { fileTimes: 'times.json' } } }) + const diagnostic: ModuleDiagnostic = { + environmentSetupDuration: 3, + prepareDuration: 5, + setupDuration: 7, + collectDuration: 100, + duration: 200, + heap: undefined, + // Per-dependency import times are already included in collectDuration. + importDurations: { dependency: { selfTime: 100, totalTime: 100 } }, + } + const module = { + moduleId: join(root, 'packages', 'a', 'tests', 'x.spec.ts'), + diagnostic: () => diagnostic, + // Vitest modules retain tests and coverage through their project. The + // timing report must read only their paths and diagnostics. + toJSON: () => { throw new Error('test module was serialized') }, + } + await reporter.onTestRunEnd([module]) + + const output = await readFile(join(root, 'times.json'), 'utf8') + expect(JSON.parse(output)).toEqual({ 'packages/a/tests/x.spec.ts': 315 }) + expect(output.endsWith('\n')).toBe(true) + expect(output).not.toContain('coverageMap') + expect(output).not.toContain('assertionResults') + }) + + it('writes an empty report when no modules finished', async () => { + const root = await temporaryRoot() + const reporter = new CoverageFileTimesReporter() + reporter.onInit({ config: { root, outputFile: { fileTimes: 'times.json' } } }) + await reporter.onTestRunEnd([]) + expect(await readFile(join(root, 'times.json'), 'utf8')).toBe('{}\n') + }) + + const missingOutputs: ResolvedConfig['outputFile'][] = ['report.json', {}, { fileTimes: '' }] + it.each(missingOutputs)( + 'rejects missing timing output %j', + (outputFile) => { + const reporter = new CoverageFileTimesReporter() + expect(() => { reporter.onInit({ config: { root: '/repo', outputFile } }) }) + .toThrow('outputFile.fileTimes is required') + }, + ) +}) diff --git a/scripts/coverage-file-times.ts b/scripts/coverage-file-times.ts new file mode 100644 index 0000000000..7d9646636e --- /dev/null +++ b/scripts/coverage-file-times.ts @@ -0,0 +1,43 @@ +/** Record coverage partition file costs without retaining test results or coverage maps. */ +import { writeFile } from 'node:fs/promises' +import { relative, resolve, sep } from 'node:path' +import type { Reporter, ResolvedConfig, TestModule } from 'vitest/node' + +/** Vitest reporter that writes repository-relative file durations to `outputFile.fileTimes`. */ +export default class CoverageFileTimesReporter implements Reporter { + private root = '' + private outputFile = '' + + /** + * Resolve the timing output owned by the coverage coordinator. + * @param vitest - repository root and configured reporter output files. + * @throws when the file-times output is absent. + */ + public onInit(vitest: { config: Pick }): void { + const output = vitest.config.outputFile + if (typeof output !== 'object' || typeof output.fileTimes !== 'string' || output.fileTimes === '') { + throw new Error('coverage-file-times: outputFile.fileTimes is required.') + } + this.root = vitest.config.root + this.outputFile = resolve(this.root, output.fileTimes) + } + + /** + * Write each file's distinct environment, preparation, setup, collection, and execution costs. + * @param testModules - finished files, including failed and skipped files. + * @returns after the compact timing report has been written. + */ + public async onTestRunEnd(testModules: readonly Pick[]): Promise { + const durations: Record = {} + for (const module of testModules) { + const diagnostic = module.diagnostic() + const file = relative(this.root, module.moduleId).split(sep).join('/') + durations[file] = diagnostic.environmentSetupDuration + + diagnostic.prepareDuration + + diagnostic.setupDuration + + diagnostic.collectDuration + + diagnostic.duration + } + await writeFile(this.outputFile, `${JSON.stringify(durations)}\n`, 'utf8') + } +} diff --git a/scripts/coverage-partitions.spec.ts b/scripts/coverage-partitions.spec.ts index 2060608248..853207bc77 100644 --- a/scripts/coverage-partitions.spec.ts +++ b/scripts/coverage-partitions.spec.ts @@ -269,17 +269,26 @@ describe('coverage file inventory', () => { expect(durations.get('packages/a/tests/y.spec.ts')).toBe(7) }) - it('extracts per-file durations from partition json reports', async () => { + it('extracts per-file durations from compact partition timing reports', async () => { const root = await temporaryRoot() - const report = join(root, 'partition-1.report.json') + const report = join(root, 'partition-1.times.json') await writeFile(report, JSON.stringify({ - testResults: [ - { name: join(root, 'packages/a/tests/x.spec.ts'), startTime: 1000, endTime: 1500 }, - { name: 'not-a-spec', startTime: 1, endTime: 2 }, - ], + 'packages/a/tests/x.spec.ts': 500, + 'packages/a/tests/y.spec.ts': 0, + 'packages/a/tests/invalid.spec.ts': -1, + 'packages/a/tests/malformed.spec.ts': 'unknown', })) - const durations = collectPartitionDurations([report], root) + const durations = collectPartitionDurations([report]) expect(durations.get('packages/a/tests/x.spec.ts')).toBe(500) + expect(durations.get('packages/a/tests/y.spec.ts')).toBe(0) + expect(durations.size).toBe(2) + }) + + it.each(['null', '[]', '42', '"invalid"', '{'])('ignores corrupt timing metadata %s', async (contents) => { + const root = await temporaryRoot() + const report = join(root, 'partition.times.json') + await writeFile(report, contents) + expect(collectPartitionDurations([report])).toEqual(new Map()) }) }) @@ -511,9 +520,10 @@ describe('coverage partition coordinator', () => { '--maxWorkers=1', '--reporter=default', '--reporter=blob', - '--reporter=json', '--testTimeout=30000', ])) + expect(command.args).not.toContain('--reporter=json') + expect(command.args).toContain('--reporter=./scripts/coverage-file-times.ts') expect(command.args).not.toContain('--shard=1/3') expect(command.args.some(argument => argument.startsWith('--config='))).toBe(true) expect(command.env).toEqual({ diff --git a/scripts/coverage-partitions.ts b/scripts/coverage-partitions.ts index 575e475564..aafef76a60 100644 --- a/scripts/coverage-partitions.ts +++ b/scripts/coverage-partitions.ts @@ -27,6 +27,7 @@ export const COVERAGE_TEST_TIMEOUT_ENV = 'DSH_COVERAGE_TEST_TIMEOUT_MS' * working directory on every platform. */ const CANONICAL_LOCATIONS_REPORTER = './scripts/coverage-canonical-locations.ts' +const FILE_TIMES_REPORTER = './scripts/coverage-file-times.ts' /** One child command owned by the coverage coordinator. */ export interface CoverageCommand { @@ -299,23 +300,23 @@ export function writeFileDurations( } /** - * Extract per-file durations from Vitest JSON reporter outputs (one per - * partition). Each `testResults` entry names an absolute spec path and carries - * `startTime`/`endTime`; the difference is the file's recorded duration. + * Extract per-file durations from compact timing reports, one per partition. + * @param reportFiles - reports mapping repository-relative file paths to milliseconds. + * @returns finite, non-negative file costs from readable reports. */ -export function collectPartitionDurations(reportFiles: readonly string[], root: string): Map { +export function collectPartitionDurations(reportFiles: readonly string[]): Map { const durations = new Map() for (const file of reportFiles) { - let report: { testResults?: Array<{ name?: unknown; startTime?: number; endTime?: number }> } + let report: unknown try { - report = JSON.parse(readFileSync(file, 'utf8')) as { testResults?: Array<{ name?: unknown; startTime?: number; endTime?: number }> } + report = JSON.parse(readFileSync(file, 'utf8')) } catch { continue } - for (const result of report.testResults ?? []) { - if (typeof result.name !== 'string' || typeof result.startTime !== 'number' || typeof result.endTime !== 'number') continue - const relativePath = relative(root, result.name).split(sep).join('/') - durations.set(relativePath, Math.max(0, result.endTime - result.startTime)) + if (report === null || typeof report !== 'object' || Array.isArray(report)) continue + for (const [path, duration] of Object.entries(report)) { + if (typeof duration !== 'number' || !Number.isFinite(duration) || duration < 0) continue + durations.set(path, duration) } } return durations @@ -532,10 +533,10 @@ export class CoveragePartitionCoordinator { private persistDurations(partitionCount: number): void { const reportFiles = Array.from( { length: partitionCount }, - (_, index) => join(this.temporaryRoot, `partition-${index + 1}.report.json`), + (_, index) => join(this.temporaryRoot, `partition-${index + 1}.times.json`), ) const currentFiles = this.projectOf.size > 0 ? [...this.projectOf.keys()] : undefined - writeFileDurations(this.root, collectPartitionDurations(reportFiles, this.root), currentFiles) + writeFileDurations(this.root, collectPartitionDurations(reportFiles), currentFiles) } /** @@ -584,7 +585,7 @@ export class CoveragePartitionCoordinator { private partitionCommand(index: number, configPath: string): CoverageCommand { const blobPath = join(this.blobsRoot, `partition-${index}.json`) const reportsDirectory = join(this.temporaryRoot, `coverage-${index}`) - const jsonReportPath = join(this.temporaryRoot, `partition-${index}.report.json`) + const timingReportPath = join(this.temporaryRoot, `partition-${index}.times.json`) const invocation = pnpmInvocation([ 'exec', 'vitest', @@ -595,10 +596,10 @@ export class CoveragePartitionCoordinator { `--config=${this.relativePath(configPath)}`, '--reporter=default', '--reporter=blob', - '--reporter=json', + `--reporter=${FILE_TIMES_REPORTER}`, `--reporter=${CANONICAL_LOCATIONS_REPORTER}`, `--outputFile.blob=${this.relativePath(blobPath)}`, - `--outputFile.json=${this.relativePath(jsonReportPath)}`, + `--outputFile.fileTimes=${this.relativePath(timingReportPath)}`, `--coverage.reportsDirectory=${this.relativePath(reportsDirectory)}`, ...this.vitestArgs, ], { npm_execpath: this.pnpmEntrypoint }) diff --git a/scripts/preview-workflow.spec.ts b/scripts/preview-workflow.spec.ts index f3d14ce484..224f5a66d3 100644 --- a/scripts/preview-workflow.spec.ts +++ b/scripts/preview-workflow.spec.ts @@ -24,14 +24,14 @@ describe('PR preview workflow', () => { expect(preview.steps.find(step => step.uses === 'actions/checkout@v6')?.with).toEqual({ 'persist-credentials': false }) }) - it('keeps the immutable full build and restore-only dependency cache', () => { + it('keeps the immutable complete artifact build and restore-only dependency cache', () => { expect(workflow.env.PRIMARY_NODE_VERSION).toBe('24') expect(workflow.env.DSH_TELEMETRY_DISABLED).toBe('1') const commands = preview.steps.map(step => step.run) expect(commands).toContain('pnpm install --frozen-lockfile') - expect(commands).toContain('pnpm run build') + expect(commands).toContain('pnpm run build --artifacts-only') expect(commands).toContain('pnpm --filter @deepseek-ai/dsh-web-frontend run build:preview') - expect(commands.indexOf('pnpm run build')).toBeLessThan(commands.indexOf('pnpm --filter @deepseek-ai/dsh-web-frontend run build:preview')) + expect(commands.indexOf('pnpm run build --artifacts-only')).toBeLessThan(commands.indexOf('pnpm --filter @deepseek-ai/dsh-web-frontend run build:preview')) expect(preview.steps.filter(step => step.uses?.startsWith('actions/cache'))).toHaveLength(1) expect(preview.steps.find(step => step.uses === 'actions/cache/restore@v4')?.with).toMatchObject({ key: "${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }}", diff --git a/scripts/release/pack.spec.ts b/scripts/release/pack.spec.ts new file mode 100644 index 0000000000..aa7314cb5e --- /dev/null +++ b/scripts/release/pack.spec.ts @@ -0,0 +1,362 @@ +/** Real-pnpm regression coverage for release packing, selection, hooks, and payload checks. */ + +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from 'node:fs' +import { createServer, type ServerResponse } from 'node:http' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { officialClientBuildEnvironment, writeClientBuildRecord } from '../client-build-environment.ts' +import { runGate, type Gate } from '../run-gates.ts' +import { removeFixtureSafely } from '../test-fixture-cleanup.ts' +import { releaseFamily, tarballName } from './families.ts' +import { packedIdentity, PUBLISH_ORDER_FILE, readPublishOrder } from './tarball.ts' + +const require = createRequire(import.meta.url) +const packScript = resolve(import.meta.dirname, 'pack.ts') +const tsxHook = pathToFileURL(require.resolve('tsx/esm')).href +const pnpmEntry = join(dirname(require.resolve('pnpm')), 'bin/pnpm.cjs') +const roots: string[] = [] +const commit = 'abcdef0' + +function write(path: string, content: string): void { + mkdirSync(dirname(path), { recursive: true }) + writeFileSync(path, content) +} + +function fixture(): string { + const root = mkdtempSync(join(tmpdir(), 'dsh-release-pack-')) + roots.push(root) + write(join(root, 'package.json'), JSON.stringify({ name: '@deepseek-ai/pack-root', private: true, version: '1.2.3' })) + write(join(root, 'pnpm-workspace.yaml'), 'packages:\n - vendor/*\n - native/*\n - packages/*/*\n - apps/*\n') + return root +} + +function packageFixture(root: string, directory: string, name: string, additions: Record = {}): void { + write(join(root, directory, 'package.json'), JSON.stringify({ + name, version: '1.2.3', files: ['lib/index.js', 'hook.cjs'], + scripts: { + prepack: 'node hook.cjs prepack', + prepare: 'node hook.cjs prepare', + postpack: 'node hook.cjs postpack', + }, + ...additions, + })) + write(join(root, directory, 'lib/index.js'), 'export const value = 1;\n') + write(join(root, directory, 'hook.cjs'), [ + "const fs = require('node:fs');", + "const path = require('node:path');", + "const manifest = require('./package.json');", + "fs.appendFileSync(path.join(process.env.DSH_PACK_FIXTURE, 'hooks.log'), `${manifest.name}:${process.argv[2]}\\n`);", + '', + ].join('\n')) +} + +function packGate(root: string, args: string[] = []): Gate { + return { + id: 'release-pack-fixture', + label: 'release pack fixture', + displayCommand: 'pnpm exec node release pack fixture', + command: process.execPath, + args: [pnpmEntry, '--dir', root, 'exec', process.execPath, '--import', tsxHook, packScript, '--family', 'vendor', '--out', 'packed', ...args], + env: { npm_execpath: pnpmEntry, DSH_PACK_FIXTURE: root, DSH_CLIENT_COMMIT_HASH: commit }, + } +} + +async function runPack(root: string, args: string[] = []) { + const controller = new AbortController() + const timer = setTimeout(() => { controller.abort() }, 90_000) + try { + const result = await runGate(packGate(root, args), controller.signal) + expect(result.error).toBeUndefined() + expect(result.aborted).toBe(false) + expect(result.signalCode).toBeNull() + return { + status: result.exitCode, + stderr: result.output.filter(chunk => chunk.stream === 'stderr').map(chunk => chunk.text).join(''), + } + } finally { + clearTimeout(timer) + } +} + +function stopped(pid: number): boolean { + if (process.platform === 'linux') { + try { + return /\)\s+Z\s/.test(readFileSync(`/proc/${pid}/stat`, 'utf8')) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return true + throw error + } + } + try { + process.kill(pid, 0) + return false + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ESRCH') return true + throw error + } +} + +function readyHookPids(root: string): number[] { + const ready: unknown = JSON.parse(readFileSync(join(root, 'hook-ready.json'), 'utf8')) + if (ready === null || typeof ready !== 'object' || !('hook' in ready) || !('child' in ready) + || typeof ready.hook !== 'number' || !Number.isSafeInteger(ready.hook) || ready.hook < 1 + || typeof ready.child !== 'number' || !Number.isSafeInteger(ready.child) || ready.child < 1) { + throw new Error('pack fixture hook readiness requires positive process ids') + } + return [ready.hook, ready.child] +} + +afterEach(() => { + for (const root of roots.splice(0)) removeFixtureSafely(root) +}) + +describe('release pack CLI', () => { + it.each([undefined, '1', '2'])('packs only public family members and records their publish order (concurrency=%s)', async (concurrency) => { + const root = fixture() + // Publication places zebra before alpha; pnpm's workspace dev edge points in the opposite direction. + packageFixture(root, 'vendor/alpha', '@deepseek-ai/alpha', { peerDependencies: { '@deepseek-ai/zebra': '*' } }) + packageFixture(root, 'vendor/zebra', '@deepseek-ai/zebra', { devDependencies: { '@deepseek-ai/alpha': 'workspace:*' } }) + write(join(root, 'vendor/zebra/node_modules/@deepseek-ai/alpha/package.json'), JSON.stringify({ name: '@deepseek-ai/alpha', version: '1.2.3' })) + packageFixture(root, 'vendor/private', '@deepseek-ai/private', { private: true }) + packageFixture(root, 'native/unselected', '@deepseek-ai/unselected') + const family = releaseFamily('vendor') + const members = family.publishOrder(family.members(root)).order + const expectedOrder = members.map(tarballName) + + const result = await runPack(root, concurrency === undefined ? [] : ['--concurrency', concurrency]) + + expect(result.status, result.stderr).toBe(0) + expect(readPublishOrder(join(root, 'packed'))).toEqual(expectedOrder) + expect(readdirSync(join(root, 'packed')).sort()).toEqual([...expectedOrder, PUBLISH_ORDER_FILE].sort()) + for (const member of members) { + expect(packedIdentity(join(root, 'packed', tarballName(member)))) + .toEqual({ name: member.name, version: member.version }) + } + const hooks = readFileSync(join(root, 'hooks.log'), 'utf8').trim().split('\n') + const expectedHooks = members.flatMap(member => ['prepack', 'prepare', 'postpack'].map(phase => `${member.name}:${phase}`)) + expect([...hooks].sort()).toEqual([...expectedHooks].sort()) + if (concurrency !== '2') expect(hooks).toEqual(expectedHooks) + }, 120_000) + + it('overlaps two independent packs and keeps later packages within the concurrency limit', async () => { + const root = fixture() + const names = ['alpha', 'bravo', 'charlie'].map(name => `@deepseek-ai/${name}`) + for (const name of names) { + const directory = `vendor/${name.slice('@deepseek-ai/'.length)}` + packageFixture(root, directory, name, { + scripts: { prepack: 'node hook.cjs start', postpack: 'node hook.cjs finish' }, + }) + write(join(root, directory, 'hook.cjs'), [ + "const { get } = require('node:http');", + "const { name } = require('./package.json');", + 'const url = `${process.env.DSH_PACK_COORDINATOR}/${process.argv[2]}?name=${encodeURIComponent(name)}&pid=${process.pid}`;', + 'const request = get(url, { agent: false }, response => {', + ' response.resume();', + " response.on('end', () => { if (response.statusCode !== 200) process.exitCode = 1; });", + '});', + "request.on('error', error => { console.error(error); process.exitCode = 1; });", + '', + ].join('\n')) + } + + const active = new Set() + const waiting = new Map() + const started: string[] = [] + const completed: string[] = [] + const pids = new Set() + const errors: string[] = [] + let maximumActive = 0 + const server = createServer((request, response) => { + const url = new URL(request.url ?? '/', 'http://localhost') + const name = url.searchParams.get('name') ?? '' + const pid = Number(url.searchParams.get('pid')) + if (!names.includes(name) || !Number.isSafeInteger(pid) || pid < 1 + || (url.pathname !== '/start' && url.pathname !== '/finish')) { + errors.push(`unexpected hook request ${request.url ?? ''}`) + response.writeHead(400).end() + return + } + pids.add(pid) + if (url.pathname === '/start') { + if (active.has(name)) errors.push(`${name} started twice`) + started.push(name) + active.add(name) + maximumActive = Math.max(maximumActive, active.size) + waiting.set(name, response) + } else { + if (!active.delete(name)) errors.push(`${name} finished without starting`) + completed.push(name) + response.end() + } + }) + const release = (name: string | undefined): void => { + const response = name === undefined ? undefined : waiting.get(name) + if (name === undefined || response === undefined) throw new Error('pack hook has not reached its barrier') + waiting.delete(name) + response.end() + } + const controller = new AbortController() + const timer = setTimeout(() => { controller.abort() }, 90_000) + let pending: ReturnType | undefined + try { + await new Promise((resolveListen, rejectListen) => { + server.once('error', rejectListen) + server.listen(0, '127.0.0.1', () => { + server.off('error', rejectListen) + resolveListen() + }) + }) + const address = server.address() + if (address === null || typeof address === 'string') throw new Error('pack coordinator has no TCP address') + const gate = packGate(root, ['--concurrency', '2']) + gate.env = { ...gate.env, DSH_PACK_COORDINATOR: `http://127.0.0.1:${address.port}` } + pending = runGate(gate, controller.signal) + + await vi.waitFor(() => { expect(started).toHaveLength(2) }, { timeout: 30_000, interval: 25 }) + expect(active.size).toBe(2) + expect([...pids].every(pid => !stopped(pid))).toBe(true) + expect(completed).toEqual([]) + release(started[0]) + + await vi.waitFor(() => { expect(started).toHaveLength(3) }, { timeout: 30_000, interval: 25 }) + expect(completed).toEqual([started[0]]) + expect(active.size).toBe(2) + expect(maximumActive).toBe(2) + release(started[1]) + release(started[2]) + + const result = await pending + expect(result.error).toBeUndefined() + expect(result.aborted).toBe(false) + expect(result.signalCode).toBeNull() + expect(result.exitCode, result.output.map(chunk => chunk.text).join('')).toBe(0) + expect(errors).toEqual([]) + expect([...started].sort()).toEqual(names) + expect([...completed].sort()).toEqual(names) + expect(active.size).toBe(0) + expect(maximumActive).toBe(2) + await vi.waitFor(() => { expect([...pids].every(stopped)).toBe(true) }, { timeout: 10_000, interval: 25 }) + expect(readPublishOrder(join(root, 'packed'))).toHaveLength(3) + } finally { + clearTimeout(timer) + controller.abort() + try { + await pending + } finally { + if (server.listening) { + const closed = new Promise((resolveClose, rejectClose) => { + server.close((error) => { + if (error === undefined) resolveClose() + else rejectClose(error) + }) + }) + server.closeAllConnections() + await closed + } + } + } + }, 120_000) + + it('does not broaden an empty parallel family to the whole workspace', async () => { + const root = fixture() + packageFixture(root, 'vendor/private', '@deepseek-ai/private', { private: true }) + packageFixture(root, 'native/unselected', '@deepseek-ai/unselected') + + const result = await runPack(root, ['--concurrency', '8']) + + expect(result.status, result.stderr).toBe(0) + expect(readFileSync(join(root, 'packed', PUBLISH_ORDER_FILE), 'utf8')).toBe('\n') + expect(readdirSync(join(root, 'packed'))).toEqual([PUBLISH_ORDER_FILE]) + expect(existsSync(join(root, 'hooks.log'))).toBe(false) + }) + + it('leaves no publish order after a pack hook fails', async () => { + const root = fixture() + packageFixture(root, 'vendor/failing', '@deepseek-ai/failing', { scripts: { prepack: 'node failing.cjs' } }) + write(join(root, 'vendor/failing/failing.cjs'), "throw new Error('pack fixture failure')\n") + + const result = await runPack(root, ['--concurrency', '8']) + + expect(result.status).not.toBe(0) + expect(result.stderr).toContain('pack fixture failure') + expect(existsSync(join(root, 'packed', PUBLISH_ORDER_FILE))).toBe(false) + }) + + it('rejects a successful command that did not produce the expected member tarball', async () => { + const root = fixture() + packageFixture(root, 'vendor/expected', '@deepseek-ai/expected', { scripts: { prepack: 'node rename.cjs' } }) + write(join(root, 'vendor/expected/rename.cjs'), [ + "const fs = require('node:fs');", + "const manifest = require('./package.json');", + "manifest.name = '@deepseek-ai/renamed';", + "fs.writeFileSync('package.json', JSON.stringify(manifest));", + '', + ].join('\n')) + + const result = await runPack(root, ['--concurrency', '8']) + + expect(result.status).not.toBe(0) + expect(result.stderr).toContain('@deepseek-ai/expected produced no tarball') + expect(existsSync(join(root, 'packed', 'deepseek-ai-renamed-1.2.3.tgz'))).toBe(true) + expect(existsSync(join(root, 'packed', PUBLISH_ORDER_FILE))).toBe(false) + }) + + it('validates every parallel tarball before recording the publish order', async () => { + const root = fixture() + packageFixture(root, 'packages/util/alpha', '@deepseek-ai/alpha') + packageFixture(root, 'packages/util/zebra', '@deepseek-ai/zebra', { files: ['lib/index.js', 'src/index.ts'] }) + write(join(root, 'packages/util/zebra/src/index.ts'), 'export const source = 1\n') + write(join(root, 'apps/web/dist/index.html'), '
') + writeClientBuildRecord(root, officialClientBuildEnvironment(root, { DSH_CLIENT_COMMIT_HASH: commit })) + + const result = await runPack(root, ['--family', 'dsh', '--concurrency', '8']) + + expect(result.status).not.toBe(0) + expect(result.stderr).toContain('@deepseek-ai/zebra publishes source file package/src/index.ts') + expect(readdirSync(join(root, 'packed')).filter(name => name.endsWith('.tgz'))).toHaveLength(2) + expect(existsSync(join(root, 'packed', PUBLISH_ORDER_FILE))).toBe(false) + }) + + it('stops the ready pack hook and its stalled descendant before fixture cleanup', async () => { + const root = fixture() + packageFixture(root, 'vendor/stalled', '@deepseek-ai/stalled', { scripts: { prepack: 'node stalled.cjs' } }) + write(join(root, 'vendor/stalled/stalled.cjs'), [ + "const { spawn } = require('node:child_process');", + "spawn(process.execPath, ['descendant.cjs'], { stdio: 'inherit' });", + '', + ].join('\n')) + write(join(root, 'vendor/stalled/descendant.cjs'), [ + "const fs = require('node:fs');", + "const path = require('node:path');", + "process.on('SIGTERM', () => {});", + 'setInterval(() => {}, 1000);', + "fs.writeFileSync(path.join(process.env.DSH_PACK_FIXTURE, 'hook-ready.json'), JSON.stringify({ hook: process.ppid, child: process.pid }));", + '', + ].join('\n')) + const controller = new AbortController() + const timer = setTimeout(() => { controller.abort() }, 90_000) + const pending = runGate(packGate(root, ['--concurrency', '8']), controller.signal) + try { + let pids: number[] = [] + await vi.waitFor(() => { + pids = readyHookPids(root) + expect(pids.every(pid => !stopped(pid))).toBe(true) + }, { timeout: 30_000, interval: 25 }) + + controller.abort() + const result = await pending + + expect(result.aborted).toBe(true) + expect(result.status).toBe('failed') + await vi.waitFor(() => { expect(pids.every(stopped)).toBe(true) }, { timeout: 10_000, interval: 25 }) + expect(existsSync(join(root, 'packed', PUBLISH_ORDER_FILE))).toBe(false) + } finally { + clearTimeout(timer) + controller.abort() + await pending + } + }, 120_000) +}) diff --git a/scripts/release/pack.ts b/scripts/release/pack.ts index 6f187e83af..c840f8e762 100644 --- a/scripts/release/pack.ts +++ b/scripts/release/pack.ts @@ -1,6 +1,7 @@ /** - * Pack one release family's whole publish set into a single directory, in - * publish order, and record that order for the publish step. + * Pack one release family's whole publish set into a single directory and + * record its publish order for the publish step. Default serial packs execute + * in that order; explicitly parallel packs use pnpm's bounded scheduler. * * The pack step is the release boundary: it runs without credentials, produces * every tarball from one commit, and hands the publish step exactly those bytes @@ -19,16 +20,13 @@ import { PUBLISH_ORDER_FILE, tarballFiles } from './tarball.ts' const DEFAULT_OUTPUT = 'dist/npm' /** - * Pack one member and check what its tarball carries. + * Check one member's expected tarball and its published files. * @param family - the release family being packed. * @param member - the member to pack. * @param destination - absolute output directory. * @returns The tarball filename. */ -async function packMember(family: ReleaseFamily, member: ReleaseMember, destination: string): Promise { - const invocation = pnpmInvocation(['--dir', member.directory, 'pack', '--pack-destination', destination]) - await runConcurrent(invocation.command, invocation.args) - +function validatePackedMember(family: ReleaseFamily, member: ReleaseMember, destination: string): string { const filename = tarballName(member) const tarball = join(destination, filename) if (!existsSync(tarball)) throw new Error(`${member.name} produced no tarball at ${tarball}`) @@ -36,6 +34,38 @@ async function packMember(family: ReleaseFamily, member: ReleaseMember, destinat return filename } +/** + * Pack serial members in publish order, or batch an explicitly parallel family. + * @param family - Release family whose payload checks apply to every tarball. + * @param members - Exact public members in the recorded publish order. + * @param destination - Absolute output directory. + * @param concurrency - Maximum active package packs; one preserves serial publication order. + * @returns Validated tarball filenames in publish order. + */ +async function packMembers( + family: ReleaseFamily, members: readonly ReleaseMember[], destination: string, concurrency: number, +): Promise { + if (concurrency === 1) { + const order: string[] = [] + for (const member of members) { + const invocation = pnpmInvocation(['--dir', member.directory, 'pack', '--pack-destination', destination]) + await runConcurrent(invocation.command, invocation.args) + order.push(validatePackedMember(family, member, destination)) + } + return order + } + if (members.length === 0) return [] + const invocation = pnpmInvocation([ + 'pack', + '--recursive', + `--workspace-concurrency=${String(concurrency)}`, + ...members.map(member => `--filter=${member.name}`), + '--pack-destination', destination, + ]) + await runConcurrent(invocation.command, invocation.args) + return members.map(member => validatePackedMember(family, member, destination)) +} + /** * @returns The validated `--concurrency` value; 1 (the default) packs the * members one at a time, exactly as the credentialed publish workflows run it. @@ -68,20 +98,7 @@ async function main(): Promise { rmSync(destination, { recursive: true, force: true }) mkdirSync(destination, { recursive: true }) - // Members pack in a bounded pool; the recorded publish order stays the - // members' order regardless of completion order, because each worker writes - // its result at the member's own position. - const order = new Array(members.length) - let cursor = 0 - await Promise.all(Array.from({ length: Math.min(concurrency, members.length) }, async () => { - while (cursor < members.length) { - const index = cursor - cursor += 1 - const member = members[index] - if (member === undefined) break - order[index] = await packMember(family, member, destination) - } - })) + const order = await packMembers(family, members, destination, concurrency) writeFileSync(join(destination, PUBLISH_ORDER_FILE), `${order.join('\n')}\n`) console.log(`release pack: family ${family.id}, ${String(order.length)} tarball(s) in ${values.out ?? DEFAULT_OUTPUT}`) diff --git a/scripts/run-ci-bench.ts b/scripts/run-ci-bench.ts new file mode 100644 index 0000000000..8d2e4c042a --- /dev/null +++ b/scripts/run-ci-bench.ts @@ -0,0 +1,19 @@ +/** Build benchmark artifacts with TypeScript diagnostics owned by required full builds. */ + +import { resolve } from 'node:path' +import { buildLibraryArtifacts, runPnpmCommand } from './compile-referenced-projects.ts' + +/** + * Build native support, libraries, benchmark workers, and Web assets before running the existing benchmark suite. + * @param root - Repository root containing all build configurations. + * @throws If any build or the benchmark suite fails. + */ +export function runCiBench(root: string): void { + runPnpmCommand(root, ['run', 'build:native-system']) + buildLibraryArtifacts(root, false, 'benchmark-emit') + runPnpmCommand(root, ['exec', 'tsdown', '--config-loader', 'native', '--config', 'benchmarks/tsdown.config.ts']) + runPnpmCommand(root, ['run', 'build:web']) + runPnpmCommand(root, ['run', 'test:bench:built']) +} + +if (import.meta.main) runCiBench(resolve(import.meta.dirname, '..')) diff --git a/scripts/run-gates.spec.ts b/scripts/run-gates.spec.ts index 8f676999e9..0cc6d41d7d 100644 --- a/scripts/run-gates.spec.ts +++ b/scripts/run-gates.spec.ts @@ -156,6 +156,13 @@ function withPlatform(platform: NodeJS.Platform, action: () => T): T { try { return action() } finally { Object.defineProperty(process, 'platform', original) } } +// Mutates worker-global state: only use for synchronous, non-concurrent graph inspection. +function withNodeVersion(version: string, action: () => T): T { + const original = Object.getOwnPropertyDescriptor(process.versions, 'node')! + Object.defineProperty(process.versions, 'node', { ...original, value: version }) + try { return action() } finally { Object.defineProperty(process.versions, 'node', original) } +} + describe('CI worker allocation', () => { it.each([1, 2, 4, 8, 16, 64])('shares a %i CPU coverage budget without multiplying pools', (cpus) => { const env = ciWorkerEnvironment('ci-coverage', {}, cpus) @@ -243,7 +250,7 @@ describe('gate graph validation', () => { await expect(runGates(subject, subject.length, execute)).resolves.toHaveLength(subject.length) }) - it('builds the native addon before benchmarks through the ci-bench script chain', () => { + it('uses the internal CI benchmark runner without changing the public benchmark scripts', () => { const subject = withPnpmEntrypoint(() => gatesForMode('ci-bench')) const { scripts } = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')) as { scripts: Record @@ -253,8 +260,8 @@ describe('gate graph validation', () => { expect(subject).toHaveLength(1) expect(subject[0]).toMatchObject({ id: 'bench', - displayCommand: 'pnpm run test:bench', - args: ['/private/pnpm.cjs', 'run', 'test:bench'], + displayCommand: 'pnpm exec tsx scripts/run-ci-bench.ts', + args: ['/private/pnpm.cjs', 'exec', 'tsx', 'scripts/run-ci-bench.ts'], }) expect(scripts['test:bench']).toBe('npm run build:bench && npm run build:web && npm run test:bench:built') expect(scripts['build:bench']).toBe( @@ -814,20 +821,76 @@ describe('Typert contract preparation', () => { }) describe('Node compatibility graph', () => { - it('runs the jsdom environment smoke on every advertised Node line', () => { - const subject = withPnpmEntrypoint(() => gatesForMode('node-compat')) + it.each(['22.19.0', '24.9.0', '26.0.0'])('runs all source compatibility smokes serially on Node %s', (version) => { + const subject = withNodeVersion(version, () => + withPnpmEntrypoint(() => gatesForMode('node-compat'))) - expect(subject.find(item => item.id === 'vitest-jsdom-smoke')).toMatchObject({ - label: 'Vitest jsdom smoke', + expect(subject.filter(item => item.id === 'source-compat-smokes')).toMatchObject([{ + label: 'source compatibility smokes', args: [ '/private/pnpm.cjs', 'exec', 'vitest', 'run', + 'packages/workflow/workflow-ptc/tests/source-runtime.compat.spec.ts', + 'packages/session/session-persistence-jsonl/tests/zstd.compat.spec.ts', + 'apps/cli/tests/source-launch.compat.spec.ts', 'scripts/vitest-environment.compat.spec.ts', + 'packages/boot/app-boot/tests/profile-resolution.spec.ts', + 'packages/boot/app-boot/tests/profile-resolution-service.spec.ts', + 'packages/boot/app-boot/tests/profile-resolution-worker-bootstrap.spec.ts', + '--no-file-parallelism', + '--maxWorkers=1', ], + env: { VITEST_MAX_WORKERS: '1' }, + }]) + }) + + it.each(['1', '8'])('keeps source compatibility smokes serial with inherited VITEST_MAX_WORKERS=%s', (workers) => { + const subject = withEnv('VITEST_MAX_WORKERS', workers, () => + withPnpmEntrypoint(() => gatesForMode('node-compat'))) + + expect(subject.find(item => item.id === 'source-compat-smokes')?.env).toEqual({ VITEST_MAX_WORKERS: '1' }) + }) + + it.each([undefined, '1'])('builds the complete Node 22 artifacts once with skip-typecheck=%s', (skipTypecheck) => { + const subject = withNodeVersion('22.19.0', () => + withEnv('DSH_NODE_COMPAT_SKIP_TYPECHECK', skipTypecheck, () => + withPnpmEntrypoint(() => gatesForMode('node-compat')))) + + expect(subject.filter(item => item.displayCommand === 'pnpm run build --artifacts-only')).toHaveLength(1) + expect(subject.find(item => item.id === 'build')?.args).toEqual([ + '/private/pnpm.cjs', 'run', 'build', '--artifacts-only', + ]) + expect(subject.map(item => item.id)).not.toContain('build:web') + expect(subject.find(item => item.id === 'cli-lazy-search-startup-smoke')).toMatchObject({ + needs: ['build'], + env: { DSH_REQUIRE_BUILT_CLI_SMOKE: '1' }, }) }) + + it.each(['passed', 'failed'] as const)('runs the Node 22 lazy-search smoke only after a passed build (%s)', async (buildStatus) => { + const subject = withNodeVersion('22.19.0', () => + withEnv('DSH_NODE_COMPAT_SKIP_TYPECHECK', '1', () => + withPnpmEntrypoint(() => gatesForMode('node-compat')))) + let buildCompleted = false + let smokeStarted = false + const results = await runGates(subject, 2, async (item) => { + if (item.id === 'build') { + buildCompleted = true + return resultFor(item, buildStatus) + } + if (item.id === 'cli-lazy-search-startup-smoke') { + expect(buildCompleted).toBe(true) + smokeStarted = true + } + return resultFor(item) + }) + + expect(smokeStarted).toBe(buildStatus === 'passed') + expect(results.find(item => item.gate.id === 'cli-lazy-search-startup-smoke')?.status) + .toBe(buildStatus === 'passed' ? 'passed' : 'skipped') + }) }) describe('Node 24 lane ownership', () => { diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index c76675aa80..25aaf53195 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -285,7 +285,7 @@ export function gatesForMode(selected: Mode): Gate[] { case 'ci-unit': return ciUnitGates() case 'ci-bench': - return [pnpmScript('bench', 'test:bench', { label: 'performance benchmarks' })] + return [pnpmExec('bench', ['tsx', 'scripts/run-ci-bench.ts'], { label: 'performance benchmarks' })] case 'ci-snapshot': return [ciBuildGate(), snapshotGate()] case 'ci-artifacts': @@ -409,45 +409,31 @@ function nodeCompatGates(): Gate[] { return [ ...typecheck, pnpmScript('build', 'build', { + ...pnpmInvocation(['run', 'build', '--artifacts-only']), + displayCommand: 'pnpm run build --artifacts-only', ...typecheck.length === 0 ? {} : { needs: ['typecheck'] }, }), - pnpmScript('build:web', 'build:web', { - label: 'Web frontend build', - needs: ['build'], - }), ...nodeCompatSmokeGates({ cliSmoke: true }), ] } function nodeCompatSmokeGates(options: { cliSmoke?: boolean } = {}): Gate[] { const gates: Gate[] = [ - pnpmExec('source-worker-smoke', [ + // Serial files keep CLI and PTC children from overlapping. Vitest's + // environment worker override takes precedence over its CLI flags. + pnpmExec('source-compat-smokes', [ 'vitest', 'run', 'packages/workflow/workflow-ptc/tests/source-runtime.compat.spec.ts', - ], { label: 'source worker smoke' }), - pnpmExec('jsonl-zstd-smoke', [ - 'vitest', - 'run', 'packages/session/session-persistence-jsonl/tests/zstd.compat.spec.ts', - ], { label: 'JSONL Zstandard smoke' }), - pnpmExec('dsh-source-launch-smoke', [ - 'vitest', - 'run', 'apps/cli/tests/source-launch.compat.spec.ts', - ], { label: 'dsh source-launch smoke' }), - pnpmExec('vitest-jsdom-smoke', [ - 'vitest', - 'run', 'scripts/vitest-environment.compat.spec.ts', - ], { label: 'Vitest jsdom smoke' }), - pnpmExec('profile-resolution-smoke', [ - 'vitest', - 'run', 'packages/boot/app-boot/tests/profile-resolution.spec.ts', 'packages/boot/app-boot/tests/profile-resolution-service.spec.ts', 'packages/boot/app-boot/tests/profile-resolution-worker-bootstrap.spec.ts', - ], { label: 'profile resolution smoke' }), + '--no-file-parallelism', + '--maxWorkers=1', + ], { label: 'source compatibility smokes', env: { VITEST_MAX_WORKERS: '1' } }), ] if (options.cliSmoke) { gates.push( @@ -458,7 +444,7 @@ function nodeCompatSmokeGates(options: { cliSmoke?: boolean } = {}): Gate[] { ], { label: 'CLI lazy-search startup smoke', env: { DSH_REQUIRE_BUILT_CLI_SMOKE: '1' }, - needs: ['build:web'], + needs: ['build'], }), ) } diff --git a/scripts/tests/ci-release-selfhosted.spec.ts b/scripts/tests/ci-release-selfhosted.spec.ts index b7da434bbb..23dcc7a8df 100644 --- a/scripts/tests/ci-release-selfhosted.spec.ts +++ b/scripts/tests/ci-release-selfhosted.spec.ts @@ -146,7 +146,9 @@ for (const [file, jobIds] of [['release.yml', ['dependencies', 'pack']], ['relea const output = family === 'dsh' ? 'dist/npm' : 'dist/npm-vendor' expect(job.steps[0]?.with?.['fetch-depth']).toBe(0) expect(commands).toContain('pnpm run release:verify --family ' + family) - expect(commands).toContain('pnpm run ' + (family === 'dsh' ? 'build:official' : 'build:lib:host')) + expect(commands).toContain(family === 'dsh' + ? "${{ github.event_name == 'pull_request' && 'pnpm run build --artifacts-only --profile official' || 'pnpm run build:official' }}" + : "${{ github.event_name == 'pull_request' && 'pnpm exec tsx scripts/compile-referenced-projects.ts host-libraries' || 'pnpm run build:lib:host' }}") expect(commands).toContain('pnpm run release:pack --family ' + family + ' --out ' + output + ' --concurrency 8') expect(commands).toContain('pnpm run release:verify-packed-install --family ' + family + ' --from ' + output + (family === 'dsh' ? ' --from dist/npm-vendor --from dist/npm-landlock' : '')) diff --git a/scripts/translation-links.spec.ts b/scripts/translation-links.spec.ts index 35efde120e..bad098e23e 100644 --- a/scripts/translation-links.spec.ts +++ b/scripts/translation-links.spec.ts @@ -4,6 +4,7 @@ import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' +import { parseMarkdown } from './markdown.ts' import { normalizeTranslationMarkdownLinks, rewriteTranslationLinkLocales, @@ -183,6 +184,33 @@ describe('translation link locale validation', () => { }) describe('translation link rewriting and normalization', () => { + it('reuses an authored tree without changing its offsets, switcher exemption, or reference resolution', () => { + const root = fixture() + const input = '# 指南\n\n[English](guide.md) | 中文\n\n[行内](reference.md?view=full&x=1#overview "标题")\n\n[引用][ref]\n\n[ref]: "标注"\n[ref]: unpaired.md\n' + const context = linkContext(root, 'docs/guide.zh.md') + const tree = parseMarkdown(input) + const originalTree = structuredClone(tree) + + expect(normalizeTranslationMarkdownLinks(input, context, ['guide.md'], tree)).toBe( + '# 指南\n\n[English](guide.md) | 中文\n\n[行内](dsh-translation-target:docs/reference.md?view=full&x=1#overview "标题")\n\n[引用][ref]\n\n[ref]: "标注"\n[ref]: unpaired.md\n', + ) + expect(translationLinkLocaleViolations(input, context, ['guide.md'], tree)).toEqual([ + { + sourcePath: 'docs/guide.zh.md', + line: 5, + url: 'reference.md?view=full&x=1#overview', + expectedUrl: 'reference.zh.md?view=full&x=1#overview', + }, + { + sourcePath: 'docs/guide.zh.md', + line: 9, + url: 'reference.md#tail', + expectedUrl: 'reference.zh.md#tail', + }, + ]) + expect(tree).toEqual(originalTree) + }) + it('rewrites only the destination while preserving the suffix and title', () => { const root = fixture() const input = '[概览](reference.md?view=full&mode=all#overview "reference.md title")\n' diff --git a/scripts/translation-links.ts b/scripts/translation-links.ts index af7c991fd4..d149d44372 100644 --- a/scripts/translation-links.ts +++ b/scripts/translation-links.ts @@ -229,8 +229,8 @@ function visitDocumentLinkNodes( markdown: string, skipTargets: readonly string[], visitor: (node: LinkNode) => void, + tree: Nodes = parseMarkdown(markdown), ): void { - const tree = parseMarkdown(markdown) const switcherOffset = languageSwitcherLinkOffset(tree, markdown, skipTargets) const referencedIdentifiers = new Set() const visitedDefinitions = new Set() @@ -255,20 +255,29 @@ function visitResolvedDocumentLinks( context: TranslationLinkContext, skipTargets: readonly string[], visitor: (node: LinkNode, destination: MarkdownDestination, resolved: ResolvedTranslationLink) => void, + tree?: Nodes, ): void { visitDocumentLinkNodes(markdown, skipTargets, (node) => { if (isExternalOrAbsoluteMarkdownUrl(node.url)) return const destination = markdownDestination(markdown, node) const resolved = resolveTranslationLink(node.url, context, destination.url) if (resolved !== undefined) visitor(node, destination, resolved) - }) + }, tree) } -/** Return one violation per wrong-locale link or link definition. */ +/** + * Return one violation per wrong-locale link or link definition. + * @param markdown - Exact authored Markdown text used for link offsets. + * @param context - Repository and locale used to resolve relative links. + * @param skipTargets - Accepted language-switcher destinations. + * @param tree - Optional GFM parse of this exact Markdown text; never a normalized or edited document. + * @returns Wrong-locale links in document order. + */ export function translationLinkLocaleViolations( markdown: string, context: TranslationLinkContext, skipTargets: readonly string[] = [], + tree?: Nodes, ): TranslationLinkLocaleViolation[] { const violations: TranslationLinkLocaleViolation[] = [] visitResolvedDocumentLinks(markdown, context, skipTargets, (node, destination, resolved) => { @@ -279,7 +288,7 @@ export function translationLinkLocaleViolations( url: destination.url, expectedUrl: resolved.expectedUrl, }) - }) + }, tree) return violations } @@ -297,11 +306,19 @@ export function rewriteTranslationLinkLocales( return { content: applyReplacements(markdown, replacements), rewritten: replacements.length } } -/** Normalize only paired-document locale paths while retaining every other byte and URL suffix. */ +/** + * Normalize only paired-document locale paths while retaining every other byte and URL suffix. + * @param markdown - Exact authored Markdown text used for link offsets. + * @param context - Repository and locale used to resolve relative links. + * @param skipTargets - Accepted language-switcher destinations. + * @param tree - Optional GFM parse of this exact Markdown text; never a normalized or edited document. + * @returns Markdown with paired-document destinations replaced by semantic targets. + */ export function normalizeTranslationMarkdownLinks( markdown: string, context: TranslationLinkContext, skipTargets: readonly string[] = [], + tree?: Nodes, ): string { const replacements: Replacement[] = [] visitResolvedDocumentLinks(markdown, context, skipTargets, (_node, destination, resolved) => { @@ -309,7 +326,7 @@ export function normalizeTranslationMarkdownLinks( destination, `dsh-translation-target:${resolved.pair.source}${resolved.suffix}`, )) - }) + }, tree) return applyReplacements(markdown, replacements) } diff --git a/scripts/translation-pairing-record.ts b/scripts/translation-pairing-record.ts index 3990ec11c2..09dd1c8f41 100644 --- a/scripts/translation-pairing-record.ts +++ b/scripts/translation-pairing-record.ts @@ -68,11 +68,10 @@ function headingText(node: Nodes): string { * Fenced code blocks and generated regions, which the pairing gate requires to * be identical on both sides, are left out of each section's blocks. */ -function sections(markdown: string): Section[] { +function sections(markdown: string, tree: Nodes = parseTranslationMarkdown(markdown)): Section[] { const frontmatter = /^---\n[\s\S]*?\n---(?:\n|$)/.exec(markdown)?.[0].length ?? 0 const result: Section[] = [{ blocks: frontmatter > 0 ? [markdown.slice(0, frontmatter)] : [] }] const regionLines = generatedRegions(markdown).map(region => [region.begin + 1, region.end + 1] as const) - const tree = parseTranslationMarkdown(markdown) if (tree.type !== 'root') throw new Error('Markdown parser returned a non-root tree') for (const node of tree.children) { if ((node.position?.start.offset ?? 0) < frontmatter) continue @@ -106,6 +105,7 @@ function sectionHash(blocks: string[]): string { * @param en - English document text. * @param zh - Chinese document text. * @param context - Link-resolution inputs for both sides. + * @param trees - Optional GFM parses of the exact English and Chinese inputs; normalized text is parsed separately when it changes. * @returns Entries in document order. * @throws Error when the two sides have different heading counts or malformed generated regions. */ @@ -114,9 +114,12 @@ export function computeTranslationPairingRecord( en: string, zh: string, context: TranslationPairingRecordContext, + trees?: { readonly en: Nodes; readonly zh: Nodes }, ): TranslationPairingRecord { - const enSections = sections(normalizeTranslationMarkdownLinks(en, { ...context, sourcePath: paths.source })) - const zhSections = sections(normalizeTranslationMarkdownLinks(zh, { ...context, sourcePath: paths.zh })) + const normalizedEn = normalizeTranslationMarkdownLinks(en, { ...context, sourcePath: paths.source }, [], trees?.en) + const normalizedZh = normalizeTranslationMarkdownLinks(zh, { ...context, sourcePath: paths.zh }, [], trees?.zh) + const enSections = sections(normalizedEn, normalizedEn === en ? trees?.en : undefined) + const zhSections = sections(normalizedZh, normalizedZh === zh ? trees?.zh : undefined) if (enSections.length !== zhSections.length) { throw new Error(`${paths.source} has ${enSections.length - 1} heading(s) but ${paths.zh} has ${zhSections.length - 1}`) } diff --git a/scripts/translation-pairing.spec.ts b/scripts/translation-pairing.spec.ts index 89ab41d09d..2e18c15175 100644 --- a/scripts/translation-pairing.spec.ts +++ b/scripts/translation-pairing.spec.ts @@ -250,6 +250,31 @@ describe('translation pairing records', () => { .toEqual(record(`${en}\nSee [bar](bar.zh.md).\n`, `${zh}\nSee [bar](bar.md).\n`)) }) + it('retains section hashes after normalized links move later heading offsets', () => { + const english = '# Guide\n\n[Reference](bar.md?x=1&y=2#tail)\n\n## Tail\n\nTail body.\n' + const chinese = '# 指南\n\n[参考](bar.zh.md?x=1&y=2#tail)\n\n## 尾部\n\n尾部正文。\n' + const trees = { en: parseTranslationMarkdown(english), zh: parseTranslationMarkdown(chinese) } + const originalTrees = structuredClone(trees) + const computed = computeTranslationPairingRecord(paths, english, chinese, context, trees) + + expect([...computed]).toEqual([ + ['/guide', { en: '3f43f06d1244d874', zh: '021cfe53b05ba084' }], + ['/guide/tail', { en: '945d8537bf1f2b5f', zh: '0a851cb395705d76' }], + ]) + expect(computed).toEqual(record(english, chinese)) + expect(trees).toEqual(originalTrees) + }) + + it('retains section hashes when the authored tree needs no link normalization', () => { + const english = '# Guide\n\nBody.\n' + const chinese = '# 指南\n\n正文。\n' + const trees = { en: parseTranslationMarkdown(english), zh: parseTranslationMarkdown(chinese) } + + expect([...computeTranslationPairingRecord(paths, english, chinese, context, trees)]).toEqual([ + ['/guide', { en: '5e1bfa97a822af7a', zh: '3409c92bef7306bb' }], + ]) + }) + it('refuses sides with different heading counts', () => { expect(() => record(en, `${zh}\n## 额外\n`)).toThrow('docs/foo.md has 3 heading(s) but docs/foo.zh.md has 4') }) diff --git a/scripts/verify-translation-pairing.ts b/scripts/verify-translation-pairing.ts index fe2f1b494b..43489c6b3c 100644 --- a/scripts/verify-translation-pairing.ts +++ b/scripts/verify-translation-pairing.ts @@ -239,8 +239,12 @@ for (const source of [...pairAnchors].sort()) { const sourceText = sourceContent.toString('utf8') const zhText = zhContent.toString('utf8') let current: ReturnType + let sourceTree: ReturnType + let zhTree: ReturnType try { - current = computeTranslationPairingRecord(paths, sourceText, zhText, recordContext) + sourceTree = parseTranslationMarkdown(sourceText) + zhTree = parseTranslationMarkdown(zhText) + current = computeTranslationPairingRecord(paths, sourceText, zhText, recordContext, { en: sourceTree, zh: zhTree }) } catch (error) { errors.push(`${source} ↔ ${zh}: ${error instanceof Error ? error.message : String(error)}`) state.set(source, 'out-of-sync') @@ -265,13 +269,13 @@ for (const source of [...pairAnchors].sort()) { sourcePath: source, isTranslationPairSource, repositoryFileExists, - }, zhSwitcherTargets), + }, zhSwitcherTargets, sourceTree), ...translationLinkLocaleViolations(zhText, { repoRoot: root, sourcePath: zh, isTranslationPairSource, repositoryFileExists, - }, sourceSwitcherTargets), + }, sourceSwitcherTargets, zhTree), ]) { errors.push(`${violation.sourcePath}:${violation.line}: link target ${JSON.stringify(violation.url)} uses the wrong locale; expected ${JSON.stringify(violation.expectedUrl)}`) state.set(source, 'out-of-sync') @@ -309,8 +313,6 @@ for (const source of [...pairAnchors].sort()) { state.set(source, 'out-of-sync') } - const sourceTree = parseTranslationMarkdown(sourceText) - const zhTree = parseTranslationMarkdown(zhText) if (!hasLanguageSwitcher(zhTree, zhText, sourceSwitcherTargets)) { errors.push(`${zh}: missing language switcher — no link to ${basename(source)}`) }