fix(webworker): retain third-party runtime sources

2026-08-25 06:43:25 +08:00
1 parent 83463aa896
commit b3081bb4be
@@ -2,5 +2,5 @@
 # side as of the last confirmed-consistent state. Both languages carry equal authority;
 # after editing either side, bring the other along and re-record with:
 #   pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-08-20-webworker-pack-lowering-and-preview.md
-2026-08-20-webworker-pack-lowering-and-preview.md: 72a6ccf856c95f835e103bd355223bf3cf42f692
-2026-08-20-webworker-pack-lowering-and-preview.zh.md: 074d44833c34a2999a5c47da809533065201b580
+2026-08-20-webworker-pack-lowering-and-preview.md: 2a05b24a3821ea905829e9da3e1edb1584135a5f
+2026-08-20-webworker-pack-lowering-and-preview.zh.md: d6c85d64038be5bc1a9ac77f1a4642c68a5d4606
@@ -10,7 +10,7 @@ The browser worker can neither compile modules at load nor be served by the prod

 ## Decision

-**Lowering happens at pack time only.** `@deepseek-ai/dsh-experimental-webworker-packer` composes the profile, materializes the closure, and lowers every JavaScript body; `LOWERING_VERSION` and `WRAPPER_PARAMS` are the pack↔worker contract and live in `src/image-layout.ts` beside the rest of the image layout. The loader wraps bodies exactly as the image holds them: a body still carrying module syntax is a refusal naming the image, and `startWorkerHost` requires the manifest's `lowered` to equal this build's contract before it mounts a single module. `lowerModuleSource` is the transform's only face and the packer its only caller; inside the worker graph, imports name the module that owns the value — never the package barrel, which is the edge that smuggled the parser in.
+**Lowering happens at pack time only.** `@deepseek-ai/dsh-experimental-webworker-packer` composes the profile, materializes the closure, and lowers every JavaScript body; `LOWERING_VERSION` and `WRAPPER_PARAMS` are the pack↔worker contract and live in `src/image-layout.ts` beside the rest of the image layout. The loader wraps bodies exactly as the image holds them: a body still carrying module syntax is a refusal naming the image, and `startWorkerHost` requires the manifest's `lowered` to equal this build's contract before it mounts a single module. `lowerModuleSource` is the transform's only face and the packer its only caller; inside the worker graph, imports name the module that owns the value — never the package barrel, which is the edge that smuggled the parser in. Source-directory exclusion applies only to workspace and vendored packages whose runtime plane is built `lib/`; installed third-party packages retain JavaScript under `src/` and `dist/` because their published entrypoints may resolve there.

 **The preview is the served page plus one tag.** One Vite build emits `dist/index.html` and `dist/preview.html` sharing every chunk; the only difference is a prepended bootstrap entry whose module connects the worker host. Startup then converges on one protocol: whichever side applies the injection table settles the `__DSH_BOOT_READY__` deferred — the served renderer resolves it in a tail script after the rendered rows, the worker bootstrap installs it before its first await and settles it after the last row — and the client entry awaits it before reading any injected state, so the chain from the stock entry onward is the served chain verbatim. The build uses a relative base so the output mounts under any static directory; the served form anchors deep SPA-fallback paths by rendering `<base href="/">` at serve time, keeping the on-disk pages byte-shared.

@@ -10,7 +10,7 @@

 ## 决定

-**Lowering 只发生在 pack 期。** `@deepseek-ai/dsh-experimental-webworker-packer` 组合 profile、物化闭包、lower 每个 JavaScript 模块体；`LOWERING_VERSION` 与 `WRAPPER_PARAMS` 是 pack↔worker 的契约，与镜像布局的其余部分一起放在 `src/image-layout.ts`。装载器完全按镜像持有的形态包装模块体：仍带模块语法的模块体是一次点名镜像的拒绝，且 `startWorkerHost` 在挂载任何模块之前要求 manifest 的 `lowered` 等于本构建的契约。`lowerModuleSource` 是转换器唯一的面、packer 是它唯一的调用方；worker 图内部的 import 一律指向拥有该值的模块——绝不指向包 barrel，那正是把解析器偷运进来的那条边。
+**Lowering 只发生在 pack 期。** `@deepseek-ai/dsh-experimental-webworker-packer` 组合 profile、物化闭包、lower 每个 JavaScript 模块体；`LOWERING_VERSION` 与 `WRAPPER_PARAMS` 是 pack↔worker 的契约，与镜像布局的其余部分一起放在 `src/image-layout.ts`。装载器完全按镜像持有的形态包装模块体：仍带模块语法的模块体是一次点名镜像的拒绝，且 `startWorkerHost` 在挂载任何模块之前要求 manifest 的 `lowered` 等于本构建的契约。`lowerModuleSource` 是转换器唯一的面、packer 是它唯一的调用方；worker 图内部的 import 一律指向拥有该值的模块——绝不指向包 barrel，那正是把解析器偷运进来的那条边。源码目录排除只用于运行期使用已构建 `lib/` 的 workspace 与 vendored 包；已安装第三方包会保留 `src/` 和 `dist/` 下的 JavaScript，因为其发布入口可能解析到这些位置。

 **preview 就是服务页面加一个标签。** 一次 Vite 构建产出共享全部 chunk 的 `dist/index.html` 与 `dist/preview.html`；唯一差异是前插的一个引导入口，其模块负责连接 worker host。启动随之汇于一个协议：应用注入表的一方 settle `__DSH_BOOT_READY__` deferred——served 渲染器在渲染完的行之后用尾部脚本 resolve，worker 引导段在首个 await 之前安装、末行生效后 settle——client 入口在读取任何注入状态前 await 它，因此从标准入口起的链路逐字就是 served 链路。构建使用相对 base，产物可挂载于任意静态目录；served 形态在 serve 期渲染 `<base href="/">` 锚定深层 SPA fallback 路径，磁盘上的两个页面保持字节共享。

@@ -2,5 +2,5 @@
 # side as of the last confirmed-consistent state. Both languages carry equal authority;
 # after editing either side, bring the other along and re-record with:
 #   pnpm run verify-translation-pairing --write packages/experimental/webworker-packer/README.md
-README.md: 39d084bc631db387a3b6e526a3a374bc9f766577
-README.zh.md: 11aa04f3db36c09525bc4d4945f77e278602cc0f
+README.md: 35ed9c92eed64b6ae1207c2a0c89901554027d36
+README.zh.md: 12edcea5e02c1f5f4b31d695e5693fdc4958381a
@@ -7,7 +7,7 @@ The VFS image packer: turns one composed profile into the gzip-compressed base t
 The pack is a three-layer standard stack:

 1. **Roster** — the composed profile's plugin rows (standard YAML parse under Include's dialect, `!!js` intact), plus the rows of every config tree the CLI declares in its `package.json` `dsh.configTrees` (agent presets), materialized as a Node-style dependency closure. External peer edges never bind the worker; workspace peers stay on the chain.
-2. **Publish view** — each workspace package contributes the slice npm would publish (`files` through picomatch) minus the rule tables in `src/rules.ts` (no sources, no workspace `dist/`; external packages keep their trees minus the same exclude globs).
+2. **Publish view** — each workspace or vendored package contributes its built npm slice (`files` through picomatch) without source or workspace `dist/`. External packages retain published JavaScript under both `src/` and `dist/` because their `main` or `exports` may point there; only generic test, map, declaration, and archive exclusions apply.
 3. **Reachability sweep** — the runtime loader's own resolution walks from every workspace export face plus the worker assembly's seeds (`IMAGE_ENTRY_SEEDS`), lowering each reached module to the wrapper contract at pack time. Page assets (`lib/client.js` behind `./client` exports) ship verbatim; an unresolvable request from our own code fails the pack, third-party ones are tolerated to fail loud at require time.

 `repository.ts` owns the repo-shaped inputs (workspace scan of `vendor/`, `packages/`, `native/landlock-run/packages/`, and `apps/`; profile composition through the real CLI dump path); `pack.ts` owns none of them, so the same library packs a different tree by being called differently. The native scan makes the Landlock entry package an ordinary published-view dependency while its executable remains a Worker platform implementation. The CLI is `dsh-pack-vfs-image --out <file> [--profile web]`; `apps/web`'s `build:preview` runs it after the preview shell build.
@@ -7,7 +7,7 @@ VFS 镜像打包器：把一份合成 profile 变成浏览器 worker 挂载为
 打包是三层标准栈：

 1. **Roster**——合成 profile 的插件行（标准 YAML 解析、Include 方言、`!!js` 原样保留），加上 CLI 在 `package.json` `dsh.configTrees` 里声明的每棵配置树（agent presets）的行，按 Node 式依赖闭包物化。外部包的 peer 边不追，workspace peer 保留在链上。
-2. **发布视图**——每个 workspace 包贡献 npm 会发布的切片（`files` 走 picomatch），再减去 `src/rules.ts` 的规则表（无源码、无 workspace `dist/`；外部包保留整棵减同一套 exclude glob）。
+2. **发布视图**——每个 workspace 或 vendored 包贡献其构建后的 npm 切片（`files` 走 picomatch），不带源码和 workspace `dist/`。外部包的 `main` 或 `exports` 可能指向 `src/` 或 `dist/`，因此两处发布 JavaScript 都会保留，只应用通用的测试、map、声明与归档排除规则。
 3. **可达性 sweep**——用运行时加载器自己的解析，从全部 workspace 导出面加 worker 装配种子（`IMAGE_ENTRY_SEEDS`）出发，pack 时把每个可达模块降低到包装契约。页面资产（`./client` 导出背后的 `lib/client.js`）原样直发；自家代码的不可解析请求打包即失败，第三方的容忍到 require 时 fail loud。

 `repository.ts` 拥有仓库形态输入（`vendor/`、`packages/`、`native/landlock-run/packages/` 与 `apps/` 的 workspace 扫描；经真 CLI dump 路径合成 profile）；`pack.ts` 一概不拥有，同一库换参即可打另一棵树。Native 扫描使 Landlock 入口包成为普通发布视图依赖，其可执行文件仍由 Worker 平台实现。CLI 为 `dsh-pack-vfs-image --out <file> [--profile web]`；`apps/web` 的 `build:preview` 在预览壳构建后运行它。
@@ -8,13 +8,12 @@
 */

 /**
- * Paths dropped from every collected tree. Source and test trees never
- * resolve at runtime (the artifact plane ships `lib/`), and sourcemaps,
- * declarations, and archives never resolve either while dominating the byte
- * count.
+ * Paths dropped from every collected tree. Test trees, sourcemaps,
+ * declarations, and archives never resolve at runtime while dominating the
+ * byte count. Third-party `src/` directories remain eligible because package
+ * entrypoints may resolve to JavaScript there.
 */
 export const EXCLUDE: readonly string[] = [
-  'src/**',
  'tests/**',
  'test/**',
  '__tests__/**',
@@ -30,11 +29,13 @@ export const EXCLUDE: readonly string[] = [
 ]

 /**
- * Additional paths dropped from workspace packages only. A workspace `dist/`
- * is a page-asset tree the static deployment serves itself; external packages
- * legitimately ship runtime code under `dist/`.
+ * Additional paths dropped from workspace and vendored packages only. Their
+ * runtime plane is built `lib/`; a workspace `dist/` is a page-asset tree the
+ * static deployment serves itself. External packages may place runtime code
+ * under either directory.
 */
 export const EXCLUDE_WORKSPACE: readonly string[] = [
+  'src/**',
  'dist/**',
 ]

@@ -37,6 +37,7 @@ const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url))
 const SUBJECT = '@deepseek-ai/dsh-timeout'
 const LANDLOCK = '@deepseek-ai/node-addon-landlock-run'
 const PLUGIN_INVENTORY = '@deepseek-ai/dsh-plugin-package-inventory-deepseek'
+const WEB_SERVER = '@deepseek-ai/dsh-host-webserver'

 const workspaces = indexWorkspacePackages(repoRoot)

@@ -103,6 +104,15 @@ const packedPluginInventory = (): ReturnType<typeof packVfsImage> => pluginInven
  entries: [],
 })

+let webServerMemo: ReturnType<typeof packVfsImage> | undefined
+const packedWebServer = (): ReturnType<typeof packVfsImage> => webServerMemo ??= packVfsImage({
+  config: `- id: subject\n  name: '${WEB_SERVER}'\n`,
+  profile: 'webserver-dependency-check',
+  workspaces,
+  resolveFrom: repoRoot,
+  entries: [],
+})
+
 /** The image's archive, inflated once: mounting reads the tar, not the gzip member. */
 let archiveMemo: Uint8Array | undefined
 const archive = async (): Promise<Uint8Array> =>
@@ -188,6 +198,25 @@ const archive = async (): Promise<Uint8Array> =>
    expect(loader.usage().modules).toBeGreaterThan(0)
  })

+  it('keeps third-party runtime JavaScript published under src', async () => {
+    const result = packedWebServer()
+    expect(result.missing).toEqual([])
+    expect(Object.hasOwn(result.files, 'node_modules/debug/src/index.js')).toBe(true)
+    expect(Object.hasOwn(result.files, 'node_modules/ms/index.js')).toBe(true)
+
+    const vfs = loadVfsImage(await inflateImage(result.image, 'the packed webserver'), DEFAULT_ROOT)
+    const loader = new WorkerModuleLoader({
+      vfs,
+      root: DEFAULT_ROOT,
+      staticModules: createNodeBuiltins(),
+      staticModulePrefixes: REPLACED_PREFIXES,
+    })
+    setActiveVfs(vfs)
+    setActiveModuleLoader(loader)
+    const webserver = loader.requireFrom(`${DEFAULT_ROOT}/workspace`)(WEB_SERVER) as { WebServer?: unknown }
+    expect(typeof webserver.WebServer).toBe('function')
+  })
+
  it('runs the unchanged Landlock entry package over the Worker platform executable', async () => {
    const result = packedLandlock()
    expect(workspaces.has(LANDLOCK)).toBe(true)
@@ -262,7 +262,7 @@ export class WorkerTunnel {
   * The image packs each bundle with a trailing `sourceURL` naming its image
   * path, so the blob shows under that name in the debugger instead of as an
   * anonymous blob entry.
-   * @param url - graph row url (`/plugins/<id>/client.js?rev=...`).
+   * @param url - Graph combo URL (`/plugins/??<id>/client.js&rev=...`).
   */
  async loadBundle(url: string): Promise<void> {
    const response = await this.fetch(url)
@@ -49,6 +49,8 @@ export const MODULE_PROXIES: Record<string, string> = {
  'node:events': './node/builtin_modules/implemented/events.ts',
  'node:timers/promises': './node/builtin_modules/implemented/timers/promises.ts',
  'node:perf_hooks': './node/builtin_modules/implemented/perf_hooks.ts',
+  'node:tty': './node/builtin_modules/implemented/tty.ts',
+  'tty': './node/builtin_modules/implemented/tty.ts',
  // Real zstd codec: session-log appends compress on every write.
  'node:zlib': './node/builtin_modules/implemented/zlib.ts',
  // The worker's own process layer: `bash -c` and the command table run against
@@ -115,6 +115,12 @@ class FakeServer {
  }
 }

+/**
+ * Constructor marker read by middleware during feature detection. Tunnel
+ * responses are synthesized objects and are never instances of this class.
+ */
+export class ServerResponse {}
+
 /**
 * Create the fake server and retain its request listener for the tunnel.
 * @param listener - the request listener the webserver installs.
@@ -172,8 +178,8 @@ export const __esModule = true
 * `net.Server` carrying sockets and a Node `RequestListener`, while this one binds
 * nothing and captures the synthesized-request listener the tunnel feeds.
 */
-type NodeFace = Partial<Omit<typeof import('node:http'), 'Server' | 'createServer'>>
-  & Record<'Server' | 'createServer', unknown>
+type NodeFace = Partial<Omit<typeof import('node:http'), 'Server' | 'ServerResponse' | 'createServer'>>
+  & Record<'Server' | 'ServerResponse' | 'createServer', unknown>

 /** CommonJS default export: the members `require()` hands a caller of this module. */
-export default { createServer, request, get, STATUS_CODES, Server: FakeServer } satisfies NodeFace
+export default { createServer, request, get, STATUS_CODES, Server: FakeServer, ServerResponse } satisfies NodeFace
@@ -21,7 +21,7 @@ export function createRequire(base: string | URL): NodeRequire {
 /** Builtin specifiers the module proxy table answers (without the `node:` prefix). */
 export const builtinModules = [
  'assert', 'async_hooks', 'buffer', 'child_process', 'crypto', 'events', 'fs', 'http', 'module',
-  'net', 'os', 'path', 'process', 'stream', 'url', 'util', 'worker_threads',
+  'net', 'os', 'path', 'process', 'stream', 'tty', 'url', 'util', 'worker_threads',
 ]

 /**
@@ -0,0 +1,19 @@
+/**
+ * `node:tty` for the browser worker. The host has no terminal-backed file
+ * descriptors, so terminal detection is always false.
+ */
+
+/**
+ * Test whether a numeric file descriptor refers to a terminal.
+ * @param _fd - File descriptor to inspect.
+ * @returns Always false in the browser worker.
+ */
+export function isatty(_fd: number): boolean {
+  return false
+}
+
+/** CommonJS interop marker: the worker loader hands `default` to default imports (see ../../builtins.ts). */
+export const __esModule = true
+
+/** CommonJS default export: the members `require()` hands a caller of this module. */
+export default { isatty } satisfies Partial<typeof import('node:tty')>
@@ -34,6 +34,7 @@ import * as nodePath from './builtin_modules/implemented/path.ts'
 import * as nodePerfHooks from './builtin_modules/implemented/perf_hooks.ts'
 import * as nodeStream from './builtin_modules/implemented/stream.ts'
 import * as nodeTimersPromises from './builtin_modules/implemented/timers/promises.ts'
+import * as nodeTty from './builtin_modules/implemented/tty.ts'
 import * as nodeUrl from './builtin_modules/implemented/url.ts'
 import * as nodeUtil from './builtin_modules/implemented/util.ts'
 import * as nodeUtilTypes from './builtin_modules/implemented/util/types.ts'
@@ -71,6 +72,7 @@ const BUILTINS: Record<string, StaticModuleFactory> = {
  sqlite: () => nodeSqlite,
  stream: () => nodeStream,
  'timers/promises': () => nodeTimersPromises,
+  tty: () => nodeTty,
  url: () => nodeUrl,
  util: () => nodeUtil,
  'util/types': () => nodeUtilTypes,
@@ -9,13 +9,15 @@ afterEach(() => {

 it('ignores script preload hints and executes script sources through the worker loader', async () => {
  const loadScript = vi.fn(async () => {})
+  const preload = '/plugins/??app-a/client.js,app-b/client.js&rev=app'
+  const bootstrap = '/plugins/??modules/client.js&rev=boot'

  await applyIndexInjections([
-    { kind: 'script-preload', src: '/plugins/preload.js' },
-    { kind: 'script-src', placement: 'head', src: '/plugins/execute.js' },
+    { kind: 'script-preload', src: preload },
+    { kind: 'script-src', placement: 'head', src: bootstrap },
  ], loadScript)

  expect(loadScript).toHaveBeenCalledOnce()
-  expect(loadScript).toHaveBeenCalledWith('/plugins/execute.js')
+  expect(loadScript).toHaveBeenCalledWith(bootstrap)
  expect(document.querySelector('link[rel="preload"]')).toBeNull()
 })
@@ -69,6 +69,12 @@ describe('module identity through the loader', () => {
    const require = loaderRequire()
    expect(require('events')).toBe(require('node:events'))
    expect(require('fs')).toBe(require('node:fs'))
+    expect(require('tty')).toBe(require('node:tty'))
+  })
+
+  it('reports that worker file descriptors are not terminals', () => {
+    const tty = loaderRequire()('tty') as { isatty(fd: number): boolean }
+    expect(tty.isatty(2)).toBe(false)
  })

  it('keeps class identity across those specifiers', () => {
@@ -13,7 +13,7 @@
 */
 import { describe, expect, it } from 'vitest'
 import {
-  createServer, get, request, requestListener, STATUS_CODES, whenRequestListener,
+  createServer, get, request, requestListener, ServerResponse, STATUS_CODES, whenRequestListener,
 } from '../../src/node/builtin_modules/implemented/http.ts'
 import type { RequestListener } from '../../src/transport/synthetic-http.ts'

@@ -46,6 +46,10 @@ describe('request listener capture', () => {
 })

 describe('binding', () => {
+  it('exposes the response prototype middleware probes during module loading', () => {
+    expect(ServerResponse.prototype).not.toHaveProperty('appendHeader')
+  })
+
  it('reports the bind through the callback the webserver fiber waits on', async () => {
    const server = createServer(listener)
    let bound = false