Merge pull request #5191 from deepseek-harness/cleanup/compact-persistence-schema

refactor: compact persistence schema type inventory

docs/cookbook/reviewing-persistence-type-changes.i18n.yaml

@@ -15,8 +15,8 @@
   en: 5fca19ac2bb1ddac
   zh: 00d78fe7e80ba4f7
 /cookbook-reviewing-persistence-type-changes/optional-inspect-the-change:
-  en: 6e2c03bf51b3c400
-  zh: 2545a193c89bbf1c
+  en: 3f85edb803afe870
+  zh: 545c651b9e5ccb56
 /cookbook-reviewing-persistence-type-changes/1-record-the-change:
   en: 342bd49412db070e
   zh: a8d14958e9a637d9

docs/cookbook/reviewing-persistence-type-changes.md

@@ -41,6 +41,8 @@ pnpm --silent run persistence-review --before .artifacts/base.schema.json --afte
 
 Record the commits supplying those files with the report. Add `--json` for structured output. This read-only comparison groups shared changes with their affected roots and uses actual literal `kind`/`form` values instead of union positions. Ambiguous alternatives remain separate additions and removals. Its compatibility section copies every root's authoritative classifier result; the structural explanation does not replace acknowledgement checks. Current catalog labels and declaration names are descriptive metadata; structural anchors and fingerprints identify types.
 
+The current machine inventory stores complete graphs in `roots`. A `types` entry contains `digest`, `names`, and `sources`, plus an explicit `schema` when roots cannot reconstruct that graph exactly. Readers resolve omitted graphs from root subgraphs by digest and validate explicit graphs directly, preserving every type and its metadata. Historical full entries remain readable. `formatVersion` identifies normalization rules; storage compaction does not change root fingerprints or require an acknowledgement.
+
 <a id="acknowledge"></a>
 ## 1. Record the change
 

docs/cookbook/reviewing-persistence-type-changes.zh.md

@@ -41,6 +41,8 @@ pnpm --silent run persistence-review --before .artifacts/base.schema.json --afte
 
 在报告旁记录这些文件对应的 commit。添加 `--json` 可获取结构化输出。此只读比较将共享变更与受影响的根类型归组，使用实际字面量 `kind`/`form` 值代替联合类型位置。无法唯一匹配的候选项保留为独立的新增与删除。兼容性部分复制每个根类型的权威分类结果；结构说明不替代确认检查。当前目录标签和声明名称是描述元数据；结构锚点和指纹标识类型。
 
+当前机器清单在 `roots` 中保存完整图。每个 `types` 条目包含 `digest`、`names` 和 `sources`；若根无法精确重建该图，条目还会保存显式 `schema`。读取器按摘要从根的子图恢复省略的图，并直接验证显式图，保留每个类型及其元数据。历史完整条目仍然可读。`formatVersion` 标识规范化规则；存储压缩不改变根指纹，也不需要确认记录。
+
 <a id="acknowledge"></a>
 ## 1. 记录变更
 

scripts/gen-persistence-catalog.ts

@@ -19,7 +19,7 @@ import { renderPersistencePair, type PersistenceArtifact } from './persistence-a
 
 export { annotateSurface, collectEventEnvelopeTypes, collectLogEvents, collectSurfaceEventTypes } from './persistence-catalog-source.ts'
 export type { AnnotatedLogEventEntry, EventEnvelopeTypeEntry, LogEventEntry } from './persistence-catalog-source.ts'
-import type { PersistenceSchemaInventory } from './persistence-schema-model.ts'
+import { persistenceSchemaSnapshot, type PersistenceSchemaInventory } from './persistence-schema-model.ts'
 import { renderPersistenceSchemaDefinitions, renderPersistenceSchemaIndex } from './render-persistence-schema.ts'
 
 const root = resolve(import.meta.dirname, '..')
@@ -178,7 +178,7 @@ export function persistenceCatalogArtifacts(scanRoot: string, schema: Persistenc
   return [
     ...renderPersistencePair(scanRoot, OUT, render(events, envelope, schema), render(events, envelope, schema, 'zh')),
     { path: OUT_RUNTIME_TYPES, content: renderKnownEventTypes(events) },
-    { path: OUT_SCHEMA, content: `${JSON.stringify(schema, null, 2)}\n` },
+    { path: OUT_SCHEMA, content: `${JSON.stringify(persistenceSchemaSnapshot(schema), null, 2)}\n` },
   ]
 }
 

scripts/persistence-changes.ts

@@ -4,7 +4,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from
 import { basename, join, resolve } from 'node:path'
 import { parseArgs } from 'node:util'
 import { JSON_SCHEMA, load } from 'js-yaml'
-import { canonicalizeSchema, schemaDigest } from './persistence-schema-model.ts'
+import { canonicalizeSchema, reachableSchemaTypes, schemaDigest } from './persistence-schema-model.ts'
 import { matchingSourceCompatibility, sourceKindGroups, validSourceCompatibility } from './persistence-source-policy.ts'
 import type { CanonicalSchema, PersistenceRoot, PersistenceSchemaInventory, SchemaNode, SchemaTupleElement } from './persistence-schema-model.ts'
 import { extractPersistenceSchema } from './persistence-schema.ts'
@@ -265,7 +265,7 @@ function parseSchema(value: unknown, label: string, formatVersion: 1 | 2): Canon
   return schema
 }
 
-/** Parse a persisted schema inventory, rejecting malformed graphs and digest drift.
+/** Parse full or compact inventories, restoring type graphs and rejecting invalid digests.
  * @param value - JSON read from the current inventory or an enforced acknowledgement snapshot.
  * @returns the validated inventory.
  */
@@ -284,8 +284,10 @@ export function parseHistoricalPersistenceSnapshot(value: unknown): PersistenceS
 function parseSnapshot(value: unknown, historical: boolean): PersistenceSchemaInventory {
   const input = record(value, 'schema inventory')
   keys(input, ['formatVersion', 'roots', 'types'], 'schema inventory')
-  if (input.formatVersion !== 1 && input.formatVersion !== 2) throw new Error('unsupported persistence schema normalization version')
+  const formatVersion = input.formatVersion
+  if (formatVersion !== 1 && formatVersion !== 2) throw new Error('unsupported persistence schema normalization version')
   const names = new Set<string>()
+  const schemas: CanonicalSchema[] = []
   for (const rawRoot of array(input.roots, 'schema roots')) {
     const root = record(rawRoot, 'schema root')
     keys(root, ['key', 'kind', 'digest', 'schema'], 'schema root', ['event', 'surface'])
@@ -298,24 +300,37 @@ function parseSnapshot(value: unknown, historical: boolean): PersistenceSchemaIn
     } else if ((root.kind !== 'header' || !['SessionHeader', 'JsonlHeaderLine'].includes(key))
       && (root.kind !== 'envelope' || key !== 'SessionEventEnvelope')) throw new Error(`invalid schema root ${key}`)
     if (root.kind !== 'event' && (root.event !== undefined || root.surface !== undefined)) throw new Error(`${key}: non-event metadata`)
-    const schema = parseSchema(root.schema, key, input.formatVersion)
+    const schema = parseSchema(root.schema, key, formatVersion)
     if (root.kind === 'event') validateEventMetadata(schema, String(root.event), root.surface === true, historical)
     if (digest(root.digest, `${key} digest`) !== schemaDigest(schema)) throw new Error(`${key}: schema digest mismatch`)
+    schemas.push(schema)
   }
-  for (const rawType of array(input.types, 'schema types')) {
+  let reachable: Map<string, CanonicalSchema> | undefined
+  const types = array(input.types, 'schema types').map((rawType) => {
     const type = record(rawType, 'schema type')
-    keys(type, ['digest', 'schema', 'names', 'sources'], 'schema type')
-    const schema = parseSchema(type.schema, 'shared schema', input.formatVersion)
-    if (digest(type.digest, 'shared digest') !== schemaDigest(schema)) throw new Error('shared schema digest mismatch')
-    for (const name of array(type.names, 'type names')) textValue(name, 'type name')
-    for (const source of array(type.sources, 'type sources')) {
+    keys(type, ['digest', 'names', 'sources'], 'schema type', ['schema'])
+    const typeDigest = digest(type.digest, 'shared digest')
+    let schema: CanonicalSchema
+    if ('schema' in type) {
+      schema = parseSchema(type.schema, 'shared schema', formatVersion)
+      if (typeDigest !== schemaDigest(schema)) throw new Error('shared schema digest mismatch')
+    } else {
+      reachable ??= reachableSchemaTypes(schemas)
+      const found = reachable.get(typeDigest)
+      if (found === undefined) throw new Error(`shared schema digest ${typeDigest} is not reachable from roots`)
+      schema = found
+    }
+    const typeNames = array(type.names, 'type names').map(name => textValue(name, 'type name'))
+    const sources = array(type.sources, 'type sources').map((source) => {
       const location = textValue(source, 'type source')
       if (historical && /:\d+(?::\d+)?$|#L\d+(?:-L\d+)?$/u.test(location)) {
         throw new Error('historical schema sources must omit line numbers')
       }
-    }
-  }
-  return input as unknown as PersistenceSchemaInventory
+      return location
+    })
+    return { digest: typeDigest, schema, names: typeNames, sources }
+  })
+  return { ...input as unknown as PersistenceSchemaInventory, types }
 }
 
 function validateEventMetadata(schema: CanonicalSchema, event: string, surface: boolean, historical: boolean): void {

scripts/persistence-schema-model.ts

@@ -4,6 +4,7 @@
  */
 
 import { createHash } from 'node:crypto'
+import { isDeepStrictEqual } from 'node:util'
 
 /** Recorded reader promise for explicitly attributed source additions. */
 export interface SourceCompatibility {
@@ -71,6 +72,53 @@ export interface PersistenceSchemaInventory {
   readonly types: readonly PersistenceType[]
 }
 
+/** Type metadata with an explicit graph when roots cannot reconstruct it exactly. */
+export interface PersistenceTypeIndex extends Omit<PersistenceType, 'schema'> {
+  readonly schema?: CanonicalSchema
+}
+
+/** Compact inventory representation; formatVersion still pins normalization. */
+export interface PersistenceSchemaSnapshot {
+  readonly formatVersion: PersistenceSchemaInventory['formatVersion']
+  readonly roots: readonly PersistenceRoot[]
+  readonly types: readonly PersistenceTypeIndex[]
+}
+
+/**
+ * Index every normalized type reachable from the supplied graphs.
+ * @param schemas - complete canonical root graphs.
+ * @returns canonical subgraphs keyed by their structural digest.
+ */
+export function reachableSchemaTypes(schemas: readonly CanonicalSchema[]): Map<string, CanonicalSchema> {
+  const types = new Map<string, CanonicalSchema>()
+  for (const schema of schemas) {
+    for (const [index] of schema.nodes.entries()) {
+      const type = canonicalizeSchema(schema.nodes, index)
+      types.set(schemaDigest(type), type)
+    }
+  }
+  return types
+}
+
+/**
+ * Omit type graphs exactly reconstructable from roots, retaining all other graphs.
+ * @param inventory - expanded inventory, including types erased from roots by normalization.
+ * @returns a lossless snapshot preserving root graphs, type order, and declaration metadata.
+ */
+export function persistenceSchemaSnapshot(inventory: PersistenceSchemaInventory): PersistenceSchemaSnapshot {
+  const reachable = reachableSchemaTypes(inventory.roots.map(root => root.schema))
+  return {
+    formatVersion: inventory.formatVersion,
+    roots: inventory.roots,
+    types: inventory.types.map(({ digest, schema, names, sources }) => ({
+      digest,
+      ...(isDeepStrictEqual(schema, reachable.get(digest)) ? {} : { schema }),
+      names,
+      sources,
+    })),
+  }
+}
+
 /**
  * Visit direct graph edges in their normalized semantic order.
  * @param node - resolved graph node.

scripts/persistence-schema-snapshot.spec.ts

@@ -0,0 +1,92 @@
+/** Compact inventories retain root fingerprints and restore shared recursive type graphs. */
+
+import { describe, expect, it } from 'vitest'
+import { parseHistoricalPersistenceSnapshot, parsePersistenceSnapshot } from './persistence-changes.ts'
+import { canonicalizeSchema, persistenceSchemaSnapshot, schemaDigest, type PersistenceSchemaInventory } from './persistence-schema-model.ts'
+import { reviewPersistenceSchemas } from './persistence-review.ts'
+import { renderPersistenceSchemaDefinitions, renderPersistenceSchemaIndex } from './render-persistence-schema.ts'
+
+function inventory(): PersistenceSchemaInventory {
+  const schema = canonicalizeSchema([
+    { kind: 'object', properties: [{ name: 'child', type: 1, optional: false }], indices: [] },
+    { kind: 'object', properties: [{ name: 'next', type: 1, optional: true }, { name: 'value', type: 2, optional: false }], indices: [] },
+    { kind: 'primitive', type: 'string' },
+  ], 0)
+  return {
+    formatVersion: 1,
+    roots: ['SessionHeader', 'JsonlHeaderLine'].map(key => ({ key, kind: 'header', digest: schemaDigest(schema), schema })),
+    types: schema.nodes.map((_, index) => {
+      const type = canonicalizeSchema(schema.nodes, index)
+      return { digest: schemaDigest(type), schema: type, names: index === 1 ? ['Recursive'] : [], sources: ['packages/example/src/types.ts'] }
+    }).sort((left, right) => left.digest.localeCompare(right.digest)),
+  }
+}
+
+describe('compact persistence snapshots', () => {
+  it('stores only root graphs and restores shared recursive definitions without changing reports', () => {
+    const original = inventory()
+    const snapshot = persistenceSchemaSnapshot(original)
+    expect(snapshot.roots).toEqual(original.roots)
+    expect(snapshot.types).toEqual(original.types.map(({ digest, names, sources }) => ({ digest, names, sources })))
+    const restored = parsePersistenceSnapshot(JSON.parse(JSON.stringify(snapshot)))
+    expect(restored).toEqual(original)
+    expect(reviewPersistenceSchemas(original, restored)).toEqual(reviewPersistenceSchemas(original, original))
+    for (const rendering of ['current', 'historical'] as const) {
+      expect(renderPersistenceSchemaIndex(restored, 'en', undefined, 2, rendering))
+        .toBe(renderPersistenceSchemaIndex(original, 'en', undefined, 2, rendering))
+      expect(renderPersistenceSchemaDefinitions(restored, 'en', undefined, 2, rendering))
+        .toBe(renderPersistenceSchemaDefinitions(original, 'en', undefined, 2, rendering))
+    }
+    expect(parseHistoricalPersistenceSnapshot(snapshot)).toEqual(original)
+  })
+
+  it.each([true, false])('retains independent recursive types with root graphs=%s', (withRoots) => {
+    const base = inventory()
+    const schema = canonicalizeSchema([
+      { kind: 'object', properties: [{ name: 'next', type: 0, optional: true }, { name: 'flag', type: 1, optional: false }], indices: [] },
+      { kind: 'literal', value: false },
+    ], 0)
+    const independent = { digest: schemaDigest(schema), schema, names: ['example.ts#Independent', 'example.ts#Alias'], sources: ['example.ts'] }
+    const original = { ...base, roots: withRoots ? base.roots : [], types: [independent, ...base.types] }
+    const snapshot = persistenceSchemaSnapshot(original)
+    expect(snapshot.types[0]).toEqual(independent)
+    if (withRoots) expect(snapshot.types.slice(1).every(type => type.schema === undefined)).toBe(true)
+    else expect(snapshot.types).toEqual(original.types)
+    const restored = parsePersistenceSnapshot(JSON.parse(JSON.stringify(snapshot)))
+    expect(restored).toEqual(original)
+    expect(parseHistoricalPersistenceSnapshot(snapshot)).toEqual(original)
+    expect(persistenceSchemaSnapshot(restored)).toEqual(snapshot)
+    expect(renderPersistenceSchemaDefinitions(restored)).toBe(renderPersistenceSchemaDefinitions(original))
+  })
+
+  it('reads legacy full graphs and keeps partial acknowledgement inventories empty', () => {
+    const original = inventory()
+    expect(parsePersistenceSnapshot(JSON.parse(JSON.stringify(original)))).toEqual(original)
+    const partial = { ...original, types: [] }
+    expect(parsePersistenceSnapshot(persistenceSchemaSnapshot(partial))).toEqual(partial)
+  })
+
+  it('rejects compact digests that do not identify a root subgraph', () => {
+    const snapshot = persistenceSchemaSnapshot(inventory())
+    const schema = canonicalizeSchema([{ kind: 'literal', value: 'unreachable' }], 0)
+    expect(() => parsePersistenceSnapshot({ ...snapshot, types: [{ digest: schemaDigest(schema), names: [], sources: [] }] }))
+      .toThrow('is not reachable from roots')
+  })
+
+  it('still validates explicit legacy graphs instead of replacing them with root graphs', () => {
+    const original = inventory()
+    const schema = canonicalizeSchema([{ kind: 'primitive', type: 'number' }], 0)
+    expect(() => parsePersistenceSnapshot({ ...original, types: original.types.map(type => ({ ...type, schema })) }))
+      .toThrow('shared schema digest mismatch')
+  })
+
+  it('rejects invalid metadata and malformed explicit schemas in compact inventories', () => {
+    const snapshot = persistenceSchemaSnapshot(inventory())
+    const type = snapshot.types[0]!
+    expect(() => parsePersistenceSnapshot({ ...snapshot, types: [{ ...type, names: [3] }] })).toThrow('type name')
+    expect(() => parsePersistenceSnapshot({ ...snapshot, types: [{ ...type, sources: [3] }] })).toThrow('type source')
+    expect(() => parsePersistenceSnapshot({ ...snapshot, types: [{ ...type, schema: null }] })).toThrow('shared schema')
+    expect(() => parseHistoricalPersistenceSnapshot({ ...snapshot, types: [{ ...type, sources: ['types.ts:3'] }] }))
+      .toThrow('must omit line numbers')
+  })
+})