From 7b8c7dfff258dc7955bb58b6ecfae89b3d3ac867 Mon Sep 17 00:00:00 2001 From: 07akioni <07akioni2@gmail.com> Date: Fri, 25 Sep 2026 17:29:15 +0800 Subject: [PATCH] feat(telemetry): upload session log events through byte-bounded OTLP --- ...-25-session-log-otel-byte-limits.i18n.yaml | 19 ++ ...2026-09-25-session-log-otel-byte-limits.md | 29 +++ ...6-09-25-session-log-otel-byte-limits.zh.md | 29 +++ ...3-session-telemetry-otel-revival.i18n.yaml | 4 +- ...26-07-23-session-telemetry-otel-revival.md | 2 + ...07-23-session-telemetry-otel-revival.zh.md | 2 + THIRD_PARTY_NOTICES.md | 1 - apps/web/tests/feedback-release.e2e.ts | 16 +- docs/config-catalog.md | 32 ++- docs/config-catalog.zh.md | 32 ++- docs/event-producer-consumer.md | 2 +- docs/event-producer-consumer.zh.md | 2 +- docs/module-graph.md | 6 +- docs/module-graph.zh.md | 6 +- docs/subsystems/product-telemetry.i18n.yaml | 4 +- docs/subsystems/product-telemetry.md | 44 +++- docs/subsystems/product-telemetry.zh.md | 44 +++- docs/subsystems/session-telemetry.i18n.yaml | 4 +- docs/subsystems/session-telemetry.md | 4 +- docs/subsystems/session-telemetry.zh.md | 4 +- packages/bundle/base/cordis.patch.yml | 3 +- .../extensions/tool-cordis/src/api-catalog.ts | 15 +- .../product-telemetry-otel/README.i18n.yaml | 16 +- .../host/product-telemetry-otel/README.md | 11 +- .../host/product-telemetry-otel/README.zh.md | 11 +- .../host/product-telemetry-otel/package.json | 6 +- .../host/product-telemetry-otel/src/index.ts | 48 ++-- .../product-telemetry-otel/src/session-log.ts | 164 +++++++++++++ .../product-telemetry-otel/src/transport.ts | 26 +++ .../tests/telemetry.spec.ts | 219 +++++++++++++++++- .../host/product-telemetry-otel/tsconfig.json | 3 + .../session-telemetry-otel/README.i18n.yaml | 12 +- .../session/session-telemetry-otel/README.md | 13 +- .../session-telemetry-otel/README.zh.md | 13 +- .../session-telemetry-otel/package.json | 7 +- .../session-telemetry-otel/src/index.ts | 83 +++---- .../tests/loader-composition.e2e.ts | 13 ++ .../session-telemetry-otel/tests/otel.spec.ts | 93 ++++---- .../session-telemetry-otel/tsconfig.json | 3 + .../session-telemetry/README.i18n.yaml | 8 +- packages/session/session-telemetry/README.md | 4 +- .../session/session-telemetry/README.zh.md | 4 +- .../session-telemetry/src/coordinator.ts | 4 +- .../session/session-telemetry/src/index.ts | 5 +- pnpm-lock.yaml | 27 +-- python/sdk-runtime/package.json | 1 + scripts/gen-cordis-catalog.ts | 1 + scripts/type-equiv.manifest.json | 10 + .../feedback-release.expected.json | 1 + 49 files changed, 885 insertions(+), 225 deletions(-) create mode 100644 .agents/notes/implemented/architecture/2026-09-25-session-log-otel-byte-limits.i18n.yaml create mode 100644 .agents/notes/implemented/architecture/2026-09-25-session-log-otel-byte-limits.md create mode 100644 .agents/notes/implemented/architecture/2026-09-25-session-log-otel-byte-limits.zh.md create mode 100644 packages/host/product-telemetry-otel/src/session-log.ts create mode 100644 packages/host/product-telemetry-otel/src/transport.ts diff --git a/.agents/notes/implemented/architecture/2026-09-25-session-log-otel-byte-limits.i18n.yaml b/.agents/notes/implemented/architecture/2026-09-25-session-log-otel-byte-limits.i18n.yaml new file mode 100644 index 0000000000..f86d612295 --- /dev/null +++ b/.agents/notes/implemented/architecture/2026-09-25-session-log-otel-byte-limits.i18n.yaml @@ -0,0 +1,19 @@ +# Bilingual-pair consistency record for 2026-09-25-session-log-otel-byte-limits.md (docs/i18n/README.md): per heading +# section, a hash of its English and Chinese blocks outside code blocks and generated regions. +# After editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-09-25-session-log-otel-byte-limits.md +/agent-note-session-log-otlp-strings-and-bounded-requests: + en: 041f7e2b55202ea3 + zh: e01834ae6c9fc7bc +/agent-note-session-log-otlp-strings-and-bounded-requests/problem: + en: 20f659a15fb55352 + zh: 49d39c95da1b8aa5 +/agent-note-session-log-otlp-strings-and-bounded-requests/decision: + en: eefcb4a88e478526 + zh: b1f38662efcfddf8 +/agent-note-session-log-otlp-strings-and-bounded-requests/alternatives-considered: + en: 0dc933b18a1a3573 + zh: 77655740322c4e45 +/agent-note-session-log-otlp-strings-and-bounded-requests/consequences: + en: ec1f5f8cf94e38d3 + zh: 21dcfe6765881aa7 diff --git a/.agents/notes/implemented/architecture/2026-09-25-session-log-otel-byte-limits.md b/.agents/notes/implemented/architecture/2026-09-25-session-log-otel-byte-limits.md new file mode 100644 index 0000000000..36e69a413a --- /dev/null +++ b/.agents/notes/implemented/architecture/2026-09-25-session-log-otel-byte-limits.md @@ -0,0 +1,29 @@ +# Agent Note: Session-log OTLP strings and bounded requests + +Status: implemented + +English | [中文](2026-09-25-session-log-otel-byte-limits.zh.md) + +## Problem + +Session logs contain nested JSON and large tool results. The product collector accepts named records with string content and limits request size; count-only SDK batching cannot bound bytes after UTF-8 encoding and JSON escaping. + +## Decision + +The product OTel package provides one Session-log reporter shared by its service and the feedback backend. Each canonical event becomes `eventName: "session-log"`, with `sessionId` and the JSON-stringified full event in `content` attributes. Redaction applies before serialization; the capture record carries an envelope without a duplicate payload. + +Session logs use their own SDK provider and queue. The exporter measures complete uncompressed OTLP JSON using the transport's SDK serializer and splits batches to at most 4,000,000 bytes. A single oversized event is rejected with a diagnostic and does not prevent other events from being sent. Attribute truncation is disabled for this queue. The owner supplies a bounded shutdown that waits for split exports. + +The [telemetry revival](../feature/2026-07-23-session-telemetry-otel-revival.md) remains authoritative for redaction, capture, and best-effort handoff. This decision replaces its SDK-only batching rule for Session-log request sizing; the SDK still owns queueing cadence and transport retry. Feedback authorization and the separate DeepSeek model-request contribution remain unchanged. + +## Alternatives considered + +**One string for the entire Session.** Rejected because one large Session would exceed the request ceiling even when each event fits. One event per record preserves event identity and permits batching. + +**Truncate oversized records or add client-only fragments.** Rejected because truncation loses content and fragments require a collector reassembly protocol. Explicit rejection retains the one-event-per-record meaning. + +**Count compressed bytes or characters.** Rejected because collector limits can apply after decompression, and characters omit UTF-8 and escaping overhead. Measuring full uncompressed requests gives a conservative bound. + +## Consequences + +Product and Session records share routing configuration but never a request. Queue overflow, single-event rejection, process exit, and transport failures can still lose data; no durable outbox or collector acknowledgement is added. Tests observe exact byte limits, nested JSON round trips, separated requests, oversize continuation, and feedback-authorized recorded-session replay. diff --git a/.agents/notes/implemented/architecture/2026-09-25-session-log-otel-byte-limits.zh.md b/.agents/notes/implemented/architecture/2026-09-25-session-log-otel-byte-limits.zh.md new file mode 100644 index 0000000000..4fd5386619 --- /dev/null +++ b/.agents/notes/implemented/architecture/2026-09-25-session-log-otel-byte-limits.zh.md @@ -0,0 +1,29 @@ +# Agent Note: Session 日志的 OTLP 字符串与有界请求 + +Status: implemented + +[English](2026-09-25-session-log-otel-byte-limits.md) | 中文 + +## 问题 + +Session 日志包含嵌套 JSON 和大型工具结果。产品采集端接收带名称和字符串内容的记录，并限制请求大小；仅按数量分批的 SDK 无法约束 UTF-8 编码与 JSON 转义后的字节数。 + +## 决策 + +产品 OTel 包提供一个由其服务和反馈后端共享的 Session 日志上报器。每条规范事件转换为 `eventName: "session-log"`，属性包含 `sessionId` 和存放完整事件 JSON 字符串的 `content`。脱敏在序列化前执行；采集记录携带不重复包含载荷的信封。 + +Session 日志使用专属 SDK provider 和队列。导出器使用传输的 SDK 序列化器计量完整未压缩 OTLP JSON，并将批次拆分到最多 4,000,000 字节。单条超限事件被拒绝并记录诊断，不阻止其他事件发送。该队列禁用属性截断。拥有方提供有界关闭过程，等待拆分导出完成。 + +[遥测恢复决策](../feature/2026-07-23-session-telemetry-otel-revival.zh.md) 仍负责脱敏、采集和尽力交接。本决策在 Session 日志请求计量方面替代其仅由 SDK 分批的规则；SDK 仍负责排队节奏和传输重试。反馈授权与独立的 DeepSeek 模型请求贡献保持不变。 + +## 考虑过的替代方案 + +**整个 Session 合为一个字符串。** 拒绝，因为即使每条事件都能放入请求，大型 Session 仍会超过请求上限。每条事件对应一条记录，既保留事件身份又允许分批。 + +**截断超限记录或仅在客户端分片。** 拒绝，因为截断会丢失内容，而分片需要采集端重组协议。显式拒绝保留一条事件对应一条记录的含义。 + +**计算压缩字节数或字符数。** 拒绝，因为采集端限制可能作用于解压后内容，而字符数不包含 UTF-8 和转义开销。计量完整未压缩请求提供保守上限。 + +## 影响 + +产品与 Session 记录共享路由配置，但绝不共用请求。队列溢出、单条事件拒绝、进程退出和传输失败仍可能丢失数据；不增加持久化 outbox 或采集端确认。测试观察精确字节上限、嵌套 JSON 往返、请求分离、超限后继续发送，以及反馈授权的录制会话回放。 diff --git a/.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.i18n.yaml b/.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.i18n.yaml index 8f2fe2b211..a06d830891 100644 --- a/.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.i18n.yaml @@ -9,8 +9,8 @@ en: d3383b495424d118 zh: 4e6ffc8ebfa7f741 /agent-note-session-telemetry-seam-with-mandatory-redaction-and-the-otel-backend/decision: - en: 85b950a67315023c - zh: cb4c08338c30b226 + en: 739b871ae583dcbf + zh: d17053e3b0a35942 /agent-note-session-telemetry-seam-with-mandatory-redaction-and-the-otel-backend/alternatives-considered: en: 01838eefca2bc117 zh: fc71f5449d095b27 diff --git a/.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.md b/.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.md index dc52ee87ef..b3b50d4be5 100644 --- a/.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.md +++ b/.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.md @@ -16,6 +16,8 @@ Every deployment that wants harness sessions in an observability stack must hand - **The `session-telemetry/record` waterfall** — the delta over the branch version and the seam's redaction extension point. Every record passes it before reaching any backend; the seam ships NO rules of its own — the innermost `next()` is a pass-through, deployments mount their rules as listeners (stacking by transforming `next()`'s return value), and a throwing rule withholds the record fail-closed. Redaction applies to the exported copy only; the canonical log is never rewritten. - **`@deepseek-ai/dsh-session-telemetry-otel`** — the reference backend: OTel JS SDK log pipeline (`LoggerProvider` → `BatchLogRecordProcessor` → OTLP/HTTP exporter), configured verbatim through `exporter`/`processor` passthroughs. `DISABLED` is the plugin default and constructs no transport. `FEEDBACK_ONLY` requires `exporter.url`; the [explicit-feedback policy](../architecture/2026-09-05-nonofficial-feedback-otel.md) requires a new submission for every bounded capture, for all providers. [Buffer-free feedback replay](../../archived/simplification/2026-08-06-buffer-free-feedback-telemetry.md) avoids a second in-memory copy of the session prefix. +The [Session-log request limits](../architecture/2026-09-25-session-log-otel-byte-limits.md) supersede the SDK-only batching rule below for the OTel Session sender; capture, redaction, and handoff rationale remain applicable. + The boundary axiom holds: the harness's aspect ends at `emit()`. Batching, retry, queueing, and loss policy are the reporting SDK's, configured through passthroughs. Delivery is best-effort: a crash can lose queued records. Backend retry or losing the same live object's module-scope cursor can duplicate lifecycle-local rows, so receivers deduplicate on `(session.id, session.format_version, event.seq)`. ## Alternatives considered diff --git a/.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.zh.md b/.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.zh.md index f647ea2896..349cd47e59 100644 --- a/.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.zh.md +++ b/.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.zh.md @@ -16,6 +16,8 @@ Status: implemented - **`session-telemetry/record` waterfall（瀑布式事件）** —— 相对分支版本的增量，也是该 seam 的脱敏扩展点。每条记录抵达任何后端前必经此处；seam 自身不带任何规则——最内层 `next()` 原样透传，部署方以监听器挂载自己的规则（通过变换 `next()` 的返回值堆叠），抛异常的规则将该记录 fail-closed 扣下。脱敏只作用于导出副本；canonical log 永不改写。 - **`@deepseek-ai/dsh-session-telemetry-otel`** —— 参考后端：OTel JS SDK 日志流水线（`LoggerProvider` → `BatchLogRecordProcessor` → OTLP/HTTP exporter），经 `exporter`/`processor` passthrough 原样配置。`DISABLED` 是插件默认值，且不构造任何传输。`FEEDBACK_ONLY` 要求 `exporter.url`；[显式反馈策略](../architecture/2026-09-05-nonofficial-feedback-otel.zh.md)要求每次有界捕获都由新提交触发，适用于所有提供方。[无缓冲反馈回放](../../archived/simplification/2026-08-06-buffer-free-feedback-telemetry.md)避免在内存中创建会话前缀的第二份副本。 +[Session 日志请求上限](../architecture/2026-09-25-session-log-otel-byte-limits.zh.md) 在 OTel Session 发送器中替代下述仅由 SDK 分批的规则；采集、脱敏和交接的理由仍然适用。 + 边界公理保持不变：harness 的职责止于 `emit()`。批处理、重试、排队与丢失策略属于 reporting SDK，并经 passthrough 配置。投递是尽力而为：崩溃可能丢失已排队记录。后端重试或丢失同一 live 对象的模块作用域游标可能重复生命周期本地 row，因此接收端基于 `(session.id, session.format_version, event.seq)` 去重。 ## 考虑过的替代方案 diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 3f325012a0..e08afa0c1c 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -56,7 +56,6 @@ External packages installed for runtime use or distributed inside the prebuilt b | [`@opentelemetry/api`](https://github.com/open-telemetry/opentelemetry-js) | Apache-2.0 | | [`@opentelemetry/api-logs`](https://github.com/open-telemetry/opentelemetry-js) | Apache-2.0 | | [`@opentelemetry/core`](https://github.com/open-telemetry/opentelemetry-js) | Apache-2.0 | -| [`@opentelemetry/exporter-logs-otlp-http`](https://github.com/open-telemetry/opentelemetry-js) | Apache-2.0 | | [`@opentelemetry/otlp-exporter-base`](https://github.com/open-telemetry/opentelemetry-js) | Apache-2.0 | | [`@opentelemetry/otlp-transformer`](https://github.com/open-telemetry/opentelemetry-js) | Apache-2.0 | | [`@opentelemetry/resources`](https://github.com/open-telemetry/opentelemetry-js) | Apache-2.0 | diff --git a/apps/web/tests/feedback-release.e2e.ts b/apps/web/tests/feedback-release.e2e.ts index 4643da9cf8..3de0ea2fca 100644 --- a/apps/web/tests/feedback-release.e2e.ts +++ b/apps/web/tests/feedback-release.e2e.ts @@ -27,6 +27,7 @@ const MODE = webSnapshotMode() interface OtlpCapture { resourceLogs: { scopeLogs: { logRecords: { + eventName: string attributes: { key: string; value: { stringValue?: string; intValue?: number | string } }[] }[] }[] }[] } @@ -53,8 +54,11 @@ describe.each(MODE === 'record' ? ['deepseek-official'] : ['deepseek-official', return capture.resourceLogs.flatMap(resource => resource.scopeLogs.flatMap(scope => scope.logRecords.map((record) => { const attribute = (key: string) => record.attributes.find(value => value.key === key)?.value - return [attribute('session.id')?.stringValue, Number(attribute('event.seq')?.intValue), - attribute('event.type')?.stringValue] as [string | undefined, number, string | undefined] + expect(record.eventName).toBe('session-log') + const content = attribute('content')?.stringValue + expect(typeof content).toBe('string') + const event = JSON.parse(content!) as SessionEvent + return [attribute('sessionId')?.stringValue, event.seq, event.type] as [string | undefined, number, string | undefined] }))) }) } @@ -71,6 +75,10 @@ describe.each(MODE === 'record' ? ['deepseek-official'] : ['deepseek-official', const expected = authorized.map(event => [sessionId, event.seq, event.type]) // No teardown, flush hook, or subsequent interaction may cause this delivery. await expect.poll(captured, { timeout: 10_000 }).toEqual(expected) + const uploadedEvents = uploads.flatMap(upload => (JSON.parse(upload) as OtlpCapture).resourceLogs + .flatMap(resource => resource.scopeLogs.flatMap(scope => scope.logRecords)) + .map(record => JSON.parse(record.attributes.find(attribute => attribute.key === 'content')!.value.stringValue!) as SessionEvent)) + expect(uploadedEvents).toEqual(authorized) suffixes.push(authorized.slice(releasedCount).map(event => event.type)) releasedCount = authorized.length expect(scaffold.ctx.agents.get(sessionId)).toBe(agent) @@ -268,7 +276,9 @@ describe.each(MODE === 'record' ? ['deepseek-official'] : ['deepseek-official', } }) await compareOrRefreshGolden(RELEASE_EXPECTED, JSON.stringify({ - mode: 'FEEDBACK_ONLY', feedback, + mode: 'FEEDBACK_ONLY', + eventName: (JSON.parse(uploads[0]!) as OtlpCapture).resourceLogs[0]!.scopeLogs[0]!.logRecords[0]!.eventName, + feedback, // The prefix is compared with each provider's actual canonical log above. laterSubmissionSuffixes: suffixes.slice(1), }, null, 2), MODE) diff --git a/docs/config-catalog.md b/docs/config-catalog.md index da7af50cea..94b7d569db 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1435,7 +1435,8 @@ export interface Config { ## `@deepseek-ai/dsh-host-product-telemetry-otel` -- `source`: [`packages/host/product-telemetry-otel/src/index.ts:37`](../packages/host/product-telemetry-otel/src/index.ts) +- `refs`: `Attributes` (`@opentelemetry/api`) · `BatchLogRecordProcessorOptions` (`@opentelemetry/sdk-logs`) · `OTLPExporterNodeConfigBase` (`@opentelemetry/otlp-exporter-base`) +- `source`: [`packages/host/product-telemetry-otel/src/index.ts:38`](../packages/host/product-telemetry-otel/src/index.ts) ```ts config-catalog /** Collector routing, application identity, and bounded in-memory batch settings. */ @@ -1462,6 +1463,25 @@ export interface Config { exportTimeoutMillis: number /** Outer shutdown wait; pending exports may be lost after this deadline. */ shutdownTimeoutMillis: number + /** Independent Session-log batching and byte limit; product-event settings do not apply. */ + sessionLog?: Pick +} + +/** Independent Session-log transport and queue settings. */ +export interface SessionLogOptions { + /** Explicit destination and SDK transport options. */ + exporter: OTLPExporterNodeConfigBase & { + /** Full HTTP(S) logs destination. */ + url: string + } + /** Session-only queue settings, independent of product-event aggregation. */ + processor?: Omit + /** May lower, but never exceed, the collector's 4,000,000-byte limit. */ + maxRequestBytes?: number + /** Application and anonymous identity carried on the OTLP resource. */ + resourceAttributes: Attributes + /** Report rejected single records and network failures without recording their content. */ + onFailure: (message: string, error?: Error) => void } ``` @@ -2703,7 +2723,7 @@ export interface Config { - `inject`: `sessions` - `refs`: `BatchLogRecordProcessorOptions` (`@opentelemetry/sdk-logs`) · `OTLPExporterNodeConfigBase` (`@opentelemetry/otlp-exporter-base`) -- `source`: [`packages/session/session-telemetry-otel/src/index.ts:100`](../packages/session/session-telemetry-otel/src/index.ts) +- `source`: [`packages/session/session-telemetry-otel/src/index.ts:86`](../packages/session/session-telemetry-otel/src/index.ts) ```ts config-catalog /** @@ -2715,10 +2735,8 @@ export interface Config { /** Defaults to `FEEDBACK_ONLY`: capture session history only when feedback is explicitly submitted. */ mode?: SessionTelemetryMode /** - * Passed verbatim to the SDK's OTLP/HTTP log exporter — the complete - * `OTLPExporterNodeConfigBase` shape (`headers`, `timeoutMillis`, - * `compression`, `keepAlive`, …), owned and documented by the SDK. `url` - * is the one field this package requires and validates itself. + * Explicit SDK HTTP transport settings. The reporter adds x-channel routing + * and does not inherit ambient credentials. URL is required while uploading. */ exporter?: OTLPExporterNodeConfigBase & { /** Full logs endpoint (e.g. `https://collector.example.com/v1/logs`). Required outside `DISABLED`; validated at load. */ @@ -2731,6 +2749,8 @@ export interface Config { processor?: Omit /** Maximum time spent awaiting the SDK provider's complete shutdown path. */ shutdownTimeoutMillis?: number + /** Uncompressed OTLP request byte limit, at most 4,000,000. */ + maxRequestBytes?: number } /** Session-sharing policy selected by {@link Config.mode}. */ diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index 41b92912b4..9a5c97b4ec 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -1437,7 +1437,8 @@ export interface Config { ## `@deepseek-ai/dsh-host-product-telemetry-otel` -- `source`: [`packages/host/product-telemetry-otel/src/index.ts:37`](../packages/host/product-telemetry-otel/src/index.ts) +- `refs`: `Attributes` (`@opentelemetry/api`) · `BatchLogRecordProcessorOptions` (`@opentelemetry/sdk-logs`) · `OTLPExporterNodeConfigBase` (`@opentelemetry/otlp-exporter-base`) +- `source`: [`packages/host/product-telemetry-otel/src/index.ts:38`](../packages/host/product-telemetry-otel/src/index.ts) ```ts config-catalog /** Collector routing, application identity, and bounded in-memory batch settings. */ @@ -1464,6 +1465,25 @@ export interface Config { exportTimeoutMillis: number /** Outer shutdown wait; pending exports may be lost after this deadline. */ shutdownTimeoutMillis: number + /** Independent Session-log batching and byte limit; product-event settings do not apply. */ + sessionLog?: Pick +} + +/** Independent Session-log transport and queue settings. */ +export interface SessionLogOptions { + /** Explicit destination and SDK transport options. */ + exporter: OTLPExporterNodeConfigBase & { + /** Full HTTP(S) logs destination. */ + url: string + } + /** Session-only queue settings, independent of product-event aggregation. */ + processor?: Omit + /** May lower, but never exceed, the collector's 4,000,000-byte limit. */ + maxRequestBytes?: number + /** Application and anonymous identity carried on the OTLP resource. */ + resourceAttributes: Attributes + /** Report rejected single records and network failures without recording their content. */ + onFailure: (message: string, error?: Error) => void } ``` @@ -2705,7 +2725,7 @@ export interface Config { - `inject`: `sessions` - `refs`: `BatchLogRecordProcessorOptions` (`@opentelemetry/sdk-logs`) · `OTLPExporterNodeConfigBase` (`@opentelemetry/otlp-exporter-base`) -- `source`: [`packages/session/session-telemetry-otel/src/index.ts:100`](../packages/session/session-telemetry-otel/src/index.ts) +- `source`: [`packages/session/session-telemetry-otel/src/index.ts:86`](../packages/session/session-telemetry-otel/src/index.ts) ```ts config-catalog /** @@ -2717,10 +2737,8 @@ export interface Config { /** Defaults to `FEEDBACK_ONLY`: capture session history only when feedback is explicitly submitted. */ mode?: SessionTelemetryMode /** - * Passed verbatim to the SDK's OTLP/HTTP log exporter — the complete - * `OTLPExporterNodeConfigBase` shape (`headers`, `timeoutMillis`, - * `compression`, `keepAlive`, …), owned and documented by the SDK. `url` - * is the one field this package requires and validates itself. + * Explicit SDK HTTP transport settings. The reporter adds x-channel routing + * and does not inherit ambient credentials. URL is required while uploading. */ exporter?: OTLPExporterNodeConfigBase & { /** Full logs endpoint (e.g. `https://collector.example.com/v1/logs`). Required outside `DISABLED`; validated at load. */ @@ -2733,6 +2751,8 @@ export interface Config { processor?: Omit /** Maximum time spent awaiting the SDK provider's complete shutdown path. */ shutdownTimeoutMillis?: number + /** Uncompressed OTLP request byte limit, at most 4,000,000. */ + maxRequestBytes?: number } /** Session-sharing policy selected by {@link Config.mode}. */ diff --git a/docs/event-producer-consumer.md b/docs/event-producer-consumer.md index 5312354d60..f3aea6091d 100644 --- a/docs/event-producer-consumer.md +++ b/docs/event-producer-consumer.md @@ -60,7 +60,7 @@ This matrix shows which packages dispatch each harness-owned event and which pac | `plugin-manager/install-log` | `emit` | [`packages/boot/plugin-manager/src/types.ts:249`](../packages/boot/plugin-manager/src/types.ts) | [`plugin-manager`](../packages/boot/plugin-manager) (`emit`) | `remotes` | | `plugin-manager/install-state` | `emit` | [`packages/boot/plugin-manager/src/types.ts:256`](../packages/boot/plugin-manager/src/types.ts) | [`plugin-manager`](../packages/boot/plugin-manager) (`emit`) | `remotes` | | `schedule/changed` | `emit` | [`packages/schedule/schedule/src/types.ts:471`](../packages/schedule/schedule/src/types.ts) | [`schedule`](../packages/schedule/schedule) (`emit`) | `remotes` | -| `session-telemetry/record` | `waterfall` | [`packages/session/session-telemetry/src/index.ts:43`](../packages/session/session-telemetry/src/index.ts) | [`session-telemetry`](../packages/session/session-telemetry) (`waterfall`) | - | +| `session-telemetry/record` | `waterfall` | [`packages/session/session-telemetry/src/index.ts:44`](../packages/session/session-telemetry/src/index.ts) | [`session-telemetry`](../packages/session/session-telemetry) (`waterfall`) | - | | `session/created` | `emit` | [`packages/core/session/src/index.ts:55`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`compaction`](../packages/compaction/compaction), [`goal`](../packages/goal/goal), [`hook-protocol`](../packages/hooks/hook-protocol), [`llm-retry`](../packages/llm/llm-retry), [`permission-presets`](../packages/interaction/permission-presets), [`plan-mode`](../packages/plan/plan-mode), [`schedule`](../packages/schedule/schedule), `server`, [`session`](../packages/core/session), `session-controller`, [`session-log-deepseek`](../packages/session/session-log-deepseek), [`session-projection`](../packages/session/session-projection), [`session-projection-cache`](../packages/session/session-projection-cache), [`session-telemetry`](../packages/session/session-telemetry), [`session-title`](../packages/session/session-title), [`time-context`](../packages/context/time-context), [`tool-todo`](../packages/todo/tool-todo), [`tool-workflow`](../packages/workflow/tool-workflow), [`tools`](../packages/core/tools), [`user-approval`](../packages/interaction/user-approval) | | `session/disposed` | `emit` | [`packages/core/session/src/index.ts:65`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`agent-loop`](../packages/core/agent-loop), `agent-team`, `file-upload`, `session-controller`, [`session-persistence-jsonl`](../packages/session/session-persistence-jsonl), [`session-projection-cache`](../packages/session/session-projection-cache), [`session-telemetry`](../packages/session/session-telemetry), [`session-title`](../packages/session/session-title), [`workspace-changes`](../packages/deliverables/workspace-changes) | | `session/event` | `emit` | [`packages/core/session/src/index.ts:77`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`acp`](../packages/acp/acp), [`agent-instructions`](../packages/context/agent-instructions), [`agent-loop`](../packages/core/agent-loop), [`agent-preset-registry`](../packages/preset/agent-preset-registry), `agent-team`, [`compaction`](../packages/compaction/compaction), [`compaction-basic`](../packages/compaction/compaction-basic), [`file-reference-local`](../packages/context/file-reference-local), `file-upload`, [`goal`](../packages/goal/goal), [`goal-round-driver`](../packages/goal/goal-round-driver), [`headless`](../packages/bundle/headless), [`hook-protocol`](../packages/hooks/hook-protocol), [`loader-smoke`](../packages/test-support/loader-smoke), `server`, [`session`](../packages/core/session), `session-controller`, [`session-persistence-jsonl`](../packages/session/session-persistence-jsonl), [`session-projection`](../packages/session/session-projection), [`session-projection-cache`](../packages/session/session-projection-cache), [`session-telemetry`](../packages/session/session-telemetry), [`session-telemetry-otel`](../packages/session/session-telemetry-otel), [`session-title`](../packages/session/session-title), [`token-meter`](../packages/llm/token-meter), [`tool-todo`](../packages/todo/tool-todo), [`tool-workflow`](../packages/workflow/tool-workflow), [`tools`](../packages/core/tools), [`user-approval`](../packages/interaction/user-approval), [`workspace-changes`](../packages/deliverables/workspace-changes) | diff --git a/docs/event-producer-consumer.zh.md b/docs/event-producer-consumer.zh.md index 6baa07d2d2..dc4391fa54 100644 --- a/docs/event-producer-consumer.zh.md +++ b/docs/event-producer-consumer.zh.md @@ -62,7 +62,7 @@ | `plugin-manager/install-log` | `emit` | [`packages/boot/plugin-manager/src/types.ts:249`](../packages/boot/plugin-manager/src/types.ts) | [`plugin-manager`](../packages/boot/plugin-manager) (`emit`) | `remotes` | | `plugin-manager/install-state` | `emit` | [`packages/boot/plugin-manager/src/types.ts:256`](../packages/boot/plugin-manager/src/types.ts) | [`plugin-manager`](../packages/boot/plugin-manager) (`emit`) | `remotes` | | `schedule/changed` | `emit` | [`packages/schedule/schedule/src/types.ts:471`](../packages/schedule/schedule/src/types.ts) | [`schedule`](../packages/schedule/schedule) (`emit`) | `remotes` | -| `session-telemetry/record` | `waterfall` | [`packages/session/session-telemetry/src/index.ts:43`](../packages/session/session-telemetry/src/index.ts) | [`session-telemetry`](../packages/session/session-telemetry) (`waterfall`) | - | +| `session-telemetry/record` | `waterfall` | [`packages/session/session-telemetry/src/index.ts:44`](../packages/session/session-telemetry/src/index.ts) | [`session-telemetry`](../packages/session/session-telemetry) (`waterfall`) | - | | `session/created` | `emit` | [`packages/core/session/src/index.ts:55`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`compaction`](../packages/compaction/compaction), [`goal`](../packages/goal/goal), [`hook-protocol`](../packages/hooks/hook-protocol), [`llm-retry`](../packages/llm/llm-retry), [`permission-presets`](../packages/interaction/permission-presets), [`plan-mode`](../packages/plan/plan-mode), [`schedule`](../packages/schedule/schedule), `server`, [`session`](../packages/core/session), `session-controller`, [`session-log-deepseek`](../packages/session/session-log-deepseek), [`session-projection`](../packages/session/session-projection), [`session-projection-cache`](../packages/session/session-projection-cache), [`session-telemetry`](../packages/session/session-telemetry), [`session-title`](../packages/session/session-title), [`time-context`](../packages/context/time-context), [`tool-todo`](../packages/todo/tool-todo), [`tool-workflow`](../packages/workflow/tool-workflow), [`tools`](../packages/core/tools), [`user-approval`](../packages/interaction/user-approval) | | `session/disposed` | `emit` | [`packages/core/session/src/index.ts:65`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`agent-loop`](../packages/core/agent-loop), `agent-team`, `file-upload`, `session-controller`, [`session-persistence-jsonl`](../packages/session/session-persistence-jsonl), [`session-projection-cache`](../packages/session/session-projection-cache), [`session-telemetry`](../packages/session/session-telemetry), [`session-title`](../packages/session/session-title), [`workspace-changes`](../packages/deliverables/workspace-changes) | | `session/event` | `emit` | [`packages/core/session/src/index.ts:77`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`acp`](../packages/acp/acp), [`agent-instructions`](../packages/context/agent-instructions), [`agent-loop`](../packages/core/agent-loop), [`agent-preset-registry`](../packages/preset/agent-preset-registry), `agent-team`, [`compaction`](../packages/compaction/compaction), [`compaction-basic`](../packages/compaction/compaction-basic), [`file-reference-local`](../packages/context/file-reference-local), `file-upload`, [`goal`](../packages/goal/goal), [`goal-round-driver`](../packages/goal/goal-round-driver), [`headless`](../packages/bundle/headless), [`hook-protocol`](../packages/hooks/hook-protocol), [`loader-smoke`](../packages/test-support/loader-smoke), `server`, [`session`](../packages/core/session), `session-controller`, [`session-persistence-jsonl`](../packages/session/session-persistence-jsonl), [`session-projection`](../packages/session/session-projection), [`session-projection-cache`](../packages/session/session-projection-cache), [`session-telemetry`](../packages/session/session-telemetry), [`session-telemetry-otel`](../packages/session/session-telemetry-otel), [`session-title`](../packages/session/session-title), [`token-meter`](../packages/llm/token-meter), [`tool-todo`](../packages/todo/tool-todo), [`tool-workflow`](../packages/workflow/tool-workflow), [`tools`](../packages/core/tools), [`user-approval`](../packages/interaction/user-approval), [`workspace-changes`](../packages/deliverables/workspace-changes) | diff --git a/docs/module-graph.md b/docs/module-graph.md index 95370f261d..85e0de2d1c 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -496,6 +496,7 @@ flowchart TD pkg_experimental_webworker_runtime --> pkg_host_webserver pkg_host_frontend_static --> pkg_client_connection pkg_host_frontend_static --> pkg_host_webserver + pkg_host_product_telemetry_otel --> pkg_session pkg_persona --> pkg_system_prompt pkg_sandbox --> pkg_llm pkg_sandbox --> pkg_session @@ -1035,6 +1036,7 @@ flowchart TD pkg_agent_preset --> pkg_agent_preset_registry pkg_session_telemetry_otel --> pkg_anonymous_user_id pkg_session_telemetry_otel --> pkg_command_feedback + pkg_session_telemetry_otel --> pkg_host_product_telemetry_otel pkg_session_telemetry_otel --> pkg_llm pkg_session_telemetry_otel --> pkg_message_feedback pkg_session_telemetry_otel --> pkg_session @@ -1462,7 +1464,6 @@ flowchart TD | [`host-directory-picker-browse`](../packages/host/directory-picker-browse) | `host` | — | | [`host-directory-picker-native`](../packages/host/directory-picker-native) | `host` | — | | [`host-open-in-app`](../packages/host/open-in-app) | `host` | — | -| [`host-product-telemetry-otel`](../packages/host/product-telemetry-otel) | `host` | — | | [`host-webserver`](../packages/host/webserver) | `host` | — | | [`invariants`](../packages/runtime-diagnostics/invariants) | `runtime-diagnostics` | — | | [`session-format`](../packages/session/session-format) | `session` | — | @@ -1515,6 +1516,7 @@ flowchart TD | [`app-boot`](../packages/boot/app-boot) | `boot` | [`home-paths`](../packages/util/home-paths), [`launch-environment`](../packages/util/launch-environment), [`system-prompt`](../packages/core/system-prompt) | | [`experimental-webworker-runtime`](../packages/experimental/webworker-runtime) | `experimental` | [`client-connection`](../packages/client/connection), [`client-modules`](../packages/client/modules), [`host-webserver`](../packages/host/webserver) | | [`host-frontend-static`](../packages/host/frontend-static) | `host` | [`client-connection`](../packages/client/connection), [`host-webserver`](../packages/host/webserver) | +| [`host-product-telemetry-otel`](../packages/host/product-telemetry-otel) | `host` | [`session`](../packages/core/session) | | [`persona`](../packages/preset/persona) | `preset` | [`system-prompt`](../packages/core/system-prompt) | | [`sandbox`](../packages/sandbox/sandbox) | `sandbox` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session) | | [`session-format-catalog`](../packages/session/session-format-catalog) | `session` | [`session`](../packages/core/session) | @@ -1637,7 +1639,7 @@ flowchart TD | [`host-plugin-inventory`](../packages/host/plugin-inventory) | `host` | [`agent-preset-registry`](../packages/preset/agent-preset-registry), [`brand`](../packages/util/brand), [`typert-protocol`](../packages/typert/protocol) | | [`mcp-client`](../packages/mcp/mcp-client) | `mcp` | [`attachment`](../packages/attachment/attachment), [`llm`](../packages/llm/llm), [`mcp-resources`](../packages/mcp/mcp-resources), [`scope`](../packages/core/scope), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) | | [`agent-preset`](../packages/preset/agent-preset) | `preset` | [`agent-preset-registry`](../packages/preset/agent-preset-registry) | -| [`session-telemetry-otel`](../packages/session/session-telemetry-otel) | `session` | [`anonymous-user-id`](../packages/identity/anonymous-user-id), [`command-feedback`](../packages/feedback/command-feedback), [`llm`](../packages/llm/llm), [`message-feedback`](../packages/feedback/message-feedback), [`session`](../packages/core/session), [`session-telemetry`](../packages/session/session-telemetry) | +| [`session-telemetry-otel`](../packages/session/session-telemetry-otel) | `session` | [`anonymous-user-id`](../packages/identity/anonymous-user-id), [`command-feedback`](../packages/feedback/command-feedback), [`host-product-telemetry-otel`](../packages/host/product-telemetry-otel), [`llm`](../packages/llm/llm), [`message-feedback`](../packages/feedback/message-feedback), [`session`](../packages/core/session), [`session-telemetry`](../packages/session/session-telemetry) | | [`tool-bash`](../packages/shell/tool-bash) | `shell` | [`agent`](../packages/core/agent), [`jobs`](../packages/jobs/jobs), [`llm`](../packages/llm/llm), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`shell`](../packages/shell/shell), [`shell-env`](../packages/shell/shell-env), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-approval`](../packages/interaction/user-approval) | | [`tool-pwsh`](../packages/shell/tool-pwsh) | `shell` | [`agent`](../packages/core/agent), [`jobs`](../packages/jobs/jobs), [`llm`](../packages/llm/llm), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`shell`](../packages/shell/shell), [`shell-env`](../packages/shell/shell-env), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-approval`](../packages/interaction/user-approval) | | [`fs-ssh`](../packages/ssh/fs-ssh) | `ssh` | [`fs`](../packages/fs/fs), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`ssh`](../packages/ssh/ssh) | diff --git a/docs/module-graph.zh.md b/docs/module-graph.zh.md index fa7306530d..fde5a05ebf 100644 --- a/docs/module-graph.zh.md +++ b/docs/module-graph.zh.md @@ -498,6 +498,7 @@ flowchart TD pkg_experimental_webworker_runtime --> pkg_host_webserver pkg_host_frontend_static --> pkg_client_connection pkg_host_frontend_static --> pkg_host_webserver + pkg_host_product_telemetry_otel --> pkg_session pkg_persona --> pkg_system_prompt pkg_sandbox --> pkg_llm pkg_sandbox --> pkg_session @@ -1037,6 +1038,7 @@ flowchart TD pkg_agent_preset --> pkg_agent_preset_registry pkg_session_telemetry_otel --> pkg_anonymous_user_id pkg_session_telemetry_otel --> pkg_command_feedback + pkg_session_telemetry_otel --> pkg_host_product_telemetry_otel pkg_session_telemetry_otel --> pkg_llm pkg_session_telemetry_otel --> pkg_message_feedback pkg_session_telemetry_otel --> pkg_session @@ -1464,7 +1466,6 @@ flowchart TD | [`host-directory-picker-browse`](../packages/host/directory-picker-browse) | `host` | — | | [`host-directory-picker-native`](../packages/host/directory-picker-native) | `host` | — | | [`host-open-in-app`](../packages/host/open-in-app) | `host` | — | -| [`host-product-telemetry-otel`](../packages/host/product-telemetry-otel) | `host` | — | | [`host-webserver`](../packages/host/webserver) | `host` | — | | [`invariants`](../packages/runtime-diagnostics/invariants) | `runtime-diagnostics` | — | | [`session-format`](../packages/session/session-format) | `session` | — | @@ -1517,6 +1518,7 @@ flowchart TD | [`app-boot`](../packages/boot/app-boot) | `boot` | [`home-paths`](../packages/util/home-paths), [`launch-environment`](../packages/util/launch-environment), [`system-prompt`](../packages/core/system-prompt) | | [`experimental-webworker-runtime`](../packages/experimental/webworker-runtime) | `experimental` | [`client-connection`](../packages/client/connection), [`client-modules`](../packages/client/modules), [`host-webserver`](../packages/host/webserver) | | [`host-frontend-static`](../packages/host/frontend-static) | `host` | [`client-connection`](../packages/client/connection), [`host-webserver`](../packages/host/webserver) | +| [`host-product-telemetry-otel`](../packages/host/product-telemetry-otel) | `host` | [`session`](../packages/core/session) | | [`persona`](../packages/preset/persona) | `preset` | [`system-prompt`](../packages/core/system-prompt) | | [`sandbox`](../packages/sandbox/sandbox) | `sandbox` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session) | | [`session-format-catalog`](../packages/session/session-format-catalog) | `session` | [`session`](../packages/core/session) | @@ -1639,7 +1641,7 @@ flowchart TD | [`host-plugin-inventory`](../packages/host/plugin-inventory) | `host` | [`agent-preset-registry`](../packages/preset/agent-preset-registry), [`brand`](../packages/util/brand), [`typert-protocol`](../packages/typert/protocol) | | [`mcp-client`](../packages/mcp/mcp-client) | `mcp` | [`attachment`](../packages/attachment/attachment), [`llm`](../packages/llm/llm), [`mcp-resources`](../packages/mcp/mcp-resources), [`scope`](../packages/core/scope), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) | | [`agent-preset`](../packages/preset/agent-preset) | `preset` | [`agent-preset-registry`](../packages/preset/agent-preset-registry) | -| [`session-telemetry-otel`](../packages/session/session-telemetry-otel) | `session` | [`anonymous-user-id`](../packages/identity/anonymous-user-id), [`command-feedback`](../packages/feedback/command-feedback), [`llm`](../packages/llm/llm), [`message-feedback`](../packages/feedback/message-feedback), [`session`](../packages/core/session), [`session-telemetry`](../packages/session/session-telemetry) | +| [`session-telemetry-otel`](../packages/session/session-telemetry-otel) | `session` | [`anonymous-user-id`](../packages/identity/anonymous-user-id), [`command-feedback`](../packages/feedback/command-feedback), [`host-product-telemetry-otel`](../packages/host/product-telemetry-otel), [`llm`](../packages/llm/llm), [`message-feedback`](../packages/feedback/message-feedback), [`session`](../packages/core/session), [`session-telemetry`](../packages/session/session-telemetry) | | [`tool-bash`](../packages/shell/tool-bash) | `shell` | [`agent`](../packages/core/agent), [`jobs`](../packages/jobs/jobs), [`llm`](../packages/llm/llm), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`shell`](../packages/shell/shell), [`shell-env`](../packages/shell/shell-env), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-approval`](../packages/interaction/user-approval) | | [`tool-pwsh`](../packages/shell/tool-pwsh) | `shell` | [`agent`](../packages/core/agent), [`jobs`](../packages/jobs/jobs), [`llm`](../packages/llm/llm), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`shell`](../packages/shell/shell), [`shell-env`](../packages/shell/shell-env), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-approval`](../packages/interaction/user-approval) | | [`fs-ssh`](../packages/ssh/fs-ssh) | `ssh` | [`fs`](../packages/fs/fs), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`ssh`](../packages/ssh/ssh) | diff --git a/docs/subsystems/product-telemetry.i18n.yaml b/docs/subsystems/product-telemetry.i18n.yaml index bd22924083..22019c0b31 100644 --- a/docs/subsystems/product-telemetry.i18n.yaml +++ b/docs/subsystems/product-telemetry.i18n.yaml @@ -3,8 +3,8 @@ # After editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/subsystems/product-telemetry.md /product-telemetry: - en: e1a8950c3b4743ed - zh: 3fff7e01a8f271e1 + en: 00f932c8124f931c + zh: 0607741db404976f /product-telemetry/record-types: en: bf5eadedbc896f5a zh: 158fce1609a85d6d diff --git a/docs/subsystems/product-telemetry.md b/docs/subsystems/product-telemetry.md index d86540d5a2..a4c33615e7 100644 --- a/docs/subsystems/product-telemetry.md +++ b/docs/subsystems/product-telemetry.md @@ -8,6 +8,8 @@ The [product telemetry plugin](../../packages/host/product-telemetry-otel/README Enqueue is synchronous and does not acknowledge delivery. The SDK owns batching and retry; local diagnostics report export failures. See the package README for configuration, shutdown, and loss limits. +`reportSessionLog()` submits one complete Session event as `eventName: "session-log"` with string attributes `sessionId` and `content`. `SessionLogReporter` exposes the same independently queued sender to feedback backends. The sender measures full OTLP JSON bytes before gzip, splits requests at 4,000,000 bytes or a lower configured limit, and rejects oversized single events without truncation. + ## Record types ```ts type-equiv @@ -33,6 +35,38 @@ interface ProductTelemetryRecord { Source: [`packages/host/product-telemetry-otel/src/index.ts`](../../packages/host/product-telemetry-otel/src/index.ts) +```ts type-equiv +/** One canonical event with its separately owned Session identity and redacted payload. */ +interface SessionLogRecord { + sessionId: SessionId + /** Complete event envelope; data is the capture policy's exported copy. */ + event: Omit & { data: unknown } + /** Additional capture metadata; sessionId and content are always assigned by the reporter. */ + attributes?: Attributes + /** Omitted values use INFO. */ + severityNumber?: SeverityNumber +} +``` + +```ts type-equiv +/** Independent Session-log transport and queue settings. */ +interface SessionLogOptions { + /** Explicit destination and SDK transport options. */ + exporter: OTLPExporterNodeConfigBase & { + /** Full HTTP(S) logs destination. */ + url: string + } + /** Session-only queue settings, independent of product-event aggregation. */ + processor?: Omit + /** May lower, but never exceed, the collector's 4,000,000-byte limit. */ + maxRequestBytes?: number + /** Application and anonymous identity carried on the OTLP resource. */ + resourceAttributes: Attributes + /** Report rejected single records and network failures without recording their content. */ + onFailure: (message: string, error?: Error) => void +} +``` + @@ -49,9 +83,15 @@ Host analytics sender. Mounting alone sends nothing; the owning fiber drains it ```ts cordis-catalog /** - * Enqueue one selected product event without waiting for network delivery. + * Enqueue one explicitly authorized Session event in the separate byte-bounded queue. * Queue admission and shutdown completion are not collector or warehouse acknowledgements. - * @param record - caller-owned event containing only approved analytics fields. + * @param record - complete Session event and its owning Session id. + */ +reportSessionLog(record: SessionLogRecord): void + +/** + * Enqueue a selected product event independently of Session-log reporting. + * @param record - approved analytics fields; no automatic Session collection. */ emit(record: ProductTelemetryRecord): void ``` diff --git a/docs/subsystems/product-telemetry.zh.md b/docs/subsystems/product-telemetry.zh.md index 52ad119363..fd413ff167 100644 --- a/docs/subsystems/product-telemetry.zh.md +++ b/docs/subsystems/product-telemetry.zh.md @@ -8,6 +8,8 @@ 入队同步完成，不代表送达确认。SDK 负责批量发送与重试；发送失败会产生本地诊断。配置、退出和丢失限制见包 README。 +`reportSessionLog()` 将一条完整 Session 事件作为 `eventName: "session-log"` 提交，字符串属性为 `sessionId` 和 `content`。`SessionLogReporter` 向反馈后端提供同一个独立排队的发送器。发送器在 gzip 前计量完整 OTLP JSON 字节数，按 4,000,000 字节或更低配置上限拆分请求，并拒绝超限单条事件而不截断。 + ## 记录类型 ```ts type-equiv @@ -33,6 +35,38 @@ interface ProductTelemetryRecord { Source: [`packages/host/product-telemetry-otel/src/index.ts`](../../packages/host/product-telemetry-otel/src/index.ts) +```ts type-equiv +/** One canonical event with its separately owned Session identity and redacted payload. */ +interface SessionLogRecord { + sessionId: SessionId + /** Complete event envelope; data is the capture policy's exported copy. */ + event: Omit & { data: unknown } + /** Additional capture metadata; sessionId and content are always assigned by the reporter. */ + attributes?: Attributes + /** Omitted values use INFO. */ + severityNumber?: SeverityNumber +} +``` + +```ts type-equiv +/** Independent Session-log transport and queue settings. */ +interface SessionLogOptions { + /** Explicit destination and SDK transport options. */ + exporter: OTLPExporterNodeConfigBase & { + /** Full HTTP(S) logs destination. */ + url: string + } + /** Session-only queue settings, independent of product-event aggregation. */ + processor?: Omit + /** May lower, but never exceed, the collector's 4,000,000-byte limit. */ + maxRequestBytes?: number + /** Application and anonymous identity carried on the OTLP resource. */ + resourceAttributes: Attributes + /** Report rejected single records and network failures without recording their content. */ + onFailure: (message: string, error?: Error) => void +} +``` + @@ -49,9 +83,15 @@ Host analytics sender. Mounting alone sends nothing; the owning fiber drains it ```ts cordis-catalog /** - * Enqueue one selected product event without waiting for network delivery. + * Enqueue one explicitly authorized Session event in the separate byte-bounded queue. * Queue admission and shutdown completion are not collector or warehouse acknowledgements. - * @param record - caller-owned event containing only approved analytics fields. + * @param record - complete Session event and its owning Session id. + */ +reportSessionLog(record: SessionLogRecord): void + +/** + * Enqueue a selected product event independently of Session-log reporting. + * @param record - approved analytics fields; no automatic Session collection. */ emit(record: ProductTelemetryRecord): void ``` diff --git a/docs/subsystems/session-telemetry.i18n.yaml b/docs/subsystems/session-telemetry.i18n.yaml index b2c2f42fb9..3b50e141b3 100644 --- a/docs/subsystems/session-telemetry.i18n.yaml +++ b/docs/subsystems/session-telemetry.i18n.yaml @@ -3,8 +3,8 @@ # After editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/subsystems/session-telemetry.md /sessiontelemetrybackend: - en: d400c8668cd76829 - zh: 67689676c19daa7f + en: b95249b397f4f069 + zh: 5ddccf5c7596e6d3 /sessiontelemetrybackend/the-logical-record: en: 9695489076496a76 zh: 457c54698bc38c2f diff --git a/docs/subsystems/session-telemetry.md b/docs/subsystems/session-telemetry.md index 18fe935e5e..5362cb45a5 100644 --- a/docs/subsystems/session-telemetry.md +++ b/docs/subsystems/session-telemetry.md @@ -2,7 +2,7 @@ English | [中文](session-telemetry.zh.md) -Outbound session reporting is split as a [capability seam](../capability-seams.md): the Service Definition and capture coordinator ([dsh-session-telemetry](../../packages/session/session-telemetry), `ctx.sessionTelemetry`) own complete canonical-event capture, the `session-telemetry/record` redaction waterfall, the handoff cursor, and the minimal backend contract; the Service Provider a deployment loads ([dsh-session-telemetry-otel](../../packages/session/session-telemetry-otel)) is the OpenTelemetry JS SDK's log pipeline configured verbatim. It is one optional capability, not part of the agent-loop spine, and nothing here reaches a model request. The boundary axiom — the harness's aspect ends at `emit()`; batching, retry, queueing, and loss policy belong to the reporting SDK — and the rejected alternatives are pinned in the [revival Agent Note](../../.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.md); the capture and cursor contracts live in the [Service Definition README](../../packages/session/session-telemetry/README.md). +The [capture service](../../packages/session/session-telemetry/README.md) owns canonical-event capture, redaction, and handoff cursors. The [feedback backend](../../packages/session/session-telemetry-otel/README.md) authorizes bounded Session prefixes and submits complete event strings through the [shared OTel reporter](../../packages/host/product-telemetry-otel/README.md), with an independent queue and a 4,000,000-byte request ceiling. No reporting data reaches model requests. Source: [`packages/session/session-telemetry/src/index.ts`](../../packages/session/session-telemetry/src/index.ts) @@ -29,6 +29,8 @@ type SessionTelemetrySeverity = 'info' | 'warn' | 'error' * identity so they can never be mistaken for ledger rows. */ interface SessionTelemetryRecord { + /** Canonical envelope without data; body carries the separately redacted payload. Absent for operational records. */ + sourceEvent?: { sessionId: SessionId; envelope: Omit } /** Ledger (session-log mirror) or ops (operational signal) channel; backends keep the two under separate instrumentation scopes. */ channel: 'ledger' | 'ops' /** Unix epoch milliseconds — the source event's append time for ledger records, the emission time for ops records. */ diff --git a/docs/subsystems/session-telemetry.zh.md b/docs/subsystems/session-telemetry.zh.md index 1b3807d2e3..02cb7e6dd2 100644 --- a/docs/subsystems/session-telemetry.zh.md +++ b/docs/subsystems/session-telemetry.zh.md @@ -2,7 +2,7 @@ [English](session-telemetry.md) | 中文 -对外的会话上报拆分为一项[能力 seam](../capability-seams.zh.md)：Service Definition 与捕获协调器（[dsh-session-telemetry](../../packages/session/session-telemetry)，`ctx.sessionTelemetry`）拥有完整的权威事件捕获、`session-telemetry/record` 脱敏 waterfall（瀑布式事件）、handoff 游标与最小后端约定；部署方加载的 Service Provider（[dsh-session-telemetry-otel](../../packages/session/session-telemetry-otel)）则是原样配置的 OpenTelemetry JS SDK 日志流水线。它是一项可选能力，不属于 agent loop（智能体循环）主干，这里也没有任何内容会进入模型请求。边界公理（harness 的职责止于 `emit()`；批处理、重试、排队与丢失策略都属于上报 SDK）连同被否决的替代方案，均已在[复活 Agent Note](../../.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.zh.md)中定案；捕获与游标约定见 [Service Definition README](../../packages/session/session-telemetry/README.zh.md)。 +[采集服务](../../packages/session/session-telemetry/README.zh.md) 负责规范事件采集、脱敏和交接游标。[反馈后端](../../packages/session/session-telemetry-otel/README.zh.md) 授权有界 Session 前缀，并通过[共享 OTel 上报器](../../packages/host/product-telemetry-otel/README.zh.md) 提交完整事件字符串，使用独立队列和 4,000,000 字节请求上限。上报数据不会进入模型请求。 源码：[`packages/session/session-telemetry/src/index.ts`](../../packages/session/session-telemetry/src/index.ts) @@ -29,6 +29,8 @@ type SessionTelemetrySeverity = 'info' | 'warn' | 'error' * identity so they can never be mistaken for ledger rows. */ interface SessionTelemetryRecord { + /** Canonical envelope without data; body carries the separately redacted payload. Absent for operational records. */ + sourceEvent?: { sessionId: SessionId; envelope: Omit } /** Ledger (session-log mirror) or ops (operational signal) channel; backends keep the two under separate instrumentation scopes. */ channel: 'ledger' | 'ops' /** Unix epoch milliseconds — the source event's append time for ledger records, the emission time for ops records. */ diff --git a/packages/bundle/base/cordis.patch.yml b/packages/bundle/base/cordis.patch.yml index db6701a9b6..20988c71e3 100644 --- a/packages/bundle/base/cordis.patch.yml +++ b/packages/bundle/base/cordis.patch.yml @@ -206,8 +206,9 @@ config: mode: !!js process.env.DSH_TELEMETRY_MODE || 'FEEDBACK_ONLY' shutdownTimeoutMillis: 3000 + maxRequestBytes: 4000000 exporter: - url: !!js process.env.DSH_TELEMETRY_OTLP_URL ?? 'https://harness-telemetry.deepseeksvc.com/v1/logs' + url: !!js process.env.DSH_TELEMETRY_OTLP_URL ?? 'https://dsh-otel-collector.deepseeksvc.com/v1/logs' compression: gzip timeoutMillis: 1000 processor: diff --git a/packages/extensions/tool-cordis/src/api-catalog.ts b/packages/extensions/tool-cordis/src/api-catalog.ts index 501831b9cc..5d8c28e3c9 100644 --- a/packages/extensions/tool-cordis/src/api-catalog.ts +++ b/packages/extensions/tool-cordis/src/api-catalog.ts @@ -1702,10 +1702,15 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ summary: 'Host analytics sender.', description: 'Host analytics sender. Mounting alone sends nothing; the owning fiber drains it on unload.', methods: [ + { + signature: 'reportSessionLog(record: SessionLogRecord): void', + description: 'Enqueue one explicitly authorized Session event in the separate byte-bounded queue. Queue admission and shutdown completion are not collector or warehouse acknowledgements.', + parameters: [{ name: 'record', description: 'complete Session event and its owning Session id.' }], + }, { signature: 'emit(record: ProductTelemetryRecord): void', - description: 'Enqueue one selected product event without waiting for network delivery. Queue admission and shutdown completion are not collector or warehouse acknowledgements.', - parameters: [{ name: 'record', description: 'caller-owned event containing only approved analytics fields.' }], + description: 'Enqueue a selected product event independently of Session-log reporting.', + parameters: [{ name: 'record', description: 'approved analytics fields; no automatic Session collection.' }], }, ], }, @@ -6561,6 +6566,10 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'SessionLogOffset', declaration: 'export type SessionLogOffset = BrandedNumber<\'SessionLogOffset\'>;', }, + { + name: 'SessionLogRecord', + declaration: 'export interface SessionLogRecord {\n sessionId: SessionId;\n event: Omit & {\n data: unknown;\n };\n attributes?: Attributes;\n severityNumber?: SeverityNumber;\n}', + }, { name: 'SessionLogSnapshot', declaration: 'export interface SessionLogSnapshot {\n session: SessionHeader;\n inheritedEventCount: SessionLogOffset;\n events: SessionEvent[];\n}', @@ -6767,7 +6776,7 @@ export const TYPE_API: readonly TypeApiEntry[] = [ }, { name: 'SessionTelemetryRecord', - declaration: 'export interface SessionTelemetryRecord {\n channel: \'ledger\' | \'ops\';\n time: number;\n severity: SessionTelemetrySeverity;\n attributes: Record;\n body: unknown;\n}', + declaration: 'export interface SessionTelemetryRecord {\n sourceEvent?: {\n sessionId: SessionId;\n envelope: Omit;\n };\n channel: \'ledger\' | \'ops\';\n time: number;\n severity: SessionTelemetrySeverity;\n attributes: Record;\n body: unknown;\n}', }, { name: 'SessionTelemetrySeverity', diff --git a/packages/host/product-telemetry-otel/README.i18n.yaml b/packages/host/product-telemetry-otel/README.i18n.yaml index e35c7672f8..e1d8950051 100644 --- a/packages/host/product-telemetry-otel/README.i18n.yaml +++ b/packages/host/product-telemetry-otel/README.i18n.yaml @@ -9,17 +9,17 @@ en: 18d7a8c944938416 zh: 4249a421e3d5f09a /deepseek-ai-dsh-host-product-telemetry-otel/summary: - en: a969cf533f6649b5 - zh: 8401a4263f70a8e0 + en: 791225419df7493e + zh: 728b142fcfe9d6e6 /deepseek-ai-dsh-host-product-telemetry-otel/table-of-contents: en: d152484eb41ac6b4 zh: 9aa34fb1dc95d8cf /deepseek-ai-dsh-host-product-telemetry-otel/use-this-package: - en: 08e0bcbc259d131b - zh: 1d7cbde2688a903f + en: a4793469bb1277c7 + zh: c89d59ee4f87a256 /deepseek-ai-dsh-host-product-telemetry-otel/understand-the-implementation: - en: 5d2c06a357a1c06a - zh: 9a1a17535889f5e3 + en: db4d50ec33ae2ce5 + zh: bb22ca3a232e1b76 /deepseek-ai-dsh-host-product-telemetry-otel/further-exploration: en: 6bfb5e8a723ff0a9 zh: fffc6b02e2c1a3cb @@ -30,8 +30,8 @@ en: b287f8e34f3634d1 zh: cd441f253ec0124b /deepseek-ai-dsh-host-product-telemetry-otel/known-limitations-and-deferred-work: - en: 20b9ab75cb7e66a1 - zh: a7dc4f4656eaf839 + en: 09804d496c1d03c4 + zh: 80377ad0c371f0d6 /deepseek-ai-dsh-host-product-telemetry-otel/known-limitations-and-deferred-work/dev-note: en: 0a1253c6f7b5e8b9 zh: 9f56528562c7c914 diff --git a/packages/host/product-telemetry-otel/README.md b/packages/host/product-telemetry-otel/README.md index af1f2c415f..ba9ebb1cc8 100644 --- a/packages/host/product-telemetry-otel/README.md +++ b/packages/host/product-telemetry-otel/README.md @@ -9,7 +9,7 @@ English | [中文](README.zh.md) ## Summary -Send selected product usage events to an OTLP/HTTP collector. Events carry a name, string summary, occurrence time, and scalar or one-level object attributes. Mounting the plugin collects nothing automatically; applications explicitly submit each event. Delivery is best effort and does not confirm warehouse ingestion. +Send selected product usage events and explicitly authorized Session logs to an OTLP/HTTP collector. Session logs use a separate queue and byte-bounded requests. Mounting the plugin collects nothing automatically. Delivery is best effort and does not confirm warehouse ingestion. ## Table of Contents @@ -48,6 +48,7 @@ Mount the plugin in a Cordis composition with the application identity; override | `timeoutMillis` | `15000` | Exporter HTTP and retry deadline | | `exportTimeoutMillis` | `20000` | Processor batch export deadline | | `shutdownTimeoutMillis` | `21000` | Outer wait for shutdown; expiry reports possible loss | +| `sessionLog.maxRequestBytes` / `sessionLog.processor` | `4,000,000` / SDK defaults | Independent Session-log request ceiling and batch settings; the byte ceiling cannot exceed 4,000,000 | The default endpoint routes explicitly submitted events to the production product collector. Test and custom deployments must override it. Only `x-channel` and SDK protocol headers reach the collector; ambient OTel headers and client certificates are not inherited. @@ -55,6 +56,10 @@ The 30-second interval batches product events; the exporter has a 15-second retr Consumers inject `productTelemetry` and call `emit()` with explicitly selected analytics fields. Event names and field semantics belong to their product and analytics owners. The plugin reads no Session, account, credential, or device identifier. Callers must exclude prompts, responses, file contents, credentials, and other unapproved values. +`reportSessionLog(record)` accepts a Session id and one complete event. It writes `eventName: "session-log"`, a string body, and `attributes.sessionId` plus `attributes.content = JSON.stringify(event)`. Optional capture metadata remains on the record. Nested objects, arrays, booleans, nulls, and fractional numbers stay inside the JSON string. The exported `SessionLogReporter` provides the same pipeline to feedback-authorizing backends without mounting the product service. Callers must authorize Session sharing before submission. + +Session records never share a request with product events. Each candidate batch is measured using the SDK OTLP JSON serializer before gzip, including resource, scope, attributes, UTF-8, and JSON escaping. Oversized batches split in event order; a single oversized record is rejected with a content-free diagnostic while other records continue. SDK attribute-length truncation is disabled for Session records. Shutdown awaits split requests as well as queued records, subject to the owner’s outer deadline. + The collector expects a string body and attributes containing strings, numbers, booleans, or objects of those scalars. Callers supply occurrence time in milliseconds; the plugin assigns observation time and defaults severity to INFO. Invalid transport configuration fails at activation. Export failures produce local warnings without making event submission wait for the network. ----- @@ -65,7 +70,7 @@ The collector expects a string body and attributes containing strings, numbers,
Implementation internals — click to expand -A private OTel logger feeds `BatchLogRecordProcessor` and the SDK HTTP delegate with its JSON log serializer. The delegate receives explicit headers and an HTTP agent; only shared timeout and compression settings use SDK environment resolution. The direct `@opentelemetry/core` dependency matches `sdk-logs` at 2.9.0 so exporter result enums share one TypeScript identity. The SDK owns queueing, transient-error retries, and compression; plugin disposal drains pending records with a bounded wait. Export completion is observed separately because SDK shutdown can resolve after a rejected export. No global OTel provider is installed. +Independent OTel providers keep product and Session-log queues separate. Both use the SDK JSON serializer and the same explicit HTTP transport configuration. The Session exporter splits oversized requests before transport; SDK batching cadence and retries remain SDK-owned. Plugin disposal drains both providers under one outer deadline. No global provider is installed. [`src/index.ts`](src/index.ts) owns configuration and submission. No runtime invariant companion is published: delivery has no independent local acknowledgement to compare with the SDK's queue. @@ -99,7 +104,7 @@ Delivery and collection remain limited to the following capabilities. - Callers own event selection, renderer-to-host transport, and any permitted identity attributes. - The queue is memory-only; overflow, network failure, and process exit can lose events. There is no durable outbox or warehouse acknowledgement. -- The SDK batches by record count, not encoded bytes. Callers must keep records within the collector's 4 MB limit and choose batch sizes appropriate to the receiver. +- Product events batch by record count. Session-log batches additionally obey the uncompressed byte ceiling; single-event rejection, queue overflow, and export failure have no durable retry guarantee. - Caller-selected strings are not redacted automatically. This package does not decide product disclosure or consent policy. diff --git a/packages/host/product-telemetry-otel/README.zh.md b/packages/host/product-telemetry-otel/README.zh.md index ffdb682d9a..c011f032de 100644 --- a/packages/host/product-telemetry-otel/README.zh.md +++ b/packages/host/product-telemetry-otel/README.zh.md @@ -9,7 +9,7 @@ kind: "package-reference" ## 概述 -将选定的产品使用事件发送到 OTLP/HTTP 接收服务。事件包含名称、字符串摘要、发生时间，以及标量或单层对象属性。挂载插件不会自动采集信息；应用需要显式提交每个事件。发送采用尽力而为方式，不代表数据已入仓。 +将选定的产品使用事件与明确授权的 Session 日志发送到 OTLP/HTTP 采集端。Session 日志使用独立队列和按字节限制的请求。挂载插件不会自动采集数据。投递为尽力而为，不确认数据仓库入库。 ## 目录 @@ -48,6 +48,7 @@ kind: "package-reference" | `timeoutMillis` | `15000` | Exporter HTTP 发送与重试的超时时间 | | `exportTimeoutMillis` | `20000` | Processor 批次导出的超时时间 | | `shutdownTimeoutMillis` | `21000` | 退出时的等待上限；超时会提示可能丢失数据 | +| `sessionLog.maxRequestBytes` / `sessionLog.processor` | `4,000,000` / SDK 默认值 | 独立的 Session 日志请求上限和批处理配置；字节上限不能超过 4,000,000 | 默认接收地址将显式提交的事件发送到生产产品 collector，测试和自定义部署必须覆盖该地址。只向 collector 发送 `x-channel` 和 SDK 协议请求头，不继承宿主 OTel 请求头或客户端证书。 @@ -55,6 +56,10 @@ kind: "package-reference" 消费方注入 `productTelemetry`，调用 `emit()` 提交明确选定的分析字段。事件名称与字段含义由产品和数据分析负责人定义。插件不读取 Session、账号、凭证或设备标识。调用方必须排除提示词、回答、文件内容、凭证及其他未经批准的数据。 +`reportSessionLog(record)` 接收 Session id 和一条完整事件，写入 `eventName: "session-log"`、字符串 body，以及 `attributes.sessionId` 和 `attributes.content = JSON.stringify(event)`。可选的采集元数据保留在记录上。嵌套对象、数组、布尔值、null 与小数保留在 JSON 字符串内。导出的 `SessionLogReporter` 为负责反馈授权的后端提供同一管线，无需挂载产品服务。调用方必须先取得 Session 分享授权再提交。 + +Session 记录不会与产品事件共用一个请求。每个候选批次都使用 SDK 的 OTLP JSON 序列化器在 gzip 前计量，涵盖 resource、scope、attributes、UTF-8 和 JSON 转义。超限批次按事件顺序拆分；单条超限记录被拒绝并记录不含内容的诊断，其余记录继续发送。Session 记录禁用 SDK 属性长度截断。关闭时等待拆分请求和排队记录，受拥有方的外层截止时间约束。 + 接收服务要求 body 为字符串，attributes 的值为字符串、数字、布尔值或由这些标量组成的对象。调用方以毫秒提供发生时间；插件填写观测时间，默认严重程度为 INFO。无效传输配置在激活时报错。发送失败产生本地警告，事件提交不等待网络。 ----- @@ -65,7 +70,7 @@ kind: "package-reference"
实现细节——点击展开 -私有 OTel logger 将记录交给 `BatchLogRecordProcessor`，再由 SDK HTTP delegate 和 JSON 日志序列化器发送。delegate 使用显式请求头和 HTTP agent，只有共享的超时与压缩配置使用 SDK 环境变量解析。直接依赖的 `@opentelemetry/core` 与 `sdk-logs` 对齐为 2.9.0，使导出结果枚举共享同一个 TypeScript 类型身份。SDK 负责队列、临时错误重试与压缩；插件卸载时在限定时间内发送剩余记录。发送完成结果单独观测，因为 SDK 在发送被拒绝后仍可能正常完成退出。不安装全局 OTel provider。 +独立的 OTel provider 将产品事件与 Session 日志队列分开。两者使用 SDK JSON 序列化器和相同的显式 HTTP 传输配置。Session 导出器在传输前拆分超限请求；批处理节奏和重试仍由 SDK 负责。插件卸载在同一个外层截止时间内排空两个 provider，不安装全局 provider。 [`src/index.ts`](src/index.ts) 负责配置与提交。不发布运行时不变量伴随模块：本地没有独立的送达确认可与 SDK 队列比较。 @@ -99,7 +104,7 @@ kind: "package-reference" - 调用方拥有事件选择、渲染进程到宿主的传输以及获准使用的标识属性。 - 队列仅存在于内存；队列溢出、网络故障和进程退出可能丢失事件。没有持久发件箱或入仓确认。 -- SDK 按记录条数而非编码字节数分批。调用方必须确保记录不超过接收服务的 4 MB 限制，并选择接收服务适用的批次大小。 +- 产品事件按记录数量分批。Session 日志批次还遵守未压缩字节上限；单条事件拒绝、队列溢出和导出失败均不提供持久化重试保证。 - 不自动脱敏调用方选择的字符串。本包不决定产品告知或同意策略。 diff --git a/packages/host/product-telemetry-otel/package.json b/packages/host/product-telemetry-otel/package.json index 29c291020d..899bf7933e 100644 --- a/packages/host/product-telemetry-otel/package.json +++ b/packages/host/product-telemetry-otel/package.json @@ -37,12 +37,14 @@ "@opentelemetry/sdk-logs": "^0.220.0" }, "peerDependencies": { - "@deepseek-ai/cordis": "workspace:~" + "@deepseek-ai/cordis": "workspace:~", + "@deepseek-ai/dsh-session": "workspace:*" }, "devDependencies": { "@deepseek-ai/cordis": "workspace:~", "@deepseek-ai/cordis-plugin-loader": "workspace:~", "@deepseek-ai/dsh-app-boot": "workspace:*", - "@deepseek-ai/dsh-loader-smoke": "workspace:*" + "@deepseek-ai/dsh-loader-smoke": "workspace:*", + "@deepseek-ai/dsh-session": "workspace:*" } } diff --git a/packages/host/product-telemetry-otel/src/index.ts b/packages/host/product-telemetry-otel/src/index.ts index eef6ec5dce..e7a76b6579 100644 --- a/packages/host/product-telemetry-otel/src/index.ts +++ b/packages/host/product-telemetry-otel/src/index.ts @@ -3,12 +3,13 @@ import { Context, Service } from '@deepseek-ai/cordis' import z from '@deepseek-ai/schemastery' import { SeverityNumber, type Logger } from '@opentelemetry/api-logs' import { validateHeaderValue } from 'node:http' -import { JsonLogsSerializer } from '@opentelemetry/otlp-transformer' -import { createOtlpHttpExportDelegate, getSharedConfigurationFromEnvironment, httpAgentFactoryFromOptions } from '@opentelemetry/otlp-exporter-base/node-http' -import { CompressionAlgorithm, getSharedConfigurationDefaults, mergeOtlpSharedConfigurationWithDefaults, OTLPExporterBase } from '@opentelemetry/otlp-exporter-base' +import { CompressionAlgorithm } from '@opentelemetry/otlp-exporter-base' import { ExportResultCode } from '@opentelemetry/core' import { resourceFromAttributes } from '@opentelemetry/resources' import { BatchLogRecordProcessor, LoggerProvider } from '@opentelemetry/sdk-logs' +import { createLogExporter } from './transport.ts' +import { resolveSessionLogLimits, SessionLogReporter, type SessionLogOptions, type SessionLogRecord } from './session-log.ts' +export { SessionLogReporter, SESSION_LOG_MAX_REQUEST_BYTES, type SessionLogOptions, type SessionLogRecord } from './session-log.ts' declare module '@deepseek-ai/cordis' { interface Context { @@ -57,6 +58,8 @@ export interface Config { exportTimeoutMillis: number /** Outer shutdown wait; pending exports may be lost after this deadline. */ shutdownTimeoutMillis: number + /** Independent Session-log batching and byte limit; product-event settings do not apply. */ + sessionLog?: Pick } const positiveInteger = () => z.number().step(1).min(1).max(2_147_483_647) @@ -74,12 +77,17 @@ export const Config: z, Config> = z.object({ timeoutMillis: positiveInteger().default(15000), exportTimeoutMillis: positiveInteger().default(20000), shutdownTimeoutMillis: positiveInteger().default(21000), + sessionLog: z.object({ + maxRequestBytes: positiveInteger().max(4_000_000), + processor: z.any(), + }), }) /** Host analytics sender. Mounting alone sends nothing; the owning fiber drains it on unload. */ export default class ProductTelemetry extends Service { static Config = Config private readonly logger: Logger + private readonly sessionLogs: SessionLogReporter constructor(ctx: Context, config: Config) { let endpoint: URL @@ -99,21 +107,17 @@ export default class ProductTelemetry extends Service { if (config.maxExportBatchSize > config.maxQueueSize) { throw new Error('product-telemetry-otel: maxExportBatchSize must not exceed maxQueueSize') } + resolveSessionLogLimits(config.sessionLog ?? {}) super(ctx, 'productTelemetry') - const shared = mergeOtlpSharedConfigurationWithDefaults({ + const transport = { + url: config.endpoint, + headers: { 'x-channel': config.channel }, timeoutMillis: config.timeoutMillis, ...(config.compression === undefined ? {} : { compression: config.compression === 'gzip' ? CompressionAlgorithm.GZIP : CompressionAlgorithm.NONE, }), - }, getSharedConfigurationFromEnvironment('LOGS'), getSharedConfigurationDefaults()) - const transport = { - ...shared, - url: config.endpoint, - // This collector must not inherit another endpoint's headers or TLS client identity. - headers: () => Promise.resolve({ 'Content-Type': 'application/json', 'x-channel': config.channel }), - agentFactory: httpAgentFactoryFromOptions({ keepAlive: true }), } - const exporter = new OTLPExporterBase(createOtlpHttpExportDelegate(transport, JsonLogsSerializer)) + const exporter = createLogExporter(transport) const provider = new LoggerProvider({ resource: resourceFromAttributes({ 'service.name': config.serviceName, @@ -138,6 +142,12 @@ export default class ProductTelemetry extends Service { })], }) this.logger = provider.getLogger('@deepseek-ai/dsh-host-product-telemetry-otel') + this.sessionLogs = new SessionLogReporter({ + ...config.sessionLog, + exporter: transport, + resourceAttributes: { 'service.name': config.serviceName, 'service.version': config.serviceVersion }, + onFailure: (message, error) => { ctx.logger.warn(message, error) }, + }) ctx.effect(() => async () => { let timer!: ReturnType const deadline = new Promise((resolve) => { @@ -147,7 +157,7 @@ export default class ProductTelemetry extends Service { }, config.shutdownTimeoutMillis) }) try { - await Promise.race([provider.shutdown(), deadline]) + await Promise.race([Promise.all([provider.shutdown(), this.sessionLogs.shutdown()]), deadline]) } finally { clearTimeout(timer) } @@ -155,9 +165,17 @@ export default class ProductTelemetry extends Service { } /** - * Enqueue one selected product event without waiting for network delivery. + * Enqueue one explicitly authorized Session event in the separate byte-bounded queue. * Queue admission and shutdown completion are not collector or warehouse acknowledgements. - * @param record - caller-owned event containing only approved analytics fields. + * @param record - complete Session event and its owning Session id. + */ + reportSessionLog(record: SessionLogRecord): void { + this.sessionLogs.reportSessionLog(record) + } + + /** + * Enqueue a selected product event independently of Session-log reporting. + * @param record - approved analytics fields; no automatic Session collection. */ emit(record: ProductTelemetryRecord): void { const severityNumber = record.severityNumber ?? SeverityNumber.INFO diff --git a/packages/host/product-telemetry-otel/src/session-log.ts b/packages/host/product-telemetry-otel/src/session-log.ts new file mode 100644 index 0000000000..47085d1540 --- /dev/null +++ b/packages/host/product-telemetry-otel/src/session-log.ts @@ -0,0 +1,164 @@ +/** Session-log records in an independent SDK queue with byte-bounded OTLP requests. */ +import type { SessionEvent, SessionId } from '@deepseek-ai/dsh-session' +import type { Attributes } from '@opentelemetry/api' +import { SeverityNumber } from '@opentelemetry/api-logs' +import { ExportResultCode, type ExportResult } from '@opentelemetry/core' +import type { OTLPExporterNodeConfigBase } from '@opentelemetry/otlp-exporter-base' +import { JsonLogsSerializer } from '@opentelemetry/otlp-transformer' +import { resourceFromAttributes } from '@opentelemetry/resources' +import { BatchLogRecordProcessor, LoggerProvider, type BatchLogRecordProcessorOptions, type LogRecordExporter, type ReadableLogRecord } from '@opentelemetry/sdk-logs' +import { createLogExporter } from './transport.ts' + +/** Collector request ceiling in uncompressed UTF-8 bytes, including the OTLP envelope. */ +export const SESSION_LOG_MAX_REQUEST_BYTES = 4_000_000 + +/** One canonical event with its separately owned Session identity and redacted payload. */ +export interface SessionLogRecord { + sessionId: SessionId + /** Complete event envelope; data is the capture policy's exported copy. */ + event: Omit & { data: unknown } + /** Additional capture metadata; sessionId and content are always assigned by the reporter. */ + attributes?: Attributes + /** Omitted values use INFO. */ + severityNumber?: SeverityNumber +} + +/** Independent Session-log transport and queue settings. */ +export interface SessionLogOptions { + /** Explicit destination and SDK transport options. */ + exporter: OTLPExporterNodeConfigBase & { + /** Full HTTP(S) logs destination. */ + url: string + } + /** Session-only queue settings, independent of product-event aggregation. */ + processor?: Omit + /** May lower, but never exceed, the collector's 4,000,000-byte limit. */ + maxRequestBytes?: number + /** Application and anonymous identity carried on the OTLP resource. */ + resourceAttributes: Attributes + /** Report rejected single records and network failures without recording their content. */ + onFailure: (message: string, error?: Error) => void +} + +/** + * Validate byte and queue settings before constructing an SDK pipeline. + * @param options - Session-specific limits supplied by the owning composition. + * @returns the resolved collector request limit. + */ +export function resolveSessionLogLimits(options: Pick): number { + const limit = options.maxRequestBytes ?? SESSION_LOG_MAX_REQUEST_BYTES + if (!Number.isSafeInteger(limit) || limit < 1 || limit > SESSION_LOG_MAX_REQUEST_BYTES) { + throw new Error(`session log maxRequestBytes must be an integer between 1 and ${SESSION_LOG_MAX_REQUEST_BYTES}`) + } + for (const key of ['maxQueueSize', 'maxExportBatchSize', 'scheduledDelayMillis', 'exportTimeoutMillis'] as const) { + const value = options.processor?.[key] + if (value !== undefined && (!Number.isSafeInteger(value) || value < 1 || value > 2_147_483_647)) { + throw new Error(`session log processor.${key} must be a positive integer no greater than 2147483647`) + } + } + return limit +} + +/** Split SDK batches using the same JSON serializer as the HTTP exporter. */ +class ByteBoundedExporter implements LogRecordExporter { + private readonly pending = new Set>() + + constructor(private readonly exporter: LogRecordExporter, private readonly limit: number, private readonly warn: SessionLogOptions['onFailure']) {} + + export(records: ReadableLogRecord[], callback: (result: ExportResult) => void): void { + const pending = this.send(records) + this.pending.add(pending) + void pending.then((result) => { + this.pending.delete(pending) + callback(result) + }, (error: unknown) => { + this.pending.delete(pending) + callback({ code: ExportResultCode.FAILED, error: error instanceof Error ? error : new Error(String(error)) }) + }) + } + + private async send(records: ReadableLogRecord[]): Promise { + const serialized = JsonLogsSerializer.serializeRequest(records) + if (serialized === undefined) throw new Error('Session log serialization produced no request') + const bytes = serialized.byteLength + if (bytes <= this.limit) { + return new Promise((resolve) => { this.exporter.export(records, resolve) }) + } + if (records.length === 1) { + const error = new Error(`Session log record exceeds maxRequestBytes: ${bytes} > ${this.limit}`) + this.warn('Session log record rejected; content was not truncated', error) + return { code: ExportResultCode.FAILED, error } + } + const middle = Math.ceil(records.length / 2) + const first = await this.send(records.slice(0, middle)) + const second = await this.send(records.slice(middle)) + return first.code === ExportResultCode.FAILED ? first : second + } + + async forceFlush(): Promise { + await this.exporter.forceFlush() + await Promise.all(this.pending) + } + + async shutdown(): Promise { + await Promise.all(this.pending) + await this.exporter.shutdown() + } +} + +/** Owns a Session-only queue; callers must drain it during their bounded shutdown. */ +export class SessionLogReporter { + private readonly provider: LoggerProvider + private readonly logger: ReturnType + + /** + * @param options - transport, resource identity, queue limits, and failure observer. + */ + constructor(options: SessionLogOptions) { + const limit = resolveSessionLogLimits(options) + const exporter = new ByteBoundedExporter(createLogExporter(options.exporter), limit, options.onFailure) + this.provider = new LoggerProvider({ + // Session content must not be truncated by ambient SDK attribute limits. + logRecordLimits: { attributeValueLengthLimit: Infinity, attributeCountLimit: Infinity }, + resource: resourceFromAttributes(options.resourceAttributes), + processors: [new BatchLogRecordProcessor({ + ...options.processor, + exporter: { + export: (records, callback) => { + exporter.export(records, (result) => { + if (result.code !== ExportResultCode.SUCCESS) options.onFailure('Session log export failed', result.error) + callback(result) + }) + }, + forceFlush: () => exporter.forceFlush(), + shutdown: () => exporter.shutdown(), + }, + })], + }) + this.logger = this.provider.getLogger('@deepseek-ai/dsh-session-telemetry-otel') + } + + /** + * Enqueue one event as a string-valued session-log record without acknowledging delivery. + * Oversized records are rejected during export; other records continue independently. + * @param record - canonical event and the Session it belongs to. + */ + reportSessionLog(record: SessionLogRecord): void { + const severityNumber = record.severityNumber ?? SeverityNumber.INFO + this.logger.emit({ + eventName: 'session-log', + body: 'session-log', + timestamp: record.event.time, + observedTimestamp: record.event.time, + severityNumber, + severityText: SeverityNumber[severityNumber], + attributes: { ...record.attributes, sessionId: record.sessionId, content: JSON.stringify(record.event) }, + }) + } + + /** + * Drain accepted records and stop the SDK pipeline; the owner supplies an outer deadline. + * @returns resolves when SDK shutdown completes, without guaranteeing collector acceptance. + */ + shutdown(): Promise { return this.provider.shutdown() } +} diff --git a/packages/host/product-telemetry-otel/src/transport.ts b/packages/host/product-telemetry-otel/src/transport.ts new file mode 100644 index 0000000000..70034133d5 --- /dev/null +++ b/packages/host/product-telemetry-otel/src/transport.ts @@ -0,0 +1,26 @@ +/** Explicit collector transport shared by product and Session-log pipelines. */ +import { createOtlpHttpExportDelegate, getSharedConfigurationFromEnvironment, httpAgentFactoryFromOptions } from '@opentelemetry/otlp-exporter-base/node-http' +import { getSharedConfigurationDefaults, mergeOtlpSharedConfigurationWithDefaults, OTLPExporterBase, type OTLPExporterNodeConfigBase } from '@opentelemetry/otlp-exporter-base' +import { JsonLogsSerializer } from '@opentelemetry/otlp-transformer' +import type { LogRecordExporter } from '@opentelemetry/sdk-logs' + +/** + * Create an SDK JSON exporter without inheriting another collector's headers or TLS identity. + * @param options - explicit endpoint, headers, agent, and SDK transport settings. + * @returns the exporter owned by one independent log pipeline. + */ +export function createLogExporter(options: OTLPExporterNodeConfigBase & { url: string }): LogRecordExporter { + const shared = mergeOtlpSharedConfigurationWithDefaults(options, getSharedConfigurationFromEnvironment('LOGS'), getSharedConfigurationDefaults()) + const transport = { + ...shared, + url: options.url, + headers: async () => ({ + 'Content-Type': 'application/json', 'x-channel': 'dsh_otel_report', + ...typeof options.headers === 'function' ? await options.headers() : options.headers, + }), + agentFactory: typeof options.httpAgentOptions === 'function' ? options.httpAgentOptions + : httpAgentFactoryFromOptions({ keepAlive: options.keepAlive ?? true, ...options.httpAgentOptions }), + ...(options.userAgent === undefined ? {} : { userAgent: options.userAgent }), + } + return new OTLPExporterBase(createOtlpHttpExportDelegate(transport, JsonLogsSerializer)) +} diff --git a/packages/host/product-telemetry-otel/tests/telemetry.spec.ts b/packages/host/product-telemetry-otel/tests/telemetry.spec.ts index 22f7df5c30..7521acba1f 100644 --- a/packages/host/product-telemetry-otel/tests/telemetry.spec.ts +++ b/packages/host/product-telemetry-otel/tests/telemetry.spec.ts @@ -1,13 +1,16 @@ -import { createServer, type IncomingHttpHeaders } from 'node:http' +import { Agent, createServer, type IncomingHttpHeaders } from 'node:http' import { once } from 'node:events' import { gunzipSync } from 'node:zlib' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { Context } from '@deepseek-ai/cordis' import { LoggerProvider } from '@opentelemetry/sdk-logs' import { SeverityNumber } from '@opentelemetry/api-logs' -import ProductTelemetry, { Config } from '../src/index.ts' +import { JsonLogsSerializer } from '@opentelemetry/otlp-transformer' +import { SessionId, SessionSeq } from '@deepseek-ai/dsh-session' +import ProductTelemetry, { Config, SESSION_LOG_MAX_REQUEST_BYTES, SessionLogReporter, type SessionLogRecord } from '../src/index.ts' interface Capture { + bytes: number headers: IncomingHttpHeaders body: { resourceLogs: { resource: unknown; scopeLogs: { logRecords: Record[] }[] }[] } } @@ -27,7 +30,7 @@ beforeEach(() => { vi.stubEnv('OTEL_EXPORTER_OTLP_LOGS_COMPRESSION', undefined) }) -async function collector(statuses = [200]) { +async function collector(statuses = [200], beforeRespond?: () => Promise) { const captures: Capture[] = [] const server = createServer((req, res) => { const chunks: Buffer[] = [] @@ -35,8 +38,11 @@ async function collector(statuses = [200]) { req.on('end', () => { const raw = Buffer.concat(chunks) const bytes = req.headers['content-encoding'] === 'gzip' ? gunzipSync(raw) : raw - captures.push({ headers: req.headers, body: JSON.parse(bytes.toString()) as Capture['body'] }) - res.writeHead(statuses.shift() ?? 200, { 'content-type': 'application/json' }).end('{}') + captures.push({ bytes: bytes.length, headers: req.headers, body: JSON.parse(bytes.toString()) as Capture['body'] }) + void (async () => { + await beforeRespond?.() + res.writeHead(statuses.shift() ?? 200, { 'content-type': 'application/json' }).end('{}') + })() }) }) cleanup.push(async () => { @@ -201,3 +207,206 @@ describe('explicit product telemetry', () => { await vi.advanceTimersByTimeAsync(0) }) }) + + +function sessionRecord(text: string, seq = 0): SessionLogRecord { + return { + sessionId: SessionId('synthetic-session'), + event: { type: 'user/message', seq: SessionSeq(seq), time: 1_800_000_000_000, surfaceOp: 'append', + data: { content: [{ type: 'text', text }], nested: { items: [null, true, false, 0, 1.5, { text }] } } }, + } +} + +function logs(capture: Capture) { + return capture.body.resourceLogs.flatMap(resource => resource.scopeLogs.flatMap(scope => scope.logRecords)) +} + +function contents(captures: Capture[]): string[] { + return captures.flatMap(capture => logs(capture).map((record) => { + const attributes = record['attributes'] as { key: string; value: { stringValue: string } }[] + return attributes.find(attribute => attribute.key === 'content')!.value.stringValue + })) +} + +function parseContent(content: string): unknown { + return JSON.parse(content) +} + +describe('Session-log reporting', () => { + it('keeps product events in separate requests and preserves the complete nested event', async () => { + vi.stubEnv('OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT', '8') + vi.stubEnv('OTEL_EXPORTER_OTLP_HEADERS', 'Authorization=Bearer%20unrelated') + const target = await collector() + const ctx = context() + const fiber = await ctx.plugin(ProductTelemetry, config(target.endpoint)) + const record = sessionRecord('中文\n"quoted"\\path 😀') + ctx.productTelemetry.emit(event) + ctx.productTelemetry.reportSessionLog(record) + await fiber.dispose() + expect(target.captures).toHaveLength(2) + const sessionCapture = target.captures.find(capture => logs(capture)[0]?.['eventName'] === 'session-log')! + expect(logs(sessionCapture)).toHaveLength(1) + expect(logs(sessionCapture)[0]).toMatchObject({ + eventName: 'session-log', body: { stringValue: 'session-log' }, + attributes: [ + { key: 'sessionId', value: { stringValue: record.sessionId } }, + { key: 'content', value: { stringValue: JSON.stringify(record.event) } }, + ], + }) + expect(JSON.parse(contents([sessionCapture])[0]!)).toEqual(record.event) + expect(sessionCapture.headers).not.toHaveProperty('authorization') + }) + + it.each(['gzip', 'none'] as const)('splits real 4 MB batches before %s compression without losing event order', async (compression) => { + const target = await collector() + const ctx = context() + const fiber = await ctx.plugin(ProductTelemetry, config(target.endpoint, { compression })) + const records = [0, 1, 2].map(seq => sessionRecord('中"\\'.repeat(100_000), seq)) + for (const record of records) ctx.productTelemetry.reportSessionLog(record) + await fiber.dispose() + expect(target.captures.length).toBeGreaterThan(1) + expect(target.captures.every(capture => capture.bytes <= SESSION_LOG_MAX_REQUEST_BYTES)).toBe(true) + expect(contents(target.captures).map(parseContent)).toEqual(records.map(record => record.event)) + }) + + it('admits an exact byte-limit request and rejects a single event one byte above it', async () => { + const target = await collector() + const ctx = context() + const record = sessionRecord('边界"\\') + const baseline = await ctx.plugin(ProductTelemetry, config(target.endpoint)) + ctx.productTelemetry.reportSessionLog(record) + await baseline.dispose() + const limit = target.captures[0]!.bytes + const exact = await ctx.plugin(ProductTelemetry, config(target.endpoint, { sessionLog: { maxRequestBytes: limit } })) + ctx.productTelemetry.reportSessionLog(record) + await exact.dispose() + expect(target.captures).toHaveLength(2) + expect(target.captures[1]!.bytes).toBe(limit) + const warn = vi.spyOn(ctx.logger, 'warn') + const tooSmall = await ctx.plugin(ProductTelemetry, config(target.endpoint, { sessionLog: { maxRequestBytes: limit - 1 } })) + ctx.productTelemetry.reportSessionLog(record) + await tooSmall.dispose() + expect(target.captures).toHaveLength(2) + expect(warn).toHaveBeenCalledWith('Session log record rejected; content was not truncated', expect.any(Error)) + }) + + it('rejects an oversized event while still sending the events before and after it', async () => { + const target = await collector() + const ctx = context() + const warn = vi.spyOn(ctx.logger, 'warn') + const fiber = await ctx.plugin(ProductTelemetry, config(target.endpoint)) + const records = [sessionRecord('before'), sessionRecord('x'.repeat(SESSION_LOG_MAX_REQUEST_BYTES), 1), sessionRecord('after', 2)] + for (const record of records) ctx.productTelemetry.reportSessionLog(record) + await fiber.dispose() + expect(contents(target.captures).map(parseContent)).toEqual([records[0]!.event, records[2]!.event]) + expect(warn).toHaveBeenCalledWith('Session log record rejected; content was not truncated', expect.any(Error)) + expect(target.captures.every(capture => capture.bytes <= SESSION_LOG_MAX_REQUEST_BYTES)).toBe(true) + }) + + it('continues later byte batches after a permanent rejection', async () => { + const target = await collector([400, 200]) + const ctx = context() + const warn = vi.spyOn(ctx.logger, 'warn') + const fiber = await ctx.plugin(ProductTelemetry, config(target.endpoint, { sessionLog: { maxRequestBytes: 1300 } })) + ctx.productTelemetry.reportSessionLog(sessionRecord('first'.repeat(25))) + ctx.productTelemetry.reportSessionLog(sessionRecord('second'.repeat(25), 1)) + await fiber.dispose() + expect(target.captures).toHaveLength(2) + expect(warn).toHaveBeenCalledWith('Session log export failed', expect.any(Error)) + }) + + it.each([0, -1, 1.5, 4_000_001, Infinity])('refuses an invalid request limit %s', (maxRequestBytes) => { + expect(() => new SessionLogReporter({ + exporter: { url: 'http://collector.test/v1/logs' }, resourceAttributes: {}, maxRequestBytes, onFailure: vi.fn(), + })).toThrow('maxRequestBytes') + }) +}) + + +it('honors explicit asynchronous headers and agent factories for Session logs', async () => { + const target = await collector() + const agent = new Agent({ keepAlive: false }) + cleanup.push(async () => { agent.destroy() }) + const failures = vi.fn() + const reporter = new SessionLogReporter({ + exporter: { url: target.endpoint, userAgent: 'session-test', headers: async () => ({ 'x-channel': 'test-channel' }), httpAgentOptions: async () => agent }, + resourceAttributes: {}, onFailure: failures, + }) + cleanup.push(() => reporter.shutdown()) + reporter.reportSessionLog(sessionRecord('explicit headers')) + await reporter.shutdown() + expect(failures).not.toHaveBeenCalled() + expect(target.captures[0]!.headers['x-channel']).toBe('test-channel') + expect(target.captures[0]!.headers['user-agent']).toContain('session-test') +}) + +it.each([new Error('serializer failed'), 'serializer failed', undefined])('reports serialization failure %s without sending a partial request', async (failure) => { + const target = await collector() + const failures = vi.fn() + const reporter = new SessionLogReporter({ + exporter: { url: target.endpoint, keepAlive: false }, resourceAttributes: {}, onFailure: failures, + }) + cleanup.push(() => reporter.shutdown()) + reporter.reportSessionLog(sessionRecord('not exported')) + vi.spyOn(JsonLogsSerializer, 'serializeRequest').mockImplementationOnce(() => { + if (failure === undefined) return undefined + throw failure + }) + await reporter.shutdown() + expect(target.captures).toEqual([]) + expect(failures).toHaveBeenCalledWith('Session log export failed', expect.any(Error)) +}) + + +it.each([ + ['maxQueueSize', 0], ['maxExportBatchSize', 0], ['scheduledDelayMillis', 0], ['exportTimeoutMillis', 0], + ['maxQueueSize', 1.5], ['exportTimeoutMillis', 2_147_483_648], +] as const)( + 'rejects invalid Session queue %s=%s before registering a service', (key, value) => { + const ctx = context() + expect(() => new ProductTelemetry(ctx, config('http://collector.test/v1/logs', { + sessionLog: { processor: { [key]: value } }, + }))).toThrow(`processor.${key}`) + expect(ctx.get('productTelemetry')).toBeUndefined() + }, +) + + +it('drains every split request when disposal starts during the first request', async () => { + const received = Promise.withResolvers() + const release = Promise.withResolvers() + const target = await collector([200], async () => { + received.resolve(undefined) + await release.promise + }) + const ctx = context() + cleanup.push(async () => { release.resolve(undefined) }) + const fiber = await ctx.plugin(ProductTelemetry, config(target.endpoint, { + sessionLog: { maxRequestBytes: 1300, processor: { maxExportBatchSize: 3, scheduledDelayMillis: 60_000 } }, + })) + const service = ctx.productTelemetry + const records = [0, 1, 2].map(seq => sessionRecord('content'.repeat(20), seq)) + for (const record of records) service.reportSessionLog(record) + await received.promise + let disposed = false + const disposal = fiber.dispose().then(() => { disposed = true }) + await Promise.resolve() + expect(disposed).toBe(false) + release.resolve(undefined) + await disposal + expect(target.captures).toHaveLength(3) + expect(contents(target.captures).map(parseContent)).toEqual(records.map(record => record.event)) + service.reportSessionLog(sessionRecord('after shutdown')) + expect(target.captures).toHaveLength(3) +}) + + +it('constructs and disposes the product service without Session-specific overrides', async () => { + const target = await collector() + const ctx = context() + const { sessionLog: _sessionLog, ...options } = config(target.endpoint) + const service = new ProductTelemetry(ctx, options) + service.reportSessionLog(sessionRecord('default Session queue')) + await ctx.fiber.dispose() + expect(target.captures).toHaveLength(1) +}) diff --git a/packages/host/product-telemetry-otel/tsconfig.json b/packages/host/product-telemetry-otel/tsconfig.json index f0063ae28d..a710d8842b 100644 --- a/packages/host/product-telemetry-otel/tsconfig.json +++ b/packages/host/product-telemetry-otel/tsconfig.json @@ -16,6 +16,9 @@ }, { "path": "../../../vendor/schemastery" + }, + { + "path": "../../core/session" } ] } diff --git a/packages/session/session-telemetry-otel/README.i18n.yaml b/packages/session/session-telemetry-otel/README.i18n.yaml index b89f0da8de..320b9cd0b8 100644 --- a/packages/session/session-telemetry-otel/README.i18n.yaml +++ b/packages/session/session-telemetry-otel/README.i18n.yaml @@ -21,11 +21,11 @@ en: bc7e598148a7f8c3 zh: 0209c6aa258ec134 /deepseek-ai-dsh-session-telemetry-otel/use-this-package/minimal-configuration: - en: 3de0d5db3156a7e4 - zh: 71199133cd9998c4 + en: afe9c8926a51303d + zh: a6cbf3a262484edf /deepseek-ai-dsh-session-telemetry-otel/use-this-package/what-leaves-the-machine: - en: b94ca8299ad60c5d - zh: 3791131f3440e3b3 + en: d3008613a4c67cdb + zh: 4ff966d8ef8ea8d6 /deepseek-ai-dsh-session-telemetry-otel/use-this-package/failures-and-shutdown: en: 20c6e78df8ad2c2a zh: fd6a00f98c4cc05c @@ -42,8 +42,8 @@ en: 9bdc01d0fa130b3a zh: 63bf30c4cc983ca0 /deepseek-ai-dsh-session-telemetry-otel/understand-the-implementation/field-mapping: - en: 38590f3afee40bb8 - zh: b2fc1f4341008d79 + en: a42e9e081fb93bee + zh: b8e817c2455278b9 /deepseek-ai-dsh-session-telemetry-otel/further-exploration: en: 155b75490ec65d86 zh: 07d37d6e4ac90c89 diff --git a/packages/session/session-telemetry-otel/README.md b/packages/session/session-telemetry-otel/README.md index 35dc351bd5..1f8a4734bd 100644 --- a/packages/session/session-telemetry-otel/README.md +++ b/packages/session/session-telemetry-otel/README.md @@ -38,7 +38,7 @@ Programmatic TypeScript configuration uses the exported `SessionTelemetryMode` e ### Minimal configuration -Uploading modes require an exporter URL and accept the SDK option blocks verbatim: +Uploading modes require an exporter URL. Processor options configure the Session-only SDK queue; the shared reporter adds collector routing and byte-bounded export. ```yaml - id: sessionTelemetry-otel @@ -46,7 +46,7 @@ Uploading modes require an exporter URL and accept the SDK option blocks verbati config: mode: FEEDBACK_ONLY # optional; defaults to FEEDBACK_ONLY shutdownTimeoutMillis: 3000 # optional; defaults to 3000 - exporter: # passed verbatim to the SDK's OTLP/HTTP log exporter + exporter: # explicit SDK transport settings url: https://collector.example.com/v1/logs headers: authorization: !!js `Bearer ${process.env.OTLP_TOKEN}` @@ -57,8 +57,9 @@ Uploading modes require an exporter URL and accept the SDK option blocks verbati |---|---|---| | `mode` | `FEEDBACK_ONLY` | Sharing policy: `FEEDBACK_ONLY` or `DISABLED` | | `exporter.url` | required in uploading modes | Full OTLP logs endpoint; must parse as `http(s)` | -| `exporter`, `processor` | — | Passed verbatim to the SDK exporter and batch processor | +| `exporter`, `processor` | — | Explicit SDK transport and Session-only batch settings; headers/TLS identity are not inherited from the environment | | `shutdownTimeoutMillis` | `3,000` | Outer deadline for the SDK's complete shutdown sequence | +| `maxRequestBytes` | `4,000,000` | Maximum complete OTLP JSON request bytes before gzip; may only be lowered | Direct `ctx.sessionTelemetry.emit()` calls are no-ops in every mode and cannot bypass feedback authorization. Inherited parent feedback does not authorize a child export: the child needs new feedback of its own. Its authorized prefix then includes inherited context. @@ -66,7 +67,9 @@ Model requests, request headers, Session creation or adoption, restoration, and ### What leaves the machine -In uploading modes, records carry the complete `event.data` as the seam's `sessionTelemetry/record` waterfall returns it — message content, tool arguments and results, the system prompt and tool schemas, todo text, compaction summaries, feedback text, and the session `cwd`. Provider credentials never appear: adapter API keys are constructor parameters, not session events, so they are structurally absent from the log and therefore from telemetry. `DISABLED` constructs no SDK pipeline and hands no capture to a backend. +Each Session event becomes one `eventName: "session-log"` record. `attributes.sessionId` identifies its Session; `attributes.content` is the JSON string of the complete event envelope and the redacted `event.data`. The resource carries application and anonymous-user identity. Contents can include messages, tool inputs/results, prompts, schemas, feedback, and paths. `DISABLED` constructs no transport. The base profile sends to `https://dsh-otel-collector.deepseeksvc.com/v1/logs` with `x-channel: dsh_otel_report`; `DSH_TELEMETRY_OTLP_URL` can override the URL. + +The [shared Session-log reporter](../../host/product-telemetry-otel/README.md) measures complete uncompressed requests, splits oversized batches, and rejects single events that exceed the limit without truncating them. Session logs never mix with product analytics in a request. Rejection and network failures produce local diagnostics; neither handoff nor shutdown confirms collector acceptance. ### Failures and shutdown @@ -98,7 +101,7 @@ The backend uses on-demand capture with stored history included. Only new own `f ### Field mapping -Each telemetry record maps to one SDK log record with captured timestamp, severity, body, and attributes. Feedback authorizes the complete unhanded prefix, not only the feedback payload. +Each capture record supplies a separately copied event envelope and redacted payload to `SessionLogReporter`; serialization retains optional surface metadata and the event sequence/time. Feedback authorizes the complete unhanded prefix, not only the feedback payload.
diff --git a/packages/session/session-telemetry-otel/README.zh.md b/packages/session/session-telemetry-otel/README.zh.md index c414e6461a..0fe5b0e1c9 100644 --- a/packages/session/session-telemetry-otel/README.zh.md +++ b/packages/session/session-telemetry-otel/README.zh.md @@ -38,7 +38,7 @@ kind: "package-reference" ### 最小配置 -上传模式需要导出器 URL，并原样接受 SDK 选项块： +上传模式要求提供导出器 URL。processor 选项配置 Session 专用 SDK 队列；共享上报器负责采集端路由和按字节限制的导出。 ```yaml - id: sessionTelemetry-otel @@ -46,7 +46,7 @@ kind: "package-reference" config: mode: FEEDBACK_ONLY # optional; defaults to FEEDBACK_ONLY shutdownTimeoutMillis: 3000 # optional; defaults to 3000 - exporter: # passed verbatim to the SDK's OTLP/HTTP log exporter + exporter: # explicit SDK transport settings url: https://collector.example.com/v1/logs headers: authorization: !!js `Bearer ${process.env.OTLP_TOKEN}` @@ -57,8 +57,9 @@ kind: "package-reference" |---|---|---| | `mode` | `FEEDBACK_ONLY` | 共享策略：`FEEDBACK_ONLY` 或 `DISABLED` | | `exporter.url` | 上传模式必填 | 完整 OTLP 日志端点；必须能解析为 `http(s)` | -| `exporter`、`processor` | — | 原样传给 SDK 导出器与批处理器 | +| `exporter`, `processor` | — | 显式 SDK 传输和 Session 专用批处理配置；不从环境继承请求头或 TLS 身份 | | `shutdownTimeoutMillis` | `3,000` | SDK 完整关闭序列的外层截止时间 | +| `maxRequestBytes` | `4,000,000` | gzip 前完整 OTLP JSON 请求的最大字节数；只能调低 | 直接调用 `ctx.sessionTelemetry.emit()` 在任何模式下都是空操作，不能绕过反馈授权。继承的父会话反馈不授权子会话导出：子会话需要新的自身反馈。授权后的前缀包含继承的上下文。 @@ -66,7 +67,9 @@ kind: "package-reference" ### 哪些数据会离开本机 -在上传模式中，记录携带 seam 的 `sessionTelemetry/record` waterfall（瀑布式事件）返回的完整 `event.data`——消息内容、工具参数与结果、系统提示词与工具 schema、todo 文本、压缩（compaction）摘要、反馈文本，以及会话 `cwd`。提供方凭据绝不会出现：适配器的 API key 是构造函数参数而非会话事件，因此它们在结构上就不存在于日志中，也就不存在于遥测中。`DISABLED` 不构造 SDK 流水线，也不把任何捕获内容交给后端。 +每条 Session 事件对应一条 `eventName: "session-log"` 记录。`attributes.sessionId` 标识所属 Session；`attributes.content` 是完整事件信封与脱敏后的 `event.data` 的 JSON 字符串。resource 携带应用与匿名用户身份。内容可包含消息、工具输入和结果、提示词、schema、反馈及路径。`DISABLED` 不构造传输。基础 profile 使用 `x-channel: dsh_otel_report` 向 `https://dsh-otel-collector.deepseeksvc.com/v1/logs` 发送；`DSH_TELEMETRY_OTLP_URL` 可覆盖 URL。 + +[共享 Session 日志上报器](../../host/product-telemetry-otel/README.zh.md) 计量完整未压缩请求，拆分超限批次，并拒绝超限单条事件而不截断内容。Session 日志不会与产品埋点混入同一个请求。拒绝和网络失败会产生本地诊断；交接和关闭都不确认采集端接受。 ### 失败与关闭 @@ -98,7 +101,7 @@ kind: "package-reference" ### 字段映射 -每条遥测记录映射为一条 SDK 日志记录，携带捕获的时间戳、严重级别、正文和属性。反馈授权的是尚未交接的完整前缀，而非只有反馈载荷。 +每条采集记录向 `SessionLogReporter` 提供单独复制的事件信封和脱敏后的载荷；序列化保留可选的呈现元数据及事件序号和时间。反馈授权整个尚未交接的前缀，而不只是反馈载荷。
diff --git a/packages/session/session-telemetry-otel/package.json b/packages/session/session-telemetry-otel/package.json index c1ae23c564..8ac62346dc 100644 --- a/packages/session/session-telemetry-otel/package.json +++ b/packages/session/session-telemetry-otel/package.json @@ -29,7 +29,6 @@ "dependencies": { "@opentelemetry/api": "^1.9.1", "@opentelemetry/api-logs": "^0.220.0", - "@opentelemetry/exporter-logs-otlp-http": "^0.220.0", "@opentelemetry/otlp-exporter-base": "^0.220.0", "@opentelemetry/resources": "^2.9.0", "@opentelemetry/sdk-logs": "^0.220.0", @@ -42,7 +41,8 @@ "@deepseek-ai/dsh-session": "workspace:*", "@deepseek-ai/dsh-session-telemetry": "workspace:*", "@deepseek-ai/dsh-anonymous-user-id": "workspace:*", - "@deepseek-ai/cordis": "workspace:~" + "@deepseek-ai/cordis": "workspace:~", + "@deepseek-ai/dsh-host-product-telemetry-otel": "workspace:*" }, "devDependencies": { "@deepseek-ai/cordis-plugin-logger-console": "workspace:~", @@ -60,6 +60,7 @@ "@deepseek-ai/dsh-subprocess-local": "workspace:*", "@deepseek-ai/dsh-anonymous-user-id": "workspace:*", "@deepseek-ai/cordis": "workspace:~", - "@deepseek-ai/dsh-http-proxy": "workspace:*" + "@deepseek-ai/dsh-http-proxy": "workspace:*", + "@deepseek-ai/dsh-host-product-telemetry-otel": "workspace:*" } } diff --git a/packages/session/session-telemetry-otel/src/index.ts b/packages/session/session-telemetry-otel/src/index.ts index 48792dbe67..764875b1c2 100644 --- a/packages/session/session-telemetry-otel/src/index.ts +++ b/packages/session/session-telemetry-otel/src/index.ts @@ -1,18 +1,13 @@ /** * OpenTelemetry Service Provider for the DeepSeek Harness telemetry capability. * - * Composes the OTel JS SDK as-is — a `LoggerProvider` with a - * `BatchLogRecordProcessor` and an OTLP/HTTP log exporter — and maps each - * record handed over by the capture coordinator onto `logger.emit()`. After that call, - * batching, retry, queueing, and loss policy use the SDK's documented behavior, configured - * verbatim through the `exporter`/`processor` passthroughs. This package owns - * capture mode and an outer shutdown deadline: the SDK's export timeout does - * not bound its preceding `forceFlush()` wait. + * Authorizes feedback-bounded capture and hands complete event strings to the + * shared SessionLogReporter. This plugin owns resource identity and an outer + * shutdown deadline; the reporter owns byte-bounded SDK delivery. * * @module @deepseek-ai/dsh-session-telemetry-otel */ -import { createRequire } from 'node:module' import z from '@deepseek-ai/schemastery' import type { Context } from '@deepseek-ai/cordis' import type {} from '@deepseek-ai/dsh-command-feedback' @@ -28,19 +23,10 @@ import { } from '@deepseek-ai/dsh-session-telemetry' import { APP_IDENTITY } from '@deepseek-ai/dsh-llm' import { getOrCreateAnonymousUserId } from '@deepseek-ai/dsh-anonymous-user-id' -import { - BatchLogRecordProcessor, - LoggerProvider, - type BatchLogRecordProcessorOptions, -} from '@opentelemetry/sdk-logs' -import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http' +import { SessionLogReporter } from '@deepseek-ai/dsh-host-product-telemetry-otel' +import type { BatchLogRecordProcessorOptions } from '@opentelemetry/sdk-logs' import type { OTLPExporterNodeConfigBase } from '@opentelemetry/otlp-exporter-base' -import { SeverityNumber, type AnyValue } from '@opentelemetry/api-logs' -import { resourceFromAttributes } from '@opentelemetry/resources' - -// The package's own manifest is the single source of the instrumentation-scope -// version (same pattern as dsh-llm's attribution identity). -const { version } = createRequire(import.meta.url)('../package.json') as { version: string } +import { SeverityNumber } from '@opentelemetry/api-logs' /** Session-sharing policy selected by {@link Config.mode}. */ export enum SessionTelemetryMode { @@ -101,10 +87,8 @@ export interface Config { /** Defaults to `FEEDBACK_ONLY`: capture session history only when feedback is explicitly submitted. */ mode?: SessionTelemetryMode /** - * Passed verbatim to the SDK's OTLP/HTTP log exporter — the complete - * `OTLPExporterNodeConfigBase` shape (`headers`, `timeoutMillis`, - * `compression`, `keepAlive`, …), owned and documented by the SDK. `url` - * is the one field this package requires and validates itself. + * Explicit SDK HTTP transport settings. The reporter adds x-channel routing + * and does not inherit ambient credentials. URL is required while uploading. */ exporter?: OTLPExporterNodeConfigBase & { /** Full logs endpoint (e.g. `https://collector.example.com/v1/logs`). Required outside `DISABLED`; validated at load. */ @@ -117,20 +101,22 @@ export interface Config { processor?: Omit /** Maximum time spent awaiting the SDK provider's complete shutdown path. */ shutdownTimeoutMillis?: number + /** Uncompressed OTLP request byte limit, at most 4,000,000. */ + maxRequestBytes?: number } /** * Schemastery validator for {@link Config}; cordis runs it before the plugin - * starts. It checks only the top-level fields; value checks live in the constructor - * so their errors name the fields. Both SDK option objects pass through unchanged: - * the SDK defines and validates their fields. Re-declaring them here would - * silently drop every field this plugin did not repeat. + * starts. The constructor validates endpoint and shutdown requirements; the + * shared reporter validates Session byte and queue limits. SDK transport and + * processor settings retain their upstream types. */ export const Config: z = z.object({ mode: z.union(Object.values(SessionTelemetryMode)).default(DEFAULT_TELEMETRY_MODE), exporter: z.any(), processor: z.any(), shutdownTimeoutMillis: z.number(), + maxRequestBytes: z.number().step(1).min(1).max(4_000_000), }) /** Default outer allowance for the SDK's complete shutdown sequence. */ @@ -157,7 +143,7 @@ export class OpenTelemetrySessionBackend extends SessionTelemetryBackend { static inject = ['sessions'] static Config = Config - private readonly provider: LoggerProvider | undefined + private readonly provider: SessionLogReporter | undefined private readonly shutdownTimeoutMillis: number override readonly sharing: SessionTelemetrySharingStatus @@ -201,37 +187,24 @@ export class OpenTelemetrySessionBackend extends SessionTelemetryBackend { throw new Error(`session-telemetry-otel: shutdownTimeoutMillis must be a positive finite number no greater than ${MAX_TIMER_DELAY_MILLIS}, got ${String(shutdownTimeoutMillis)}`) } this.shutdownTimeoutMillis = shutdownTimeoutMillis - this.provider = new LoggerProvider({ - resource: resourceFromAttributes({ + const reporter = new SessionLogReporter({ + exporter: { ...config.exporter, url }, + ...(config.processor === undefined ? {} : { processor: config.processor }), + ...(config.maxRequestBytes === undefined ? {} : { maxRequestBytes: config.maxRequestBytes }), + resourceAttributes: { 'service.name': APP_IDENTITY.product, 'service.version': APP_IDENTITY.version, - // OTel semconv's standard user attribute, carried once per export - // batch on the Resource rather than per record: the collector - // aggregates by Resource, and the id is process-stable anyway. 'user.id': getOrCreateAnonymousUserId(), - }), - processors: [ - new BatchLogRecordProcessor({ - ...config.processor, - // The complete validated exporter object, verbatim: every SDK - // option (`timeoutMillis`, `compression`, `keepAlive`, …) reaches - // the exporter — rebuilding selected fields here would silently - // ignore the rest. App identity travels in the Resource - // (service.name/version); the transport-level user-agent is the - // SDK's own, per the axiom. - exporter: new OTLPLogExporter(config.exporter), - }), - ], + }, + onFailure: (message, error) => { ctx.logger.warn(message, error) }, }) - const ledger = this.provider.getLogger('@deepseek-ai/dsh-session-telemetry-otel', version) + this.provider = reporter const enqueue: SessionTelemetrySink['emit'] = (record) => { - ledger.emit({ - timestamp: record.time, - observedTimestamp: record.time, - ...SEVERITY[record.severity], - // JSON-serializable by the seam's contract (validated at Session.append), - // which is exactly the AnyValue subset. - body: record.body as AnyValue, + if (record.sourceEvent === undefined) return + reporter.reportSessionLog({ + sessionId: record.sourceEvent.sessionId, + event: { ...record.sourceEvent.envelope, data: record.body }, + severityNumber: SEVERITY[record.severity].severityNumber, attributes: record.attributes, }) } diff --git a/packages/session/session-telemetry-otel/tests/loader-composition.e2e.ts b/packages/session/session-telemetry-otel/tests/loader-composition.e2e.ts index af28e1e892..3d0d01e64d 100644 --- a/packages/session/session-telemetry-otel/tests/loader-composition.e2e.ts +++ b/packages/session/session-telemetry-otel/tests/loader-composition.e2e.ts @@ -4,6 +4,7 @@ * only the redacted authorized prefix; the canonical log keeps every event. */ +import type { SessionEvent } from '@deepseek-ai/dsh-session' import { readFile, readdir } from 'node:fs/promises' import { join } from 'node:path' import { fileURLToPath } from 'node:url' @@ -24,6 +25,7 @@ const FIXTURE_SECRET = 'sk-e2efixture1234567890' const FIXTURE_PLACEHOLDER = '[E2E-REDACTED]' interface OtlpLogRecord { + eventName: string attributes?: { key: string; value: Record }[] body?: unknown } @@ -102,6 +104,17 @@ describe('session-telemetry-otel through the production headless profile', () => const records = allRecords(output.captures) expect(records.length).toBeGreaterThan(0) + for (const { record } of records) { + expect(record.eventName).toBe('session-log') + expect(record.attributes?.find(attribute => attribute.key === 'sessionId')?.value.stringValue).toBeTypeOf('string') + const content = record.attributes?.find(attribute => attribute.key === 'content')?.value.stringValue + expect(typeof content).toBe('string') + const event = JSON.parse(content as string) as SessionEvent + expect(event.type).toBeTypeOf('string') + expect(event.seq).toBeTypeOf('number') + expect(event.time).toBeTypeOf('number') + expect(event).toHaveProperty('data') + } const types = eventTypes(output.captures) for (const expected of ['turn/start', 'user/message', 'tool/call', 'tool/result', 'assistant/message', 'turn/end']) { diff --git a/packages/session/session-telemetry-otel/tests/otel.spec.ts b/packages/session/session-telemetry-otel/tests/otel.spec.ts index 72a8e5ef10..b3969e6c57 100644 --- a/packages/session/session-telemetry-otel/tests/otel.spec.ts +++ b/packages/session/session-telemetry-otel/tests/otel.spec.ts @@ -187,58 +187,13 @@ describe('OpenTelemetrySessionBackend wire', () => { expect(end?.record.severityText).toBe('ERROR') const assistant = ledger.find(r => r.record.attributes?.some(a => a.key === 'event.type' && a.value.stringValue === 'assistant/message')) - const body = assistant?.record.body as { - kvlistValue: { values: { key: string; value: unknown }[] } + const content = assistant?.record.attributes?.find(a => a.key === 'content')?.value.stringValue + expect(typeof content).toBe('string') + expect(JSON.parse(content as string)).toEqual(session.snapshotEvents().find(event => event.type === 'assistant/message')) + for (const { record } of ledger) { + expect(record).toMatchObject({ eventName: 'session-log', body: { stringValue: 'session-log' } }) + expect(record.attributes).toContainEqual({ key: 'sessionId', value: { stringValue: session.id } }) } - expect(body.kvlistValue.values.find(value => value.key === 'stream')?.value).toEqual({ - arrayValue: { - values: [ - { - kvlistValue: { - values: [ - { key: 'type', value: { stringValue: 'text-chunks' } }, - { key: 'time0', value: { intValue: 1_000 } }, - { key: 'index', value: { intValue: 0 } }, - { key: 'dt', value: { arrayValue: { values: [{ intValue: 7 }] } } }, - { - key: 'texts', - value: { - arrayValue: { - values: [ - { stringValue: 'first complete chunk' }, - { stringValue: 'second complete chunk' }, - ], - }, - }, - }, - ], - }, - }, - { - kvlistValue: { - values: [ - { key: 'type', value: { stringValue: 'chunk' } }, - { key: 'time', value: { intValue: 1_007 } }, - { - key: 'chunk', - value: { - kvlistValue: { - values: [ - { key: 'type', value: { stringValue: 'finish' } }, - { - key: 'reason', - value: { kvlistValue: { values: [{ key: 'kind', value: { stringValue: 'stop' } }] } }, - }, - ], - }, - }, - }, - ], - }, - }, - ], - }, - }) expect(eventTypes(captures)).not.toContain('manual') expect(eventTypes(captures)).toEqual(session.snapshotEvents().map(event => event.type)) expect(ops).toHaveLength(0) @@ -842,3 +797,39 @@ describe('dsh-session-telemetry-otel real-load-path guard', () => { await fiber.dispose() }) }) + + +it('reports oversized records without failing feedback or exporting their contents', async () => { + const { url, captures } = await mockCollector() + const ctx = new Context() + try { + await ctx.plugin(SessionStore) + const warn = vi.spyOn(ctx.logger, 'warn') + const fiber = await ctx.plugin(OpenTelemetrySessionBackend, { + mode: SessionTelemetryMode.FEEDBACK_ONLY, exporter: { url }, maxRequestBytes: 1, + }) + const session = ctx.sessions.create(SessionId('oversized')) + expect(() => { recordFeedback(session, { text: 'feedback remains local' }) }).not.toThrow() + await fiber.dispose() + expect(captures).toEqual([]) + expect(warn).toHaveBeenCalledWith('Session log export failed', expect.any(Error)) + } finally { + await ctx.fiber.dispose() + } +}) + +it('does not upload a record whose redaction policy removes its canonical envelope', async () => { + const { url, captures } = await mockCollector() + const { ctx, fiber } = await boot(url) + try { + ctx.on('session-telemetry/record', (_record, next) => { + const { sourceEvent: _sourceEvent, ...redacted } = next() + return redacted + }) + recordFeedback(ctx.sessions.create(SessionId('withheld-envelope')), { text: 'withheld' }) + await fiber.dispose() + expect(captures).toEqual([]) + } finally { + await ctx.fiber.dispose() + } +}) diff --git a/packages/session/session-telemetry-otel/tsconfig.json b/packages/session/session-telemetry-otel/tsconfig.json index 87f43bdc9a..d281b88bf0 100644 --- a/packages/session/session-telemetry-otel/tsconfig.json +++ b/packages/session/session-telemetry-otel/tsconfig.json @@ -34,6 +34,9 @@ }, { "path": "../../identity/anonymous-user-id" + }, + { + "path": "../../host/product-telemetry-otel" } ] } diff --git a/packages/session/session-telemetry/README.i18n.yaml b/packages/session/session-telemetry/README.i18n.yaml index 894c36f1e5..df0c1d695e 100644 --- a/packages/session/session-telemetry/README.i18n.yaml +++ b/packages/session/session-telemetry/README.i18n.yaml @@ -24,8 +24,8 @@ en: e3cc8ee7774a0ad5 zh: 3c039339870b8cf9 /deepseek-ai-dsh-session-telemetry/use-this-package/what-gets-captured: - en: ed96f8ed3cf72fe8 - zh: b7f2b546b8639f43 + en: 5b7fbbe1c55997e8 + zh: d62d9118fd8df2f4 /deepseek-ai-dsh-session-telemetry/use-this-package/the-sharing-disclosure: en: 5c325a8aaadf17c7 zh: 5c493d2333f82b28 @@ -36,8 +36,8 @@ en: cad1e9d195916553 zh: 2192f3ce5e2196bd /deepseek-ai-dsh-session-telemetry/understand-the-implementation/design-concept: - en: 3bac6e75e9e442eb - zh: 88062e9d93a782eb + en: 6a606a2a6549b063 + zh: a6e23d4bf5c522ab /deepseek-ai-dsh-session-telemetry/understand-the-implementation/source-map: en: 5dc345ec3dda38b8 zh: 189d673734a2c20e diff --git a/packages/session/session-telemetry/README.md b/packages/session/session-telemetry/README.md index c7d7668641..b48d8e4a7a 100644 --- a/packages/session/session-telemetry/README.md +++ b/packages/session/session-telemetry/README.md @@ -39,6 +39,8 @@ A backend implements three members: `emit(record)` must be a non-blocking enqueu Capture runs in one of two modes. `live` capture follows session events as they are appended, replays already-live sessions at mount time, and records lifecycle markers; `on-demand` capture reads the canonical session log only when the backend requests a prefix through `captureSession(session, throughSeq?)`. Coordinator options select whether stored history is included. Every canonical session event maps to one ledger record in order. An `assistant/message` or `assistant/attempt` record carries its complete embedded compact stream, including failed and retried output. Each ledger record also carries `session.id`, `session.format_version`, the numeric event identity, optional header facts, and a pre-mapped severity (`error` for `tool/result.isError`, `turn/end` error reasons, and `agent-error`; `info` otherwise). +Ledger records carry `sourceEvent` with the owning Session id and a copied event envelope excluding `data`. The redaction waterfall receives `body` as the only payload copy. Backends can reconstruct a complete exported event from the envelope and redacted body without bypassing redaction. + ### The sharing disclosure @@ -63,7 +65,7 @@ This section explains the capture design; the observable behavior is fully cover ### Design concept -The seam is built on one boundary: the harness's aspect ends at `emit()`. Complete event capture, redaction, and the handoff cursor live here; batching, retry, queueing, and loss policy are the reporting SDK's, deliberately not modelled or wrapped. The design and rejected alternatives are pinned in the [revival Agent Note](../../../.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.md). +The capture package owns complete event capture, redaction, and the handoff cursor. Backends own delivery; the OTel implementation uses SDK queues and retries with collector-specific byte-bounded requests. The [revival Agent Note](../../../.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.md) records the capture and redaction rationale. ### Source map diff --git a/packages/session/session-telemetry/README.zh.md b/packages/session/session-telemetry/README.zh.md index 2e9bc7e661..a4c6317e38 100644 --- a/packages/session/session-telemetry/README.zh.md +++ b/packages/session/session-telemetry/README.zh.md @@ -39,6 +39,8 @@ kind: "package-library" 捕获以两种模式之一运行。`live` 捕获在追加时跟随会话事件、在挂载时回放已存活会话并记录生命周期标记；`on-demand` 捕获只在后端通过 `captureSession(session, throughSeq?)` 请求前缀时读取权威会话日志。协调器选项决定是否包含存储历史。每条权威会话事件都按顺序映射为一条 ledger 记录。`assistant/message` 或 `assistant/attempt` 记录会携带完整的嵌入式紧凑流，包括失败和重试输出。每条 ledger 记录还携带 `session.id`、`session.format_version`、数值型事件标识、可选 header 事实与预先映射的严重级别（`tool/result.isError`、`turn/end` 的错误原因与 `agent-error` 映射为 `error`；其余为 `info`）。 +ledger 记录携带 `sourceEvent`，其中包含所属 Session id 和不含 `data` 的事件信封副本。脱敏 waterfall 接收的 `body` 是唯一的载荷副本。后端可从信封和脱敏后的 body 重建完整导出事件，而不绕过脱敏。 + ### 共享披露 @@ -63,7 +65,7 @@ kind: "package-library" ### 设计理念 -seam 建立在一个边界之上：harness 的职责止于 `emit()`。完整事件捕获、脱敏与 handoff 游标都在这里；批处理、重试、排队与丢失策略属于上报 SDK，本包有意不建模也不包装。设计与被否决的替代方案见[复活 Agent Note](../../../.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.zh.md)。 +采集包负责完整事件采集、脱敏和交接游标。后端负责投递；OTel 实现使用 SDK 队列和重试，并按采集端的字节上限限制请求。[恢复遥测的 Agent Note](../../../.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.zh.md) 记录采集与脱敏的理由。 ### 源码地图 diff --git a/packages/session/session-telemetry/src/coordinator.ts b/packages/session/session-telemetry/src/coordinator.ts index 7f3b5d6052..fa21a5ae41 100644 --- a/packages/session/session-telemetry/src/coordinator.ts +++ b/packages/session/session-telemetry/src/coordinator.ts @@ -178,15 +178,17 @@ export class SessionTelemetryCoordinator { /** Copy, redact, and hand one canonical event to the backend. */ private captureEvent(session: Session, event: SessionEvent): void { + const { data, ...envelope } = event this.deliver(session, { record: this.redact({ + sourceEvent: { sessionId: session.id, envelope: structuredClone(envelope) }, channel: 'ledger', time: event.time, severity: severityOf(event), attributes: identityOf(session, event), // The canonical event object is mutable and the backend serializes // later; append-time validation guarantees this clone cannot throw. - body: structuredClone(event.data), + body: structuredClone(data), }), seq: event.seq, }) diff --git a/packages/session/session-telemetry/src/index.ts b/packages/session/session-telemetry/src/index.ts index 392df907f7..54347e4a16 100644 --- a/packages/session/session-telemetry/src/index.ts +++ b/packages/session/session-telemetry/src/index.ts @@ -7,7 +7,7 @@ * forwarding), live versus on-demand canonical-log capture, and the HMR * cursor. Everything downstream of * {@link SessionTelemetryBackend.emit} — batching, retry, queueing, and loss policy — is the - * reporting SDK's territory and is deliberately not modelled here. The + * backend's responsibility and is deliberately not modelled here. The * design and its trade-offs are pinned in * .agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.md. * @@ -15,6 +15,7 @@ */ import { Context, Service } from '@deepseek-ai/cordis' +import type { SessionEvent, SessionId } from '@deepseek-ai/dsh-session' declare module '@deepseek-ai/cordis' { interface Context { @@ -62,6 +63,8 @@ export type SessionTelemetrySeverity = 'info' | 'warn' | 'error' * identity so they can never be mistaken for ledger rows. */ export interface SessionTelemetryRecord { + /** Canonical envelope without data; body carries the separately redacted payload. Absent for operational records. */ + sourceEvent?: { sessionId: SessionId; envelope: Omit } /** Ledger (session-log mirror) or ops (operational signal) channel; backends keep the two under separate instrumentation scopes. */ channel: 'ledger' | 'ops' /** Unix epoch milliseconds — the source event's append time for ledger records, the emission time for ops records. */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 49642d4755..0f3b736f3f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9026,6 +9026,9 @@ importers: '@deepseek-ai/dsh-loader-smoke': specifier: workspace:* version: link:../../test-support/loader-smoke + '@deepseek-ai/dsh-session': + specifier: workspace:* + version: link:../../core/session packages/host/webserver: dependencies: @@ -10917,9 +10920,6 @@ importers: '@opentelemetry/api-logs': specifier: ^0.220.0 version: 0.220.0 - '@opentelemetry/exporter-logs-otlp-http': - specifier: ^0.220.0 - version: 0.220.0(@opentelemetry/api@1.9.1) '@opentelemetry/otlp-exporter-base': specifier: ^0.220.0 version: 0.220.0(@opentelemetry/api@1.9.1) @@ -10951,6 +10951,9 @@ importers: '@deepseek-ai/dsh-command-feedback': specifier: workspace:* version: link:../../feedback/command-feedback + '@deepseek-ai/dsh-host-product-telemetry-otel': + specifier: workspace:* + version: link:../../host/product-telemetry-otel '@deepseek-ai/dsh-http-proxy': specifier: workspace:* version: link:../../util/http-proxy @@ -13980,6 +13983,9 @@ importers: '@deepseek-ai/dsh-host-plugin-inventory': specifier: workspace:* version: link:../../packages/host/plugin-inventory + '@deepseek-ai/dsh-host-product-telemetry-otel': + specifier: workspace:* + version: link:../../packages/host/product-telemetry-otel '@deepseek-ai/dsh-host-webserver': specifier: workspace:* version: link:../../packages/host/webserver @@ -16161,12 +16167,6 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/exporter-logs-otlp-http@0.220.0': - resolution: {integrity: sha512-8186thl+pTw64iz/qEEen5oJZoZ/gO73XruChdaGlYdWOdBIQ42r+vHLf6a7vIDqTD4b8ZOoMlyxptanECaI9A==} - engines: {node: ^18.19.0 || >=20.6.0} - peerDependencies: - '@opentelemetry/api': ^1.3.0 - '@opentelemetry/otlp-exporter-base@0.220.0': resolution: {integrity: sha512-CXYo8UD5Mn9YbgebO2EL4wejtA+gxLmLiu6HCk2KH2BR7XhFN6/6p1UlCb23DYCjeYkndevLHuejCCN1yx4+OQ==} engines: {node: ^18.19.0 || >=20.6.0} @@ -23477,15 +23477,6 @@ snapshots: '@opentelemetry/api': 1.9.1 '@opentelemetry/semantic-conventions': 1.43.0 - '@opentelemetry/exporter-logs-otlp-http@0.220.0(@opentelemetry/api@1.9.1)': - dependencies: - '@opentelemetry/api': 1.9.1 - '@opentelemetry/api-logs': 0.220.0 - '@opentelemetry/core': 2.9.0(@opentelemetry/api@1.9.1) - '@opentelemetry/otlp-exporter-base': 0.220.0(@opentelemetry/api@1.9.1) - '@opentelemetry/otlp-transformer': 0.220.0(@opentelemetry/api@1.9.1) - '@opentelemetry/sdk-logs': 0.220.0(@opentelemetry/api@1.9.1) - '@opentelemetry/otlp-exporter-base@0.220.0(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 diff --git a/python/sdk-runtime/package.json b/python/sdk-runtime/package.json index 3083c4ad9f..6978667f80 100644 --- a/python/sdk-runtime/package.json +++ b/python/sdk-runtime/package.json @@ -55,6 +55,7 @@ "@deepseek-ai/dsh-hooks-claude-code": "workspace:*", "@deepseek-ai/dsh-hooks-codex": "workspace:*", "@deepseek-ai/dsh-host-plugin-inventory": "workspace:*", + "@deepseek-ai/dsh-host-product-telemetry-otel": "workspace:*", "@deepseek-ai/dsh-host-webserver": "workspace:*", "@deepseek-ai/dsh-http-proxy": "workspace:*", "@deepseek-ai/dsh-invariants": "workspace:*", diff --git a/scripts/gen-cordis-catalog.ts b/scripts/gen-cordis-catalog.ts index facc1d7222..72c622e90f 100644 --- a/scripts/gen-cordis-catalog.ts +++ b/scripts/gen-cordis-catalog.ts @@ -274,6 +274,7 @@ export const EVENT_WALK_EXEMPTIONS: Record = { */ export const LINK_MAP: Readonly> = { ProductTelemetryRecord: 'product-telemetry.md', + SessionLogRecord: 'product-telemetry.md', ProductTelemetryScalar: 'product-telemetry.md', WorkspaceChangesSummary: 'deliverables.md', WorkspaceFileDiff: 'deliverables.md', diff --git a/scripts/type-equiv.manifest.json b/scripts/type-equiv.manifest.json index 190df80ee0..4f31c159c9 100644 --- a/scripts/type-equiv.manifest.json +++ b/scripts/type-equiv.manifest.json @@ -2336,6 +2336,16 @@ "doc": "docs/subsystems/product-telemetry.md", "symbol": "ProductTelemetryRecord", "source": "packages/host/product-telemetry-otel/src/index.ts" + }, + { + "doc": "docs/subsystems/product-telemetry.md", + "symbol": "SessionLogRecord", + "source": "packages/host/product-telemetry-otel/src/session-log.ts" + }, + { + "doc": "docs/subsystems/product-telemetry.md", + "symbol": "SessionLogOptions", + "source": "packages/host/product-telemetry-otel/src/session-log.ts" } ] } diff --git a/snapshots/web/feedback-release/feedback-release.expected.json b/snapshots/web/feedback-release/feedback-release.expected.json index 994953b4ef..501d8582bf 100644 --- a/snapshots/web/feedback-release/feedback-release.expected.json +++ b/snapshots/web/feedback-release/feedback-release.expected.json @@ -1,5 +1,6 @@ { "mode": "FEEDBACK_ONLY", + "eventName": "session-log", "feedback": [ { "type": "feedback/record",