From f673eec90383a8a5699026d25c18b981ea1edc2f Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Mon, 26 Oct 2026 13:52:16 +0800 Subject: [PATCH 1/3] fix(web): decode a malformed multi-label charset declaration --- ...26-malformed-charset-declaration.i18n.yaml | 22 ++++ ...026-10-26-malformed-charset-declaration.md | 45 ++++++++ ...-10-26-malformed-charset-declaration.zh.md | 45 ++++++++ packages/web/web-fetch-http/README.i18n.yaml | 12 +- packages/web/web-fetch-http/README.md | 6 +- packages/web/web-fetch-http/README.zh.md | 6 +- packages/web/web-fetch-http/src/policy.ts | 108 +++++++++++++++--- packages/web/web-fetch-http/src/provider.ts | 15 +-- .../web-fetch-http/tests/fetch-http.spec.ts | 78 ++++++++++++- 9 files changed, 294 insertions(+), 43 deletions(-) create mode 100644 .agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.i18n.yaml create mode 100644 .agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.md create mode 100644 .agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.zh.md diff --git a/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.i18n.yaml new file mode 100644 index 0000000000..f77ef8fa89 --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.i18n.yaml @@ -0,0 +1,22 @@ +# Bilingual-pair consistency record for 2026-10-26-malformed-charset-declaration.md (docs/i18n/README.md): per heading +# section, a hash of its English and Chinese blocks outside code blocks and generated regions. +# After editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.md +/agent-note-a-malformed-charset-declaration-is-resolved-against-the-body: + en: ce3a059a679d3c55 + zh: 5f2b426f9cb84d6e +/agent-note-a-malformed-charset-declaration-is-resolved-against-the-body/problem: + en: 111d4f151c7c209e + zh: b5dcc01b300c74c3 +/agent-note-a-malformed-charset-declaration-is-resolved-against-the-body/decision: + en: 1700647432adb909 + zh: 5453577aad4deeae +/agent-note-a-malformed-charset-declaration-is-resolved-against-the-body/verification: + en: 88d3b3325d2603e7 + zh: 4b2a22f427cbc4c0 +/agent-note-a-malformed-charset-declaration-is-resolved-against-the-body/alternatives-considered: + en: 6aac113f16de50d8 + zh: 9cd553b7f71950cc +/agent-note-a-malformed-charset-declaration-is-resolved-against-the-body/consequences: + en: 4d4d530ec9bc82e5 + zh: cfbe5498901ec215 diff --git a/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.md b/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.md new file mode 100644 index 0000000000..1a56bca323 --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.md @@ -0,0 +1,45 @@ +# Agent Note: A malformed charset declaration is resolved against the body + +Status: implemented + +English | [中文](2026-10-26-malformed-charset-declaration.zh.md) + +## Problem + +A `Content-Type` header can join two encoding labels in one `charset` parameter (`text/html; charset=utf-8,gbk`). The whole parameter reaches `TextDecoder` as a single label, which rejects it, so `web_fetch` failed with `WEB_UNSUPPORTED_CONTENT_TYPE` before reading a body that is valid UTF-8 and renders in a browser. The header is malformed, but its body stays readable, and refusing the page costs more than tolerating the declaration. + +## Decision + +`charsetCandidates` splits the declared parameter on comma and whitespace, replaces each label with its canonical `TextDecoder` name (`gb2312` and `gbk` collapse into one candidate), keeps declaration order, drops labels no encoding has, returns the UTF-8 default when the response declares none, and throws the existing `WEB_UNSUPPORTED_CONTENT_TYPE` when a declaration names no encoding at all. + +`decodeBody` decodes with a single candidate as declared and does not probe. Several candidates mean the header names encodings a server cannot mean simultaneously, so the body decides: a UTF-8 candidate wins when the bytes are strictly valid UTF-8; otherwise the first candidate whose strict decode succeeds wins; when no candidate decodes the body, the fetch fails naming the candidates rather than returning replacement characters. + +The UTF-8 precedence comes from how far each decode result can be trusted. A strict UTF-8 decode of GBK text fails, while `TextDecoder('gbk')` accepted four of six UTF-8 test strings, so a legacy label's success does not identify the encoding and only UTF-8 validity does. Single-byte legacy labels such as `windows-1252` accept every byte sequence, so without the precedence an `iso-8859-1,utf-8` declaration over UTF-8 text would decode as `windows-1252`. + +The strict probe decodes with `fatal: true` and `stream: true`, so an incomplete final sequence cut by `maxResponseBytes` stays buffered instead of failing the probe; the chosen label's ordinary decode then replaces it. Candidates resolve before the body is read, keeping the path that cancels the stream when a declaration names no encoding. + +## Verification + +- [`fetch-http.spec.ts`](../../../../packages/web/web-fetch-http/tests/fetch-http.spec.ts) covers the candidate rules (`utf-8,gbk`, the `gb2312`/`gbk` alias pair, an unrecognized label beside a valid one, a declaration naming nothing, the UTF-8 default), the single-candidate path, UTF-8 over `windows-1252`, declaration-order fallback, the no-candidate failure, and a body cut inside a character. +- The same file fetches through a local HTTP server with `charset=utf-8,gbk` over UTF-8 HTML, with `charset=gbk,utf-8` over GBK bytes, and with a body neither label decodes (rejected). +- `pnpm exec vitest run packages/web/web-fetch-http/tests --coverage --coverage.include='packages/web/web-fetch-http/src/**/*.ts'` reports 100% statements, branches, functions, and lines for the package. + +## Alternatives considered + +**Use the text before the first comma.** The reported header decodes correctly that way, but it presumes the first label is the intended one: `gbk,utf-8` over UTF-8 text and `iso-8859-1,utf-8` over UTF-8 text would decode as the first label and produce mojibake. Rejected. + +**Treat a declaration no encoding has as absent and fall back to UTF-8.** A browser reaches its `` prescan through that fallback, but here it would decode genuinely GBK content as UTF-8 without a signal. Rejected; a declaration naming no encoding stays a loud failure. + +**Probe the candidates in declaration order.** Legacy decoders accept UTF-8 byte sequences too readily (four of six samples), so the last declared label would decide UTF-8 content whenever a legacy label precedes it. Rejected. + +**Implement the `` prescan instead.** That is the specification's path for a transport declaration a browser cannot use, but it adds HTML scanning to the transport layer, does not help non-HTML text responses, and the reported page decodes once the declaration itself is tolerated. Deferred, and the package README keeps `` as an ignored declaration. + +**Keep failing with a clearer error.** Readable content is the common case behind such a header; refusing it costs more than the ambiguity of a permissive decode that still requires the bytes to decode cleanly. + +## Consequences + +- The model now receives pages whose header joins labels when the body decodes; when no candidate decodes the body, the failure names the candidates and no replacement-character text reaches the model. +- A short body can be valid in more than one candidate encoding. The UTF-8 precedence decides then, so a short GBK body declared `gbk,utf-8` can decode as UTF-8; that ambiguity becomes negligible as the body grows. +- A single-label declaration behaves as before, including the failure for an unrecognized label and the replacement characters from a declared-but-wrong encoding. +- A declaration naming many encodings costs one strict decode per distinct encoding, bounded by `maxResponseBytes`. +- `` remains ignored. diff --git a/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.zh.md b/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.zh.md new file mode 100644 index 0000000000..2184d84edc --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.zh.md @@ -0,0 +1,45 @@ +# Agent Note: 畸形的 charset 声明由正文判定解码编码 + +Status: implemented + +[English](2026-10-26-malformed-charset-declaration.md) | 中文 + +## Problem + +`Content-Type` 标头可以在一个 `charset` 参数里拼接两个编码标签（`text/html; charset=utf-8,gbk`）。整个参数作为单个标签交给 `TextDecoder` 会被拒绝，因此 `web_fetch` 在读取正文之前就以 `WEB_UNSUPPORTED_CONTENT_TYPE` 失败，而该正文是合法 UTF-8 且浏览器可以正常渲染。标头是畸形的，但正文仍然可读，拒绝整页的代价高于容忍这个声明。 + +## Decision + +`charsetCandidates` 按逗号与空白切分声明的参数，把每个标签替换为其 `TextDecoder` 规范名称（`gb2312` 与 `gbk` 合并为一个候选），保留声明顺序，丢弃任何编码都不认识的标签，响应未声明时返回 UTF-8 默认值，声明存在但不含任何已知编码时抛出既有的 `WEB_UNSUPPORTED_CONTENT_TYPE`。 + +`decodeBody` 在只有单个候选时按声明解码且不做探测。存在多个候选说明标头同时点名了服务器不可能同时表达的编码，此时由正文判定：正文字节严格合法 UTF-8 时使用 UTF-8 候选；否则使用第一个严格解码成功的候选；没有任何候选能解码正文时抓取失败并列出候选，而不是返回替换字符。 + +UTF-8 优先级来自两类解码结果可信度的差距。GBK 文本的严格 UTF-8 解码会失败，而 `TextDecoder('gbk')` 接受了 6 条 UTF-8 测试字符串中的 4 条，因此传统编码解码成功并不能识别编码，只有 UTF-8 的合法性可以。`windows-1252` 这类单字节传统编码接受任意字节序列，若没有该优先级，`iso-8859-1,utf-8` 声明下的 UTF-8 文本会被当作 `windows-1252` 解码。 + +严格探测使用 `fatal: true` 与 `stream: true` 解码，因此被 `maxResponseBytes` 截断的末尾不完整序列会留在缓冲区，而不是让探测失败；随后所选编码的常规解码会把该不完整序列替换掉。候选项在读取正文之前解析完成，保留了声明不含任何已知编码时取消流的失败路径。 + +## Verification + +- [`fetch-http.spec.ts`](../../../../packages/web/web-fetch-http/tests/fetch-http.spec.ts) 覆盖候选规则（`utf-8,gbk`、`gb2312`/`gbk` 别名对、无法识别的标签与有效标签并存、声明不含任何已知编码、UTF-8 默认值）、单候选路径、UTF-8 优先于 `windows-1252`、按声明顺序回退、无候选可解码的失败，以及在字符内部被截断的正文。 +- 同一文件通过本地 HTTP 服务抓取 `charset=utf-8,gbk` 的 UTF-8 HTML、`charset=gbk,utf-8` 的 GBK 字节，以及两个标签都无法解码的正文（被拒绝）。 +- `pnpm exec vitest run packages/web/web-fetch-http/tests --coverage --coverage.include='packages/web/web-fetch-http/src/**/*.ts'` 对该包报告语句、分支、函数与行 100% 覆盖。 + +## Alternatives considered + +**取第一个逗号之前的文本。** 这样报告的标头能正确解码，但它假定第一个标签就是服务器本意：`gbk,utf-8` 下的 UTF-8 正文与 `iso-8859-1,utf-8` 下的 UTF-8 正文都会按第一个标签解码并产生乱码。拒绝。 + +**把不含任何已知编码的声明当作未声明并回退 UTF-8。** 浏览器正是经由该回退走到 `` 预扫描，但在这里会把真正的 GBK 内容无提示地当作 UTF-8 解码。拒绝；不含任何已知编码的声明保持显式失败。 + +**按声明顺序探测候选。** 传统编码解码器接受 UTF-8 字节序列的比例过高（6 条样本中 4 条），只要传统标签排在前面，UTF-8 正文就会由最后一个声明标签决定。拒绝。 + +**改为实现 `` 预扫描。** 那是规范为浏览器无法使用的传输声明准备的路径，但会把 HTML 扫描加入传输层，对非 HTML 文本响应没有帮助，而所报页面在容忍声明本身之后即可正确解码。延期处理，包 README 仍把 `` 记为被忽略的声明。 + +**继续失败并给出更清晰的错误。** 这类标头背后通常是可读内容；在仍然要求正文能干净解码的前提下，容忍解码的模糊性代价更低。 + +## Consequences + +- 当正文可解码时，模型现在能读到标头拼接了多个标签的页面；没有任何候选能解码正文时，失败信息会列出候选，且不会有替换字符文本进入模型。 +- 较短的正文可能在多个候选编码下都合法，此时由 UTF-8 优先级决定，因此 `gbk,utf-8` 声明下较短的 GBK 正文仍可能被当作 UTF-8 解码；随正文变长该模糊性趋近于零。 +- 单标签声明与之前一致，包括无法识别标签的失败，以及声明错误编码产生的替换字符。 +- 声明点名大量编码时，每个不同编码会消耗一次严格解码，受 `maxResponseBytes` 限制。 +- `` 仍然被忽略。 diff --git a/packages/web/web-fetch-http/README.i18n.yaml b/packages/web/web-fetch-http/README.i18n.yaml index fcea215f61..1e5155ec85 100644 --- a/packages/web/web-fetch-http/README.i18n.yaml +++ b/packages/web/web-fetch-http/README.i18n.yaml @@ -27,8 +27,8 @@ en: 2d18cdcc9b53dd0f zh: 3bc6ca7c5933ee5d /deepseek-ai-dsh-web-fetch-http/use-this-package/transport-behavior: - en: 584054eb465ae615 - zh: c421860ba79bfffc + en: 8d528f9875fc9ef7 + zh: 0cfba2d8ac297d8b /deepseek-ai-dsh-web-fetch-http/use-this-package/failures-and-recovery: en: 35407f106dc65deb zh: 02118072fba68b8f @@ -42,8 +42,8 @@ en: 1f355ec2726ff61b zh: 076a0c259c94f22a /deepseek-ai-dsh-web-fetch-http/understand-the-implementation/read-path: - en: f13cd16555fe39a5 - zh: 8f48cb659f2c272c + en: 6eccff20f66757e4 + zh: 2c344e25d59e7d96 /deepseek-ai-dsh-web-fetch-http/further-exploration: en: eeaaf0b455673a1c zh: 0874f9f44ff9a220 @@ -54,8 +54,8 @@ en: 40294c1c31b74824 zh: d75d48f485447a74 /deepseek-ai-dsh-web-fetch-http/known-limitations-and-deferred-work: - en: aa351dc3edaa6355 - zh: 5bb556857b389df7 + en: 334c100db26b665a + zh: 2443f50d829e8b97 /deepseek-ai-dsh-web-fetch-http/known-limitations-and-deferred-work/dev-note: en: e67f633dc6fe7773 zh: 3bb0c20c93f16f98 diff --git a/packages/web/web-fetch-http/README.md b/packages/web/web-fetch-http/README.md index ac56125e1a..ba1793d4db 100644 --- a/packages/web/web-fetch-http/README.md +++ b/packages/web/web-fetch-http/README.md @@ -61,7 +61,7 @@ const page = await ctx.web.fetch({ url: 'https://example.com' }) ### Transport behavior -The provider keeps requests anonymous and bounded: it accepts only `http:` and `https:` URLs without embedded credentials and rejects URLs over 2,048 characters. Requests go through the process-wide dispatcher, so the installed proxy policy applies and the system resolver chooses the address; any destination address is allowed. Same-origin redirects are followed; cross-origin redirects fail and require a fresh call. The provider also enforces byte, character, hop, and time caps, rejects unsupported content types, and sends an explicit product `User-Agent`. +The provider keeps requests anonymous and bounded: it accepts only `http:` and `https:` URLs without embedded credentials and rejects URLs over 2,048 characters. Requests go through the process-wide dispatcher, so the installed proxy policy applies and the system resolver chooses the address; any destination address is allowed. Same-origin redirects are followed; cross-origin redirects fail and require a fresh call. The provider also enforces byte, character, hop, and time caps, rejects unsupported content types, and sends an explicit product `User-Agent`. It also decodes a malformed charset declaration that comma-joins several labels (`charset=utf-8,gbk`): a UTF-8 candidate is used when the body is valid UTF-8, otherwise the first candidate that strictly decodes the body, and a body no candidate decodes fails with `WEB_UNSUPPORTED_CONTENT_TYPE`. ### Failures and recovery @@ -94,7 +94,7 @@ The package is built on one separation and one layered timeout: ### Read path -A fetch validates the URL and sends the request. It repeats URL validation for each same-origin redirect; a cross-origin redirect fails before response bytes are accepted. The final response is classified by `Content-Type`, decoded from its declared charset, and read under the byte cap; the decoded text is then truncated to the character cap. +A fetch validates the URL and sends the request. It repeats URL validation for each same-origin redirect; a cross-origin redirect fails before response bytes are accepted. The final response is classified by `Content-Type`, decoded from its declared charset (a malformed multi-label declaration is resolved against the body), and read under the byte cap; the decoded text is then truncated to the character cap. @@ -132,7 +132,7 @@ These limits define when the provider is unsafe or a poor fit. They are current - **Private destinations are reachable** — loopback, private-network, and cloud metadata addresses are fetched like any other host. A deployment that must keep the model away from them needs a `tools/pre-execute` policy, network confinement, or no fetch tool. - **Only textual content decodes** — html/xhtml and `text/*` plus JSON/XML families; a missing `Content-Type` or any binary type throws `WEB_UNSUPPORTED_CONTENT_TYPE`, and text-extractable PDF decoding is named deferred work. -- **Charset comes only from the `Content-Type` header** (UTF-8 default) — an HTML `` declaration is ignored, and a declared-but-unrecognized charset label throws rather than falling back. +- **Charset comes only from the `Content-Type` header** (UTF-8 default) — an HTML `` declaration is ignored, and a declared label that names no encoding throws rather than falling back. ### Dev Note diff --git a/packages/web/web-fetch-http/README.zh.md b/packages/web/web-fetch-http/README.zh.md index 662d3e132c..66e5322e58 100644 --- a/packages/web/web-fetch-http/README.zh.md +++ b/packages/web/web-fetch-http/README.zh.md @@ -61,7 +61,7 @@ const page = await ctx.web.fetch({ url: 'https://example.com' }) ### 传输行为 -提供方保持请求匿名且有界：只接受不含内嵌凭据且不超过 2,048 个字符的 `http:` 与 `https:` URL。请求经过进程级 dispatcher，因此已安装的代理策略生效，目标地址由系统解析器选择；任何目标地址都被允许。同源重定向会被跟随；跨源重定向会失败并要求重新调用。提供方还强制执行字节、字符、跳数和时间上限，拒绝不支持的内容类型，并发送显式产品 `User-Agent`。 +提供方保持请求匿名且有界：只接受不含内嵌凭据且不超过 2,048 个字符的 `http:` 与 `https:` URL。请求经过进程级 dispatcher，因此已安装的代理策略生效，目标地址由系统解析器选择；任何目标地址都被允许。同源重定向会被跟随；跨源重定向会失败并要求重新调用。提供方还强制执行字节、字符、跳数和时间上限，拒绝不支持的内容类型，并发送显式产品 `User-Agent`。它还会解码以逗号拼接多个标签的畸形 charset 声明（`charset=utf-8,gbk`）：正文字节是合法 UTF-8 时使用 UTF-8 候选，否则使用第一个能严格解码正文的候选；若没有任何候选能解码正文，则以 `WEB_UNSUPPORTED_CONTENT_TYPE` 失败。 ### 失败与恢复 @@ -94,7 +94,7 @@ const page = await ctx.web.fetch({ url: 'https://example.com' }) ### 读取路径 -抓取先校验 URL，再发送请求。每次同源重定向都重复 URL 校验；跨源重定向在接收响应字节前失败。最终响应按 `Content-Type` 分类、依声明的 charset 解码，并在字节上限内读取；解码后的文本再截断到字符上限。 +抓取先校验 URL，再发送请求。每次同源重定向都重复 URL 校验；跨源重定向在接收响应字节前失败。最终响应按 `Content-Type` 分类、依声明的 charset 解码（畸形的多标签声明会结合正文判定），并在字节上限内读取；解码后的文本再截断到字符上限。 @@ -132,7 +132,7 @@ const page = await ctx.web.fetch({ url: 'https://example.com' }) - **可以访问私有目标**——loopback、内网与云元数据地址与其他主机一样会被抓取。需要阻止模型访问这些地址的部署必须添加 `tools/pre-execute` 策略、网络隔离，或不挂载抓取工具。 - **只解码文本内容**——包括 html/xhtml 与 `text/*` 加 JSON/XML 家族；缺少 `Content-Type` 或任何二进制类型都会抛出 `WEB_UNSUPPORTED_CONTENT_TYPE`，可提取文本的 PDF 解码属于明确的延期工作。 -- **charset 只来自 `Content-Type` 标头**（默认 UTF-8）——HTML `` 声明会被忽略；声明但无法识别的 charset 标签会抛出异常，而非回退。 +- **charset 只来自 `Content-Type` 标头**（默认 UTF-8）——HTML `` 声明会被忽略；声明了但不含任何已知编码的标签会抛出异常，而非回退。 ### 开发备注 diff --git a/packages/web/web-fetch-http/src/policy.ts b/packages/web/web-fetch-http/src/policy.ts index 47b8fe14ab..d8f71672a3 100644 --- a/packages/web/web-fetch-http/src/policy.ts +++ b/packages/web/web-fetch-http/src/policy.ts @@ -1,7 +1,8 @@ /** - * URL validation and content-type classification for the local HTTP(S) fetch - * provider — the pure, network-free half. The provider's `fetch()` composes - * these with transport (redirect following, byte caps, decoding). + * URL validation, content-type classification, and charset resolution for the + * local HTTP(S) fetch provider — the pure, network-free half. The provider's + * `fetch()` composes these with transport (redirect following, byte caps, + * decoding). * * @module @deepseek-ai/dsh-web-fetch-http/policy */ @@ -84,34 +85,103 @@ export function classifyContentType(contentType: string | null): FetchableKind | /** * Extract the `charset` parameter from a response `Content-Type`, lower-cased, - * or `undefined` when absent. The provider feeds this label to `TextDecoder` - * so a non-UTF-8 response is decoded with its declared encoding rather than - * silently mangled into replacement characters. + * or `undefined` when absent. {@link charsetCandidates} resolves the declared + * label into decoders; keeping the parameter unprocessed lets a malformed + * declaration be reported exactly as the server sent it. * * @param contentType - the raw `Content-Type` header, or `null` when the * response carries none. - * @returns the lower-cased charset label, or `undefined` when none is declared. + * @returns the lower-cased charset parameter, or `undefined` when none is + * declared. */ export function parseCharset(contentType: string | null): string | undefined { const match = /;\s*charset\s*=\s*"?([^";]+)"?/i.exec(contentType ?? '') return match?.[1]?.trim().toLowerCase() } +/** Canonical `TextDecoder` name of UTF-8, and the default when no charset is declared. */ +const UTF8_ENCODING = 'utf-8' + /** - * Build a `TextDecoder` for the declared charset, falling back to UTF-8 when - * none is declared. Throws {@link WebError} `WEB_UNSUPPORTED_CONTENT_TYPE` when - * the label is present but not a charset `TextDecoder` recognizes — better to - * fail loudly than return mojibake. + * Resolve a declared charset parameter into the encoding labels that can decode + * the body, in declaration order. An absent parameter yields the UTF-8 default. + * A parameter is split on the comma and whitespace separators that a malformed + * header uses to join several labels (`charset=utf-8,gbk`); each label is + * replaced by its canonical `TextDecoder` name, so `gb2312` and `gbk` collapse + * into one candidate, and labels no encoding has are dropped. Throws + * {@link WebError} `WEB_UNSUPPORTED_CONTENT_TYPE` when a declaration is present + * but names no recognizable encoding. * - * @param charset - the declared charset label (from {@link parseCharset}), or - * `undefined` to default to UTF-8. - * @returns a decoder for the declared (or defaulted) encoding. + * @param charset - the declared charset parameter (from {@link parseCharset}), + * or `undefined` when the response declares none. + * @returns the candidate encoding labels; never empty. */ -export function decoderForCharset(charset: string | undefined): TextDecoder { - if (charset === undefined) return new TextDecoder('utf-8') +export function charsetCandidates(charset: string | undefined): string[] { + if (charset === undefined) return [UTF8_ENCODING] + const candidates: string[] = [] + for (const label of charset.split(/[\s,]+/)) { + if (label.length === 0) continue + const encoding = canonicalEncoding(label) + if (encoding !== undefined && !candidates.includes(encoding)) candidates.push(encoding) + } + if (candidates.length === 0) { + throw new WebError(`unsupported charset "${charset}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE') + } + return candidates +} + +/** + * Decode a response body with its candidate encodings. One candidate is used as + * declared. Several candidates mean the header joined labels a server cannot + * mean simultaneously, so the bytes decide: a UTF-8 candidate wins when the body + * is valid UTF-8 (a strict decode of legacy multi-byte text as UTF-8 fails, + * while those legacy decoders accept UTF-8 sequences too readily to count as + * evidence), otherwise the first candidate whose strict decode succeeds. When + * no candidate decodes the body the fetch fails rather than returning + * replacement characters. + * + * @param bytes - the complete response body, possibly cut at the byte cap. + * @param candidates - the encoding labels from {@link charsetCandidates}. + * @returns the decoded text. + */ +export function decodeBody(bytes: Uint8Array, candidates: readonly string[]): string { + const declared = candidates.length === 1 ? candidates[0] : undefined + if (declared !== undefined) return new TextDecoder(declared).decode(bytes) + const chosen = chooseEncoding(bytes, candidates) + return new TextDecoder(chosen).decode(bytes) +} + +/** Canonical `TextDecoder` name of a label, or `undefined` when no encoding has it. */ +function canonicalEncoding(label: string): string | undefined { try { - return new TextDecoder(charset) - } catch (error: unknown) { - throw new WebError(`unsupported charset "${charset}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE', { cause: error }) + return new TextDecoder(label).encoding + } catch { + // A RangeError here means this label names no encoding; a malformed + // declaration may still name a usable encoding beside it. + return undefined } } + +/** Whether an encoding decodes the body, tolerating a final sequence cut at the byte cap. */ +function decodesStrictly(encoding: string, bytes: Uint8Array): boolean { + try { + new TextDecoder(encoding, { fatal: true }).decode(bytes, { stream: true }) + return true + } catch { + // A RangeError here means the body holds a sequence this encoding cannot + // decode, so the next candidate gets the bytes. + return false + } +} + +/** Pick the candidate encoding for a multi-label declaration, or throw when none decodes the body. */ +function chooseEncoding(bytes: Uint8Array, candidates: readonly string[]): string { + if (candidates.includes(UTF8_ENCODING) && decodesStrictly(UTF8_ENCODING, bytes)) return UTF8_ENCODING + for (const encoding of candidates) { + if (decodesStrictly(encoding, bytes)) return encoding + } + throw new WebError( + `cannot decode the response body with any declared charset (${candidates.join(', ')})`, + 'WEB_UNSUPPORTED_CONTENT_TYPE', + ) +} diff --git a/packages/web/web-fetch-http/src/provider.ts b/packages/web/web-fetch-http/src/provider.ts index 3357124e38..e1bfb26baf 100644 --- a/packages/web/web-fetch-http/src/provider.ts +++ b/packages/web/web-fetch-http/src/provider.ts @@ -10,7 +10,7 @@ import { WebError } from '@deepseek-ai/dsh-web' import type { WebFetchBody, WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web' import { deadline, timeoutOf } from '@deepseek-ai/dsh-timeout' import type { Response } from 'undici' -import { classifyContentType, decoderForCharset, isSameOrigin, parseCharset, validateFetchUrl } from './policy.ts' +import { charsetCandidates, classifyContentType, decodeBody, isSameOrigin, parseCharset, validateFetchUrl } from './policy.ts' /** Resolved provider limits (the plugin's schemastery Config supplies defaults). */ export interface HttpFetchLimits { @@ -121,18 +121,19 @@ export class HttpFetchProvider implements WebFetchProvider { throw new WebError(`unsupported content type "${contentType ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE') } - // Resolve the decoder BEFORE reading the body so an unsupported charset - // fails without consuming the stream — but cancel the body on that failure - // so the socket does not leak (matching the unsupported-content-type path). - let decoder: TextDecoder + // Resolve the charset candidates BEFORE reading the body so an unsupported + // declaration fails without consuming the stream — but cancel the body on + // that failure so the socket does not leak (matching the + // unsupported-content-type path). + let candidates: string[] try { - decoder = decoderForCharset(parseCharset(contentType)) + candidates = charsetCandidates(parseCharset(contentType)) } catch (error: unknown) { await response.body?.cancel() throw error } const { bytes, truncatedByBytes } = await this.readCapped(response, signal) - const decoded = decoder.decode(bytes) + const decoded = decodeBody(bytes, candidates) const truncatedByChars = decoded.length > this.limits.maxBodyChars const content = truncatedByChars ? decoded.slice(0, this.limits.maxBodyChars) : decoded const body: WebFetchBody = kind === 'html' ? { kind: 'html', content } : { kind: 'text', content } diff --git a/packages/web/web-fetch-http/tests/fetch-http.spec.ts b/packages/web/web-fetch-http/tests/fetch-http.spec.ts index ab2b713936..971599ebda 100644 --- a/packages/web/web-fetch-http/tests/fetch-http.spec.ts +++ b/packages/web/web-fetch-http/tests/fetch-http.spec.ts @@ -7,8 +7,9 @@ import { HttpFetchProvider, LOCAL_FETCH_PROVIDER_ID } from '@deepseek-ai/dsh-web import type { HttpFetchLimits } from '@deepseek-ai/dsh-web-fetch-http' import * as fetchPlugin from '@deepseek-ai/dsh-web-fetch-http' import { + charsetCandidates, classifyContentType, - decoderForCharset, + decodeBody, isSameOrigin, parseCharset, parseFetchUrl, @@ -83,10 +84,49 @@ describe('policy helpers', () => { expect(parseCharset(null)).toBeUndefined() }) - it('builds a decoder for a charset and defaults to UTF-8', () => { - expect(decoderForCharset(undefined).encoding).toBe('utf-8') - expect(decoderForCharset('iso-8859-1').encoding).toBe('windows-1252') - expect(() => decoderForCharset('not-a-charset')).toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) + it('resolves charset candidates and defaults to UTF-8', () => { + expect(charsetCandidates(undefined)).toEqual(['utf-8']) + expect(charsetCandidates('utf-8')).toEqual(['utf-8']) + expect(charsetCandidates('gb2312, gbk')).toEqual(['gbk']) + expect(charsetCandidates('not-a-charset, utf-8')).toEqual(['utf-8']) + expect(charsetCandidates('utf-8,gbk')).toEqual(['utf-8', 'gbk']) + expect(() => charsetCandidates('not-a-charset')) + .toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE', message: 'unsupported charset "not-a-charset"' })) + expect(() => charsetCandidates(',')) + .toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) + }) + + it('decodes with a single declared charset without probing the body', () => { + // 0xE9 is "é" in windows-1252; a strict probe would reject these bytes. + expect(decodeBody(Uint8Array.of(0x63, 0x61, 0x66, 0xE9), ['windows-1252'])).toBe('café') + }) + + it('lets a UTF-8 candidate win when the body is valid UTF-8', () => { + const utf8 = new TextEncoder().encode('中文') + expect(decodeBody(utf8, ['gbk', 'utf-8'])).toBe('中文') + // windows-1252 decodes every byte sequence, so only the UTF-8 preference reaches the text. + expect(decodeBody(utf8, ['windows-1252', 'utf-8'])).toBe('中文') + }) + + it('falls back to the first candidate that decodes the body', () => { + // 中文 in GBK: invalid UTF-8, so declaration order decides. + const gbk = Uint8Array.of(0xD6, 0xD0, 0xCE, 0xC4) + expect(decodeBody(gbk, ['gbk', 'utf-8'])).toBe('中文') + expect(decodeBody(gbk, ['gbk', 'windows-1252'])).toBe('中文') + }) + + it('rejects a body no candidate decodes instead of returning replacement characters', () => { + // 0x81 starts neither a GBK nor a UTF-8 sequence, and 0x20 cannot continue either. + expect(() => decodeBody(Uint8Array.of(0x81, 0x20), ['gbk', 'utf-8'])) + .toThrow(expect.objectContaining({ + code: 'WEB_UNSUPPORTED_CONTENT_TYPE', + message: 'cannot decode the response body with any declared charset (gbk, utf-8)', + })) + }) + + it('accepts a body cut at the byte cap inside a character', () => { + // '中文' cut to 5 bytes leaves the second character incomplete. + expect(decodeBody(new TextEncoder().encode('中文').subarray(0, 5), ['utf-8', 'gbk'])).toBe('中\uFFFD') }) }) @@ -179,6 +219,34 @@ describe('HttpFetchProvider caps', () => { await expect(provider().fetch({ url: base })) .rejects.toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) }) + + it('decodes a comma-joined charset declaration when the body is UTF-8', async () => { + handler = (_req, res) => { + res.writeHead(200, { 'content-type': 'text/html; charset=utf-8,gbk' }) + res.end(Buffer.from('

你好

', 'utf8')) + } + const result = await provider().fetch({ url: base }) + expect(result.body).toEqual({ kind: 'html', content: '

你好

' }) + }) + + it('follows declaration order for a conflicting comma-joined charset whose body is not UTF-8', async () => { + // 中文 in GBK; the body is invalid UTF-8, so the first declared label decodes it. + handler = (_req, res) => { + res.writeHead(200, { 'content-type': 'text/plain; charset=gbk,utf-8' }) + res.end(Buffer.from([0xD6, 0xD0, 0xCE, 0xC4])) + } + const result = await provider().fetch({ url: base }) + expect(result.body.content).toBe('中文') + }) + + it('rejects a comma-joined charset whose labels decode none of the body', async () => { + handler = (_req, res) => { + res.writeHead(200, { 'content-type': 'text/plain; charset=gbk,utf-8' }) + res.end(Buffer.from([0x81, 0x20])) + } + await expect(provider().fetch({ url: base })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) + }) }) describe('HttpFetchProvider redirects', () => { From 0d7ae41c8f56d5d3b1170d7b389124e14c74bced Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Mon, 26 Oct 2026 14:39:21 +0800 Subject: [PATCH 2/3] fix(web): constrain malformed charset recovery to strict UTF-8 --- ...26-malformed-charset-declaration.i18n.yaml | 33 ++-- ...026-10-26-malformed-charset-declaration.md | 38 ++--- ...-10-26-malformed-charset-declaration.zh.md | 38 ++--- packages/web/web-fetch-http/README.i18n.yaml | 12 +- packages/web/web-fetch-http/README.md | 9 +- packages/web/web-fetch-http/README.zh.md | 9 +- packages/web/web-fetch-http/src/policy.ts | 104 +++--------- packages/web/web-fetch-http/src/provider.ts | 23 ++- .../web-fetch-http/tests/fetch-http.spec.ts | 148 +++++++++++------- .../web-fetch/web-fetch-fixture-server.mjs | 8 +- 10 files changed, 192 insertions(+), 230 deletions(-) diff --git a/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.i18n.yaml index f77ef8fa89..22e56fbdd0 100644 --- a/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.i18n.yaml +++ b/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.i18n.yaml @@ -2,21 +2,18 @@ # section, a hash of its English and Chinese blocks outside code blocks and generated regions. # After editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.md -/agent-note-a-malformed-charset-declaration-is-resolved-against-the-body: - en: ce3a059a679d3c55 - zh: 5f2b426f9cb84d6e -/agent-note-a-malformed-charset-declaration-is-resolved-against-the-body/problem: - en: 111d4f151c7c209e - zh: b5dcc01b300c74c3 -/agent-note-a-malformed-charset-declaration-is-resolved-against-the-body/decision: - en: 1700647432adb909 - zh: 5453577aad4deeae -/agent-note-a-malformed-charset-declaration-is-resolved-against-the-body/verification: - en: 88d3b3325d2603e7 - zh: 4b2a22f427cbc4c0 -/agent-note-a-malformed-charset-declaration-is-resolved-against-the-body/alternatives-considered: - en: 6aac113f16de50d8 - zh: 9cd553b7f71950cc -/agent-note-a-malformed-charset-declaration-is-resolved-against-the-body/consequences: - en: 4d4d530ec9bc82e5 - zh: cfbe5498901ec215 +/agent-note-malformed-charset-recovery-prefers-utf-8: + en: d8a759b84c36b308 + zh: 3b6ff2eb5519afb1 +/agent-note-malformed-charset-recovery-prefers-utf-8/problem: + en: dce96445b3801a7f + zh: e026d26fa680556a +/agent-note-malformed-charset-recovery-prefers-utf-8/decision: + en: ba74ad57757231ca + zh: 46999cd0100814ae +/agent-note-malformed-charset-recovery-prefers-utf-8/alternatives-considered: + en: 9fafffb6a7a78a29 + zh: ee1dcc27d0315e68 +/agent-note-malformed-charset-recovery-prefers-utf-8/consequences: + en: c8aad3cc3f02e81e + zh: 29d9313afd1c7b5b diff --git a/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.md b/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.md index 1a56bca323..9e36964e09 100644 --- a/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.md +++ b/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.md @@ -1,4 +1,4 @@ -# Agent Note: A malformed charset declaration is resolved against the body +# Agent Note: Malformed charset recovery prefers UTF-8 Status: implemented @@ -6,40 +6,28 @@ English | [中文](2026-10-26-malformed-charset-declaration.zh.md) ## Problem -A `Content-Type` header can join two encoding labels in one `charset` parameter (`text/html; charset=utf-8,gbk`). The whole parameter reaches `TextDecoder` as a single label, which rejects it, so `web_fetch` failed with `WEB_UNSUPPORTED_CONTENT_TYPE` before reading a body that is valid UTF-8 and renders in a browser. The header is malformed, but its body stays readable, and refusing the page costs more than tolerating the declaration. +A response can declare `charset=utf-8,gbk` over readable UTF-8 HTML. Passing that entire parameter to `TextDecoder` rejects the page before reading it. Compatibility needs an explicit encoding preference because successful decoding alone cannot identify the server's intended encoding. ## Decision -`charsetCandidates` splits the declared parameter on comma and whitespace, replaces each label with its canonical `TextDecoder` name (`gb2312` and `gbk` collapse into one candidate), keeps declaration order, drops labels no encoding has, returns the UTF-8 default when the response declares none, and throws the existing `WEB_UNSUPPORTED_CONTENT_TYPE` when a declaration names no encoding at all. +`decoderForCharset` preserves valid single-label declarations and the absent-header UTF-8 default. If a declaration is unsupported as a whole, it checks its comma- and whitespace-separated labels for UTF-8, including standard aliases. A match selects a fatal UTF-8 decoder; otherwise the declaration fails before the response body is read and the provider cancels the stream. -`decodeBody` decodes with a single candidate as declared and does not probe. Several candidates mean the header names encodings a server cannot mean simultaneously, so the body decides: a UTF-8 candidate wins when the bytes are strictly valid UTF-8; otherwise the first candidate whose strict decode succeeds wins; when no candidate decodes the body, the fetch fails naming the candidates rather than returning replacement characters. +The provider decodes once. Recovery rejects invalid UTF-8 with `WEB_UNSUPPORTED_CONTENT_TYPE` and preserves the decoding error as its cause. Only an actual byte-cap truncation enables streaming decode, which omits an incomplete suffix and keeps `truncated: true`. Complete responses, including those exactly at the cap, validate the final sequence at EOF. -The UTF-8 precedence comes from how far each decode result can be trusted. A strict UTF-8 decode of GBK text fails, while `TextDecoder('gbk')` accepted four of six UTF-8 test strings, so a legacy label's success does not identify the encoding and only UTF-8 validity does. Single-byte legacy labels such as `windows-1252` accept every byte sequence, so without the precedence an `iso-8859-1,utf-8` declaration over UTF-8 text would decode as `windows-1252`. - -The strict probe decodes with `fatal: true` and `stream: true`, so an incomplete final sequence cut by `maxResponseBytes` stays buffered instead of failing the probe; the chosen label's ordinary decode then replaces it. Candidates resolve before the body is read, keeping the path that cancels the stream when a declaration names no encoding. - -## Verification - -- [`fetch-http.spec.ts`](../../../../packages/web/web-fetch-http/tests/fetch-http.spec.ts) covers the candidate rules (`utf-8,gbk`, the `gb2312`/`gbk` alias pair, an unrecognized label beside a valid one, a declaration naming nothing, the UTF-8 default), the single-candidate path, UTF-8 over `windows-1252`, declaration-order fallback, the no-candidate failure, and a body cut inside a character. -- The same file fetches through a local HTTP server with `charset=utf-8,gbk` over UTF-8 HTML, with `charset=gbk,utf-8` over GBK bytes, and with a body neither label decodes (rejected). -- `pnpm exec vitest run packages/web/web-fetch-http/tests --coverage --coverage.include='packages/web/web-fetch-http/src/**/*.ts'` reports 100% statements, branches, functions, and lines for the package. +UTF-8 recovery is an explicit heuristic, not proof of the original encoding. The bytes `C2 A9` represent `©` in UTF-8 and `漏` in GBK; repeating them preserves that ambiguity at any length. The provider accepts that ambiguity for UTF-8 recovery but does not extend it to arbitrary legacy-encoding guesses. ## Alternatives considered -**Use the text before the first comma.** The reported header decodes correctly that way, but it presumes the first label is the intended one: `gbk,utf-8` over UTF-8 text and `iso-8859-1,utf-8` over UTF-8 text would decode as the first label and produce mojibake. Rejected. +**Choose the first declared encoding that decodes.** A GBK body can also decode as `windows-1252` with different text. Declaration order does not resolve the conflict, and single-byte encodings can turn a decoding failure into silent corruption. Rejected. -**Treat a declaration no encoding has as absent and fall back to UTF-8.** A browser reaches its `` prescan through that fallback, but here it would decode genuinely GBK content as UTF-8 without a signal. Rejected; a declaration naming no encoding stays a loud failure. +**Use strict probes followed by ordinary decoding.** This decodes successful bodies twice. An unflushed streaming probe also accepts an incomplete EOF sequence that the ordinary decoder replaces. One fatal decode with explicit truncation state is sufficient. -**Probe the candidates in declaration order.** Legacy decoders accept UTF-8 byte sequences too readily (four of six samples), so the last declared label would decide UTF-8 content whenever a legacy label precedes it. Rejected. - -**Implement the `` prescan instead.** That is the specification's path for a transport declaration a browser cannot use, but it adds HTML scanning to the transport layer, does not help non-HTML text responses, and the reported page decodes once the declaration itself is tolerated. Deferred, and the package README keeps `` as an ignored declaration. - -**Keep failing with a clearer error.** Readable content is the common case behind such a header; refusing it costs more than the ambiguity of a permissive decode that still requires the bytes to decode cleanly. +**Read BOM and HTML meta declarations.** This would add HTML encoding identification and a dependency or parser, with a separate policy for missing metadata and non-HTML content. It remains deferred; the bounded UTF-8 preference resolves the reported response without claiming browser-equivalent detection. ## Consequences -- The model now receives pages whose header joins labels when the body decodes; when no candidate decodes the body, the failure names the candidates and no replacement-character text reaches the model. -- A short body can be valid in more than one candidate encoding. The UTF-8 precedence decides then, so a short GBK body declared `gbk,utf-8` can decode as UTF-8; that ambiguity becomes negligible as the body grows. -- A single-label declaration behaves as before, including the failure for an unrecognized label and the replacement characters from a declared-but-wrong encoding. -- A declaration naming many encodings costs one strict decode per distinct encoding, bounded by `maxResponseBytes`. -- `` remains ignored. +- Malformed declarations containing UTF-8 can return readable text; a failed UTF-8 recovery never falls back to another encoding or inserts replacement characters. +- Malformed declarations without a UTF-8 label remain unsupported, including repeated legacy aliases. Valid single-label legacy decoding is unchanged. +- [`fetch-http.spec.ts`](../../../../packages/web/web-fetch-http/tests/fetch-http.spec.ts) covers recovery, conflicting labels, invalid bytes, EOF, truncation, and response-stream cancellation. +- The recorded [`web-fetch`](../../../../snapshots/session/web-fetch/snapshot.yml) scenario serves literal UTF-8 characters with `charset=gbk,utf-8` through the real HTTP provider and verifies the same model-visible text. +- HTML `` remains ignored. The package README documents the remaining ambiguity. diff --git a/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.zh.md b/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.zh.md index 2184d84edc..8e9d1f31ed 100644 --- a/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.zh.md +++ b/.agents/notes/implemented/bug-fix/2026-10-26-malformed-charset-declaration.zh.md @@ -1,4 +1,4 @@ -# Agent Note: 畸形的 charset 声明由正文判定解码编码 +# Agent Note: 畸形 charset 声明的恢复优先使用 UTF-8 Status: implemented @@ -6,40 +6,28 @@ Status: implemented ## Problem -`Content-Type` 标头可以在一个 `charset` 参数里拼接两个编码标签（`text/html; charset=utf-8,gbk`）。整个参数作为单个标签交给 `TextDecoder` 会被拒绝，因此 `web_fetch` 在读取正文之前就以 `WEB_UNSUPPORTED_CONTENT_TYPE` 失败，而该正文是合法 UTF-8 且浏览器可以正常渲染。标头是畸形的，但正文仍然可读，拒绝整页的代价高于容忍这个声明。 +响应可能在可读的 UTF-8 HTML 上声明 `charset=utf-8,gbk`。把整个参数交给 `TextDecoder` 会在读取前拒绝页面。兼容处理需要明确的编码优先规则，因为解码成功本身无法识别服务器原本使用的编码。 ## Decision -`charsetCandidates` 按逗号与空白切分声明的参数，把每个标签替换为其 `TextDecoder` 规范名称（`gb2312` 与 `gbk` 合并为一个候选），保留声明顺序，丢弃任何编码都不认识的标签，响应未声明时返回 UTF-8 默认值，声明存在但不含任何已知编码时抛出既有的 `WEB_UNSUPPORTED_CONTENT_TYPE`。 +`decoderForCharset` 保留合法的单标签声明，以及未声明时默认使用 UTF-8 的行为。如果整个声明不受支持，它检查按逗号和空白分隔的标签是否包含 UTF-8（含标准别名）。匹配时选择 fatal UTF-8 解码器；否则在读取正文前拒绝声明，由提供方取消响应流。 -`decodeBody` 在只有单个候选时按声明解码且不做探测。存在多个候选说明标头同时点名了服务器不可能同时表达的编码，此时由正文判定：正文字节严格合法 UTF-8 时使用 UTF-8 候选；否则使用第一个严格解码成功的候选；没有任何候选能解码正文时抓取失败并列出候选，而不是返回替换字符。 +提供方只解码一次。恢复路径以 `WEB_UNSUPPORTED_CONTENT_TYPE` 拒绝非法 UTF-8，并保留解码错误作为 cause。只有真正触发字节上限截断时才启用流式解码，省略不完整的末尾序列并保持 `truncated: true`。完整响应（包括恰好达到上限的响应）在 EOF 校验末尾序列。 -UTF-8 优先级来自两类解码结果可信度的差距。GBK 文本的严格 UTF-8 解码会失败，而 `TextDecoder('gbk')` 接受了 6 条 UTF-8 测试字符串中的 4 条，因此传统编码解码成功并不能识别编码，只有 UTF-8 的合法性可以。`windows-1252` 这类单字节传统编码接受任意字节序列，若没有该优先级，`iso-8859-1,utf-8` 声明下的 UTF-8 文本会被当作 `windows-1252` 解码。 - -严格探测使用 `fatal: true` 与 `stream: true` 解码，因此被 `maxResponseBytes` 截断的末尾不完整序列会留在缓冲区，而不是让探测失败；随后所选编码的常规解码会把该不完整序列替换掉。候选项在读取正文之前解析完成，保留了声明不含任何已知编码时取消流的失败路径。 - -## Verification - -- [`fetch-http.spec.ts`](../../../../packages/web/web-fetch-http/tests/fetch-http.spec.ts) 覆盖候选规则（`utf-8,gbk`、`gb2312`/`gbk` 别名对、无法识别的标签与有效标签并存、声明不含任何已知编码、UTF-8 默认值）、单候选路径、UTF-8 优先于 `windows-1252`、按声明顺序回退、无候选可解码的失败，以及在字符内部被截断的正文。 -- 同一文件通过本地 HTTP 服务抓取 `charset=utf-8,gbk` 的 UTF-8 HTML、`charset=gbk,utf-8` 的 GBK 字节，以及两个标签都无法解码的正文（被拒绝）。 -- `pnpm exec vitest run packages/web/web-fetch-http/tests --coverage --coverage.include='packages/web/web-fetch-http/src/**/*.ts'` 对该包报告语句、分支、函数与行 100% 覆盖。 +UTF-8 恢复是明确的启发式规则，不是原始编码的证明。字节 `C2 A9` 在 UTF-8 下表示 `©`，在 GBK 下表示 `漏`；重复这些字节后，任意长度都保留该歧义。提供方为 UTF-8 恢复接受这种歧义，但不将其扩展为任意传统编码的猜测。 ## Alternatives considered -**取第一个逗号之前的文本。** 这样报告的标头能正确解码，但它假定第一个标签就是服务器本意：`gbk,utf-8` 下的 UTF-8 正文与 `iso-8859-1,utf-8` 下的 UTF-8 正文都会按第一个标签解码并产生乱码。拒绝。 +**选择声明中第一个能解码的编码。** GBK 正文也可能按 `windows-1252` 解出不同文本。声明顺序不能解决冲突，单字节编码还可能把解码失败变成静默的内容损坏。拒绝。 -**把不含任何已知编码的声明当作未声明并回退 UTF-8。** 浏览器正是经由该回退走到 `` 预扫描，但在这里会把真正的 GBK 内容无提示地当作 UTF-8 解码。拒绝；不含任何已知编码的声明保持显式失败。 +**先严格探测，再常规解码。** 这会对成功的正文解码两次。未结束的流式探测还会接受 EOF 处不完整的序列，随后被常规解码器替换。一次 fatal 解码配合显式截断状态已经足够。 -**按声明顺序探测候选。** 传统编码解码器接受 UTF-8 字节序列的比例过高（6 条样本中 4 条），只要传统标签排在前面，UTF-8 正文就会由最后一个声明标签决定。拒绝。 - -**改为实现 `` 预扫描。** 那是规范为浏览器无法使用的传输声明准备的路径，但会把 HTML 扫描加入传输层，对非 HTML 文本响应没有帮助，而所报页面在容忍声明本身之后即可正确解码。延期处理，包 README 仍把 `` 记为被忽略的声明。 - -**继续失败并给出更清晰的错误。** 这类标头背后通常是可读内容；在仍然要求正文能干净解码的前提下，容忍解码的模糊性代价更低。 +**读取 BOM 和 HTML meta 声明。** 这会增加 HTML 编码识别能力及依赖或解析器，还需单独规定缺少元数据和非 HTML 内容的策略。继续延期；有范围的 UTF-8 优先规则可以处理所报告的响应，无需宣称与浏览器等价的编码检测。 ## Consequences -- 当正文可解码时，模型现在能读到标头拼接了多个标签的页面；没有任何候选能解码正文时，失败信息会列出候选，且不会有替换字符文本进入模型。 -- 较短的正文可能在多个候选编码下都合法，此时由 UTF-8 优先级决定，因此 `gbk,utf-8` 声明下较短的 GBK 正文仍可能被当作 UTF-8 解码；随正文变长该模糊性趋近于零。 -- 单标签声明与之前一致，包括无法识别标签的失败，以及声明错误编码产生的替换字符。 -- 声明点名大量编码时，每个不同编码会消耗一次严格解码，受 `maxResponseBytes` 限制。 -- `` 仍然被忽略。 +- 含有 UTF-8 的畸形声明可以返回可读文本；UTF-8 恢复失败时不会回退到其他编码，也不会插入替换字符。 +- 不含 UTF-8 标签的畸形声明仍不受支持，包括重复的传统编码别名。合法的单标签传统编码解码保持不变。 +- [`fetch-http.spec.ts`](../../../../packages/web/web-fetch-http/tests/fetch-http.spec.ts) 覆盖恢复、冲突标签、非法字节、EOF、截断和响应流取消。 +- 已录制的 [`web-fetch`](../../../../snapshots/session/web-fetch/snapshot.yml) 场景通过真实 HTTP 提供方，以 `charset=gbk,utf-8` 提供包含原样 UTF-8 字符的页面，并验证相同的模型可见文本。 +- HTML `` 仍然被忽略。包 README 记录了剩余歧义。 diff --git a/packages/web/web-fetch-http/README.i18n.yaml b/packages/web/web-fetch-http/README.i18n.yaml index 1e5155ec85..073d491a38 100644 --- a/packages/web/web-fetch-http/README.i18n.yaml +++ b/packages/web/web-fetch-http/README.i18n.yaml @@ -27,8 +27,8 @@ en: 2d18cdcc9b53dd0f zh: 3bc6ca7c5933ee5d /deepseek-ai-dsh-web-fetch-http/use-this-package/transport-behavior: - en: 8d528f9875fc9ef7 - zh: 0cfba2d8ac297d8b + en: 3e54fca1e3ee5f12 + zh: 1f284264ce8a2a54 /deepseek-ai-dsh-web-fetch-http/use-this-package/failures-and-recovery: en: 35407f106dc65deb zh: 02118072fba68b8f @@ -42,8 +42,8 @@ en: 1f355ec2726ff61b zh: 076a0c259c94f22a /deepseek-ai-dsh-web-fetch-http/understand-the-implementation/read-path: - en: 6eccff20f66757e4 - zh: 2c344e25d59e7d96 + en: 4836d06bc065477d + zh: 5a7ddc805a7c67d8 /deepseek-ai-dsh-web-fetch-http/further-exploration: en: eeaaf0b455673a1c zh: 0874f9f44ff9a220 @@ -54,8 +54,8 @@ en: 40294c1c31b74824 zh: d75d48f485447a74 /deepseek-ai-dsh-web-fetch-http/known-limitations-and-deferred-work: - en: 334c100db26b665a - zh: 2443f50d829e8b97 + en: e2c71fbfe37d473a + zh: 987e4c667d24d16c /deepseek-ai-dsh-web-fetch-http/known-limitations-and-deferred-work/dev-note: en: e67f633dc6fe7773 zh: 3bb0c20c93f16f98 diff --git a/packages/web/web-fetch-http/README.md b/packages/web/web-fetch-http/README.md index ba1793d4db..8ee1c951ad 100644 --- a/packages/web/web-fetch-http/README.md +++ b/packages/web/web-fetch-http/README.md @@ -61,7 +61,9 @@ const page = await ctx.web.fetch({ url: 'https://example.com' }) ### Transport behavior -The provider keeps requests anonymous and bounded: it accepts only `http:` and `https:` URLs without embedded credentials and rejects URLs over 2,048 characters. Requests go through the process-wide dispatcher, so the installed proxy policy applies and the system resolver chooses the address; any destination address is allowed. Same-origin redirects are followed; cross-origin redirects fail and require a fresh call. The provider also enforces byte, character, hop, and time caps, rejects unsupported content types, and sends an explicit product `User-Agent`. It also decodes a malformed charset declaration that comma-joins several labels (`charset=utf-8,gbk`): a UTF-8 candidate is used when the body is valid UTF-8, otherwise the first candidate that strictly decodes the body, and a body no candidate decodes fails with `WEB_UNSUPPORTED_CONTENT_TYPE`. +The provider keeps requests anonymous and bounded: it accepts only `http:` and `https:` URLs without embedded credentials and rejects URLs over 2,048 characters. Requests go through the process-wide dispatcher, so the installed proxy policy applies and the system resolver chooses the address; any destination address is allowed. Same-origin redirects are followed; cross-origin redirects fail and require a fresh call. The provider also enforces byte, character, hop, and time caps, rejects unsupported content types, and sends an explicit product `User-Agent`. + +A malformed comma- or whitespace-joined charset declaration containing a UTF-8 label (such as `charset=utf-8,gbk`) is decoded strictly as UTF-8. Invalid UTF-8 fails with `WEB_UNSUPPORTED_CONTENT_TYPE`; the provider does not try another encoding. When the byte cap cuts a character, recovery omits the incomplete suffix and returns `truncated: true`; a complete response must pass the EOF check. Valid single-label declarations retain their ordinary decoding behavior. ### Failures and recovery @@ -94,7 +96,7 @@ The package is built on one separation and one layered timeout: ### Read path -A fetch validates the URL and sends the request. It repeats URL validation for each same-origin redirect; a cross-origin redirect fails before response bytes are accepted. The final response is classified by `Content-Type`, decoded from its declared charset (a malformed multi-label declaration is resolved against the body), and read under the byte cap; the decoded text is then truncated to the character cap. +A fetch validates the URL and sends the request. It repeats URL validation for each same-origin redirect; a cross-origin redirect fails before response bytes are accepted. The final response is classified by `Content-Type`, decoded from its declared charset (malformed declarations containing UTF-8 use strict UTF-8 recovery), and read under the byte cap; the decoded text is then truncated to the character cap. @@ -132,7 +134,8 @@ These limits define when the provider is unsafe or a poor fit. They are current - **Private destinations are reachable** — loopback, private-network, and cloud metadata addresses are fetched like any other host. A deployment that must keep the model away from them needs a `tools/pre-execute` policy, network confinement, or no fetch tool. - **Only textual content decodes** — html/xhtml and `text/*` plus JSON/XML families; a missing `Content-Type` or any binary type throws `WEB_UNSUPPORTED_CONTENT_TYPE`, and text-extractable PDF decoding is named deferred work. -- **Charset comes only from the `Content-Type` header** (UTF-8 default) — an HTML `` declaration is ignored, and a declared label that names no encoding throws rather than falling back. +- **Charset comes only from the `Content-Type` header** (UTF-8 default) — an HTML `` declaration is ignored, and unsupported declarations without a UTF-8 recovery label throw. +- **UTF-8 recovery is a preference, not encoding detection** — bytes can be valid as both UTF-8 and another encoding, even in a long response. Recovery chooses UTF-8 in that case; malformed declarations without a UTF-8 label remain unsupported. ### Dev Note diff --git a/packages/web/web-fetch-http/README.zh.md b/packages/web/web-fetch-http/README.zh.md index 66e5322e58..895d3f1094 100644 --- a/packages/web/web-fetch-http/README.zh.md +++ b/packages/web/web-fetch-http/README.zh.md @@ -61,7 +61,9 @@ const page = await ctx.web.fetch({ url: 'https://example.com' }) ### 传输行为 -提供方保持请求匿名且有界：只接受不含内嵌凭据且不超过 2,048 个字符的 `http:` 与 `https:` URL。请求经过进程级 dispatcher，因此已安装的代理策略生效，目标地址由系统解析器选择；任何目标地址都被允许。同源重定向会被跟随；跨源重定向会失败并要求重新调用。提供方还强制执行字节、字符、跳数和时间上限，拒绝不支持的内容类型，并发送显式产品 `User-Agent`。它还会解码以逗号拼接多个标签的畸形 charset 声明（`charset=utf-8,gbk`）：正文字节是合法 UTF-8 时使用 UTF-8 候选，否则使用第一个能严格解码正文的候选；若没有任何候选能解码正文，则以 `WEB_UNSUPPORTED_CONTENT_TYPE` 失败。 +提供方保持请求匿名且有界：只接受不含内嵌凭据且不超过 2,048 个字符的 `http:` 与 `https:` URL。请求经过进程级 dispatcher，因此已安装的代理策略生效，目标地址由系统解析器选择；任何目标地址都被允许。同源重定向会被跟随；跨源重定向会失败并要求重新调用。提供方还强制执行字节、字符、跳数和时间上限，拒绝不支持的内容类型，并发送显式产品 `User-Agent`。 + +对于以逗号或空白拼接、且含有 UTF-8 标签的畸形 charset 声明（例如 `charset=utf-8,gbk`），提供方严格按 UTF-8 解码。非法 UTF-8 以 `WEB_UNSUPPORTED_CONTENT_TYPE` 失败，提供方不会尝试其他编码。字节上限在字符内部截断时，恢复路径省略不完整的末尾序列并返回 `truncated: true`；完整响应必须通过 EOF 检查。合法的单标签声明保持原有的常规解码行为。 ### 失败与恢复 @@ -94,7 +96,7 @@ const page = await ctx.web.fetch({ url: 'https://example.com' }) ### 读取路径 -抓取先校验 URL，再发送请求。每次同源重定向都重复 URL 校验；跨源重定向在接收响应字节前失败。最终响应按 `Content-Type` 分类、依声明的 charset 解码（畸形的多标签声明会结合正文判定），并在字节上限内读取；解码后的文本再截断到字符上限。 +抓取先校验 URL，再发送请求。每次同源重定向都重复 URL 校验；跨源重定向在接收响应字节前失败。最终响应按 `Content-Type` 分类、依声明的 charset 解码（含有 UTF-8 的畸形声明使用严格 UTF-8 恢复），并在字节上限内读取；解码后的文本再截断到字符上限。 @@ -132,7 +134,8 @@ const page = await ctx.web.fetch({ url: 'https://example.com' }) - **可以访问私有目标**——loopback、内网与云元数据地址与其他主机一样会被抓取。需要阻止模型访问这些地址的部署必须添加 `tools/pre-execute` 策略、网络隔离，或不挂载抓取工具。 - **只解码文本内容**——包括 html/xhtml 与 `text/*` 加 JSON/XML 家族；缺少 `Content-Type` 或任何二进制类型都会抛出 `WEB_UNSUPPORTED_CONTENT_TYPE`，可提取文本的 PDF 解码属于明确的延期工作。 -- **charset 只来自 `Content-Type` 标头**（默认 UTF-8）——HTML `` 声明会被忽略；声明了但不含任何已知编码的标签会抛出异常，而非回退。 +- **charset 只来自 `Content-Type` 标头**（默认 UTF-8）——HTML `` 声明会被忽略；不支持且不含 UTF-8 恢复标签的声明会抛出异常。 +- **UTF-8 恢复是一种优先规则，不是编码检测**——同一组字节可能同时是合法的 UTF-8 和其他编码，长响应也不例外。恢复路径在这种情况下选择 UTF-8；不含 UTF-8 标签的畸形声明仍不受支持。 ### 开发备注 diff --git a/packages/web/web-fetch-http/src/policy.ts b/packages/web/web-fetch-http/src/policy.ts index d8f71672a3..a106958f1e 100644 --- a/packages/web/web-fetch-http/src/policy.ts +++ b/packages/web/web-fetch-http/src/policy.ts @@ -84,104 +84,42 @@ export function classifyContentType(contentType: string | null): FetchableKind | } /** - * Extract the `charset` parameter from a response `Content-Type`, lower-cased, - * or `undefined` when absent. {@link charsetCandidates} resolves the declared - * label into decoders; keeping the parameter unprocessed lets a malformed - * declaration be reported exactly as the server sent it. + * Extract the lower-cased `charset` parameter from a response `Content-Type`. * - * @param contentType - the raw `Content-Type` header, or `null` when the - * response carries none. - * @returns the lower-cased charset parameter, or `undefined` when none is - * declared. + * @param contentType - the raw header, or `null` when absent. + * @returns the charset parameter, or `undefined` when none is declared. */ export function parseCharset(contentType: string | null): string | undefined { const match = /;\s*charset\s*=\s*"?([^";]+)"?/i.exec(contentType ?? '') return match?.[1]?.trim().toLowerCase() } -/** Canonical `TextDecoder` name of UTF-8, and the default when no charset is declared. */ -const UTF8_ENCODING = 'utf-8' - /** - * Resolve a declared charset parameter into the encoding labels that can decode - * the body, in declaration order. An absent parameter yields the UTF-8 default. - * A parameter is split on the comma and whitespace separators that a malformed - * header uses to join several labels (`charset=utf-8,gbk`); each label is - * replaced by its canonical `TextDecoder` name, so `gb2312` and `gbk` collapse - * into one candidate, and labels no encoding has are dropped. Throws - * {@link WebError} `WEB_UNSUPPORTED_CONTENT_TYPE` when a declaration is present - * but names no recognizable encoding. + * Resolve the declared charset, defaulting to UTF-8 when absent. A malformed + * comma- or whitespace-joined declaration containing a UTF-8 label uses a fatal + * UTF-8 decoder; other unsupported declarations throw `WEB_UNSUPPORTED_CONTENT_TYPE`. + * UTF-8 recovery is a preference, not proof of the original encoding. * - * @param charset - the declared charset parameter (from {@link parseCharset}), - * or `undefined` when the response declares none. - * @returns the candidate encoding labels; never empty. + * @param charset - the declared charset parameter, or `undefined` to default. + * @returns a decoder; callers must report fatal decoding failures and flush at + * EOF unless they deliberately truncated the bytes. */ -export function charsetCandidates(charset: string | undefined): string[] { - if (charset === undefined) return [UTF8_ENCODING] - const candidates: string[] = [] - for (const label of charset.split(/[\s,]+/)) { - if (label.length === 0) continue - const encoding = canonicalEncoding(label) - if (encoding !== undefined && !candidates.includes(encoding)) candidates.push(encoding) - } - if (candidates.length === 0) { - throw new WebError(`unsupported charset "${charset}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE') - } - return candidates -} - -/** - * Decode a response body with its candidate encodings. One candidate is used as - * declared. Several candidates mean the header joined labels a server cannot - * mean simultaneously, so the bytes decide: a UTF-8 candidate wins when the body - * is valid UTF-8 (a strict decode of legacy multi-byte text as UTF-8 fails, - * while those legacy decoders accept UTF-8 sequences too readily to count as - * evidence), otherwise the first candidate whose strict decode succeeds. When - * no candidate decodes the body the fetch fails rather than returning - * replacement characters. - * - * @param bytes - the complete response body, possibly cut at the byte cap. - * @param candidates - the encoding labels from {@link charsetCandidates}. - * @returns the decoded text. - */ -export function decodeBody(bytes: Uint8Array, candidates: readonly string[]): string { - const declared = candidates.length === 1 ? candidates[0] : undefined - if (declared !== undefined) return new TextDecoder(declared).decode(bytes) - const chosen = chooseEncoding(bytes, candidates) - return new TextDecoder(chosen).decode(bytes) -} - -/** Canonical `TextDecoder` name of a label, or `undefined` when no encoding has it. */ -function canonicalEncoding(label: string): string | undefined { +export function decoderForCharset(charset: string | undefined): TextDecoder { + const declared = charset ?? 'utf-8' try { - return new TextDecoder(label).encoding - } catch { - // A RangeError here means this label names no encoding; a malformed - // declaration may still name a usable encoding beside it. - return undefined + return new TextDecoder(declared) + } catch (error: unknown) { + if (declared.split(/[\s,]+/).some(isUtf8Label)) return new TextDecoder('utf-8', { fatal: true }) + throw new WebError(`unsupported charset "${declared}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE', { cause: error }) } } -/** Whether an encoding decodes the body, tolerating a final sequence cut at the byte cap. */ -function decodesStrictly(encoding: string, bytes: Uint8Array): boolean { +/** Whether a label names UTF-8, including its standard aliases. */ +function isUtf8Label(label: string): boolean { try { - new TextDecoder(encoding, { fatal: true }).decode(bytes, { stream: true }) - return true - } catch { - // A RangeError here means the body holds a sequence this encoding cannot - // decode, so the next candidate gets the bytes. + return new TextDecoder(label).encoding === 'utf-8' + } catch (_error: unknown) { + // An unsupported label does not authorize UTF-8 recovery. return false } } - -/** Pick the candidate encoding for a multi-label declaration, or throw when none decodes the body. */ -function chooseEncoding(bytes: Uint8Array, candidates: readonly string[]): string { - if (candidates.includes(UTF8_ENCODING) && decodesStrictly(UTF8_ENCODING, bytes)) return UTF8_ENCODING - for (const encoding of candidates) { - if (decodesStrictly(encoding, bytes)) return encoding - } - throw new WebError( - `cannot decode the response body with any declared charset (${candidates.join(', ')})`, - 'WEB_UNSUPPORTED_CONTENT_TYPE', - ) -} diff --git a/packages/web/web-fetch-http/src/provider.ts b/packages/web/web-fetch-http/src/provider.ts index e1bfb26baf..1842e30c8e 100644 --- a/packages/web/web-fetch-http/src/provider.ts +++ b/packages/web/web-fetch-http/src/provider.ts @@ -10,7 +10,7 @@ import { WebError } from '@deepseek-ai/dsh-web' import type { WebFetchBody, WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web' import { deadline, timeoutOf } from '@deepseek-ai/dsh-timeout' import type { Response } from 'undici' -import { charsetCandidates, classifyContentType, decodeBody, isSameOrigin, parseCharset, validateFetchUrl } from './policy.ts' +import { classifyContentType, decoderForCharset, isSameOrigin, parseCharset, validateFetchUrl } from './policy.ts' /** Resolved provider limits (the plugin's schemastery Config supplies defaults). */ export interface HttpFetchLimits { @@ -121,19 +121,26 @@ export class HttpFetchProvider implements WebFetchProvider { throw new WebError(`unsupported content type "${contentType ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE') } - // Resolve the charset candidates BEFORE reading the body so an unsupported - // declaration fails without consuming the stream — but cancel the body on - // that failure so the socket does not leak (matching the - // unsupported-content-type path). - let candidates: string[] + // Reject unsupported declarations before reading, releasing the response stream. + let decoder: TextDecoder try { - candidates = charsetCandidates(parseCharset(contentType)) + decoder = decoderForCharset(parseCharset(contentType)) } catch (error: unknown) { await response.body?.cancel() throw error } const { bytes, truncatedByBytes } = await this.readCapped(response, signal) - const decoded = decodeBody(bytes, candidates) + let decoded: string + try { + // Only a real byte-cap truncation permits an incomplete UTF-8 suffix. + decoded = decoder.decode(bytes, { stream: decoder.fatal && truncatedByBytes }) + } catch (error: unknown) { + throw new WebError( + 'response body is not valid UTF-8 for the malformed charset declaration', + 'WEB_UNSUPPORTED_CONTENT_TYPE', + { cause: error }, + ) + } const truncatedByChars = decoded.length > this.limits.maxBodyChars const content = truncatedByChars ? decoded.slice(0, this.limits.maxBodyChars) : decoded const body: WebFetchBody = kind === 'html' ? { kind: 'html', content } : { kind: 'text', content } diff --git a/packages/web/web-fetch-http/tests/fetch-http.spec.ts b/packages/web/web-fetch-http/tests/fetch-http.spec.ts index 971599ebda..622788d573 100644 --- a/packages/web/web-fetch-http/tests/fetch-http.spec.ts +++ b/packages/web/web-fetch-http/tests/fetch-http.spec.ts @@ -7,9 +7,8 @@ import { HttpFetchProvider, LOCAL_FETCH_PROVIDER_ID } from '@deepseek-ai/dsh-web import type { HttpFetchLimits } from '@deepseek-ai/dsh-web-fetch-http' import * as fetchPlugin from '@deepseek-ai/dsh-web-fetch-http' import { - charsetCandidates, classifyContentType, - decodeBody, + decoderForCharset, isSameOrigin, parseCharset, parseFetchUrl, @@ -84,49 +83,24 @@ describe('policy helpers', () => { expect(parseCharset(null)).toBeUndefined() }) - it('resolves charset candidates and defaults to UTF-8', () => { - expect(charsetCandidates(undefined)).toEqual(['utf-8']) - expect(charsetCandidates('utf-8')).toEqual(['utf-8']) - expect(charsetCandidates('gb2312, gbk')).toEqual(['gbk']) - expect(charsetCandidates('not-a-charset, utf-8')).toEqual(['utf-8']) - expect(charsetCandidates('utf-8,gbk')).toEqual(['utf-8', 'gbk']) - expect(() => charsetCandidates('not-a-charset')) - .toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE', message: 'unsupported charset "not-a-charset"' })) - expect(() => charsetCandidates(',')) - .toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) + it('builds a decoder for a charset and defaults to UTF-8', () => { + expect(decoderForCharset(undefined).encoding).toBe('utf-8') + expect(decoderForCharset('iso-8859-1').encoding).toBe('windows-1252') + expect(decoderForCharset('gb2312').encoding).toBe('gbk') + expect(decoderForCharset('utf-8').fatal).toBe(false) + expect(() => decoderForCharset('not-a-charset')).toThrow(expect.objectContaining({ + code: 'WEB_UNSUPPORTED_CONTENT_TYPE', message: 'unsupported charset "not-a-charset"', + })) }) - it('decodes with a single declared charset without probing the body', () => { - // 0xE9 is "é" in windows-1252; a strict probe would reject these bytes. - expect(decodeBody(Uint8Array.of(0x63, 0x61, 0x66, 0xE9), ['windows-1252'])).toBe('café') + it.each(['utf-8,gbk', 'gbk,utf8', 'not-a-charset, utf-8', 'utf-8 utf8'])('requires strict UTF-8 for %s', (charset) => { + const decoder = decoderForCharset(charset) + expect(decoder.encoding).toBe('utf-8') + expect(decoder.fatal).toBe(true) }) - it('lets a UTF-8 candidate win when the body is valid UTF-8', () => { - const utf8 = new TextEncoder().encode('中文') - expect(decodeBody(utf8, ['gbk', 'utf-8'])).toBe('中文') - // windows-1252 decodes every byte sequence, so only the UTF-8 preference reaches the text. - expect(decodeBody(utf8, ['windows-1252', 'utf-8'])).toBe('中文') - }) - - it('falls back to the first candidate that decodes the body', () => { - // 中文 in GBK: invalid UTF-8, so declaration order decides. - const gbk = Uint8Array.of(0xD6, 0xD0, 0xCE, 0xC4) - expect(decodeBody(gbk, ['gbk', 'utf-8'])).toBe('中文') - expect(decodeBody(gbk, ['gbk', 'windows-1252'])).toBe('中文') - }) - - it('rejects a body no candidate decodes instead of returning replacement characters', () => { - // 0x81 starts neither a GBK nor a UTF-8 sequence, and 0x20 cannot continue either. - expect(() => decodeBody(Uint8Array.of(0x81, 0x20), ['gbk', 'utf-8'])) - .toThrow(expect.objectContaining({ - code: 'WEB_UNSUPPORTED_CONTENT_TYPE', - message: 'cannot decode the response body with any declared charset (gbk, utf-8)', - })) - }) - - it('accepts a body cut at the byte cap inside a character', () => { - // '中文' cut to 5 bytes leaves the second character incomplete. - expect(decodeBody(new TextEncoder().encode('中文').subarray(0, 5), ['utf-8', 'gbk'])).toBe('中\uFFFD') + it.each(['gb2312,gbk', 'windows-1252,gbk', ','])('rejects a malformed declaration without UTF-8: %s', (charset) => { + expect(() => decoderForCharset(charset)).toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) }) }) @@ -219,34 +193,99 @@ describe('HttpFetchProvider caps', () => { await expect(provider().fetch({ url: base })) .rejects.toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) }) +}) - it('decodes a comma-joined charset declaration when the body is UTF-8', async () => { +describe('HttpFetchProvider charset recovery', () => { + it.each(['utf-8,gbk', 'gbk,utf8', 'not-a-charset, utf-8'])('decodes UTF-8 HTML declared as %s', async (charset) => { handler = (_req, res) => { - res.writeHead(200, { 'content-type': 'text/html; charset=utf-8,gbk' }) - res.end(Buffer.from('

你好

', 'utf8')) + res.writeHead(200, { 'content-type': `text/html; charset=${charset}` }) + res.end('

你好

') } const result = await provider().fetch({ url: base }) expect(result.body).toEqual({ kind: 'html', content: '

你好

' }) + expect(result.truncated).toBe(false) }) - it('follows declaration order for a conflicting comma-joined charset whose body is not UTF-8', async () => { - // 中文 in GBK; the body is invalid UTF-8, so the first declared label decodes it. + it.each([ + ['gbk,utf-8', [0xE4, 0xB8]], // Valid GBK, but an incomplete UTF-8 sequence at EOF. + ['gbk,utf-8', [0xE4]], + ['gbk,utf-8', [0x81, 0x20]], + ['gbk,utf-8', [0xD6, 0xD0, 0xCE, 0xC4]], + ['not-a-charset,utf-8', [0xD6, 0xD0, 0xCE, 0xC4]], + ['utf8,utf-8', [0x81, 0x20]], + ] as const)('rejects non-UTF-8 bytes for %s: %j', async (charset, bytes) => { handler = (_req, res) => { - res.writeHead(200, { 'content-type': 'text/plain; charset=gbk,utf-8' }) + res.writeHead(200, { 'content-type': `text/plain; charset=${charset}` }) + res.end(Buffer.from(bytes)) + } + await expect(provider().fetch({ url: base })).rejects.toThrow(expect.objectContaining({ + code: 'WEB_UNSUPPORTED_CONTENT_TYPE', + message: 'response body is not valid UTF-8 for the malformed charset declaration', + })) + }) + + it.each(['gb2312,gbk', 'windows-1252,gbk'])('rejects a legacy-only compound declaration: %s', async (charset) => { + handler = (_req, res) => { + res.writeHead(200, { 'content-type': `text/plain; charset=${charset}` }) res.end(Buffer.from([0xD6, 0xD0, 0xCE, 0xC4])) } - const result = await provider().fetch({ url: base }) - expect(result.body.content).toBe('中文') + await expect(provider().fetch({ url: base })).rejects.toThrow(expect.objectContaining({ + code: 'WEB_UNSUPPORTED_CONTENT_TYPE', message: `unsupported charset "${charset}"`, + })) }) - it('rejects a comma-joined charset whose labels decode none of the body', async () => { + it('keeps single-label GBK decoding', async () => { + handler = (_req, res) => { + res.writeHead(200, { 'content-type': 'text/plain; charset=gbk' }) + res.end(Buffer.from([0xD6, 0xD0, 0xCE, 0xC4])) + } + expect((await provider().fetch({ url: base })).body.content).toBe('中文') + }) + + it('keeps replacement decoding for a single declared charset', async () => { + handler = (_req, res) => { + res.writeHead(200, { 'content-type': 'text/plain; charset=utf-8' }) + res.end(Buffer.from([0xE4, 0xB8])) + } + expect((await provider().fetch({ url: base })).body.content).toBe('\uFFFD') + }) + + it('omits an incomplete UTF-8 suffix only when the byte cap cuts the response', async () => { handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain; charset=gbk,utf-8' }) - res.end(Buffer.from([0x81, 0x20])) + res.end('中文') } - await expect(provider().fetch({ url: base })) + const result = await provider({ maxResponseBytes: 5 }).fetch({ url: base }) + expect(result.body.content).toBe('中') + expect(result.truncated).toBe(true) + }) + + it('validates EOF when the response exactly fills the byte cap', async () => { + handler = (_req, res) => { + res.writeHead(200, { 'content-type': 'text/plain; charset=gbk,utf-8' }) + res.end(Buffer.from([0xE4, 0xB8])) + } + await expect(provider({ maxResponseBytes: 2 }).fetch({ url: base })) .rejects.toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) }) + + it('rejects invalid UTF-8 before the truncated suffix', async () => { + handler = (_req, res) => { + res.writeHead(200, { 'content-type': 'text/plain; charset=gbk,utf-8' }) + res.end(Buffer.from([0x81, 0x20, 0x61])) + } + await expect(provider({ maxResponseBytes: 2 }).fetch({ url: base })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) + }) + + it('uses UTF-8 even when a long body also forms valid GBK', async () => { + // C2 A9 represents © in UTF-8 and 漏 in GBK; length cannot disambiguate it. + handler = (_req, res) => { + res.writeHead(200, { 'content-type': 'text/plain; charset=gbk,utf-8' }) + res.end('©'.repeat(1000)) + } + expect((await provider().fetch({ url: base })).body.content).toBe('©'.repeat(1000)) + }) }) describe('HttpFetchProvider redirects', () => { @@ -410,7 +449,6 @@ describe('HttpFetchProvider invalid URLs and abort', () => { await expect(provider().fetch({ url: 'http://127.0.0.1:1/' })) .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) }) - }) describe('HttpFetchProvider body cancellation on error paths', () => { @@ -432,8 +470,8 @@ describe('HttpFetchProvider body cancellation on error paths', () => { await closed }) - it('cancels the body when an unsupported charset is rejected', async () => { - const closed = respondWithOpenBody(200, { 'content-type': 'text/plain; charset=not-a-charset' }) + it.each(['not-a-charset', 'gb2312,gbk', 'windows-1252,gbk'])('cancels the body when charset %s is rejected', async (charset) => { + const closed = respondWithOpenBody(200, { 'content-type': `text/plain; charset=${charset}` }) await expect(provider().fetch({ url: base })) .rejects.toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) await closed diff --git a/snapshots/session/web-fetch/web-fetch-fixture-server.mjs b/snapshots/session/web-fetch/web-fetch-fixture-server.mjs index f580910d03..84fab269dd 100644 --- a/snapshots/session/web-fetch/web-fetch-fixture-server.mjs +++ b/snapshots/session/web-fetch/web-fetch-fixture-server.mjs @@ -1,6 +1,6 @@ /** * Deterministic HTTP provider for the web-fetch snapshot scenario: a small - * HTML page (headings, named entities, a GFM table, nested formatting) on a + * UTF-8 HTML page with a malformed charset declaration on an * OS-assigned loopback port behind the real HTTP transport. Recording * and replay therefore exercise fetch and markdown rendering without external * network while retaining the recorded request URL. @@ -14,10 +14,10 @@ const RECORDED_URL = 'http://public.test:43117/menu.html' const PAGE = ` Menu -

Café menu

+

Café menu

Prices include service & tax — updated daily.

-
DrinkPrice
Espresso€2
Flat white€3
+
DrinkPrice
Espresso€2
Flat white€3

See today’s specials.

` @@ -62,7 +62,7 @@ export async function apply(ctx) { label: 'web-fetch-fixture-server', requestListener: (req, res) => { if (req.url === '/menu.html') { - res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }) + res.writeHead(200, { 'content-type': 'text/html; charset=gbk,utf-8' }) res.end(PAGE) return } From b062e52bf6b624dc48894752250e65faf9df523c Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Mon, 26 Oct 2026 15:05:10 +0800 Subject: [PATCH 3/3] test(web): strengthen charset recovery boundary coverage --- packages/web/web-fetch-http/README.i18n.yaml | 12 +++++----- packages/web/web-fetch-http/README.md | 6 ++--- packages/web/web-fetch-http/README.zh.md | 6 ++--- .../web-fetch-http/tests/fetch-http.spec.ts | 24 ++++++++++++------- 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/packages/web/web-fetch-http/README.i18n.yaml b/packages/web/web-fetch-http/README.i18n.yaml index 073d491a38..e38ab6fa9e 100644 --- a/packages/web/web-fetch-http/README.i18n.yaml +++ b/packages/web/web-fetch-http/README.i18n.yaml @@ -27,11 +27,11 @@ en: 2d18cdcc9b53dd0f zh: 3bc6ca7c5933ee5d /deepseek-ai-dsh-web-fetch-http/use-this-package/transport-behavior: - en: 3e54fca1e3ee5f12 - zh: 1f284264ce8a2a54 + en: 4c5618124337446e + zh: 45c7378c8b0bba45 /deepseek-ai-dsh-web-fetch-http/use-this-package/failures-and-recovery: - en: 35407f106dc65deb - zh: 02118072fba68b8f + en: 34095d9d4231d73a + zh: be0ed903adbb124f /deepseek-ai-dsh-web-fetch-http/understand-the-implementation: en: 9842c69ad3636bb3 zh: 3cdc3f816cbdd0cb @@ -54,8 +54,8 @@ en: 40294c1c31b74824 zh: d75d48f485447a74 /deepseek-ai-dsh-web-fetch-http/known-limitations-and-deferred-work: - en: e2c71fbfe37d473a - zh: 987e4c667d24d16c + en: 74d63a0c0e7649e1 + zh: 73a59fcca72daba5 /deepseek-ai-dsh-web-fetch-http/known-limitations-and-deferred-work/dev-note: en: e67f633dc6fe7773 zh: 3bb0c20c93f16f98 diff --git a/packages/web/web-fetch-http/README.md b/packages/web/web-fetch-http/README.md index 8ee1c951ad..4aecd7b649 100644 --- a/packages/web/web-fetch-http/README.md +++ b/packages/web/web-fetch-http/README.md @@ -63,11 +63,11 @@ const page = await ctx.web.fetch({ url: 'https://example.com' }) The provider keeps requests anonymous and bounded: it accepts only `http:` and `https:` URLs without embedded credentials and rejects URLs over 2,048 characters. Requests go through the process-wide dispatcher, so the installed proxy policy applies and the system resolver chooses the address; any destination address is allowed. Same-origin redirects are followed; cross-origin redirects fail and require a fresh call. The provider also enforces byte, character, hop, and time caps, rejects unsupported content types, and sends an explicit product `User-Agent`. -A malformed comma- or whitespace-joined charset declaration containing a UTF-8 label (such as `charset=utf-8,gbk`) is decoded strictly as UTF-8. Invalid UTF-8 fails with `WEB_UNSUPPORTED_CONTENT_TYPE`; the provider does not try another encoding. When the byte cap cuts a character, recovery omits the incomplete suffix and returns `truncated: true`; a complete response must pass the EOF check. Valid single-label declarations retain their ordinary decoding behavior. +A malformed comma- or whitespace-joined charset declaration containing a UTF-8 label (such as `charset=utf-8,gbk`) is decoded strictly as UTF-8. Recovery rejects invalid UTF-8 without trying another encoding. When the byte cap cuts a character, recovery omits the incomplete suffix and returns `truncated: true`; a complete response using recovery must pass the EOF check. Valid single-label declarations retain ordinary replacement decoding. ### Failures and recovery -Failures throw `WebError` with a machine-routable code: `WEB_INVALID_URL`, `WEB_BLOCKED_URL`, `WEB_FETCH_TOO_LARGE`, `WEB_FETCH_TIMEOUT`, `WEB_REDIRECT_BLOCKED`, `WEB_UNSUPPORTED_CONTENT_TYPE`, `WEB_ABORTED`, or `WEB_PROVIDER_ERROR`. Direct callers can route on the code; the model-facing `web_fetch` tool surfaces the failure text to the model under its own error wrapper. +Failures throw `WebError` with a machine-routable code: `WEB_INVALID_URL`, `WEB_BLOCKED_URL`, `WEB_FETCH_TOO_LARGE`, `WEB_FETCH_TIMEOUT`, `WEB_REDIRECT_BLOCKED`, `WEB_UNSUPPORTED_CONTENT_TYPE`, `WEB_ABORTED`, or `WEB_PROVIDER_ERROR`. `WEB_UNSUPPORTED_CONTENT_TYPE` covers unsupported media types, unsupported charset declarations, and invalid UTF-8 during malformed-declaration recovery. Direct callers can route on the code; the model-facing `web_fetch` tool surfaces the failure text to the model under its own error wrapper. ----- @@ -134,7 +134,7 @@ These limits define when the provider is unsafe or a poor fit. They are current - **Private destinations are reachable** — loopback, private-network, and cloud metadata addresses are fetched like any other host. A deployment that must keep the model away from them needs a `tools/pre-execute` policy, network confinement, or no fetch tool. - **Only textual content decodes** — html/xhtml and `text/*` plus JSON/XML families; a missing `Content-Type` or any binary type throws `WEB_UNSUPPORTED_CONTENT_TYPE`, and text-extractable PDF decoding is named deferred work. -- **Charset comes only from the `Content-Type` header** (UTF-8 default) — an HTML `` declaration is ignored, and unsupported declarations without a UTF-8 recovery label throw. +- **Charset comes only from the `Content-Type` header** (UTF-8 default) — an HTML `` declaration is ignored, and unsupported declarations without a UTF-8 label throw. - **UTF-8 recovery is a preference, not encoding detection** — bytes can be valid as both UTF-8 and another encoding, even in a long response. Recovery chooses UTF-8 in that case; malformed declarations without a UTF-8 label remain unsupported. diff --git a/packages/web/web-fetch-http/README.zh.md b/packages/web/web-fetch-http/README.zh.md index 895d3f1094..ba4092e23a 100644 --- a/packages/web/web-fetch-http/README.zh.md +++ b/packages/web/web-fetch-http/README.zh.md @@ -63,11 +63,11 @@ const page = await ctx.web.fetch({ url: 'https://example.com' }) 提供方保持请求匿名且有界：只接受不含内嵌凭据且不超过 2,048 个字符的 `http:` 与 `https:` URL。请求经过进程级 dispatcher，因此已安装的代理策略生效，目标地址由系统解析器选择；任何目标地址都被允许。同源重定向会被跟随；跨源重定向会失败并要求重新调用。提供方还强制执行字节、字符、跳数和时间上限，拒绝不支持的内容类型，并发送显式产品 `User-Agent`。 -对于以逗号或空白拼接、且含有 UTF-8 标签的畸形 charset 声明（例如 `charset=utf-8,gbk`），提供方严格按 UTF-8 解码。非法 UTF-8 以 `WEB_UNSUPPORTED_CONTENT_TYPE` 失败，提供方不会尝试其他编码。字节上限在字符内部截断时，恢复路径省略不完整的末尾序列并返回 `truncated: true`；完整响应必须通过 EOF 检查。合法的单标签声明保持原有的常规解码行为。 +对于以逗号或空白拼接、且含有 UTF-8 标签的畸形 charset 声明（例如 `charset=utf-8,gbk`），提供方严格按 UTF-8 解码。恢复路径拒绝非法 UTF-8，不会尝试其他编码。字节上限在字符内部截断时，恢复路径省略不完整的末尾序列并返回 `truncated: true`；使用恢复路径的完整响应必须通过 EOF 检查。合法的单标签声明仍在遇到非法字节时使用替换字符。 ### 失败与恢复 -失败会抛出 `WebError`，其中包含可供程序路由的错误码：`WEB_INVALID_URL`、`WEB_BLOCKED_URL`、`WEB_FETCH_TOO_LARGE`、`WEB_FETCH_TIMEOUT`、`WEB_REDIRECT_BLOCKED`、`WEB_UNSUPPORTED_CONTENT_TYPE`、`WEB_ABORTED` 或 `WEB_PROVIDER_ERROR`。直接调用方可以按错误码路由；面向模型的 `web_fetch` 工具会在自己的错误包装层内把失败文本呈现给模型。 +失败会抛出 `WebError`，其中包含可供程序路由的错误码：`WEB_INVALID_URL`、`WEB_BLOCKED_URL`、`WEB_FETCH_TOO_LARGE`、`WEB_FETCH_TIMEOUT`、`WEB_REDIRECT_BLOCKED`、`WEB_UNSUPPORTED_CONTENT_TYPE`、`WEB_ABORTED` 或 `WEB_PROVIDER_ERROR`。`WEB_UNSUPPORTED_CONTENT_TYPE` 涵盖不支持的媒体类型、不支持的 charset 声明，以及畸形声明恢复过程中遇到的非法 UTF-8。直接调用方可以按错误码路由；面向模型的 `web_fetch` 工具会在自己的错误包装层内把失败文本呈现给模型。 ----- @@ -134,7 +134,7 @@ const page = await ctx.web.fetch({ url: 'https://example.com' }) - **可以访问私有目标**——loopback、内网与云元数据地址与其他主机一样会被抓取。需要阻止模型访问这些地址的部署必须添加 `tools/pre-execute` 策略、网络隔离，或不挂载抓取工具。 - **只解码文本内容**——包括 html/xhtml 与 `text/*` 加 JSON/XML 家族；缺少 `Content-Type` 或任何二进制类型都会抛出 `WEB_UNSUPPORTED_CONTENT_TYPE`，可提取文本的 PDF 解码属于明确的延期工作。 -- **charset 只来自 `Content-Type` 标头**（默认 UTF-8）——HTML `` 声明会被忽略；不支持且不含 UTF-8 恢复标签的声明会抛出异常。 +- **charset 只来自 `Content-Type` 标头**（默认 UTF-8）——HTML `` 声明会被忽略；不支持且不含 UTF-8 标签的声明会抛出异常。 - **UTF-8 恢复是一种优先规则，不是编码检测**——同一组字节可能同时是合法的 UTF-8 和其他编码，长响应也不例外。恢复路径在这种情况下选择 UTF-8；不含 UTF-8 标签的畸形声明仍不受支持。 diff --git a/packages/web/web-fetch-http/tests/fetch-http.spec.ts b/packages/web/web-fetch-http/tests/fetch-http.spec.ts index 622788d573..c66876cc5b 100644 --- a/packages/web/web-fetch-http/tests/fetch-http.spec.ts +++ b/packages/web/web-fetch-http/tests/fetch-http.spec.ts @@ -250,14 +250,18 @@ describe('HttpFetchProvider charset recovery', () => { expect((await provider().fetch({ url: base })).body.content).toBe('\uFFFD') }) - it('omits an incomplete UTF-8 suffix only when the byte cap cuts the response', async () => { + it.each([ + ['中文', 5, true], + ['中文', 3, true], + ['中', 3, false], + ] as const)('returns complete UTF-8 characters for %j with byte cap %i (truncated: %s)', async (body, maxResponseBytes, truncated) => { handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain; charset=gbk,utf-8' }) - res.end('中文') + res.end(body) } - const result = await provider({ maxResponseBytes: 5 }).fetch({ url: base }) + const result = await provider({ maxResponseBytes }).fetch({ url: base }) expect(result.body.content).toBe('中') - expect(result.truncated).toBe(true) + expect(result.truncated).toBe(truncated) }) it('validates EOF when the response exactly fills the byte cap', async () => { @@ -265,8 +269,10 @@ describe('HttpFetchProvider charset recovery', () => { res.writeHead(200, { 'content-type': 'text/plain; charset=gbk,utf-8' }) res.end(Buffer.from([0xE4, 0xB8])) } - await expect(provider({ maxResponseBytes: 2 }).fetch({ url: base })) - .rejects.toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) + await expect(provider({ maxResponseBytes: 2 }).fetch({ url: base })).rejects.toThrow(expect.objectContaining({ + code: 'WEB_UNSUPPORTED_CONTENT_TYPE', + message: 'response body is not valid UTF-8 for the malformed charset declaration', + })) }) it('rejects invalid UTF-8 before the truncated suffix', async () => { @@ -274,8 +280,10 @@ describe('HttpFetchProvider charset recovery', () => { res.writeHead(200, { 'content-type': 'text/plain; charset=gbk,utf-8' }) res.end(Buffer.from([0x81, 0x20, 0x61])) } - await expect(provider({ maxResponseBytes: 2 }).fetch({ url: base })) - .rejects.toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) + await expect(provider({ maxResponseBytes: 2 }).fetch({ url: base })).rejects.toThrow(expect.objectContaining({ + code: 'WEB_UNSUPPORTED_CONTENT_TYPE', + message: 'response body is not valid UTF-8 for the malformed charset declaration', + })) }) it('uses UTF-8 even when a long body also forms valid GBK', async () => {