diff --git a/packages/web/web-fetch-http/README.i18n.yaml b/packages/web/web-fetch-http/README.i18n.yaml index 5315b70e40..c342205b78 100644 --- a/packages/web/web-fetch-http/README.i18n.yaml +++ b/packages/web/web-fetch-http/README.i18n.yaml @@ -27,8 +27,8 @@ en: 2d18cdcc9b53dd0f zh: 3bc6ca7c5933ee5d /deepseek-ai-dsh-web-fetch-http/use-this-package/transport-behavior: - en: c432147daf2877cf - zh: c14d507dd66e124a + en: 2c2dc6d5bc6d9e83 + zh: 2dba700520be5a9e /deepseek-ai-dsh-web-fetch-http/use-this-package/failures-and-recovery: en: 35407f106dc65deb zh: 02118072fba68b8f @@ -54,8 +54,8 @@ en: 40294c1c31b74824 zh: d75d48f485447a74 /deepseek-ai-dsh-web-fetch-http/known-limitations-and-deferred-work: - en: 4d935fd8f466a234 - zh: c398eb7aa737f5ea + en: 25ad864700ba1bf2 + zh: 1771da9930708906 /deepseek-ai-dsh-web-fetch-http/known-limitations-and-deferred-work/dev-note: en: e67f633dc6fe7773 zh: 3bb0c20c93f16f98 diff --git a/packages/web/web-fetch-http/README.md b/packages/web/web-fetch-http/README.md index ab2e82d179..c92f971177 100644 --- a/packages/web/web-fetch-http/README.md +++ b/packages/web/web-fetch-http/README.md @@ -61,7 +61,7 @@ const page = await ctx.web.fetch({ url: 'https://example.com' }) ### Transport behavior -The provider keeps requests anonymous and bounded: it accepts only `http:` and `https:` URLs without embedded credentials and rejects URLs over 2,048 characters. It resolves each hostname once, rejects the complete result if any IPv4 or IPv6 address is not public unicast, and pins the connection to that validated set. IPv6 checks discover the active DNS64 prefix and reject translations to non-public IPv4. Each same-origin redirect repeats resolution and pinning; cross-origin redirects fail and require a fresh call. The provider also enforces byte, character, hop, and time caps, rejects unsupported content types, and sends an explicit product `User-Agent`. +The provider keeps requests anonymous and bounded: it accepts only `http:` and `https:` URLs without embedded credentials and rejects URLs over 2,048 characters. It resolves each hostname once, rejects the complete result if any IPv4 or IPv6 address is not public unicast, and pins the connection to that validated set. A hostname answer in `198.18.0.0/15`, the range fake-IP DNS of TUN-mode proxies such as Clash returns for every name, counts as public because the proxy resolves the real host; an IP literal in that range stays blocked. IPv6 checks discover the active DNS64 prefix and reject translations to non-public IPv4. Each same-origin redirect repeats resolution and pinning; cross-origin redirects fail and require a fresh call. The provider also enforces byte, character, hop, and time caps, rejects unsupported content types, and sends an explicit product `User-Agent`. ### Failures and recovery @@ -132,6 +132,7 @@ No direct invalidation; the named consumer owns any request-prefix changes. These limits define when the provider is unsafe or a poor fit. They are current package constraints. - **Only textual content decodes** — html/xhtml and `text/*` plus JSON/XML families; a missing `Content-Type` or any binary type throws `WEB_UNSUPPORTED_CONTENT_TYPE`, and text-extractable PDF decoding is named deferred work. +- **Fake-IP answers delegate address checks to the proxy** — under a TUN-mode fake-IP proxy every hostname resolves into `198.18.0.0/15`, so the provider cannot see whether the proxy's own resolution reaches a non-public host, the same limit that applies to a hop routed through `HTTP_PROXY`. - **Charset comes only from the `Content-Type` header** (UTF-8 default) — an HTML `` declaration is ignored, and a declared-but-unrecognized charset label throws rather than falling back. diff --git a/packages/web/web-fetch-http/README.zh.md b/packages/web/web-fetch-http/README.zh.md index 2ae10c4b7d..c2e48b0eca 100644 --- a/packages/web/web-fetch-http/README.zh.md +++ b/packages/web/web-fetch-http/README.zh.md @@ -61,7 +61,7 @@ const page = await ctx.web.fetch({ url: 'https://example.com' }) ### 传输行为 -提供方保持请求匿名且有界:只接受不含内嵌凭据且不超过 2,048 个字符的 `http:` 与 `https:` URL。它只解析一次主机名;只要结果中有任何 IPv4 或 IPv6 地址不是公共单播地址,就拒绝整个结果,并把连接固定到已校验的地址集合。IPv6 检查会发现活动 DNS64 前缀,并拒绝指向非公开 IPv4 的转换地址。每次同源重定向都会重复解析与固定;跨源重定向会失败并要求重新调用。提供方还强制执行字节、字符、跳数和时间上限,拒绝不支持的内容类型,并发送显式产品 `User-Agent`。 +提供方保持请求匿名且有界:只接受不含内嵌凭据且不超过 2,048 个字符的 `http:` 与 `https:` URL。它只解析一次主机名;只要结果中有任何 IPv4 或 IPv6 地址不是公共单播地址,就拒绝整个结果,并把连接固定到已校验的地址集合。主机名解析到 `198.18.0.0/15`(Clash 等 TUN 模式代理的 fake-IP DNS 为所有域名返回的地址段)时视为公开地址,因为真实主机由代理解析;该地址段的 IP 字面量仍被拒绝。IPv6 检查会发现活动 DNS64 前缀,并拒绝指向非公开 IPv4 的转换地址。每次同源重定向都会重复解析与固定;跨源重定向会失败并要求重新调用。提供方还强制执行字节、字符、跳数和时间上限,拒绝不支持的内容类型,并发送显式产品 `User-Agent`。 ### 失败与恢复 @@ -132,6 +132,7 @@ const page = await ctx.web.fetch({ url: 'https://example.com' }) 这些限制说明提供方何时不安全或不合适。它们是当前包约束。 - **只解码文本内容**——包括 html/xhtml 与 `text/*` 加 JSON/XML 家族;缺少 `Content-Type` 或任何二进制类型都会抛出 `WEB_UNSUPPORTED_CONTENT_TYPE`,可提取文本的 PDF 解码属于明确的延期工作。 +- **fake-IP 应答把地址检查交给代理**——在 TUN 模式 fake-IP 代理下,所有主机名都解析到 `198.18.0.0/15`,提供方无法得知代理自身的解析是否到达非公开主机;经 `HTTP_PROXY` 路由的请求也有同样限制。 - **charset 只来自 `Content-Type` 标头**(默认 UTF-8)——HTML `` 声明会被忽略;声明但无法识别的 charset 标签会抛出异常,而非回退。 diff --git a/packages/web/web-fetch-http/src/network.ts b/packages/web/web-fetch-http/src/network.ts index f14bf276b4..f9ea77ca1d 100644 --- a/packages/web/web-fetch-http/src/network.ts +++ b/packages/web/web-fetch-http/src/network.ts @@ -37,6 +37,12 @@ export type AddressResolver = (hostname: string, options: { all: true; order: 'v const RFC6052_PREFIX_LENGTHS = [32, 40, 48, 56, 64, 96] as const const IPV4ONLY_DISCOVERY_HOST = 'ipv4only.arpa' const IPV4ONLY_SENTINELS = new Set(['192.0.0.170', '192.0.0.171']) +/** + * RFC 2544 benchmarking block that fake-IP DNS in TUN-mode proxies (Clash/Mihomo, Surge, sing-box) + * returns for every hostname. The proxy maps the address back to the hostname and resolves it, so a + * DNS answer in this block is treated like a proxied hop; an IP literal in it stays blocked. + */ +const FAKE_IP_RANGE = ipaddr.parseCIDR('198.18.0.0/15') interface Nat64Prefix { readonly bytes: readonly number[] @@ -65,7 +71,8 @@ export function isPublicIpAddress(input: string): boolean { /** * Resolve a hostname once and reject the complete answer set if any destination - * is not public. The returned addresses are the only ones the transport may use. + * is not public. A hostname's answer in the fake-IP block counts as public. The returned addresses + * are the only ones the transport may use. * * @param hostname - URL hostname, including brackets when it is an IPv6 literal. * @param signal - aborts the wait for system resolution; an in-flight OS lookup may finish unused. @@ -97,7 +104,7 @@ export async function resolvePublicAddresses( if ((entry.family !== 4 && entry.family !== 6) || isIP(entry.address) !== entry.family) { throw new WebError(`hostname "${hostname}" resolved to an invalid IP address`, 'WEB_PROVIDER_ERROR') } - if (!isPublicIpAddress(entry.address)) { + if (!isPublicIpAddress(entry.address) && !(literalFamily === 0 && isFakeIpAddress(entry.address))) { throw new WebError(`URL hostname "${hostname}" resolves to a non-public IP address`, 'WEB_BLOCKED_URL') } const translatedIpv4 = translatedIpv4Address(entry.address, nat64Prefixes) @@ -109,6 +116,12 @@ export async function resolvePublicAddresses( return addresses } +/** Whether an IPv4 address is in {@link FAKE_IP_RANGE}. */ +function isFakeIpAddress(address: string): boolean { + const parsed = ipaddr.parse(address) + return parsed instanceof ipaddr.IPv4 && parsed.match(FAKE_IP_RANGE) +} + /** Discover the active DNS64 prefix set using RFC 7050's reserved hostname. */ async function discoverNat64Prefixes(signal: AbortSignal, resolver: AddressResolver): Promise { const discovered = await raceWithSignal( diff --git a/packages/web/web-fetch-http/tests/fetch-http.spec.ts b/packages/web/web-fetch-http/tests/fetch-http.spec.ts index 1a134f200f..2ad21e75c8 100644 --- a/packages/web/web-fetch-http/tests/fetch-http.spec.ts +++ b/packages/web/web-fetch-http/tests/fetch-http.spec.ts @@ -142,6 +142,15 @@ describe('public-network policy', () => { .rejects.toThrow(expect.objectContaining({ code: 'WEB_BLOCKED_URL' })) }) + it('accepts a fake-IP DNS answer for a hostname but not the same address as a literal', async () => { + const resolver = vi.fn(async () => [{ address: '198.18.0.138', family: 4 }]) + await expect(resolvePublicAddresses('www.example.com', new AbortController().signal, resolver)) + .resolves.toEqual([{ address: '198.18.0.138', family: 4 }]) + await expect(resolvePublicAddresses('198.19.255.1', new AbortController().signal, resolver)) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_BLOCKED_URL' })) + expect(resolver).toHaveBeenCalledOnce() + }) + it('rejects empty and invalid resolver results', async () => { await expect(resolvePublicAddresses('empty.test', new AbortController().signal, async () => [])) .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' }))