test(llm): archive recorded provider model listings

Replies captured on 2026-09-02 from OpenRouter, models.dev, and DeepSeek replay through the discovery parser, and OpenRouter's nested top_provider.max_completion_tokens now feeds the output-token cap.
2026-09-02 21:55:06 +08:00
1 parent 8179d929ab
commit 31065cf1ab
@@ -2,5 +2,5 @@
 # side as of the last confirmed-consistent state. Both languages carry equal authority;
 # after editing either side, bring the other along and re-record with:
 #   pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.md
-2026-09-02-protocol-specific-model-listing-discovery.md: 187c27cc997f1dd8b4d57bc72589c314c9be0537
-2026-09-02-protocol-specific-model-listing-discovery.zh.md: 176a1493986c84e35af2a704fc28cb06535d297e
+2026-09-02-protocol-specific-model-listing-discovery.md: 9e14f0ca82f74c2e428c97cb4d6303a1415c9fec
+2026-09-02-protocol-specific-model-listing-discovery.zh.md: 28c43cb5f6e35198a007cb820a460aa862965c93
@@ -40,4 +40,4 @@ The supported formats remain an explicit compatibility set rather than arbitrary

 ## Testing

-Local HTTP-server tests pin both accepted response formats, field normalization, name fallback, ignored malformed entries, Anthropic headers, the maximum-page query, and both spellings of the Anthropic root.
+Local HTTP-server tests pin both accepted response formats, field normalization, name fallback, ignored malformed entries, Anthropic headers, the maximum-page query, and both spellings of the Anthropic root. Replies recorded from OpenRouter, models.dev, and DeepSeek on 2026-09-02 live under `packages/llm/llm-pi-ai/tests/fixtures/model-listings/` and replay through the parser, so the accepted field spellings are pinned to real endpoints rather than to hand-written samples.
@@ -40,4 +40,4 @@ Models 页面无需改变请求身份，即可询问 OpenAI 兼容网关与 Anth

 ## 测试

-本地 HTTP 服务器测试钉住两种受支持响应格式、字段归一化、名称回退、忽略畸形条目、Anthropic 标头、最大页查询，以及 Anthropic 根地址的两种写法。
+本地 HTTP 服务器测试钉住两种受支持响应格式、字段归一化、名称回退、忽略畸形条目、Anthropic 标头、最大页查询，以及 Anthropic 根地址的两种写法。2026-09-02 从 OpenRouter、models.dev 与 DeepSeek 录得的回复存放在 `packages/llm/llm-pi-ai/tests/fixtures/model-listings/` 下并经解析器回放，因此受支持的字段拼写钉在真实端点上，而不是手写样例上。
@@ -63,6 +63,11 @@ interface ListingLimit {
  output?: unknown
 }

+/** Per-route capacities OpenRouter nests under each entry. */
+interface ListingTopProvider {
+  max_completion_tokens?: unknown
+}
+
 /** One entry of a supported `GET /models` reply. */
 interface ListingEntry {
  id?: unknown
@@ -79,6 +84,7 @@ interface ListingEntry {
  max_output_tokens?: unknown
  maxTokens?: unknown
  limit?: ListingLimit | null
+  top_provider?: ListingTopProvider | null
 }

 /** A positive integer field of a listing entry, or `undefined` when absent or unusable. */
@@ -211,6 +217,7 @@ function readListing(body: unknown): LlmDiscoveredModel[] {
      entry?.maxTokens,
      entry?.max_tokens,
      entry?.limit?.output,
+      entry?.top_provider?.max_completion_tokens,
    )
    models.push({
      id,
@@ -1,3 +1,4 @@
+import { readFile } from 'node:fs/promises'
 import { createServer } from 'node:http'
 import type { IncomingMessage, Server, ServerResponse } from 'node:http'
 import { afterEach, describe, expect, it, vi } from 'vitest'
@@ -482,3 +483,50 @@ describe('probe key format', () => {
    expect(headers.has('authorization')).toBe(false)
  })
 })
+
+/**
+ * Replies recorded from live endpoints on 2026-09-02. Each file keeps the
+ * reply's top-level fields and entry objects verbatim; only the entry list is
+ * cut down to the named entries so the archive stays small.
+ */
+const RECORDED_LISTINGS = [
+  {
+    name: 'OpenRouter GET /api/v1/models',
+    file: 'openrouter-2026-09-02.json',
+    models: [
+      { id: 'anthropic/claude-fable-5.1', name: 'Anthropic: Claude Fable 5.1', contextWindow: 1_000_000, maxTokens: 128_000 },
+      // The router's own aggregate route reports no completion cap.
+      { id: 'openrouter/auto-beta', name: 'Auto Router (Beta)', contextWindow: 2_000_000 },
+      { id: 'deepseek/deepseek-v4-flash', name: 'DeepSeek: DeepSeek V4 Flash 0423', contextWindow: 1_048_576, maxTokens: 384_000 },
+    ],
+  },
+  {
+    name: 'the models.dev anthropic provider object',
+    file: 'models-dev-anthropic-2026-09-02.json',
+    models: [
+      { id: 'claude-opus-4-7', name: 'Claude Opus 4.7', contextWindow: 1_000_000, maxTokens: 128_000 },
+      { id: 'claude-fable-5-1', name: 'Claude Fable 5.1', contextWindow: 1_000_000, maxTokens: 128_000 },
+      { id: 'claude-haiku-4-5', name: 'Claude Haiku 4.5 (latest)', contextWindow: 200_000, maxTokens: 64_000 },
+    ],
+  },
+  {
+    name: 'DeepSeek GET /models',
+    file: 'deepseek-2026-09-02.json',
+    models: [
+      { id: 'deepseek-v4-flash', name: 'deepseek-v4-flash' },
+      { id: 'deepseek-v4-pro', name: 'deepseek-v4-pro' },
+      { id: 'deepseek-v4-flash-vision-exp', name: 'deepseek-v4-flash-vision-exp' },
+    ],
+  },
+]
+
+describe('recorded provider listings', () => {
+  it.each(RECORDED_LISTINGS)('reads $name as recorded', async ({ file, models }) => {
+    const body = await readFile(new URL(`./fixtures/model-listings/${file}`, import.meta.url), 'utf8')
+    const server = await listingServer({ body })
+    const ctx = await harness()
+
+    await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: server.url, api: 'openai-completions' }))
+      .resolves.toEqual(models)
+  })
+})
@@ -0,0 +1,20 @@
+{
+  "object": "list",
+  "data": [
+    {
+      "id": "deepseek-v4-flash",
+      "object": "model",
+      "owned_by": "deepseek"
+    },
+    {
+      "id": "deepseek-v4-pro",
+      "object": "model",
+      "owned_by": "deepseek"
+    },
+    {
+      "id": "deepseek-v4-flash-vision-exp",
+      "object": "model",
+      "owned_by": "deepseek"
+    }
+  ]
+}
@@ -0,0 +1,146 @@
+{
+  "id": "anthropic",
+  "env": [
+    "ANTHROPIC_API_KEY"
+  ],
+  "npm": "@ai-sdk/anthropic",
+  "name": "Anthropic",
+  "doc": "https://docs.anthropic.com/en/docs/about-claude/models",
+  "models": {
+    "claude-opus-4-7": {
+      "id": "claude-opus-4-7",
+      "name": "Claude Opus 4.7",
+      "description": "Stronger Opus tier for advanced software work and high-stakes reasoning",
+      "family": "claude-opus",
+      "attachment": true,
+      "reasoning": true,
+      "reasoning_options": [
+        {
+          "type": "effort",
+          "values": [
+            "low",
+            "medium",
+            "high",
+            "xhigh",
+            "max"
+          ]
+        }
+      ],
+      "tool_call": true,
+      "structured_output": true,
+      "temperature": false,
+      "knowledge": "2026-01-31",
+      "release_date": "2026-04-14",
+      "last_updated": "2026-04-16",
+      "modalities": {
+        "input": [
+          "text",
+          "image",
+          "pdf"
+        ],
+        "output": [
+          "text"
+        ]
+      },
+      "open_weights": false,
+      "limit": {
+        "context": 1000000,
+        "output": 128000
+      },
+      "cost": {
+        "input": 5,
+        "output": 25,
+        "cache_read": 0.5,
+        "cache_write": 6.25
+      }
+    },
+    "claude-fable-5-1": {
+      "id": "claude-fable-5-1",
+      "name": "Claude Fable 5.1",
+      "description": "Claude model for demanding reasoning and long-horizon agentic work",
+      "family": "claude-fable",
+      "attachment": true,
+      "reasoning": true,
+      "reasoning_options": [
+        {
+          "type": "effort",
+          "values": [
+            "low",
+            "medium",
+            "high",
+            "xhigh",
+            "max"
+          ]
+        }
+      ],
+      "tool_call": true,
+      "structured_output": true,
+      "temperature": false,
+      "knowledge": "2026-06",
+      "release_date": "2026-09-01",
+      "last_updated": "2026-09-01",
+      "modalities": {
+        "input": [
+          "text",
+          "image",
+          "pdf"
+        ],
+        "output": [
+          "text"
+        ]
+      },
+      "open_weights": false,
+      "limit": {
+        "context": 1000000,
+        "output": 128000
+      },
+      "cost": {
+        "input": 10,
+        "output": 50,
+        "cache_read": 0.25,
+        "cache_write": 12.5
+      }
+    },
+    "claude-haiku-4-5": {
+      "id": "claude-haiku-4-5",
+      "name": "Claude Haiku 4.5 (latest)",
+      "description": "Fast Claude lane for lightweight agents, office tasks, and responsive chat",
+      "family": "claude-haiku",
+      "attachment": true,
+      "reasoning": true,
+      "reasoning_options": [
+        {
+          "type": "budget_tokens",
+          "min": 1024
+        }
+      ],
+      "tool_call": true,
+      "structured_output": true,
+      "temperature": true,
+      "knowledge": "2025-02-28",
+      "release_date": "2025-10-15",
+      "last_updated": "2025-10-15",
+      "modalities": {
+        "input": [
+          "text",
+          "image",
+          "pdf"
+        ],
+        "output": [
+          "text"
+        ]
+      },
+      "open_weights": false,
+      "limit": {
+        "context": 200000,
+        "output": 64000
+      },
+      "cost": {
+        "input": 1,
+        "output": 5,
+        "cache_read": 0.1,
+        "cache_write": 1.25
+      }
+    }
+  }
+}
@@ -0,0 +1,299 @@
+{
+  "data": [
+    {
+      "id": "anthropic/claude-fable-5.1",
+      "canonical_slug": "anthropic/claude-fable-5.1-20260831",
+      "hugging_face_id": null,
+      "name": "Anthropic: Claude Fable 5.1",
+      "created": 1788285838,
+      "description": "Claude Fable 5.1 improves on Claude Fable 5 across the board, with the biggest gains in agentic coding, long-running agentic workflows, and knowledge work: long code refactors, front-end and visual...",
+      "context_length": 1000000,
+      "architecture": {
+        "modality": "text+image+file->text",
+        "input_modalities": [
+          "text",
+          "image",
+          "file"
+        ],
+        "output_modalities": [
+          "text"
+        ],
+        "tokenizer": "Claude",
+        "instruct_type": null
+      },
+      "pricing": {
+        "prompt": "0.00001",
+        "completion": "0.00005",
+        "web_search": "0.01",
+        "input_cache_read": "0.00000025",
+        "input_cache_write": "0.0000125",
+        "input_cache_write_1h": "0.00002"
+      },
+      "top_provider": {
+        "context_length": 1000000,
+        "max_completion_tokens": 128000,
+        "is_moderated": true
+      },
+      "per_request_limits": null,
+      "supported_parameters": [
+        "include_reasoning",
+        "max_completion_tokens",
+        "max_tokens",
+        "reasoning",
+        "reasoning_effort",
+        "response_format",
+        "stop",
+        "structured_outputs",
+        "tools",
+        "verbosity"
+      ],
+      "default_parameters": {},
+      "supported_voices": null,
+      "knowledge_cutoff": null,
+      "expiration_date": null,
+      "links": {
+        "details": "/api/v1/models/anthropic/claude-fable-5.1-20260831/endpoints"
+      },
+      "benchmarks": {
+        "design_arena": [
+          {
+            "arena": "models",
+            "category": "website",
+            "elo": 1351,
+            "win_rate": 58.5,
+            "rank": 2
+          }
+        ],
+        "artificial_analysis": {
+          "intelligence_index": 65.7,
+          "coding_index": 81.6,
+          "agentic_index": 61.3
+        }
+      },
+      "reasoning": {
+        "mandatory": true,
+        "supported_efforts": [
+          "max",
+          "xhigh",
+          "high",
+          "medium",
+          "low"
+        ],
+        "default_effort": "high"
+      }
+    },
+    {
+      "id": "openrouter/auto-beta",
+      "canonical_slug": "openrouter/auto-beta",
+      "hugging_face_id": null,
+      "name": "Auto Router (Beta)",
+      "created": 1784311165,
+      "description": "Auto Router (Beta) is a task-aware router from OpenRouter. It classifies each request, then routes it the [most popular model](/rankings#task-spend) for that task based on aggregate spend, filtered by your...",
+      "context_length": 2000000,
+      "architecture": {
+        "modality": "text+image+file+audio+video->text+image",
+        "input_modalities": [
+          "text",
+          "image",
+          "audio",
+          "file",
+          "video"
+        ],
+        "output_modalities": [
+          "text",
+          "image"
+        ],
+        "tokenizer": "Router",
+        "instruct_type": null
+      },
+      "pricing": {
+        "prompt": "-1",
+        "completion": "-1"
+      },
+      "top_provider": {
+        "context_length": null,
+        "max_completion_tokens": null,
+        "is_moderated": false
+      },
+      "per_request_limits": null,
+      "supported_parameters": [
+        "frequency_penalty",
+        "include_reasoning",
+        "logit_bias",
+        "logprobs",
+        "max_tokens",
+        "min_p",
+        "prediction",
+        "presence_penalty",
+        "reasoning",
+        "reasoning_effort",
+        "repetition_penalty",
+        "response_format",
+        "seed",
+        "stop",
+        "structured_outputs",
+        "temperature",
+        "tool_choice",
+        "tools",
+        "top_a",
+        "top_k",
+        "top_logprobs",
+        "top_p",
+        "web_search_options"
+      ],
+      "default_parameters": {
+        "temperature": null,
+        "top_p": null,
+        "top_k": null,
+        "frequency_penalty": null,
+        "presence_penalty": null,
+        "repetition_penalty": null
+      },
+      "supported_voices": null,
+      "knowledge_cutoff": null,
+      "expiration_date": null,
+      "links": {
+        "details": "/api/v1/models/openrouter/auto-beta/endpoints"
+      }
+    },
+    {
+      "id": "deepseek/deepseek-v4-flash",
+      "canonical_slug": "deepseek/deepseek-v4-flash-20260423",
+      "hugging_face_id": "deepseek-ai/DeepSeek-V4-Flash",
+      "name": "DeepSeek: DeepSeek V4 Flash 0423",
+      "created": 1777000666,
+      "description": "DeepSeek V4 Flash is an efficiency-optimized Mixture-of-Experts model from DeepSeek with 284B total parameters and 13B activated parameters, supporting a 1M-token context window. It is designed for fast inference and...",
+      "context_length": 1048576,
+      "architecture": {
+        "modality": "text->text",
+        "input_modalities": [
+          "text"
+        ],
+        "output_modalities": [
+          "text"
+        ],
+        "tokenizer": "DeepSeek",
+        "instruct_type": null
+      },
+      "pricing": {
+        "prompt": "0.00000008708",
+        "completion": "0.00000017416",
+        "input_cache_read": "0.000000017416"
+      },
+      "top_provider": {
+        "context_length": 1024000,
+        "max_completion_tokens": 384000,
+        "is_moderated": false
+      },
+      "per_request_limits": null,
+      "supported_parameters": [
+        "frequency_penalty",
+        "include_reasoning",
+        "logit_bias",
+        "logprobs",
+        "max_completion_tokens",
+        "max_tokens",
+        "min_p",
+        "presence_penalty",
+        "reasoning",
+        "reasoning_effort",
+        "repetition_penalty",
+        "response_format",
+        "seed",
+        "stop",
+        "structured_outputs",
+        "temperature",
+        "tool_choice",
+        "tools",
+        "top_a",
+        "top_k",
+        "top_logprobs",
+        "top_p"
+      ],
+      "default_parameters": {},
+      "supported_voices": null,
+      "knowledge_cutoff": null,
+      "expiration_date": null,
+      "links": {
+        "details": "/api/v1/models/deepseek/deepseek-v4-flash-20260423/endpoints"
+      },
+      "benchmarks": {
+        "design_arena": [
+          {
+            "arena": "models",
+            "category": "3d",
+            "elo": 1229,
+            "win_rate": 49.3,
+            "rank": 40
+          },
+          {
+            "arena": "models",
+            "category": "asciiart",
+            "elo": 1138,
+            "win_rate": 42.8,
+            "rank": 47
+          },
+          {
+            "arena": "models",
+            "category": "codecategories",
+            "elo": 1226,
+            "win_rate": 48.9,
+            "rank": 43
+          },
+          {
+            "arena": "models",
+            "category": "dataviz",
+            "elo": 1148,
+            "win_rate": 40.6,
+            "rank": 74
+          },
+          {
+            "arena": "models",
+            "category": "gamedev",
+            "elo": 1232,
+            "win_rate": 50.2,
+            "rank": 39
+          },
+          {
+            "arena": "models",
+            "category": "svg",
+            "elo": 1193,
+            "win_rate": 48.4,
+            "rank": 34
+          },
+          {
+            "arena": "models",
+            "category": "uicomponent",
+            "elo": 1188,
+            "win_rate": 44.7,
+            "rank": 58
+          },
+          {
+            "arena": "models",
+            "category": "website",
+            "elo": 1224,
+            "win_rate": 49.1,
+            "rank": 45
+          }
+        ],
+        "artificial_analysis": {
+          "intelligence_index": 42.1,
+          "coding_index": 56.2,
+          "agentic_index": 33.7
+        }
+      },
+      "reasoning": {
+        "mandatory": false,
+        "supported_efforts": [
+          "xhigh",
+          "high"
+        ],
+        "default_effort": "high"
+      }
+    }
+  ],
+  "total_count": 421,
+  "links": {
+    "next": null
+  }
+}