From 31065cf1ab44a6b59bbd5f0024fbea2207bcbf36 Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Wed, 2 Sep 2026 21:55:06 +0800 Subject: [PATCH] test(llm): archive recorded provider model listings Replies captured on 2026-09-02 from OpenRouter, models.dev, and DeepSeek replay through the discovery parser, and OpenRouter's nested top_provider.max_completion_tokens now feeds the output-token cap. --- ...specific-model-listing-discovery.i18n.yaml | 4 +- ...otocol-specific-model-listing-discovery.md | 2 +- ...col-specific-model-listing-discovery.zh.md | 2 +- packages/llm/llm-pi-ai/src/discovery.ts | 7 + .../llm/llm-pi-ai/tests/discovery.spec.ts | 48 +++ .../model-listings/deepseek-2026-09-02.json | 20 ++ .../models-dev-anthropic-2026-09-02.json | 146 +++++++++ .../model-listings/openrouter-2026-09-02.json | 299 ++++++++++++++++++ 8 files changed, 524 insertions(+), 4 deletions(-) create mode 100644 packages/llm/llm-pi-ai/tests/fixtures/model-listings/deepseek-2026-09-02.json create mode 100644 packages/llm/llm-pi-ai/tests/fixtures/model-listings/models-dev-anthropic-2026-09-02.json create mode 100644 packages/llm/llm-pi-ai/tests/fixtures/model-listings/openrouter-2026-09-02.json diff --git a/.agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.i18n.yaml b/.agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.i18n.yaml index d58d8a514b..17118efaa7 100644 --- a/.agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.md -2026-09-02-protocol-specific-model-listing-discovery.md: 187c27cc997f1dd8b4d57bc72589c314c9be0537 -2026-09-02-protocol-specific-model-listing-discovery.zh.md: 176a1493986c84e35af2a704fc28cb06535d297e +2026-09-02-protocol-specific-model-listing-discovery.md: 9e14f0ca82f74c2e428c97cb4d6303a1415c9fec +2026-09-02-protocol-specific-model-listing-discovery.zh.md: 28c43cb5f6e35198a007cb820a460aa862965c93 diff --git a/.agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.md b/.agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.md index 187c27cc99..9e14f0ca82 100644 --- a/.agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.md +++ b/.agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.md @@ -40,4 +40,4 @@ The supported formats remain an explicit compatibility set rather than arbitrary ## Testing -Local HTTP-server tests pin both accepted response formats, field normalization, name fallback, ignored malformed entries, Anthropic headers, the maximum-page query, and both spellings of the Anthropic root. +Local HTTP-server tests pin both accepted response formats, field normalization, name fallback, ignored malformed entries, Anthropic headers, the maximum-page query, and both spellings of the Anthropic root. Replies recorded from OpenRouter, models.dev, and DeepSeek on 2026-09-02 live under `packages/llm/llm-pi-ai/tests/fixtures/model-listings/` and replay through the parser, so the accepted field spellings are pinned to real endpoints rather than to hand-written samples. diff --git a/.agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.zh.md b/.agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.zh.md index 176a149398..28c43cb5f6 100644 --- a/.agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.zh.md +++ b/.agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.zh.md @@ -40,4 +40,4 @@ Models 页面无需改变请求身份，即可询问 OpenAI 兼容网关与 Anth ## 测试 -本地 HTTP 服务器测试钉住两种受支持响应格式、字段归一化、名称回退、忽略畸形条目、Anthropic 标头、最大页查询，以及 Anthropic 根地址的两种写法。 +本地 HTTP 服务器测试钉住两种受支持响应格式、字段归一化、名称回退、忽略畸形条目、Anthropic 标头、最大页查询，以及 Anthropic 根地址的两种写法。2026-09-02 从 OpenRouter、models.dev 与 DeepSeek 录得的回复存放在 `packages/llm/llm-pi-ai/tests/fixtures/model-listings/` 下并经解析器回放，因此受支持的字段拼写钉在真实端点上，而不是手写样例上。 diff --git a/packages/llm/llm-pi-ai/src/discovery.ts b/packages/llm/llm-pi-ai/src/discovery.ts index 1fef22a5e6..09be7b6207 100644 --- a/packages/llm/llm-pi-ai/src/discovery.ts +++ b/packages/llm/llm-pi-ai/src/discovery.ts @@ -63,6 +63,11 @@ interface ListingLimit { output?: unknown } +/** Per-route capacities OpenRouter nests under each entry. */ +interface ListingTopProvider { + max_completion_tokens?: unknown +} + /** One entry of a supported `GET /models` reply. */ interface ListingEntry { id?: unknown @@ -79,6 +84,7 @@ interface ListingEntry { max_output_tokens?: unknown maxTokens?: unknown limit?: ListingLimit | null + top_provider?: ListingTopProvider | null } /** A positive integer field of a listing entry, or `undefined` when absent or unusable. */ @@ -211,6 +217,7 @@ function readListing(body: unknown): LlmDiscoveredModel[] { entry?.maxTokens, entry?.max_tokens, entry?.limit?.output, + entry?.top_provider?.max_completion_tokens, ) models.push({ id, diff --git a/packages/llm/llm-pi-ai/tests/discovery.spec.ts b/packages/llm/llm-pi-ai/tests/discovery.spec.ts index c844a250a2..02b389ac8a 100644 --- a/packages/llm/llm-pi-ai/tests/discovery.spec.ts +++ b/packages/llm/llm-pi-ai/tests/discovery.spec.ts @@ -1,3 +1,4 @@ +import { readFile } from 'node:fs/promises' import { createServer } from 'node:http' import type { IncomingMessage, Server, ServerResponse } from 'node:http' import { afterEach, describe, expect, it, vi } from 'vitest' @@ -482,3 +483,50 @@ describe('probe key format', () => { expect(headers.has('authorization')).toBe(false) }) }) + +/** + * Replies recorded from live endpoints on 2026-09-02. Each file keeps the + * reply's top-level fields and entry objects verbatim; only the entry list is + * cut down to the named entries so the archive stays small. + */ +const RECORDED_LISTINGS = [ + { + name: 'OpenRouter GET /api/v1/models', + file: 'openrouter-2026-09-02.json', + models: [ + { id: 'anthropic/claude-fable-5.1', name: 'Anthropic: Claude Fable 5.1', contextWindow: 1_000_000, maxTokens: 128_000 }, + // The router's own aggregate route reports no completion cap. + { id: 'openrouter/auto-beta', name: 'Auto Router (Beta)', contextWindow: 2_000_000 }, + { id: 'deepseek/deepseek-v4-flash', name: 'DeepSeek: DeepSeek V4 Flash 0423', contextWindow: 1_048_576, maxTokens: 384_000 }, + ], + }, + { + name: 'the models.dev anthropic provider object', + file: 'models-dev-anthropic-2026-09-02.json', + models: [ + { id: 'claude-opus-4-7', name: 'Claude Opus 4.7', contextWindow: 1_000_000, maxTokens: 128_000 }, + { id: 'claude-fable-5-1', name: 'Claude Fable 5.1', contextWindow: 1_000_000, maxTokens: 128_000 }, + { id: 'claude-haiku-4-5', name: 'Claude Haiku 4.5 (latest)', contextWindow: 200_000, maxTokens: 64_000 }, + ], + }, + { + name: 'DeepSeek GET /models', + file: 'deepseek-2026-09-02.json', + models: [ + { id: 'deepseek-v4-flash', name: 'deepseek-v4-flash' }, + { id: 'deepseek-v4-pro', name: 'deepseek-v4-pro' }, + { id: 'deepseek-v4-flash-vision-exp', name: 'deepseek-v4-flash-vision-exp' }, + ], + }, +] + +describe('recorded provider listings', () => { + it.each(RECORDED_LISTINGS)('reads $name as recorded', async ({ file, models }) => { + const body = await readFile(new URL(`./fixtures/model-listings/${file}`, import.meta.url), 'utf8') + const server = await listingServer({ body }) + const ctx = await harness() + + await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: server.url, api: 'openai-completions' })) + .resolves.toEqual(models) + }) +}) diff --git a/packages/llm/llm-pi-ai/tests/fixtures/model-listings/deepseek-2026-09-02.json b/packages/llm/llm-pi-ai/tests/fixtures/model-listings/deepseek-2026-09-02.json new file mode 100644 index 0000000000..85b8bb4529 --- /dev/null +++ b/packages/llm/llm-pi-ai/tests/fixtures/model-listings/deepseek-2026-09-02.json @@ -0,0 +1,20 @@ +{ + "object": "list", + "data": [ + { + "id": "deepseek-v4-flash", + "object": "model", + "owned_by": "deepseek" + }, + { + "id": "deepseek-v4-pro", + "object": "model", + "owned_by": "deepseek" + }, + { + "id": "deepseek-v4-flash-vision-exp", + "object": "model", + "owned_by": "deepseek" + } + ] +} diff --git a/packages/llm/llm-pi-ai/tests/fixtures/model-listings/models-dev-anthropic-2026-09-02.json b/packages/llm/llm-pi-ai/tests/fixtures/model-listings/models-dev-anthropic-2026-09-02.json new file mode 100644 index 0000000000..e8d827f353 --- /dev/null +++ b/packages/llm/llm-pi-ai/tests/fixtures/model-listings/models-dev-anthropic-2026-09-02.json @@ -0,0 +1,146 @@ +{ + "id": "anthropic", + "env": [ + "ANTHROPIC_API_KEY" + ], + "npm": "@ai-sdk/anthropic", + "name": "Anthropic", + "doc": "https://docs.anthropic.com/en/docs/about-claude/models", + "models": { + "claude-opus-4-7": { + "id": "claude-opus-4-7", + "name": "Claude Opus 4.7", + "description": "Stronger Opus tier for advanced software work and high-stakes reasoning", + "family": "claude-opus", + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": [ + "low", + "medium", + "high", + "xhigh", + "max" + ] + } + ], + "tool_call": true, + "structured_output": true, + "temperature": false, + "knowledge": "2026-01-31", + "release_date": "2026-04-14", + "last_updated": "2026-04-16", + "modalities": { + "input": [ + "text", + "image", + "pdf" + ], + "output": [ + "text" + ] + }, + "open_weights": false, + "limit": { + "context": 1000000, + "output": 128000 + }, + "cost": { + "input": 5, + "output": 25, + "cache_read": 0.5, + "cache_write": 6.25 + } + }, + "claude-fable-5-1": { + "id": "claude-fable-5-1", + "name": "Claude Fable 5.1", + "description": "Claude model for demanding reasoning and long-horizon agentic work", + "family": "claude-fable", + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": [ + "low", + "medium", + "high", + "xhigh", + "max" + ] + } + ], + "tool_call": true, + "structured_output": true, + "temperature": false, + "knowledge": "2026-06", + "release_date": "2026-09-01", + "last_updated": "2026-09-01", + "modalities": { + "input": [ + "text", + "image", + "pdf" + ], + "output": [ + "text" + ] + }, + "open_weights": false, + "limit": { + "context": 1000000, + "output": 128000 + }, + "cost": { + "input": 10, + "output": 50, + "cache_read": 0.25, + "cache_write": 12.5 + } + }, + "claude-haiku-4-5": { + "id": "claude-haiku-4-5", + "name": "Claude Haiku 4.5 (latest)", + "description": "Fast Claude lane for lightweight agents, office tasks, and responsive chat", + "family": "claude-haiku", + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "budget_tokens", + "min": 1024 + } + ], + "tool_call": true, + "structured_output": true, + "temperature": true, + "knowledge": "2025-02-28", + "release_date": "2025-10-15", + "last_updated": "2025-10-15", + "modalities": { + "input": [ + "text", + "image", + "pdf" + ], + "output": [ + "text" + ] + }, + "open_weights": false, + "limit": { + "context": 200000, + "output": 64000 + }, + "cost": { + "input": 1, + "output": 5, + "cache_read": 0.1, + "cache_write": 1.25 + } + } + } +} diff --git a/packages/llm/llm-pi-ai/tests/fixtures/model-listings/openrouter-2026-09-02.json b/packages/llm/llm-pi-ai/tests/fixtures/model-listings/openrouter-2026-09-02.json new file mode 100644 index 0000000000..58c03a65ea --- /dev/null +++ b/packages/llm/llm-pi-ai/tests/fixtures/model-listings/openrouter-2026-09-02.json @@ -0,0 +1,299 @@ +{ + "data": [ + { + "id": "anthropic/claude-fable-5.1", + "canonical_slug": "anthropic/claude-fable-5.1-20260831", + "hugging_face_id": null, + "name": "Anthropic: Claude Fable 5.1", + "created": 1788285838, + "description": "Claude Fable 5.1 improves on Claude Fable 5 across the board, with the biggest gains in agentic coding, long-running agentic workflows, and knowledge work: long code refactors, front-end and visual...", + "context_length": 1000000, + "architecture": { + "modality": "text+image+file->text", + "input_modalities": [ + "text", + "image", + "file" + ], + "output_modalities": [ + "text" + ], + "tokenizer": "Claude", + "instruct_type": null + }, + "pricing": { + "prompt": "0.00001", + "completion": "0.00005", + "web_search": "0.01", + "input_cache_read": "0.00000025", + "input_cache_write": "0.0000125", + "input_cache_write_1h": "0.00002" + }, + "top_provider": { + "context_length": 1000000, + "max_completion_tokens": 128000, + "is_moderated": true + }, + "per_request_limits": null, + "supported_parameters": [ + "include_reasoning", + "max_completion_tokens", + "max_tokens", + "reasoning", + "reasoning_effort", + "response_format", + "stop", + "structured_outputs", + "tools", + "verbosity" + ], + "default_parameters": {}, + "supported_voices": null, + "knowledge_cutoff": null, + "expiration_date": null, + "links": { + "details": "/api/v1/models/anthropic/claude-fable-5.1-20260831/endpoints" + }, + "benchmarks": { + "design_arena": [ + { + "arena": "models", + "category": "website", + "elo": 1351, + "win_rate": 58.5, + "rank": 2 + } + ], + "artificial_analysis": { + "intelligence_index": 65.7, + "coding_index": 81.6, + "agentic_index": 61.3 + } + }, + "reasoning": { + "mandatory": true, + "supported_efforts": [ + "max", + "xhigh", + "high", + "medium", + "low" + ], + "default_effort": "high" + } + }, + { + "id": "openrouter/auto-beta", + "canonical_slug": "openrouter/auto-beta", + "hugging_face_id": null, + "name": "Auto Router (Beta)", + "created": 1784311165, + "description": "Auto Router (Beta) is a task-aware router from OpenRouter. It classifies each request, then routes it the [most popular model](/rankings#task-spend) for that task based on aggregate spend, filtered by your...", + "context_length": 2000000, + "architecture": { + "modality": "text+image+file+audio+video->text+image", + "input_modalities": [ + "text", + "image", + "audio", + "file", + "video" + ], + "output_modalities": [ + "text", + "image" + ], + "tokenizer": "Router", + "instruct_type": null + }, + "pricing": { + "prompt": "-1", + "completion": "-1" + }, + "top_provider": { + "context_length": null, + "max_completion_tokens": null, + "is_moderated": false + }, + "per_request_limits": null, + "supported_parameters": [ + "frequency_penalty", + "include_reasoning", + "logit_bias", + "logprobs", + "max_tokens", + "min_p", + "prediction", + "presence_penalty", + "reasoning", + "reasoning_effort", + "repetition_penalty", + "response_format", + "seed", + "stop", + "structured_outputs", + "temperature", + "tool_choice", + "tools", + "top_a", + "top_k", + "top_logprobs", + "top_p", + "web_search_options" + ], + "default_parameters": { + "temperature": null, + "top_p": null, + "top_k": null, + "frequency_penalty": null, + "presence_penalty": null, + "repetition_penalty": null + }, + "supported_voices": null, + "knowledge_cutoff": null, + "expiration_date": null, + "links": { + "details": "/api/v1/models/openrouter/auto-beta/endpoints" + } + }, + { + "id": "deepseek/deepseek-v4-flash", + "canonical_slug": "deepseek/deepseek-v4-flash-20260423", + "hugging_face_id": "deepseek-ai/DeepSeek-V4-Flash", + "name": "DeepSeek: DeepSeek V4 Flash 0423", + "created": 1777000666, + "description": "DeepSeek V4 Flash is an efficiency-optimized Mixture-of-Experts model from DeepSeek with 284B total parameters and 13B activated parameters, supporting a 1M-token context window. It is designed for fast inference and...", + "context_length": 1048576, + "architecture": { + "modality": "text->text", + "input_modalities": [ + "text" + ], + "output_modalities": [ + "text" + ], + "tokenizer": "DeepSeek", + "instruct_type": null + }, + "pricing": { + "prompt": "0.00000008708", + "completion": "0.00000017416", + "input_cache_read": "0.000000017416" + }, + "top_provider": { + "context_length": 1024000, + "max_completion_tokens": 384000, + "is_moderated": false + }, + "per_request_limits": null, + "supported_parameters": [ + "frequency_penalty", + "include_reasoning", + "logit_bias", + "logprobs", + "max_completion_tokens", + "max_tokens", + "min_p", + "presence_penalty", + "reasoning", + "reasoning_effort", + "repetition_penalty", + "response_format", + "seed", + "stop", + "structured_outputs", + "temperature", + "tool_choice", + "tools", + "top_a", + "top_k", + "top_logprobs", + "top_p" + ], + "default_parameters": {}, + "supported_voices": null, + "knowledge_cutoff": null, + "expiration_date": null, + "links": { + "details": "/api/v1/models/deepseek/deepseek-v4-flash-20260423/endpoints" + }, + "benchmarks": { + "design_arena": [ + { + "arena": "models", + "category": "3d", + "elo": 1229, + "win_rate": 49.3, + "rank": 40 + }, + { + "arena": "models", + "category": "asciiart", + "elo": 1138, + "win_rate": 42.8, + "rank": 47 + }, + { + "arena": "models", + "category": "codecategories", + "elo": 1226, + "win_rate": 48.9, + "rank": 43 + }, + { + "arena": "models", + "category": "dataviz", + "elo": 1148, + "win_rate": 40.6, + "rank": 74 + }, + { + "arena": "models", + "category": "gamedev", + "elo": 1232, + "win_rate": 50.2, + "rank": 39 + }, + { + "arena": "models", + "category": "svg", + "elo": 1193, + "win_rate": 48.4, + "rank": 34 + }, + { + "arena": "models", + "category": "uicomponent", + "elo": 1188, + "win_rate": 44.7, + "rank": 58 + }, + { + "arena": "models", + "category": "website", + "elo": 1224, + "win_rate": 49.1, + "rank": 45 + } + ], + "artificial_analysis": { + "intelligence_index": 42.1, + "coding_index": 56.2, + "agentic_index": 33.7 + } + }, + "reasoning": { + "mandatory": false, + "supported_efforts": [ + "xhigh", + "high" + ], + "default_effort": "high" + } + } + ], + "total_count": 421, + "links": { + "next": null + } +}