Merge pull request #5214 from deepseek-harness/worktree/deepseek-stale-file-batch-invalidate

fix(llm): remove every stale DeepSeek file mapping in one index update

packages/llm/llm-deepseek/README.i18n.yaml

@@ -39,8 +39,8 @@
   en: f863390198cc177c
   zh: 8856c62ba3fb36ca
 /deepseek-ai-dsh-llm-deepseek/use-this-package/failures-and-recovery:
-  en: d31bd24c8c419f88
-  zh: 91205cb3cd754f39
+  en: 7a09ab81593cd9c1
+  zh: 919dc0eb6347e9fd
 /deepseek-ai-dsh-llm-deepseek/understand-the-implementation:
   en: 1598ac8a8bb9b6bf
   zh: fdd089c5ae7722cd

packages/llm/llm-deepseek/README.md

@@ -115,7 +115,7 @@ Configuration accepts Messages only and has no `protocol` field. If resolution r
 
 Successful Files responses must contain valid JSON. JSON decoding failures from upload, list, retrieve, and delete throw `INVALID_RESPONSE` with the operation and HTTP status in the message, the status in `LlmError.failure`, and the original parser error as `cause`. Body-read transport and cancellation errors retain their identity.
 
-Non-2xx responses fail with stable codes: `AUTH` (401/403), `QUOTA`, `RATE_LIMIT`, `CONTEXT_WINDOW_EXCEEDED`, `INVALID_REQUEST`, `SERVER`, and `HTTP_<status>` otherwise; pre-response transport failures throw `TRANSPORT`, caller aborts throw `ABORTED`, and stream-idle expiry throws `TIMEOUT`. Request-extension preparation, field collision, or post-2xx acceptance fails with `REQUEST_EXTENSION`. A normalized-image rejection names every plausible attachment and its durable position when the provider does not identify a file id. Stale-file rejection invalidates the named mappings (or every mapping used by the attempt) and permits one replacement model request. Protocol violations throw `STREAM_CLOSED` or `MALFORMED_RESPONSE`, and a terminal `stop` with no content blocks becomes `EMPTY_RESPONSE`, which the default retry policy retries. A request on the official route without an API key fails with `MISSING_CREDENTIAL`, and a malformed credential fails with `INVALID_CREDENTIAL` naming the reference to fix — never any part of the key.
+Non-2xx responses fail with stable codes: `AUTH` (401/403), `QUOTA`, `RATE_LIMIT`, `CONTEXT_WINDOW_EXCEEDED`, `INVALID_REQUEST`, `SERVER`, and `HTTP_<status>` otherwise; pre-response transport failures throw `TRANSPORT`, caller aborts throw `ABORTED`, and stream-idle expiry throws `TIMEOUT`. Request-extension preparation, field collision, or post-2xx acceptance fails with `REQUEST_EXTENSION`. A normalized-image rejection names every plausible attachment and its durable position when the provider does not identify a file id. Stale-file rejection removes every named mapping (or every mapping used by the attempt) in one upload-index update and permits one replacement model request. Protocol violations throw `STREAM_CLOSED` or `MALFORMED_RESPONSE`, and a terminal `stop` with no content blocks becomes `EMPTY_RESPONSE`, which the default retry policy retries. A request on the official route without an API key fails with `MISSING_CREDENTIAL`, and a malformed credential fails with `INVALID_CREDENTIAL` naming the reference to fix — never any part of the key.
 
 Provider plugins own catalog availability; only the account route requires a stored grant for discovery. Their catalogs are configured independently; the transport supplies shared default model metadata and capability resolution.
 

packages/llm/llm-deepseek/README.zh.md

@@ -115,7 +115,7 @@ Files 模式通过 `maxRequestFilesBytes` 与 `maxImagesPerRequest` 限制保留
 
 成功的 Files 响应必须包含有效 JSON。上传、列举、获取和删除操作的 JSON 解码失败抛出 `INVALID_RESPONSE`，消息包含操作名称与 HTTP 状态，`LlmError.failure` 保留该状态，`cause` 保留原始解析错误。读取响应体时的传输和取消错误保留其原有身份。
 
-非 2xx 响应以稳定 code 失败：`AUTH`（401/403）、`QUOTA`、`RATE_LIMIT`、`CONTEXT_WINDOW_EXCEEDED`、`INVALID_REQUEST`、`SERVER` 以及其他情况的 `HTTP_<status>`；响应前传输失败抛出 `TRANSPORT`，调用方中止抛出 `ABORTED`，流空闲超时抛出 `TIMEOUT`。请求扩展准备、字段冲突或 2xx 后接受失败使用 `REQUEST_EXTENSION`。当提供方未指出 file id 时，规范化图片拒绝会列出所有可能附件及其持久位置。陈旧文件拒绝会使点名映射（或该次尝试使用的全部映射）失效，并允许一次替换模型请求。协议违规抛出 `STREAM_CLOSED` 或 `MALFORMED_RESPONSE`；不带内容块的终止 `stop` 变成 `EMPTY_RESPONSE`，默认重试策略会重试它。官方路由缺少 API Key 的请求以 `MISSING_CREDENTIAL` 失败；格式错误的凭据以 `INVALID_CREDENTIAL` 失败，并点名需要修复的引用——绝不包含密钥的任何部分。
+非 2xx 响应以稳定 code 失败：`AUTH`（401/403）、`QUOTA`、`RATE_LIMIT`、`CONTEXT_WINDOW_EXCEEDED`、`INVALID_REQUEST`、`SERVER` 以及其他情况的 `HTTP_<status>`；响应前传输失败抛出 `TRANSPORT`，调用方中止抛出 `ABORTED`，流空闲超时抛出 `TIMEOUT`。请求扩展准备、字段冲突或 2xx 后接受失败使用 `REQUEST_EXTENSION`。当提供方未指出 file id 时，规范化图片拒绝会列出所有可能附件及其持久位置。陈旧文件拒绝会在一次上传索引更新中删除全部点名映射（或该次尝试使用的全部映射），并允许一次替换模型请求。协议违规抛出 `STREAM_CLOSED` 或 `MALFORMED_RESPONSE`；不带内容块的终止 `stop` 变成 `EMPTY_RESPONSE`，默认重试策略会重试它。官方路由缺少 API Key 的请求以 `MISSING_CREDENTIAL` 失败；格式错误的凭据以 `INVALID_CREDENTIAL` 失败，并点名需要修复的引用——绝不包含密钥的任何部分。
 
 提供方插件负责目录可用性；仅账号路由要求存有凭据才能发现模型。两者的目录独立配置；传输层提供共享的默认模型元数据和能力解析。
 

packages/llm/llm-deepseek/src/file-store.ts

@@ -244,21 +244,15 @@ export class DeepSeekFileStore {
   }
 
   /**
-   * Invalidate one exact local mapping after a model request rejects its remote id.
-   * @param version - request-image version whose remote generation failed.
-   * @param fileId - exact rejected file id.
+   * Invalidate exact local mappings in one index update after a model request rejects their remote ids.
+   * @param generations - request-image variants with the exact file id the request used for each.
    * @param connection - endpoint and API-key snapshot.
    */
   async invalidate(
-    version: RequestImageAttachment,
-    fileId: DeepSeekFileId,
+    generations: readonly Pick<DeepSeekUploadRecord, 'variantId' | 'fileId'>[],
     connection: DeepSeekFileConnection,
   ): Promise<void> {
-    await this.index.remove(
-      fileScope(connection),
-      version.variantId,
-      fileId,
-    )
+    await this.index.remove(fileScope(connection), generations)
   }
 
   /**
@@ -284,7 +278,7 @@ export class DeepSeekFileStore {
     )
     if (record === undefined) return false
     await this.client(connection).delete(record.fileId, signal)
-    await this.index.remove(scope, version.variantId, record.fileId)
+    await this.index.remove(scope, [{ variantId: version.variantId, fileId: record.fileId }])
     return true
   }
 

packages/llm/llm-deepseek/src/request-files.ts

@@ -128,13 +128,16 @@ export class RequestFiles {
   }
 
   /**
-   * Invalidate rejected mappings; only the first stale-id response permits another request.
+   * Invalidate every rejected mapping in one index update; only the first stale-id response permits another request.
    * @param detail - provider error fields used for stale-id classification.
    * @returns whether the caller should serialize and dispatch again.
    */
   async retry(detail: string): Promise<boolean> {
     if (this.used.length === 0 || !providerRejectedFileId(detail)) return false
-    await Promise.all(staleMappings(this.used, detail).map(file => this.files.invalidate(file.version, file.fileId, this.connection)))
+    await this.files.invalidate(
+      staleMappings(this.used, detail).map(file => ({ variantId: file.version.variantId, fileId: file.fileId })),
+      this.connection,
+    )
     if (this.retried) return false
     this.retried = true
     return true

packages/llm/llm-deepseek/src/upload-index.ts

@@ -190,21 +190,20 @@ export class DeepSeekUploadIndex {
   }
 
   /**
-   * Remove one exact mapping without deleting a concurrently installed successor.
+   * Remove exact mappings in one locked rewrite without deleting concurrently installed successors.
    * @param scope - endpoint/API-key namespace.
-   * @param variantId - complete request-image transformation identity.
-   * @param fileId - exact remote generation being invalidated.
+   * @param generations - exact remote generations being invalidated; pairs absent from the index are ignored.
    */
   async remove(
     scope: DeepSeekFileScopeType,
-    variantId: ImageVariantIdType,
-    fileId: DeepSeekFileIdType,
+    generations: readonly Pick<DeepSeekUploadRecord, 'variantId' | 'fileId'>[],
   ): Promise<void> {
+    const invalidated = new Set(generations.map(generation => `${generation.variantId}\0${generation.fileId}`))
     await mkdir(dirname(this.path), { recursive: true, mode: 0o700 })
     await withFileLock(this.path, async () => {
       const index = await this.load()
       const records = index.records.filter(record => !(
-        record.scope === scope && record.variantId === variantId && record.fileId === fileId
+        record.scope === scope && invalidated.has(`${record.variantId}\0${record.fileId}`)
       ))
       if (records.length !== index.records.length) await this.save({ formatVersion: 3, records })
     })

packages/llm/llm-deepseek/tests/file-store.spec.ts

@@ -98,10 +98,10 @@ describe('DeepSeekFileStore', () => {
     expect((await store.ensureUploaded(VERSION, { ...CONNECTION, baseURL: `${CONNECTION.baseURL}/v1/` }, POLICY)).record).toEqual(first.record)
     const reopened = new DeepSeekFileStore({ index, fetch: remote.fetchImpl, now: () => now })
     expect((await reopened.ensureUploaded(VERSION, CONNECTION, POLICY)).record).toEqual(first.record)
-    await reopened.invalidate(VERSION, other.record.fileId, CONNECTION)
+    await reopened.invalidate([{ variantId: VERSION.variantId, fileId: other.record.fileId }], CONNECTION)
     expect((await reopened.ensureUploaded(VERSION, CONNECTION, POLICY)).record).toEqual(first.record)
     expect(remote.uploads()).toBe(2)
-    await reopened.invalidate(VERSION, first.record.fileId, CONNECTION)
+    await reopened.invalidate([{ variantId: VERSION.variantId, fileId: first.record.fileId }], CONNECTION)
     const replacement = await reopened.ensureUploaded(VERSION, CONNECTION, POLICY)
     expect(replacement.record.fileId).not.toBe(first.record.fileId)
     expect((await reopened.ensureUploaded(VERSION, alternate, POLICY)).record).toEqual(other.record)

packages/llm/llm-deepseek/tests/files.spec.ts

@@ -73,7 +73,7 @@ describe('Messages Files requests', () => {
     vi.stubGlobal('fetch', fetchImpl)
     const h = harness()
     await chunks(h.adapter.stream(request([ref, second])))
-    expect(h.invalidate).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ attachment: ref }), 'file-a', expect.objectContaining({}))
+    expect(h.invalidate).toHaveBeenCalledExactlyOnceWith([{ variantId: version(ref).variantId, fileId: 'file-a' }], expect.objectContaining({}))
     expect(fetchImpl).toHaveBeenCalledTimes(2)
     expect(h.prepare).toHaveBeenCalledTimes(2)
     expect(h.ensureUploaded).toHaveBeenCalledTimes(4)
@@ -85,7 +85,7 @@ describe('Messages Files requests', () => {
     const h = harness()
     await expect(chunks(h.adapter.stream(request([ref, second])))).rejects.toMatchObject({ code: 'INVALID_REQUEST' })
     expect(fetchImpl).toHaveBeenCalledTimes(2)
-    expect(h.invalidate).toHaveBeenCalledTimes(4)
+    expect(h.invalidate.mock.calls.map(call => call[0].length)).toEqual([2, 2])
     expect(fetchImpl.mock.calls.every(call => !body(call[1]).includes('base64'))).toBe(true)
   })
 

packages/llm/llm-deepseek/tests/runtime.spec.ts

@@ -1017,6 +1017,57 @@ describe('DeepSeekAdapter against a mock server', () => {
       ])
   })
 
+  it('recovers in one replacement request when the provider lists dozens of missing file ids', async () => {
+    const refs = Array.from({ length: 40 }, (_, index): ImageAttachmentRef => ({
+      ...imageRef,
+      attachmentId: AttachmentId(`sha256:${(index + 1).toString(16).padStart(64, '0')}`),
+    }))
+    const missing = Array.from({ length: 30 }, (_, index) => `file-api-${index + 1}`)
+    const server = await mockServer([
+      {
+        kind: 'http-error',
+        status: 400,
+        body: JSON.stringify({
+          error: {
+            type: 'invalid_request_error',
+            message: 'messages.0.content.1.tool_result: the following file_ids do not exist or are not created under your account: '
+              + `${missing.join(', ')} (request_id: 00000000-0000-4000-8000-000000000002)`,
+          },
+        }),
+      },
+      { kind: 'sse', events: textEvents },
+    ])
+    const attachments = attachmentStoreOf(ref => Promise.resolve({
+      ...requestImage(ref),
+      variantId: ImageVariantId(String(ref.attachmentId)),
+    })).store
+    const adapter = adapterOf({
+      baseURL: server.url,
+      models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }],
+    }, attachments)
+
+    await drain(adapter.stream({
+      provider: 'deepseek-official',
+      model: 'deepseek-v4-flash-vision-exp',
+      messages: [createUserMessage({
+        content: refs.map(ref => ({ type: 'image' as const, attachment: ref })),
+        source: { kind: 'model', provider: 'deepseek-official', model: 'deepseek-v4-flash' },
+      })],
+    }))
+
+    expect(server.requests).toHaveLength(2)
+    expect(server.fileRequests.filter(request => request.method === 'POST')).toHaveLength(70)
+    const attempts = server.requests as Array<{ messages: Array<{ content: Array<{ type: string; source?: { file_id?: string } }> }> }>
+    const fileIds = (attempt: number) => attempts[attempt]?.messages[0]?.content
+      .filter(block => block.type === 'image')
+      .map(block => block.source?.file_id)
+    expect(fileIds(0)).toEqual(refs.map((_, index) => `file-api-${index + 1}`))
+    expect(fileIds(1)).toEqual([
+      ...missing.map((_, index) => `file-api-${41 + index}`),
+      ...refs.slice(30).map((_, index) => `file-api-${31 + index}`),
+    ])
+  })
+
   it('invalidates every used mapping when a stale-file response does not identify one file id', async () => {
     const secondRef: ImageAttachmentRef = {
       ...imageRef,

packages/llm/llm-deepseek/tests/upload-index.spec.ts

@@ -56,12 +56,36 @@ describe('DeepSeekUploadIndex', () => {
     await index.commit(first, 1, 1)
 
     await expect(index.commit(duplicate, 2, 1)).resolves.toEqual({ record: first, accepted: false })
-    await index.remove(scope, VARIANT, duplicate.fileId)
+    await index.remove(scope, [{ variantId: VARIANT, fileId: duplicate.fileId }])
     await expect(index.get(scope, VARIANT, 2, 1)).resolves.toEqual(first)
-    await index.remove(scope, VARIANT, first.fileId)
+    await index.remove(scope, [{ variantId: VARIANT, fileId: first.fileId }])
     await expect(index.get(scope, VARIANT, 2, 1)).resolves.toBeUndefined()
   })
 
+  it('removes several exact generations in one update and keeps unlisted mappings', async () => {
+    const dir = await mkdtemp(join(tmpdir(), 'dsh-upload-index-'))
+    roots.push(dir)
+    const index = new DeepSeekUploadIndex(join(dir, 'index.json'))
+    const scope = deepSeekFileScope('https://api.deepseek.com', 'key')
+    const record = (digest: string) => ({
+      scope, attachmentId: ATTACHMENT, variantId: ImageVariantId(`sha256:${digest.repeat(64)}`),
+      fileId: DeepSeekFileId(`file-api-${digest}`), bytes: 3, createdAt: 1, expiresAt: 10_000,
+    })
+    const first = record('b')
+    const second = record('c')
+    const third = record('d')
+    for (const entry of [first, second, third]) await index.commit(entry, 1, 1)
+
+    await index.remove(scope, [
+      { variantId: first.variantId, fileId: first.fileId },
+      { variantId: second.variantId, fileId: second.fileId },
+      { variantId: third.variantId, fileId: DeepSeekFileId('file-api-superseded') },
+    ])
+    await expect(index.get(scope, first.variantId, 2, 1)).resolves.toBeUndefined()
+    await expect(index.get(scope, second.variantId, 2, 1)).resolves.toBeUndefined()
+    await expect(index.get(scope, third.variantId, 2, 1)).resolves.toEqual(third)
+  })
+
   it('treats a corrupt upload cache as empty and repairs it on the next commit', async () => {
     const dir = await mkdtemp(join(tmpdir(), 'dsh-upload-index-'))
     roots.push(dir)