feat(llm): cache the DeepSeek account listing per credential

This commit is contained in:
Yichen Jiang committed 2026-09-24 13:08:00 +08:00
1 parent 2817b9e205
commit 31ce1ad5f1
2 files changed
+275

No files matched your search

@@ -0,0 +1,127 @@
/** Credential-scoped `/models` cache with single-flight loads, background refresh, and last-known-good retention. */
import { createHash } from 'node:crypto'
import type { CatalogSettings } from './config.ts'
import type { ModelListing } from './listing.ts'
/** Host-owned inputs of one catalog instance. */
export interface CatalogDependencies {
/** Current endpoint settings, re-read on every access. */
readonly settings: () => CatalogSettings
/** The account token allowed for `url`, or `undefined` while signed out. */
readonly token: (url: string) => Promise<string | undefined>
/** Perform one listing request authenticated with `token`. */
readonly load: (url: string, token: string, signal: AbortSignal) => Promise<ModelListing>
/** Wall clock used for listing age. */
readonly now: () => number
/** Report the first failure of a failing streak. */
readonly warn: (message: string) => void
}
/** What request resolution reads. */
export interface CatalogView {
/** Whether an account token is available for the listing endpoint. */
readonly signedIn: boolean
/** Last successful listing for this credential and endpoint; `undefined` until one succeeds. */
readonly listing: ModelListing | undefined
}
/** Wait for `task` unless `signal` aborts first; the task keeps running. */
async function abandonable(task: Promise<void>, signal: AbortSignal | undefined): Promise<void> {
if (signal === undefined) return task
const abandoned = Promise.withResolvers<never>()
const abandon = (): void => { abandoned.reject(signal.reason) }
signal.addEventListener('abort', abandon, { once: true })
try {
await Promise.race([task, abandoned.promise])
} finally {
signal.removeEventListener('abort', abandon)
}
}
/** One account route's listing cache; each endpoint and credential pair has its own scope. */
export class AccountModelCatalog {
private scope: string | undefined
private listing: ModelListing | undefined
/** Settle time of the latest load in this scope; `undefined` before the first one settles. */
private settledAt: number | undefined
private inflight: Promise<void> | undefined
private request: AbortController | undefined
private generation = 0
private failing = false
private closed = false
/** @param dependencies - settings, credential, transport, clock, and diagnostic callbacks. */
constructor(private readonly dependencies: CatalogDependencies) {}
/**
* The last published view, without I/O.
* @returns the signed-in state and listing of the current scope.
*/
current(): CatalogView {
return { signedIn: this.scope !== undefined, listing: this.listing }
}
/**
* Resolve the view for the current credential. The first access of a scope
* waits for its load, which the request deadline bounds; later accesses
* return at once and refresh a stale listing in the background.
* @param signal - abandons this wait only; the shared load continues.
* @returns the view after any first load of the scope settles.
*/
async view(signal?: AbortSignal): Promise<CatalogView> {
const settings = this.dependencies.settings()
const token = await this.dependencies.token(settings.url)
signal?.throwIfAborted()
if (this.closed) return this.current()
const scope = token === undefined ? undefined : `${settings.url}\n${createHash('sha256').update(token).digest('hex')}`
if (scope !== this.scope) this.rescope(scope)
if (token !== undefined) {
if (this.settledAt === undefined) await abandonable(this.start(settings, token), signal)
else if (this.dependencies.now() - this.settledAt >= settings.maxAgeMs) void this.start(settings, token)
}
return this.current()
}
/** Abort in-flight loads; later results and accesses change nothing. */
dispose(): void {
this.closed = true
this.generation += 1
this.request?.abort()
}
private rescope(scope: string | undefined): void {
this.request?.abort()
this.scope = scope
this.listing = undefined
this.settledAt = undefined
this.inflight = undefined
this.failing = false
this.generation += 1
}
private start(settings: CatalogSettings, token: string): Promise<void> {
if (this.inflight !== undefined) return this.inflight
this.generation += 1
const generation = this.generation
const request = new AbortController()
this.request = request
const run = this.dependencies.load(settings.url, token, AbortSignal.any([request.signal, AbortSignal.timeout(settings.timeoutMs)]))
.then((listing) => {
if (generation !== this.generation) return
this.listing = listing
this.settledAt = this.dependencies.now()
this.failing = false
}, (error: unknown) => {
if (generation !== this.generation) return
if (!this.failing) {
const fallback = this.listing === undefined ? 'using the bundled catalog' : 'keeping the previous catalog'
this.dependencies.warn(`llm-deepseek-account: model listing from ${settings.url} failed (${String(error)}); ${fallback}`)
}
this.settledAt = this.dependencies.now()
this.failing = true
})
.finally(() => { if (this.inflight === run) this.inflight = undefined })
this.inflight = run
return run
}
}
@@ -0,0 +1,148 @@
/** Credential-scoped listing cache: first-use waits, background refresh, retention, and disposal. */
import { expect, it, vi } from 'vitest'
import { AccountModelCatalog } from '../src/catalog.ts'
import type { ModelListing } from '../src/listing.ts'
const listing = (...ids: string[]): ModelListing => ({
skipped: 0,
entries: ids.map(id => ({ id, requirement: { kind: 'none' }, capabilities: { usable: true, facts: { inputModalities: ['text'] } } })),
})
function harness(timeoutMs = 60_000, honorsAbort = true) {
let clock = 0
let token: string | undefined = 'token-a'
const loads: Array<{ token: string; signal: AbortSignal; settle: PromiseWithResolvers<ModelListing> }> = []
const warn = vi.fn()
const catalog = new AccountModelCatalog({
settings: () => ({ url: 'https://api.example/models', timeoutMs, maxAgeMs: 100 }),
token: () => Promise.resolve(token),
load: (_url, value, signal) => {
const settle = Promise.withResolvers<ModelListing>()
if (honorsAbort) signal.addEventListener('abort', () => { settle.reject(signal.reason) }, { once: true })
loads.push({ token: value, signal, settle })
return settle.promise
},
now: () => clock,
warn,
})
const flush = () => new Promise(resolve => setImmediate(resolve))
return {
catalog, loads, warn, flush,
advance: (ms: number) => { clock += ms },
account: (value: string | undefined) => { token = value },
}
}
it('reports a signed-out view without loading', async () => {
const { catalog, loads, account } = harness()
account(undefined)
await expect(catalog.view()).resolves.toEqual({ signedIn: false, listing: undefined })
expect(loads).toEqual([])
})
it('shares one first load between concurrent callers', async () => {
const { catalog, loads, flush } = harness()
const first = catalog.view()
const second = catalog.view()
await flush()
expect(loads).toHaveLength(1)
loads[0]!.settle.resolve(listing('a'))
await expect(first).resolves.toEqual({ signedIn: true, listing: listing('a') })
await expect(second).resolves.toEqual({ signedIn: true, listing: listing('a') })
expect(catalog.current()).toEqual({ signedIn: true, listing: listing('a') })
})
it('bounds the first wait by the request deadline and serves no listing after a failure', async () => {
const { catalog, warn } = harness(5)
await expect(catalog.view()).resolves.toEqual({ signedIn: true, listing: undefined })
expect(warn).toHaveBeenCalledExactlyOnceWith(expect.stringContaining('using the bundled catalog'))
})
it('abandons only the caller\'s wait when its signal aborts', async () => {
const { catalog, loads, flush } = harness()
const controller = new AbortController()
const waiting = catalog.view(controller.signal)
await flush()
controller.abort(new Error('caller left'))
await expect(waiting).rejects.toThrow('caller left')
expect(loads[0]!.signal.aborted).toBe(false)
loads[0]!.settle.resolve(listing('a'))
await flush()
await expect(catalog.view()).resolves.toMatchObject({ listing: listing('a') })
})
it('rejects a caller whose signal is already aborted', async () => {
const { catalog } = harness()
await expect(catalog.view(AbortSignal.abort(new Error('gone')))).rejects.toThrow('gone')
})
it('refreshes a stale listing in the background and keeps it through a failing streak', async () => {
const { catalog, loads, warn, flush, advance } = harness()
const first = catalog.view()
await flush()
loads[0]!.settle.resolve(listing('a'))
await first
advance(50)
await catalog.view()
expect(loads).toHaveLength(1)
advance(50)
await expect(catalog.view()).resolves.toMatchObject({ listing: listing('a') })
expect(loads).toHaveLength(2)
loads[1]!.settle.reject(new Error('down'))
await flush()
expect(warn).toHaveBeenCalledExactlyOnceWith(expect.stringContaining('keeping the previous catalog'))
advance(100)
await catalog.view()
loads[2]!.settle.reject(new Error('still down'))
await flush()
expect(warn).toHaveBeenCalledTimes(1)
advance(100)
await catalog.view()
loads[3]!.settle.resolve(listing('a', 'b'))
await flush()
expect(catalog.current().listing).toEqual(listing('a', 'b'))
advance(100)
await catalog.view()
loads[4]!.settle.reject(new Error('down again'))
await flush()
expect(warn).toHaveBeenCalledTimes(2)
})
it('rescopes on another account and discards the previous account\'s result', async () => {
const { catalog, loads, flush, account } = harness()
const first = catalog.view()
await flush()
account('token-b')
const second = catalog.view()
await flush()
expect(loads[0]!.signal.aborted).toBe(true)
expect(loads[1]!.token).toBe('token-b')
loads[1]!.settle.resolve(listing('b'))
await expect(second).resolves.toMatchObject({ listing: listing('b') })
await first
expect(catalog.current().listing).toEqual(listing('b'))
account(undefined)
await expect(catalog.view()).resolves.toEqual({ signedIn: false, listing: undefined })
})
it('aborts in-flight work on dispose and loads nothing afterwards', async () => {
const { catalog, loads, warn, flush } = harness()
const waiting = catalog.view()
await flush()
catalog.dispose()
expect(loads[0]!.signal.aborted).toBe(true)
await waiting
await expect(catalog.view()).resolves.toEqual({ signedIn: true, listing: undefined })
expect(loads).toHaveLength(1)
expect(warn).not.toHaveBeenCalled()
})
it('drops a successful result that settles after disposal', async () => {
const { catalog, loads, flush } = harness(60_000, false)
const waiting = catalog.view()
await flush()
catalog.dispose()
loads[0]!.settle.resolve(listing('late'))
await waiting
expect(catalog.current().listing).toBeUndefined()
})