feat(llm): cache the DeepSeek account listing per credential
This commit is contained in:
1 parent
2817b9e205
commit
31ce1ad5f1
2 files changed
+275
No files matched your search
@@ -0,0 +1,127 @@
|
||||
/** Credential-scoped `/models` cache with single-flight loads, background refresh, and last-known-good retention. */
|
||||
import { createHash } from 'node:crypto'
|
||||
import type { CatalogSettings } from './config.ts'
|
||||
import type { ModelListing } from './listing.ts'
|
||||
|
||||
/** Host-owned inputs of one catalog instance. */
|
||||
export interface CatalogDependencies {
|
||||
/** Current endpoint settings, re-read on every access. */
|
||||
readonly settings: () => CatalogSettings
|
||||
/** The account token allowed for `url`, or `undefined` while signed out. */
|
||||
readonly token: (url: string) => Promise<string | undefined>
|
||||
/** Perform one listing request authenticated with `token`. */
|
||||
readonly load: (url: string, token: string, signal: AbortSignal) => Promise<ModelListing>
|
||||
/** Wall clock used for listing age. */
|
||||
readonly now: () => number
|
||||
/** Report the first failure of a failing streak. */
|
||||
readonly warn: (message: string) => void
|
||||
}
|
||||
|
||||
/** What request resolution reads. */
|
||||
export interface CatalogView {
|
||||
/** Whether an account token is available for the listing endpoint. */
|
||||
readonly signedIn: boolean
|
||||
/** Last successful listing for this credential and endpoint; `undefined` until one succeeds. */
|
||||
readonly listing: ModelListing | undefined
|
||||
}
|
||||
|
||||
/** Wait for `task` unless `signal` aborts first; the task keeps running. */
|
||||
async function abandonable(task: Promise<void>, signal: AbortSignal | undefined): Promise<void> {
|
||||
if (signal === undefined) return task
|
||||
const abandoned = Promise.withResolvers<never>()
|
||||
const abandon = (): void => { abandoned.reject(signal.reason) }
|
||||
signal.addEventListener('abort', abandon, { once: true })
|
||||
try {
|
||||
await Promise.race([task, abandoned.promise])
|
||||
} finally {
|
||||
signal.removeEventListener('abort', abandon)
|
||||
}
|
||||
}
|
||||
|
||||
/** One account route's listing cache; each endpoint and credential pair has its own scope. */
|
||||
export class AccountModelCatalog {
|
||||
private scope: string | undefined
|
||||
private listing: ModelListing | undefined
|
||||
/** Settle time of the latest load in this scope; `undefined` before the first one settles. */
|
||||
private settledAt: number | undefined
|
||||
private inflight: Promise<void> | undefined
|
||||
private request: AbortController | undefined
|
||||
private generation = 0
|
||||
private failing = false
|
||||
private closed = false
|
||||
|
||||
/** @param dependencies - settings, credential, transport, clock, and diagnostic callbacks. */
|
||||
constructor(private readonly dependencies: CatalogDependencies) {}
|
||||
|
||||
/**
|
||||
* The last published view, without I/O.
|
||||
* @returns the signed-in state and listing of the current scope.
|
||||
*/
|
||||
current(): CatalogView {
|
||||
return { signedIn: this.scope !== undefined, listing: this.listing }
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the view for the current credential. The first access of a scope
|
||||
* waits for its load, which the request deadline bounds; later accesses
|
||||
* return at once and refresh a stale listing in the background.
|
||||
* @param signal - abandons this wait only; the shared load continues.
|
||||
* @returns the view after any first load of the scope settles.
|
||||
*/
|
||||
async view(signal?: AbortSignal): Promise<CatalogView> {
|
||||
const settings = this.dependencies.settings()
|
||||
const token = await this.dependencies.token(settings.url)
|
||||
signal?.throwIfAborted()
|
||||
if (this.closed) return this.current()
|
||||
const scope = token === undefined ? undefined : `${settings.url}\n${createHash('sha256').update(token).digest('hex')}`
|
||||
if (scope !== this.scope) this.rescope(scope)
|
||||
if (token !== undefined) {
|
||||
if (this.settledAt === undefined) await abandonable(this.start(settings, token), signal)
|
||||
else if (this.dependencies.now() - this.settledAt >= settings.maxAgeMs) void this.start(settings, token)
|
||||
}
|
||||
return this.current()
|
||||
}
|
||||
|
||||
/** Abort in-flight loads; later results and accesses change nothing. */
|
||||
dispose(): void {
|
||||
this.closed = true
|
||||
this.generation += 1
|
||||
this.request?.abort()
|
||||
}
|
||||
|
||||
private rescope(scope: string | undefined): void {
|
||||
this.request?.abort()
|
||||
this.scope = scope
|
||||
this.listing = undefined
|
||||
this.settledAt = undefined
|
||||
this.inflight = undefined
|
||||
this.failing = false
|
||||
this.generation += 1
|
||||
}
|
||||
|
||||
private start(settings: CatalogSettings, token: string): Promise<void> {
|
||||
if (this.inflight !== undefined) return this.inflight
|
||||
this.generation += 1
|
||||
const generation = this.generation
|
||||
const request = new AbortController()
|
||||
this.request = request
|
||||
const run = this.dependencies.load(settings.url, token, AbortSignal.any([request.signal, AbortSignal.timeout(settings.timeoutMs)]))
|
||||
.then((listing) => {
|
||||
if (generation !== this.generation) return
|
||||
this.listing = listing
|
||||
this.settledAt = this.dependencies.now()
|
||||
this.failing = false
|
||||
}, (error: unknown) => {
|
||||
if (generation !== this.generation) return
|
||||
if (!this.failing) {
|
||||
const fallback = this.listing === undefined ? 'using the bundled catalog' : 'keeping the previous catalog'
|
||||
this.dependencies.warn(`llm-deepseek-account: model listing from ${settings.url} failed (${String(error)}); ${fallback}`)
|
||||
}
|
||||
this.settledAt = this.dependencies.now()
|
||||
this.failing = true
|
||||
})
|
||||
.finally(() => { if (this.inflight === run) this.inflight = undefined })
|
||||
this.inflight = run
|
||||
return run
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
/** Credential-scoped listing cache: first-use waits, background refresh, retention, and disposal. */
|
||||
import { expect, it, vi } from 'vitest'
|
||||
import { AccountModelCatalog } from '../src/catalog.ts'
|
||||
import type { ModelListing } from '../src/listing.ts'
|
||||
|
||||
const listing = (...ids: string[]): ModelListing => ({
|
||||
skipped: 0,
|
||||
entries: ids.map(id => ({ id, requirement: { kind: 'none' }, capabilities: { usable: true, facts: { inputModalities: ['text'] } } })),
|
||||
})
|
||||
|
||||
function harness(timeoutMs = 60_000, honorsAbort = true) {
|
||||
let clock = 0
|
||||
let token: string | undefined = 'token-a'
|
||||
const loads: Array<{ token: string; signal: AbortSignal; settle: PromiseWithResolvers<ModelListing> }> = []
|
||||
const warn = vi.fn()
|
||||
const catalog = new AccountModelCatalog({
|
||||
settings: () => ({ url: 'https://api.example/models', timeoutMs, maxAgeMs: 100 }),
|
||||
token: () => Promise.resolve(token),
|
||||
load: (_url, value, signal) => {
|
||||
const settle = Promise.withResolvers<ModelListing>()
|
||||
if (honorsAbort) signal.addEventListener('abort', () => { settle.reject(signal.reason) }, { once: true })
|
||||
loads.push({ token: value, signal, settle })
|
||||
return settle.promise
|
||||
},
|
||||
now: () => clock,
|
||||
warn,
|
||||
})
|
||||
const flush = () => new Promise(resolve => setImmediate(resolve))
|
||||
return {
|
||||
catalog, loads, warn, flush,
|
||||
advance: (ms: number) => { clock += ms },
|
||||
account: (value: string | undefined) => { token = value },
|
||||
}
|
||||
}
|
||||
|
||||
it('reports a signed-out view without loading', async () => {
|
||||
const { catalog, loads, account } = harness()
|
||||
account(undefined)
|
||||
await expect(catalog.view()).resolves.toEqual({ signedIn: false, listing: undefined })
|
||||
expect(loads).toEqual([])
|
||||
})
|
||||
|
||||
it('shares one first load between concurrent callers', async () => {
|
||||
const { catalog, loads, flush } = harness()
|
||||
const first = catalog.view()
|
||||
const second = catalog.view()
|
||||
await flush()
|
||||
expect(loads).toHaveLength(1)
|
||||
loads[0]!.settle.resolve(listing('a'))
|
||||
await expect(first).resolves.toEqual({ signedIn: true, listing: listing('a') })
|
||||
await expect(second).resolves.toEqual({ signedIn: true, listing: listing('a') })
|
||||
expect(catalog.current()).toEqual({ signedIn: true, listing: listing('a') })
|
||||
})
|
||||
|
||||
it('bounds the first wait by the request deadline and serves no listing after a failure', async () => {
|
||||
const { catalog, warn } = harness(5)
|
||||
await expect(catalog.view()).resolves.toEqual({ signedIn: true, listing: undefined })
|
||||
expect(warn).toHaveBeenCalledExactlyOnceWith(expect.stringContaining('using the bundled catalog'))
|
||||
})
|
||||
|
||||
it('abandons only the caller\'s wait when its signal aborts', async () => {
|
||||
const { catalog, loads, flush } = harness()
|
||||
const controller = new AbortController()
|
||||
const waiting = catalog.view(controller.signal)
|
||||
await flush()
|
||||
controller.abort(new Error('caller left'))
|
||||
await expect(waiting).rejects.toThrow('caller left')
|
||||
expect(loads[0]!.signal.aborted).toBe(false)
|
||||
loads[0]!.settle.resolve(listing('a'))
|
||||
await flush()
|
||||
await expect(catalog.view()).resolves.toMatchObject({ listing: listing('a') })
|
||||
})
|
||||
|
||||
it('rejects a caller whose signal is already aborted', async () => {
|
||||
const { catalog } = harness()
|
||||
await expect(catalog.view(AbortSignal.abort(new Error('gone')))).rejects.toThrow('gone')
|
||||
})
|
||||
|
||||
it('refreshes a stale listing in the background and keeps it through a failing streak', async () => {
|
||||
const { catalog, loads, warn, flush, advance } = harness()
|
||||
const first = catalog.view()
|
||||
await flush()
|
||||
loads[0]!.settle.resolve(listing('a'))
|
||||
await first
|
||||
advance(50)
|
||||
await catalog.view()
|
||||
expect(loads).toHaveLength(1)
|
||||
advance(50)
|
||||
await expect(catalog.view()).resolves.toMatchObject({ listing: listing('a') })
|
||||
expect(loads).toHaveLength(2)
|
||||
loads[1]!.settle.reject(new Error('down'))
|
||||
await flush()
|
||||
expect(warn).toHaveBeenCalledExactlyOnceWith(expect.stringContaining('keeping the previous catalog'))
|
||||
advance(100)
|
||||
await catalog.view()
|
||||
loads[2]!.settle.reject(new Error('still down'))
|
||||
await flush()
|
||||
expect(warn).toHaveBeenCalledTimes(1)
|
||||
advance(100)
|
||||
await catalog.view()
|
||||
loads[3]!.settle.resolve(listing('a', 'b'))
|
||||
await flush()
|
||||
expect(catalog.current().listing).toEqual(listing('a', 'b'))
|
||||
advance(100)
|
||||
await catalog.view()
|
||||
loads[4]!.settle.reject(new Error('down again'))
|
||||
await flush()
|
||||
expect(warn).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('rescopes on another account and discards the previous account\'s result', async () => {
|
||||
const { catalog, loads, flush, account } = harness()
|
||||
const first = catalog.view()
|
||||
await flush()
|
||||
account('token-b')
|
||||
const second = catalog.view()
|
||||
await flush()
|
||||
expect(loads[0]!.signal.aborted).toBe(true)
|
||||
expect(loads[1]!.token).toBe('token-b')
|
||||
loads[1]!.settle.resolve(listing('b'))
|
||||
await expect(second).resolves.toMatchObject({ listing: listing('b') })
|
||||
await first
|
||||
expect(catalog.current().listing).toEqual(listing('b'))
|
||||
account(undefined)
|
||||
await expect(catalog.view()).resolves.toEqual({ signedIn: false, listing: undefined })
|
||||
})
|
||||
|
||||
it('aborts in-flight work on dispose and loads nothing afterwards', async () => {
|
||||
const { catalog, loads, warn, flush } = harness()
|
||||
const waiting = catalog.view()
|
||||
await flush()
|
||||
catalog.dispose()
|
||||
expect(loads[0]!.signal.aborted).toBe(true)
|
||||
await waiting
|
||||
await expect(catalog.view()).resolves.toEqual({ signedIn: true, listing: undefined })
|
||||
expect(loads).toHaveLength(1)
|
||||
expect(warn).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('drops a successful result that settles after disposal', async () => {
|
||||
const { catalog, loads, flush } = harness(60_000, false)
|
||||
const waiting = catalog.view()
|
||||
await flush()
|
||||
catalog.dispose()
|
||||
loads[0]!.settle.resolve(listing('late'))
|
||||
await waiting
|
||||
expect(catalog.current().listing).toBeUndefined()
|
||||
})
|
||||
Reference in new issue
Block a user