/** Host reader of configured plugin market indexes for the Discover page and the `plugin_market` tool. */ import { isAbsolute, join, resolve } from 'node:path' import type { Context, Volatile } from '@deepseek-ai/cordis' import z from '@deepseek-ai/schemastery' import semver from 'semver' import { evaluatePluginCompatibility, getDshRuntimeVersion } from '@deepseek-ai/dsh-app-boot' import { marketInstallTarget, type MarketIndex, type MarketPlugin, type MarketVersion } from '@deepseek-ai/dsh-plugin-market-index' import type { BundleInfo } from '@deepseek-ai/dsh-plugin-manager/types' import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' import { Remote, RemoteError, TypertRemoteService } from '@deepseek-ai/dsh-typert-protocol' import { DetailFailure, detailImageResponse, languageChain, NPM_FILE_SOURCES, npmFileSourceOrder, npmFiles, } from '@deepseek-ai/dsh-plugin-detail' import type { NpmFileSource } from '@deepseek-ai/dsh-plugin-detail/types' import { DetailReader, repositoryOf, type DetailLocation, type RepositoryLocation } from './detail.ts' import { repositoryFiles, type RepositoryFileOptions } from './detail-files.ts' import { normalizeIndexUrl } from './index-url.ts' import { RepositoryCache } from './repository-cache.ts' import { parseIndexInWorker } from './parse-thread.ts' import { MarketSearch } from './search.ts' // Type-only: the `pluginManager` service, `settings.update`, the Loader's `loader/volatile-update` event, and the // `connection` Fetch routes. import type {} from '@deepseek-ai/dsh-plugin-manager' import type {} from '@deepseek-ai/dsh-settings' import type {} from '@deepseek-ai/cordis-plugin-loader' import type {} from '@deepseek-ai/dsh-client-connection' import type { MarketCompatibility, MarketIndexSummary, MarketInstallation, MarketListingEntry, MarketListingVersion, MarketQuery, MarketSearchResult, PluginDetail, } from './types.ts' export type * from './types.ts' declare module '@deepseek-ai/cordis' { interface Context { /** Configured plugin market index sources and their validated listings. */ pluginMarket: PluginMarket } } /** Index sources and fetch limits. */ export interface Config { /** Index URLs in display order; Settings and `setIndexes` write this field without remounting the service. */ indexes: Volatile /** Deadline for one index fetch, including the body. */ fetchTimeoutMs: number /** Largest accepted index body in bytes. */ maxBytes: number /** Lifetime of a successfully read index document, a detail read result, and a repository's default-branch commit. */ cacheTtlMs: number /** Largest detail document in bytes. */ detailMaxBytes: number /** Largest image beside a detail document in bytes. */ detailImageMaxBytes: number /** Largest amount in bytes one Git command adds to a partial clone; a repository whose fetch adds more is not read. */ detailRepositoryMaxBytes: number /** Largest total size of the partial clones; least recently used ones are deleted beyond it. */ detailCacheMaxBytes: number /** Directory of the partial clones, relative to the profile directory; absent means `.plugin-market/repositories`. */ detailCacheDir?: string /** Deadline of one Git command of the clone fallback. */ detailGitTimeoutMs: number /** npm file CDNs read in order; a read's `npmFileSources` moves the CDNs it names first. */ detailNpmFileSources: NpmFileSource[] /** Hosts serving the GitLab repository files API. */ detailGitlabHosts: string[] } /** The exact diagnostic of a failure, including a non-Error rejection. */ function messageOf(error: unknown): string { return error instanceof Error ? error.message : String(error) } /** Causes followed below a rejection; a deeper or cyclic chain is cut there. */ const MAX_CAUSES = 4 /** A rejection with its cause chain: a Node `fetch` failure is a bare `fetch failed` whose reason is its cause. */ function causeChainOf(error: unknown, depth = 0): string { return error instanceof Error && error.cause !== undefined && depth < MAX_CAUSES ? `${error.message}: ${causeChainOf(error.cause, depth + 1)}` : messageOf(error) } /** The Fetch route serving images beside detail documents. */ const DETAIL_FILE_PATH = '/api/plugin-market/detail-file' /** Registries whose packages the public npm file CDNs serve. */ const PUBLIC_REGISTRIES = new Set(['registry.npmjs.org', 'registry.npmmirror.com']) /** Largest page `search` returns. */ const MAX_LIMIT = 100 /** One fetched index document with its search state. */ interface IndexDocument { readonly index: MarketIndex /** Unix milliseconds. */ readonly fetchedAt: number readonly search: MarketSearch } interface CachedIndex { readonly document: IndexDocument readonly expiresAt: number } /** Fetches, validates, and caches configured market indexes; annotates entries with the profile's installation facts. */ export default class PluginMarket extends TypertRemoteService { static inject = ['pluginManager'] static Config = z.object({ indexes: z.array(z.string().pattern(/^https?:\/\/\S+$/u)).default([]).volatile(), fetchTimeoutMs: z.natural().min(1).max(MAX_TIMER_DELAY_MS).default(15000), maxBytes: z.natural().min(1).default(128 * 1024 * 1024), cacheTtlMs: z.natural().default(600000), detailMaxBytes: z.natural().min(1).default(1024 * 1024), detailImageMaxBytes: z.natural().min(1).default(5 * 1024 * 1024), detailRepositoryMaxBytes: z.natural().min(1).default(50 * 1024 * 1024), detailCacheMaxBytes: z.natural().min(1).default(500 * 1024 * 1024), detailCacheDir: z.string(), detailGitTimeoutMs: z.natural().min(1).max(MAX_TIMER_DELAY_MS).default(60000), detailNpmFileSources: z.array(z.union(NPM_FILE_SOURCES)).min(1).default([...NPM_FILE_SOURCES]), detailGitlabHosts: z.array(z.string()).default(['gitlab.com']), }) private readonly lifetime = new AbortController() private readonly cache = new Map() private readonly pending = new Map>() private readonly errors = new Map() private readonly entryId: string | undefined private readonly repositories: RepositoryCache private readonly details: DetailReader constructor(ctx: Context, private readonly config: Config) { super(ctx, 'pluginMarket') this.entryId = ctx.fiber.entry?.options.id const profile = ctx.get('profileContext') const cacheDir = profile === undefined ? config.detailCacheDir : resolve(profile.dir, config.detailCacheDir ?? join('.plugin-market', 'repositories')) if (cacheDir === undefined || !isAbsolute(cacheDir)) throw new Error('plugin-market: detailCacheDir must be an absolute path when the service runs without a dsh profile') const lifetime = this.lifetime.signal this.repositories = new RepositoryCache({ dir: cacheDir, maxRepositoryBytes: config.detailRepositoryMaxBytes, maxCacheBytes: config.detailCacheMaxBytes, // The URL forms repositories are read from, and ssh, which `url..insteadOf` may route them to. timeoutMs: config.detailGitTimeoutMs, ttlMs: config.cacheTtlMs, transports: ['https', 'http', 'ssh'], lifetime, // Bounds how far a fetch overshoots `detailRepositoryMaxBytes`; directory walks cost little at this rate. sizePollMs: 200, }) this.details = new DetailReader( { maxDocumentBytes: config.detailMaxBytes, maxImageBytes: config.detailImageMaxBytes, ttlMs: config.cacheTtlMs }, this.detailRepositoryOptions(), ) ctx.inject(['connection'], (scope) => { scope.effect(() => scope.connection.fetch.register({ path: DETAIL_FILE_PATH, methods: ['GET'], requestBody: 'buffered', fetch: request => this.serveDetailFile(request), }), `plugin-market: ${DETAIL_FILE_PATH}`) }) ctx.on('loader/volatile-update', () => { const configured = new Set(this.sources().flatMap(source => source.error === undefined ? [source.url] : [])) for (const url of [...this.cache.keys(), ...this.errors.keys()]) { if (!configured.has(url)) { this.cache.delete(url) this.errors.delete(url) } } ctx.emit('plugin-market/changed') }) ctx.effect(() => async () => { this.lifetime.abort(new Error('Plugin market disposed')) await Promise.allSettled([...this.pending.values(), this.repositories.settled()]) }) } /** * List the configured index sources without fetching them. * @returns every source in order, with the title of its last successful read and, when the last read failed, its error. */ @Remote listIndexes(): MarketIndexSummary[] { return this.sources().map(({ url, error: invalid }) => { const index = this.cache.get(url)?.document.index const error = invalid ?? this.errors.get(url) return { url, ...index === undefined ? {} : { title: index.title, ...index.description === undefined ? {} : { description: index.description } }, ...error === undefined ? {} : { error }, } }) } /** * Replace the configured index sources through Settings, without remounting this service. * `plugin-market/changed` announces the committed list. A URL already configured is kept as written, so a client can * add or remove sources beside one that `listIndexes` reports as unusable. * @param urls - index URLs in display order. * @throws {RemoteError} `plugin-market/invalid-index-url` for an invalid new URL or a repeated URL; * `plugin-market/settings-unavailable` without Settings or a profile entry. */ @Remote async setIndexes(urls: string[]): Promise { const configured = new Set(this.config.indexes.get()) const normalized = urls.map(url => configured.has(url) ? url : normalizeIndexUrl(url)) // A kept URL compares by its canonical form when it has one, so a new spelling of it counts as a repeat. const keys = normalized.map(url => URL.canParse(url) ? new URL(url).href : url) const repeated = normalized.find((_url, position) => keys.indexOf(keys[position] as string) !== position) if (repeated !== undefined) { throw new RemoteError('plugin-market/invalid-index-url', `Index source listed twice: ${repeated}`, { url: repeated, reason: 'duplicate' }) } const settings = this.ctx.get('settings') if (settings === undefined || this.entryId === undefined) { throw new RemoteError('plugin-market/settings-unavailable', 'Index sources require the settings service and a profile entry', {}) } await settings.update(this.entryId, { indexes: normalized }) } /** * Return one page of the plugins of a configured index that match a query. Concurrent reads of a URL share one * fetch; a successful document is reused for `cacheTtlMs` unless `refresh` is set. Matching and ordering read only * the document; installation status and compatibility are computed for the returned page on every call. * @param url - a configured index URL. * @param query - text, filters, order, page, and whether to bypass the cached document. * @returns the index header, the page's plugins with each plugin's default version and installation status and each * version's compatibility and install arguments, the number of matches, and the next page's offset. * @throws {RemoteError} `plugin-market/invalid-query`, `plugin-market/unknown-index`, `plugin-market/invalid-index-url`, * `plugin-market/fetch-failed`, or `plugin-market/invalid-index`. */ @Remote async search(url: string, query: MarketQuery): Promise { if (!Number.isSafeInteger(query.offset) || query.offset < 0) { throw new RemoteError('plugin-market/invalid-query', 'offset must be a non-negative integer', { field: 'offset', reason: 'range' }) } if (!Number.isInteger(query.limit) || query.limit < 1 || query.limit > MAX_LIMIT) { throw new RemoteError('plugin-market/invalid-query', `limit must be an integer from 1 to ${String(MAX_LIMIT)}`, { field: 'limit', reason: 'range' }) } const document = await this.document(url, query.refresh === true, query.fetchedAt) const matching = await document.search.select(query) const page = matching.slice(query.offset, query.offset + query.limit) const listing = await this.annotator() const next = query.offset + page.length const { index } = document return { url, title: index.title, ...index.description === undefined ? {} : { description: index.description }, metadata: index.metadata, fetchedAt: document.fetchedAt, entries: page.map(listing), total: matching.length, nextOffset: next < matching.length ? next : null, } } /** * Read one plugin of a configured index, using the cached document as `search` does. * @param url - a configured index URL. * @param name - the plugin's package name. * @returns the plugin with its default version, installation status, and annotated versions; undefined when the * index does not list it. * @throws {RemoteError} `plugin-market/unknown-index`, `plugin-market/invalid-index-url`, `plugin-market/fetch-failed`, * or `plugin-market/invalid-index`. */ async entry(url: string, name: string): Promise { const plugin = (await this.document(url, false)).index.plugins.find(candidate => candidate.name === name) return plugin === undefined ? undefined : (await this.annotator())(plugin) } /** * Read the long description of one listed version of a plugin market entry, from the version's source: the npm file * CDNs for a public npm version, or its pinned Git commit. The Markdown file the package's locale `meta.detail` names * in `language` comes first, then the package's README, then the README of the entry's `repository`, or for an npm * version without one, of its `package.json` `repository`, at the default branch. An npm version from another * registry has no readable package files, so only the entry's `repository` is read. * @param url - a configured index URL. * @param name - the plugin's package name. * @param version - one listed version. * @param language - the display language, such as `zh-CN`; documents fall back through its prefixes to English. * @param options - `refresh` reads again instead of reusing a result cached for `cacheTtlMs`; `npmFileSources` names * npm file CDNs to ask first for an npm version, in order; configured CDNs it omits follow, and CDNs that are not * configured are not asked. * @returns the document with its origin and image document id, or `none`. * @throws {RemoteError} the failures of `search`, `plugin-market/unknown-plugin` for an unlisted plugin or version, * `plugin-market/invalid-query`, or `plugin-market/detail-unavailable`. */ @Remote async readEntryDetail( url: string, name: string, version: string, language: string, options?: { refresh?: boolean; npmFileSources?: NpmFileSource[] }, ): Promise { const languages = this.languages(language) const plugin = (await this.document(url, false)).index.plugins.find(candidate => candidate.name === name) const listed = plugin?.versions.find(candidate => candidate.version === version) if (plugin === undefined || listed === undefined) { throw new RemoteError('plugin-market/unknown-plugin', `${url} lists no ${name}@${version}`, { name, version }) } const declared = plugin.repository const found = declared === undefined ? undefined : repositoryOf(declared.url) let repository: RepositoryLocation | undefined = found === undefined ? undefined : { ...found, directory: declared?.directory ?? '' } const { source } = listed let location: DetailLocation if (source.type === 'git') { const ref = repositoryOf(source.url) /* v8 ignore next -- parseMarketIndex admits only https Git URLs with an owner and a project */ if (ref === undefined) return { status: 'none' } const dir = source.path ?? '' // A repository that names the source directory itself adds nothing to the source. if (repository?.url === ref.url && repository.directory === dir) repository = undefined location = { files: repositoryFiles(ref, source.commit, this.detailRepositoryOptions()), dir, repository } } else if (source.registry === undefined || PUBLIC_REGISTRIES.has(new URL(source.registry).host)) { location = this.npmLocation(name, version, repository ?? 'manifest', options?.npmFileSources ?? []) } else { location = { dir: '', repository } } return await this.readDetail(location, languages, options?.refresh === true) } private languages(language: string): string[] { try { return languageChain(language) } catch (error) { throw new RemoteError('plugin-market/invalid-query', messageOf(error), { field: 'language', reason: 'format' }, { cause: error }) } } private detailRepositoryOptions(): RepositoryFileOptions { const { config } = this return { timeoutMs: config.fetchTimeoutMs, lifetime: this.lifetime.signal, gitlabHosts: config.detailGitlabHosts.map(host => host.toLowerCase()), cache: this.repositories, } } /** The detail location of a published npm version on the configured CDNs, the preferred ones first. */ private npmLocation(name: string, version: string, repository: RepositoryLocation | 'manifest', preferred: readonly string[]): DetailLocation { const sources = npmFileSourceOrder(this.config.detailNpmFileSources, preferred) return { dir: '', repository, files: npmFiles(name, version, sources, { timeoutMs: this.config.fetchTimeoutMs, lifetime: this.lifetime.signal }) } } private async readDetail(location: DetailLocation, languages: readonly string[], refresh: boolean): Promise { try { return await this.details.read(location, languages, refresh) } catch (error) { if (!(error instanceof DetailFailure)) throw error throw new RemoteError('plugin-market/detail-unavailable', error.message, { reason: error.reason }, { cause: error }) } } /** Answer `GET ${DETAIL_FILE_PATH}?document=&path=` with one image beside a read detail document. */ private async serveDetailFile(request: Request): Promise { const params = new URL(request.url).searchParams const document = params.get('document') const path = params.get('path') // Images stay cached as long as their document's result. const cache = `private, max-age=${String(Math.floor(this.config.cacheTtlMs / 1000))}` if (document === null || path === null) return detailImageResponse('invalid', cache) try { return detailImageResponse(await this.details.image(document, path), cache) } catch (error) { if (!(error instanceof DetailFailure)) throw error return detailImageResponse(error.reason, cache) } } private async document(url: string, refresh: boolean, fetchedAt?: number): Promise { const source = this.sources().find(candidate => candidate.url === url) if (source === undefined) throw new RemoteError('plugin-market/unknown-index', `Not a configured index source: ${url}`, { url }) if (source.error !== undefined) { throw new RemoteError('plugin-market/invalid-index-url', source.error, { url, reason: 'configured' }) } const cached = this.cache.get(url) const usable = cached !== undefined && (cached.expiresAt > Date.now() || cached.document.fetchedAt === fetchedAt) return !refresh && usable ? cached.document : await this.fetchShared(url) } /** Annotate plugins with the profile's bundles and exemptions as they are now. */ private async annotator(): Promise<(plugin: MarketPlugin) => MarketListingEntry> { const [bundles, { exemptions }] = [await this.ctx.pluginManager.listBundles(), this.ctx.pluginManager.listVersionExemptions()] const runtimeVersion = getDshRuntimeVersion() return plugin => listingOf(plugin, bundles, exemptions, runtimeVersion) } /** * The configured sources as written, each with the refusal that makes it unusable. Settings and profile files can * write any http(s) string, so every use validates instead of the schema; an unusable source is listed with its error * and never fetched, and a repeated URL counts once. */ private sources(): { readonly url: string; readonly error?: string }[] { const seen = new Set() return this.config.indexes.get().flatMap((url) => { if (seen.has(url)) return [] seen.add(url) try { return normalizeIndexUrl(url) === url ? [{ url }] : [{ url, error: `Index URL is not in canonical form; use ${normalizeIndexUrl(url)}` }] } catch (error) { return [{ url, error: messageOf(error) }] } }) } private configured(url: string): boolean { return this.sources().some(source => source.url === url && source.error === undefined) } private fetchShared(url: string): Promise { let pending = this.pending.get(url) if (pending === undefined) { pending = this.fetchIndex(url).then((index) => { // Distinct per URL even when two fetches finish within one millisecond. const fetchedAt = Math.max(Date.now(), (this.cache.get(url)?.document.fetchedAt ?? 0) + 1) const document = { index, fetchedAt, search: new MarketSearch(index) } if (this.configured(url)) { this.cache.set(url, { document, expiresAt: Date.now() + this.config.cacheTtlMs }) this.errors.delete(url) } return document }, (error: unknown) => { if (this.configured(url)) this.errors.set(url, messageOf(error)) throw error }).finally(() => { this.pending.delete(url) }) this.pending.set(url, pending) } return pending } private async fetchIndex(url: string): Promise { const signal = AbortSignal.any([this.lifetime.signal, AbortSignal.timeout(this.config.fetchTimeoutMs)]) const failed = (reason: string, cause?: unknown): RemoteError => new RemoteError('plugin-market/fetch-failed', `Could not fetch ${url}: ${reason}`, { url, reason }, { cause }) let response: Response try { response = await fetch(url, { signal, redirect: 'error', headers: { accept: 'application/json' } }) } catch (error) { throw failed(causeChainOf(error), error) } if (!response.ok) { await response.body?.cancel() throw failed(`HTTP ${String(response.status)}`) } const bytes = await this.readBody(response, failed) const result = await parseIndexInWorker(bytes, url, this.lifetime.signal).catch((error: unknown) => { throw failed(messageOf(error), error) }) if (result.ok) return result.index throw new RemoteError('plugin-market/invalid-index', `${url}: ${result.message}`, { url, path: result.path, reason: result.reason }) } /** Read the whole body, refusing it as soon as it exceeds `maxBytes`. */ private async readBody(response: Response, failed: (reason: string, cause?: unknown) => RemoteError): Promise { const limit = this.config.maxBytes const tooLarge = `body exceeds ${String(limit)} bytes` if (Number(response.headers.get('content-length') ?? 0) > limit) { await response.body?.cancel() throw failed(tooLarge) } const chunks: Uint8Array[] = [] let size = 0 try { for await (const chunk of response.body ?? []) { size += chunk.byteLength if (size > limit) throw failed(tooLarge) chunks.push(chunk) } } catch (error) { if (error instanceof RemoteError) throw error throw failed(messageOf(error), error) } const body = new Uint8Array(size) let offset = 0 for (const chunk of chunks) { body.set(chunk, offset) offset += chunk.byteLength } return body } } function listingOf( plugin: MarketPlugin, bundles: readonly BundleInfo[], exemptions: Readonly>, runtimeVersion: string, ): MarketListingEntry { const versions = plugin.versions.map((version): MarketListingVersion => ({ ...version, install: marketInstallTarget(plugin.name, version), compatibility: compatibilityOf(plugin.name, version, exemptions, runtimeVersion), })) const compatible = versions.filter(version => version.compatibility.compatible) const stable = plugin.versions.some(version => semver.prerelease(version.version) === null) const defaultVersion = compatible.find(version => !stable || semver.prerelease(version.version) === null)?.version /* v8 ignore next -- parseMarketIndex guarantees a non-empty version list */ const measured = defaultVersion ?? versions[0]?.version ?? '' return { ...plugin, versions, ...defaultVersion === undefined ? {} : { defaultVersion }, installation: installationOf(plugin.name, measured, bundles), } } function installationOf(name: string, version: string, bundles: readonly BundleInfo[]): MarketInstallation { const bundle = bundles.find(candidate => candidate.name === name) // An on-demand catalog entry is listed before the profile installs it. if (bundle === undefined || (!bundle.installed && bundle.availability !== 'installation')) return { status: 'not-installed' } if (!bundle.installed) return { status: 'provided', ...bundle.version === undefined ? {} : { installedVersion: bundle.version } } if (bundle.version === undefined) return { status: 'different-version' } const installedVersion = bundle.version if (installedVersion === version) return { status: 'installed', installedVersion } const newer = semver.valid(installedVersion) !== null && semver.gt(version, installedVersion) return { status: newer ? 'update-available' : 'different-version', installedVersion } } function compatibilityOf( name: string, version: MarketVersion, exemptions: Readonly>, runtimeVersion: string, ): MarketCompatibility { const manifest = { name, version: version.version, ...version.engines.dsh === undefined ? {} : { engines: { dsh: version.engines.dsh } } } const issue = evaluatePluginCompatibility(manifest, exemptions, runtimeVersion) if (issue === undefined) return { compatible: true } return { compatible: false, runtimeVersion: issue.runtimeVersion, requirements: issue.requirements, exempted: issue.exempted } }