From 716e2e2683b56b3fd2d67d3308d52206df075e60 Mon Sep 17 00:00:00 2001 From: Yichen Jiang Date: Fri, 18 Sep 2026 14:34:12 +0800 Subject: [PATCH] feat(plugin-manager): ask configured registries in turn while one is unreachable Add registry and fallbackRegistries to the manager Config, ask them in order from inspect and installBundle with --registry, and move on only for failures another registry can change: unreachable, timed out, or a copy the registry lacks. A git or tarball spec's own host is never retried through a registry, and a registry outside the configured set is asked alone. The lookup runs without pnpm's retries so a dead registry is reported within inspectTimeoutMs; the install keeps pnpm's defaults. Results carry the registries asked, the install-state event names each attempt, and the plugin_manager tool takes a registry. --- packages/boot/plugin-manager/src/index.ts | 148 ++++++++++++----- .../plugin-manager/src/install-failure.ts | 2 +- .../boot/plugin-manager/src/install-spec.ts | 27 +++- .../boot/plugin-manager/src/operations.ts | 25 ++- packages/boot/plugin-manager/src/registry.ts | 62 ++++++++ packages/boot/plugin-manager/src/tools.ts | 2 + packages/boot/plugin-manager/src/types.ts | 27 +++- .../plugin-manager/tests/install-spec.spec.ts | 15 +- .../boot/plugin-manager/tests/manager.spec.ts | 149 +++++++++++++++--- .../plugin-manager/tests/operations.spec.ts | 16 +- .../plugin-manager/tests/registry.spec.ts | 79 ++++++++++ .../boot/plugin-manager/tests/tools.spec.ts | 2 + 12 files changed, 471 insertions(+), 83 deletions(-) create mode 100644 packages/boot/plugin-manager/src/registry.ts create mode 100644 packages/boot/plugin-manager/tests/registry.spec.ts diff --git a/packages/boot/plugin-manager/src/index.ts b/packages/boot/plugin-manager/src/index.ts index 2d9451d844..e3ef0cd496 100644 --- a/packages/boot/plugin-manager/src/index.ts +++ b/packages/boot/plugin-manager/src/index.ts @@ -15,21 +15,24 @@ import { } from '@deepseek-ai/dsh-app-boot' import type {} from '@deepseek-ai/dsh-hmr' import type { ProfileContext, ProfileManifest } from '@deepseek-ai/dsh-app-boot' -import { bundleManifest, runProfilePnpm, saveManifest, viewProfilePackage } from './operations.ts' +import { bundleManifest, registryArguments, runProfilePnpm, saveManifest, viewProfilePackage } from './operations.ts' import { classifyInstallFailure } from './install-failure.ts' -import { InvalidInstallSpecError, parseInstallSpec } from './install-spec.ts' +import { InvalidInstallSpecError, parseInstallSpec, type ParsedInstallSpec } from './install-spec.ts' +import { askNextRegistry, normalizeRegistry, registryPlan } from './registry.ts' import { writePluginEnabled } from './patch.ts' import { ManagementFailure } from './failure.ts' import { approveBuilds, readPendingBuilds } from './build-approval.ts' import type { - BundleInfo, BundleRowInfo, ChangeResult, InstallBundleOptions, ManagementError, PackageResult, PluginChange, PluginEntryId, PluginInfo, - PluginInspectProblem, PluginInstallCancellation, PluginInstallProgress, PluginInstallRequestId, PluginSpecInspection, + BundleInfo, BundleRowInfo, ChangeResult, InspectOptions, InstallBundleOptions, ManagementError, PackageResult, PluginChange, + PluginEntryId, PluginInfo, PluginInspectProblem, PluginInstallCancellation, PluginInstallProgress, PluginInstallRequestId, + PluginRegistries, PluginSpecInspection, Registry, } from './types.ts' export type * from './types.ts' export { classifyInstallFailure, type InstallFailureFacts } from './install-failure.ts' export { InvalidInstallSpecError, parseInstallSpec, type ParsedInstallSpec } from './install-spec.ts' +export { askNextRegistry, normalizeRegistry, registryPlan } from './registry.ts' -/** The pnpm executable and the limits for package diagnostics and registry lookups. */ +/** The pnpm executable, the registries asked, and the limits for package diagnostics and registry lookups. */ export interface Config { /** The pnpm executable name or path; resolved through `PATH` like the `dsh plugin` command. */ pnpmCommand?: string @@ -39,8 +42,18 @@ export interface Config { lockWaitMs?: number /** Bound on one registry lookup an inspection runs, in milliseconds. */ inspectTimeoutMs?: number + /** The registry lookups and installations ask first, as an http(s) URL; absent, the one pnpm's own configuration names. */ + registry?: string + /** + * Registries asked in turn, as http(s) URLs, while the one before is unreachable or holds no copy of the package. + * A registry outside this set and `registry` is asked alone. + */ + fallbackRegistries?: string[] } +/** An http(s) URL, as pnpm's `--registry` takes it. */ +const REGISTRY_URL = /^https?:\/\/\S+$/ + const protectedModules = new Set([ '@deepseek-ai/dsh-plugin-manager', '@deepseek-ai/cordis-plugin-loader', '@deepseek-ai/cordis-plugin-include', '@deepseek-ai/dsh-api-gateway', @@ -101,7 +114,7 @@ interface InstallationManifest { } /** What a package manifest says about the package: identity, one-liner, and whether it is a bundle. */ -function inspectionOf(kind: 'registry' | 'path', manifest: object): Extract { +function inspectionOf(kind: 'registry' | 'path', manifest: object, registry: Registry): Extract { const dsh = (manifest as { dsh?: unknown }).dsh const declared = typeof dsh === 'object' && dsh !== null ? dsh as { bundle?: unknown } : undefined const bundle = declared !== undefined && typeof declared.bundle === 'object' && declared.bundle !== null @@ -109,7 +122,7 @@ function inspectionOf(kind: 'registry' | 'path', manifest: object): Extract>() @@ -143,6 +169,7 @@ export class PluginManager extends TypertRemoteService { private readonly lockWaitMs: number private readonly inspectTimeoutMs: number private readonly pnpmCommand: string + private readonly configuredRegistries: PluginRegistries private readonly ownerContext: Context private readonly abort = new AbortController() /** Installations by request id, from their call until it settles. */ @@ -157,6 +184,10 @@ export class PluginManager extends TypertRemoteService { this.lockWaitMs = (config as Required).lockWaitMs this.inspectTimeoutMs = (config as Required).inspectTimeoutMs this.pnpmCommand = (config as Required).pnpmCommand + this.configuredRegistries = { + registry: config.registry === undefined ? null : normalizeRegistry(config.registry), + fallbackRegistries: (config as Required).fallbackRegistries.map(normalizeRegistry), + } ctx.effect(() => async () => { this.abort.abort() await Promise.allSettled([...this.packageOperations]) @@ -224,13 +255,22 @@ export class PluginManager extends TypertRemoteService { return Promise.resolve(bundles) } + /** Read the registries this manager asks: the configured first one, then its fallbacks in order. + * @returns The registries in pnpm's comparison form; null is the one pnpm's own configuration names. + */ + @Remote + registries(): Promise { + return Promise.resolve({ ...this.configuredRegistries, fallbackRegistries: [...this.configuredRegistries.fallbackRegistries] }) + } + /** Read what a spec names before installing it. * @param spec One package spec: a registry name, an absolute path, a git address, or a tarball. + * @param options The registry asked first. * @param signal Ends a registry lookup early. * @returns The package the spec names, or why it is refused. */ @Remote - async inspect(spec: string, signal?: AbortSignal): Promise { + async inspect(spec: string, options?: InspectOptions, signal?: AbortSignal): Promise { let parsed try { parsed = parseInstallSpec(spec) @@ -244,11 +284,13 @@ export class PluginManager extends TypertRemoteService { const known = new Set([ ...manifest.dsh?.profile?.bundles ?? [], ...Object.keys(manifest.dependencies ?? {}), ...Object.keys(installation.dependencies ?? {}), ]) + const plan = registryPlan(options?.registry, this.configuredRegistries) + const registry = plan[0] as Registry switch (parsed.kind) { - case 'git': return { status: 'accepted', kind: 'git', bundle: null } + case 'git': return { status: 'accepted', kind: 'git', bundle: null, registry } case 'tarball': if (parsed.path !== undefined && !existsSync(parsed.path)) return refused('not-a-package', 'the tarball does not exist') - return { status: 'accepted', kind: 'tarball', bundle: null } + return { status: 'accepted', kind: 'tarball', bundle: null, registry } case 'path': { if (!existsSync(parsed.path)) return refused('not-a-package', 'the path does not exist') let read: object @@ -257,7 +299,7 @@ export class PluginManager extends TypertRemoteService { } catch (error) { return refused('not-a-package', `no readable package.json at the path: ${messageOf(error)}`) } - const inspection = inspectionOf('path', read) + const inspection = inspectionOf('path', read, registry) if (inspection.name === undefined) return refused('not-a-package', 'the package.json names no package') if (known.has(inspection.name)) return refused('already-installed', `${inspection.name} is already installed`) if (!inspection.bundle) return refused('not-a-bundle', `${inspection.name} declares no dsh.bundle`) @@ -265,31 +307,41 @@ export class PluginManager extends TypertRemoteService { } case 'registry': { if (known.has(parsed.name)) return refused('already-installed', `${parsed.name} is already installed`) - const view = await viewProfilePackage(this.profile.dir, spec.trim(), { - ...this.profile.packageManager ?? { command: this.pnpmCommand }, - timeoutMs: this.inspectTimeoutMs, ...signal === undefined ? {} : { signal }, - }) - const log = `${view.stderr}${view.cause === undefined ? '' : `${messageOf(view.cause)}\n`}`.trim() - if (view.exitCode !== 0 || view.cause !== undefined || view.timedOut) { - const kind = classifyInstallFailure({ log, timedOut: view.timedOut, ...view.cause === undefined ? {} : { cause: view.cause } }) - const reason = log || view.stdout.trim() || `pnpm view exited with ${String(view.exitCode)}` - if (kind === 'not-found' || kind === 'no-matching-version') return refused('not-found', reason) - if (kind === 'network') return refused('network', reason) - return refused('unknown', view.timedOut ? `pnpm view timed out after ${String(this.inspectTimeoutMs)}ms` : reason) + const registries: Registry[] = [] + const refusedBy = (problem: PluginInspectProblem, reason: string): PluginSpecInspection => + ({ ...refused(problem, reason), registries }) + for (const current of plan) { + registries.push(current) + const view = await viewProfilePackage(this.profile.dir, spec.trim(), { + ...this.profile.packageManager ?? { command: this.pnpmCommand }, + timeoutMs: this.inspectTimeoutMs, ...signal === undefined ? {} : { signal }, registry: current, + }) + const log = `${view.stderr}${view.cause === undefined ? '' : `${messageOf(view.cause)}\n`}`.trim() + if (view.exitCode !== 0 || view.cause !== undefined || view.timedOut) { + const kind = classifyInstallFailure({ log, timedOut: view.timedOut, ...view.cause === undefined ? {} : { cause: view.cause } }) + // A lookup the caller dropped is not carried to the next registry. + if (registries.length < plan.length && signal?.aborted !== true && askNextRegistry(kind, log, parsed)) continue + const reason = log || view.stdout.trim() || `pnpm view exited with ${String(view.exitCode)}` + if (kind === 'not-found' || kind === 'no-matching-version') return refusedBy('not-found', reason) + if (kind === 'network') return refusedBy('network', reason) + return refusedBy('unknown', view.timedOut ? `pnpm view timed out after ${String(this.inspectTimeoutMs)}ms` : reason) + } + let answer: unknown + try { + answer = JSON.parse(view.stdout.replace(ANSI_SEQUENCE, '').trim() || 'null') + } catch (error) { + return refusedBy('unknown', `unreadable pnpm view output: ${messageOf(error)}`) + } + // A range answers one object per matching version, oldest first. + const latest: unknown = Array.isArray(answer) ? answer.at(-1) : answer + if (typeof latest !== 'object' || latest === null) return refusedBy('unknown', 'pnpm view answered no package') + const inspection = inspectionOf('registry', latest, current) + const named = inspection.name === undefined ? { ...inspection, name: parsed.name } : inspection + if (!named.bundle) return refusedBy('not-a-bundle', `${named.name} declares no dsh.bundle`) + return named } - let answer: unknown - try { - answer = JSON.parse(view.stdout.replace(ANSI_SEQUENCE, '').trim() || 'null') - } catch (error) { - return refused('unknown', `unreadable pnpm view output: ${messageOf(error)}`) - } - // A range answers one object per matching version, oldest first. - const latest: unknown = Array.isArray(answer) ? answer.at(-1) : answer - if (typeof latest !== 'object' || latest === null) return refused('unknown', 'pnpm view answered no package') - const inspection = inspectionOf('registry', latest) - const named = inspection.name === undefined ? { ...inspection, name: parsed.name } : inspection - if (!named.bundle) return refused('not-a-bundle', `${named.name} declares no dsh.bundle`) - return named + /* v8 ignore next -- the plan is never empty: every attempt returns or continues to the next */ + throw new Error('no registry was asked') } } } @@ -330,9 +382,9 @@ export class PluginManager extends TypertRemoteService { * that fails, is cancelled, or adds a package without a bundle patch restores * `package.json` and `pnpm-lock.yaml` as they were; downloaded files can stay. * @param spec One package spec, including local paths relative to the invocation directory. - * @param options Whether to activate the installed bundle (defaults to true), the request id a cancellation names, and - * the pending build scripts to allow for this profile before pnpm runs. - * @returns Package-manager diagnostics and observed activation outcome. + * @param options Whether to activate the installed bundle (defaults to true), the request id a cancellation names, + * the pending build scripts to allow for this profile before pnpm runs, and the registry asked first. + * @returns Package-manager diagnostics, the registries asked, and the observed activation outcome. */ @Remote installBundle(spec: string, options?: InstallBundleOptions): Promise { @@ -340,8 +392,8 @@ export class PluginManager extends TypertRemoteService { const control: InstallControl = { abort: new AbortController(), phase: 'installing', settled: Promise.resolve() } const stopped = (): boolean => control.abort.signal.aborted if (requestId !== undefined) this.installs.set(requestId, control) - const announce = (phase: PluginInstallProgress['phase']): void => { - if (requestId !== undefined) this.ownerContext.emit('plugin-manager/install-state', { requestId, phase }) + const announce = (phase: PluginInstallProgress['phase'], attempt?: PluginInstallProgress['attempt']): void => { + if (requestId !== undefined) this.ownerContext.emit('plugin-manager/install-state', { requestId, phase, ...attempt === undefined ? {} : { attempt } }) } const result = this.change(async (result) => { if (spec.trim() === '' || spec.startsWith('-')) throw new ManagementFailure('invalid-spec') @@ -352,11 +404,21 @@ export class PluginManager extends TypertRemoteService { } const files = await this.readRestoredFiles() const before = readProfileManifest('dsh', this.profile.dir).dependencies ?? {} - announce('installing') + const plan = registryPlan(options?.registry, this.configuredRegistries) let name: string try { - result.packageResult = await this.runPnpm(['add', spec], control.abort.signal, requestId) - if (stopped()) throw new InstallCancelledError() + result.registries = [] + for (const [index, registry] of plan.entries()) { + if (index > 0) await this.restoreFiles(files) + result.registries.push(registry) + announce('installing', { registry, index: index + 1, total: plan.length }) + result.packageResult = await this.runPnpm(['add', spec, ...registryArguments(registry)], control.abort.signal, requestId) + if (stopped()) throw new InstallCancelledError() + const { exitCode, kind, output } = result.packageResult + /* v8 ignore next 2 -- runPnpm classifies every failed run, so kind is never absent here */ + if (exitCode === 0 || index === plan.length - 1 || kind === undefined) break + if (!askNextRegistry(kind, output, parsedForRegistry(spec))) break + } if (result.packageResult.exitCode !== 0) { // pnpm-workspace.yaml is not restored, so the names pnpm left undecided there can be offered for approval. try { result.pendingBuilds = await readPendingBuilds(this.profile.dir) } diff --git a/packages/boot/plugin-manager/src/install-failure.ts b/packages/boot/plugin-manager/src/install-failure.ts index a855963fe7..dc5888ead3 100644 --- a/packages/boot/plugin-manager/src/install-failure.ts +++ b/packages/boot/plugin-manager/src/install-failure.ts @@ -25,7 +25,7 @@ const LOG_KINDS: readonly [PluginInstallFailureKind, RegExp][] = [ ['disk-full', /\bENOSPC\b|no space left on device/i], ['permission', /\bEACCES\b|\bEPERM\b|permission denied/i], ['integrity', /ERR_PNPM_TARBALL_INTEGRITY|ERR_PNPM_BAD_TARBALL_SIZE|\bEINTEGRITY\b/], - ['network', /\bENOTFOUND\b|\bECONNRESET\b|\bETIMEDOUT\b|\bECONNREFUSED\b|\bEAI_AGAIN\b|ERR_PNPM_META_FETCH_FAIL|ERR_PNPM_FETCH_5\d\d|ERR_PNPM_FETCH_TIMEOUT|socket hang up|Could not resolve host|unable to access/], + ['network', /\bENOTFOUND\b|\bECONNRESET\b|\bETIMEDOUT\b|\bECONNREFUSED\b|\bEAI_AGAIN\b|ERR_PNPM_META_FETCH_FAIL|ERR_PNPM_NO_OFFLINE_META|ERR_PNPM_FETCH_5\d\d|ERR_PNPM_FETCH_TIMEOUT|\bFETCH_ERROR\b|socket hang up|Could not resolve host|unable to access/], ] /** diff --git a/packages/boot/plugin-manager/src/install-spec.ts b/packages/boot/plugin-manager/src/install-spec.ts index fa43fd9487..3ef4615584 100644 --- a/packages/boot/plugin-manager/src/install-spec.ts +++ b/packages/boot/plugin-manager/src/install-spec.ts @@ -6,17 +6,25 @@ import { isAbsolute } from 'node:path' -/** One spec read into its form and the parts an inspection needs. */ +/** + * One spec read into its form and the parts an inspection needs. A git spec + * and a tarball URL carry the `host` pnpm fetches them from, which no registry + * stands in for; only their dependencies come from the registry. + */ export type ParsedInstallSpec = | { readonly kind: 'registry'; readonly spec: string; readonly name: string; readonly range?: string } | { readonly kind: 'path'; readonly spec: string; readonly path: string } - | { readonly kind: 'tarball'; readonly spec: string; readonly path?: string } - | { readonly kind: 'git'; readonly spec: string } + | { readonly kind: 'tarball'; readonly spec: string; readonly path?: string; readonly host?: string } + | { readonly kind: 'git'; readonly spec: string; readonly host: string } /** The forms pnpm resolves through a git host: a host shorthand, a git URL, or a hosted repository URL. */ const GIT_SHORTHAND = /^(?:github|gitlab|bitbucket|gist):/i const GIT_URL = /^git(?:\+[a-z]+)?:\/\/|^git@[^:]+:/i const HOSTED_REPOSITORY_URL = /^https?:\/\/[^/]+\/[^/]+\/[^/#]+(?:\.git)?(?:#.*)?$/i +/** The hosts pnpm's shorthands stand for. */ +const GIT_SHORTHAND_HOSTS: Readonly> = { + github: 'github.com', gitlab: 'gitlab.com', bitbucket: 'bitbucket.org', gist: 'gist.github.com', +} /** A tarball, on disk or over HTTP. */ const TARBALL_SPEC = /\.(?:tgz|tar\.gz)(?:#.*)?$/i /** An npm package name: lowercase URL-safe segments, an optional scope, no leading dot or underscore. */ @@ -39,6 +47,15 @@ function invalid(spec: string, reason: string): InvalidInstallSpecError { return new InvalidInstallSpecError(spec, reason) } +/** The host a git spec is cloned from: the shorthand's host, the scp-like user@host, or the URL's host with its port. */ +function gitHost(spec: string): string { + const shorthand = /^([a-z]+):/i.exec(spec)?.[1]?.toLowerCase() + if (shorthand !== undefined && Object.hasOwn(GIT_SHORTHAND_HOSTS, shorthand)) return GIT_SHORTHAND_HOSTS[shorthand] as string + const scp = /^git@([^:]+):/i.exec(spec)?.[1] + if (scp !== undefined) return scp.toLowerCase() + return new URL(spec.replace(/^git\+/i, '')).host +} + /** * Read a spec into its form. A path must be absolute: the Host's working * directory means nothing to the person typing into a browser, and a @@ -58,9 +75,9 @@ export function parseInstallSpec(raw: string): ParsedInstallSpec { } if (/^\.{1,2}(?:[\\/]|$)/.test(spec)) throw invalid(spec, 'a local path must be absolute') const git = GIT_SHORTHAND.test(spec) || GIT_URL.test(spec) || HOSTED_REPOSITORY_URL.test(spec) - if (git && !TARBALL_SPEC.test(spec)) return { kind: 'git', spec } + if (git && !TARBALL_SPEC.test(spec)) return { kind: 'git', spec, host: gitHost(spec) } if (/^https?:\/\//i.test(spec)) { - if (TARBALL_SPEC.test(spec)) return { kind: 'tarball', spec } + if (TARBALL_SPEC.test(spec)) return { kind: 'tarball', spec, host: new URL(spec).host } throw invalid(spec, 'a URL must point at a git repository or a tarball') } const at = spec.indexOf('@', 1) diff --git a/packages/boot/plugin-manager/src/operations.ts b/packages/boot/plugin-manager/src/operations.ts index 9954739d42..88ef88cc33 100644 --- a/packages/boot/plugin-manager/src/operations.ts +++ b/packages/boot/plugin-manager/src/operations.ts @@ -9,7 +9,7 @@ import { resolveBundleDir, resolveProfileDir, loadOverlayPatches, type ProfileManifest, } from '@deepseek-ai/dsh-app-boot' import { scrubbedParentEnv } from '@deepseek-ai/dsh-subprocess' -import type { PackageResult } from './types.ts' +import type { PackageResult, Registry } from './types.ts' /** Profile and invocation locations supplied by the launcher. */ export interface PackageOperationContext { @@ -209,18 +209,35 @@ export interface PackageViewOptions { signal?: AbortSignal /** Bound on the lookup, in milliseconds. */ timeoutMs: number + /** The registry asked; null asks the one pnpm's own configuration names. */ + registry?: Registry +} + +/** + * The argument that sends one pnpm command to a registry. + * @param registry - the registry, or null for the one pnpm's own configuration names. + * @returns `--registry=` for a URL; nothing for null. + */ +export function registryArguments(registry: Registry): string[] { + return registry === null ? [] : [`--registry=${registry}`] } /** * Ask the registry what a spec names through `pnpm view`, run in the profile - * directory so the registry, proxy, and authentication settings of an install apply. + * directory so the registry, proxy, and authentication settings of an install + * apply. The lookup makes one request without pnpm's own retries: a registry + * that does not answer is reported within `timeoutMs`, and the registries + * configured after it are the retry. * @param dir Profile directory. * @param spec One registry spec: a package name with an optional range. - * @param options Cancellation and the time bound. + * @param options The registry, cancellation, and the time bound. * @returns pnpm's exit, output, and how the lookup ended. */ export async function viewProfilePackage(dir: string, spec: string, options: PackageViewOptions): Promise { - const result = await execa(options.command ?? 'pnpm', [...options.args ?? [], 'view', spec, 'name', 'version', 'description', 'dsh', '--json'], { + const result = await execa(options.command ?? 'pnpm', [ + ...options.args ?? [], 'view', spec, 'name', 'version', 'description', 'dsh', '--json', + ...registryArguments(options.registry ?? null), '--config.fetch-retries=0', + ], { cwd: dir, env: { ...scrubbedParentEnv(), ...options.env }, extendEnv: false, reject: false, stdin: 'ignore', timeout: options.timeoutMs, ...options.signal === undefined ? {} : { cancelSignal: options.signal }, }) diff --git a/packages/boot/plugin-manager/src/registry.ts b/packages/boot/plugin-manager/src/registry.ts new file mode 100644 index 0000000000..80e14ead03 --- /dev/null +++ b/packages/boot/plugin-manager/src/registry.ts @@ -0,0 +1,62 @@ +/** + * Which registries one package operation asks, in order, and whether a failed + * attempt sends it to the next one. A registry is asked with pnpm's + * `--registry`; `null` is the one pnpm's own configuration names. + * @module @deepseek-ai/dsh-plugin-manager/registry + */ + +import type { ParsedInstallSpec } from './install-spec.ts' +import type { PluginInstallFailureKind, PluginRegistries, Registry } from './types.ts' + +/** + * Parse a registry URL into the form pnpm compares registries in: lower-case host, trailing slash. + * @param url - the registry as configured or requested. + * @returns the normalized URL. + * @throws {Error} for anything but an http(s) URL. + */ +export function normalizeRegistry(url: string): string { + let parsed: URL | undefined + try { parsed = new URL(url) } + catch { /* named below: only an http(s) URL is a registry */ } + if (parsed === undefined || (parsed.protocol !== 'http:' && parsed.protocol !== 'https:')) { + throw new Error(`a registry must be an http(s) URL: ${url}`) + } + if (!parsed.pathname.endsWith('/')) parsed.pathname += '/' + return parsed.href +} + +/** + * The registries one operation asks, first to last. + * @param requested - the caller's registry; undefined defers to the configured first one. + * @param configured - the configured first registry and the fallbacks after it. + * @returns the requested or configured registry first, then the rest of the configured set; a requested + * registry outside that set is asked alone, so a private registry never falls through to a public one. + */ +export function registryPlan(requested: Registry | undefined, configured: PluginRegistries): Registry[] { + const normalize = (registry: Registry): Registry => registry === null ? null : normalizeRegistry(registry) + const known: Registry[] = [] + for (const registry of [configured.registry, ...configured.fallbackRegistries]) { + const normalized = normalize(registry) + if (!known.includes(normalized)) known.push(normalized) + } + const first = normalize(requested === undefined ? configured.registry : requested) + if (!known.includes(first)) return [first] + return [first, ...known.filter(registry => registry !== first)] +} + +/** The failures after which another registry can answer differently: this one was unreachable, or its copy may be stale. */ +const NEXT_REGISTRY_KINDS: ReadonlySet = new Set(['network', 'timeout', 'not-found', 'no-matching-version']) + +/** + * Whether the next registry is worth asking after a failed attempt. + * @param kind - how the attempt failed. + * @param log - what the attempt printed. + * @param spec - the spec the attempt installed. + * @returns true for a failure another registry can change; false for one it cannot, and for a failure that + * names the host a git or tarball spec is fetched from, which no registry stands in for. + */ +export function askNextRegistry(kind: PluginInstallFailureKind, log: string, spec: ParsedInstallSpec): boolean { + if (!NEXT_REGISTRY_KINDS.has(kind)) return false + const host = spec.kind === 'git' || spec.kind === 'tarball' ? spec.host : undefined + return host === undefined || !log.toLowerCase().includes(host.toLowerCase()) +} diff --git a/packages/boot/plugin-manager/src/tools.ts b/packages/boot/plugin-manager/src/tools.ts index e7c09ba9ae..a5b1cd6837 100644 --- a/packages/boot/plugin-manager/src/tools.ts +++ b/packages/boot/plugin-manager/src/tools.ts @@ -23,6 +23,7 @@ export function apply(ctx: Context): void { target: { type: 'string', description: 'Plugin entry id, bundle package name, or installation spec, according to action.' }, enabled: { type: 'boolean', description: 'Required for set operations; defaults to true for installation.' }, approvedBuilds: { type: 'array', items: { type: 'string' }, description: 'For install_bundle: pass names from pendingBuilds only after the user explicitly approves running their install scripts in the conversation. This grants persistent permission for this profile.' }, + registry: { type: 'string', description: 'For install_bundle: the npm registry URL asked first, when the user names one; otherwise the configured registry is asked, and its configured fallbacks while a registry is unreachable.' }, offset: { type: 'number', description: 'Zero-based list offset; defaults to 0.' }, limit: { type: 'number', description: 'List page size, from 1 to 100; defaults to 25.' }, }, @@ -64,6 +65,7 @@ export function apply(ctx: Context): void { return JSON.stringify(await manager.installBundle(args.target, { ...args.enabled === undefined ? {} : { enabled: args.enabled }, ...args.approvedBuilds === undefined ? {} : { approvedBuilds: args.approvedBuilds }, + ...args.registry === undefined ? {} : { registry: args.registry }, })) case 'remove_bundle': if (args.target === undefined) throw new Error('target bundle name is required') diff --git a/packages/boot/plugin-manager/src/types.ts b/packages/boot/plugin-manager/src/types.ts index 52d29ebbcb..9917a4c4b2 100644 --- a/packages/boot/plugin-manager/src/types.ts +++ b/packages/boot/plugin-manager/src/types.ts @@ -52,6 +52,15 @@ export interface BundleInfo { overrides: string[] } +/** A registry to install from: an http(s) URL, or null for the one pnpm's own configuration names. */ +export type Registry = string | null + +/** The registries the manager asks: the configured first one, then its fallbacks in order. */ +export interface PluginRegistries { + readonly registry: Registry + readonly fallbackRegistries: readonly string[] +} + /** How a pnpm run failed, read off how it ended and what it printed. */ export type PluginInstallFailureKind = | 'pnpm-missing' @@ -94,6 +103,8 @@ export interface ChangeResult { pendingBuilds?: string[] /** Package script permissions saved before this installation attempt. */ approvedBuilds?: string[] + /** The registries the installation asked, in order; `packageResult` is the last one's run. */ + registries?: Registry[] } /** Identifies one installation from its start to its settlement, including its log chunks and cancellation. */ @@ -105,6 +116,14 @@ export interface InstallBundleOptions { requestId?: PluginInstallRequestId /** Explicitly allow these pending packages' scripts for this profile, then install; a name no longer pending refuses the call. */ approvedBuilds?: string[] + /** The registry asked first; absent, the configured one. The configured fallbacks follow while a registry is unreachable or stale. */ + registry?: Registry +} + +/** Where an inspection asks. */ +export interface InspectOptions { + /** The registry asked first; absent, the configured one. */ + readonly registry?: Registry } /** The form one install spec takes, in pnpm's vocabulary. */ @@ -135,18 +154,24 @@ export type PluginSpecInspection = readonly description?: string /** Whether the package declares a bundle patch; null when the spec's form does not say. */ readonly bundle: boolean | null + /** The registry that answered for a name, and for the other forms the one an install of the spec asks first. */ + readonly registry: Registry } | { readonly status: 'refused' readonly problem: PluginInspectProblem /** What pnpm, the registry, or the file system said. */ readonly reason: string + /** The registries asked, in order, when the refusal came from asking them; `reason` is the last one's. */ + readonly registries?: Registry[] } -/** The Host phase of one installation, before its install call settles. */ +/** The Host phase of one installation, before its install call settles; `installing` is announced once per registry asked. */ export interface PluginInstallProgress { readonly requestId: PluginInstallRequestId readonly phase: 'installing' | 'cancelling' | 'applying' + /** With `installing`: the registry this attempt asks, its one-based position, and how many the installation may ask. */ + readonly attempt?: { readonly registry: Registry; readonly index: number; readonly total: number } } /** Cancellation is confirmed only after process exit and file restoration. */ diff --git a/packages/boot/plugin-manager/tests/install-spec.spec.ts b/packages/boot/plugin-manager/tests/install-spec.spec.ts index f4ea1d7d9f..7165e80412 100644 --- a/packages/boot/plugin-manager/tests/install-spec.spec.ts +++ b/packages/boot/plugin-manager/tests/install-spec.spec.ts @@ -17,11 +17,16 @@ describe('parseInstallSpec', () => { expect(parseInstallSpec('/packs/dsh-x-1.0.0.tgz')).toEqual({ kind: 'tarball', spec: '/packs/dsh-x-1.0.0.tgz', path: '/packs/dsh-x-1.0.0.tgz' }) }) - it('reads git hosts and tarball URLs', () => { - for (const spec of ['github:someone/dsh-plugin', 'gitlab:a/b#main', 'git+ssh://git@github.com/a/b.git', 'git://host/a/b', 'git@github.com:a/b.git', 'https://github.com/a/b', 'https://github.com/a/b.git#v1']) { - expect(parseInstallSpec(spec)).toEqual({ kind: 'git', spec }) + it('reads git hosts and tarball URLs, naming the host the spec itself is fetched from', () => { + const hosts: Record = { + 'github:someone/dsh-plugin': 'github.com', 'gitlab:a/b#main': 'gitlab.com', 'bitbucket:a/b': 'bitbucket.org', 'gist:0123abcd': 'gist.github.com', + 'git+ssh://git@github.com/a/b.git': 'github.com', 'git://host/a/b': 'host', 'git@github.com:a/b.git': 'github.com', + 'https://github.com/a/b': 'github.com', 'https://github.com/a/b.git#v1': 'github.com', 'git+https://Git.Example.com:8443/a/b.git': 'git.example.com:8443', } - expect(parseInstallSpec('https://cdn.example.com/x/y/z/dsh-x-1.0.0.tgz')).toEqual({ kind: 'tarball', spec: 'https://cdn.example.com/x/y/z/dsh-x-1.0.0.tgz' }) + for (const [spec, host] of Object.entries(hosts)) { + expect(parseInstallSpec(spec)).toEqual({ kind: 'git', spec, host }) + } + expect(parseInstallSpec('https://cdn.example.com/x/y/z/dsh-x-1.0.0.tgz')).toEqual({ kind: 'tarball', spec: 'https://cdn.example.com/x/y/z/dsh-x-1.0.0.tgz', host: 'cdn.example.com' }) }) it('refuses what neither the registry nor pnpm would take, naming why', () => { @@ -65,6 +70,8 @@ describe('classifyInstallFailure', () => { expect(classifyInstallFailure({ log: 'ECONNRESET' })).toBe('network') expect(classifyInstallFailure({ log: 'ERR_PNPM_FETCH_502 GET https://registry/x: Bad Gateway' })).toBe('network') expect(classifyInstallFailure({ log: 'fatal: unable to access https://github.com/a/b/: Could not resolve host' })).toBe('network') + expect(classifyInstallFailure({ log: 'ERR_PNPM_NO_OFFLINE_META Failed to resolve x@1 in package mirror' })).toBe('network') + expect(classifyInstallFailure({ log: 'npm error code FETCH_ERROR\nnpm error errno FETCH_ERROR' })).toBe('network') expect(classifyInstallFailure({ log: 'exited with 1', cause: new Error('no code') })).toBe('unknown') expect(classifyInstallFailure({ log: '' })).toBe('unknown') }) diff --git a/packages/boot/plugin-manager/tests/manager.spec.ts b/packages/boot/plugin-manager/tests/manager.spec.ts index 7592ab2919..c61f9cd9cb 100644 --- a/packages/boot/plugin-manager/tests/manager.spec.ts +++ b/packages/boot/plugin-manager/tests/manager.spec.ts @@ -286,10 +286,11 @@ it('restores the manifest and lockfile after a failed package run, classifying t }) onTestFinished(() => { install.mockRestore() }) const before = readFileSync(join(dir, 'package.json'), 'utf8') + // A registry that does not answer sends the installation on to the configured fallback; the failure reported is the last one's. expect(await manager.installBundle('partial')).toMatchObject({ changed: false, application: 'failed', stage: 'install', target: 'partial', error: { code: 'operation-error', diagnostic: expect.stringContaining('ENOTFOUND') as string }, - packageResult: { exitCode: 42, kind: 'network' }, + packageResult: { exitCode: 42, kind: 'network' }, registries: [null, MIRROR], }) expect(readFileSync(join(dir, 'package.json'), 'utf8')).toBe(before) // A lockfile the run created is removed; one that existed is put back. @@ -298,7 +299,7 @@ it('restores the manifest and lockfile after a failed package run, classifying t expect(await manager.installBundle('partial')).toMatchObject({ changed: false, application: 'failed' }) expect(readFileSync(lockPath, 'utf8')).toBe('original lockfile\n') expect((await manager.listBundles()).some(row => row.name === 'partial')).toBe(false) - expect(install).toHaveBeenCalledTimes(2) + expect(install).toHaveBeenCalledTimes(4) }) it('keeps saved changes after activation failure and allows a corrected configuration to retry', async () => { @@ -504,7 +505,7 @@ it('streams pnpm output, reports the installation phases, and names the installe { requestId, jobId, argv: ['pnpm', 'add', 'streamed'], cwd: dir, stream: 'stderr', text: 'warning\n' }, { requestId, jobId, argv: ['pnpm', 'add', 'streamed'], cwd: dir, stream: 'stdout', text: '', exitCode: 0 }, ]) - expect(phases).toEqual([{ requestId, phase: 'installing' }, { requestId, phase: 'applying' }]) + expect(phases).toEqual([{ requestId, phase: 'installing', attempt: { registry: null, index: 1, total: 2 } }, { requestId, phase: 'applying' }]) expect(changes).toEqual([{ reason: 'install' }]) // A run without a request id streams too, unidentified. await manager.removeBundle('streamed') @@ -537,7 +538,7 @@ it('stops a run on request, restores the files, and answers not-running or too-l const cancelled = await run expect(cancelled).toMatchObject({ application: 'cancelled', changed: false, stage: 'install', packageResult: { exitCode: 1 } }) expect(cancelled.error).toBeUndefined() - expect(phases).toEqual([{ requestId, phase: 'installing' }, { requestId, phase: 'cancelling' }]) + expect(phases).toEqual([{ requestId, phase: 'installing', attempt: { registry: null, index: 1, total: 2 } }, { requestId, phase: 'cancelling' }]) expect(await manager.cancelInstall(requestId)).toEqual({ status: 'not-running' }) // Once pnpm has exited and the bundle is being applied, the run cannot be stopped. install.mockImplementation(async (_context, args) => { @@ -561,22 +562,22 @@ it('stops a run on request, restores the files, and answers not-running or too-l }) it('reads what a spec names before installing it', async () => { - const { manager, dir, profile } = await fixture(undefined, false, undefined, { inspectTimeoutMs: 1000, pnpmCommand: 'pnpm-test' }) + const { manager, dir, profile } = await fixture(undefined, false, undefined, { inspectTimeoutMs: 1000, pnpmCommand: 'pnpm-test', fallbackRegistries: [] }) const view = vi.spyOn(operations, 'viewProfilePackage') onTestFinished(() => { view.mockRestore() }) const answers = (stdout: string) => view.mockResolvedValueOnce({ exitCode: 0, stdout, stderr: '', timedOut: false }) answers(JSON.stringify({ name: 'dsh-x', version: '1.4.2', description: 'A sidebar.', dsh: { bundle: { patch: './cordis.patch.yml' } } })) expect(await manager.inspect('dsh-x')).toEqual({ - status: 'accepted', kind: 'registry', name: 'dsh-x', version: '1.4.2', description: 'A sidebar.', bundle: true, + status: 'accepted', kind: 'registry', name: 'dsh-x', version: '1.4.2', description: 'A sidebar.', bundle: true, registry: null, }) - expect(view).toHaveBeenCalledWith(dir, 'dsh-x', { command: 'pnpm-test', timeoutMs: 1000 }) + expect(view).toHaveBeenCalledWith(dir, 'dsh-x', { command: 'pnpm-test', timeoutMs: 1000, registry: null }) const signal = AbortSignal.abort() answers(JSON.stringify([{ name: 'dsh-lib', version: '1.0.0', dsh: { bundle: {} } }, { name: 'dsh-lib', version: '1.1.0', dsh: null }])) - expect(await manager.inspect('dsh-lib@^1', signal)).toEqual({ status: 'refused', problem: 'not-a-bundle', reason: 'dsh-lib declares no dsh.bundle' }) - expect(view).toHaveBeenLastCalledWith(dir, 'dsh-lib@^1', { command: 'pnpm-test', timeoutMs: 1000, signal }) + expect(await manager.inspect('dsh-lib@^1', undefined, signal)).toEqual({ status: 'refused', problem: 'not-a-bundle', reason: 'dsh-lib declares no dsh.bundle', registries: [null] }) + expect(view).toHaveBeenLastCalledWith(dir, 'dsh-lib@^1', { command: 'pnpm-test', timeoutMs: 1000, signal, registry: null }) // An answer that names no package keeps the name the spec gave; colour escapes around the JSON are dropped. answers('\x1b[36m' + JSON.stringify({ version: '0.0.1', description: '', dsh: { bundle: { patch: './p.yml' } } }) + '\x1b[39m\n') - expect(await manager.inspect('dsh-bare')).toEqual({ status: 'accepted', kind: 'registry', name: 'dsh-bare', version: '0.0.1', bundle: true }) + expect(await manager.inspect('dsh-bare')).toEqual({ status: 'accepted', kind: 'registry', name: 'dsh-bare', version: '0.0.1', bundle: true, registry: null }) const failure = (stderr: string, exitCode: number | null = 1, more: Partial = {}) => view.mockResolvedValueOnce({ exitCode, stdout: '', stderr, timedOut: false, ...more }) failure('npm error code E404\nnpm error 404 Not Found - GET https://registry/nope\n') @@ -586,28 +587,28 @@ it('reads what a spec names before installing it', async () => { failure('ERR_PNPM_META_FETCH_FAIL request failed, reason: getaddrinfo ENOTFOUND registry\n') expect(await manager.inspect('far')).toMatchObject({ status: 'refused', problem: 'network' }) view.mockResolvedValueOnce({ exitCode: 3, stdout: 'plain text\n', stderr: '', timedOut: false }) - expect(await manager.inspect('odd')).toEqual({ status: 'refused', problem: 'unknown', reason: 'plain text' }) + expect(await manager.inspect('odd')).toEqual({ status: 'refused', problem: 'unknown', reason: 'plain text', registries: [null] }) failure('', 4) - expect(await manager.inspect('quiet')).toEqual({ status: 'refused', problem: 'unknown', reason: 'pnpm view exited with 4' }) + expect(await manager.inspect('quiet')).toEqual({ status: 'refused', problem: 'unknown', reason: 'pnpm view exited with 4', registries: [null] }) failure('', null, { timedOut: true }) - expect(await manager.inspect('slow')).toEqual({ status: 'refused', problem: 'unknown', reason: 'pnpm view timed out after 1000ms' }) + expect(await manager.inspect('slow')).toEqual({ status: 'refused', problem: 'unknown', reason: 'pnpm view timed out after 1000ms', registries: [null] }) failure('', null, { cause: Object.assign(new Error('spawn pnpm ENOENT'), { code: 'ENOENT' }) }) expect(await manager.inspect('gone')).toMatchObject({ status: 'refused', problem: 'unknown', reason: expect.stringContaining('ENOENT') as string }) answers('not json') expect(await manager.inspect('garbled')).toMatchObject({ status: 'refused', problem: 'unknown', reason: expect.stringContaining('unreadable pnpm view output') as string }) answers('"just a string"') - expect(await manager.inspect('scalar')).toEqual({ status: 'refused', problem: 'unknown', reason: 'pnpm view answered no package' }) + expect(await manager.inspect('scalar')).toEqual({ status: 'refused', problem: 'unknown', reason: 'pnpm view answered no package', registries: [null] }) answers('') - expect(await manager.inspect('silent')).toEqual({ status: 'refused', problem: 'unknown', reason: 'pnpm view answered no package' }) + expect(await manager.inspect('silent')).toEqual({ status: 'refused', problem: 'unknown', reason: 'pnpm view answered no package', registries: [null] }) // What is installed, or supplied by the installation, is refused before the registry is asked. expect(await manager.inspect('extra')).toEqual({ status: 'refused', problem: 'already-installed', reason: 'extra is already installed' }) expect(await manager.inspect('./relative')).toEqual({ status: 'refused', problem: 'invalid-spec', reason: 'a local path must be absolute' }) - expect(await manager.inspect('github:acme/dsh-remote')).toEqual({ status: 'accepted', kind: 'git', bundle: null }) + expect(await manager.inspect('github:acme/dsh-remote')).toEqual({ status: 'accepted', kind: 'git', bundle: null, registry: null }) const tarball = join(profile.home, 'pack.tgz') expect(await manager.inspect(tarball)).toEqual({ status: 'refused', problem: 'not-a-package', reason: 'the tarball does not exist' }) writeFileSync(tarball, '') - expect(await manager.inspect(tarball)).toEqual({ status: 'accepted', kind: 'tarball', bundle: null }) - expect(await manager.inspect('https://cdn.example.com/x/y/z/dsh-x-1.0.0.tgz')).toEqual({ status: 'accepted', kind: 'tarball', bundle: null }) + expect(await manager.inspect(tarball)).toEqual({ status: 'accepted', kind: 'tarball', bundle: null, registry: null }) + expect(await manager.inspect('https://cdn.example.com/x/y/z/dsh-x-1.0.0.tgz')).toEqual({ status: 'accepted', kind: 'tarball', bundle: null, registry: null }) // A directory answers from its own manifest. const local = join(profile.home, 'dev', 'dsh-local') mkdirSync(local, { recursive: true }) @@ -618,13 +619,13 @@ it('reads what a spec names before installing it', async () => { writeFileSync(join(local, 'package.json'), JSON.stringify({ name: 'dsh-local', version: '0.1.0', description: 'Local.' })) expect(await manager.inspect(local)).toEqual({ status: 'refused', problem: 'not-a-bundle', reason: 'dsh-local declares no dsh.bundle' }) writeFileSync(join(local, 'package.json'), JSON.stringify({ name: 'dsh-local', version: '0.1.0', description: 'Local.', dsh: { bundle: { patch: './p.yml' } } })) - expect(await manager.inspect(`file:${local}`)).toEqual({ status: 'accepted', kind: 'path', name: 'dsh-local', version: '0.1.0', description: 'Local.', bundle: true }) + expect(await manager.inspect(`file:${local}`)).toEqual({ status: 'accepted', kind: 'path', name: 'dsh-local', version: '0.1.0', description: 'Local.', bundle: true, registry: null }) writeFileSync(join(local, 'package.json'), JSON.stringify({ name: 'core', dsh: { bundle: { patch: './p.yml' } } })) expect(await manager.inspect(local)).toEqual({ status: 'refused', problem: 'already-installed', reason: 'core is already installed' }) // A profile and an installation that list nothing know nothing. writeFileSync(join(dir, 'package.json'), '{}') writeFileSync(profile.installAnchor, '{}') - expect(await manager.inspect(local)).toEqual({ status: 'accepted', kind: 'path', name: 'core', bundle: true }) + expect(await manager.inspect(local)).toEqual({ status: 'accepted', kind: 'path', name: 'core', bundle: true, registry: null }) expect(view).toHaveBeenCalledTimes(13) }) @@ -767,3 +768,111 @@ it('installs and removes with the bundled pnpm when PATH contains no pnpm', asyn expect(removed.packageResult?.exitCode).toBe(0) expect(readProfileManifest('test', dir).dependencies ?? {}).not.toHaveProperty('@test/desktop-manager') }) + +const MIRROR = 'https://registry.npmmirror.com/' + +it('asks the registries in turn while one is unreachable or stale, and names the one that answered', async () => { + const { manager } = await fixture(undefined, false, undefined, { inspectTimeoutMs: 1000, fallbackRegistries: ['https://REGISTRY.npmmirror.com'] }) + const view = vi.spyOn(operations, 'viewProfilePackage') + onTestFinished(() => { view.mockRestore() }) + const failure = (stderr: string, exitCode = 1) => view.mockResolvedValueOnce({ exitCode, stdout: '', stderr, timedOut: false }) + const answers = (stdout: string) => view.mockResolvedValueOnce({ exitCode: 0, stdout, stderr: '', timedOut: false }) + const asked = (from: number) => view.mock.calls.slice(from).map(call => call[2].registry) + const manifest = JSON.stringify({ name: 'dsh-x', version: '1.0.0', dsh: { bundle: { patch: './p.yml' } } }) + // pnpm's own registry is unreachable; the mirror answers and the install is told to start there. + failure('ERR_PNPM_META_FETCH_FAIL GET https://registry.npmjs.org/dsh-x: ETIMEDOUT\n') + answers(manifest) + expect(await manager.inspect('dsh-x')).toEqual({ status: 'accepted', kind: 'registry', name: 'dsh-x', version: '1.0.0', bundle: true, registry: MIRROR }) + expect(asked(0)).toEqual([null, MIRROR]) + expect(view).toHaveBeenLastCalledWith(expect.any(String), 'dsh-x', { command: 'pnpm', timeoutMs: 1000, registry: MIRROR }) + // A requested registry goes first. A mirror's 404 may be a copy not yet synced, so the lookup goes on; every registry's 404 is not-found. + failure('ERR_PNPM_FETCH_404 GET https://registry.npmmirror.com/dsh-x: Not Found - 404\n') + failure('ERR_PNPM_FETCH_404 GET https://registry.npmjs.org/dsh-x: Not Found - 404\n') + expect(await manager.inspect('dsh-x', { registry: MIRROR })).toMatchObject({ status: 'refused', problem: 'not-found', registries: [MIRROR, null] }) + expect(asked(2)).toEqual([MIRROR, null]) + // A registry outside the configured set is asked alone. + failure('ERR_PNPM_META_FETCH_FAIL GET https://npm.corp.example/dsh-x: ECONNREFUSED\n') + expect(await manager.inspect('dsh-x', { registry: 'https://npm.corp.example' })).toMatchObject({ status: 'refused', problem: 'network', registries: ['https://npm.corp.example/'] }) + expect(asked(4)).toEqual(['https://npm.corp.example/']) + // A failure no registry changes ends the round at once. + failure('', 4) + expect(await manager.inspect('dsh-x')).toMatchObject({ status: 'refused', problem: 'unknown', registries: [null] }) + expect(asked(5)).toEqual([null]) + // A lookup the caller dropped is not carried to the next registry. + const controller = new AbortController() + view.mockImplementationOnce(async () => { controller.abort(); return { exitCode: 1, stdout: '', stderr: 'ECONNRESET\n', timedOut: false } }) + expect(await manager.inspect('dsh-x', {}, controller.signal)).toMatchObject({ status: 'refused', problem: 'network', registries: [null] }) + expect(asked(6)).toEqual([null]) + // The other forms ask no registry and carry the one the install starts with. + expect(await manager.inspect('github:acme/dsh-remote', { registry: MIRROR })).toEqual({ status: 'accepted', kind: 'git', bundle: null, registry: MIRROR }) + expect(await manager.inspect('github:acme/dsh-remote')).toEqual({ status: 'accepted', kind: 'git', bundle: null, registry: null }) + expect(view).toHaveBeenCalledTimes(7) +}) + +it('installs from the next registry after one is unreachable, restoring the files between attempts', async () => { + const { ctx, manager, dir, bundle } = await fixture() + const phases: PluginInstallProgress[] = [] + const chunks: PluginInstallLogChunk[] = [] + ctx.on('plugin-manager/install-state', (progress) => { phases.push(progress) }) + ctx.on('plugin-manager/install-log', (chunk) => { chunks.push(chunk) }) + const lockPath = join(dir, 'pnpm-lock.yaml') + const install = vi.spyOn(operations, 'runProfilePnpm') + .mockImplementationOnce(async () => { + writeFileSync(lockPath, 'partial lockfile\n') + return { exitCode: 1, output: 'ERR_PNPM_META_FETCH_FAIL GET https://registry.npmjs.org/fallen: ETIMEDOUT', truncated: false, logPath: join(dir, 'pnpm.log') } + }) + .mockImplementationOnce(async (_context, args) => { + // The failed attempt's lockfile is gone before the next registry is asked. + expect(existsSync(lockPath)).toBe(false) + const name = String(args[1]) + bundle(name, [{ id: name, name: './plugin.mjs', config: { service: name } }]) + const manifest = readProfileManifest('test', dir) + manifest.dependencies = { ...manifest.dependencies, [name]: '1.0.0' } + writeFileSync(join(dir, 'package.json'), JSON.stringify(manifest)) + return { exitCode: 0, output: 'installed', truncated: false, logPath: join(dir, 'pnpm.log') } + }) + onTestFinished(() => { install.mockRestore() }) + const requestId = 'f2340b6d-40bb-46b7-8b94-217bdf5010bd' as PluginInstallRequestId + expect(await manager.installBundle('fallen', { enabled: false, requestId })).toMatchObject({ + application: 'applied', bundle: 'fallen', registries: [null, MIRROR], packageResult: { exitCode: 0 }, + }) + expect(install.mock.calls.map(call => call[1])).toEqual([['add', 'fallen'], ['add', 'fallen', `--registry=${MIRROR}`]]) + expect(phases).toEqual([ + { requestId, phase: 'installing', attempt: { registry: null, index: 1, total: 2 } }, + { requestId, phase: 'installing', attempt: { registry: MIRROR, index: 2, total: 2 } }, + { requestId, phase: 'applying' }, + ]) + // Each attempt streams as its own run, the command line naming the registry it asked. + expect(new Set(chunks.map(chunk => chunk.jobId)).size).toBe(2) + expect(chunks.at(-1)).toMatchObject({ requestId, argv: ['pnpm', 'add', 'fallen', `--registry=${MIRROR}`], exitCode: 0 }) +}) + +it('stops at a failure no registry changes, at a host the spec itself is fetched from, and after a registry outside the configured set', async () => { + const { manager } = await fixture() + const failing = (output: string) => ({ exitCode: 1, output, truncated: false, logPath: '/dev/null' }) + const install = vi.spyOn(operations, 'runProfilePnpm').mockResolvedValue(failing('ERR_PNPM_IGNORED_BUILDS Ignored build scripts: native')) + onTestFinished(() => { install.mockRestore() }) + expect(await manager.installBundle('blocked')).toMatchObject({ application: 'failed', registries: [null], packageResult: { kind: 'build-blocked' } }) + expect(install).toHaveBeenCalledTimes(1) + install.mockResolvedValue(failing('fatal: unable to access \'https://github.com/acme/dsh-x/\': Could not resolve host: github.com')) + expect(await manager.installBundle('github:acme/dsh-x')).toMatchObject({ application: 'failed', registries: [null], packageResult: { kind: 'network' } }) + expect(install).toHaveBeenCalledTimes(2) + install.mockResolvedValue(failing('ERR_PNPM_META_FETCH_FAIL GET https://npm.corp.example/dsh-x: ECONNREFUSED')) + expect(await manager.installBundle('dsh-x', { registry: 'https://npm.corp.example' })).toMatchObject({ + application: 'failed', registries: ['https://npm.corp.example/'], packageResult: { kind: 'network' }, + }) + expect(install).toHaveBeenCalledTimes(3) + expect(install).toHaveBeenLastCalledWith(expect.anything(), ['add', 'dsh-x', '--registry=https://npm.corp.example/'], expect.anything()) + // Every configured registry unreachable: the failure is the last attempt's, and all of them are named. + install.mockResolvedValue(failing('ERR_PNPM_META_FETCH_FAIL GET https://registry/dsh-x: ETIMEDOUT')) + expect(await manager.installBundle('dsh-x')).toMatchObject({ application: 'failed', registries: [null, MIRROR], packageResult: { kind: 'network' } }) + expect(install).toHaveBeenCalledTimes(5) +}) + +it('answers the configured registries in pnpm\'s comparison form, and refuses one that is not an http(s) URL at load', async () => { + expect(await (await fixture()).manager.registries()).toEqual({ registry: null, fallbackRegistries: [MIRROR] }) + const { manager } = await fixture(undefined, false, undefined, { registry: 'https://NPM.corp.example', fallbackRegistries: [] }) + expect(await manager.registries()).toEqual({ registry: 'https://npm.corp.example/', fallbackRegistries: [] }) + expect(() => PluginManager.Config({ registry: 'npm.corp.example' })).toThrow() + expect(() => PluginManager.Config({ fallbackRegistries: ['ftp://npm.corp.example/'] })).toThrow() +}) diff --git a/packages/boot/plugin-manager/tests/operations.spec.ts b/packages/boot/plugin-manager/tests/operations.spec.ts index 23e6a44d0f..fb706017ea 100644 --- a/packages/boot/plugin-manager/tests/operations.spec.ts +++ b/packages/boot/plugin-manager/tests/operations.spec.ts @@ -212,15 +212,21 @@ it('settles inherited CLI descriptors without requiring captured streams', async expect(await runPluginCommand(context, ['approve-builds'], { execution: 'cli', outputBytes: 100 })).toMatchObject({ exitCode: 0, output: '' }) }) -it('asks the registry through pnpm view in the profile directory and reports how the lookup ended', async () => { +it('asks the registry through pnpm view in the profile directory, without pnpm\'s own retries, and reports how the lookup ended', async () => { const { dir } = fixture() const answer = (value: object) => command.run.mockResolvedValueOnce(value as never) answer({ exitCode: 0, stdout: '{"name":"x"}', stderr: '', timedOut: false, isCanceled: false }) expect(await viewProfilePackage(dir, 'x@^1', { timeoutMs: 5 })).toEqual({ exitCode: 0, stdout: '{"name":"x"}', stderr: '', timedOut: false }) - expect(command.run).toHaveBeenLastCalledWith('pnpm', ['view', 'x@^1', 'name', 'version', 'description', 'dsh', '--json'], expect.objectContaining({ - cwd: dir, timeout: 5, reject: false, stdin: 'ignore', - })) + expect(command.run).toHaveBeenLastCalledWith('pnpm', ['view', 'x@^1', 'name', 'version', 'description', 'dsh', '--json', '--config.fetch-retries=0'], + expect.objectContaining({ cwd: dir, timeout: 5, reject: false, stdin: 'ignore' })) expect((command.run.mock.lastCall as unknown[])[2]).not.toHaveProperty('cancelSignal') + // A registry asked by URL goes on the command line; null leaves the choice to pnpm's own configuration. + answer({ exitCode: 0, stdout: '{"name":"x"}', stderr: '', timedOut: false, isCanceled: false }) + await viewProfilePackage(dir, 'x', { timeoutMs: 5, registry: 'https://registry.npmmirror.com/' }) + expect((command.run.mock.lastCall as unknown[])[1]).toEqual(['view', 'x', 'name', 'version', 'description', 'dsh', '--json', '--registry=https://registry.npmmirror.com/', '--config.fetch-retries=0']) + answer({ exitCode: 0, stdout: '{"name":"x"}', stderr: '', timedOut: false, isCanceled: false }) + await viewProfilePackage(dir, 'x', { timeoutMs: 5, registry: null }) + expect((command.run.mock.lastCall as unknown[])[1]).toEqual(['view', 'x', 'name', 'version', 'description', 'dsh', '--json', '--config.fetch-retries=0']) const signal = AbortSignal.abort() answer({ exitCode: undefined, stdout: '', stderr: '', timedOut: true, isCanceled: false }) expect(await viewProfilePackage(dir, 'x', { timeoutMs: 5, signal })).toEqual({ exitCode: null, stdout: '', stderr: '', timedOut: true }) @@ -244,6 +250,6 @@ it('uses application-owned executable arguments and environment for package oper command.run.mockResolvedValueOnce(Object.assign({ exitCode: 0, failed: false }, { stdout: '{}', stderr: '', timedOut: false })) await viewProfilePackage(dir, 'example', { ...runtime, timeoutMs: 1000 }) expect(command.run).toHaveBeenLastCalledWith(runtime.command, - [...runtime.args, 'view', 'example', 'name', 'version', 'description', 'dsh', '--json'], + [...runtime.args, 'view', 'example', 'name', 'version', 'description', 'dsh', '--json', '--config.fetch-retries=0'], expect.objectContaining({ env: expect.objectContaining(runtime.env) as unknown })) }) diff --git a/packages/boot/plugin-manager/tests/registry.spec.ts b/packages/boot/plugin-manager/tests/registry.spec.ts new file mode 100644 index 0000000000..95c77abb14 --- /dev/null +++ b/packages/boot/plugin-manager/tests/registry.spec.ts @@ -0,0 +1,79 @@ +/** Which registries an operation asks, in order, and when a failed attempt sends it to the next one: pure, table-driven. */ + +import { describe, expect, it } from 'vitest' +import { askNextRegistry, normalizeRegistry, parseInstallSpec, registryPlan } from '@deepseek-ai/dsh-plugin-manager' + +const MIRROR = 'https://registry.npmmirror.com/' +const CORP = 'https://npm.corp.example/' + +describe('normalizeRegistry', () => { + it('parses an http(s) URL into pnpm\'s comparison form: lower-case host, trailing slash', () => { + expect(normalizeRegistry('https://REGISTRY.npmmirror.com')).toBe(MIRROR) + expect(normalizeRegistry('http://npm.corp.example:4873/prefix')).toBe('http://npm.corp.example:4873/prefix/') + expect(normalizeRegistry(MIRROR)).toBe(MIRROR) + }) + + it('refuses anything but an http(s) URL', () => { + for (const url of ['registry.npmmirror.com', 'ftp://x/', 'file:///tmp', '', 'https://']) { + expect(() => normalizeRegistry(url), url).toThrow(/http\(s\) URL/) + } + }) +}) + +describe('registryPlan', () => { + const configured = { registry: null, fallbackRegistries: [MIRROR] } + + it('asks the configured registry first, then the fallbacks', () => { + expect(registryPlan(undefined, configured)).toEqual([null, MIRROR]) + }) + + it('moves a requested registry to the front when it is one of the configured set', () => { + expect(registryPlan(MIRROR, configured)).toEqual([MIRROR, null]) + expect(registryPlan('https://REGISTRY.npmmirror.com', configured)).toEqual([MIRROR, null]) + expect(registryPlan(null, { registry: MIRROR, fallbackRegistries: [] })).toEqual([null]) + }) + + it('asks a registry outside the configured set alone, never followed by a public one', () => { + expect(registryPlan(CORP, configured)).toEqual([CORP]) + expect(registryPlan(undefined, { registry: CORP, fallbackRegistries: [] })).toEqual([CORP]) + expect(registryPlan(undefined, { registry: CORP, fallbackRegistries: [MIRROR] })).toEqual([CORP, MIRROR]) + }) + + it('lists each registry once', () => { + expect(registryPlan(undefined, { registry: MIRROR, fallbackRegistries: [MIRROR, CORP] })).toEqual([MIRROR, CORP]) + }) +}) + +describe('askNextRegistry', () => { + const name = parseInstallSpec('dsh-x') + const git = parseInstallSpec('github:acme/dsh-x') + const tarball = parseInstallSpec('https://cdn.example.com/dsh-x-1.0.0.tgz') + const registryLog = 'ERR_PNPM_META_FETCH_FAIL GET https://registry.npmjs.org/dsh-x: getaddrinfo ENOTFOUND registry.npmjs.org' + + it('asks the next registry when this one was unreachable or may hold a stale copy', () => { + for (const kind of ['network', 'timeout', 'not-found', 'no-matching-version'] as const) { + expect(askNextRegistry(kind, registryLog, name), kind).toBe(true) + } + }) + + it('stops on failures no other registry would change', () => { + for (const kind of ['build-blocked', 'disk-full', 'permission', 'integrity', 'pnpm-missing', 'unknown'] as const) { + expect(askNextRegistry(kind, registryLog, name), kind).toBe(false) + } + }) + + it('keeps a failure that names the host a git or tarball spec is fetched from with that spec', () => { + expect(askNextRegistry('network', 'fatal: unable to access \'https://github.com/acme/dsh-x/\': Could not resolve host: github.com', git)).toBe(false) + expect(askNextRegistry('network', 'ssh: Could not resolve hostname GITHUB.COM: nodename nor servname provided', git)).toBe(false) + expect(askNextRegistry('network', 'ERR_PNPM_FETCH_502 GET https://cdn.example.com/dsh-x-1.0.0.tgz: Bad Gateway', tarball)).toBe(false) + // The same specs' dependencies still come from the registry. + expect(askNextRegistry('network', registryLog, git)).toBe(true) + expect(askNextRegistry('network', registryLog, tarball)).toBe(true) + expect(askNextRegistry('not-found', 'ERR_PNPM_FETCH_404 GET https://registry.npmjs.org/left-pad: Not Found - 404', git)).toBe(true) + }) + + it('reads a failure that names no host as the registry\'s', () => { + expect(askNextRegistry('network', 'ECONNRESET', git)).toBe(true) + expect(askNextRegistry('network', 'socket hang up', name)).toBe(true) + }) +}) diff --git a/packages/boot/plugin-manager/tests/tools.spec.ts b/packages/boot/plugin-manager/tests/tools.spec.ts index fb867e50cd..b321f01e3a 100644 --- a/packages/boot/plugin-manager/tests/tools.spec.ts +++ b/packages/boot/plugin-manager/tests/tools.spec.ts @@ -170,6 +170,8 @@ it('forwards all mutation actions and renders the returned outcome', async () => expect(manager.installBundle).toHaveBeenLastCalledWith('bundle', { enabled: false }) await call({ action: 'install_bundle', target: 'bundle', approvedBuilds: ['native'] }) expect(manager.installBundle).toHaveBeenLastCalledWith('bundle', { approvedBuilds: ['native'] }) + await call({ action: 'install_bundle', target: 'bundle', registry: 'https://registry.npmmirror.com/' }) + expect(manager.installBundle).toHaveBeenLastCalledWith('bundle', { registry: 'https://registry.npmmirror.com/' }) expect(resultText(await call({ action: 'remove_bundle', target: 'bundle' }))).toContain('"application":"failed"') expect(manager.removeBundle).toHaveBeenCalledWith('bundle') })