From 14abe9543cb7ebebbc85c5ddc92de83ac9f311b3 Mon Sep 17 00:00:00 2001 From: turtle1999 Date: Wed, 14 Oct 2026 12:18:26 +0800 Subject: [PATCH] feat(plugin-market-generator): record the source repository and parse options with commander Copy each plugin's manifest repository as its web page and package directory, converting shorthands and Git URLs with hosted-git-info. Parse options and generate help with commander. --- THIRD_PARTY_NOTICES.md | 2 + apps/plugin-market-generator/README.i18n.yaml | 8 +- apps/plugin-market-generator/README.md | 3 +- apps/plugin-market-generator/README.zh.md | 3 +- apps/plugin-market-generator/package.json | 3 + apps/plugin-market-generator/src/cli.ts | 146 +++++++++--------- apps/plugin-market-generator/src/generate.ts | 5 +- apps/plugin-market-generator/src/metadata.ts | 52 ++++++- apps/plugin-market-generator/src/packages.ts | 1 + .../plugin-market-generator/tests/cli.spec.ts | 48 +++++- .../tests/discovery.spec.ts | 35 ++++- .../tests/generate.spec.ts | 7 +- pnpm-lock.yaml | 22 +++ 13 files changed, 241 insertions(+), 94 deletions(-) diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 7d2bd4ba3d..67f867a790 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -92,6 +92,7 @@ External packages installed for runtime use or distributed inside the prebuilt b | [`fast-xml-parser`](https://github.com/NaturalIntelligence/fast-xml-parser) | MIT | | [`fflate`](https://github.com/101arrowz/fflate) | MIT | | [`got`](https://github.com/sindresorhus/got) | MIT | +| [`hosted-git-info`](https://github.com/npm/hosted-git-info) | ISC | | [`immer`](https://github.com/immerjs/immer) | MIT | | [`ipaddr.js`](https://github.com/whitequark/ipaddr.js) | MIT | | [`js-yaml`](https://github.com/nodeca/js-yaml) | MIT | @@ -204,6 +205,7 @@ External packages **directly declared** for development, tests, types, or toolin | [`@types/babel__code-frame`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | | [`@types/big.js`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | | [`@types/compression`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | +| [`@types/hosted-git-info`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | | [`@types/js-yaml`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | | [`@types/jsdom`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | | [`@types/mdast`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | diff --git a/apps/plugin-market-generator/README.i18n.yaml b/apps/plugin-market-generator/README.i18n.yaml index dd7ee3a523..f873926151 100644 --- a/apps/plugin-market-generator/README.i18n.yaml +++ b/apps/plugin-market-generator/README.i18n.yaml @@ -6,11 +6,11 @@ en: 531349889ccc3124 zh: 08769a2d9fc586fd /plugin-market-index-generator/usage: - en: 528513d86b87fe4e - zh: 59e2ecba16dbe336 + en: ed2a83d06f6789ef + zh: e5c1e26486dfefb0 /plugin-market-index-generator/what-gets-listed: - en: d12b091789de1751 - zh: 8d57fd273c77d804 + en: 7457f51ab0e41cbe + zh: 6c690391aaa7cf0a /plugin-market-index-generator/caching-and-repeated-runs: en: a62c9bf12310c33c zh: 60aa01288f46133c diff --git a/apps/plugin-market-generator/README.md b/apps/plugin-market-generator/README.md index 85668b1a44..5be2f3bbdc 100644 --- a/apps/plugin-market-generator/README.md +++ b/apps/plugin-market-generator/README.md @@ -37,6 +37,7 @@ pnpm --filter @deepseek-ai/dsh-plugin-market-generator start \ | `--inline-icon-bytes ` | `16384` | Largest icon embedded as a `data:` URL, at most 32768, the format's inline icon limit. | | `--concurrency ` | `4` | Sources read at the same time. | | `--timeout-ms ` | `60000` | Deadline for one request or Git command. | +| `-h`, `--help` | | Print these options and the exit statuses. | The exit status is 0 when every source was read, 1 when a source could not be read, and 2 for invalid arguments or an invalid list. A list line that the index format would reject, such as an `http:` URL, a URL with credentials, a query, or a fragment, a URL with fewer than two path segments, or a tarball URL, is an invalid list. A source that cannot be read keeps its last successful result, so a temporary outage does not remove plugins from the index; the failure is still reported. @@ -54,7 +55,7 @@ Each package becomes one plugin with one or more versions. A version is listed o `engines.dsh` is the manifest's `engines.dsh`, trimmed. Without one, it is the conjunction of the manifest's `@deepseek-ai/dsh` and `@deepseek-ai/dsh-*` `peerDependencies` ranges; a peer of exactly `workspace:*`, `workspace:^`, or `workspace:~` means the running DSH version and adds nothing, and identical ranges appear once. A `*` range admits every version, so a version whose ranges are all `*` lists no `engines.dsh`. A version is skipped with a warning when installation would refuse it on every DSH version: an empty range, a `workspace:` value in `engines.dsh` or any other `workspace:` peer, an invalid range, or ranges that no version satisfies together. -Display fields come from the highest listed version and follow the rules DSH uses for installed plugins. `./locale/en.json` and `./locale/.json` are resolved through the package `exports` map with the `import`, `node`, and `default` conditions, or literally without one, and every locale file must sit in the English file's directory; their `meta.title` and `meta.description` fall back to the manifest `name` and `description`. The icon is the manifest `icon`, a relative path, or else the exported `./icon`: SVG, PNG, JPEG, or WebP, at most 256 KiB, inside the package. `author` is the name part of the manifest `author` string, without its `` and `(url)`, or the `name` of an author object. `homepage` is the manifest `homepage` as written when it is an absolute http(s) URL; nothing else, such as `repository`, stands in for a missing one. Malformed locale files, invalid locale names, and rejected icons are reported as warnings. Translations without English text or a manifest fallback are dropped because the format requires English. Every entry is checked with [`dsh-plugin-market-index`](../../packages/util/plugin-market-index/README.md) before it is written. +Display fields come from the highest listed version and follow the rules DSH uses for installed plugins. `./locale/en.json` and `./locale/.json` are resolved through the package `exports` map with the `import`, `node`, and `default` conditions, or literally without one, and every locale file must sit in the English file's directory; their `meta.title` and `meta.description` fall back to the manifest `name` and `description`. The icon is the manifest `icon`, a relative path, or else the exported `./icon`: SVG, PNG, JPEG, or WebP, at most 256 KiB, inside the package. `author` is the name part of the manifest `author` string, without its `` and `(url)`, or the `name` of an author object. `homepage` is the manifest `homepage` as written when it is an absolute http(s) URL; nothing else, such as `repository`, stands in for a missing one. `repository` is the web page of the manifest `repository`, a shorthand such as `github:o/r` or a URL, through [hosted-git-info](https://github.com/npm/hosted-git-info) for hosts it knows, without any `#` committish; another http(s) URL loses a `git+` prefix, credentials, query, fragment, and `.git` suffix. Its `directory` is copied when the format accepts it. A repository with no http(s) page, or an unusable `directory`, is reported and omitted. Malformed locale files, invalid locale names, and rejected icons are reported as warnings. Translations without English text or a manifest fallback are dropped because the format requires English. Every entry is checked with [`dsh-plugin-market-index`](../../packages/util/plugin-market-index/README.md) before it is written. `metadata.keywords` is an optional array of strings for discovery. npm `package.json` `keywords` is the convention; index generators may supplement or adjust it. This reference generator copies the highest listed version's manifest `keywords` unchanged, including empty strings, duplicates, and an empty array. An absent field is omitted; a non-array value or an array containing any non-string produces a warning and omits the whole field without dropping the package. diff --git a/apps/plugin-market-generator/README.zh.md b/apps/plugin-market-generator/README.zh.md index 2f3506bb40..0a6fba94ce 100644 --- a/apps/plugin-market-generator/README.zh.md +++ b/apps/plugin-market-generator/README.zh.md @@ -37,6 +37,7 @@ pnpm --filter @deepseek-ai/dsh-plugin-market-generator start \ | `--inline-icon-bytes ` | `16384` | 以 `data:` URL 内嵌的图标的最大字节数，不超过格式的内联图标上限 32768。 | | `--concurrency ` | `4` | 同时读取的来源数。 | | `--timeout-ms ` | `60000` | 单个请求或 Git 命令的时限。 | +| `-h`、`--help` | | 打印这些选项和退出状态。 | 所有来源都读取成功时退出状态为 0，有来源无法读取时为 1，参数无效或列表无效时为 2。索引格式会拒绝的列表行属于无效列表，例如 `http:` URL、带凭据、查询或片段的 URL、路径段少于两个的 URL，以及 tarball URL。无法读取的来源保留其上一次成功的结果，因此临时故障不会把插件从索引中移除；生成器仍会报告该失败。 @@ -54,7 +55,7 @@ pnpm --filter @deepseek-ai/dsh-plugin-market-generator start \ `engines.dsh` 取 manifest 的 `engines.dsh` 并去掉首尾空白。没有该字段时，它是 manifest 中 `@deepseek-ai/dsh` 和 `@deepseek-ai/dsh-*` 的 `peerDependencies` 范围的合取；恰为 `workspace:*`、`workspace:^` 或 `workspace:~` 的 peer 表示当前运行的 DSH 版本，不增加限制；相同的范围只出现一次。`*` 范围接受所有版本，因此范围全为 `*` 的版本不列出 `engines.dsh`。如果安装时在所有 DSH 版本上都会拒绝某个版本，该版本会被跳过并发出警告：范围为空、`engines.dsh` 中出现 `workspace:` 值或 peer 中出现其他 `workspace:` 范围、范围无效，或者没有任何版本能同时满足这些范围。 -展示字段来自最高的列出版本，并遵循 DSH 读取已安装插件时的规则。`./locale/en.json` 和 `./locale/.json` 通过包的 `exports` 映射以 `import`、`node` 和 `default` 条件解析，没有该映射时按字面路径读取，并且每个 locale 文件都必须与英文文件位于同一目录；其 `meta.title` 和 `meta.description` 缺失时回退到 manifest 的 `name` 和 `description`。图标是 manifest 的 `icon`（相对路径），否则是导出的 `./icon`：格式为 SVG、PNG、JPEG 或 WebP，不超过 256 KiB，并位于包内。`author` 是 manifest `author` 字符串中去掉 `` 和 `(url)` 后的名字部分，或 author 对象的 `name`。`homepage` 在 manifest 的 `homepage` 是绝对 http(s) URL 时按原样复制；缺少时不会用 `repository` 等其他字段代替。格式错误的 locale 文件、无效的 locale 名称和被拒绝的图标会作为警告报告。由于该格式要求英文，没有英文文本也没有 manifest 回退值的翻译会被丢弃。每个条目在写出前都经过 [`dsh-plugin-market-index`](../../packages/util/plugin-market-index/README.zh.md) 校验。 +展示字段来自最高的列出版本，并遵循 DSH 读取已安装插件时的规则。`./locale/en.json` 和 `./locale/.json` 通过包的 `exports` 映射以 `import`、`node` 和 `default` 条件解析，没有该映射时按字面路径读取，并且每个 locale 文件都必须与英文文件位于同一目录；其 `meta.title` 和 `meta.description` 缺失时回退到 manifest 的 `name` 和 `description`。图标是 manifest 的 `icon`（相对路径），否则是导出的 `./icon`：格式为 SVG、PNG、JPEG 或 WebP，不超过 256 KiB，并位于包内。`author` 是 manifest `author` 字符串中去掉 `` 和 `(url)` 后的名字部分，或 author 对象的 `name`。`homepage` 在 manifest 的 `homepage` 是绝对 http(s) URL 时按原样复制；缺少时不会用 `repository` 等其他字段代替。`repository` 是 manifest `repository`（如 `github:o/r` 这样的简写或 URL）对应的网页：已知主机通过 [hosted-git-info](https://github.com/npm/hosted-git-info) 转换，并去掉 `#` 后的 committish；其他 http(s) URL 去掉 `git+` 前缀、凭据、查询、片段和 `.git` 后缀。格式接受时复制其 `directory`。没有 http(s) 网页的仓库或不可用的 `directory` 会被报告并省略。格式错误的 locale 文件、无效的 locale 名称和被拒绝的图标会作为警告报告。由于该格式要求英文，没有英文文本也没有 manifest 回退值的翻译会被丢弃。每个条目在写出前都经过 [`dsh-plugin-market-index`](../../packages/util/plugin-market-index/README.zh.md) 校验。 `metadata.keywords` 是用于发现插件的可选字符串数组。按约定使用 npm `package.json` 的 `keywords`；索引生成器可以补充或调整它。本参考生成器原样复制最高列出版本 manifest 的 `keywords`，包括空字符串、重复值和空数组。字段缺省时省略；如果值不是数组，或数组包含任何非字符串成员，则发出警告并省略整个字段，但保留该包。 diff --git a/apps/plugin-market-generator/package.json b/apps/plugin-market-generator/package.json index 4210e1ac7c..48362d3a31 100644 --- a/apps/plugin-market-generator/package.json +++ b/apps/plugin-market-generator/package.json @@ -10,6 +10,8 @@ "license": "MIT", "dependencies": { "@deepseek-ai/dsh-plugin-market-index": "workspace:*", + "commander": "^15.0.0", + "hosted-git-info": "^9.0.3", "resolve.exports": "^2.0.3", "semver": "^7.8.5", "tar": "^7.5.22", @@ -17,6 +19,7 @@ }, "devDependencies": { "@deepseek-ai/dsh-package-manifest": "workspace:*", + "@types/hosted-git-info": "^3.0.5", "@types/semver": "^7.8.0", "tsx": "^4.22.4" } diff --git a/apps/plugin-market-generator/src/cli.ts b/apps/plugin-market-generator/src/cli.ts index b40f3de3a8..b288716414 100644 --- a/apps/plugin-market-generator/src/cli.ts +++ b/apps/plugin-market-generator/src/cli.ts @@ -10,7 +10,7 @@ */ import { existsSync, lstatSync, readdirSync, readFileSync, rmSync } from 'node:fs' import { dirname, join, resolve } from 'node:path' -import { parseArgs } from 'node:util' +import { Command, CommanderError, InvalidArgumentError, Option } from 'commander' import { MAX_INLINE_ICON_BYTES } from '@deepseek-ai/dsh-plugin-market-index' import { writeIfChanged } from './files.ts' import { generate } from './generate.ts' @@ -18,32 +18,78 @@ import { GENERATED_ICON } from './metadata.ts' import { parseSources } from './sources.ts' const PUBLIC_REGISTRY = 'https://registry.npmjs.org/' -/** Names of icon files the generator writes, which it may also remove. */ -const HELP = `Usage: plugin-market-generator --input --output --title [options] +/** Parsed command-line options. */ +interface Options { + readonly input: string + readonly output: string + readonly title: string + readonly description?: string + readonly cache?: string + readonly registry: string + readonly inlineIconBytes: number + readonly concurrency: number + readonly timeoutMs: number +} -Options: - --input source list: one npm package name or https Git URL per line; lines starting with # are comments - --output index file to write; large icons go to icons/ beside it, and stale generated icons there are removed - --title market title - --description market description - --cache cache directory (default: .plugin-market-cache beside the input list) - --registry npm registry (default: ${PUBLIC_REGISTRY}) - --inline-icon-bytes largest icon embedded as a data URL, at most ${String(MAX_INLINE_ICON_BYTES)} (default: 16384) - --concurrency sources read at once (default: 4) - --timeout-ms deadline for one request or git command (default: 60000) - -h, --help show this help - -Exit status: 0 when every source was read, 1 when some source fell back to its cached result or was omitted, -2 for invalid arguments or input.` - -function integer(value: string | undefined, fallback: number, name: string, max?: number): number { - if (value === undefined) return fallback - const parsed = Number(value) - if (!Number.isInteger(parsed) || parsed < 1 || (max !== undefined && parsed > max)) { - throw new Error(`--${name} must be a positive integer${max === undefined ? '' : ` up to ${String(max)}`}`) +/** A positive integer option value, at most `max` when one is given. */ +function integer(max?: number): (value: string) => number { + return (value) => { + const parsed = Number(value) + if (!Number.isInteger(parsed) || parsed < 1 || (max !== undefined && parsed > max)) { + throw new InvalidArgumentError(`must be a positive integer${max === undefined ? '' : ` up to ${String(max)}`}`) + } + return parsed } - return parsed +} + +function program(): Command { + return new Command('plugin-market-generator') + .description('Write a plugin market index from a list of npm packages and Git repositories.') + .requiredOption('--input ', 'source list: one npm package name or https Git URL per line; lines starting with # are comments') + .requiredOption('--output ', 'index file to write; large icons go to icons/ beside it, and stale generated icons there are removed') + .requiredOption('--title ', 'market title') + .option('--description ', 'market description') + .option('--cache ', 'cache directory (default: .plugin-market-cache beside the input list)') + .option('--registry ', 'npm registry', PUBLIC_REGISTRY) + .addOption(new Option('--inline-icon-bytes ', `largest icon embedded as a data URL, at most ${String(MAX_INLINE_ICON_BYTES)}`) + .argParser(integer(MAX_INLINE_ICON_BYTES)).default(16384)) + .addOption(new Option('--concurrency ', 'sources read at once').argParser(integer()).default(4)) + .addOption(new Option('--timeout-ms ', 'deadline for one request or git command').argParser(integer()).default(60000)) + .helpOption('-h, --help', 'show this help') + .addHelpText('after', '\nExit status: 0 when every source was read, 1 when some source fell back to its cached result or was omitted,\n2 for invalid arguments or input.') + .showHelpAfterError('(run with --help for usage)') + .exitOverride() +} + +async function run(options: Options): Promise { + // `pnpm --filter … start` runs in the package directory; paths are relative to where the user ran pnpm. + const base = process.env.INIT_CWD ?? process.cwd() + const input = resolve(base, options.input) + const output = resolve(base, options.output) + const registry = new URL(options.registry).href.replace(/\/?$/u, '/') + const result = await generate({ + sources: parseSources(readFileSync(input, 'utf8')), + cacheDir: options.cache === undefined ? join(dirname(input), '.plugin-market-cache') : resolve(base, options.cache), + registry, + recordRegistry: registry !== PUBLIC_REGISTRY, + title: options.title, + ...options.description === undefined ? {} : { description: options.description }, + inlineIconBytes: options.inlineIconBytes, + limits: { timeoutMs: options.timeoutMs, maxTarballBytes: 64 * 1024 * 1024 }, + concurrency: options.concurrency, + warn: (message) => { console.error(`warning: ${message}`) }, + }) + for (const [path, cached] of result.icons) writeIfChanged(join(dirname(output), path), readFileSync(cached)) + const iconDir = join(dirname(output), 'icons') + for (const name of existsSync(iconDir) ? readdirSync(iconDir) : []) { + const file = join(iconDir, name) + if (GENERATED_ICON.test(name) && !result.icons.has(`icons/${name}`) && lstatSync(file).isFile()) rmSync(file, { force: true }) + } + const changed = writeIfChanged(output, `${JSON.stringify(result.index, null, 2)}\n`) + const plugins = result.index.plugins.length + console.log(`${changed ? 'wrote' : 'unchanged'} ${output}: ${plugins} plugin(s), ${result.failures.length} failed source(s)`) + return result.failures.length === 0 ? 0 : 1 } /** @@ -52,56 +98,16 @@ function integer(value: string | undefined, fallback: number, name: string, max? * @returns the process exit status. */ export async function main(argv: readonly string[]): Promise { - let values + let options: Options try { - ({ values } = parseArgs({ - args: [...argv], - options: { - input: { type: 'string' }, output: { type: 'string' }, title: { type: 'string' }, description: { type: 'string' }, - cache: { type: 'string' }, registry: { type: 'string' }, 'inline-icon-bytes': { type: 'string' }, - concurrency: { type: 'string' }, 'timeout-ms': { type: 'string' }, help: { type: 'boolean', short: 'h' }, - }, - })) + options = program().parse([...argv], { from: 'user' }).opts() } catch (error) { - console.error(`${(error as Error).message}\n\n${HELP}`) - return 2 - } - if (values.help === true) { - console.log(HELP) - return 0 + // Commander has already printed the help or the parse error. + if (!(error instanceof CommanderError)) throw error + return error.code === 'commander.helpDisplayed' ? 0 : 2 } try { - if (values.input === undefined || values.output === undefined || values.title === undefined) throw new Error('--input, --output, and --title are required') - // `pnpm --filter … start` runs in the package directory; paths are relative to where the user ran pnpm. - const base = process.env.INIT_CWD ?? process.cwd() - const input = resolve(base, values.input) - const output = resolve(base, values.output) - const registry = new URL(values.registry ?? PUBLIC_REGISTRY).href.replace(/\/?$/u, '/') - const inlineIconBytes = integer(values['inline-icon-bytes'], 16384, 'inline-icon-bytes', MAX_INLINE_ICON_BYTES) - const timeoutMs = integer(values['timeout-ms'], 60000, 'timeout-ms') - const concurrency = integer(values.concurrency, 4, 'concurrency') - const result = await generate({ - sources: parseSources(readFileSync(input, 'utf8')), - cacheDir: values.cache === undefined ? join(dirname(input), '.plugin-market-cache') : resolve(base, values.cache), - registry, - recordRegistry: registry !== PUBLIC_REGISTRY, - title: values.title, - ...values.description === undefined ? {} : { description: values.description }, - inlineIconBytes, - limits: { timeoutMs, maxTarballBytes: 64 * 1024 * 1024 }, - concurrency, - warn: (message) => { console.error(`warning: ${message}`) }, - }) - for (const [path, cached] of result.icons) writeIfChanged(join(dirname(output), path), readFileSync(cached)) - const iconDir = join(dirname(output), 'icons') - for (const name of existsSync(iconDir) ? readdirSync(iconDir) : []) { - const file = join(iconDir, name) - if (GENERATED_ICON.test(name) && !result.icons.has(`icons/${name}`) && lstatSync(file).isFile()) rmSync(file, { force: true }) - } - const changed = writeIfChanged(output, `${JSON.stringify(result.index, null, 2)}\n`) - const plugins = result.index.plugins.length - console.log(`${changed ? 'wrote' : 'unchanged'} ${output}: ${plugins} plugin(s), ${result.failures.length} failed source(s)`) - return result.failures.length === 0 ? 0 : 1 + return await run(options) } catch (error) { console.error((error as Error).message) return 2 diff --git a/apps/plugin-market-generator/src/generate.ts b/apps/plugin-market-generator/src/generate.ts index 078fa8a6c4..09fbf4fa3a 100644 --- a/apps/plugin-market-generator/src/generate.ts +++ b/apps/plugin-market-generator/src/generate.ts @@ -59,7 +59,7 @@ interface Release { /** Base URL only used to validate relative icon references. */ const VALIDATION_URL = 'https://index.invalid/index.json' /** Part of every cache key; change it when the way entries are derived changes, so cached results are rebuilt. */ -const CACHE_VERSION = 6 +const CACHE_VERSION = 7 /** Why the index format rejects these plugins, or undefined when it accepts them. */ function invalid(plugins: unknown): string | undefined { @@ -74,11 +74,12 @@ function invalid(plugins: unknown): string | undefined { /** Combine releases of one package into an entry whose display fields come from the highest version. */ function entryOf(name: string, releases: readonly Release[]): GeneratedEntry { const sorted = [...releases].sort((left, right) => semver.rcompare(left.version.version, right.version.version)) - const { title, description, icon, author, homepage, metadata } = sorted[0]?.display ?? {} + const { title, description, icon, author, homepage, repository, metadata } = sorted[0]?.display ?? {} return { name, ...title === undefined ? {} : { title }, ...description === undefined ? {} : { description }, ...icon === undefined ? {} : { icon }, ...author === undefined ? {} : { author }, ...homepage === undefined ? {} : { homepage }, + ...repository === undefined ? {} : { repository }, ...metadata === undefined ? {} : { metadata }, versions: sorted.map(release => release.version), } diff --git a/apps/plugin-market-generator/src/metadata.ts b/apps/plugin-market-generator/src/metadata.ts index 7b5554e02c..24eaf993b7 100644 --- a/apps/plugin-market-generator/src/metadata.ts +++ b/apps/plugin-market-generator/src/metadata.ts @@ -5,6 +5,8 @@ import { createHash } from 'node:crypto' import { posix } from 'node:path' import type { LocalizedText } from '@deepseek-ai/dsh-package-manifest' +import hostedGitInfo from 'hosted-git-info' +import type { MarketRepository } from '@deepseek-ai/dsh-plugin-market-index' import { exports as resolveExports, type Package } from 'resolve.exports' import { PackageFiles } from './files.ts' import type { Manifest } from './packages.ts' @@ -20,6 +22,8 @@ export interface Display { readonly author?: string /** The manifest `homepage`, copied as written. */ readonly homepage?: string + /** The web URL of the manifest `repository`, with its `directory`. */ + readonly repository?: MarketRepository /** Optional discovery metadata; keywords are copied from the manifest as written. */ readonly metadata?: { readonly keywords: readonly string[] } } @@ -44,6 +48,9 @@ const LANGUAGE_ID = /^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*$/u const MAX_ICON_BYTES = 256 * 1024 const ICON_TYPES = new Map([['.svg', 'image/svg+xml'], ['.png', 'image/png'], ['.jpg', 'image/jpeg'], ['.jpeg', 'image/jpeg'], ['.webp', 'image/webp']]) +/** One segment of a package directory, as the index format accepts it; keep it equal to the parser's rule. */ +const PATH_SEGMENT = /^[A-Za-z0-9._@+-]+$/u + function text(value: unknown): string | undefined { return typeof value === 'string' && value.trim() !== '' ? value : undefined } @@ -123,6 +130,43 @@ function localized(field: keyof Fields, locales: Map, fallback: return Object.fromEntries([['en', english], ...translated]) as LocalizedText } +/** + * The web URL of a manifest `repository` string or object `url`: a shorthand or URL of a host npm knows becomes its + * browse URL without a committish, and any other http(s) URL, after a `git+` prefix, loses its credentials, query, fragment, and `.git` + * suffix. + */ +function repositoryUrl(declared: string): string | undefined { + // A committish such as `#semver:^1` names a revision, not the repository page. + const hosted = hostedGitInfo.fromUrl(declared.replace(/#.*$/u, '')) + if (hosted !== undefined) return hosted.browse() + const plain = declared.replace(/^git\+/u, '') + if (!URL.canParse(plain)) return undefined + const url = new URL(plain) + if (url.protocol !== 'https:' && url.protocol !== 'http:') return undefined + url.username = '' + url.password = '' + url.search = '' + url.hash = '' + url.pathname = url.pathname.replace(/\.git$/u, '') + return url.href +} + +/** The repository a manifest declares, or why it is omitted; a directory the format cannot hold is dropped with a warning. */ +function readRepository(value: unknown, warn: (message: string) => void): MarketRepository | undefined { + if (value === undefined) return undefined + const object = typeof value === 'object' && value !== null ? value as { url?: unknown; directory?: unknown } : undefined + const declared = text(object === undefined ? value : object.url)?.trim() + const url = declared === undefined ? undefined : repositoryUrl(declared) + if (url === undefined) { + warn(`package.json repository ${JSON.stringify(value)} names no http(s) repository page; it is omitted`) + return undefined + } + const directory = text(object?.directory)?.replace(/^\.\/|\/$/gu, '') + const valid = directory !== undefined && directory.split('/').every(segment => segment !== '.' && segment !== '..' && PATH_SEGMENT.test(segment)) + if (object?.directory !== undefined && !valid) warn(`package.json repository.directory ${JSON.stringify(object.directory)} is not a relative directory; it is omitted`) + return { url, ...valid ? { directory } : {} } +} + /** The icon a package declares, or why it is not used: the manifest `icon` takes priority over the `./icon` export. */ async function readIcon( files: PackageFiles, manifest: Manifest, @@ -147,8 +191,10 @@ async function readIcon( } /** - * Read display metadata, author, homepage, and icon of a package. `homepage` is copied from the manifest as written; a - * value that is not an absolute http(s) URL is reported and omitted, and nothing else stands in for it. + * Read display metadata, author, homepage, repository, and icon of a package. `homepage` is copied from the manifest as + * written; a value that is not an absolute http(s) URL is reported and omitted, and nothing else stands in for it. + * `repository` becomes the repository's web URL and its directory; one that names no http(s) page is reported and + * omitted. * Manifest `keywords` arrays containing only strings are copied unchanged into `metadata.keywords`; any other * present value is reported and omitted without dropping the package. * @param files - the package's files. @@ -168,12 +214,14 @@ export async function readDisplayMetadata( const homepage = text(manifest.homepage) const linkable = homepage !== undefined && URL.canParse(homepage) && ['https:', 'http:'].includes(new URL(homepage).protocol) if (homepage !== undefined && !linkable) options.warn(`package.json homepage ${JSON.stringify(homepage)} is not an http(s) URL; it is omitted`) + const repository = readRepository(manifest.repository, options.warn) const keywords = Array.isArray(manifest.keywords) && manifest.keywords.every((keyword): keyword is string => typeof keyword === 'string') ? manifest.keywords : undefined if (manifest.keywords !== undefined && keywords === undefined) options.warn('package.json keywords must be an array of strings; it is omitted') const display = { ...title === undefined ? {} : { title }, ...description === undefined ? {} : { description }, ...author === undefined ? {} : { author }, ...linkable ? { homepage } : {}, + ...repository === undefined ? {} : { repository }, ...keywords === undefined ? {} : { metadata: { keywords } }, } const icon = await readIcon(files, manifest) diff --git a/apps/plugin-market-generator/src/packages.ts b/apps/plugin-market-generator/src/packages.ts index b372120dcd..743d05d57b 100644 --- a/apps/plugin-market-generator/src/packages.ts +++ b/apps/plugin-market-generator/src/packages.ts @@ -12,6 +12,7 @@ export interface Manifest { readonly icon?: unknown readonly author?: unknown readonly homepage?: unknown + readonly repository?: unknown readonly keywords?: unknown readonly deprecated?: unknown readonly workspaces?: unknown diff --git a/apps/plugin-market-generator/tests/cli.spec.ts b/apps/plugin-market-generator/tests/cli.spec.ts index ab3ba52706..c2c758edde 100644 --- a/apps/plugin-market-generator/tests/cli.spec.ts +++ b/apps/plugin-market-generator/tests/cli.spec.ts @@ -1,15 +1,49 @@ -/** Command-line validation of the generator's options. */ +/** Command-line parsing and help of the generator. */ import { afterEach, expect, it, vi } from 'vitest' import { main } from '../src/cli.ts' afterEach(() => { vi.restoreAllMocks() }) +/** Run the CLI and collect what it writes to stdout and stderr. */ +async function cli(...argv: string[]) { + const out: string[] = [] + const err: string[] = [] + vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { out.push(String(chunk)); return true }) + vi.spyOn(process.stderr, 'write').mockImplementation((chunk) => { err.push(String(chunk)); return true }) + const status = await main(argv) + return { status, out: out.join(''), err: err.join('') } +} + +const REQUIRED = ['--input', '/nonexistent/list.txt', '--output', '/nonexistent/index.json', '--title', 'T'] + +it('prints help with every option, its default, and the exit statuses', async () => { + const { status, out } = await cli('--help') + expect(status).toBe(0) + for (const text of ['--input ', '--inline-icon-bytes ', 'at most 32768', '(default: 16384)', 'Exit status: 0 when every source was read']) { + expect(out).toContain(text) + } +}) + it.each([ - ['32769', '--inline-icon-bytes must be a positive integer up to 32768'], - ['0', '--inline-icon-bytes must be a positive integer up to 32768'], -])('refuses --inline-icon-bytes %s before reading the source list', async (value, message) => { + [['--inline-icon-bytes', '32769'], "option '--inline-icon-bytes ' argument '32769' is invalid. must be a positive integer up to 32768"], + [['--inline-icon-bytes', '0'], 'must be a positive integer up to 32768'], + [['--concurrency', '1.5'], "option '--concurrency ' argument '1.5' is invalid. must be a positive integer"], + [['--bogus'], "unknown option '--bogus'"], +])('refuses %j before reading the source list', async (extra, message) => { + const { status, err } = await cli(...REQUIRED, ...extra) + expect(status).toBe(2) + expect(err).toContain(message) +}) + +it('names a missing required option', async () => { + const { status, err } = await cli('--input', 'list.txt') + expect(status).toBe(2) + expect(err).toContain("required option '--output ' not specified") + expect(err).toContain('(run with --help for usage)') +}) + +it('reports a source list it cannot read', async () => { const error = vi.spyOn(console, 'error').mockImplementation(() => {}) - const status = await main(['--input', '/nonexistent/list.txt', '--output', '/nonexistent/index.json', '--title', 'T', '--inline-icon-bytes', value]) - expect(status).toBe(2) - expect(error).toHaveBeenCalledWith(message) + expect(await main(REQUIRED)).toBe(2) + expect(String(error.mock.calls[0]?.[0])).toContain('ENOENT') }) diff --git a/apps/plugin-market-generator/tests/discovery.spec.ts b/apps/plugin-market-generator/tests/discovery.spec.ts index ce4f98fec8..eefbd58349 100644 --- a/apps/plugin-market-generator/tests/discovery.spec.ts +++ b/apps/plugin-market-generator/tests/discovery.spec.ts @@ -171,13 +171,44 @@ describe('readDisplayMetadata', () => { }) }) + it.each([ + ['github:jane/dsh-a', { url: 'https://github.com/jane/dsh-a' }], + ['jane/dsh-a', { url: 'https://github.com/jane/dsh-a' }], + [' github:jane/dsh-a ', { url: 'https://github.com/jane/dsh-a' }], + ['github:jane/dsh-a#semver:^1', { url: 'https://github.com/jane/dsh-a' }], + [{ type: 'git', url: 'git+ssh://git@github.com/jane/plugins.git', directory: 'packages/a' }, { url: 'https://github.com/jane/plugins', directory: 'packages/a' }], + [{ url: 'git+https://u:p@git.example/g/sub/r.git?x=1#main', directory: './packages/a/' }, { url: 'https://git.example/g/sub/r', directory: 'packages/a' }], + [{ url: 'http://git.intranet/o/r' }, { url: 'http://git.intranet/o/r' }], + ])('converts the manifest repository %j to its web page', async (repository, expected) => { + expect(await readDisplay(tree({}), { name: 'dsh-a', repository })).toEqual({ display: { title: 'dsh-a', repository: expected }, warnings: [] }) + }) + + it.each([ + 'git://git.example/o/r.git', + { url: 'ssh://git@git.example/o/r.git' }, + { directory: 'packages/a' }, + 3, + ])('warns and omits a repository %j without an http(s) page', async (repository) => { + expect(await readDisplay(tree({}), { name: 'dsh-a', repository })).toEqual({ + display: { title: 'dsh-a' }, + warnings: [`package.json repository ${JSON.stringify(repository)} names no http(s) repository page; it is omitted`], + }) + }) + + it.each(['../a', 'a/./b', 'a b', 4])('keeps the repository but omits the directory %j', async (directory) => { + expect(await readDisplay(tree({}), { name: 'dsh-a', repository: { url: 'https://github.com/o/r', directory } })).toEqual({ + display: { title: 'dsh-a', repository: { url: 'https://github.com/o/r' } }, + warnings: [`package.json repository.directory ${JSON.stringify(directory)} is not a relative directory; it is omitted`], + }) + }) + it('copies an http(s) homepage as written and reports any other value without a substitute', async () => { const homepage = 'https://github.com/jane/dsh-a#readme' expect((await readDisplay(tree({}), { name: 'dsh-a', homepage })).display).toEqual({ title: 'dsh-a', homepage }) - // `repository` is present to show that it does not replace a rejected homepage. + // A repository stays its own field and does not replace a rejected homepage. const manifest = { name: 'dsh-a', homepage: 'javascript:alert(1)', repository: 'https://github.com/jane/dsh-a' } const rejected = await readDisplay(tree({}), manifest) - expect(rejected.display).toEqual({ title: 'dsh-a' }) + expect(rejected.display).toEqual({ title: 'dsh-a', repository: { url: 'https://github.com/jane/dsh-a' } }) expect(rejected.warnings).toEqual(['package.json homepage "javascript:alert(1)" is not an http(s) URL; it is omitted']) expect((await readDisplay(tree({}), { name: 'dsh-a', homepage: 'docs/readme' })).display).toEqual({ title: 'dsh-a' }) }) diff --git a/apps/plugin-market-generator/tests/generate.spec.ts b/apps/plugin-market-generator/tests/generate.spec.ts index 9ffe581227..f204eb1a39 100644 --- a/apps/plugin-market-generator/tests/generate.spec.ts +++ b/apps/plugin-market-generator/tests/generate.spec.ts @@ -106,7 +106,7 @@ it('lists every non-deprecated bundle version from the registry document and rea time: '2026-06-01T00:00:00.000Z', manifest: { ...BUNDLE, author: { name: 'Jane Doe', email: 'jane@example.com' } }, tarball: await pack({ - 'package.json': bundleManifest('dsh-a', '2.0.0-rc.1', { author: { name: 'Jane Doe' }, icon: 'icon.png', homepage: 'https://example.com/dsh-a', keywords: ['alpha', '', 'alpha'] }), + 'package.json': bundleManifest('dsh-a', '2.0.0-rc.1', { author: { name: 'Jane Doe' }, icon: 'icon.png', homepage: 'https://example.com/dsh-a', repository: { type: 'git', url: 'git+https://github.com/jane/plugins.git', directory: 'packages/a' }, keywords: ['alpha', '', 'alpha'] }), 'locale/en.json': JSON.stringify({ meta: { title: 'Alpha' } }), 'locale/zh.json': JSON.stringify({ meta: { title: '甲' } }), 'locale/x_y.json': '{}', @@ -136,6 +136,7 @@ it('lists every non-deprecated bundle version from the registry document and rea expect(first.index.plugins).toEqual([ { name: 'dsh-a', title: { en: 'Alpha', zh: '甲' }, icon: expect.stringMatching(/^icons\/[0-9a-f]{16}\.png$/u) as string, author: 'Jane Doe', homepage: 'https://example.com/dsh-a', + repository: { url: 'https://github.com/jane/plugins', directory: 'packages/a' }, metadata: { keywords: ['alpha', '', 'alpha'] }, versions: [ { version: '2.0.0-rc.1', publishTimestamp: 1780272000, source: npm }, @@ -252,10 +253,6 @@ it('writes the index and icons once, keeps the cache beside the input, and remov const other = join(scratch(), 'index.json') expect(await main(['--input', list, '--output', other, '--title', 'T', '--description', 'D', '--registry', server.url, '--concurrency', '2', '--cache', join(lists, 'other-cache')])).toBe(1) expect((JSON.parse(readFileSync(other, 'utf8')) as { description: string }).description).toBe('D') - expect(await main(['--input', list])).toBe(2) - expect(await main(['--unknown'])).toBe(2) - expect(await main(['--help'])).toBe(0) - expect(await main([...args, '--concurrency', '0'])).toBe(2) writeFileSync(list, 'http://example.com/o/r\n') expect(await main(['--input', list, '--output', other, '--title', 'T'])).toBe(2) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 42dc762e7e..0445f02ac2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -833,6 +833,12 @@ importers: '@deepseek-ai/dsh-plugin-market-index': specifier: workspace:* version: link:../../packages/util/plugin-market-index + commander: + specifier: ^15.0.0 + version: 15.0.0 + hosted-git-info: + specifier: ^9.0.3 + version: 9.0.3 resolve.exports: specifier: ^2.0.3 version: 2.0.3 @@ -849,6 +855,9 @@ importers: '@deepseek-ai/dsh-package-manifest': specifier: workspace:* version: link:../../packages/util/package-manifest + '@types/hosted-git-info': + specifier: ^3.0.5 + version: 3.0.5 '@types/semver': specifier: ^7.8.0 version: 7.8.0 @@ -18916,6 +18925,9 @@ packages: '@types/hast@3.0.5': resolution: {integrity: sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==} + '@types/hosted-git-info@3.0.5': + resolution: {integrity: sha512-Dmngh7U003cOHPhKGyA7LWqrnvcTyILNgNPmNCxlx7j8MIi54iBliiT8XqVLIQ3GchoOjVAyBzNJVyuaJjqokg==} + '@types/http-cache-semantics@4.2.0': resolution: {integrity: sha512-L3LgimLHXtGkWikKnsPg0/VFx9OGZaC+eN1u4r+OB1XRqH3meBIAVC2zr1WdMH+RHmnRkqliQAOHNJ/E0j/e0Q==} @@ -20861,6 +20873,10 @@ packages: resolution: {integrity: sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==} engines: {node: '>=10'} + hosted-git-info@9.0.3: + resolution: {integrity: sha512-Hc+ghLoSt6QaYZUv0WBiIvmMDZuZZ7oaDvdH8MbfOO4lOsxdXLEvuC6ePoGs9H1X9oCLyq6+NVN0MKqD+ydxyg==} + engines: {node: ^20.17.0 || >=22.9.0} + html-encoding-sniffer@3.0.0: resolution: {integrity: sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==} engines: {node: '>=12'} @@ -26174,6 +26190,8 @@ snapshots: dependencies: '@types/unist': 3.0.3 + '@types/hosted-git-info@3.0.5': {} + '@types/http-cache-semantics@4.2.0': {} '@types/http-errors@2.0.5': {} @@ -28491,6 +28509,10 @@ snapshots: dependencies: lru-cache: 6.0.0 + hosted-git-info@9.0.3: + dependencies: + lru-cache: 11.5.1 + html-encoding-sniffer@3.0.0: dependencies: whatwg-encoding: 2.0.0