From 434dc116f0b99b25143ecf6e5dce1f80f665508e Mon Sep 17 00:00:00 2001 From: turtle1999 Date: Wed, 14 Oct 2026 12:02:56 +0800 Subject: [PATCH] feat(plugin-market-generator): add a reference index generator Generate a validated format-1 plugin market index from a list of npm package names and https Git repository URLs, without running package code. - List every non-deprecated npm bundle version, and Git versions at release tags and the default branch, each with its engines.dsh range. - Read display fields, keywords, homepage, and icons the way DSH reads installed plugins; inline small icons up to the format's 32 KiB limit. - Cache results per immutable input and rewrite output only when it changes. --- THIRD_PARTY_NOTICES.md | 2 +- apps/plugin-market-generator/README.i18n.yaml | 22 ++ apps/plugin-market-generator/README.md | 77 ++++ apps/plugin-market-generator/README.zh.md | 77 ++++ apps/plugin-market-generator/package.json | 23 ++ apps/plugin-market-generator/src/bin.ts | 4 + apps/plugin-market-generator/src/cli.ts | 109 ++++++ apps/plugin-market-generator/src/fetchers.ts | 261 +++++++++++++ apps/plugin-market-generator/src/files.ts | 129 +++++++ apps/plugin-market-generator/src/generate.ts | 357 ++++++++++++++++++ apps/plugin-market-generator/src/metadata.ts | 186 +++++++++ apps/plugin-market-generator/src/packages.ts | 196 ++++++++++ apps/plugin-market-generator/src/sources.ts | 48 +++ .../plugin-market-generator/tests/cli.spec.ts | 15 + .../tests/discovery.spec.ts | 270 +++++++++++++ .../tests/generate.spec.ts | 261 +++++++++++++ .../plugin-market-generator/tests/git.spec.ts | 182 +++++++++ apps/plugin-market-generator/tests/support.ts | 45 +++ apps/plugin-market-generator/tsconfig.json | 14 + docs/user/guide/plugin-market.i18n.yaml | 11 +- docs/user/guide/plugin-market.md | 21 ++ docs/user/guide/plugin-market.zh.md | 21 ++ pnpm-lock.yaml | 28 ++ scripts/check-workspace-constraints.spec.ts | 7 + scripts/check-workspace-constraints.ts | 14 +- tsconfig.host.json | 2 + 26 files changed, 2374 insertions(+), 8 deletions(-) create mode 100644 apps/plugin-market-generator/README.i18n.yaml create mode 100644 apps/plugin-market-generator/README.md create mode 100644 apps/plugin-market-generator/README.zh.md create mode 100644 apps/plugin-market-generator/package.json create mode 100644 apps/plugin-market-generator/src/bin.ts create mode 100644 apps/plugin-market-generator/src/cli.ts create mode 100644 apps/plugin-market-generator/src/fetchers.ts create mode 100644 apps/plugin-market-generator/src/files.ts create mode 100644 apps/plugin-market-generator/src/generate.ts create mode 100644 apps/plugin-market-generator/src/metadata.ts create mode 100644 apps/plugin-market-generator/src/packages.ts create mode 100644 apps/plugin-market-generator/src/sources.ts create mode 100644 apps/plugin-market-generator/tests/cli.spec.ts create mode 100644 apps/plugin-market-generator/tests/discovery.spec.ts create mode 100644 apps/plugin-market-generator/tests/generate.spec.ts create mode 100644 apps/plugin-market-generator/tests/git.spec.ts create mode 100644 apps/plugin-market-generator/tests/support.ts create mode 100644 apps/plugin-market-generator/tsconfig.json diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 6733a4bce7..7d2bd4ba3d 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -130,6 +130,7 @@ External packages installed for runtime use or distributed inside the prebuilt b | [`shiki`](https://github.com/shikijs/shiki) | MIT | | [`simple-icons`](https://github.com/simple-icons/simple-icons) | CC0-1.0 | | [`supports-color`](https://github.com/chalk/supports-color) | MIT | +| [`tar`](https://github.com/isaacs/node-tar) | BlueOak-1.0.0 | | [`tsx`](https://github.com/privatenumber/tsx) | MIT | | [`turndown`](https://github.com/mixmark-io/turndown) | MIT | | [`typescript`](https://github.com/microsoft/TypeScript) | Apache-2.0 | @@ -253,7 +254,6 @@ External packages **directly declared** for development, tests, types, or toolin | [`selfsigned`](https://github.com/jfromaniello/selfsigned) | MIT | | [`smol-toml`](https://github.com/squirrelchat/smol-toml) | BSD-3-Clause | | [`spdx-expression-parse`](https://github.com/jslicense/spdx-expression-parse.js) | MIT | -| [`tar`](https://github.com/isaacs/node-tar) | BlueOak-1.0.0 | | [`tsdown`](https://github.com/rolldown/tsdown) | MIT | | [`typescript-language-server`](https://github.com/typescript-language-server/typescript-language-server) | Apache-2.0 | | [`vite`](https://github.com/vitejs/vite) | MIT | diff --git a/apps/plugin-market-generator/README.i18n.yaml b/apps/plugin-market-generator/README.i18n.yaml new file mode 100644 index 0000000000..dd7ee3a523 --- /dev/null +++ b/apps/plugin-market-generator/README.i18n.yaml @@ -0,0 +1,22 @@ +# Bilingual-pair consistency record for README.md (docs/i18n/README.md): per heading +# section, a hash of its English and Chinese blocks outside code blocks and generated regions. +# After editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write apps/plugin-market-generator/README.md +/plugin-market-index-generator: + en: 531349889ccc3124 + zh: 08769a2d9fc586fd +/plugin-market-index-generator/usage: + en: 528513d86b87fe4e + zh: 59e2ecba16dbe336 +/plugin-market-index-generator/what-gets-listed: + en: d12b091789de1751 + zh: 8d57fd273c77d804 +/plugin-market-index-generator/caching-and-repeated-runs: + en: a62c9bf12310c33c + zh: 60aa01288f46133c +/plugin-market-index-generator/security: + en: de5821128b676a61 + zh: aad295eeb80cad7e +/plugin-market-index-generator/limits: + en: b5e02017e961e818 + zh: 0f0aaa26d519acb1 diff --git a/apps/plugin-market-generator/README.md b/apps/plugin-market-generator/README.md new file mode 100644 index 0000000000..85668b1a44 --- /dev/null +++ b/apps/plugin-market-generator/README.md @@ -0,0 +1,77 @@ +# Plugin market index generator + +English | [中文](README.zh.md) + +A reference generator for [plugin market index](../../.agents/notes/implemented/feature/2026-10-02-plugin-market-index-format.md) files. It reads a plain list of npm package names and Git repository URLs and writes one `index.json` that the DSH **Discover** page can show without further requests. It is a standalone Node program, not a Cordis application, and it never installs or runs the code of the packages it describes. + +## Usage + +Write a source list, one entry per line. Blank lines and lines starting with `#` are ignored. + +```text +# npm packages: every non-deprecated bundle version is listed +@scope/dsh-example +dsh-another-plugin +# Git repositories: release tags and the default branch are listed +https://github.com/author/dsh-plugin +https://gitlab.example.com/group/dsh-plugins +``` + +Run the generator from the repository: + +```sh +pnpm --filter @deepseek-ai/dsh-plugin-market-generator start \ + --input plugins.txt --output site/index.json --title "Example Market" +``` + +`--input` and `--output` are relative to the directory where you run pnpm. Publish the output directory over https, then add the `index.json` URL as an index source in Discover. + +| Option | Default | Meaning | +|---|---|---| +| `--input ` | required | Source list. | +| `--output ` | required | Index file. Icons too large to inline are written to `icons/` next to it; generated icon files the index no longer uses are removed from there. | +| `--title ` | required | Market title. | +| `--description ` | none | Market description. | +| `--cache ` | `.plugin-market-cache` next to the input list | Result cache. It stays out of the output directory, which is published. | +| `--registry ` | `https://registry.npmjs.org/` | npm registry. Any other registry is recorded in each npm version. | +| `--inline-icon-bytes ` | `16384` | Largest icon embedded as a `data:` URL, at most 32768, the format's inline icon limit. | +| `--concurrency ` | `4` | Sources read at the same time. | +| `--timeout-ms ` | `60000` | Deadline for one request or Git command. | + +The exit status is 0 when every source was read, 1 when a source could not be read, and 2 for invalid arguments or an invalid list. A list line that the index format would reject, such as an `http:` URL, a URL with credentials, a query, or a fragment, a URL with fewer than two path segments, or a tarball URL, is an invalid list. A source that cannot be read keeps its last successful result, so a temporary outage does not remove plugins from the index; the failure is still reported. + +## What gets listed + +Each package becomes one plugin with one or more versions. A version is listed only when its `package.json` declares `dsh.bundle` with `patch` set to a file path or a list of file paths. + +- **npm package**: every version in the registry document that is not deprecated, declares a bundle, and has a publish time. Versions come from the registry document alone; only the tarball of the highest listed version is downloaded, for its display fields. Its `package.json` must name the requested package and version. +- **Git repository**: bundle packages are found on the default branch, in this order: + 1. the root `package.json` declares `dsh.bundle`: the root package only; + 2. `package.json` `workspaces` or `pnpm-workspace.yaml` `packages` match packages that declare `dsh.bundle`: those packages. Each glob segment is a literal name, a name with `*`, or `**`; negated globs are ignored, and a glob that matches nothing is reported; + 3. otherwise, every package declaring `dsh.bundle` within three directory levels, outside `node_modules`, `test`, `tests`, `fixtures`, `example`, and `examples`. + + Each package's versions come from release tags and the default branch. A `vX.Y.Z` or `X.Y.Z` tag applies to every package, and a `@X.Y.Z` tag, such as `@scope/dsh-a@1.2.0`, to the package with that name; the tag version must be an exact SemVer version. A tag lists a package when the package's directory at the tagged commit holds a bundle manifest with the same name and the tag's version. The default branch commit lists each package's current version unless a tag already lists it. When several tags list one version, a `@` tag wins over a unified tag, then the earlier commit wins. The publish time is the commit's committer time. + +`engines.dsh` is the manifest's `engines.dsh`, trimmed. Without one, it is the conjunction of the manifest's `@deepseek-ai/dsh` and `@deepseek-ai/dsh-*` `peerDependencies` ranges; a peer of exactly `workspace:*`, `workspace:^`, or `workspace:~` means the running DSH version and adds nothing, and identical ranges appear once. A `*` range admits every version, so a version whose ranges are all `*` lists no `engines.dsh`. A version is skipped with a warning when installation would refuse it on every DSH version: an empty range, a `workspace:` value in `engines.dsh` or any other `workspace:` peer, an invalid range, or ranges that no version satisfies together. + +Display fields come from the highest listed version and follow the rules DSH uses for installed plugins. `./locale/en.json` and `./locale/.json` are resolved through the package `exports` map with the `import`, `node`, and `default` conditions, or literally without one, and every locale file must sit in the English file's directory; their `meta.title` and `meta.description` fall back to the manifest `name` and `description`. The icon is the manifest `icon`, a relative path, or else the exported `./icon`: SVG, PNG, JPEG, or WebP, at most 256 KiB, inside the package. `author` is the name part of the manifest `author` string, without its `` and `(url)`, or the `name` of an author object. `homepage` is the manifest `homepage` as written when it is an absolute http(s) URL; nothing else, such as `repository`, stands in for a missing one. Malformed locale files, invalid locale names, and rejected icons are reported as warnings. Translations without English text or a manifest fallback are dropped because the format requires English. Every entry is checked with [`dsh-plugin-market-index`](../../packages/util/plugin-market-index/README.md) before it is written. + +`metadata.keywords` is an optional array of strings for discovery. npm `package.json` `keywords` is the convention; index generators may supplement or adjust it. This reference generator copies the highest listed version's manifest `keywords` unchanged, including empty strings, duplicates, and an empty array. An absent field is omitted; a non-array value or an array containing any non-string produces a warning and omits the whole field without dropping the package. + +When several sources produce the same package name, the first one in the list wins. + +## Caching and repeated runs + +Each run reads the registry document of every npm source and the refs of every Git repository. Results are cached per immutable input: an npm plugin per registry, icon threshold, package, and set of listed versions; a Git repository per default branch commit and release tags; and each Git package per commit and directory. A Git run therefore fetches only the commits whose packages are not cached. Cache files are written atomically, and a cache file that is damaged or that the index format rejects is rebuilt. Output is deterministic: entries follow the list order, and the index and icon files are rewritten only when their contents change, so running the generator again without upstream changes leaves the output untouched. + +## Security + +The generator downloads and reads files only. npm tarballs are read in memory: a tarball and its unpacked files may not exceed 64 MiB each, the bytes must match the registry integrity hash, and only regular files outside `node_modules` are kept, without paths that leave the package. Git objects are fetched with depth 1 into a private temporary bare repository and read as blobs, so no working tree exists and no checkout filter, LFS download, or symbolic link takes effect; symbolic links and submodules count as absent. Git runs with hooks and the file-system monitor disabled, no submodules, only the https and ssh transports, and without inherited `GIT_*` variables except `GIT_SSH` and `GIT_SSH_COMMAND`. Your ssh program and `GIT_SSH_COMMAND` keep precedence; without either, ssh runs in batch mode, so it never waits for a prompt. Manifests, locale files, and workspace files over 1 MiB are refused. Private repositories use your Git credential configuration; an https URL can be routed to ssh with `url..insteadOf`. + +Index content is what the packages claim about themselves. Review the list you publish: a listed repository can use any package name. + +## Limits + +- A Git server must allow fetching a commit by its SHA, as GitHub, GitLab, and Gitea do. When a server refuses, the generator fetches the tags and the default branch by name, which fails if a ref moved after it was listed. +- Registry tokens from `.npmrc` are not applied; only public or unauthenticated registries work. +- Git never prompts for credentials. A repository that needs them must be readable through a credential helper or an ssh agent. diff --git a/apps/plugin-market-generator/README.zh.md b/apps/plugin-market-generator/README.zh.md new file mode 100644 index 0000000000..2f3506bb40 --- /dev/null +++ b/apps/plugin-market-generator/README.zh.md @@ -0,0 +1,77 @@ +# 插件市场索引生成器 + +[English](README.md) | 中文 + +[插件市场索引](../../.agents/notes/implemented/feature/2026-10-02-plugin-market-index-format.zh.md)文件的参考生成器。它读取一份由 npm 包名和 Git 仓库 URL 组成的纯文本列表，写出一个 `index.json`，DSH 的 **发现** 页面无需额外请求即可展示该文件。它是独立的 Node 程序，不是 Cordis 应用，也从不安装或运行其所描述包的代码。 + +## 用法 + +编写来源列表，每行一个条目。生成器忽略空行和以 `#` 开头的行。 + +```text +# npm packages: every non-deprecated bundle version is listed +@scope/dsh-example +dsh-another-plugin +# Git repositories: release tags and the default branch are listed +https://github.com/author/dsh-plugin +https://gitlab.example.com/group/dsh-plugins +``` + +在仓库中运行生成器： + +```sh +pnpm --filter @deepseek-ai/dsh-plugin-market-generator start \ + --input plugins.txt --output site/index.json --title "Example Market" +``` + +`--input` 和 `--output` 相对于你运行 pnpm 的目录。通过 https 发布输出目录，然后在「发现」中把该 `index.json` 的 URL 添加为索引来源。 + +| 选项 | 默认值 | 含义 | +|---|---|---| +| `--input ` | 必填 | 来源列表。 | +| `--output ` | 必填 | 索引文件。过大而无法内联的图标写入其旁边的 `icons/`；索引不再使用的已生成图标文件会从该目录删除。 | +| `--title ` | 必填 | 市场标题。 | +| `--description ` | 无 | 市场描述。 | +| `--cache ` | 输入列表旁的 `.plugin-market-cache` | 结果缓存。它不在会被发布的输出目录中。 | +| `--registry ` | `https://registry.npmjs.org/` | npm 注册表。使用其他注册表时，该注册表会记录在每个 npm 版本中。 | +| `--inline-icon-bytes ` | `16384` | 以 `data:` URL 内嵌的图标的最大字节数，不超过格式的内联图标上限 32768。 | +| `--concurrency ` | `4` | 同时读取的来源数。 | +| `--timeout-ms ` | `60000` | 单个请求或 Git 命令的时限。 | + +所有来源都读取成功时退出状态为 0，有来源无法读取时为 1，参数无效或列表无效时为 2。索引格式会拒绝的列表行属于无效列表，例如 `http:` URL、带凭据、查询或片段的 URL、路径段少于两个的 URL，以及 tarball URL。无法读取的来源保留其上一次成功的结果，因此临时故障不会把插件从索引中移除；生成器仍会报告该失败。 + +## 列出的内容 + +每个包成为一个插件，带有一个或多个版本。只有 `package.json` 声明了 `dsh.bundle`、且其 `patch` 是一个文件路径或文件路径列表的版本才会列出。 + +- **npm 包**：注册表文档中未弃用、声明了组合包（bundle）且有发布时间的每个版本。版本只取自注册表文档；生成器只下载最高列出版本的 tarball，用于读取展示字段。该 tarball 的 `package.json` 必须与所请求的包名和版本一致。 +- **Git 仓库**：生成器按以下顺序在默认分支上查找组合包： + 1. 根 `package.json` 声明了 `dsh.bundle`：仅使用根包； + 2. `package.json` 的 `workspaces` 或 `pnpm-workspace.yaml` 的 `packages` 匹配到声明了 `dsh.bundle` 的包：使用这些包。每个 glob 段可以是字面名称、含 `*` 的名称或 `**`；生成器忽略否定 glob，并报告未匹配任何内容的 glob； + 3. 否则，使用三层目录以内、位于 `node_modules`、`test`、`tests`、`fixtures`、`example` 和 `examples` 之外的所有声明了 `dsh.bundle` 的包。 + + 每个包的版本来自发布标签（tag）和默认分支。`vX.Y.Z` 或 `X.Y.Z` 标签适用于所有包，`@X.Y.Z` 标签（例如 `@scope/dsh-a@1.2.0`）适用于同名的包；标签中的版本必须是确切的 SemVer 版本。当包目录在打标签的提交中有一个组合包 manifest（元数据清单），且其名称相同、版本等于标签版本时，该标签列出这个包。默认分支的提交列出每个包的当前版本，除非某个标签已经列出该版本。多个标签列出同一版本时，`@` 标签优先于统一标签，其次是较早的提交。发布时间是该提交的提交者时间。 + +`engines.dsh` 取 manifest 的 `engines.dsh` 并去掉首尾空白。没有该字段时，它是 manifest 中 `@deepseek-ai/dsh` 和 `@deepseek-ai/dsh-*` 的 `peerDependencies` 范围的合取；恰为 `workspace:*`、`workspace:^` 或 `workspace:~` 的 peer 表示当前运行的 DSH 版本，不增加限制；相同的范围只出现一次。`*` 范围接受所有版本，因此范围全为 `*` 的版本不列出 `engines.dsh`。如果安装时在所有 DSH 版本上都会拒绝某个版本，该版本会被跳过并发出警告：范围为空、`engines.dsh` 中出现 `workspace:` 值或 peer 中出现其他 `workspace:` 范围、范围无效，或者没有任何版本能同时满足这些范围。 + +展示字段来自最高的列出版本，并遵循 DSH 读取已安装插件时的规则。`./locale/en.json` 和 `./locale/.json` 通过包的 `exports` 映射以 `import`、`node` 和 `default` 条件解析，没有该映射时按字面路径读取，并且每个 locale 文件都必须与英文文件位于同一目录；其 `meta.title` 和 `meta.description` 缺失时回退到 manifest 的 `name` 和 `description`。图标是 manifest 的 `icon`（相对路径），否则是导出的 `./icon`：格式为 SVG、PNG、JPEG 或 WebP，不超过 256 KiB，并位于包内。`author` 是 manifest `author` 字符串中去掉 `` 和 `(url)` 后的名字部分，或 author 对象的 `name`。`homepage` 在 manifest 的 `homepage` 是绝对 http(s) URL 时按原样复制；缺少时不会用 `repository` 等其他字段代替。格式错误的 locale 文件、无效的 locale 名称和被拒绝的图标会作为警告报告。由于该格式要求英文，没有英文文本也没有 manifest 回退值的翻译会被丢弃。每个条目在写出前都经过 [`dsh-plugin-market-index`](../../packages/util/plugin-market-index/README.zh.md) 校验。 + +`metadata.keywords` 是用于发现插件的可选字符串数组。按约定使用 npm `package.json` 的 `keywords`；索引生成器可以补充或调整它。本参考生成器原样复制最高列出版本 manifest 的 `keywords`，包括空字符串、重复值和空数组。字段缺省时省略；如果值不是数组，或数组包含任何非字符串成员，则发出警告并省略整个字段，但保留该包。 + +多个来源产生相同包名时，列表中靠前的来源生效。 + +## 缓存与重复运行 + +每次运行时，生成器读取每个 npm 来源的注册表文档和每个 Git 仓库的引用（ref）。结果按不可变的输入缓存：npm 插件按注册表、图标阈值、包和列出版本的集合缓存；Git 仓库按默认分支提交和发布标签缓存；每个 Git 包按提交和目录缓存。因此 Git 运行只拉取其包尚未缓存的提交。生成器以原子方式写入缓存文件，并重建损坏的或索引格式会拒绝的缓存文件。输出是确定性的：条目按列表顺序排列，索引文件和图标文件只在内容变化时才重写，因此上游没有变化时再次运行生成器不会改动输出。 + +## 安全 + +生成器只下载和读取文件。npm tarball 在内存中读取：tarball 及其解包后的文件各自不能超过 64 MiB，字节必须与注册表的完整性哈希一致，并且只保留 `node_modules` 之外的普通文件，不接受离开包目录的路径。Git 对象以深度 1 拉取到一个私有的临时裸仓库，并作为 blob 读取，因此不存在工作树，检出过滤器、LFS 下载和符号链接都不会生效；符号链接和子模块视为不存在。Git 运行时禁用钩子和文件系统监视器、不处理子模块、只允许 https 和 ssh 传输协议，并且不继承除 `GIT_SSH` 和 `GIT_SSH_COMMAND` 之外的 `GIT_*` 变量。你配置的 ssh 程序和 `GIT_SSH_COMMAND` 优先；两者都没有时，ssh 以批处理模式运行，因此不会等待输入。超过 1 MiB 的 manifest、语言文件和工作区文件会被拒绝。私有仓库使用你的 Git 凭据配置；可以用 `url..insteadOf` 把 https URL 改为通过 ssh 访问。 + +索引内容是各个包对自身的声明。请审查你发布的列表：列出的仓库可以使用任意包名。 + +## 限制 + +- Git 服务器必须允许按 SHA 拉取提交，GitHub、GitLab 和 Gitea 都允许。服务器拒绝时，生成器按名称拉取标签和默认分支；如果某个引用在列出之后移动了，这一步会失败。 +- 生成器不使用 `.npmrc` 中的注册表令牌；只支持公开或无需认证的注册表。 +- Git 从不提示输入凭据。需要凭据的仓库必须能通过凭据助手或 ssh agent 读取。 diff --git a/apps/plugin-market-generator/package.json b/apps/plugin-market-generator/package.json new file mode 100644 index 0000000000..4210e1ac7c --- /dev/null +++ b/apps/plugin-market-generator/package.json @@ -0,0 +1,23 @@ +{ + "name": "@deepseek-ai/dsh-plugin-market-generator", + "description": "Reference generator of plugin market index files from a list of npm packages and Git repositories", + "version": "0.2.1-alpha.3", + "private": true, + "type": "module", + "scripts": { + "start": "tsx src/bin.ts" + }, + "license": "MIT", + "dependencies": { + "@deepseek-ai/dsh-plugin-market-index": "workspace:*", + "resolve.exports": "^2.0.3", + "semver": "^7.8.5", + "tar": "^7.5.22", + "yaml": "^2.9.0" + }, + "devDependencies": { + "@deepseek-ai/dsh-package-manifest": "workspace:*", + "@types/semver": "^7.8.0", + "tsx": "^4.22.4" + } +} diff --git a/apps/plugin-market-generator/src/bin.ts b/apps/plugin-market-generator/src/bin.ts new file mode 100644 index 0000000000..d001e0c244 --- /dev/null +++ b/apps/plugin-market-generator/src/bin.ts @@ -0,0 +1,4 @@ +/** Command-line entry: run the generator with the process arguments and exit with its status. */ +import { main } from './cli.ts' + +process.exitCode = await main(process.argv.slice(2)) diff --git a/apps/plugin-market-generator/src/cli.ts b/apps/plugin-market-generator/src/cli.ts new file mode 100644 index 0000000000..b40f3de3a8 --- /dev/null +++ b/apps/plugin-market-generator/src/cli.ts @@ -0,0 +1,109 @@ +/** + * Reference plugin market index generator. + * + * Usage: plugin-market-generator --input --output --title [options] + * + * The list holds one npm package name or https Git repository URL per line. The generator lists every non-deprecated + * npm bundle version, and the versions at Git release tags and the default branch, never runs package code, and writes + * a validated format-1 index. Unchanged releases and commits are read from the cache directory, and the index file is + * rewritten only when its content changes. + */ +import { existsSync, lstatSync, readdirSync, readFileSync, rmSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { parseArgs } from 'node:util' +import { MAX_INLINE_ICON_BYTES } from '@deepseek-ai/dsh-plugin-market-index' +import { writeIfChanged } from './files.ts' +import { generate } from './generate.ts' +import { GENERATED_ICON } from './metadata.ts' +import { parseSources } from './sources.ts' + +const PUBLIC_REGISTRY = 'https://registry.npmjs.org/' +/** Names of icon files the generator writes, which it may also remove. */ + +const HELP = `Usage: plugin-market-generator --input --output --title [options] + +Options: + --input source list: one npm package name or https Git URL per line; lines starting with # are comments + --output index file to write; large icons go to icons/ beside it, and stale generated icons there are removed + --title market title + --description market description + --cache cache directory (default: .plugin-market-cache beside the input list) + --registry npm registry (default: ${PUBLIC_REGISTRY}) + --inline-icon-bytes largest icon embedded as a data URL, at most ${String(MAX_INLINE_ICON_BYTES)} (default: 16384) + --concurrency sources read at once (default: 4) + --timeout-ms deadline for one request or git command (default: 60000) + -h, --help show this help + +Exit status: 0 when every source was read, 1 when some source fell back to its cached result or was omitted, +2 for invalid arguments or input.` + +function integer(value: string | undefined, fallback: number, name: string, max?: number): number { + if (value === undefined) return fallback + const parsed = Number(value) + if (!Number.isInteger(parsed) || parsed < 1 || (max !== undefined && parsed > max)) { + throw new Error(`--${name} must be a positive integer${max === undefined ? '' : ` up to ${String(max)}`}`) + } + return parsed +} + +/** + * Run the generator. + * @param argv - command-line arguments after the program name. + * @returns the process exit status. + */ +export async function main(argv: readonly string[]): Promise { + let values + try { + ({ values } = parseArgs({ + args: [...argv], + options: { + input: { type: 'string' }, output: { type: 'string' }, title: { type: 'string' }, description: { type: 'string' }, + cache: { type: 'string' }, registry: { type: 'string' }, 'inline-icon-bytes': { type: 'string' }, + concurrency: { type: 'string' }, 'timeout-ms': { type: 'string' }, help: { type: 'boolean', short: 'h' }, + }, + })) + } catch (error) { + console.error(`${(error as Error).message}\n\n${HELP}`) + return 2 + } + if (values.help === true) { + console.log(HELP) + return 0 + } + try { + if (values.input === undefined || values.output === undefined || values.title === undefined) throw new Error('--input, --output, and --title are required') + // `pnpm --filter … start` runs in the package directory; paths are relative to where the user ran pnpm. + const base = process.env.INIT_CWD ?? process.cwd() + const input = resolve(base, values.input) + const output = resolve(base, values.output) + const registry = new URL(values.registry ?? PUBLIC_REGISTRY).href.replace(/\/?$/u, '/') + const inlineIconBytes = integer(values['inline-icon-bytes'], 16384, 'inline-icon-bytes', MAX_INLINE_ICON_BYTES) + const timeoutMs = integer(values['timeout-ms'], 60000, 'timeout-ms') + const concurrency = integer(values.concurrency, 4, 'concurrency') + const result = await generate({ + sources: parseSources(readFileSync(input, 'utf8')), + cacheDir: values.cache === undefined ? join(dirname(input), '.plugin-market-cache') : resolve(base, values.cache), + registry, + recordRegistry: registry !== PUBLIC_REGISTRY, + title: values.title, + ...values.description === undefined ? {} : { description: values.description }, + inlineIconBytes, + limits: { timeoutMs, maxTarballBytes: 64 * 1024 * 1024 }, + concurrency, + warn: (message) => { console.error(`warning: ${message}`) }, + }) + for (const [path, cached] of result.icons) writeIfChanged(join(dirname(output), path), readFileSync(cached)) + const iconDir = join(dirname(output), 'icons') + for (const name of existsSync(iconDir) ? readdirSync(iconDir) : []) { + const file = join(iconDir, name) + if (GENERATED_ICON.test(name) && !result.icons.has(`icons/${name}`) && lstatSync(file).isFile()) rmSync(file, { force: true }) + } + const changed = writeIfChanged(output, `${JSON.stringify(result.index, null, 2)}\n`) + const plugins = result.index.plugins.length + console.log(`${changed ? 'wrote' : 'unchanged'} ${output}: ${plugins} plugin(s), ${result.failures.length} failed source(s)`) + return result.failures.length === 0 ? 0 : 1 + } catch (error) { + console.error((error as Error).message) + return 2 + } +} diff --git a/apps/plugin-market-generator/src/fetchers.ts b/apps/plugin-market-generator/src/fetchers.ts new file mode 100644 index 0000000000..44af180997 --- /dev/null +++ b/apps/plugin-market-generator/src/fetchers.ts @@ -0,0 +1,261 @@ +/** + * Download package files without running their code. npm tarballs are streamed under a size limit, verified against the + * registry integrity, and unpacked in memory. Git objects are fetched into a private temporary bare repository with + * hooks, the file-system monitor, submodules, and all but the allowed transports disabled, and files are read as blobs: + * no working tree is created, so no checkout filter, LFS download, or symbolic link takes effect. + */ +import { createHash, getHashes } from 'node:crypto' +import { execFile } from 'node:child_process' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' +import { Parser, type ReadEntry } from 'tar' +import { PackageFiles, type FileEntry } from './files.ts' +import { readManifest, type Manifest } from './packages.ts' + +const run = promisify(execFile) + +/** Network limits for one source. */ +export interface FetchLimits { + readonly timeoutMs: number + /** Largest npm tarball accepted, in bytes, both compressed and unpacked. */ + readonly maxTarballBytes: number +} + +/** The registry document of one npm package; only the fields the generator reads are typed. */ +export interface Packument { + readonly 'dist-tags'?: { readonly latest?: unknown } + readonly time?: Readonly> + readonly versions?: Readonly> +} + +/** + * Hash a value into a short cache-path component. + * @param value - the identity to hash. + * @returns 24 hexadecimal characters. + */ +export function cacheKey(value: string): string { + return createHash('sha256').update(value).digest('hex').slice(0, 24) +} + +/** + * Read the registry document of an npm package. + * @param registry - registry base URL ending in `/`. + * @param name - package name. + * @param limits - request deadline. + * @returns the parsed document. + */ +export async function npmPackument(registry: string, name: string, limits: FetchLimits): Promise { + const response = await fetch(new URL(name.replace('/', '%2f'), registry), { signal: AbortSignal.timeout(limits.timeoutMs) }) + if (!response.ok) throw new Error(`${name}: registry answered HTTP ${response.status}`) + return await response.json() as Packument +} + +/** Whether `bytes` match one supported hash of a subresource-integrity string. */ +function matchesIntegrity(bytes: Uint8Array, integrity: string): boolean { + const supported = new Set(getHashes()) + const hashes = integrity.trim().split(/\s+/u).map(token => /^(sha\d+)-([A-Za-z0-9+/=]+)/u.exec(token)).filter(match => match !== null) + const usable = hashes.filter(([, algorithm]) => supported.has(algorithm as string)) + return usable.length > 0 && usable.some(([, algorithm, digest]) => createHash(algorithm as string).update(bytes).digest('base64') === digest) +} + +/** Unpack the regular files of an npm tarball below its top directory, without `node_modules`, into memory. */ +function unpack(bytes: Uint8Array, label: string, maxBytes: number): Promise { + return new Promise((resolve, reject) => { + const files = new Map() + let total = 0 + const parser = new Parser({ strict: true }) + parser.on('entry', (entry: ReadEntry) => { + const path = PackageFiles.resolve('', entry.path.split('/').slice(1).join('/')) + if (entry.type !== 'File' || path === undefined || path === '' || path.split('/').includes('node_modules')) { + entry.resume() + return + } + const chunks: Buffer[] = [] + entry.on('data', (chunk: Buffer) => { + total += chunk.length + if (total > maxBytes) parser.abort(new Error(`unpacked tarball exceeds ${maxBytes} bytes`)) + else chunks.push(chunk) + }) + entry.on('end', () => { + const contents = Buffer.concat(chunks) + files.set(path, { size: contents.length, read: () => Promise.resolve(contents) }) + }) + }) + parser.on('error', (error: Error) => { reject(new Error(`${label}: ${error.message}`)) }) + parser.on('end', () => { resolve(new PackageFiles(files)) }) + parser.end(Buffer.from(bytes)) + }) +} + +/** + * Download, verify, and unpack an npm tarball in memory. Its `package.json` must name the requested package and version. + * @param name - package name. + * @param version - exact version. + * @param dist - the version's registry `dist` fields. + * @param limits - request deadline and size bound. + * @returns the package files and its manifest. + */ +export async function npmTarball( + name: string, version: string, dist: { readonly tarball?: unknown; readonly integrity?: unknown } | undefined, limits: FetchLimits, +): Promise<{ files: PackageFiles; manifest: Manifest }> { + const label = `${name}@${version}` + if (typeof dist?.tarball !== 'string' || typeof dist.integrity !== 'string') throw new Error(`${label}: registry lists no tarball with integrity`) + const response = await fetch(dist.tarball, { signal: AbortSignal.timeout(limits.timeoutMs) }) + if (!response.ok) throw new Error(`${label}: tarball answered HTTP ${response.status}`) + const chunks: Uint8Array[] = [] + let total = 0 + // A successful response has a body; throwing inside the loop cancels it. + for await (const chunk of response.body as ReadableStream) { + total += chunk.length + if (total > limits.maxTarballBytes) throw new Error(`${label}: tarball exceeds ${limits.maxTarballBytes} bytes`) + chunks.push(chunk) + } + const bytes = Buffer.concat(chunks) + if (!matchesIntegrity(bytes, dist.integrity)) throw new Error(`${label}: tarball does not match its registry integrity`) + const files = await unpack(bytes, label, limits.maxTarballBytes) + const manifest = await readManifest(files, '') + if (manifest?.name !== name || manifest.version !== version) throw new Error(`${label}: tarball package.json does not name ${label}`) + return { files, manifest } +} + +/** Git transport names, such as `https`, `ssh`, or `file`. */ +export type GitTransports = readonly string[] + +/** The default branch commit and the tags of a remote repository. */ +export interface RemoteRefs { + readonly head: string + /** Commit of each tag by tag name; annotated tags are peeled. */ + readonly tags: ReadonlyMap +} + +/** The environment of every Git command: the caller's, without Git overrides, and without prompts or LFS downloads. */ +function gitEnv(): NodeJS.ProcessEnv { + const env = Object.fromEntries(Object.entries(process.env).filter(([name]) => !name.startsWith('GIT_') || name === 'GIT_SSH' || name === 'GIT_SSH_COMMAND')) + // A user's own ssh program keeps precedence; the default only keeps ssh from waiting for a prompt. + const ssh = env.GIT_SSH === undefined && env.GIT_SSH_COMMAND === undefined ? { GIT_SSH_COMMAND: 'ssh -o BatchMode=yes' } : {} + return { ...env, GIT_TERMINAL_PROMPT: '0', GIT_LFS_SKIP_SMUDGE: '1', ...ssh } +} + +/** + * Run Git with options that keep it from running repository-supplied programs and allow only `transports`; + * `url..insteadOf` in the user's configuration still applies, so an https URL can be routed to ssh. + */ +function git( + args: readonly string[], transports: GitTransports, limits: FetchLimits, cwd?: string, input?: string, +): Promise<{ stdout: Buffer }> { + const safety = [ + 'core.hooksPath=/dev/null', 'core.fsmonitor=false', 'submodule.recurse=false', 'gc.auto=0', 'maintenance.auto=false', 'protocol.allow=never', + ...transports.map(transport => `protocol.${transport}.allow=always`), + ].flatMap(setting => ['-c', setting]) + const command = run('git', [...safety, ...args], { cwd, env: gitEnv(), timeout: limits.timeoutMs, encoding: 'buffer', maxBuffer: 256 * 1024 * 1024 }) + // Git may exit before it reads its input; the exit status reports that failure, so the broken pipe is not a + // second one. Whether the write races the exit depends on scheduling, so no test reaches the handler reliably. + /* v8 ignore next */ + command.child.stdin?.on('error', () => {}) + command.child.stdin?.end(input ?? '') + return command +} + +/** + * List the default branch commit and the tags of a repository. + * @param url - repository URL. + * @param transports - allowed Git transports. + * @param limits - command deadline. + * @returns the refs. + */ +export async function gitRefs(url: string, transports: GitTransports, limits: FetchLimits): Promise { + const { stdout } = await git(['ls-remote', '--', url, 'HEAD', 'refs/tags/*'], transports, limits) + let head: string | undefined + const tags = new Map() + for (const [, commit, ref] of stdout.toString('utf8').matchAll(/^([0-9a-f]{40})\t(\S+)$/gmu)) { + if (ref === 'HEAD') head = commit + const tag = /^refs\/tags\/(.+?)(\^\{\})?$/u.exec(ref as string) + // A peeled `^{}` line follows its annotated tag and replaces the tag object with the commit. + if (tag !== null) tags.set(tag[1] as string, commit as string) + } + if (head === undefined) throw new Error(`${url}: no HEAD commit`) + return { head, tags } +} + +/** Objects fetched from one repository into a private bare repository. */ +export interface GitObjects { + /** + * Fetch commits with depth 1 in one request, skipping commits already fetched. When the server refuses a request by + * commit, the refs that point to the commits are fetched by name instead. + * @param commits - commits listed by {@link gitRefs}. + */ + fetch(commits: readonly string[]): Promise + /** + * @param commit - a commit listed by {@link gitRefs}; it is fetched first when necessary. + * @param path - the directory to list, `''` for the whole tree. + * @returns the regular files below `path`, with paths relative to it, read on demand. + */ + tree(commit: string, path: string): Promise + /** + * @param commit - a fetched commit. + * @returns its committer time in Unix seconds. + */ + committed(commit: string): Promise +} + +/** + * Run `read` with a private temporary bare repository for `url`, then delete it. + * @param url - repository URL. + * @param refs - the repository's refs, which name the commits for a fetch by name. + * @param transports - allowed Git transports. + * @param limits - command deadline. + * @param read - fetches and reads objects. + * @returns what `read` returns. + */ +export async function withGitObjects( + url: string, refs: RemoteRefs, transports: GitTransports, limits: FetchLimits, read: (objects: GitObjects) => Promise, +): Promise { + const dir = mkdtempSync(join(tmpdir(), 'plugin-market-git-')) + const inRepository = (args: readonly string[], input?: string) => git(args, transports, limits, dir, input) + const names = new Map([...[...refs.tags].map(([tag, commit]) => [commit, `refs/tags/${tag}`] as const), [refs.head, 'HEAD']]) + const fetched = new Set() + const listTree = async (commit: string, path: string) => { + const scope = path === '' ? [] : ['--', `${path}/`] + const listing = (await inRepository(['ls-tree', '-r', '-l', '-z', '--full-tree', commit, ...scope])).stdout.toString('utf8') + const prefix = path === '' ? '' : `${path}/` + const files = new Map() + for (const line of listing.split('\0')) { + // Symbolic links (120000) and submodules (160000) are not files. + const fields = /^100(?:644|755) blob (?[0-9a-f]+) +(?\d+)\t(?.+)$/su.exec(line)?.groups + if (fields === undefined) continue + const { blob, size, file } = fields as { blob: string; size: string; file: string } + files.set(file.slice(prefix.length), { size: Number(size), read: async () => (await inRepository(['cat-file', 'blob', blob])).stdout }) + } + return new PackageFiles(files) + } + const objects: GitObjects = { + async fetch(commits) { + const wanted = [...new Set(commits)].filter(commit => !fetched.has(commit)) + if (wanted.length === 0) return + // Targets go through standard input, so any number of commits fits on the command line. + const fetch = (targets: readonly string[]) => inRepository(['fetch', '--quiet', '--depth', '1', '--no-tags', '--stdin', '--', url], `${targets.join('\n')}\n`) + await fetch(wanted).catch((_refused: unknown) => fetch(wanted.map(commit => names.get(commit) as string))) + for (const commit of wanted) fetched.add(commit) + }, + async tree(commit, path) { + await objects.fetch([commit]) + return listTree(commit, path) + }, + async committed(commit) { + // The raw commit object, so no user formatting option such as signature display runs on repository data. + const header = (await inRepository(['cat-file', 'commit', commit])).stdout.toString('utf8') + const time = /^committer .* (\d+) [+-]\d{4}$/mu.exec(header)?.[1] + /* v8 ignore next -- Git writes a committer line into every commit object */ + if (time === undefined) throw new Error(`${commit}: commit has no committer time`) + return Number(time) + }, + } + try { + await inRepository(['init', '--quiet', '--bare']) + return await read(objects) + } finally { + rmSync(dir, { recursive: true, force: true }) + } +} diff --git a/apps/plugin-market-generator/src/files.ts b/apps/plugin-market-generator/src/files.ts new file mode 100644 index 0000000000..c5ab2671cb --- /dev/null +++ b/apps/plugin-market-generator/src/files.ts @@ -0,0 +1,129 @@ +/** Atomic file writes shared by the cache and the output, and the read-only file trees packages are read from. */ +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs' +import { dirname, posix } from 'node:path' + +/** + * Write `contents` through a temporary sibling and a rename, leaving the file untouched when it already holds them. + * A reader never sees a partially written file. + * @param file - destination path. + * @param contents - complete file contents. + * @returns whether the file changed. + */ +export function writeIfChanged(file: string, contents: Uint8Array | string): boolean { + const bytes = Buffer.from(contents) + if (existsSync(file) && readFileSync(file).equals(bytes)) return false + mkdirSync(dirname(file), { recursive: true }) + const temporary = `${file}.tmp-${process.pid}-${Math.random().toString(36).slice(2)}` + writeFileSync(temporary, bytes) + renameSync(temporary, file) + return true +} + +/** + * Read a JSON file written by {@link writeIfChanged}. + * @param file - path to read. + * @returns the parsed value, or undefined when the file is absent or is not valid JSON. + */ +export function readJsonFile(file: string): unknown { + if (!existsSync(file)) return undefined + try { + return JSON.parse(readFileSync(file, 'utf8')) + } catch (_malformed) { + // A file from an older or interrupted generator is treated as absent and rebuilt. + return undefined + } +} + +/** One regular file of a {@link PackageFiles} tree. */ +export interface FileEntry { + readonly size: number + readonly read: () => Promise +} + +/** + * The regular files of an npm tarball or a Git tree, read without a file system. Paths are normalized POSIX paths + * relative to the tree root `''`; symbolic links, submodules, and directories without files do not exist. + */ +/** Largest manifest, locale, or workspace file read, in bytes. */ +export const MAX_TEXT_FILE_BYTES = 1024 * 1024 + +export class PackageFiles { + readonly #files: ReadonlyMap + /** Names of the directories and files directly inside each directory. */ + readonly #children = new Map; readonly files: string[] }>() + + /** @param files - regular files by normalized relative path. */ + constructor(files: ReadonlyMap) { + this.#files = files + for (const path of files.keys()) { + const segments = path.split('/') + segments.forEach((name, depth) => { + const dir = segments.slice(0, depth).join('/') + let children = this.#children.get(dir) + if (children === undefined) this.#children.set(dir, children = { directories: new Set(), files: [] }) + if (depth === segments.length - 1) children.files.push(name) + else children.directories.add(name) + }) + } + } + + /** + * Resolve a relative reference inside the tree. + * @param base - directory the reference is relative to. + * @param reference - relative path, possibly starting with `./`. + * @returns the normalized path, or undefined when the reference is absolute or leaves the tree. + */ + static resolve(base: string, reference: string): string | undefined { + if (posix.isAbsolute(reference)) return undefined + const path = posix.normalize(posix.join(base, reference)).replace(/\/$/u, '') + if (path === '..' || path.startsWith('../')) return undefined + return path === '.' ? '' : path + } + + /** + * @param path - normalized file path. + * @returns the regular file with that path, or undefined. + */ + file(path: string): FileEntry | undefined { + return this.#files.get(path) + } + + /** + * Read a text file such as a manifest or locale file. + * @param path - normalized file path. + * @returns the file contents, or undefined when no regular file has that path. + * @throws when the file exceeds {@link MAX_TEXT_FILE_BYTES}. + */ + async read(path: string): Promise { + const file = this.#files.get(path) + if (file !== undefined && file.size > MAX_TEXT_FILE_BYTES) throw new Error(`${path} exceeds ${MAX_TEXT_FILE_BYTES} bytes`) + return await file?.read() + } + + /** + * @param dir - normalized directory path. + * @returns the names of the directories directly inside `dir`, sorted. + */ + directories(dir: string): string[] { + return [...this.#children.get(dir)?.directories ?? []].sort() + } + + /** + * @param dir - normalized directory path. + * @returns the names of the regular files directly inside `dir`, sorted. + */ + fileNames(dir: string): string[] { + return [...this.#children.get(dir)?.files ?? []].sort() + } + + /** + * @param dir - normalized directory path. + * @returns the files below `dir`, with paths relative to it. + */ + within(dir: string): PackageFiles { + if (dir === '') return this + const prefix = `${dir}/` + const inside = [...this.#files].flatMap(([path, entry]) => path.startsWith(prefix) ? [[path.slice(prefix.length), entry] as const] : []) + return new PackageFiles(new Map(inside)) + } +} diff --git a/apps/plugin-market-generator/src/generate.ts b/apps/plugin-market-generator/src/generate.ts new file mode 100644 index 0000000000..078fa8a6c4 --- /dev/null +++ b/apps/plugin-market-generator/src/generate.ts @@ -0,0 +1,357 @@ +/** Turn a source list into a validated format-1 index, reusing cached results for unchanged releases and commits. */ +import { existsSync } from 'node:fs' +import { join, posix } from 'node:path' +import { parseMarketIndex } from '@deepseek-ai/dsh-plugin-market-index' +import semver from 'semver' +import { cacheKey, gitRefs, npmPackument, npmTarball, withGitObjects, type FetchLimits, type GitObjects, type GitTransports } from './fetchers.ts' +import { readJsonFile, writeIfChanged, type PackageFiles } from './files.ts' +import { GENERATED_ICON, readDisplayMetadata, type Display, type IconFile } from './metadata.ts' +import { dshRange, findBundlePackages, isBundle, readManifest, type Manifest } from './packages.ts' +import type { Source } from './sources.ts' + +/** One generated version, in the field order of the format. */ +export interface GeneratedVersion { + readonly version: string + readonly publishTimestamp: number + readonly source: Readonly> + readonly engines?: { readonly dsh: string } +} + +/** One generated plugin, in the field order of the format. */ +export interface GeneratedEntry extends Display { + readonly name: string + readonly versions: readonly GeneratedVersion[] +} + +/** Everything one run needs besides the network. */ +export interface GenerateOptions { + readonly sources: readonly Source[] + readonly cacheDir: string + /** npm registry base URL ending in `/`. */ + readonly registry: string + /** Whether versions name `registry`; false for the public registry, which clients use by default. */ + readonly recordRegistry: boolean + readonly title: string + readonly description?: string + readonly inlineIconBytes: number + readonly limits: FetchLimits + readonly concurrency: number + readonly warn: (message: string) => void + /** Git transports allowed besides none; `['https', 'ssh']` when absent. Tests add `file` for local repositories. */ + readonly gitTransports?: GitTransports +} + +/** A finished run: the index document, the icon files it refers to, and the sources that could not be read. */ +export interface GenerateResult { + readonly index: { readonly format: 1; readonly title: string; readonly description?: string; readonly plugins: readonly GeneratedEntry[] } + /** Icon files by index-relative path; contents live under the cache. */ + readonly icons: ReadonlyMap + readonly failures: readonly string[] +} + +/** One listable version of a package; `display` is read from that version's files when they were read. */ +interface Release { + readonly name: string + readonly version: GeneratedVersion + readonly display?: Display +} + +/** Base URL only used to validate relative icon references. */ +const VALIDATION_URL = 'https://index.invalid/index.json' +/** Part of every cache key; change it when the way entries are derived changes, so cached results are rebuilt. */ +const CACHE_VERSION = 6 + +/** Why the index format rejects these plugins, or undefined when it accepts them. */ +function invalid(plugins: unknown): string | undefined { + try { + parseMarketIndex({ format: 1, title: 'check', plugins }, VALIDATION_URL) + return undefined + } catch (error) { + return (error as Error).message + } +} + +/** Combine releases of one package into an entry whose display fields come from the highest version. */ +function entryOf(name: string, releases: readonly Release[]): GeneratedEntry { + const sorted = [...releases].sort((left, right) => semver.rcompare(left.version.version, right.version.version)) + const { title, description, icon, author, homepage, metadata } = sorted[0]?.display ?? {} + return { + name, + ...title === undefined ? {} : { title }, ...description === undefined ? {} : { description }, + ...icon === undefined ? {} : { icon }, ...author === undefined ? {} : { author }, ...homepage === undefined ? {} : { homepage }, + ...metadata === undefined ? {} : { metadata }, + versions: sorted.map(release => release.version), + } +} + +/** The result cache. A record is used only when the index format accepts its plugins and their icon files exist. */ +class Cache { + constructor(private readonly dir: string) {} + + /** @returns the record file of an immutable identity. */ + file(...identity: unknown[]): string { + return join(this.dir, 'results', `${cacheKey(JSON.stringify([CACHE_VERSION, ...identity]))}.json`) + } + + usable(plugins: unknown): plugins is GeneratedEntry[] { + return invalid(plugins) === undefined + && (plugins as GeneratedEntry[]).every(({ icon }) => icon === undefined || icon.startsWith('data:') + || (icon.startsWith('icons/') && GENERATED_ICON.test(icon.slice('icons/'.length)) && existsSync(join(this.dir, icon)))) + } + + /** @returns cached plugins, or undefined when the record is absent or unusable. */ + plugins(file: string): GeneratedEntry[] | undefined { + const plugins = (readJsonFile(file) as { plugins?: unknown } | undefined)?.plugins + return this.usable(plugins) ? plugins : undefined + } + + /** @returns cached display fields, or undefined when the record is absent or unusable. */ + display(file: string): Display | undefined { + const display = (readJsonFile(file) as { display?: Display } | undefined)?.display + // The format validates display fields only as part of a plugin, so the record is checked inside a minimal one. + const minimal = { ...display, name: 'placeholder', versions: [{ version: '0.0.0', publishTimestamp: 0, source: { type: 'npm' } }] } + return display !== undefined && this.usable([minimal]) ? display : undefined + } + + /** @returns a cached release, null for a cached absence, or undefined when the record is absent or unusable. */ + release(file: string): Release | null | undefined { + const release = (readJsonFile(file) as { release?: Release | null } | undefined)?.release + if (release === null) return null + return release !== undefined && this.usable([entryOf(release.name, [release])]) ? release : undefined + } + + write(file: string, value: object, icons: readonly IconFile[]): void { + for (const icon of icons) writeIfChanged(join(this.dir, icon.path), icon.bytes) + writeIfChanged(file, `${JSON.stringify(value)}\n`) + } +} + +/** Build a release from a version's manifest, or warn why the version is not listed. */ +function releaseOf( + name: string, manifest: Manifest, version: string, publishTimestamp: number, source: Record, + warn: (message: string) => void, +): Release | undefined { + const range = dshRange(manifest) + const release = { + name, + version: { version, publishTimestamp, source, ...typeof range === 'string' ? { engines: { dsh: range } } : {} }, + } + const problem = range instanceof Error ? range.message : invalid([entryOf(name, [release])])?.replace('$.plugins[0]', 'entry') + if (problem === undefined) return release + warn(`${name}@${version} skipped: ${problem}`) + return undefined +} + +async function readDisplay( + files: PackageFiles, manifest: Manifest, label: string, options: GenerateOptions, icons: IconFile[], +): Promise { + const { display, iconFile } = await readDisplayMetadata(files, manifest, { + inlineBytes: options.inlineIconBytes, warn: (message) => { options.warn(`${label}: ${message}`) }, + }) + if (iconFile !== undefined) icons.push(iconFile) + return display +} + +/** + * Read every non-deprecated bundle version from the registry document, and the display fields from the tarball of the + * highest one, through a cache keyed by the listed versions. + */ +async function readNpm(source: Extract, options: GenerateOptions, cache: Cache): Promise { + const packument = await npmPackument(options.registry, source.name, options.limits) + const npmSource = { type: 'npm', ...options.recordRegistry ? { registry: options.registry } : {} } + const warn = (message: string) => { options.warn(`${source.line}: ${message}`) } + const releases = Object.entries(packument.versions ?? {}).flatMap(([version, manifest]) => { + const published = packument.time?.[version] + const deprecated = typeof manifest.deprecated === 'string' && manifest.deprecated !== '' + if (deprecated || !isBundle(manifest) || typeof published !== 'string') return [] + return releaseOf(source.name, manifest, version, Math.floor(Date.parse(published) / 1000), npmSource, warn) ?? [] + }) + const highest = entryOf(source.name, releases).versions[0]?.version + if (highest === undefined) { + warn('no published version declares dsh.bundle') + return [] + } + const file = cache.file('npm', options.registry, options.inlineIconBytes, source.name, releases.map(release => release.version.version).sort()) + const cached = cache.plugins(file) + if (cached !== undefined) return cached + const { files, manifest } = await npmTarball(source.name, highest, packument.versions?.[highest]?.dist, options.limits) + const icons: IconFile[] = [] + const display = await readDisplay(files, manifest, `${source.name}@${highest}`, options, icons) + const plugins = [entryOf(source.name, releases.map(release => release.version.version === highest ? { ...release, display } : release))] + cache.write(file, { plugins }, icons) + return plugins +} + +/** A tag that names a release: `vX.Y.Z` and `X.Y.Z` for every package, `@X.Y.Z` for one package. */ +interface ReleaseTag { + readonly tag: string + readonly commit: string + readonly version: string + /** The one package the tag applies to; absent for every package. */ + readonly name?: string +} + +function releaseTag(tag: string, commit: string): ReleaseTag | undefined { + const { name, version = '' } = /^(?:(?.+)@|v)?(?[^@]+)$/u.exec(tag)?.groups ?? {} + return semver.valid(version) === version ? { tag, commit, version, ...name === undefined ? {} : { name } } : undefined +} + +/** + * Read the bundle packages on a repository's default branch, with their versions at release tags and at the default + * branch commit. Results are cached for the listed refs, and per commit and package directory. + */ +async function readGit(source: Extract, options: GenerateOptions, cache: Cache): Promise { + const transports = options.gitTransports ?? ['https', 'ssh'] + const refs = await gitRefs(source.url, transports, options.limits) + const tags = [...refs.tags].flatMap(([tag, commit]) => releaseTag(tag, commit) ?? []) + const file = cache.file('git', options.inlineIconBytes, source.url, refs.head, tags.map(tag => [tag.tag, tag.commit])) + const cached = cache.plugins(file) + if (cached !== undefined) return cached + const warn = (message: string) => { options.warn(`${source.line}: ${message}`) } + const unitFile = (kind: string, commit: string, path: string) => cache.file(kind, options.inlineIconBytes, source.url, commit, path) + const plugins = await withGitObjects(source.url, refs, transports, options.limits, async (objects) => { + const packages = await headPackages(await objects.tree(refs.head, ''), warn) + // Lower ranks win when several commits carry one version: `@` tags, then unified tags, then the default branch. + const units = [ + ...tags.flatMap(tag => packages.filter(found => tag.name === undefined || tag.name === found.name) + .map(found => ({ found, commit: tag.commit, version: tag.version, rank: tag.name === undefined ? 1 : 0 }))), + ...packages.map(found => ({ found, commit: refs.head, version: undefined, rank: 2 })), + ].map(unit => ({ ...unit, file: unitFile('release', unit.commit, unit.found.path) })) + const cached = new Map(units.map(unit => [unit.file, cache.release(unit.file)])) + await objects.fetch(units.flatMap(unit => cached.get(unit.file) === undefined ? [unit.commit] : [])) + const releases = new Map() + const candidates: { release: Release; rank: number; path: string }[] = [] + for (const unit of units) { + const known = releases.has(unit.file) ? releases.get(unit.file) : cached.get(unit.file) + const release = known === undefined ? await readRelease(objects, unit.file, unit.commit, unit.found.path) : known + releases.set(unit.file, release) + if (release?.name === unit.found.name && (unit.version ?? release.version.version) === release.version.version) { + candidates.push({ release, rank: unit.rank, path: unit.found.path }) + } + } + const time = (candidate: { release: Release }) => candidate.release.version.publishTimestamp + candidates.sort((left, right) => left.rank - right.rank || time(left) - time(right)) + const chosen = new Map(packages.map(found => [found.name, new Map()])) + for (const { release, path } of candidates) { + const versions = chosen.get(release.name) as Map + if (!versions.has(release.version.version)) versions.set(release.version.version, { release, path }) + } + const entries: GeneratedEntry[] = [] + for (const [name, versions] of chosen) { + const highest = entryOf(name, [...versions.values()].map(choice => choice.release)).versions[0] + if (highest === undefined) continue + const { path } = versions.get(highest.version) as { path: string } + const display = await readGitDisplay(objects, highest.source.commit as string, path, `${name}@${highest.version}`) + const listed = [...versions.values()].map(({ release }) => release.version === highest ? { ...release, display } : release) + entries.push(entryOf(name, listed)) + } + return entries + }) + cache.write(file, { plugins }, []) + return plugins + + /** + * Read and cache the release at one commit and package directory; null when it lists none. A commit that cannot be + * read, such as a release-named tag on a blob, is reported and lists none, so it cannot fail the repository. + */ + async function readRelease(objects: GitObjects, releaseFile: string, commit: string, path: string): Promise { + let release: Release | null + try { + const files = await objects.tree(commit, path) + const manifest = await readManifest(files, '') + const gitSource = { type: 'git', url: source.url, commit, ...path === '' ? {} : { path } } + release = isBundle(manifest) && typeof manifest?.name === 'string' && typeof manifest.version === 'string' + ? releaseOf(manifest.name, manifest, manifest.version, await objects.committed(commit), gitSource, warn) ?? null : null + } catch (error) { + warn(`${commit}:${path} skipped: ${(error as Error).message}`) + release = null + } + cache.write(releaseFile, { release }, []) + return release + } + + /** Read and cache the display fields at one commit and package directory. */ + async function readGitDisplay(objects: GitObjects, commit: string, path: string, label: string): Promise { + const displayFile = unitFile('display', commit, path) + const known = cache.display(displayFile) + if (known !== undefined) return known + const files = await objects.tree(commit, path) + const icons: IconFile[] = [] + const display = await readDisplay(files, await readManifest(files, '') as Manifest, label, options, icons) + cache.write(displayFile, { display }, icons) + return display + } +} + +/** Names of the bundle packages on the default branch, by package directory. */ +async function headPackages(files: PackageFiles, warn: (message: string) => void): Promise<{ path: string; name: string }[]> { + const paths = await findBundlePackages(files, warn) + if (paths.length === 0) warn('no package declares dsh.bundle') + const packages: { path: string; name: string }[] = [] + for (const path of paths) { + const name = (await readManifest(files.within(path), ''))?.name + if (typeof name === 'string') packages.push({ path, name }) + else warn(`${posix.join(path, 'package.json')} has no name`) + } + return packages +} + +/** Run `work` over `items` with at most `limit` in flight, keeping results in input order. */ +async function mapLimited(items: readonly T[], limit: number, work: (item: T, index: number) => Promise): Promise { + const results: R[] = new Array(items.length) + let next = 0 + const worker = async (): Promise => { + while (next < items.length) { + const index = next++ + results[index] = await work(items[index] as T, index) + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)) + return results +} + +/** + * Generate an index. A source that cannot be read keeps its last successful result, so a transient outage does not + * drop plugins; it is still reported in `failures`. + * @param options - sources, cache, and limits. + * @returns the index document, the icon files it refers to, and the failed sources in list order. + */ +export async function generate(options: GenerateOptions): Promise { + const cache = new Cache(options.cacheDir) + const failed: (string | undefined)[] = new Array(options.sources.length) + const results = await mapLimited(options.sources, options.concurrency, async (source, position) => { + const lastFile = join(options.cacheDir, 'last', `${cacheKey(JSON.stringify([CACHE_VERSION, options.registry, options.inlineIconBytes, source.line]))}.json`) + try { + const plugins = source.kind === 'npm' ? await readNpm(source, options, cache) : await readGit(source, options, cache) + cache.write(lastFile, { plugins }, []) + return plugins + } catch (error) { + // Any failure of one source keeps that source's last result instead of failing the run. + const message = `${source.line}: ${(error as Error).message}` + const last = cache.plugins(lastFile) + failed[position] = message + options.warn(last === undefined ? message : `${message}; keeping the last successful result`) + return last ?? [] + } + }) + const names = new Set() + const plugins: GeneratedEntry[] = [] + const icons = new Map() + for (const entry of results.flat()) { + if (names.has(entry.name)) { + options.warn(`${entry.name}: listed by more than one source; keeping the first`) + continue + } + names.add(entry.name) + plugins.push(entry) + if (entry.icon?.startsWith('icons/') === true) icons.set(entry.icon, join(options.cacheDir, entry.icon)) + } + const index = { + format: 1 as const, + title: options.title, + ...options.description === undefined ? {} : { description: options.description }, + plugins, + } + parseMarketIndex(index, VALIDATION_URL) + return { index, icons, failures: failed.filter(message => message !== undefined) } +} diff --git a/apps/plugin-market-generator/src/metadata.ts b/apps/plugin-market-generator/src/metadata.ts new file mode 100644 index 0000000000..7b5554e02c --- /dev/null +++ b/apps/plugin-market-generator/src/metadata.ts @@ -0,0 +1,186 @@ +/** + * Read a package's display metadata from its files the way DSH reads installed plugin metadata: locale files and the + * icon are resolved through the package `exports` map, and nothing is evaluated. + */ +import { createHash } from 'node:crypto' +import { posix } from 'node:path' +import type { LocalizedText } from '@deepseek-ai/dsh-package-manifest' +import { exports as resolveExports, type Package } from 'resolve.exports' +import { PackageFiles } from './files.ts' +import type { Manifest } from './packages.ts' + +/** Name of an icon file the generator writes under `icons/`: a content hash and the image extension. */ +export const GENERATED_ICON = /^[0-9a-f]{16}\.(?:svg|png|jpg|jpeg|webp)$/u + +/** Display fields of one plugin; `icon` is a data URL or a path relative to the index file. */ +export interface Display { + readonly title?: LocalizedText + readonly description?: LocalizedText + readonly icon?: string + readonly author?: string + /** The manifest `homepage`, copied as written. */ + readonly homepage?: string + /** Optional discovery metadata; keywords are copied from the manifest as written. */ + readonly metadata?: { readonly keywords: readonly string[] } +} + +/** How icons are emitted and where diagnostics go. */ +export interface DisplayOptions { + /** Largest icon, in bytes, embedded as a data URL; larger icons are returned as files. */ + readonly inlineBytes: number + /** Receives diagnostics about invalid manifest fields, locale files, and icons that are not used. */ + readonly warn: (message: string) => void +} + +/** An icon too large to inline: the index refers to `path`, and the caller writes `bytes` there. */ +export interface IconFile { + readonly path: string + readonly bytes: Uint8Array +} + +type Fields = { title?: string; description?: string } + +const LANGUAGE_ID = /^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*$/u +const MAX_ICON_BYTES = 256 * 1024 +const ICON_TYPES = new Map([['.svg', 'image/svg+xml'], ['.png', 'image/png'], ['.jpg', 'image/jpeg'], ['.jpeg', 'image/jpeg'], ['.webp', 'image/webp']]) + +function text(value: unknown): string | undefined { + return typeof value === 'string' && value.trim() !== '' ? value : undefined +} + +/** + * Resolve a package subpath such as `./icon` to a file path: through `exports` with the import, node, and default + * conditions when the manifest has one, otherwise literally. A subpath that is not exported, or whose target leaves + * the package, has no path. + */ +function resource(manifest: Manifest, subpath: string): string | undefined { + if (manifest.exports === undefined) return PackageFiles.resolve('', subpath) + let targets: string[] + try { + // With an `exports` field, resolve.exports returns a non-empty target list or throws for an unexported subpath. + targets = resolveExports({ name: '', exports: manifest.exports as Package['exports'] }, subpath) as string[] + } catch (_notExported) { + return undefined + } + return PackageFiles.resolve('', targets[0] as string) +} + +/** Parse one locale file's `meta` fields, or explain why the file is not used. */ +function localeFields(bytes: Uint8Array): Fields | string { + let parsed: unknown + try { + parsed = JSON.parse(Buffer.from(bytes).toString('utf8')) + } catch (error) { + return String(error) + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) return 'must be a JSON object' + const meta: unknown = (parsed as { meta?: unknown }).meta + if (meta === undefined) return {} + if (typeof meta !== 'object' || meta === null || Array.isArray(meta)) return 'meta must be an object' + const fields: Fields = {} + for (const key of ['title', 'description'] as const) { + const value: unknown = (meta as Fields)[key] + if (value === undefined) continue + if (text(value) === undefined) return `meta.${key} must be a non-empty string` + fields[key] = value as string + } + return fields +} + +/** + * Locale `meta` fields by lowercase language id. The English file `./locale/en.json` locates the locale directory; + * every other `.json` file there is read through `./locale/` and must resolve into the same directory. + */ +async function dictionaries(files: PackageFiles, manifest: Manifest, warn: (message: string) => void): Promise> { + const result = new Map() + const english = resource(manifest, './locale/en.json') + if (english === undefined || files.file(english) === undefined) return result + const dir = posix.dirname(english) + for (const name of files.fileNames(dir === '.' ? '' : dir)) { + if (!name.endsWith('.json')) continue + const language = name.slice(0, -5) + const path = resource(manifest, `./locale/${name}`) + const bytes = path === undefined || posix.dirname(path) !== dir ? undefined : await files.read(path) + const fields = !LANGUAGE_ID.test(language) ? 'its name is not a language id' + : result.has(language.toLowerCase()) ? `duplicates locale ${language.toLowerCase()}` + : bytes === undefined ? `./locale/${name} does not resolve into ${dir}` : localeFields(bytes) + if (typeof fields === 'string') warn(`locale file ${posix.join(dir, name)} ignored: ${fields}`) + else result.set(language.toLowerCase(), fields) + } + return result +} + +/** + * Combine locale values with the manifest fallback: a plain string without translations, otherwise an `en`-keyed map. + * The format requires English text, so translations without an English value or manifest fallback are dropped. + */ +function localized(field: keyof Fields, locales: Map, fallback: string | undefined): LocalizedText | undefined { + const translated = [...locales].flatMap(([language, fields]) => fields[field] === undefined ? [] : [[language, fields[field]] as const]) + if (translated.length === 0) return fallback + const english = locales.get('en')?.[field] ?? fallback + if (english === undefined) return undefined + if (translated.length === 1 && translated[0]?.[0] === 'en') return english + return Object.fromEntries([['en', english], ...translated]) as LocalizedText +} + +/** The icon a package declares, or why it is not used: the manifest `icon` takes priority over the `./icon` export. */ +async function readIcon( + files: PackageFiles, manifest: Manifest, +): Promise<{ type: string; path: string; bytes: Buffer } | string | undefined> { + let path: string | undefined + if (manifest.icon === undefined) { + // As for Node module resolution, an `./icon` subpath without a file is absent. + path = resource(manifest, './icon') + if (path === undefined || files.file(path) === undefined) return undefined + } else { + const declared = text(manifest.icon) + if (declared === undefined || /^[A-Za-z][A-Za-z\d+.-]*:/u.test(declared)) return 'icon must be a relative file path' + path = PackageFiles.resolve('', declared) + if (path === undefined) return `icon ${declared} is outside the package` + } + const type = ICON_TYPES.get(posix.extname(path).toLowerCase()) + if (type === undefined) return `icon ${path} must be SVG, PNG, JPEG, or WebP` + const file = files.file(path) + if (file === undefined) return `icon ${path} is not a file in the package` + if (file.size > MAX_ICON_BYTES) return `icon ${path} exceeds 256 KiB` + return { type, path, bytes: Buffer.from(await file.read()) } +} + +/** + * Read display metadata, author, homepage, and icon of a package. `homepage` is copied from the manifest as written; a + * value that is not an absolute http(s) URL is reported and omitted, and nothing else stands in for it. + * Manifest `keywords` arrays containing only strings are copied unchanged into `metadata.keywords`; any other + * present value is reported and omitted without dropping the package. + * @param files - the package's files. + * @param manifest - its parsed `package.json`. + * @param options - icon inlining threshold and diagnostics. + * @returns the display fields, and the icon file to write when the icon is too large to inline. + */ +export async function readDisplayMetadata( + files: PackageFiles, manifest: Manifest, options: DisplayOptions, +): Promise<{ display: Display; iconFile?: IconFile }> { + const locales = await dictionaries(files, manifest, options.warn) + const title = localized('title', locales, text(manifest.name)) + const description = localized('description', locales, text(manifest.description)) + // npm's person string is `Name (url)`; only the name is published, so no address spreads into the index. + const person = typeof manifest.author === 'object' && manifest.author !== null ? (manifest.author as { name?: unknown }).name : manifest.author + const author = text(typeof person === 'string' ? person.replace(/<[^>]*>|\([^)]*\)/gu, '').trim() : person) + const homepage = text(manifest.homepage) + const linkable = homepage !== undefined && URL.canParse(homepage) && ['https:', 'http:'].includes(new URL(homepage).protocol) + if (homepage !== undefined && !linkable) options.warn(`package.json homepage ${JSON.stringify(homepage)} is not an http(s) URL; it is omitted`) + const keywords = Array.isArray(manifest.keywords) && manifest.keywords.every((keyword): keyword is string => typeof keyword === 'string') + ? manifest.keywords : undefined + if (manifest.keywords !== undefined && keywords === undefined) options.warn('package.json keywords must be an array of strings; it is omitted') + const display = { + ...title === undefined ? {} : { title }, ...description === undefined ? {} : { description }, ...author === undefined ? {} : { author }, + ...linkable ? { homepage } : {}, + ...keywords === undefined ? {} : { metadata: { keywords } }, + } + const icon = await readIcon(files, manifest) + if (typeof icon === 'string') options.warn(`${icon}; the icon is omitted`) + if (typeof icon !== 'object') return { display } + const { type, path, bytes } = icon + if (bytes.length <= options.inlineBytes) return { display: { ...display, icon: `data:${type};base64,${bytes.toString('base64')}` } } + const file = `icons/${createHash('sha256').update(bytes).digest('hex').slice(0, 16)}${posix.extname(path).toLowerCase()}` + return { display: { ...display, icon: file }, iconFile: { path: file, bytes } } +} diff --git a/apps/plugin-market-generator/src/packages.ts b/apps/plugin-market-generator/src/packages.ts new file mode 100644 index 0000000000..b372120dcd --- /dev/null +++ b/apps/plugin-market-generator/src/packages.ts @@ -0,0 +1,196 @@ +/** Read package manifests and find the bundle packages of a repository tree without running any of their code. */ +import { posix } from 'node:path' +import semver from 'semver' +import { parse as parseYaml } from 'yaml' +import type { PackageFiles } from './files.ts' + +/** A package manifest as read from a file; only the fields the generator reads are typed. */ +export interface Manifest { + readonly name?: unknown + readonly version?: unknown + readonly description?: unknown + readonly icon?: unknown + readonly author?: unknown + readonly homepage?: unknown + readonly keywords?: unknown + readonly deprecated?: unknown + readonly workspaces?: unknown + readonly exports?: unknown + readonly engines?: unknown + readonly peerDependencies?: unknown + readonly dsh?: { readonly bundle?: unknown } +} + +/** Depth limit of the fallback scan, counted in directories below the repository root. */ +const SCAN_DEPTH = 3 +const SKIPPED_DIRECTORIES = new Set(['node_modules', 'test', 'tests', 'fixtures', 'examples', 'example']) +const DSH_PEER = /^@deepseek-ai\/dsh(?:-|$)/u +/** Ranges that installation reads as the running DSH version. */ +const CURRENT_DSH = new Set(['workspace:*', 'workspace:^', 'workspace:~']) + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +/** + * Read a `package.json`. + * @param files - the tree to read. + * @param dir - package directory in the tree. + * @returns the parsed object, or undefined when the file is absent or is not a JSON object. + */ +export async function readManifest(files: PackageFiles, dir: string): Promise { + const bytes = await files.read(posix.join(dir, 'package.json')) + if (bytes === undefined) return undefined + let value: unknown + try { + value = JSON.parse(Buffer.from(bytes).toString('utf8')) + } catch (_malformed) { + // A malformed manifest declares no package. + return undefined + } + return isObject(value) ? value : undefined +} + +/** + * Whether a manifest declares a DSH bundle. + * @param manifest - parsed manifest, or undefined for a directory without one. + * @returns true when `dsh.bundle` is an object whose `patch` is a string or an array of strings. + */ +export function isBundle(manifest: Manifest | undefined): boolean { + const bundle = manifest?.dsh?.bundle + if (!isObject(bundle)) return false + return typeof bundle.patch === 'string' || (Array.isArray(bundle.patch) && bundle.patch.every(file => typeof file === 'string')) +} + +/** Whether installation reads `range` as a SemVer range; an empty range and other `workspace:` ranges never match. */ +const valid = (range: string): boolean => range !== '' && !range.startsWith('workspace:') && semver.validRange(range) !== null +const invalidRange = (range: string): Error => new Error(`DSH range ${JSON.stringify(range)} is not a valid SemVer range`) + +/** Most `||` alternatives the conjunction of DSH peer ranges may produce; a version needing more is not listed. */ +const MAX_RANGE_ALTERNATIVES = 256 + +/** + * The `engines.dsh` range an index lists for a manifest: its own `engines.dsh` string, otherwise the conjunction of + * its `@deepseek-ai/dsh` and `@deepseek-ai/dsh-*` peer ranges. A peer of exactly `workspace:*`, `workspace:^`, or + * `workspace:~` means the running DSH version, and a `*` range admits every version; both impose nothing. A single + * remaining range is listed as written. Several ranges are combined: the conjunction combines one `||` alternative of + * each range, keeps each comparator once, and drops combinations that no version satisfies, prereleases included. + * @param manifest - the version's manifest. + * @returns the range, undefined for any DSH version, or an error when installation finds the version incompatible + * with every DSH version: an empty, `workspace:`, or invalid range, or ranges that no version satisfies together. + */ +export function dshRange(manifest: Manifest): string | undefined | Error { + const declared = isObject(manifest.engines) ? manifest.engines.dsh : undefined + const ranges = typeof declared === 'string' ? [declared] + : Object.entries(isObject(manifest.peerDependencies) ? manifest.peerDependencies : {}) + .flatMap(([name, range]) => DSH_PEER.test(name) && typeof range === 'string' && !CURRENT_DSH.has(range) ? [range] : []) + const used = [...new Set(ranges.map(range => range.trim()))].filter(range => range !== '*') + if (used.length === 0) return undefined + let combinations: string[][] = [[]] + for (const range of used) { + if (!valid(range)) return invalidRange(range) + const alternatives = new semver.Range(range, { includePrerelease: true }).set + .map(set => set.map(comparator => comparator.value).filter(Boolean)) + if (combinations.length * alternatives.length > MAX_RANGE_ALTERNATIVES) { + return new Error(`DSH peer ranges combine into more than ${MAX_RANGE_ALTERNATIVES} alternatives`) + } + combinations = combinations.flatMap(prefix => alternatives.map(alternative => [...new Set([...prefix, ...alternative])])) + .filter(combination => semver.minVersion(new semver.Range(combination.join(' ') || '*', { includePrerelease: true })) !== null) + } + if (combinations.length === 0) return new Error(`no DSH version satisfies ${used.length === 1 ? 'the DSH range' : 'every DSH peer range'}: ${used.join(', ')}`) + const written = [...new Set(combinations.map(combination => combination.join(' ') || '*'))] + if (written.includes('*')) return undefined + return used.length === 1 ? used[0] : written.join(' || ') +} + +/** String members of a value that is an array, or none. */ +function strings(value: unknown): string[] { + return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string') : [] +} + +/** Workspace globs from `package.json` `workspaces` or `pnpm-workspace.yaml` `packages`; negated globs are dropped. */ +async function workspaceGlobs(files: PackageFiles, manifest: Manifest | undefined): Promise { + const declared = manifest?.workspaces + const fromManifest = Array.isArray(declared) ? strings(declared) : isObject(declared) ? strings(declared.packages) : [] + const pnpmFile = await files.read('pnpm-workspace.yaml') + let parsed: unknown + try { + parsed = pnpmFile === undefined ? undefined : parseYaml(Buffer.from(pnpmFile).toString('utf8')) + } catch (_malformed) { + // A malformed workspace file declares no packages. + parsed = undefined + } + return [...fromManifest, ...isObject(parsed) ? strings(parsed.packages) : []].filter(glob => !glob.startsWith('!')) +} + +function directories(files: PackageFiles, dir: string): string[] { + return files.directories(dir).filter(name => !name.startsWith('.') && name !== 'node_modules') +} + +/** Match one path segment against a glob segment in which `*` stands for any characters. */ +function segmentMatches(pattern: string, name: string): boolean { + return new RegExp(`^${pattern.split('*').map(part => part.replace(/[.+?^${}()|[\]\\]/gu, '\\$&')).join('.*')}$`, 'u').test(name) +} + +/** + * Expand one workspace glob into existing directories. Each `/`-separated segment is a literal name, a name with `*` + * wildcards, or `**` for zero or more directories up to the scan depth; parent-relative globs match nothing. + */ +function expandGlob(files: PackageFiles, glob: string): string[] { + const segments = posix.normalize(glob.replace(/^\.\//u, '').replace(/\/+$/u, '')).split('/').filter(segment => segment !== '' && segment !== '.') + if (segments.includes('..')) return [] + const found = new Set() + const walk = (relative: string, rest: string[], depth: number): void => { + const [head, ...tail] = rest + if (head === undefined) { + found.add(relative) + return + } + if (head === '**') { + walk(relative, tail, depth) + if (depth < SCAN_DEPTH) for (const name of directories(files, relative)) walk(posix.join(relative, name), rest, depth + 1) + return + } + for (const name of directories(files, relative)) { + if (segmentMatches(head, name)) walk(posix.join(relative, name), tail, depth + 1) + } + } + walk('', segments, 0) + return [...found] +} + +/** Bundle package directories up to the scan depth, skipping test and example trees. */ +async function scan(files: PackageFiles): Promise { + const found: string[] = [] + const walk = async (relative: string, depth: number): Promise => { + for (const name of directories(files, relative)) { + if (SKIPPED_DIRECTORIES.has(name)) continue + const child = posix.join(relative, name) + if (isBundle(await readManifest(files, child))) found.push(child) + else if (depth < SCAN_DEPTH) await walk(child, depth + 1) + } + } + await walk('', 1) + return found +} + +/** + * Locate bundle packages in a repository tree, in this order of precedence: + * 1. the root manifest declares a bundle: the root only; + * 2. workspace globs match bundle packages: those packages; + * 3. otherwise: every bundle package found within three directory levels, outside test and example trees. + * @param files - repository tree. + * @param warn - receives workspace globs that match no directory. + * @returns repository-relative POSIX package directories, `''` for the root, sorted. + */ +export async function findBundlePackages(files: PackageFiles, warn: (message: string) => void = () => {}): Promise { + const manifest = await readManifest(files, '') + if (isBundle(manifest)) return [''] + const declared = new Set() + for (const glob of await workspaceGlobs(files, manifest)) { + const matches = expandGlob(files, glob) + if (matches.length === 0) warn(`workspace glob ${JSON.stringify(glob)} matches no directory`) + for (const dir of matches) if (isBundle(await readManifest(files, dir))) declared.add(dir) + } + return (declared.size > 0 ? [...declared] : await scan(files)).sort() +} diff --git a/apps/plugin-market-generator/src/sources.ts b/apps/plugin-market-generator/src/sources.ts new file mode 100644 index 0000000000..51793e5504 --- /dev/null +++ b/apps/plugin-market-generator/src/sources.ts @@ -0,0 +1,48 @@ +/** Read the source list: one npm package name or Git repository URL per line. */ +import { parseMarketIndex, type MarketIndexError } from '@deepseek-ai/dsh-plugin-market-index' + +/** One line of the source list. */ +export type Source = + | { readonly kind: 'npm'; readonly line: string; readonly name: string } + | { readonly kind: 'git'; readonly line: string; readonly url: string } + +/** Throw the reason the index format rejects a plugin with this name and source. */ +function check(name: string, source: object): void { + const version = { version: '0.0.0', publishTimestamp: 0, source } + parseMarketIndex({ format: 1, title: 'check', plugins: [{ name, versions: [version] }] }, 'https://index.invalid/') +} + +/** + * Parse a source list. Blank lines and lines starting with `#` are ignored; a trailing `/` or `.git` on a repository + * URL is dropped, so `https://github.com/o/r/` and `https://github.com/o/r.git` name the same repository. Every line + * must be a package name or a Git URL the index format accepts. + * @param text - the list file contents. + * @returns the sources in file order, without repeats. + * @throws when a line is neither an npm package name nor an https repository URL. + */ +export function parseSources(text: string): Source[] { + const sources: Source[] = [] + const seen = new Set() + for (const [index, raw] of text.split(/\r?\n/u).entries()) { + const line = raw.trim() + if (line === '' || line.startsWith('#')) continue + let source: Source + try { + if (line.includes('://')) { + const url = URL.canParse(line) ? new URL(line).href.replace(/\/+$/u, '').replace(/\.git$/u, '') : line + check('placeholder', { type: 'git', url, commit: '0'.repeat(40) }) + source = { kind: 'git', line, url } + } else { + check(line, { type: 'npm' }) + source = { kind: 'npm', line, name: line } + } + } catch (error) { + throw new Error(`line ${index + 1}: not an npm package name or an https Git repository URL (${(error as MarketIndexError).reason}): ${line}`) + } + const key = source.kind === 'npm' ? `npm:${source.name}` : `git:${source.url}` + if (seen.has(key)) continue + seen.add(key) + sources.push(source) + } + return sources +} diff --git a/apps/plugin-market-generator/tests/cli.spec.ts b/apps/plugin-market-generator/tests/cli.spec.ts new file mode 100644 index 0000000000..ab3ba52706 --- /dev/null +++ b/apps/plugin-market-generator/tests/cli.spec.ts @@ -0,0 +1,15 @@ +/** Command-line validation of the generator's options. */ +import { afterEach, expect, it, vi } from 'vitest' +import { main } from '../src/cli.ts' + +afterEach(() => { vi.restoreAllMocks() }) + +it.each([ + ['32769', '--inline-icon-bytes must be a positive integer up to 32768'], + ['0', '--inline-icon-bytes must be a positive integer up to 32768'], +])('refuses --inline-icon-bytes %s before reading the source list', async (value, message) => { + const error = vi.spyOn(console, 'error').mockImplementation(() => {}) + const status = await main(['--input', '/nonexistent/list.txt', '--output', '/nonexistent/index.json', '--title', 'T', '--inline-icon-bytes', value]) + expect(status).toBe(2) + expect(error).toHaveBeenCalledWith(message) +}) diff --git a/apps/plugin-market-generator/tests/discovery.spec.ts b/apps/plugin-market-generator/tests/discovery.spec.ts new file mode 100644 index 0000000000..ce4f98fec8 --- /dev/null +++ b/apps/plugin-market-generator/tests/discovery.spec.ts @@ -0,0 +1,270 @@ +import { describe, expect, it } from 'vitest' +import { PackageFiles } from '../src/files.ts' +import { readDisplayMetadata } from '../src/metadata.ts' +import { dshRange, findBundlePackages, isBundle, readManifest, type Manifest } from '../src/packages.ts' +import { parseSources } from '../src/sources.ts' + +/** An in-memory tree from relative path → file contents. */ +function tree(contents: Record): PackageFiles { + return new PackageFiles(new Map(Object.entries(contents).map(([path, value]) => { + const bytes = Buffer.from(value) + return [path, { size: bytes.length, read: () => Promise.resolve(bytes) }] + }))) +} + +const bundle = (name: string, extra: object = {}): string => JSON.stringify({ name, version: '1.0.0', dsh: { bundle: { patch: './cordis.patch.yml' } }, ...extra }) + +async function readDisplay(files: PackageFiles, manifest: Manifest, inlineBytes = 100) { + const warnings: string[] = [] + const result = await readDisplayMetadata(files, manifest, { inlineBytes, warn: (message) => { warnings.push(message) } }) + return { ...result, warnings } +} + +describe('parseSources', () => { + it('reads package names and repository URLs, ignoring comments, blanks, and repeats', () => { + expect(parseSources('# market\n\n@scope/dsh-a\ndsh-b\nhttps://github.com/o/r.git\nhttps://github.com/o/r/\ndsh-b\nhttps://gitlab.example.com/g/sub/r\n')).toEqual([ + { kind: 'npm', line: '@scope/dsh-a', name: '@scope/dsh-a' }, + { kind: 'npm', line: 'dsh-b', name: 'dsh-b' }, + { kind: 'git', line: 'https://github.com/o/r.git', url: 'https://github.com/o/r' }, + { kind: 'git', line: 'https://gitlab.example.com/g/sub/r', url: 'https://gitlab.example.com/g/sub/r' }, + ]) + }) + + it.each([ + ['Not A Name', 'line 1: not an npm package name or an https Git repository URL (must be a valid npm package name): Not A Name'], + ['x'.repeat(215), 'must be a valid npm package name'], + ['http://github.com/o/r', 'must use https:'], + ['ssh://git@github.com/o/r', 'must use https:'], + ['\nhttps://u:p@github.com/o/r', 'line 2: not an npm package name or an https Git repository URL (must not contain credentials)'], + ['https://github.com/o/r#main', 'must not contain a query or fragment'], + ['https://github.com/o/r?ref=main', 'must not contain a query or fragment'], + ['https://github.com/r', 'must name a repository with at least two path segments'], + ['https://example.com/o/r/archive.tar.gz', 'must name a repository, not a tarball'], + ['https://', 'must be an absolute URL'], + ])('refuses %j', (line, message) => { + expect(() => parseSources(line)).toThrow(message) + }) +}) + +describe('isBundle and dshRange', () => { + it('accepts only a bundle whose patch is a string or a list of strings', () => { + expect([ + { patch: 'a.yml' }, { patch: ['a.yml', 'b.yml'] }, { patch: [] }, { patch: [1] }, {}, [], null, 'a.yml', + ].map(value => isBundle({ dsh: { bundle: value } }))).toEqual([true, true, true, false, false, false, false, false]) + expect(isBundle(undefined)).toBe(false) + }) + + it('copies engines.dsh, else joins DSH peer ranges, and refuses ranges installation rejects', () => { + expect(dshRange({ engines: { dsh: '>=0.2.0' }, peerDependencies: { '@deepseek-ai/dsh': '1' } })).toBe('>=0.2.0') + expect(dshRange({ peerDependencies: { react: '18', '@deepseek-ai/dsh-agent': 'workspace:*', '@deepseek-ai/dsh': '*' } })).toBeUndefined() + // Installation refuses an empty range, any `workspace:` engines.dsh, and a padded workspace peer on every DSH version. + for (const manifest of [ + { peerDependencies: { '@deepseek-ai/dsh': ' ' } }, { engines: { dsh: '' } }, { engines: { dsh: 'workspace:*' } }, + { peerDependencies: { '@deepseek-ai/dsh': ' workspace:*' } }, + ]) expect(dshRange(manifest)).toBeInstanceOf(Error) + expect(dshRange({ engines: { dsh: '>1 <1' } })).toEqual(new Error('no DSH version satisfies the DSH range: >1 <1')) + expect(dshRange({ engines: { dsh: ' * ' } })).toBeUndefined() + expect(dshRange({ engines: { node: '22' }, peerDependencies: [] })).toBeUndefined() + expect(dshRange({ + peerDependencies: { '@deepseek-ai/dsh-a': '^0.2.0 || ^0.3.0', '@deepseek-ai/dsh-b': '>=0.2.1', '@deepseek-ai/dsh-c': '>=0.2.1', '@deepseek-ai/dsh-d': '<1 ||' }, + })).toBe('>=0.2.0 <0.3.0-0 >=0.2.1 || >=0.3.0 <0.4.0-0 >=0.2.1') + // A hyphen range is expanded before combining, and combinations no version satisfies are dropped. + expect(dshRange({ peerDependencies: { '@deepseek-ai/dsh': '1.0.0 - 2.0.0', '@deepseek-ai/dsh-x': '>=1.5.0 || <0.5.0' } })).toBe('>=1.0.0-0 <2.0.1-0 >=1.5.0') + expect(dshRange({ peerDependencies: { '@deepseek-ai/dsh': '^0.1.0', '@deepseek-ai/dsh-x': '^0.2.0' } })) + .toEqual(new Error('no DSH version satisfies every DSH peer range: ^0.1.0, ^0.2.0')) + expect(dshRange({ peerDependencies: { '@deepseek-ai/dsh': '^0.2.0-rc.1' } })).toBe('^0.2.0-rc.1') + expect(dshRange({ peerDependencies: { '@deepseek-ai/dsh': '>=0.1.0 || *', '@deepseek-ai/dsh-x': '^0.2.0 || *' } })).toBeUndefined() + expect(dshRange({ peerDependencies: { '@deepseek-ai/dsh': '<0.1.0', '@deepseek-ai/dsh-x': '>=0.2.0' } })) + .toEqual(new Error('no DSH version satisfies every DSH peer range: <0.1.0, >=0.2.0')) + expect(dshRange({ peerDependencies: { '@deepseek-ai/dsh': '*', '@deepseek-ai/dsh-x': '>=0' } })).toBeUndefined() + expect(dshRange({ peerDependencies: { '@deepseek-ai/dsh-llm': '*', '@deepseek-ai/dsh-x': '*' } })).toBeUndefined() + const wide = Object.fromEntries(Array.from({ length: 3 }, (_, peer) => [`@deepseek-ai/dsh-${peer}`, Array.from({ length: 10 }, (_, minor) => `>=0.${minor}.${peer}`).join(' || ')])) + expect(dshRange({ peerDependencies: wide })).toEqual(new Error('DSH peer ranges combine into more than 256 alternatives')) + expect(dshRange({ peerDependencies: { '@deepseek-ai/dsh-a': 'workspace:>=1' } })).toEqual(new Error('DSH range "workspace:>=1" is not a valid SemVer range')) + expect(dshRange({ engines: { dsh: 'not a range' } })).toBeInstanceOf(Error) + expect(dshRange({ peerDependencies: { '@deepseek-ai/dsh': '>=0.2.0', '@deepseek-ai/dsh-x': 'not a range' } })).toEqual(new Error('DSH range "not a range" is not a valid SemVer range')) + }) +}) + +describe('findBundlePackages', () => { + it('takes the root package when it is a bundle, even with workspaces', async () => { + expect(await findBundlePackages(tree({ 'package.json': bundle('root', { workspaces: ['packages/*'] }), 'packages/a/package.json': bundle('a') }))).toEqual(['']) + }) + + it('expands package.json and pnpm workspace globs', async () => { + const files = tree({ + 'package.json': JSON.stringify({ name: 'mono', private: true, workspaces: { packages: ['plugins/*'] } }), + 'pnpm-workspace.yaml': 'packages:\n - tools/**\n - single\n - "!plugins/excluded"\n - "packages/*/*"\n - "dsh-*"\n - missing\n', + 'plugins/a/package.json': bundle('a'), + 'plugins/lib/package.json': JSON.stringify({ name: 'lib', version: '1.0.0' }), + 'tools/group/b/package.json': bundle('b'), + 'single/package.json': bundle('single'), + 'packages/core/c/package.json': bundle('c'), + 'dsh-x/package.json': bundle('x'), + 'other/package.json': bundle('other'), + }) + const warnings: string[] = [] + expect(await findBundlePackages(files, (message) => { warnings.push(message) })).toEqual(['dsh-x', 'packages/core/c', 'plugins/a', 'single', 'tools/group/b']) + expect(warnings).toEqual(['workspace glob "missing" matches no directory']) + }) + + it('scans three levels without a root manifest, skipping test, example, hidden, and node_modules trees', async () => { + const files = tree({ + 'dsh-pet/package.json': bundle('pet'), + 'dsh-pet/runtime/helper/package.json': bundle('nested-under-bundle'), + 'plugins/dsh/package.json': bundle('dsh'), + 'a/b/c/package.json': bundle('deep'), + 'a/b/c/d/package.json': bundle('under-bundle'), + 'x/y/z/w/package.json': bundle('too-deep'), + 'examples/template/package.json': bundle('example'), + 'node_modules/x/package.json': bundle('dependency'), + '.hidden/package.json': bundle('hidden'), + 'broken/package.json': '{', + 'array/package.json': '[]', + }) + expect(await findBundlePackages(files)).toEqual(['a/b/c', 'dsh-pet', 'plugins/dsh']) + expect(await readManifest(files, 'broken')).toBeUndefined() + expect(await readManifest(files, 'array')).toBeUndefined() + }) + + it('ignores a malformed workspace file and parent-relative globs', async () => { + const files = tree({ 'pnpm-workspace.yaml': 'packages: [', 'package.json': JSON.stringify({ workspaces: ['../x', 'missing/*'] }), 'p/package.json': bundle('p') }) + expect(await findBundlePackages(files)).toEqual(['p']) + expect(await findBundlePackages(tree({ 'pnpm-workspace.yaml': 'packages: p', 'p/package.json': bundle('p') }))).toEqual(['p']) + }) +}) + +describe('PackageFiles', () => { + it('resolves references inside the tree only', () => { + expect([['', './a/b'], ['a', '../c'], ['a', '../..'], ['', '/etc/passwd'], ['a', '.'], ['', 'x/']].map(([base, path]) => PackageFiles.resolve(base as string, path as string))) + .toEqual(['a/b', 'c', undefined, undefined, 'a', 'x']) + }) + + it('lists directories and files and reroots subtrees', async () => { + const files = tree({ 'a/x.json': '1', 'a/b/y.json': '2', 'z': '3' }) + expect([files.directories(''), files.fileNames(''), files.fileNames('a'), files.directories('missing'), files.fileNames('missing')]) + .toEqual([['a'], ['z'], ['x.json'], [], []]) + expect(files.within('')).toBe(files) + expect(Buffer.from(await files.within('a').read('b/y.json') ?? []).toString()).toBe('2') + expect(await files.read('missing')).toBeUndefined() + }) +}) + +describe('readDisplayMetadata', () => { + it.each([ + { keywords: ['search', '', ' search ', 'search', '搜索'] }, + { keywords: [] }, + ])('preserves manifest keywords $keywords', async ({ keywords }) => { + const result = await readDisplay(tree({}), { name: 'dsh-a', keywords }) + expect(result).toEqual({ display: { title: 'dsh-a', metadata: { keywords } }, warnings: [] }) + }) + + it('omits metadata without manifest keywords', async () => { + expect(await readDisplay(tree({}), { name: 'dsh-a' })).toEqual({ display: { title: 'dsh-a' }, warnings: [] }) + }) + + it.each([['search', 1], [null], 'search', null, 1, false, {}].map(keywords => ({ keywords })))('warns and omits invalid keywords $keywords entirely', async ({ keywords }) => { + const result = await readDisplay(tree({}), { name: 'dsh-a', keywords }) + expect(result).toEqual({ + display: { title: 'dsh-a' }, + warnings: ['package.json keywords must be an array of strings; it is omitted'], + }) + }) + + it('copies an http(s) homepage as written and reports any other value without a substitute', async () => { + const homepage = 'https://github.com/jane/dsh-a#readme' + expect((await readDisplay(tree({}), { name: 'dsh-a', homepage })).display).toEqual({ title: 'dsh-a', homepage }) + // `repository` is present to show that it does not replace a rejected homepage. + const manifest = { name: 'dsh-a', homepage: 'javascript:alert(1)', repository: 'https://github.com/jane/dsh-a' } + const rejected = await readDisplay(tree({}), manifest) + expect(rejected.display).toEqual({ title: 'dsh-a' }) + expect(rejected.warnings).toEqual(['package.json homepage "javascript:alert(1)" is not an http(s) URL; it is omitted']) + expect((await readDisplay(tree({}), { name: 'dsh-a', homepage: 'docs/readme' })).display).toEqual({ title: 'dsh-a' }) + }) + + it('combines locale meta with manifest fallbacks, reports unusable locale files, and reads the author', async () => { + const files = tree({ + 'locale/en.json': JSON.stringify({ meta: { title: 'Alpha' } }), + 'locale/fr.json': JSON.stringify({ meta: {} }), + 'locale/zh.json': JSON.stringify({ meta: { title: '甲', description: '做甲' } }), + 'locale/FR.json': JSON.stringify({ meta: { title: 'Alpha (fr)' } }), + 'locale/Bad_Name.json': '{}', + 'locale/de.json': '{', + 'locale/es.json': '[]', + 'locale/it.json': JSON.stringify({ meta: 'x' }), + 'locale/ja.json': JSON.stringify({ meta: { title: 1 } }), + 'locale/ko.json': JSON.stringify({}), + 'locale/readme.md': 'not a locale', + }) + const { display, warnings } = await readDisplay(files, { name: 'dsh-a', description: 'Does A', author: { name: 'Jane', email: 'j@example.com' } }) + expect(display).toEqual({ title: { en: 'Alpha', fr: 'Alpha (fr)', zh: '甲' }, description: { en: 'Does A', zh: '做甲' }, author: 'Jane' }) + expect(warnings).toEqual([ + 'locale file locale/Bad_Name.json ignored: its name is not a language id', + expect.stringMatching(/^locale file locale\/de\.json ignored: SyntaxError/u) as string, + 'locale file locale/es.json ignored: must be a JSON object', + 'locale file locale/fr.json ignored: duplicates locale fr', + 'locale file locale/it.json ignored: meta must be an object', + 'locale file locale/ja.json ignored: meta.title must be a non-empty string', + ]) + }) + + it('reads no locale directory without locale/en.json and returns plain strings without translations', async () => { + expect((await readDisplay(tree({ 'locale/zh.json': JSON.stringify({ meta: { title: '甲' } }) }), { name: 'dsh-a', author: 'Jane ' })).display) + .toEqual({ title: 'dsh-a', author: 'Jane' }) + expect((await readDisplay(tree({ 'locale/en.json': JSON.stringify({ meta: { description: 'English only' } }) }), { name: 'dsh-a', author: { url: 'x' } })).display) + .toEqual({ title: 'dsh-a', description: 'English only' }) + expect((await readDisplay(tree({ 'locale/en.json': '{}', 'locale/zh.json': JSON.stringify({ meta: { description: '只有中文' } }) }), { name: 'dsh-a' })).display) + .toEqual({ title: 'dsh-a' }) + }) + + it('resolves locale files and the icon through the exports map', async () => { + const files = tree({ + 'dist/locale/en.json': JSON.stringify({ meta: { title: 'Alpha' } }), + 'dist/locale/zh.json': JSON.stringify({ meta: { title: '甲' } }), + 'dist/locale/fr.json': JSON.stringify({ meta: { title: 'Alpha (fr)' } }), + 'other/fr.json': JSON.stringify({ meta: { title: 'elsewhere' } }), + 'locale/en.json': JSON.stringify({ meta: { title: 'not exported' } }), + 'art/icon.svg': '', + 'icon.svg': 'unexported', + }) + const { display, warnings } = await readDisplay(files, { + name: 'dsh-a', + exports: { './locale/fr.json': './other/fr.json', './locale/*': './dist/locale/*', './icon': { types: './icon.d.ts', default: './art/icon.svg' } }, + }) + expect(display).toEqual({ title: { en: 'Alpha', zh: '甲' }, icon: 'data:image/svg+xml;base64,PHN2Zy8+' }) + expect(warnings).toEqual(['locale file dist/locale/fr.json ignored: ./locale/fr.json does not resolve into dist/locale']) + expect((await readDisplay(files, { name: 'dsh-a', exports: { '.': './index.js' } })).display).toEqual({ title: 'dsh-a' }) + const flat = tree({ 'en.json': JSON.stringify({ meta: { title: 'Alpha' } }), 'zh.json': JSON.stringify({ meta: { title: '甲' } }), 'package.json': '{}' }) + expect((await readDisplay(flat, { exports: { './locale/*': './*' } })).display).toEqual({ title: { en: 'Alpha', zh: '甲' } }) + expect((await readDisplay(flat, {})).display).toEqual({}) + expect((await readDisplay(files, { name: 'dsh-a', exports: { './locale/en.json': './../outside.json' } })).display).toEqual({ title: 'dsh-a' }) + }) + + it('inlines small icons, returns larger ones as files, and prefers the manifest icon', async () => { + const files = tree({ 'icon.svg': '', 'big.png': new Uint8Array(200), 'assets/i.webp': new Uint8Array(4) }) + expect((await readDisplay(files, { name: 'a', icon: './icon.svg', exports: { './icon': './assets/i.webp' } })).display.icon).toBe('data:image/svg+xml;base64,PHN2Zy8+') + const large = await readDisplay(files, { name: 'a', icon: 'big.png' }) + expect(large.display.icon).toMatch(/^icons\/[0-9a-f]{16}\.png$/u) + expect(large.iconFile?.bytes.length).toBe(200) + expect((await readDisplay(files, { name: 'a', exports: { './icon': { import: { default: './assets/i.webp' } } } })).display.icon).toBe('data:image/webp;base64,AAAAAA==') + expect((await readDisplay(files, { name: 'a', exports: { './icon': [] } })).warnings).toEqual([]) + }) + + it('reports icons that are not used', async () => { + const files = tree({ 'icon.gif': 'GIF89a', 'huge.png': new Uint8Array(256 * 1024 + 1), 'folder.png/x': '' }) + const cases: [unknown, string][] = [ + ['/etc/secret.svg', 'icon /etc/secret.svg is outside the package'], + ['../secret.svg', 'icon ../secret.svg is outside the package'], + ['https://cdn.example/i.png', 'icon must be a relative file path'], + [3, 'icon must be a relative file path'], + ['missing.svg', 'icon missing.svg is not a file in the package'], + ['folder.png', 'icon folder.png is not a file in the package'], + ['icon.gif', 'icon icon.gif must be SVG, PNG, JPEG, or WebP'], + ['huge.png', 'icon huge.png exceeds 256 KiB'], + ] + for (const [icon, message] of cases) { + const result = await readDisplay(files, { name: 'a', icon }) + expect([result.display.icon, result.warnings]).toEqual([undefined, [`${message}; the icon is omitted`]]) + } + }) +}) diff --git a/apps/plugin-market-generator/tests/generate.spec.ts b/apps/plugin-market-generator/tests/generate.spec.ts new file mode 100644 index 0000000000..9ffe581227 --- /dev/null +++ b/apps/plugin-market-generator/tests/generate.spec.ts @@ -0,0 +1,261 @@ +import { createHash } from 'node:crypto' +import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs' +import { createServer } from 'node:http' +import { join } from 'node:path' +import { c as createTar } from 'tar' +import { expect, it } from 'vitest' +import { main } from '../src/cli.ts' +import { generate, type GenerateOptions } from '../src/generate.ts' +import { onCleanup, scratch } from './support.ts' + +/** Pack a package directory the way npm does: every file under `package/`. */ +async function pack(files: Record): Promise { + const dir = scratch() + for (const [path, contents] of Object.entries(files)) { + mkdirSync(join(dir, 'package', path, '..'), { recursive: true }) + writeFileSync(join(dir, 'package', path), contents) + } + await createTar({ gzip: true, cwd: dir, file: join(dir, 'out.tgz'), portable: true }, ['package']) + return readFileSync(join(dir, 'out.tgz')) +} + +/** One published version: manifest fields besides name and version, and its tarball. */ +interface Published { + readonly manifest?: object + readonly time?: string + /** Tarball bytes, or `oversized` for a 2 MiB body sent without a content length. */ + readonly tarball?: Buffer | 'oversized' + readonly integrity?: string +} + +/** Packages by name: versions in publication order, or an HTTP status for the registry document. */ +type Packages = Record> | number> + +/** Serve registry documents under `/` and `/mirror/`, and tarballs; `tarballs` lists downloaded tarball paths. */ +async function registry(packages: Packages) { + const tarballs: string[] = [] + const server = createServer((request, response) => { + const url = decodeURIComponent(request.url ?? '') + const tarball = /^\/tarballs\/(.+)\/([^/]+)\.tgz$/u.exec(url) + if (tarball !== null) { + tarballs.push(url) + const published = (packages[tarball[1] as string] as Record)[tarball[2] as string]?.tarball + if (published === undefined) return void response.writeHead(404).end() + if (published !== 'oversized') return void response.writeHead(200).end(published) + response.writeHead(200) + for (let sent = 0; sent < 2 << 20; sent += 1 << 16) response.write(Buffer.alloc(1 << 16)) + return void response.end() + } + const name = url.replace(/^\/(?:mirror\/)?/u, '') + const versions = url.startsWith('/mirror/') ? 503 : packages[name] ?? 404 + if (typeof versions === 'number') return void response.writeHead(versions).end() + response.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify({ + 'dist-tags': { latest: Object.keys(versions).at(-1) }, + time: Object.fromEntries(Object.entries(versions).flatMap(([version, { time }]) => time === undefined ? [] : [[version, time]])), + // A package without versions has a registry document without a `versions` member. + ...Object.keys(versions).length === 0 ? {} : { versions: Object.fromEntries(Object.entries(versions).map(([version, published]) => { + const bytes = published.tarball + const integrity = published.integrity ?? (Buffer.isBuffer(bytes) ? `sha512-${createHash('sha512').update(bytes).digest('base64')}` : undefined) + return [version, { + name, version, ...published.manifest, + dist: { tarball: `${base}/tarballs/${encodeURIComponent(name)}/${version}.tgz`, ...integrity === undefined ? {} : { integrity } }, + }] + })) }, + })) + }) + await new Promise((resolve) => { server.listen(0, '127.0.0.1', resolve) }) + const address = server.address() + if (address === null || typeof address === 'string') throw new Error('no port') + const base = `http://127.0.0.1:${address.port}` + onCleanup(() => new Promise((resolve) => { + server.close(() => { resolve() }) + server.closeAllConnections() + })) + return { url: `${base}/`, tarballs } +} + +const BUNDLE = { dsh: { bundle: { patch: './cordis.patch.yml' } } } + +const bundleManifest = (name: string, version: string, extra: object = {}) => JSON.stringify({ name, version, ...BUNDLE, ...extra }) + +function options(registryUrl: string, cacheDir: string, lines: string[], warnings: string[]): GenerateOptions { + return { + sources: lines.map(name => ({ kind: 'npm' as const, line: name, name })), + cacheDir, registry: registryUrl, recordRegistry: true, title: 'Test Market', description: 'For tests', + inlineIconBytes: 64, limits: { timeoutMs: 10000, maxTarballBytes: 1 << 20 }, concurrency: 2, + warn: (message) => { warnings.push(message) }, + } +} + +it('lists every non-deprecated bundle version from the registry document and reads display fields from the highest tarball', async () => { + const packages: Packages = { + 'dsh-a': { + '0.9.0': { time: '2026-01-01T00:00:00.000Z' }, + '1.0.0': { time: '2026-02-01T00:00:00.000Z', manifest: { ...BUNDLE, keywords: ['old'], engines: { dsh: '>=0.2.0' }, peerDependencies: { '@deepseek-ai/dsh': '9' } } }, + '1.1.0': { time: '2026-03-01T00:00:00.000Z', manifest: { ...BUNDLE, deprecated: 'broken' } }, + '1.2.0': { + time: '2026-04-01T00:00:00.000Z', + manifest: { + ...BUNDLE, deprecated: '', + peerDependencies: { '@deepseek-ai/dsh': '>=0.2.0', '@deepseek-ai/dsh-agent': '>=0.2.0', '@deepseek-ai/dsh-tools': '^0.2.0 || ^0.3.0', react: '18' }, + }, + }, + '1.3.0': { time: '2026-05-01T00:00:00.000Z', manifest: { ...BUNDLE, peerDependencies: { '@deepseek-ai/dsh-agent': 'workspace:>=1' } } }, + '1.4.0': { manifest: BUNDLE }, + '2.0.0-rc.1': { + time: '2026-06-01T00:00:00.000Z', + manifest: { ...BUNDLE, author: { name: 'Jane Doe', email: 'jane@example.com' } }, + tarball: await pack({ + 'package.json': bundleManifest('dsh-a', '2.0.0-rc.1', { author: { name: 'Jane Doe' }, icon: 'icon.png', homepage: 'https://example.com/dsh-a', keywords: ['alpha', '', 'alpha'] }), + 'locale/en.json': JSON.stringify({ meta: { title: 'Alpha' } }), + 'locale/zh.json': JSON.stringify({ meta: { title: '甲' } }), + 'locale/x_y.json': '{}', + 'icon.png': new Uint8Array(100), + 'node_modules/dep/package.json': '{}', + }), + }, + }, + 'dsh-b': { '1.0.0': { time: '2026-06-02T00:00:00.000Z', manifest: { ...BUNDLE, author: 'Bob' }, tarball: await pack({ 'package.json': bundleManifest('dsh-b', '1.0.0') }) } }, + 'plain-lib': { '1.0.0': { time: '2026-06-01T00:00:00.000Z' } }, + 'dsh-imposter': { '1.0.0': { time: '2026-06-01T00:00:00.000Z', manifest: BUNDLE, tarball: await pack({ 'package.json': bundleManifest('dsh-other', '1.0.0') }) } }, + 'dsh-tampered': { '1.0.0': { time: '2026-06-01T00:00:00.000Z', manifest: BUNDLE, tarball: await pack({ 'package.json': bundleManifest('dsh-tampered', '1.0.0') }), integrity: 'sha512-AAAA' } }, + 'dsh-unsigned': { '1.0.0': { time: '2026-06-01T00:00:00.000Z', manifest: BUNDLE } }, + 'dsh-huge': { '1.0.0': { time: '2026-06-01T00:00:00.000Z', manifest: BUNDLE, tarball: 'oversized', integrity: 'sha512-AAAA' } }, + 'dsh-bomb': { '1.0.0': { time: '2026-06-01T00:00:00.000Z', manifest: BUNDLE, tarball: await pack({ 'package.json': bundleManifest('dsh-bomb', '1.0.0'), 'zeros.bin': new Uint8Array(2 << 20) }) } }, + 'dsh-no-tarball': { '1.0.0': { time: '2026-06-01T00:00:00.000Z', manifest: BUNDLE, integrity: 'sha512-AAAA' } }, + 'dsh-unpublished': {}, + 'dsh-missing': 404, + } + const server = await registry(packages) + const cacheDir = scratch() + const warnings: string[] = [] + const lines = ['dsh-a', 'dsh-b', 'plain-lib', 'dsh-imposter', 'dsh-tampered', 'dsh-unsigned', 'dsh-huge', 'dsh-bomb', 'dsh-no-tarball', 'dsh-unpublished', 'dsh-missing'] + const first = await generate(options(server.url, cacheDir, lines, warnings)) + const npm = { type: 'npm', registry: server.url } + expect(first.index).toMatchObject({ format: 1, title: 'Test Market', description: 'For tests' }) + expect(first.index.plugins).toEqual([ + { + name: 'dsh-a', title: { en: 'Alpha', zh: '甲' }, icon: expect.stringMatching(/^icons\/[0-9a-f]{16}\.png$/u) as string, author: 'Jane Doe', homepage: 'https://example.com/dsh-a', + metadata: { keywords: ['alpha', '', 'alpha'] }, + versions: [ + { version: '2.0.0-rc.1', publishTimestamp: 1780272000, source: npm }, + { version: '1.2.0', publishTimestamp: 1775001600, source: npm, engines: { dsh: '>=0.2.0 <0.3.0-0 || >=0.2.0 >=0.3.0 <0.4.0-0' } }, + { version: '1.0.0', publishTimestamp: 1769904000, source: npm, engines: { dsh: '>=0.2.0' } }, + ], + }, + { name: 'dsh-b', title: 'dsh-b', versions: [{ version: '1.0.0', publishTimestamp: 1780358400, source: npm }] }, + ]) + expect(existsSync(first.icons.get(first.index.plugins[0]?.icon ?? '') ?? '')).toBe(true) + expect(server.tarballs.filter(path => path.includes('dsh-a'))).toEqual(['/tarballs/dsh-a/2.0.0-rc.1.tgz']) + expect(first.failures).toEqual([ + 'dsh-imposter: dsh-imposter@1.0.0: tarball package.json does not name dsh-imposter@1.0.0', + 'dsh-tampered: dsh-tampered@1.0.0: tarball does not match its registry integrity', + 'dsh-unsigned: dsh-unsigned@1.0.0: registry lists no tarball with integrity', + `dsh-huge: dsh-huge@1.0.0: tarball exceeds ${1 << 20} bytes`, + `dsh-bomb: dsh-bomb@1.0.0: unpacked tarball exceeds ${1 << 20} bytes`, + 'dsh-no-tarball: dsh-no-tarball@1.0.0: tarball answered HTTP 404', + 'dsh-missing: dsh-missing: registry answered HTTP 404', + ]) + expect(warnings).toEqual(expect.arrayContaining([ + 'dsh-a: dsh-a@1.3.0 skipped: DSH range "workspace:>=1" is not a valid SemVer range', + 'dsh-a@2.0.0-rc.1: locale file locale/x_y.json ignored: its name is not a language id', + 'plain-lib: no published version declares dsh.bundle', + 'dsh-unpublished: no published version declares dsh.bundle', + ])) + + // Unchanged version lists come from the cache; a failing source keeps its last result for the same registry only. + packages['dsh-b'] = 500 + const downloads = server.tarballs.length + const second = await generate(options(server.url, cacheDir, lines, warnings)) + expect(server.tarballs.length - downloads).toBe(5) + expect(second.index.plugins).toEqual(first.index.plugins) + expect(warnings).toContain('dsh-b: dsh-b: registry answered HTTP 500; keeping the last successful result') + const mirror = await generate(options(`${server.url}mirror/`, cacheDir, ['dsh-b'], warnings)) + expect([mirror.index.plugins, mirror.failures]).toEqual([[], ['dsh-b: dsh-b: registry answered HTTP 503']]) +}) + +it('keeps packages with absent or invalid keywords and preserves empty keywords in generated and cached output', async () => { + const cases = [ + { name: 'dsh-absent', fields: {}, metadata: undefined }, + { name: 'dsh-empty', fields: { keywords: [] }, metadata: { keywords: [] } }, + { name: 'dsh-mixed', fields: { keywords: ['valid', 1] }, metadata: undefined }, + { name: 'dsh-nonarray', fields: { keywords: 'search' }, metadata: undefined }, + ] + const packages: Packages = Object.fromEntries(await Promise.all(cases.map(async ({ name, fields }) => [name, { + '1.0.0': { time: '2026-06-01T00:00:00.000Z', manifest: BUNDLE, tarball: await pack({ 'package.json': bundleManifest(name, '1.0.0', fields) }) }, + }] as const))) + const server = await registry(packages) + const warnings: string[] = [] + const opts = options(server.url, scratch(), cases.map(({ name }) => name), warnings) + const first = await generate(opts) + expect(first.failures).toEqual([]) + expect(first.index.plugins.map(({ name, metadata }) => ({ name, metadata }))) + .toEqual(cases.map(({ name, metadata }) => ({ name, metadata }))) + for (const plugin of first.index.plugins) { + if (plugin.name !== 'dsh-empty') expect(plugin).not.toHaveProperty('metadata') + } + expect(warnings.sort()).toEqual([ + 'dsh-mixed@1.0.0: package.json keywords must be an array of strings; it is omitted', + 'dsh-nonarray@1.0.0: package.json keywords must be an array of strings; it is omitted', + ]) + const cached = await generate(opts) + expect(cached).toEqual(first) + expect(server.tarballs).toHaveLength(cases.length) + for (const { name } of cases) packages[name] = 503 + expect((await generate(opts)).index).toEqual(first.index) +}) + +it('rebuilds a cached record the index format rejects', async () => { + const server = await registry({ 'dsh-a': { '1.0.0': { time: '2026-06-01T00:00:00.000Z', manifest: BUNDLE, tarball: await pack({ 'package.json': bundleManifest('dsh-a', '1.0.0') }) } } }) + const cacheDir = scratch() + const first = await generate(options(server.url, cacheDir, ['dsh-a'], [])) + const results = join(cacheDir, 'results') + for (const record of ['{', JSON.stringify({ plugins: [{ name: 'Bad Name', versions: [] }] })]) { + for (const file of readdirSync(results)) writeFileSync(join(results, file), record) + expect((await generate(options(server.url, cacheDir, ['dsh-a'], []))).index.plugins).toEqual(first.index.plugins) + } + expect(server.tarballs).toHaveLength(3) +}) + +it('keeps the first source of a repeated package name', async () => { + const tarball = await pack({ 'package.json': bundleManifest('dsh-a', '1.0.0') }) + const server = await registry({ 'dsh-a': { '1.0.0': { time: '2026-06-01T00:00:00.000Z', manifest: BUNDLE, tarball } } }) + const warnings: string[] = [] + const source = { kind: 'npm' as const, line: 'dsh-a', name: 'dsh-a' } + const result = await generate({ ...options(server.url, scratch(), [], warnings), recordRegistry: false, sources: [source, { ...source, line: 'dsh-a ' }] }) + expect(result.index.plugins.map(entry => [entry.name, entry.versions[0]?.source])).toEqual([['dsh-a', { type: 'npm' }]]) + expect(warnings).toEqual(['dsh-a: listed by more than one source; keeping the first']) +}) + +it('writes the index and icons once, keeps the cache beside the input, and removes only stale generated icon files', async () => { + const server = await registry({ + 'dsh-a': { '1.0.0': { time: '2026-06-01T00:00:00.000Z', manifest: BUNDLE, tarball: await pack({ 'package.json': bundleManifest('dsh-a', '1.0.0', { icon: 'i.png' }), 'i.png': new Uint8Array(20000) }) } }, + }) + const lists = scratch() + const out = join(scratch(), 'site') + const list = join(lists, 'plugins.txt') + writeFileSync(list, 'dsh-a\n') + const args = ['--input', list, '--output', join(out, 'index.json'), '--title', 'T', '--registry', server.url] + expect(await main(args)).toBe(0) + const written = readFileSync(join(out, 'index.json'), 'utf8') + const icon = (JSON.parse(written) as { plugins: { icon: string }[] }).plugins[0]?.icon ?? '' + expect(readFileSync(join(out, icon)).length).toBe(20000) + expect([existsSync(join(lists, '.plugin-market-cache')), existsSync(join(out, '.plugin-market-cache'))]).toEqual([true, false]) + writeFileSync(join(out, 'icons', '0123456789abcdef.png'), 'stale') + writeFileSync(join(out, 'icons', 'keep.txt'), 'mine') + mkdirSync(join(out, 'icons', 'fedcba9876543210.svg')) + expect(await main(args)).toBe(0) + expect(readFileSync(join(out, 'index.json'), 'utf8')).toBe(written) + expect(['0123456789abcdef.png', 'keep.txt', 'fedcba9876543210.svg', icon.slice('icons/'.length)].map(name => existsSync(join(out, 'icons', name)))) + .toEqual([false, true, true, true]) + writeFileSync(list, 'dsh-gone\n') + const other = join(scratch(), 'index.json') + expect(await main(['--input', list, '--output', other, '--title', 'T', '--description', 'D', '--registry', server.url, '--concurrency', '2', '--cache', join(lists, 'other-cache')])).toBe(1) + expect((JSON.parse(readFileSync(other, 'utf8')) as { description: string }).description).toBe('D') + expect(await main(['--input', list])).toBe(2) + expect(await main(['--unknown'])).toBe(2) + expect(await main(['--help'])).toBe(0) + expect(await main([...args, '--concurrency', '0'])).toBe(2) + writeFileSync(list, 'http://example.com/o/r\n') + expect(await main(['--input', list, '--output', other, '--title', 'T'])).toBe(2) +}) diff --git a/apps/plugin-market-generator/tests/git.spec.ts b/apps/plugin-market-generator/tests/git.spec.ts new file mode 100644 index 0000000000..cbedd07569 --- /dev/null +++ b/apps/plugin-market-generator/tests/git.spec.ts @@ -0,0 +1,182 @@ +import { execFileSync } from 'node:child_process' +import { existsSync, mkdirSync, readdirSync, symlinkSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' +import { expect, it } from 'vitest' +import { generate, type GenerateOptions } from '../src/generate.ts' +import { scratch, setEnv } from './support.ts' + +const REPOSITORY = 'https://git.example/team/plugins' + +/** Git subprocesses run under CI load, so each test gets more than the default deadline. */ +const GIT_TEST = { timeout: 60000 } + +/** A Git repository fixture built without the caller's Git environment or configuration. */ +function repository() { + const dir = scratch() + const home = scratch() + const env = { + ...Object.fromEntries(Object.entries(process.env).filter(([name]) => !name.startsWith('GIT_'))), + HOME: home, XDG_CONFIG_HOME: home, GIT_CONFIG_NOSYSTEM: '1', + } + const git = (args: string[], time = 0) => execFileSync('git', ['-c', 'user.name=Test', '-c', 'user.email=test@example.com', ...args], { + cwd: dir, env: { ...env, GIT_AUTHOR_DATE: `${time} +0000`, GIT_COMMITTER_DATE: `${time} +0000` }, encoding: 'utf8', + }).trim() + git(['init', '--quiet', '--initial-branch=main']) + const write = (files: Record) => { + for (const [path, contents] of Object.entries(files)) { + mkdirSync(join(dir, path, '..'), { recursive: true }) + writeFileSync(join(dir, path), contents) + } + } + const commit = (time: number) => { + git(['add', '--all']) + git(['commit', '--quiet', '--message', `at ${time}`], time) + return git(['rev-parse', 'HEAD']) + } + return { dir, git, write, commit } +} + +const bundle = (name: string, version: string, extra: object = {}) => JSON.stringify({ name, version, dsh: { bundle: { patch: './cordis.patch.yml' } }, ...extra }) + +/** + * Configure Git for the generator through a private home directory: the https REPOSITORY is routed to the fixture, protocol + * version 0 refuses commits that are not advertised ref tips, and a required filter would mark any checkout. + */ +function gitHome(fixture: string): { marker: string; temporary: string } { + const home = scratch() + const marker = join(home, 'filtered') + // Git config values treat a backslash as an escape, so Windows paths are written with forward slashes. + const slashed = (path: string) => path.replaceAll('\\', '/') + writeFileSync(join(home, '.gitconfig'), [ + `[url "${pathToFileURL(fixture).href}"]`, `\tinsteadOf = ${REPOSITORY}`, + '[protocol]', '\tversion = 0', + '[filter "marker"]', `\tsmudge = touch '${slashed(marker)}'; cat`, `\tclean = touch '${slashed(marker)}'; cat`, '\trequired = true', + ].join('\n')) + const temporary = scratch() + setEnv('HOME', home) + setEnv('XDG_CONFIG_HOME', home) + setEnv('TMPDIR', temporary) + return { marker, temporary } +} + +function options(cacheDir: string, warnings: string[]): GenerateOptions { + return { + sources: [{ kind: 'git', line: REPOSITORY, url: REPOSITORY }], cacheDir, registry: 'https://registry.invalid/', recordRegistry: false, title: 'T', + inlineIconBytes: 1024, limits: { timeoutMs: 20000, maxTarballBytes: 1 << 20 }, concurrency: 1, gitTransports: ['file'], + warn: (message) => { warnings.push(message) }, + } +} + +it('lists Git versions from release tags and the default branch by reading objects', GIT_TEST, async () => { + const repo = repository() + repo.write({ + 'package.json': JSON.stringify({ name: 'mono', private: true }), + 'pnpm-workspace.yaml': 'packages:\n - packages/*\n', + '.gitattributes': '* filter=marker\n', + 'packages/a/package.json': bundle('@s/a', '1.0.0'), + 'packages/b/package.json': bundle('@s/b', '0.9.0'), + }) + const first = repo.commit(1790000000) + repo.git(['tag', 'v1.0.0']) + repo.git(['tag', 'not-a-version']) + repo.write({ 'packages/b/package.json': bundle('@s/b', '2.0.0', { keywords: ['tagged'] }) }) + const second = repo.commit(1790000100) + repo.git(['tag', '--annotate', '--message', 'b release', '@s/b@2.0.0'], 1790000150) + repo.write({ 'packages/b/README.md': 'docs' }) + repo.commit(1790000200) + repo.git(['tag', '1.0.0']) + repo.git(['tag', '@s/missing@1.0.0']) + repo.git(['tag', 'broken@']) + mkdirSync(join(repo.dir, 'packages/c'), { recursive: true }) + symlinkSync('../a/package.json', join(repo.dir, 'packages/c/package.json')) + repo.write({ + 'packages/a/package.json': JSON.stringify({ ...JSON.parse(bundle('@s/a', '1.1.0')), author: 'Ann', icon: 'icon.svg', keywords: ['alpha'] }), + 'packages/a/locale/en.json': JSON.stringify({ meta: { title: 'Alpha', description: 'Does A' } }), + 'packages/a/icon.svg': '', + 'packages/b/package.json': bundle('@s/b', '2.0.0', { keywords: ['head'] }), + 'packages/d/package.json': JSON.stringify({ ...JSON.parse(bundle('@s/d', '0.1.0')), peerDependencies: { '@deepseek-ai/dsh': 'workspace:>=1' } }), + }) + const head = repo.commit(1790000300) + repo.git(['tag', 'v2.0.0']) + repo.git(['tag', '@s/a@1.1.0']) + const cacheDir = scratch() + const { marker, temporary } = gitHome(repo.dir) + const warnings: string[] = [] + + const result = await generate(options(cacheDir, warnings)) + const source = (commit: string, path: string) => ({ type: 'git', url: REPOSITORY, commit, path }) + expect(result.index.plugins).toEqual([ + { + name: '@s/a', title: 'Alpha', description: 'Does A', icon: 'data:image/svg+xml;base64,PHN2Zy8+', author: 'Ann', + metadata: { keywords: ['alpha'] }, + versions: [ + { version: '1.1.0', publishTimestamp: 1790000300, source: source(head, 'packages/a') }, + { version: '1.0.0', publishTimestamp: 1790000000, source: source(first, 'packages/a') }, + ], + }, + { name: '@s/b', title: '@s/b', metadata: { keywords: ['tagged'] }, versions: [{ version: '2.0.0', publishTimestamp: 1790000100, source: source(second, 'packages/b') }] }, + ]) + expect([result.failures, warnings]).toEqual([[], [`${REPOSITORY}: @s/d@0.1.0 skipped: DSH range "workspace:>=1" is not a valid SemVer range`]]) + expect([existsSync(marker), readdirSync(temporary)]).toEqual([false, []]) + expect(await generate(options(cacheDir, warnings))).toEqual(result) + + // An untagged version follows the default branch; tagged versions and their display fields come from the cache. + repo.write({ 'packages/b/package.json': bundle('@s/b', '2.1.0') }) + const next = repo.commit(1790000400) + const updated = await generate(options(cacheDir, warnings)) + expect(updated.index.plugins.map(plugin => plugin.versions.map(version => [version.version, version.source.commit]))) + .toEqual([[['1.1.0', head], ['1.0.0', first]], [['2.1.0', next], ['2.0.0', second]]]) + expect(updated.index.plugins[0]).toEqual(result.index.plugins[0]) + expect(updated.index.plugins[1]).not.toHaveProperty('metadata') +}) + +it('reads a root package, reports repositories without one, and refuses transports that are not allowed', GIT_TEST, async () => { + const repo = repository() + gitHome(repo.dir) + const warnings: string[] = [] + const run = async (defaultTransports = false) => { + const withFile = options(scratch(), warnings) + const { gitTransports: _file, ...defaults } = withFile + const result = await generate(defaultTransports ? defaults : withFile) + return [result.index.plugins, result.failures] + } + expect(await run()).toEqual([[], [`${REPOSITORY}: ${REPOSITORY}: no HEAD commit`]]) + repo.write({ 'README.md': 'nothing here' }) + repo.commit(1790000000) + expect(await run(true)).toEqual([[], [expect.stringContaining("transport 'file' not allowed") as string]]) + expect(await run()).toEqual([[], []]) + repo.write({ 'package.json': JSON.stringify({ version: '1.0.0', dsh: { bundle: { patch: 'p.yml' } } }) }) + repo.commit(1790000100) + expect(await run()).toEqual([[], []]) + repo.write({ 'package.json': bundle('dsh-root', '1.0.0') }) + const commit = repo.commit(1790000200) + expect(await run()).toEqual([[{ + name: 'dsh-root', title: 'dsh-root', versions: [{ version: '1.0.0', publishTimestamp: 1790000200, source: { type: 'git', url: REPOSITORY, commit } }], + }], []]) + expect(warnings).toEqual([`${REPOSITORY}: ${REPOSITORY}: no HEAD commit`, expect.stringContaining('not allowed') as string, `${REPOSITORY}: no package declares dsh.bundle`, `${REPOSITORY}: package.json has no name`]) +}) + +it('skips a release tag it cannot read, refuses oversized manifests, and ignores Git overrides from the environment', GIT_TEST, async () => { + const repo = repository() + repo.write({ + 'package.json': bundle('dsh-root', '1.0.0'), + }) + repo.commit(1790000000) + const blob = repo.git(['hash-object', '-w', 'package.json']) + repo.git(['tag', 'v0.5.0', blob]) + gitHome(repo.dir) + // A Git override from the caller would redirect the private repository; the user's ssh command is kept. + setEnv('GIT_DIR', join(repo.dir, '.git')) + setEnv('GIT_SSH_COMMAND', 'ssh -o BatchMode=yes -o ConnectTimeout=1') + const warnings: string[] = [] + const result = await generate(options(scratch(), warnings)) + expect(result.index.plugins.map(plugin => plugin.versions.map(version => version.version))).toEqual([['1.0.0']]) + expect(result.failures).toEqual([]) + expect(warnings).toEqual([expect.stringMatching(new RegExp(`^${REPOSITORY}: ${blob}: skipped: `, 'u')) as string]) + + repo.write({ 'package.json': JSON.stringify({ name: 'dsh-root', version: '1.1.0', padding: 'x'.repeat(1024 * 1024) }) }) + repo.commit(1790000100) + const oversized = await generate(options(scratch(), warnings)) + expect(oversized.failures).toEqual([`${REPOSITORY}: package.json exceeds 1048576 bytes`]) +}) diff --git a/apps/plugin-market-generator/tests/support.ts b/apps/plugin-market-generator/tests/support.ts new file mode 100644 index 0000000000..d5cd7c598c --- /dev/null +++ b/apps/plugin-market-generator/tests/support.ts @@ -0,0 +1,45 @@ +/** Per-test temporary directories and cleanups shared by the generator specs. */ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, vi } from 'vitest' + +const cleanups: (() => void | Promise)[] = [] + +/** Every cleanup runs after each test, in reverse order, even when an earlier one fails. */ +afterEach(async () => { + const errors: unknown[] = [] + for (const cleanup of cleanups.splice(0).reverse()) { + try { + await cleanup() + } catch (error) { + errors.push(error) + } + } + if (errors.length > 0) throw new AggregateError(errors, 'test cleanup failed') +}) + +/** + * Register a cleanup for the current test. + * @param cleanup - runs after the test. + */ +export function onCleanup(cleanup: () => void | Promise): void { + cleanups.push(cleanup) +} + +/** @returns a new temporary directory deleted after the current test. */ +export function scratch(): string { + const dir = mkdtempSync(join(tmpdir(), 'market-generator-')) + onCleanup(() => { rmSync(dir, { recursive: true, force: true }) }) + return dir +} + +/** + * Set a process environment variable until the current test ends. + * @param name - variable name. + * @param value - value for the test. + */ +export function setEnv(name: string, value: string): void { + vi.stubEnv(name, value) + onCleanup(() => { vi.unstubAllEnvs() }) +} diff --git a/apps/plugin-market-generator/tsconfig.json b/apps/plugin-market-generator/tsconfig.json new file mode 100644 index 0000000000..6267ae46f4 --- /dev/null +++ b/apps/plugin-market-generator/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib/types" + }, + "include": [ + "src" + ], + "references": [ + { "path": "../../packages/util/plugin-market-index" }, + { "path": "../../packages/util/package-manifest" } + ] +} diff --git a/docs/user/guide/plugin-market.i18n.yaml b/docs/user/guide/plugin-market.i18n.yaml index 4ef6d68a2b..e05a0c2801 100644 --- a/docs/user/guide/plugin-market.i18n.yaml +++ b/docs/user/guide/plugin-market.i18n.yaml @@ -9,8 +9,8 @@ en: 2716b194b8a12aa7 zh: b90773d095851969 /plugin-markets/table-of-contents: - en: 9400db64624524b7 - zh: 92cae6a2675135fc + en: a8d1e6785c3c3c30 + zh: 7ea1eaa8204a5a1e /plugin-markets/turn-on-discover: en: 96b2fd994f8f3e7e zh: 203e8460ec6aed7a @@ -18,8 +18,11 @@ en: cf616bba26958533 zh: 1e32df16a6f4bba7 /plugin-markets/publish-an-index: - en: b37011e8397c9fad - zh: 57b5a3b9b17e357e + en: b85b82b22c1e14b8 + zh: 920c0d6f20956c07 +/plugin-markets/generate-an-index-with-the-reference-generator: + en: 57a48912c00e1c85 + zh: 69f2934eed0fbadf /plugin-markets/index-format-reference: en: d058d12c27433f64 zh: f43b85e4934817f3 diff --git a/docs/user/guide/plugin-market.md b/docs/user/guide/plugin-market.md index 8ebdc4c753..91152f23ed 100644 --- a/docs/user/guide/plugin-market.md +++ b/docs/user/guide/plugin-market.md @@ -11,6 +11,7 @@ A plugin market is one JSON index file that lists installable DSH bundles with t - [Turn on Discover](#turn-on) - [Browse and install](#browse-and-install) - [Publish an index](#publish-an-index) +- [Generate an index with the reference generator](#generate-an-index) - [Index format reference](#index-format) - [Further Exploration](#further-exploration) @@ -80,6 +81,26 @@ This is a minimal index with one npm plugin and one plugin from a Git monorepo: DSH rejects the whole index when any rule below is violated and shows the reason, with the JSON path of the offending value, under the source in **Manage index sources…**. + +## Generate an index with the reference generator + +The repository includes a [reference generator](../../../apps/plugin-market-generator/README.md) that builds an index from a plain list. Write one npm package name or https Git repository URL per line; blank lines and lines starting with `#` are ignored: + +```text +@team/dsh-foo +https://gitlab.example.com/team/dsh-plugins +``` + +Run it from a DSH checkout: + +```sh +pnpm --filter @deepseek-ai/dsh-plugin-market-generator start --input plugins.txt --output site/index.json --title "Team plugins" +``` + +For an npm package, the index lists every version that is not deprecated and declares `dsh.bundle`. For a Git repository, it finds the bundle packages on the default branch and lists their versions at `vX.Y.Z`, `X.Y.Z`, and `@X.Y.Z` tags, plus each package's version on the default branch. Titles, descriptions, icons, authors, and homepages come from each plugin's highest version. Paths are relative to the directory where you run pnpm. + +The generator only downloads and reads files; it never installs packages or runs their code. It keeps a cache in `.plugin-market-cache` next to the list, so a repeated run downloads only new versions and commits. Publish the output directory, `index.json` together with its `icons/` directory, as described above. + ## Index format reference diff --git a/docs/user/guide/plugin-market.zh.md b/docs/user/guide/plugin-market.zh.md index f14a74bc25..3b7ca0ea38 100644 --- a/docs/user/guide/plugin-market.zh.md +++ b/docs/user/guide/plugin-market.zh.md @@ -11,6 +11,7 @@ - [打开「发现」](#turn-on) - [浏览和安装](#browse-and-install) - [发布索引](#publish-an-index) +- [用参考生成器生成索引](#generate-an-index) - [索引格式参考](#index-format) - [延伸阅读](#further-exploration) @@ -80,6 +81,26 @@ DSH 不内置任何索引源。选择**发现**，点击**添加索引源**， 违反下列任何一条规则时，DSH 会拒绝整个索引，并在**管理索引源…**中该索引源下方显示原因及违规值的 JSON 路径。 + +## 用参考生成器生成索引 + +仓库包含一个[参考生成器](../../../apps/plugin-market-generator/README.zh.md)，可以从一份纯文本列表构建索引。每行写一个 npm 包名或 https Git 仓库 URL；空行和以 `#` 开头的行会被忽略： + +```text +@team/dsh-foo +https://gitlab.example.com/team/dsh-plugins +``` + +在 DSH 代码仓库中运行它： + +```sh +pnpm --filter @deepseek-ai/dsh-plugin-market-generator start --input plugins.txt --output site/index.json --title "Team plugins" +``` + +对于 npm 包，索引列出所有未弃用且声明了 `dsh.bundle` 的版本。对于 Git 仓库，生成器在默认分支上查找组合包，列出它们在 `vX.Y.Z`、`X.Y.Z` 和 `@X.Y.Z` 标签处的版本，以及每个包在默认分支上的版本。标题、描述、图标、作者和主页取自每个插件的最高版本。路径相对于你运行 pnpm 的目录。 + +生成器只下载和读取文件，从不安装包或运行其代码。它在列表旁的 `.plugin-market-cache` 中保存缓存，因此再次运行时只下载新的版本和提交。按上文所述发布输出目录，即 `index.json` 及其 `icons/` 目录。 + ## 索引格式参考 diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0e9dcd02cb..42dc762e7e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -828,6 +828,34 @@ importers: specifier: workspace:* version: link:../../packages/skill/skill + apps/plugin-market-generator: + dependencies: + '@deepseek-ai/dsh-plugin-market-index': + specifier: workspace:* + version: link:../../packages/util/plugin-market-index + resolve.exports: + specifier: ^2.0.3 + version: 2.0.3 + semver: + specifier: ^7.8.5 + version: 7.8.5 + tar: + specifier: ^7.5.22 + version: 7.5.22 + yaml: + specifier: ^2.9.0 + version: 2.9.0 + devDependencies: + '@deepseek-ai/dsh-package-manifest': + specifier: workspace:* + version: link:../../packages/util/package-manifest + '@types/semver': + specifier: ^7.8.0 + version: 7.8.0 + tsx: + specifier: ^4.22.4 + version: 4.23.13 + apps/web: devDependencies: '@deepseek-ai/cordis-plugin-group': diff --git a/scripts/check-workspace-constraints.spec.ts b/scripts/check-workspace-constraints.spec.ts index efad15c367..a69eed0281 100644 --- a/scripts/check-workspace-constraints.spec.ts +++ b/scripts/check-workspace-constraints.spec.ts @@ -350,6 +350,13 @@ describe('package payload constraints', () => { }, ) + it('exempts the private plugin market generator from release rules in its own directory only', () => { + const dir = 'apps/plugin-market-generator' + const manifest = JSON.parse(readFileSync(new URL(`../${dir}/package.json`, import.meta.url), 'utf8')) as WorkspaceManifest['manifest'] + expect(checkWorkspaceManifest({ dir, manifest })).toEqual([]) + expect(checkWorkspaceManifest({ dir: 'apps/other-tool', manifest })).not.toEqual([]) + }) + it('rejects locale publication entries without their resource exports', () => { const dir = 'packages/experimental/auto-review' const manifest = JSON.parse(readFileSync(new URL(`../${dir}/package.json`, import.meta.url), 'utf8')) as WorkspaceManifest['manifest'] diff --git a/scripts/check-workspace-constraints.ts b/scripts/check-workspace-constraints.ts index 5d9b1e7ca5..cf9493a5d2 100644 --- a/scripts/check-workspace-constraints.ts +++ b/scripts/check-workspace-constraints.ts @@ -57,8 +57,16 @@ const publicationSourceAllowlist: Readonly> = const publishedRepositoryUrl = 'git+https://github.com/deepseek-ai/deepseek-harness.git' /** Packages that participate in the experimental policy. */ const experimentalPackageDirectory = /^packages\/experimental\/[^/]+$/ -/** Ordinary directories whose packages this repository publishes: one release member each. */ -const standardReleaseMemberDirectory = /^(?:packages\/(?!experimental\/)[^/]+\/[^/]+|apps\/(?!desktop(?:-host)?$)[^/]+|vendor\/[^/]+)$/ +/** Private reference tool run from the repository; it publishes no files. */ +const pluginMarketGeneratorDirectory = 'apps/plugin-market-generator' +/** + * Ordinary directories whose packages this repository publishes: one release member each. The reference plugin + * market generator is a private repository tool, not a published application. + */ +const standardReleaseMemberDirectory = new RegExp( + `^(?:packages/(?!experimental/)[^/]+/[^/]+|apps/(?!desktop(?:-host)?$|${pluginMarketGeneratorDirectory.slice('apps/'.length)}$)[^/]+|vendor/[^/]+)$`, + 'u', +) /** Installable application assembled by electron-builder rather than published to npm. */ const desktopApplicationDirectory = 'apps/desktop' const localArtifactDirs = new Set(['node_modules']) @@ -484,7 +492,7 @@ export function checkWorkspaceManifest({ dir, manifest }: WorkspaceManifest): st } } - if (dir.startsWith('apps/') && dir !== desktopApplicationDirectory && manifest.name?.startsWith('@deepseek-ai/')) { + if (dir.startsWith('apps/') && dir !== desktopApplicationDirectory && dir !== pluginMarketGeneratorDirectory && manifest.name?.startsWith('@deepseek-ai/')) { const expectedFiles = appPackageFiles[manifest.name] if (expectedFiles === undefined) { errors.push(`${label}: app package has no publication files policy`) diff --git a/tsconfig.host.json b/tsconfig.host.json index f481764d40..aab5dbbb28 100644 --- a/tsconfig.host.json +++ b/tsconfig.host.json @@ -200,6 +200,7 @@ "apps/web/tests/workflow-run.e2e.ts", "apps/web/stress-tests/reasoning-chunks.stress.ts", "apps/cli/tests/**/*.ts", + "apps/plugin-market-generator/tests/**/*.ts", "apps/desktop/scripts/**/*.ts", "apps/desktop/tests/**/*.ts", "apps/desktop-host/tests/**/*.ts", @@ -526,6 +527,7 @@ { "path": "./packages/lsp/lsp-stdio" }, { "path": "./packages/lsp/tool-lsp" }, { "path": "./apps/cli" }, + { "path": "./apps/plugin-market-generator" }, { "path": "./apps/desktop-host" }, { "path": "./apps/desktop/tsconfig.host.json" } ]