fix: download optional Python runtime resources explicitly
This commit is contained in:
1 parent
15c10e0e87
commit
89f35fe2e9
34 files changed
+1159
-259
No files matched your search
@@ -1,8 +1,8 @@
|
||||
/** Prepare pinned, relocatable script interpreters without installing into the build host. */
|
||||
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { cpSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { cp } from 'node:fs/promises'
|
||||
import { createRequire } from 'node:module'
|
||||
import { tmpdir } from 'node:os'
|
||||
@@ -10,30 +10,97 @@ import { dirname, join, resolve } from 'node:path'
|
||||
import { parseArgs } from 'node:util'
|
||||
import extractZip from 'extract-zip'
|
||||
import { x as extractTar } from 'tar'
|
||||
import { parsePrimaryRuntime, workspaceDependencyPaths, type PrimaryRuntimeManifest } from '../../packages/skill/tool-workspace-dependencies/src/index.ts'
|
||||
import { parsePrimaryRuntime, workspaceDependencyPaths, type PrimaryRuntimeManifest } from '@deepseek-ai/dsh-tool-workspace-dependencies'
|
||||
import lock from './lock.json' with { type: 'json' }
|
||||
|
||||
/**
|
||||
* Download or reuse an archive only when its bytes match the release lock.
|
||||
* @param url - Locked archive URL.
|
||||
* @param sha256 - Expected SHA-256 digest.
|
||||
* @param sha256 - Expected hexadecimal digest for the selected algorithm.
|
||||
* @param cache - Download cache directory.
|
||||
* @param algorithm - Digest algorithm; npm archives use SHA-512.
|
||||
* @returns Verified local archive path.
|
||||
*/
|
||||
export async function downloadPrimaryRuntimeAsset(url: string, sha256: string, cache: string): Promise<string> {
|
||||
export async function downloadPrimaryRuntimeAsset(url: string, sha256: string, cache: string, algorithm: 'sha256' | 'sha512' = 'sha256'): Promise<string> {
|
||||
mkdirSync(cache, { recursive: true })
|
||||
const destination = join(cache, sha256)
|
||||
let bytes: Buffer
|
||||
try { bytes = readFileSync(destination) } catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error
|
||||
const response = await fetch(url)
|
||||
const timeout = Number(process.env.DSH_RESOURCE_DOWNLOAD_TIMEOUT_MS ?? '300000')
|
||||
if (!Number.isSafeInteger(timeout) || timeout < 1 || timeout > 2_147_483_647) throw new Error('DSH_RESOURCE_DOWNLOAD_TIMEOUT_MS must be an integer from 1 to 2147483647')
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(timeout) })
|
||||
if (!response.ok) throw new Error(`primary runtime download: ${String(response.status)} ${url}`)
|
||||
bytes = Buffer.from(await response.arrayBuffer())
|
||||
}
|
||||
if (createHash('sha256').update(bytes).digest('hex') !== sha256) throw new Error(`primary runtime download: checksum mismatch for ${url}`)
|
||||
writeFileSync(destination, bytes)
|
||||
if (createHash(algorithm).update(bytes).digest('hex') !== sha256) {
|
||||
rmSync(destination, { force: true })
|
||||
throw new Error(`primary runtime download: checksum mismatch for ${url}`)
|
||||
}
|
||||
const temporary = join(cache, `.${randomUUID()}`)
|
||||
try {
|
||||
writeFileSync(temporary, bytes)
|
||||
renameSync(temporary, destination)
|
||||
} finally {
|
||||
rmSync(temporary, { force: true })
|
||||
}
|
||||
return destination
|
||||
}
|
||||
|
||||
/** Pinned npm tarball, without dependency resolution or lifecycle scripts. */
|
||||
export interface RuntimeNpmArchive {
|
||||
/** Exact published package name. */
|
||||
readonly name: string
|
||||
/** Exact published version. */
|
||||
readonly version: string
|
||||
/** Registry tarball URL. */
|
||||
readonly url: string
|
||||
/** npm SHA-512 integrity. */
|
||||
readonly integrity: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Download and unpack one locked npm package without running its scripts.
|
||||
* @param artifact - Published tarball identity and integrity.
|
||||
* @param destination - Empty package directory.
|
||||
* @param cache - Verified archive cache.
|
||||
* @returns Resolves after package identity validation.
|
||||
*/
|
||||
export async function downloadRuntimeNpmPackage(artifact: RuntimeNpmArchive, destination: string, cache: string): Promise<void> {
|
||||
if (!/^sha512-[A-Za-z0-9+/]+={0,2}$/u.test(artifact.integrity)) throw new Error(`runtime download: invalid integrity for ${artifact.name}`)
|
||||
const checksum = Buffer.from(artifact.integrity.slice(7), 'base64').toString('hex')
|
||||
const archive = await downloadPrimaryRuntimeAsset(artifact.url, checksum, cache, 'sha512')
|
||||
mkdirSync(destination, { recursive: true })
|
||||
await extractTar({ file: archive, cwd: destination, strip: 1 })
|
||||
const installed = JSON.parse(readFileSync(join(destination, 'package.json'), 'utf8')) as { name?: string; version?: string }
|
||||
if (installed.name !== artifact.name || installed.version !== artifact.version) throw new Error(`runtime download: package identity mismatch for ${artifact.name}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare the locked standalone Node independently of Python and pnpm.
|
||||
* @param target - Target interpreter platform.
|
||||
* @param destination - Directory receiving bin/node and LICENSE.
|
||||
* @param cache - Verified archive cache.
|
||||
* @returns Resolves after extracting the interpreter.
|
||||
*/
|
||||
export async function downloadNodeRuntime(target: PrimaryRuntimeTarget, destination: string, cache: string): Promise<void> {
|
||||
const artifact = lock.targets[target]
|
||||
const filename = `node-v${lock.nodeVersion}-${artifact.nodeArchive}`
|
||||
const archive = await downloadPrimaryRuntimeAsset(`https://nodejs.org/dist/v${lock.nodeVersion}/${filename}`, artifact.nodeSha256, cache)
|
||||
const staging = mkdtempSync(join(tmpdir(), 'dsh-node-'))
|
||||
try {
|
||||
if (target === 'win-x64') await extractZip(archive, { dir: staging })
|
||||
else await extractTar({ file: archive, cwd: staging })
|
||||
const source = join(staging, filename.replace(/\.(?:zip|tar\.gz)$/u, ''))
|
||||
mkdirSync(join(destination, 'bin'), { recursive: true })
|
||||
cpSync(join(source, ...(target === 'win-x64' ? ['node.exe'] : ['bin', 'node'])),
|
||||
join(destination, 'bin', target === 'win-x64' ? 'node.exe' : 'node'))
|
||||
cpSync(join(source, 'LICENSE'), join(destination, 'LICENSE'))
|
||||
} finally {
|
||||
rmSync(staging, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
async function pythonArchive(target: keyof typeof lock.targets, cache: string): Promise<string> {
|
||||
const artifact = lock.targets[target]
|
||||
const filename = `cpython-${lock.pythonVersion}+${lock.pythonRelease}-${artifact.pythonTarget}-install_only_stripped.tar.gz`
|
||||
@@ -91,7 +158,7 @@ export async function prepareOfficeSkillAssets(source: string, destination: stri
|
||||
/** A locked interpreter and wheel target. */
|
||||
export type PrimaryRuntimeTarget = keyof typeof lock.targets
|
||||
|
||||
/** Build-only inputs shared by Desktop and SDK carriers. */
|
||||
/** Resource preparation inputs shared by builds and explicit SDK downloads. */
|
||||
export interface PreparePrimaryRuntimeOptions {
|
||||
/** Target whose archives and wheels are downloaded. */
|
||||
readonly target: PrimaryRuntimeTarget
|
||||
@@ -101,6 +168,10 @@ export interface PreparePrimaryRuntimeOptions {
|
||||
readonly cache: string
|
||||
/** Carrier release recorded in the legacy desktopVersion manifest field. */
|
||||
readonly version: string
|
||||
/** Locked npm pnpm tarball for installed carriers; build hosts otherwise copy their installed pnpm. */
|
||||
readonly pnpmArchive?: RuntimeNpmArchive
|
||||
/** Ordinary filesystem Office assets; build hosts otherwise resolve the workspace package. */
|
||||
readonly skillSource?: string
|
||||
/** Omit Node.js and pnpm for carriers providing only Python. */
|
||||
readonly pythonOnly?: boolean
|
||||
}
|
||||
@@ -113,7 +184,6 @@ export interface PreparePrimaryRuntimeOptions {
|
||||
export async function preparePrimaryRuntime(options: PreparePrimaryRuntimeOptions): Promise<void> {
|
||||
const { target } = options
|
||||
const paths = { runtime: resolve(options.output), downloads: resolve(options.cache) }
|
||||
const artifact = lock.targets[target]
|
||||
mkdirSync(paths.runtime, { recursive: true })
|
||||
mkdirSync(paths.downloads, { recursive: true })
|
||||
const staging = mkdtempSync(join(tmpdir(), 'dsh-primary-'))
|
||||
@@ -123,23 +193,18 @@ export async function preparePrimaryRuntime(options: PreparePrimaryRuntimeOption
|
||||
mkdirSync(dependencies, { recursive: true })
|
||||
let pnpmVersion: string | undefined
|
||||
if (!options.pythonOnly) {
|
||||
const nodeFilename = `node-v${lock.nodeVersion}-${artifact.nodeArchive}`
|
||||
const nodeArchive = await downloadPrimaryRuntimeAsset(`https://nodejs.org/dist/v${lock.nodeVersion}/${nodeFilename}`, artifact.nodeSha256, paths.downloads)
|
||||
const unpackedNode = join(staging, 'node')
|
||||
mkdirSync(unpackedNode)
|
||||
if (target === 'win-x64') await extractZip(nodeArchive, { dir: unpackedNode })
|
||||
else await extractTar({ file: nodeArchive, cwd: unpackedNode })
|
||||
const nodeSource = join(unpackedNode, nodeFilename.replace(/\.(?:zip|tar\.gz)$/u, ''))
|
||||
mkdirSync(join(dependencies, 'node', 'bin'), { recursive: true })
|
||||
mkdirSync(join(dependencies, 'node', 'node_modules'))
|
||||
writeFileSync(join(dependencies, 'node', 'node_modules', 'README.txt'), 'Reserved for bundled Node packages. pnpm uses its default installation directories.\n')
|
||||
cpSync(join(nodeSource, ...(target === 'win-x64' ? ['node.exe'] : ['bin', 'node'])),
|
||||
join(dependencies, 'node', 'bin', target === 'win-x64' ? 'node.exe' : 'node'))
|
||||
cpSync(join(nodeSource, 'LICENSE'), join(dependencies, 'node', 'LICENSE'))
|
||||
const require = createRequire(import.meta.url)
|
||||
const pnpmManifest = require.resolve('pnpm')
|
||||
pnpmVersion = (JSON.parse(readFileSync(pnpmManifest, 'utf8')) as { version: string }).version
|
||||
await cp(dirname(pnpmManifest), join(dependencies, 'pnpm'), { recursive: true, dereference: true })
|
||||
const node = join(dependencies, 'node')
|
||||
await downloadNodeRuntime(target, node, paths.downloads)
|
||||
mkdirSync(join(node, 'node_modules'))
|
||||
writeFileSync(join(node, 'node_modules', 'README.txt'), 'Reserved for bundled Node packages. pnpm uses its default installation directories.\n')
|
||||
if (options.pnpmArchive !== undefined) {
|
||||
pnpmVersion = options.pnpmArchive.version
|
||||
await downloadRuntimeNpmPackage(options.pnpmArchive, join(dependencies, 'pnpm'), paths.downloads)
|
||||
} else {
|
||||
const pnpmManifest = createRequire(import.meta.url).resolve('pnpm')
|
||||
pnpmVersion = (JSON.parse(readFileSync(pnpmManifest, 'utf8')) as { version: string }).version
|
||||
await cp(dirname(pnpmManifest), join(dependencies, 'pnpm'), { recursive: true, dereference: true })
|
||||
}
|
||||
}
|
||||
await extractTar({ file: await pythonArchive(target, paths.downloads), cwd: dependencies })
|
||||
const manifest: PrimaryRuntimeManifest = {
|
||||
@@ -152,7 +217,7 @@ export async function preparePrimaryRuntime(options: PreparePrimaryRuntimeOption
|
||||
pythonPackages: lock.pythonPackages,
|
||||
}
|
||||
const entries = workspaceDependencyPaths(output, manifest)
|
||||
for (const wheel of [...artifact.wheels, ...lock.wheels]) {
|
||||
for (const wheel of [...lock.targets[target].wheels, ...lock.wheels]) {
|
||||
await unpackPrimaryRuntimeWheel(await downloadPrimaryRuntimeAsset(wheel.url, wheel.sha256, paths.downloads), entries.pythonPackages)
|
||||
}
|
||||
writeFileSync(join(output, 'runtime.json'), `${JSON.stringify(manifest, undefined, 2)}\n`)
|
||||
@@ -162,8 +227,8 @@ export async function preparePrimaryRuntime(options: PreparePrimaryRuntimeOption
|
||||
} finally {
|
||||
rmSync(staging, { recursive: true, force: true })
|
||||
}
|
||||
const require = createRequire(import.meta.url)
|
||||
await prepareOfficeSkillAssets(join(dirname(require.resolve('@deepseek-ai/dsh-skill-office/package.json')), 'assets'),
|
||||
const assets = options.skillSource ?? join(dirname(createRequire(import.meta.url).resolve('@deepseek-ai/dsh-skill-office/package.json')), 'assets')
|
||||
await prepareOfficeSkillAssets(assets,
|
||||
join(paths.runtime, 'office-skills'))
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user