v0.8.1: add ISO by URL, zero-touch admin bootstrap
Ease-of-use pass inspired by Bootimus (Dnsmasq-PXE is a manual dnsmasq setup guide — nothing to adopt; OpenPXE already replaces that stack). Add ISO by URL: - New http-api `fetch` module: a small FetchJobs registry + a background streaming download (reqwest) that pipes a remote .iso through the same UploadHandle + introspection path as an upload, so a URL-fetched image classifies and gains boot entries identically. Progress is polled by the Storage view and rendered as rows, mirroring uploads. - Routes POST/GET/DELETE /api/isos/fetch. http/https only; .iso-only filename derived from Content-Disposition / URL basename with path traversal stripped; 16 GiB cap; cancel; credential-stripped URL display. Operator-gated, no boot-time outbound — offline boot is untouched. - Storage upload card gains an "Or add by URL" field with progress + cancel. Zero-touch admin bootstrap: - OPENPXE_ADMIN_USERNAME + OPENPXE_ADMIN_PASSWORD (or _PASSWORD_FILE for Docker/K8s secrets) auto-create the admin on first run, so a fresh container is usable with no setup wizard. Seeds the first run only — a lingering env var can't reset a rotated password. Tests: URL parse / filename / Content-Disposition unit tests + a wiremock end-to-end fetch-into-store integration test. clippy/fmt/node clean. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
1c262a6d61
commit
5f98e6e03f
Generated
+8
-8
@@ -2758,7 +2758,7 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
|
||||
|
||||
[[package]]
|
||||
name = "openpxe"
|
||||
version = "0.8.0"
|
||||
version = "0.8.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum",
|
||||
@@ -2778,7 +2778,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openpxe-core"
|
||||
version = "0.8.0"
|
||||
version = "0.8.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
@@ -2803,7 +2803,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openpxe-dhcp-proxy"
|
||||
version = "0.8.0"
|
||||
version = "0.8.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"dhcproto",
|
||||
@@ -2818,7 +2818,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openpxe-http-api"
|
||||
version = "0.8.0"
|
||||
version = "0.8.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum",
|
||||
@@ -2852,7 +2852,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openpxe-ipxe-assets"
|
||||
version = "0.8.0"
|
||||
version = "0.8.1"
|
||||
dependencies = [
|
||||
"openpxe-core",
|
||||
"rust-embed",
|
||||
@@ -2861,7 +2861,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openpxe-iso-store"
|
||||
version = "0.8.0"
|
||||
version = "0.8.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bcrypt",
|
||||
@@ -2887,7 +2887,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openpxe-tftp"
|
||||
version = "0.8.0"
|
||||
version = "0.8.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"openpxe-core",
|
||||
@@ -2899,7 +2899,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openpxe-webui"
|
||||
version = "0.8.0"
|
||||
version = "0.8.1"
|
||||
|
||||
[[package]]
|
||||
name = "p256"
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ members = [
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.8.0"
|
||||
version = "0.8.1"
|
||||
edition = "2021"
|
||||
rust-version = "1.95"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -65,7 +65,10 @@ OpenPXE collapses that whole stack into **one statically-linked binary in one co
|
||||
hierarchy — generated fresh on every request from current settings.
|
||||
|
||||
#### ISO management & remote libraries
|
||||
- **Drag-and-drop chunked uploads** that don't 502 on multi-GB images.
|
||||
- **Drag-and-drop chunked uploads** that don't 502 on multi-GB images — drop several at once
|
||||
and they upload concurrently.
|
||||
- **Add by URL** — paste an ISO link and the server streams it straight into storage and
|
||||
auto-detects it, with progress; no download-then-reupload.
|
||||
- **Automatic introspection** — detects the distro family and generates the right
|
||||
kernel+initrd or Windows `wimboot` chain. No manual config.
|
||||
- **Remote ISO libraries, streamed on demand** (no local cache) over **SMB, NFS, or SFTP** —
|
||||
@@ -88,6 +91,9 @@ OpenPXE collapses that whole stack into **one statically-linked binary in one co
|
||||
- **Prometheus `/metrics`**, a built-in operator **terminal**, live tracing log, and
|
||||
`/healthz` · `/readyz` probes.
|
||||
- **Layered config** — defaults → TOML file → `OPENPXE_*` env, in that order.
|
||||
- **Zero-touch first run** — set `OPENPXE_ADMIN_USERNAME` + `OPENPXE_ADMIN_PASSWORD` and a
|
||||
fresh container comes up with the admin already created, no setup wizard. Automate the rest
|
||||
from scripts/Postman with a per-install **API key** (`x-api-key`), shown in Settings → Advanced.
|
||||
|
||||
## Built in Rust
|
||||
|
||||
@@ -236,6 +242,8 @@ All settings have defaults and layer **defaults → TOML (`--config` / `OPENPXE_
|
||||
| `OPENPXE_ISO_DIR` | `/var/lib/openpxe/isos` | Uploaded ISOs |
|
||||
| `OPENPXE_WORK_DIR` | `/var/lib/openpxe/work` | Scratch, settings, share + branding state |
|
||||
| `OPENPXE_LOG` | `info,openpxe=info` | `tracing` filter |
|
||||
| `OPENPXE_ADMIN_USERNAME` | — | First-run only: with `OPENPXE_ADMIN_PASSWORD`, auto-creates the admin so no setup wizard is needed. Ignored once an admin exists. |
|
||||
| `OPENPXE_ADMIN_PASSWORD` | — | First-run admin password. Use `OPENPXE_ADMIN_PASSWORD_FILE` to read it from a file (Docker/K8s secret). |
|
||||
|
||||
## OpenShift
|
||||
|
||||
|
||||
@@ -94,6 +94,17 @@ pub fn build_router(state: AppState) -> Router {
|
||||
// JSON API.
|
||||
.route("/api/isos", get(api_list_isos).post(api_upload_iso))
|
||||
.route("/api/isos/{id}", delete(api_delete_iso))
|
||||
// v0.8.1: add ISO by URL — server-side streaming download + progress
|
||||
// polling. Static `fetch` coexists with `{id}` above (matchit
|
||||
// prioritizes the literal), same as `/api/queue/join` vs `{entry_id}`.
|
||||
.route(
|
||||
"/api/isos/fetch",
|
||||
get(crate::fetch::api_iso_fetch_list).post(crate::fetch::api_iso_fetch_start),
|
||||
)
|
||||
.route(
|
||||
"/api/isos/fetch/{id}",
|
||||
delete(crate::fetch::api_iso_fetch_cancel),
|
||||
)
|
||||
.route("/api/uploads", post(api_upload_begin))
|
||||
.route(
|
||||
"/api/uploads/{upload_id}",
|
||||
@@ -1857,6 +1868,12 @@ async fn api_docs() -> Json<serde_json::Value> {
|
||||
"summary": "Clear an ISO's boot password."},
|
||||
{"method": "PUT", "path": "/api/isos/{id}/category",
|
||||
"summary": "Set the menu category. Body: { \"category\": \"os\" | \"tools\" }."},
|
||||
{"method": "POST", "path": "/api/isos/fetch",
|
||||
"summary": "Add an ISO by URL. Body: { \"url\", \"filename\"? }. The server streams the .iso into storage and introspects it. Returns { fetch_id }."},
|
||||
{"method": "GET", "path": "/api/isos/fetch",
|
||||
"summary": "Poll URL-fetch progress — [{ id, filename, url, downloaded, total, state }]. Completed jobs are returned once."},
|
||||
{"method": "DELETE", "path": "/api/isos/fetch/{id}",
|
||||
"summary": "Cancel an in-flight URL fetch, or dismiss a finished/failed one."},
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
@@ -0,0 +1,485 @@
|
||||
//! Server-side "add ISO by URL" — stream a remote `.iso` straight into the
|
||||
//! store, reusing the chunked-upload handle + introspection pipeline so a
|
||||
//! URL-fetched image classifies and gains boot entries exactly like an
|
||||
//! uploaded one. A small in-memory job registry tracks progress; the web UI
|
||||
//! polls it and renders rows just like browser uploads.
|
||||
//!
|
||||
//! This is operator-initiated and auth-gated (`/api/*`), never runs at boot,
|
||||
//! and adds no CDN assets — so it doesn't touch OpenPXE's offline-boot
|
||||
//! guarantee. On an air-gapped network it simply goes unused (upload
|
||||
//! instead). It's the same class of optional outbound the server already
|
||||
//! makes for webhooks and the update check.
|
||||
|
||||
use crate::state::AppState;
|
||||
use axum::{
|
||||
extract::{Path, State},
|
||||
http::{header, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use futures::StreamExt;
|
||||
use openpxe_core::{Error, Result};
|
||||
use openpxe_iso_store::IsoStore;
|
||||
use parking_lot::Mutex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::json;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Hard ceiling on a URL-fetched image — matches the HTTP upload body cap
|
||||
/// (`DefaultBodyLimit` in `app.rs`).
|
||||
const MAX_ISO_BYTES: u64 = 16 * 1024 * 1024 * 1024;
|
||||
|
||||
/// How long a finished-with-error job lingers so the operator can read the
|
||||
/// failure before it's swept. Successful jobs are read-once (see
|
||||
/// [`FetchJobs::snapshot`]).
|
||||
const FAILED_TTL: Duration = Duration::from_mins(10);
|
||||
|
||||
#[derive(Clone)]
|
||||
enum Phase {
|
||||
Downloading,
|
||||
Done { iso_id: String },
|
||||
Failed { error: String },
|
||||
Canceled,
|
||||
}
|
||||
|
||||
struct Job {
|
||||
/// Best-known target filename (provisional from the URL, refined once
|
||||
/// the response headers arrive).
|
||||
filename: String,
|
||||
/// Display-safe source URL — any `user:pass@` userinfo is stripped so
|
||||
/// the UI/logs never echo embedded credentials.
|
||||
url: String,
|
||||
downloaded: u64,
|
||||
/// Total bytes from `Content-Length`, or `0` when the server didn't
|
||||
/// send one (progress then shows bytes-so-far without a percentage).
|
||||
total: u64,
|
||||
phase: Phase,
|
||||
cancel: bool,
|
||||
finished_at: Option<Instant>,
|
||||
}
|
||||
|
||||
/// One row in the fetch-progress list the UI polls.
|
||||
#[derive(Serialize)]
|
||||
pub struct JobDto {
|
||||
id: String,
|
||||
filename: String,
|
||||
url: String,
|
||||
downloaded: u64,
|
||||
total: u64,
|
||||
/// `downloading` | `done` | `failed` | `canceled`.
|
||||
state: &'static str,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
iso_id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
/// In-memory registry of in-flight and recently-finished URL fetches.
|
||||
/// Cheap to clone (Arc-shared); contention is nil (a handful of jobs, brief
|
||||
/// per-chunk progress bumps).
|
||||
#[derive(Clone, Default)]
|
||||
pub struct FetchJobs {
|
||||
inner: Arc<Mutex<HashMap<String, Arc<Mutex<Job>>>>>,
|
||||
}
|
||||
|
||||
impl FetchJobs {
|
||||
/// Validate the URL, register a job, and spawn the background download.
|
||||
/// Returns the job id. Fails fast on a bad/unsupported URL so the POST
|
||||
/// gets an immediate error instead of a job that dies a moment later.
|
||||
pub fn start(
|
||||
&self,
|
||||
store: IsoStore,
|
||||
raw_url: &str,
|
||||
filename_hint: Option<&str>,
|
||||
) -> Result<String> {
|
||||
let (url, display) = parse_and_sanitize(raw_url)?;
|
||||
// Provisional filename for the first render; the task refines it
|
||||
// from Content-Disposition / the post-redirect URL.
|
||||
let provisional = filename_hint
|
||||
.map(sanitize_filename)
|
||||
.or_else(|| basename(&url))
|
||||
.unwrap_or_else(|| "download.iso".to_string());
|
||||
|
||||
let id = Uuid::new_v4().simple().to_string();
|
||||
let job = Arc::new(Mutex::new(Job {
|
||||
filename: provisional,
|
||||
url: display,
|
||||
downloaded: 0,
|
||||
total: 0,
|
||||
phase: Phase::Downloading,
|
||||
cancel: false,
|
||||
finished_at: None,
|
||||
}));
|
||||
self.inner.lock().insert(id.clone(), job.clone());
|
||||
|
||||
let hint = filename_hint.map(str::to_string);
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = download(&store, &job, url, hint).await {
|
||||
let mut g = job.lock();
|
||||
// A cancel flips the phase itself; don't overwrite it.
|
||||
if !matches!(g.phase, Phase::Canceled) {
|
||||
g.phase = Phase::Failed {
|
||||
error: format!("{e}"),
|
||||
};
|
||||
}
|
||||
g.finished_at = Some(Instant::now());
|
||||
}
|
||||
});
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Snapshot every job for the UI, then sweep the terminal ones:
|
||||
/// `Done`/`Canceled` are **read-once** (removed after this call, so the
|
||||
/// UI reacts to completion exactly once and never loops on a lingering
|
||||
/// "done" row), while `Failed` is retained until [`FAILED_TTL`] so the
|
||||
/// error stays visible.
|
||||
pub fn snapshot(&self) -> Vec<JobDto> {
|
||||
let mut g = self.inner.lock();
|
||||
let mut out = Vec::with_capacity(g.len());
|
||||
for (id, job) in g.iter() {
|
||||
let j = job.lock();
|
||||
let (state, iso_id, error) = match &j.phase {
|
||||
Phase::Downloading => ("downloading", None, None),
|
||||
Phase::Done { iso_id } => ("done", Some(iso_id.clone()), None),
|
||||
Phase::Failed { error } => ("failed", None, Some(error.clone())),
|
||||
Phase::Canceled => ("canceled", None, None),
|
||||
};
|
||||
out.push(JobDto {
|
||||
id: id.clone(),
|
||||
filename: j.filename.clone(),
|
||||
url: j.url.clone(),
|
||||
downloaded: j.downloaded,
|
||||
total: j.total,
|
||||
state,
|
||||
iso_id,
|
||||
error,
|
||||
});
|
||||
}
|
||||
let now = Instant::now();
|
||||
g.retain(|_, job| {
|
||||
let j = job.lock();
|
||||
match &j.phase {
|
||||
Phase::Downloading => true,
|
||||
Phase::Done { .. } | Phase::Canceled => false, // read-once
|
||||
Phase::Failed { .. } => j
|
||||
.finished_at
|
||||
.is_none_or(|t| now.duration_since(t) < FAILED_TTL),
|
||||
}
|
||||
});
|
||||
out
|
||||
}
|
||||
|
||||
/// Cancel an in-flight download, or dismiss a terminal one. Returns
|
||||
/// `true` if a job with that id existed.
|
||||
pub fn cancel(&self, id: &str) -> bool {
|
||||
let g = self.inner.lock();
|
||||
let Some(job) = g.get(id) else { return false };
|
||||
let mut j = job.lock();
|
||||
if matches!(j.phase, Phase::Downloading) {
|
||||
j.cancel = true; // the download loop checks this each chunk
|
||||
} else {
|
||||
drop(j);
|
||||
drop(g);
|
||||
self.inner.lock().remove(id);
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// The background download: GET the URL, derive + validate the filename,
|
||||
/// then stream the body through an `UploadHandle` (which hashes, writes the
|
||||
/// `.partial`, and on `finish` renames + introspects).
|
||||
async fn download(
|
||||
store: &IsoStore,
|
||||
job: &Arc<Mutex<Job>>,
|
||||
url: reqwest::Url,
|
||||
filename_hint: Option<String>,
|
||||
) -> Result<()> {
|
||||
let client = reqwest::Client::builder()
|
||||
.connect_timeout(Duration::from_secs(15))
|
||||
.user_agent(concat!("OpenPXE/", env!("CARGO_PKG_VERSION")))
|
||||
.build()
|
||||
.map_err(|e| Error::Other(e.into()))?;
|
||||
|
||||
let resp = client
|
||||
.get(url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::Invalid(format!("request failed: {e}")))?;
|
||||
if !resp.status().is_success() {
|
||||
return Err(Error::Invalid(format!("server returned {}", resp.status())));
|
||||
}
|
||||
|
||||
let cd = resp
|
||||
.headers()
|
||||
.get(header::CONTENT_DISPOSITION)
|
||||
.and_then(|v| v.to_str().ok());
|
||||
let filename = pick_filename(filename_hint.as_deref(), cd, resp.url())?;
|
||||
let total = resp.content_length().unwrap_or(0);
|
||||
if total > MAX_ISO_BYTES {
|
||||
return Err(Error::Invalid(format!(
|
||||
"declared size {total} exceeds the {MAX_ISO_BYTES}-byte cap"
|
||||
)));
|
||||
}
|
||||
{
|
||||
let mut g = job.lock();
|
||||
g.filename.clone_from(&filename);
|
||||
g.total = total;
|
||||
}
|
||||
|
||||
let mut handle = store.begin_upload(&filename).await?;
|
||||
let mut stream = resp.bytes_stream();
|
||||
let mut received: u64 = 0;
|
||||
while let Some(item) = stream.next().await {
|
||||
if job.lock().cancel {
|
||||
let _ = handle.abort().await;
|
||||
job.lock().phase = Phase::Canceled;
|
||||
job.lock().finished_at = Some(Instant::now());
|
||||
return Ok(());
|
||||
}
|
||||
let chunk = item.map_err(|e| Error::Invalid(format!("transfer error: {e}")))?;
|
||||
received += chunk.len() as u64;
|
||||
if received > MAX_ISO_BYTES {
|
||||
let _ = handle.abort().await;
|
||||
return Err(Error::Invalid(format!(
|
||||
"download exceeded the {MAX_ISO_BYTES}-byte cap"
|
||||
)));
|
||||
}
|
||||
if let Err(e) = handle.write_chunk(&chunk).await {
|
||||
let _ = handle.abort().await;
|
||||
return Err(e);
|
||||
}
|
||||
job.lock().downloaded = received;
|
||||
}
|
||||
|
||||
let meta = handle.finish(store).await?;
|
||||
tracing::info!(
|
||||
target: "openpxe::http::fetch",
|
||||
filename = %filename, bytes = received, family = ?meta.introspection.family,
|
||||
"fetched ISO from URL"
|
||||
);
|
||||
let mut g = job.lock();
|
||||
g.downloaded = received;
|
||||
g.phase = Phase::Done { iso_id: meta.id };
|
||||
g.finished_at = Some(Instant::now());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Parse the URL, require an `http`/`https` scheme (no `file:`/`gopher:`/…),
|
||||
/// and return it alongside a credential-stripped display form.
|
||||
fn parse_and_sanitize(raw: &str) -> Result<(reqwest::Url, String)> {
|
||||
let url = reqwest::Url::parse(raw.trim())
|
||||
.map_err(|_| Error::Invalid("not a valid URL".to_string()))?;
|
||||
if !matches!(url.scheme(), "http" | "https") {
|
||||
return Err(Error::Invalid(
|
||||
"only http:// and https:// URLs are accepted".to_string(),
|
||||
));
|
||||
}
|
||||
let mut display = url.clone();
|
||||
let _ = display.set_username("");
|
||||
let _ = display.set_password(None);
|
||||
Ok((url, display.to_string()))
|
||||
}
|
||||
|
||||
/// Choose the target filename: explicit hint > `Content-Disposition` >
|
||||
/// post-redirect URL basename. Must end in `.iso` (case-insensitive).
|
||||
fn pick_filename(
|
||||
explicit: Option<&str>,
|
||||
content_disposition: Option<&str>,
|
||||
final_url: &reqwest::Url,
|
||||
) -> Result<String> {
|
||||
let candidate = explicit
|
||||
.map(sanitize_filename)
|
||||
.or_else(|| content_disposition.and_then(filename_from_disposition))
|
||||
.or_else(|| basename(final_url))
|
||||
.ok_or_else(|| Error::Invalid("could not determine a filename".to_string()))?;
|
||||
if !candidate.to_ascii_lowercase().ends_with(".iso") {
|
||||
return Err(Error::Invalid(format!(
|
||||
"URL does not point at an .iso (got '{candidate}')"
|
||||
)));
|
||||
}
|
||||
Ok(candidate)
|
||||
}
|
||||
|
||||
/// Last path segment of a URL, percent-decoded and reduced to a bare
|
||||
/// filename. `None` for a pathless URL.
|
||||
fn basename(url: &reqwest::Url) -> Option<String> {
|
||||
let seg = url.path_segments()?.next_back()?;
|
||||
if seg.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let decoded = percent_decode(seg);
|
||||
Some(sanitize_filename(&decoded))
|
||||
}
|
||||
|
||||
/// Pull `filename="x.iso"` (or bare `filename=x.iso`) out of a
|
||||
/// `Content-Disposition` header. RFC 5987 `filename*` is ignored — the
|
||||
/// common case is enough, and the URL basename is the fallback.
|
||||
fn filename_from_disposition(cd: &str) -> Option<String> {
|
||||
let idx = cd.to_ascii_lowercase().find("filename=")?;
|
||||
let rest = &cd[idx + "filename=".len()..];
|
||||
let val = rest.trim_start().trim_start_matches('"');
|
||||
let end = val.find(['"', ';']).unwrap_or(val.len());
|
||||
let name = val[..end].trim();
|
||||
if name.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(sanitize_filename(name))
|
||||
}
|
||||
}
|
||||
|
||||
/// Reduce any path-ish string to a safe bare filename: last component only,
|
||||
/// no `/`, `\`, or NULs. Prevents a crafted `Content-Disposition`/URL from
|
||||
/// escaping the ISO directory.
|
||||
fn sanitize_filename(s: &str) -> String {
|
||||
s.rsplit(['/', '\\'])
|
||||
.next()
|
||||
.unwrap_or(s)
|
||||
.replace('\0', "")
|
||||
.trim()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// Minimal percent-decoding for a single path segment (enough for `%20`
|
||||
/// spaces in an ISO name); leaves malformed escapes untouched.
|
||||
fn percent_decode(s: &str) -> String {
|
||||
let bytes = s.as_bytes();
|
||||
let mut out = Vec::with_capacity(bytes.len());
|
||||
let mut i = 0;
|
||||
while i < bytes.len() {
|
||||
if bytes[i] == b'%' && i + 2 < bytes.len() {
|
||||
if let (Some(h), Some(l)) = (hexval(bytes[i + 1]), hexval(bytes[i + 2])) {
|
||||
out.push(h << 4 | l);
|
||||
i += 3;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
out.push(bytes[i]);
|
||||
i += 1;
|
||||
}
|
||||
String::from_utf8_lossy(&out).into_owned()
|
||||
}
|
||||
|
||||
fn hexval(b: u8) -> Option<u8> {
|
||||
match b {
|
||||
b'0'..=b'9' => Some(b - b'0'),
|
||||
b'a'..=b'f' => Some(b - b'a' + 10),
|
||||
b'A'..=b'F' => Some(b - b'A' + 10),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Handlers ──────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct FetchBody {
|
||||
pub url: String,
|
||||
#[serde(default)]
|
||||
pub filename: Option<String>,
|
||||
}
|
||||
|
||||
/// `POST /api/isos/fetch` — start a URL download.
|
||||
pub async fn api_iso_fetch_start(
|
||||
State(state): State<AppState>,
|
||||
Json(body): Json<FetchBody>,
|
||||
) -> Response {
|
||||
match state
|
||||
.fetch_jobs
|
||||
.start(state.iso_store.clone(), &body.url, body.filename.as_deref())
|
||||
{
|
||||
Ok(id) => (StatusCode::ACCEPTED, Json(json!({ "fetch_id": id }))).into_response(),
|
||||
Err(Error::Invalid(msg)) => {
|
||||
(StatusCode::BAD_REQUEST, Json(json!({ "error": msg }))).into_response()
|
||||
}
|
||||
Err(e) => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(json!({ "error": format!("{e}") })),
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /api/isos/fetch` — poll progress. Successful jobs appear once.
|
||||
pub async fn api_iso_fetch_list(State(state): State<AppState>) -> Response {
|
||||
(
|
||||
StatusCode::OK,
|
||||
Json(json!({ "jobs": state.fetch_jobs.snapshot() })),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
/// `DELETE /api/isos/fetch/{id}` — cancel an in-flight download or dismiss a
|
||||
/// finished/failed row.
|
||||
pub async fn api_iso_fetch_cancel(
|
||||
State(state): State<AppState>,
|
||||
Path(id): Path<String>,
|
||||
) -> Response {
|
||||
if state.fetch_jobs.cancel(&id) {
|
||||
StatusCode::NO_CONTENT.into_response()
|
||||
} else {
|
||||
(
|
||||
StatusCode::NOT_FOUND,
|
||||
Json(json!({ "error": "no such fetch job" })),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn scheme_is_restricted_and_credentials_are_stripped() {
|
||||
assert!(parse_and_sanitize("file:///etc/passwd").is_err());
|
||||
assert!(parse_and_sanitize("gopher://x/1").is_err());
|
||||
assert!(parse_and_sanitize("not a url").is_err());
|
||||
let (_, display) = parse_and_sanitize("https://user:[email protected]/a.iso").unwrap();
|
||||
assert!(
|
||||
!display.contains("secret"),
|
||||
"credentials must be stripped: {display}"
|
||||
);
|
||||
assert!(display.starts_with("https://example.com/"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filename_prefers_explicit_then_disposition_then_url() {
|
||||
let u = reqwest::Url::parse("https://example.com/path/final.iso").unwrap();
|
||||
// Explicit wins, and path traversal is stripped.
|
||||
assert_eq!(
|
||||
pick_filename(Some("../../evil/custom.iso"), None, &u).unwrap(),
|
||||
"custom.iso"
|
||||
);
|
||||
// Content-Disposition next.
|
||||
assert_eq!(
|
||||
pick_filename(None, Some(r#"attachment; filename="rescue.iso""#), &u).unwrap(),
|
||||
"rescue.iso"
|
||||
);
|
||||
// URL basename fallback (with percent-decoding).
|
||||
let sp = reqwest::Url::parse("https://example.com/System%20Rescue.iso").unwrap();
|
||||
assert_eq!(pick_filename(None, None, &sp).unwrap(), "System Rescue.iso");
|
||||
// Non-.iso is rejected.
|
||||
assert!(pick_filename(
|
||||
None,
|
||||
None,
|
||||
&reqwest::Url::parse("https://x/y.tar.gz").unwrap()
|
||||
)
|
||||
.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disposition_parsing_handles_quotes_and_bare() {
|
||||
assert_eq!(
|
||||
filename_from_disposition(r#"attachment; filename="a.iso"; size=1"#).as_deref(),
|
||||
Some("a.iso")
|
||||
);
|
||||
assert_eq!(
|
||||
filename_from_disposition("inline; filename=b.iso").as_deref(),
|
||||
Some("b.iso")
|
||||
);
|
||||
assert_eq!(filename_from_disposition("attachment").as_deref(), None);
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@
|
||||
pub mod app;
|
||||
pub mod auth;
|
||||
pub mod error;
|
||||
pub mod fetch;
|
||||
pub mod grub_script;
|
||||
pub mod ipxe_script;
|
||||
pub mod log_stream;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use crate::auth::SessionStore;
|
||||
use crate::fetch::FetchJobs;
|
||||
use crate::saml_routes::SamlRuntime;
|
||||
use crate::uploads::UploadSessions;
|
||||
use openpxe_core::{
|
||||
@@ -108,6 +109,10 @@ pub struct AppState {
|
||||
/// through `IsoStore`, but the UI uses sessions so large ISO transfers
|
||||
/// can show deterministic progress and leave visible partial files.
|
||||
pub uploads: UploadSessions,
|
||||
/// v0.8.1: server-side "add ISO by URL" jobs. Background downloads
|
||||
/// streamed straight into the store (reusing the upload handle +
|
||||
/// introspection); the Storage view polls their progress.
|
||||
pub fetch_jobs: FetchJobs,
|
||||
/// Live log bus consumed by the Terminal tab via SSE. Operator-issued
|
||||
/// terminal commands also push synthetic lines onto it so the tail
|
||||
/// shows them inline.
|
||||
|
||||
@@ -126,6 +126,7 @@ async fn build_state() -> (AppState, tempfile::TempDir) {
|
||||
sftp_shares,
|
||||
unattended,
|
||||
uploads: openpxe_http_api::uploads::UploadSessions::default(),
|
||||
fetch_jobs: openpxe_http_api::fetch::FetchJobs::default(),
|
||||
log_bus,
|
||||
started_at: time::OffsetDateTime::now_utc(),
|
||||
public_base_url: "http://127.0.0.1".into(),
|
||||
@@ -191,6 +192,55 @@ async fn api_key_authenticates_gated_endpoints() {
|
||||
assert_eq!(res.status(), StatusCode::OK, "valid key must authenticate");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fetch_iso_by_url_downloads_into_store() {
|
||||
// v0.8.1: add-ISO-by-URL. Serve a real ISO over HTTP, POST its URL, poll
|
||||
// the fetch registry until the background download finishes, then assert
|
||||
// the image landed in the store (classified like an upload).
|
||||
use wiremock::matchers::{method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
let (state, _dir) = build_state().await;
|
||||
let app = build_router(state);
|
||||
|
||||
let iso = fake_alpine_iso();
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/rescue.iso"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_bytes(iso))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let url = format!("{}/rescue.iso", server.uri());
|
||||
|
||||
let (code, _) = post_json(&app, "/api/isos/fetch", &format!(r#"{{"url":"{url}"}}"#)).await;
|
||||
assert_eq!(code, StatusCode::ACCEPTED, "fetch should start");
|
||||
|
||||
// Bounded poll for completion (the download runs on a spawned task).
|
||||
let mut done = false;
|
||||
for _ in 0..100 {
|
||||
let (_, body) = get(&app, "/api/isos/fetch").await;
|
||||
let v: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
let jobs = v["jobs"].as_array().cloned().unwrap_or_default();
|
||||
assert!(
|
||||
!jobs.iter().any(|j| j["state"] == "failed"),
|
||||
"fetch failed: {jobs:?}"
|
||||
);
|
||||
if jobs.iter().any(|j| j["state"] == "done") {
|
||||
done = true;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
}
|
||||
assert!(done, "fetch did not complete in time");
|
||||
|
||||
// The fetched ISO is now in the store under the URL basename.
|
||||
let (_, body) = get(&app, "/api/isos").await;
|
||||
assert!(
|
||||
String::from_utf8_lossy(&body).contains("rescue.iso"),
|
||||
"fetched ISO should appear in /api/isos"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn health_and_ready_endpoints() {
|
||||
let (state, _dir) = build_state().await;
|
||||
|
||||
@@ -120,6 +120,38 @@ async fn main() -> anyhow::Result<()> {
|
||||
let boot_log = openpxe_core::BootLog::load_or_default(&config.paths.work_dir);
|
||||
let branding = openpxe_core::BrandingStore::load_or_default(&config.paths.work_dir);
|
||||
let admin = openpxe_core::AdminStore::load_or_default(&config.paths.work_dir);
|
||||
// v0.8.1: zero-touch first-run bootstrap. If no admin exists yet and the
|
||||
// operator supplied OPENPXE_ADMIN_USERNAME + OPENPXE_ADMIN_PASSWORD (or
|
||||
// …_PASSWORD_FILE, for Docker/K8s secrets), create the admin now so a
|
||||
// fresh container is usable without the web setup wizard. Seeds the
|
||||
// first run only — once an admin exists (including one made in the UI)
|
||||
// this is a no-op, so a lingering env var can't reset a rotated password.
|
||||
if !admin.is_configured() {
|
||||
if let Ok(username) = std::env::var("OPENPXE_ADMIN_USERNAME") {
|
||||
let password = std::env::var("OPENPXE_ADMIN_PASSWORD_FILE")
|
||||
.ok()
|
||||
.and_then(|p| std::fs::read_to_string(p).ok())
|
||||
.map(|s| s.trim_end_matches(['\n', '\r']).to_string())
|
||||
.or_else(|| std::env::var("OPENPXE_ADMIN_PASSWORD").ok());
|
||||
if let Some(password) = password {
|
||||
match admin.bootstrap(&username, &password) {
|
||||
Ok(p) => tracing::info!(
|
||||
target: "openpxe::auth", username = %p.username,
|
||||
"admin bootstrapped from environment"
|
||||
),
|
||||
Err(e) => tracing::warn!(
|
||||
target: "openpxe::auth",
|
||||
"env admin bootstrap failed ({e}); use the web setup wizard"
|
||||
),
|
||||
}
|
||||
} else {
|
||||
tracing::warn!(
|
||||
target: "openpxe::auth",
|
||||
"OPENPXE_ADMIN_USERNAME set without OPENPXE_ADMIN_PASSWORD[_FILE]; skipping bootstrap"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let sso = openpxe_core::SsoStore::load_or_default(&config.paths.work_dir);
|
||||
let notify = openpxe_core::NotifyStore::load_or_default(&config.paths.work_dir);
|
||||
let api_key = openpxe_core::ApiKeyStore::load_or_init(&config.paths.work_dir);
|
||||
@@ -213,6 +245,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
sftp_shares: sftp_shares.clone(),
|
||||
unattended: unattended.clone(),
|
||||
uploads: openpxe_http_api::uploads::UploadSessions::default(),
|
||||
fetch_jobs: openpxe_http_api::fetch::FetchJobs::default(),
|
||||
log_bus: log_bus.clone(),
|
||||
started_at: time::OffsetDateTime::now_utc(),
|
||||
public_base_url: public_base_url.clone(),
|
||||
|
||||
+75
-2
@@ -768,6 +768,75 @@
|
||||
}
|
||||
}
|
||||
|
||||
// v0.8.1: add ISO by URL. Paste a link and the server streams it
|
||||
// straight into the store and auto-detects it — no download-then-
|
||||
// reupload. Progress polls /api/isos/fetch and shows rows below,
|
||||
// mirroring uploads. On an air-gapped network, use the drop zone.
|
||||
const urlInput = el('input', {type:'url', id:'iso-url', style:'flex:1',
|
||||
placeholder:'https://example.com/systemrescue.iso'});
|
||||
const fetchBtn = el('button', {class:'ghost', type:'button', style:'margin-left:8px'}, 'Fetch');
|
||||
const fetchMsg = el('div', {class:'msg', style:'margin-top:6px'});
|
||||
const fetchList = el('div', {id:'fetches', style:'display:grid;gap:12px;margin-top:12px'});
|
||||
const urlRow = el('div', {style:'margin-top:14px'}, [
|
||||
el('label', {class:'field', style:'margin-bottom:0'}, [
|
||||
el('span', {class:'name'}, 'Or add by URL'),
|
||||
el('div', {style:'display:flex;align-items:center'}, [urlInput, fetchBtn]),
|
||||
el('span', {class:'hint'},
|
||||
'The server downloads the .iso into storage and auto-detects it — same result as a drag-drop. Any http(s) .iso link works.'),
|
||||
]),
|
||||
fetchMsg,
|
||||
]);
|
||||
|
||||
let fetchTimer = null;
|
||||
const renderFetchRows = (jobs) => {
|
||||
fetchList.replaceChildren(...jobs.map(j => {
|
||||
const pct = j.total > 0 ? Math.min(100, (j.downloaded / j.total) * 100)
|
||||
: (j.state === 'done' ? 100 : 0);
|
||||
let text, cls = '';
|
||||
if (j.state === 'downloading')
|
||||
text = 'Downloading ' + fmtBytes(j.downloaded) +
|
||||
(j.total ? ' of ' + fmtBytes(j.total) + ' (' + pct.toFixed(0) + '%)' : '');
|
||||
else if (j.state === 'done') { text = 'Downloaded and analyzed.'; cls = 'ok'; }
|
||||
else if (j.state === 'failed') { text = 'Failed: ' + (j.error || 'unknown error'); cls = 'err'; }
|
||||
else text = 'Canceled — partial discarded.';
|
||||
const btn = el('button', {class:'danger', type:'button', style:'margin-top:8px'},
|
||||
j.state === 'downloading' ? 'Cancel' : 'Dismiss');
|
||||
btn.onclick = async () => {
|
||||
try { await fetch('/api/isos/fetch/' + encodeURIComponent(j.id), {method:'DELETE'}); } catch (_) {}
|
||||
pollFetches();
|
||||
};
|
||||
return el('div', {}, [
|
||||
el('div', {style:'font-weight:600;font-size:13px;margin-bottom:6px;word-break:break-all'},
|
||||
j.filename + ' · ' + j.url),
|
||||
el('div', {class:'progress' + (j.state === 'downloading' ? ' active' : '')},
|
||||
el('div', {class:'bar', style:'width:' + pct.toFixed(1) + '%'})),
|
||||
el('div', {class:'msg ' + cls}, text),
|
||||
btn,
|
||||
]);
|
||||
}));
|
||||
};
|
||||
async function pollFetches() {
|
||||
if (fetchTimer) { clearTimeout(fetchTimer); fetchTimer = null; }
|
||||
let jobs = [];
|
||||
try { jobs = (await getJSON('/api/isos/fetch')).jobs || []; } catch (_) {}
|
||||
renderFetchRows(jobs);
|
||||
// A successful fetch is read-once on the server, so refreshing here
|
||||
// shows the new image and won't re-trigger on the next poll. Keep
|
||||
// polling only while a download is still in flight.
|
||||
if (jobs.some(j => j.state === 'done')) { render('storage'); return; }
|
||||
if (jobs.some(j => j.state === 'downloading')) fetchTimer = setTimeout(pollFetches, 1500);
|
||||
}
|
||||
async function startFetch() {
|
||||
const url = urlInput.value.trim();
|
||||
if (!url) return;
|
||||
fetchMsg.textContent = 'Starting…'; fetchMsg.className = 'msg';
|
||||
const r = await postJSON('/api/isos/fetch', { url });
|
||||
if (r.ok) { urlInput.value = ''; fetchMsg.textContent = ''; pollFetches(); }
|
||||
else { fetchMsg.textContent = 'Could not start: ' + (await r.text()).slice(0, 160); fetchMsg.className = 'msg err'; }
|
||||
}
|
||||
fetchBtn.onclick = startFetch;
|
||||
urlInput.addEventListener('keydown', e => { if (e.key === 'Enter') { e.preventDefault(); startFetch(); } });
|
||||
|
||||
// ── ISO table (mixed local + SMB) ──
|
||||
// Each row gets a "Password" cell that toggles a small inline
|
||||
// editor (a checkbox + a password field + Save button) inside the
|
||||
@@ -1397,11 +1466,11 @@
|
||||
]),
|
||||
]);
|
||||
|
||||
return el('div', {}, [el('div', {class:'grid'}, [
|
||||
const root = el('div', {}, [el('div', {class:'grid'}, [
|
||||
diskCard,
|
||||
el('div', {class:'card'}, [
|
||||
el('header', {}, el('h2', {}, 'Upload ISO')),
|
||||
el('div', {class:'body'}, [drop, file, uploadsList]),
|
||||
el('div', {class:'body'}, [drop, file, urlRow, uploadsList, fetchList]),
|
||||
]),
|
||||
// v0.5.1: SMB + NFS unified into one "Remote shares" card with a
|
||||
// protocol dropdown. Backend endpoints are unchanged; this is a
|
||||
@@ -1440,6 +1509,10 @@
|
||||
isoPager,
|
||||
]),
|
||||
]), unattendedAdvanced]);
|
||||
// v0.8.1: resume/kick URL-fetch progress polling; stop it on view swap.
|
||||
root._cleanup = () => { if (fetchTimer) clearTimeout(fetchTimer); };
|
||||
pollFetches();
|
||||
return root;
|
||||
},
|
||||
|
||||
hosts: async () => {
|
||||
|
||||
Reference in New Issue
Block a user