From 55ace0c25ceb1c143f87f787bf87e60ffdf73b51 Mon Sep 17 00:00:00 2001 From: Miles Ward Date: Sun, 24 May 2026 13:45:35 -0400 Subject: [PATCH] v0.4.1: harden ISO uploads and beta UI polish Add browser-safe chunked ISO uploads with progress, partial-file visibility, offset validation, and abort cleanup while keeping the legacy multipart endpoint for API clients. Record host-log validation coverage, keep the queue/status UI copy clean, move release docs to 0.4.1, and tighten the dark theme to a near-black Netbox-style palette. --- Cargo.toml | 2 +- README.md | 21 ++-- crates/core/src/host_bindings.rs | 9 +- crates/http-api/Cargo.toml | 1 + crates/http-api/src/app.rs | 157 ++++++++++++++++++++++-- crates/http-api/src/ipxe_script.rs | 4 +- crates/http-api/src/lib.rs | 1 + crates/http-api/src/state.rs | 5 + crates/http-api/src/uploads.rs | 180 ++++++++++++++++++++++++++++ crates/http-api/tests/full_flow.rs | 130 +++++++++++++++++++- crates/iso-store/src/store.rs | 16 +++ crates/openpxe/src/main.rs | 3 +- crates/webui/src/app.css | 10 +- crates/webui/src/app.js | 120 +++++++++++-------- crates/webui/src/index.html | 2 +- deploy/openshift/30-deployment.yaml | 2 +- deploy/unraid/README.md | 14 +-- docs/NEXT_PHASE.md | 2 +- docs/architecture.md | 7 +- 19 files changed, 579 insertions(+), 107 deletions(-) create mode 100644 crates/http-api/src/uploads.rs diff --git a/Cargo.toml b/Cargo.toml index fdef9aa..5eca210 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "0.4.0" +version = "0.4.1" edition = "2021" rust-version = "1.80" license = "MIT OR Apache-2.0" diff --git a/README.md b/README.md index d604fca..758abaf 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,11 @@ Container-native PXE boot server. A Rust reimplementation of for Docker/OCI and OpenShift. Upload `.iso` files via the web UI; network clients PXE-boot them. -> **Status:** v0.3.2 / pre-beta. Phases 1–5 complete: full PXE stack, +> **Status:** v0.4.1 / pre-beta. Phases 1–5 complete: full PXE stack, > Queued Deployment queue, NFS-share ISO sources, live tracing log + an -> operator terminal, per-MAC host bindings (Tinkerbell-style), -> Prometheus `/metrics`, light/dark theme toggle, animated OpenPXE -> imaging-progress widget, and per-ISO boot passwords. The test suite and +> operator terminal, per-MAC host bindings, Prometheus `/metrics`, +> light/dark theme toggle, animated OpenPXE imaging-progress widget, +> chunked ISO uploads, and per-ISO boot passwords. The test suite and > clippy are part of the release checklist. Ready for real-hardware validation. ## Design non-negotiables @@ -51,8 +51,7 @@ clients PXE-boot them. widget when devices are imaging. All assets served from the binary — no external requests. 8. **Per-MAC host bindings.** Pin a MAC to a boot target and the client - skips the menu, chains straight through. Inspired by Tinkerbell's - `smee` MAC-prepended URL pattern. + skips the menu, chains straight through. 9. **Prometheus metrics** at `/metrics` — DHCP replies by arch, TFTP transfer counts and bytes, HTTP request counts by route, queue / imaging gauges, uptime, build info. Plain text exposition format, @@ -81,7 +80,7 @@ skip TFTP and respond with an HTTP URL. ./scripts/fetch-ipxe.sh # 2. Build the container image (~3 min first time). -docker buildx build -f deploy/docker/Dockerfile -t openpxe:0.3.2 --load . +docker buildx build -f deploy/docker/Dockerfile -t openpxe:0.4.1 --load . # 3. Run it on the box plugged into your PXE network. Set PUBLIC_IP to # this host's LAN address so advertised iPXE URLs are reachable. @@ -91,7 +90,7 @@ docker run -d --name openpxe \ -e OPENPXE_DHCP_MODE=proxy \ -v $PWD/data/isos:/var/lib/openpxe/isos \ -v $PWD/data/work:/var/lib/openpxe/work \ - openpxe:0.3.2 + openpxe:0.4.1 # 4. Open the UI and drop an ISO in. open http://10.0.0.5 @@ -122,7 +121,7 @@ docker buildx create --name openpxe-multi --driver docker-container --use # Build + push both linux/amd64 and linux/arm64 under one tag. docker buildx build --builder openpxe-multi \ --platform linux/amd64,linux/arm64 \ - -t ghcr.io/YOUR-ORG/openpxe:0.3.2 \ + -t ghcr.io/YOUR-ORG/openpxe:0.4.1 \ --push \ -f deploy/docker/Dockerfile . ``` @@ -155,10 +154,10 @@ docker run --rm \ -v /my/iso-library:/seed:ro \ -v openpxe-data:/var/lib/openpxe/isos \ -e OPENPXE_PUBLIC_IP=10.0.0.5 \ - openpxe:0.3.2 seed --from /seed + openpxe:0.4.1 seed --from /seed # Dry run first to see what would be imported: -docker run --rm -v /my/iso-library:/seed:ro openpxe:0.3.2 seed --from /seed --dry-run +docker run --rm -v /my/iso-library:/seed:ro openpxe:0.4.1 seed --from /seed --dry-run ``` ### Environment overrides diff --git a/crates/core/src/host_bindings.rs b/crates/core/src/host_bindings.rs index 05a7194..9e7be03 100644 --- a/crates/core/src/host_bindings.rs +++ b/crates/core/src/host_bindings.rs @@ -1,10 +1,9 @@ //! Per-MAC host bindings. //! -//! Inspired by the Tinkerbell `smee` "MAC-prepended URL" pattern: an -//! operator can attach a preferred boot target (a `BootEntry::id`) to a -//! specific MAC address. When a client with that MAC arrives, the -//! top-level boot script chains straight to that target instead of -//! showing the interactive menu. +//! Operators can attach a preferred boot target (a `BootEntry::id`) to a +//! specific MAC address. When a client with that MAC arrives, the top-level +//! boot script chains straight to that target instead of showing the +//! interactive menu. //! //! Use cases: //! - "This rack of Dell servers always images with Ubuntu Server 24.04" diff --git a/crates/http-api/Cargo.toml b/crates/http-api/Cargo.toml index 89ba2fb..0969d47 100644 --- a/crates/http-api/Cargo.toml +++ b/crates/http-api/Cargo.toml @@ -31,6 +31,7 @@ bytes.workspace = true futures.workspace = true mime.workspace = true mime_guess.workspace = true +uuid.workspace = true [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "time"] } diff --git a/crates/http-api/src/app.rs b/crates/http-api/src/app.rs index 09c817f..52c5d1e 100644 --- a/crates/http-api/src/app.rs +++ b/crates/http-api/src/app.rs @@ -22,19 +22,19 @@ use crate::log_stream; use crate::state::AppState; use crate::terminal; use axum::{ - body::Body, + body::{Body, Bytes}, extract::{ConnectInfo, DefaultBodyLimit, Multipart, Path as AxumPath, Query, State}, http::{header, HeaderMap, HeaderValue, StatusCode}, response::{IntoResponse, Response}, - routing::{delete, get, post}, + routing::{delete, get, post, put}, Json, Router, }; -use openpxe_core::{BootEvent, ClientEvent, Settings}; -use std::net::SocketAddr; +use openpxe_core::{BootEvent, ClientEvent, Error, Settings}; use openpxe_ipxe_assets::asset_bytes; use openpxe_iso_store::{IsoMeta, NfsAddRequest}; use serde::Deserialize; use serde_json::json; +use std::net::SocketAddr; use std::time::Duration; use tokio::io::{AsyncReadExt, AsyncSeekExt}; use tower_http::trace::TraceLayer; @@ -62,6 +62,11 @@ pub fn build_router(state: AppState) -> Router { // JSON API. .route("/api/isos", get(api_list_isos).post(api_upload_iso)) .route("/api/isos/:id", delete(api_delete_iso)) + .route("/api/uploads", post(api_upload_begin)) + .route( + "/api/uploads/:upload_id", + put(api_upload_chunk).delete(api_upload_abort), + ) // Per-ISO password prompt. PUT body `{ "password": "..." }` // sets, `{ "password": null }` (or DELETE) clears. .route( @@ -88,11 +93,11 @@ pub fn build_router(state: AppState) -> Router { .route("/api/log/clear", post(log_stream::clear)) // Phase 4: operator terminal commands (whitelisted). .route("/api/terminal", post(terminal::run_command)) - // Phase 5: per-MAC host bindings (Tinkerbell-style). Operator + // Phase 5: per-MAC host bindings. Operator // pins a MAC to a boot entry; /boot.ipxe?mac=... chains directly. .route("/api/hosts", get(api_hosts_list).post(api_hosts_upsert)) .route("/api/hosts/:mac", delete(api_hosts_remove)) - // v0.4.0: rolling "host log" of boot events — what image actually + // Rolling "host log" of boot events: what image actually // started installing on what MAC/IP, and when. Persisted to disk. .route("/api/boot-log", get(api_boot_log)) // Phase 5: Prometheus scrape endpoint. Plain text exposition @@ -638,8 +643,7 @@ async fn api_upload_iso(State(state): State, mut multipart: Multipart) target: "openpxe::http::upload", filename = %filename, "rejecting non-.iso upload" ); - return (StatusCode::BAD_REQUEST, "only .iso uploads accepted") - .into_response(); + return (StatusCode::BAD_REQUEST, "only .iso uploads accepted").into_response(); } tracing::info!( target: "openpxe::http::upload", @@ -725,8 +729,7 @@ async fn api_upload_iso(State(state): State, mut multipart: Multipart) filename = %filename, error = %e, "finish failed (rename/introspect)" ); - return (StatusCode::INTERNAL_SERVER_ERROR, format!("{e}")) - .into_response(); + return (StatusCode::INTERNAL_SERVER_ERROR, format!("{e}")).into_response(); } }; tracing::info!( @@ -758,6 +761,140 @@ async fn api_upload_iso(State(state): State, mut multipart: Multipart) } } +#[derive(Debug, Deserialize)] +struct UploadBeginBody { + filename: String, + #[serde(default)] + size_bytes: Option, +} + +async fn api_upload_begin( + State(state): State, + Json(body): Json, +) -> Response { + match state + .uploads + .begin(&state.iso_store, &body.filename, body.size_bytes) + .await + { + Ok(started) => { + tracing::info!( + target: "openpxe::http::upload", + upload_id = %started.upload_id, + iso = %started.iso_id, + filename = %started.filename, + expected_size = ?body.size_bytes, + "chunked upload started" + ); + (StatusCode::CREATED, Json(started)).into_response() + } + Err(Error::Invalid(e)) if e.contains("already exists") => { + (StatusCode::CONFLICT, e).into_response() + } + Err(Error::Invalid(e)) => (StatusCode::BAD_REQUEST, e).into_response(), + Err(e) => { + tracing::error!(target: "openpxe::http::upload", error = %e, "chunked upload begin failed"); + (StatusCode::INTERNAL_SERVER_ERROR, format!("{e}")).into_response() + } + } +} + +async fn api_upload_chunk( + State(state): State, + AxumPath(upload_id): AxumPath, + headers: HeaderMap, + chunk: Bytes, +) -> Response { + let Some(offset) = parse_u64_header(&headers, "x-openpxe-upload-offset") else { + return ( + StatusCode::BAD_REQUEST, + "missing or invalid x-openpxe-upload-offset", + ) + .into_response(); + }; + let complete = bool_header(&headers, "x-openpxe-upload-complete"); + + match state + .uploads + .append(&state.iso_store, &upload_id, offset, chunk, complete) + .await + { + Ok(crate::uploads::UploadAppend::Progress { offset }) => ( + StatusCode::ACCEPTED, + Json(json!({ + "ok": true, + "upload_id": upload_id, + "offset": offset, + "complete": false, + })), + ) + .into_response(), + Ok(crate::uploads::UploadAppend::Complete { offset, iso }) => { + tracing::info!( + target: "openpxe::http::upload", + upload_id = %upload_id, + iso = %iso.id, + size = iso.size_bytes, + family = ?iso.introspection.family, + entries = iso.boot_entries.len(), + "chunked upload finished" + ); + ( + StatusCode::CREATED, + Json(json!({ + "ok": true, + "upload_id": upload_id, + "offset": offset, + "complete": true, + "iso": iso, + })), + ) + .into_response() + } + Err(Error::Invalid(e)) if e.starts_with("expected offset") => { + (StatusCode::CONFLICT, e).into_response() + } + Err(Error::Invalid(e)) if e.starts_with("no such upload") => { + (StatusCode::NOT_FOUND, e).into_response() + } + Err(Error::Invalid(e)) => (StatusCode::BAD_REQUEST, e).into_response(), + Err(e) => { + tracing::error!( + target: "openpxe::http::upload", + upload_id = %upload_id, + error = %e, + "chunked upload failed" + ); + (StatusCode::INTERNAL_SERVER_ERROR, format!("{e}")).into_response() + } + } +} + +async fn api_upload_abort( + State(state): State, + AxumPath(upload_id): AxumPath, +) -> Response { + match state.uploads.abort(&upload_id).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(Error::Invalid(e)) if e.starts_with("no such upload") => { + (StatusCode::NOT_FOUND, e).into_response() + } + Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, format!("{e}")).into_response(), + } +} + +fn parse_u64_header(headers: &HeaderMap, name: &'static str) -> Option { + headers.get(name)?.to_str().ok()?.trim().parse::().ok() +} + +fn bool_header(headers: &HeaderMap, name: &'static str) -> bool { + headers + .get(name) + .and_then(|v| v.to_str().ok()) + .map(str::trim) + .is_some_and(|v| matches!(v, "1" | "true" | "TRUE" | "yes" | "YES")) +} + // ─── health / readiness ─────────────────────────────────────────────────── async fn healthz() -> Response { diff --git a/crates/http-api/src/ipxe_script.rs b/crates/http-api/src/ipxe_script.rs index 1faf3c8..77ce2b6 100644 --- a/crates/http-api/src/ipxe_script.rs +++ b/crates/http-api/src/ipxe_script.rs @@ -183,7 +183,7 @@ pub fn render_family_menu(isos: &[IsoMeta], base_url: &str, is_windows: bool) -> "iseq ${{target}} back && chain {base}/boot.ipxe || goto menu" ); // Pass `?mac=${mac}` so the per-entry handler can record the booting - // client into the Host log (v0.4.0). iPXE substitutes `${mac}` before + // client into the Host log. iPXE substitutes `${mac}` before // the HTTP fetch; if the firmware can't resolve it the literal // `${mac}` is sent and the server treats it as "unknown". let _ = writeln!( @@ -469,7 +469,7 @@ pub fn render_password_prompt(entry_id: &str, iso_filename: &str, base_url: &str let _ = writeln!(s, ":submit"); let _ = writeln!(s, "echo Verifying..."); // Carry `mac=${mac}` alongside the token so a successful unlock - // records the actual client MAC into the Host log (v0.4.0). On + // records the actual client MAC into the Host log. On // older iPXE that can't resolve `${mac}` the server just stores it // as "unknown" rather than refusing to boot. let _ = writeln!( diff --git a/crates/http-api/src/lib.rs b/crates/http-api/src/lib.rs index 584c25e..ddf6f0c 100644 --- a/crates/http-api/src/lib.rs +++ b/crates/http-api/src/lib.rs @@ -19,6 +19,7 @@ pub mod iso_fs; pub mod log_stream; pub mod state; pub mod terminal; +pub mod uploads; pub use app::build_router; pub use state::AppState; diff --git a/crates/http-api/src/state.rs b/crates/http-api/src/state.rs index e622564..79b362c 100644 --- a/crates/http-api/src/state.rs +++ b/crates/http-api/src/state.rs @@ -1,3 +1,4 @@ +use crate::uploads::UploadSessions; use openpxe_core::{ BootLog, ClientRegistry, DeploymentQueue, HostBindings, LogBus, Metrics, SettingsStore, }; @@ -31,6 +32,10 @@ pub struct AppState { /// available in the runtime image. Surfaces errors per-mount rather /// than failing the global state. pub nfs: NfsManager, + /// Browser chunked upload state. Multipart uploads still go straight + /// through `IsoStore`, but the UI uses sessions so large ISO transfers + /// can show deterministic progress and leave visible partial files. + pub uploads: UploadSessions, /// Live log bus consumed by the Terminal tab via SSE. Operator-issued /// terminal commands also push synthetic lines onto it so the tail /// shows them inline. diff --git a/crates/http-api/src/uploads.rs b/crates/http-api/src/uploads.rs new file mode 100644 index 0000000..9141a01 --- /dev/null +++ b/crates/http-api/src/uploads.rs @@ -0,0 +1,180 @@ +//! Chunked upload sessions for browser-driven ISO uploads. +//! +//! The legacy multipart endpoint still exists for simple API clients, but +//! browsers get a better failure mode with raw chunks: progress advances after +//! each acknowledged write, partial files appear in the ISO directory +//! immediately, and reverse proxies are less likely to buffer an entire DVD +//! image before OpenPXE sees byte one. + +use bytes::Bytes; +use openpxe_core::{Error, Result}; +use openpxe_iso_store::{IsoMeta, IsoStore, UploadHandle}; +use serde::Serialize; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::Mutex; +use uuid::Uuid; + +const DEFAULT_CHUNK_SIZE: u64 = 8 * 1024 * 1024; + +#[derive(Clone, Default)] +pub struct UploadSessions { + inner: Arc>>>>, +} + +struct UploadSession { + filename: String, + expected_size: Option, + offset: u64, + handle: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub struct UploadStarted { + pub upload_id: String, + pub iso_id: String, + pub filename: String, + pub offset: u64, + pub chunk_size: u64, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum UploadAppend { + Progress { offset: u64 }, + Complete { offset: u64, iso: Box }, +} + +impl UploadSessions { + pub async fn begin( + &self, + store: &IsoStore, + filename: &str, + expected_size: Option, + ) -> Result { + if !filename.to_ascii_lowercase().ends_with(".iso") { + return Err(Error::Invalid("only .iso uploads accepted".to_string())); + } + + let handle = store.begin_upload(filename).await?; + let iso_id = handle.id.clone(); + let upload_id = Uuid::new_v4().to_string(); + let session = UploadSession { + filename: filename.to_string(), + expected_size, + offset: 0, + handle: Some(handle), + }; + + self.inner + .lock() + .await + .insert(upload_id.clone(), Arc::new(Mutex::new(session))); + + Ok(UploadStarted { + upload_id, + iso_id, + filename: filename.to_string(), + offset: 0, + chunk_size: DEFAULT_CHUNK_SIZE, + }) + } + + pub async fn append( + &self, + store: &IsoStore, + upload_id: &str, + offset: u64, + chunk: Bytes, + complete: bool, + ) -> Result { + let Some(session_lock) = self.inner.lock().await.get(upload_id).cloned() else { + return Err(Error::Invalid(format!("no such upload '{upload_id}'"))); + }; + + let mut session = session_lock.lock().await; + if session.offset != offset { + return Err(Error::Invalid(format!( + "expected offset {}, got {offset}", + session.offset + ))); + } + + let new_offset = session + .offset + .checked_add(chunk.len() as u64) + .ok_or_else(|| Error::Invalid("upload offset overflow".to_string()))?; + + if let Some(expected) = session.expected_size { + if new_offset > expected { + return Err(Error::Invalid(format!( + "chunk exceeds declared upload size {expected}" + ))); + } + } + + let Some(handle) = session.handle.as_mut() else { + return Err(Error::Invalid("upload already completed".to_string())); + }; + if let Err(e) = handle.write_chunk(&chunk).await { + let handle = session.handle.take(); + drop(session); + self.inner.lock().await.remove(upload_id); + if let Some(handle) = handle { + let _ = handle.abort().await; + } + return Err(e); + } + + session.offset = new_offset; + if !complete { + return Ok(UploadAppend::Progress { offset: new_offset }); + } + + if let Some(expected) = session.expected_size { + if new_offset != expected { + return Err(Error::Invalid(format!( + "final chunk ended at {new_offset}, expected {expected}" + ))); + } + } + + let Some(handle) = session.handle.take() else { + return Err(Error::Invalid("upload already completed".to_string())); + }; + let filename = session.filename.clone(); + drop(session); + + tracing::info!( + target: "openpxe::http::upload", + upload_id, + filename = %filename, + received_bytes = new_offset, + "chunked upload body complete; introspecting" + ); + + let meta = match handle.finish(store).await { + Ok(meta) => meta, + Err(e) => { + self.inner.lock().await.remove(upload_id); + return Err(e); + } + }; + self.inner.lock().await.remove(upload_id); + Ok(UploadAppend::Complete { + offset: new_offset, + iso: Box::new(meta), + }) + } + + pub async fn abort(&self, upload_id: &str) -> Result<()> { + let Some(session_lock) = self.inner.lock().await.remove(upload_id) else { + return Err(Error::Invalid(format!("no such upload '{upload_id}'"))); + }; + let mut session = session_lock.lock().await; + if let Some(handle) = session.handle.take() { + handle.abort().await?; + } + Ok(()) + } +} diff --git a/crates/http-api/tests/full_flow.rs b/crates/http-api/tests/full_flow.rs index dbecfd5..c06be51 100644 --- a/crates/http-api/tests/full_flow.rs +++ b/crates/http-api/tests/full_flow.rs @@ -110,6 +110,7 @@ async fn build_state() -> (AppState, tempfile::TempDir) { metrics, smb: None, nfs, + uploads: openpxe_http_api::uploads::UploadSessions::default(), log_bus, started_at: time::OffsetDateTime::now_utc(), public_base_url: "http://127.0.0.1".into(), @@ -1002,11 +1003,7 @@ async fn boot_log_records_entry_serve_with_mac() { assert_eq!(upload.status(), StatusCode::CREATED); // Fetch the per-entry script with ?mac=... - let (s, _) = get( - &app, - "/boot/fake-alpine-linux.ipxe?mac=AA:BB:CC:00:00:09", - ) - .await; + let (s, _) = get(&app, "/boot/fake-alpine-linux.ipxe?mac=AA:BB:CC:00:00:09").await; assert_eq!(s, StatusCode::OK); // The boot log should now contain exactly one entry, with the @@ -1019,7 +1016,7 @@ async fn boot_log_records_entry_serve_with_mac() { let ev = &events[0]; assert_eq!(ev["target_id"], "fake-alpine-linux"); assert_eq!(ev["mac"], "aa:bb:cc:00:00:09"); // normalized - // Title should include the filename and entry title. + // Title should include the filename and entry title. let title = ev["target_title"].as_str().unwrap(); assert!(title.contains("fake-alpine.iso"), "title was {title}"); } @@ -1083,3 +1080,124 @@ async fn upload_rejects_non_iso_filename_with_clear_message() { let text = std::str::from_utf8(&body).unwrap(); assert!(text.contains("only .iso uploads accepted"), "got: {text}"); } + +#[tokio::test] +async fn chunked_upload_writes_progressively_and_finishes_iso() { + let (state, dir) = build_state().await; + let app = build_router(state); + let iso = fake_alpine_iso(); + + let start = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/uploads") + .header("content-type", "application/json") + .body(Body::from( + r#"{"filename":"chunked-alpine.iso","size_bytes":65536}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(start.status(), StatusCode::CREATED); + let body = axum::body::to_bytes(start.into_body(), usize::MAX) + .await + .unwrap(); + let started: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let upload_id = started["upload_id"].as_str().unwrap(); + + let split = 8192usize; + let first = app + .clone() + .oneshot( + Request::builder() + .method("PUT") + .uri(format!("/api/uploads/{upload_id}")) + .header("x-openpxe-upload-offset", "0") + .body(Body::from(iso[..split].to_vec())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(first.status(), StatusCode::ACCEPTED); + let body = axum::body::to_bytes(first.into_body(), usize::MAX) + .await + .unwrap(); + let progress: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(progress["offset"].as_u64().unwrap(), split as u64); + assert!(!progress["complete"].as_bool().unwrap()); + assert!( + dir.path().join("isos/chunked-alpine.partial").exists(), + "chunked upload should leave a visible partial file while in progress" + ); + + let final_chunk = app + .clone() + .oneshot( + Request::builder() + .method("PUT") + .uri(format!("/api/uploads/{upload_id}")) + .header("x-openpxe-upload-offset", split.to_string()) + .header("x-openpxe-upload-complete", "true") + .body(Body::from(iso[split..].to_vec())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(final_chunk.status(), StatusCode::CREATED); + let body = axum::body::to_bytes(final_chunk.into_body(), usize::MAX) + .await + .unwrap(); + let finished: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert!(finished["complete"].as_bool().unwrap()); + assert_eq!(finished["iso"]["id"], "chunked-alpine"); + assert!(dir.path().join("isos/chunked-alpine.iso").exists()); + assert!(!dir.path().join("isos/chunked-alpine.partial").exists()); +} + +#[tokio::test] +async fn chunked_upload_rejects_offset_mismatch_without_advancing() { + let (state, _dir) = build_state().await; + let app = build_router(state); + + let start = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/uploads") + .header("content-type", "application/json") + .body(Body::from( + r#"{"filename":"offset-test.iso","size_bytes":16}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(start.status(), StatusCode::CREATED); + let body = axum::body::to_bytes(start.into_body(), usize::MAX) + .await + .unwrap(); + let started: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let upload_id = started["upload_id"].as_str().unwrap(); + + let mismatch = app + .oneshot( + Request::builder() + .method("PUT") + .uri(format!("/api/uploads/{upload_id}")) + .header("x-openpxe-upload-offset", "8") + .body(Body::from(vec![1, 2, 3, 4])) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(mismatch.status(), StatusCode::CONFLICT); + let body = axum::body::to_bytes(mismatch.into_body(), usize::MAX) + .await + .unwrap(); + let text = String::from_utf8(body.to_vec()).unwrap(); + assert!(text.contains("expected offset 0"), "got: {text}"); +} diff --git a/crates/iso-store/src/store.rs b/crates/iso-store/src/store.rs index cc5d515..6899b65 100644 --- a/crates/iso-store/src/store.rs +++ b/crates/iso-store/src/store.rs @@ -216,6 +216,9 @@ impl IsoStore { return Err(Error::Invalid(format!("iso '{id}' already exists"))); } let partial_path = self.iso_dir.join(format!("{id}.partial")); + if partial_path.exists() { + return Err(Error::Invalid(format!("iso '{id}' is already uploading"))); + } let file = tokio::fs::File::create(&partial_path).await?; Ok(UploadHandle { id, @@ -594,6 +597,19 @@ mod tests { assert!(matches!(r, Err(Error::Invalid(_)))); } + #[tokio::test] + async fn begin_upload_rejects_existing_partial_file() { + let dir = tempdir().unwrap(); + let store = IsoStore::new(dir.path().to_path_buf()); + store.ensure_dirs().await.unwrap(); + tokio::fs::write(dir.path().join("ubuntu.partial"), b"in-flight") + .await + .unwrap(); + + let r = store.begin_upload("ubuntu.iso").await; + assert!(matches!(r, Err(Error::Invalid(_)))); + } + #[tokio::test] async fn password_persists_via_meta_json_for_local_isos() { // Hash makes it onto disk so it survives a restart. diff --git a/crates/openpxe/src/main.rs b/crates/openpxe/src/main.rs index 687e0a6..20b6ac9 100644 --- a/crates/openpxe/src/main.rs +++ b/crates/openpxe/src/main.rs @@ -39,7 +39,7 @@ enum Command { /// docker run --rm \ /// -v /my/isos:/seed:ro \ /// -v openpxe-data:/var/lib/openpxe/isos \ - /// openpxe:0.3.2 seed --from /seed + /// openpxe:0.4.1 seed --from /seed Seed { /// Source directory containing one or more `.iso` files. #[arg(long)] @@ -145,6 +145,7 @@ async fn main() -> anyhow::Result<()> { metrics: metrics.clone(), smb: Some(smb.clone()), nfs: nfs.clone(), + uploads: openpxe_http_api::uploads::UploadSessions::default(), log_bus: log_bus.clone(), started_at: time::OffsetDateTime::now_utc(), public_base_url: public_base_url.clone(), diff --git a/crates/webui/src/app.css b/crates/webui/src/app.css index 897ac22..82b0dea 100644 --- a/crates/webui/src/app.css +++ b/crates/webui/src/app.css @@ -9,10 +9,10 @@ :root { /* Jet-black dark palette (default). Modelled on Netbox Labs's - near-black product chrome — surfaces step from #000 → #0d → #16 → #1c + near-black product chrome, with surfaces stepping subtly upward rather than the previous blue-tinted ramp, so the UI reads as a genuine "dark" rather than "dim navy". */ - --bg: #000000; + --bg: #030303; --bg-panel: #0a0a0a; --bg-panel-2: #141414; --bg-elev: #1c1c1c; @@ -26,7 +26,7 @@ --ok: #4ade80; --border: #1f1f1f; --border-soft: #141414; - --terminal-bg: #000000; + --terminal-bg: #050505; --shadow-card: 0 1px 0 rgba(255,255,255,0.02), 0 8px 24px rgba(0,0,0,0.55); --radius: 6px; --radius-lg: 10px; @@ -41,7 +41,7 @@ consistency. Designed against Netbox Labs's reference screenshot: near-white surfaces, soft grey dividers, dark text. */ --bg: #f6f8fb; - --bg-panel: #ffffff; + --bg-panel: #fbfcfe; --bg-panel-2: #f0f3f8; --bg-elev: #e6ebf2; --fg: #1c2330; @@ -253,7 +253,7 @@ button, .btn { cursor: pointer; transition: background 0.12s ease; } -button:hover, .btn:hover { background: var(--accent-dim); color: #fff; } +button:hover, .btn:hover { background: var(--accent-dim); color: #f4fffd; } button.ghost { background: transparent; color: var(--fg); border: 1px solid var(--border); } button.ghost:hover { background: var(--bg-panel-2); color: var(--fg); } button.danger { background: transparent; color: var(--err); border: 1px solid color-mix(in srgb, var(--err) 35%, transparent); } diff --git a/crates/webui/src/app.js b/crates/webui/src/app.js index 0c74c36..46ab90d 100644 --- a/crates/webui/src/app.js +++ b/crates/webui/src/app.js @@ -346,68 +346,84 @@ }); file.onchange = () => { if (file.files[0]) upload(file.files[0]); }; - // Upload telemetry. We surface bytes-sent + percent + ETA so when - // an upload stalls (e.g. a reverse proxy is buffering or rejecting - // a >100MB body) the operator can see it instead of staring at a - // 0% bar. We also tag the most common failure modes — timeout, - // network drop, HTTP 413/502/504 — with hints so the path forward - // is obvious from the UI. - function upload(f) { + // Chunked upload telemetry. The old browser path posted one huge + // multipart body, which left operators staring at 0% when a reverse + // proxy buffered or rejected the request before OpenPXE saw it. This + // path writes small raw chunks; each acknowledged chunk advances the + // bar and leaves a visible .partial file in the ISO directory. + async function upload(f) { const started = Date.now(); const bar = $('#bar'); const setStatus = (text, cls) => { upMsg.textContent = text; upMsg.className = 'msg ' + (cls || ''); }; - setStatus('Uploading ' + f.name + ' (' + fmtBytes(f.size) + ')…'); - prog.classList.add('active'); - bar.style.width = '0%'; - const fd = new FormData(); fd.append('file', f); - const xhr = new XMLHttpRequest(); - // 4-hour ceiling for very large ISOs over slow links. Browser - // default is 0 (never time out); we set an explicit cap so a - // stalled connection doesn't masquerade as "still uploading". - xhr.timeout = 4 * 60 * 60 * 1000; - xhr.upload.onprogress = e => { - if (!e.lengthComputable) return; - const pct = (e.loaded / e.total) * 100; + const update = (loaded, total, phase) => { + const pct = total > 0 ? Math.min(100, (loaded / total) * 100) : 100; bar.style.width = pct.toFixed(1) + '%'; - const elapsed = (Date.now() - started) / 1000; - const rate = elapsed > 0 ? e.loaded / elapsed : 0; - const remain = rate > 0 ? (e.total - e.loaded) / rate : 0; + const elapsed = Math.max(0.001, (Date.now() - started) / 1000); + const rate = loaded > 0 ? loaded / elapsed : 0; + const remain = rate > 0 ? (total - loaded) / rate : 0; setStatus( - 'Uploading ' + f.name + ' — ' + - fmtBytes(e.loaded) + ' of ' + fmtBytes(e.total) + + phase + ' ' + f.name + ' - ' + + fmtBytes(loaded) + ' of ' + fmtBytes(total) + ' (' + pct.toFixed(1) + '%, ' + fmtBytes(rate) + '/s' + (remain > 0 ? ', ' + Math.ceil(remain) + 's left' : '') + ')'); }; - xhr.onload = () => { - prog.classList.remove('active'); - bar.style.width = '0'; - if (xhr.status >= 200 && xhr.status < 300) { - setStatus('Uploaded & analyzed: ' + f.name + ' (' + fmtBytes(f.size) + ')', 'ok'); - render('storage'); - return; - } + const failText = async (r) => { + const text = (await r.text()).slice(0, 240); let hint = ''; - if (xhr.status === 413) hint = ' — body too large. A reverse proxy in front of OpenPXE (Cloudflare free tier caps at 100 MB) likely rejected it. Try the LAN IP directly.'; - else if (xhr.status === 502) hint = ' — bad gateway. Reverse proxy lost the upstream mid-stream.'; - else if (xhr.status === 504) hint = ' — gateway timeout. The upload took longer than the proxy allows; try the LAN IP.'; - else if (xhr.status === 409) hint = ' — an ISO with this name already exists. Remove the old one or rename.'; - setStatus('Upload failed: HTTP ' + xhr.status + ' ' + (xhr.responseText || '').slice(0, 200) + hint, 'err'); + if (r.status === 413) hint = ' - body too large. A proxy likely rejected this chunk.'; + else if (r.status === 502) hint = ' - bad gateway. Proxy lost the upstream mid-stream.'; + else if (r.status === 504) hint = ' - gateway timeout. Try the LAN IP directly.'; + else if (r.status === 409) hint = ' - name conflict or offset mismatch. Remove the old ISO and retry.'; + return 'HTTP ' + r.status + ' ' + text + hint; }; - xhr.onerror = () => { + + let uploadId = null; + setStatus('Preparing upload for ' + f.name + ' (' + fmtBytes(f.size) + ')'); + prog.classList.add('active'); + bar.style.width = '1%'; + + try { + const begin = await postJSON('/api/uploads', { + filename: f.name, + size_bytes: f.size, + }); + if (!begin.ok) throw new Error(await failText(begin)); + const session = await begin.json(); + uploadId = session.upload_id; + const chunkSize = Math.max(1024 * 1024, Number(session.chunk_size || 8 * 1024 * 1024)); + + let offset = Number(session.offset || 0); + let finished = null; + do { + const end = Math.min(offset + chunkSize, f.size); + const complete = end >= f.size; + const r = await fetch('/api/uploads/' + encodeURIComponent(uploadId), { + method: 'PUT', + headers: { + 'x-openpxe-upload-offset': String(offset), + 'x-openpxe-upload-complete': complete ? 'true' : 'false', + }, + body: f.slice(offset, end), + }); + if (!r.ok) throw new Error(await failText(r)); + const j = await r.json(); + offset = Number(j.offset || end); + update(offset, f.size, complete ? 'Analyzing' : 'Uploading'); + if (j.complete) finished = j.iso || true; + } while (!finished); + + setStatus('Uploaded and analyzed: ' + f.name + ' (' + fmtBytes(f.size) + ')', 'ok'); + render('storage'); + } catch (err) { + if (uploadId) { + try { await fetch('/api/uploads/' + encodeURIComponent(uploadId), {method: 'DELETE'}); } + catch {} + } + setStatus('Upload failed: ' + (err && err.message ? err.message : String(err)), 'err'); + } finally { prog.classList.remove('active'); - setStatus('Upload failed: network error or connection closed mid-stream. ' + - 'If you went through a reverse proxy, try the server\'s LAN IP directly.', 'err'); - }; - xhr.ontimeout = () => { - prog.classList.remove('active'); - setStatus('Upload timed out after 4 hours.', 'err'); - }; - xhr.onabort = () => { - prog.classList.remove('active'); - setStatus('Upload aborted.', 'err'); - }; - xhr.open('POST', '/api/isos'); - xhr.send(fd); + if (!upMsg.className.includes('ok')) bar.style.width = '0'; + } } // ── ISO table (mixed local + NFS) ── @@ -987,7 +1003,7 @@ } // Set the sidebar footer "Service status:" line. The chip itself moved - // off the topbar in v0.4.0 — operators wanted readiness, advertised + // off the topbar in v0.4.x: operators wanted readiness, advertised // URL, and the boot IP grouped together as the bottom-left summary. function setReady(state) { const dot = $('[data-bind=ready_dot]'); diff --git a/crates/webui/src/index.html b/crates/webui/src/index.html index 5def73b..c56cc88 100644 --- a/crates/webui/src/index.html +++ b/crates/webui/src/index.html @@ -29,7 +29,7 @@
OpenPXE -
v0.4.0
+
v0.4.1