Compare commits

...
1 Commits
Author SHA1 Message Date
Miles WardandClaude Opus 4.8 29040e8a5a v0.7.1: walk the ladder once ever — persistent learned modes, rule pins, same-boot iPXE recovery
Answers the operational question 'can a machine try all three boot
binaries in one go?' The protocol can't carry three NBPs in one cycle
(one boot file per DHCP round, the Secure-Boot refusal happens after
handoff with no error report, and the broken-NIC case specifically needs
the firmware itself to load builtin-driver iPXE — GRUB's network rides
the same broken firmware stack). What we CAN do is make the walk a
once-per-machine-ever event and give operators a way to skip it:

- Learned driver modes persist (<work_dir>/driver_modes.json). A MAC
  that reaches the Shim rung, or confirms an iPXE handoff at Builtin,
  is pinned to disk: immune to the 30-min TTL, reloaded at startup.
  The file only carries exceptions — a healthy fleet never writes it.
  Corrupt file starts empty (standard crash-cache policy).
- Boot rules gain an optional driver_mode pin (auto/firmware/builtin/
  shim), consulted by the DHCP proxy BEFORE the escalation ladder:
  'this OUI is a Secure Boot rack -> serve shim immediately' = zero
  failed cycles. Mode-only rules coexist with target rules (a pin
  doesn't shadow a later target match). Editor column on Hosts tab.
- grub.cfg now tries to chainload all-drivers iPXE before showing the
  signed menu: with SB off the chainload succeeds and the client gets
  the full iPXE feature set back in the SAME boot (self-healing for
  mis-escalations, and the handoff then pins the working mode); with
  SB on, shim's verifier refuses it inline — no reboot — and the
  signed menu appears.

DhcpProxyServer now takes the escalation table + rules store from main
(persistence path comes from the configured work dir).

Validation: clippy clean, fmt clean, 299 workspace tests green (+9:
persistence round-trip across restart, Shim pin survives TTL, learned
Builtin survives TTL, corrupt-file recovery, default-mode-never-
persisted, rule-pin matching incl. unknown-mode tolerance and
pin/target coexistence, GRUB chainload-before-menu ordering, API
round-trip of the driver_mode field).

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
2026-06-09 21:16:22 -04:00
10 changed files with 517 additions and 100 deletions
Generated
+10 -8
View File
@@ -2836,7 +2836,7 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
[[package]] [[package]]
name = "openpxe" name = "openpxe"
version = "0.7.0" version = "0.7.1"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"axum", "axum",
@@ -2858,7 +2858,7 @@ dependencies = [
[[package]] [[package]]
name = "openpxe-core" name = "openpxe-core"
version = "0.7.0" version = "0.7.1"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"base64", "base64",
@@ -2885,14 +2885,16 @@ dependencies = [
[[package]] [[package]]
name = "openpxe-dhcp-proxy" name = "openpxe-dhcp-proxy"
version = "0.7.0" version = "0.7.1"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bytes", "bytes",
"dhcproto", "dhcproto",
"openpxe-core", "openpxe-core",
"parking_lot", "parking_lot",
"serde_json",
"socket2", "socket2",
"tempfile",
"thiserror", "thiserror",
"tokio", "tokio",
"tracing", "tracing",
@@ -2900,7 +2902,7 @@ dependencies = [
[[package]] [[package]]
name = "openpxe-http-api" name = "openpxe-http-api"
version = "0.7.0" version = "0.7.1"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"axum", "axum",
@@ -2936,7 +2938,7 @@ dependencies = [
[[package]] [[package]]
name = "openpxe-ipxe-assets" name = "openpxe-ipxe-assets"
version = "0.7.0" version = "0.7.1"
dependencies = [ dependencies = [
"openpxe-core", "openpxe-core",
"rust-embed", "rust-embed",
@@ -2946,7 +2948,7 @@ dependencies = [
[[package]] [[package]]
name = "openpxe-iso-store" name = "openpxe-iso-store"
version = "0.7.0" version = "0.7.1"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bcrypt", "bcrypt",
@@ -2975,7 +2977,7 @@ dependencies = [
[[package]] [[package]]
name = "openpxe-tftp" name = "openpxe-tftp"
version = "0.7.0" version = "0.7.1"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bytes", "bytes",
@@ -2989,7 +2991,7 @@ dependencies = [
[[package]] [[package]]
name = "openpxe-webui" name = "openpxe-webui"
version = "0.7.0" version = "0.7.1"
[[package]] [[package]]
name = "p256" name = "p256"
+1 -1
View File
@@ -12,7 +12,7 @@ members = [
] ]
[workspace.package] [workspace.package]
version = "0.7.0" version = "0.7.1"
edition = "2021" edition = "2021"
rust-version = "1.95" rust-version = "1.95"
license = "MIT OR Apache-2.0" license = "MIT OR Apache-2.0"
+87 -2
View File
@@ -41,7 +41,16 @@ pub struct BootRule {
pub arch: String, pub arch: String,
/// Boot entry id (a `BootEntry::id`) or reserved menu name /// Boot entry id (a `BootEntry::id`) or reserved menu name
/// (`_local`, `_queue`, …) to chain to when this rule matches. /// (`_local`, `_queue`, …) to chain to when this rule matches.
/// May be empty for a rule that only pins a driver mode.
#[serde(default)]
pub target: String, pub target: String,
/// v0.7.1: optional first-boot binary pin — `""` (auto: let the
/// escalation ladder decide), `"firmware"`, `"builtin"`, or
/// `"shim"`. Lets an operator declare "this rack is all Secure
/// Boot → serve the signed chain immediately", skipping the
/// learn-by-failing walk entirely for known fleets.
#[serde(default)]
pub driver_mode: String,
/// Rules can be parked without deleting them. /// Rules can be parked without deleting them.
#[serde(default = "default_true")] #[serde(default = "default_true")]
pub enabled: bool, pub enabled: bool,
@@ -111,6 +120,7 @@ impl BootRulesStore {
r.mac_prefix = normalize_mac(&r.mac_prefix); r.mac_prefix = normalize_mac(&r.mac_prefix);
r.arch = r.arch.trim().to_ascii_lowercase(); r.arch = r.arch.trim().to_ascii_lowercase();
r.target = r.target.trim().to_string(); r.target = r.target.trim().to_string();
r.driver_mode = r.driver_mode.trim().to_ascii_lowercase();
r.note = r.note.trim().to_string(); r.note = r.note.trim().to_string();
} }
cfg.webhook_url = cfg.webhook_url.trim().to_string(); cfg.webhook_url = cfg.webhook_url.trim().to_string();
@@ -134,10 +144,40 @@ impl BootRulesStore {
/// passed one along, `None` otherwise (older chains). /// passed one along, `None` otherwise (older chains).
#[must_use] #[must_use]
pub fn match_target(&self, mac: &str, arch: Option<&str>) -> Option<String> { pub fn match_target(&self, mac: &str, arch: Option<&str>) -> Option<String> {
self.first_match(mac, arch, |r| {
(!r.target.is_empty()).then(|| r.target.clone())
})
}
/// v0.7.1: first enabled rule that pins a driver mode for `(mac,
/// arch)`. Consulted by the DHCP proxy *before* the automatic
/// escalation ladder — an operator who knows a rack is all Secure
/// Boot pins it to `shim` and those machines never walk the ladder.
/// Unknown mode strings are ignored (forward compatibility).
#[must_use]
pub fn driver_mode_hint(&self, mac: &str, arch: Option<&str>) -> Option<crate::DriverMode> {
self.first_match(mac, arch, |r| match r.driver_mode.as_str() {
"firmware" => Some(crate::DriverMode::Firmware),
"builtin" => Some(crate::DriverMode::Builtin),
"shim" => Some(crate::DriverMode::Shim),
_ => None,
})
}
/// Shared rule-matching walk: returns the first `extract` result from
/// an enabled rule whose selectors match. Rules that match but yield
/// `None` from `extract` (e.g. no target set, or no driver mode set)
/// don't stop the walk — target rules and mode-pin rules coexist.
fn first_match<T>(
&self,
mac: &str,
arch: Option<&str>,
extract: impl Fn(&BootRule) -> Option<T>,
) -> Option<T> {
let mac = normalize_mac(mac); let mac = normalize_mac(mac);
let g = self.inner.read(); let g = self.inner.read();
for r in &g.rules { for r in &g.rules {
if !r.enabled || r.target.is_empty() { if !r.enabled {
continue; continue;
} }
if !r.mac_prefix.is_empty() && !mac.starts_with(r.mac_prefix.as_str()) { if !r.mac_prefix.is_empty() && !mac.starts_with(r.mac_prefix.as_str()) {
@@ -151,7 +191,9 @@ impl BootRulesStore {
_ => continue, _ => continue,
} }
} }
return Some(r.target.clone()); if let Some(v) = extract(r) {
return Some(v);
}
} }
None None
} }
@@ -189,11 +231,54 @@ mod tests {
mac_prefix: mac_prefix.into(), mac_prefix: mac_prefix.into(),
arch: arch.into(), arch: arch.into(),
target: target.into(), target: target.into(),
driver_mode: String::new(),
enabled: true, enabled: true,
note: String::new(), note: String::new(),
} }
} }
#[test]
fn driver_mode_hint_pins_known_modes_and_ignores_unknown() {
let dir = tempdir().unwrap();
let s = BootRulesStore::load_or_default(dir.path());
let mut sb_rack = rule("aa:bb:cc", "", "");
sb_rack.driver_mode = "SHIM".into(); // normalized on replace
let mut weird = rule("11:22:33", "", "");
weird.driver_mode = "quantum".into(); // unknown → ignored
s.replace(BootRulesConfig {
rules: vec![sb_rack, weird],
webhook_url: String::new(),
});
assert_eq!(
s.driver_mode_hint("aa:bb:cc:00:00:01", None),
Some(crate::DriverMode::Shim)
);
assert_eq!(s.driver_mode_hint("11:22:33:00:00:01", None), None);
assert_eq!(s.driver_mode_hint("99:99:99:00:00:01", None), None);
}
#[test]
fn mode_pin_rule_does_not_shadow_later_target_rule() {
// A mode-only rule and a target rule can both apply to the same
// client: the mode pin must not consume the target walk.
let dir = tempdir().unwrap();
let s = BootRulesStore::load_or_default(dir.path());
let mut pin = rule("aa:bb", "", "");
pin.driver_mode = "builtin".into();
s.replace(BootRulesConfig {
rules: vec![pin, rule("aa:bb", "", "rack-image")],
webhook_url: String::new(),
});
assert_eq!(
s.driver_mode_hint("aa:bb:00:00:00:01", None),
Some(crate::DriverMode::Builtin)
);
assert_eq!(
s.match_target("aa:bb:00:00:00:01", None).as_deref(),
Some("rack-image")
);
}
#[test] #[test]
fn empty_config_matches_nothing() { fn empty_config_matches_nothing() {
let dir = tempdir().unwrap(); let dir = tempdir().unwrap();
+5
View File
@@ -19,3 +19,8 @@ thiserror.workspace = true
anyhow.workspace = true anyhow.workspace = true
bytes.workspace = true bytes.workspace = true
parking_lot.workspace = true parking_lot.workspace = true
# v0.7.1: learned driver modes persist to <work_dir>/driver_modes.json.
serde_json.workspace = true
[dev-dependencies]
tempfile = "3.12"
+265 -36
View File
@@ -1,24 +1,43 @@
//! Automatic per-MAC NIC driver-mode escalation (v0.6.1). //! Automatic per-MAC boot-binary escalation (v0.6.1, extended v0.7.x).
//! //!
//! OpenPXE serves the firmware-net iPXE build (`snponly`/`undionly`) by //! OpenPXE serves the firmware-net iPXE build (`snponly`/`undionly`) by
//! default — it's the most reliable choice for chainloading because the //! default — it's the most reliable choice for chainloading because the
//! firmware just proved its network works by downloading the NBP. A minority //! firmware just proved its network works by downloading the NBP. Two
//! of NICs have a missing or buggy firmware UNDI/SNP stack; those clients //! classes of machine can't run it:
//! TFTP the binary fine, but then iPXE can't bring the link up, so the
//! tell-tale second DHCP DISCOVER carrying the `iPXE` user-class never arrives
//! and the machine eventually re-PXE-boots.
//! //!
//! We detect exactly that: a *fresh* firmware DISCOVER from a MAC whose //! * a minority of NICs have a missing or buggy firmware UNDI/SNP stack —
//! previous firmware attempt was never confirmed by an iPXE handoff means the //! they TFTP the binary fine but iPXE can't bring the link up;
//! firmware-net build failed → escalate that MAC to [`DriverMode::Builtin`] //! * Secure-Boot firmware downloads it fine but refuses to *execute* an
//! (iPXE's own NIC drivers). The decision is sticky — once a MAC settles on a //! unsigned image.
//! mode that completes the handoff, later boots go straight to it. There is no //!
//! operator toggle; it just works, and the default (firmware) path is //! Both look identical from here: the tell-tale second DHCP DISCOVER
//! unchanged so hardware that already boots never regresses. //! carrying the `iPXE` user-class never arrives and the machine
//! re-PXE-boots. So a fresh firmware DISCOVER from a MAC whose previous
//! attempt was never confirmed climbs one rung:
//! `Firmware → Builtin → Shim` (the signed shim+GRUB chain). The decision
//! is sticky; there is no operator toggle; the default path is unchanged
//! so hardware that already boots never regresses.
//!
//! v0.7.1 — **learned modes persist**. Walking the ladder costs one or
//! two failed boot cycles, so a machine should pay it once *ever*, not
//! once per idle window or server restart. Two events pin a MAC's mode
//! to disk (`<work_dir>/driver_modes.json`):
//!
//! * a confirmed iPXE handoff at a non-default mode (Builtin proved to
//! work — also Shim, via the GRUB→iPXE same-boot chainload);
//! * reaching the terminal Shim rung (Secure-Boot machines never produce
//! an iPXE handoff from the signed menu, so escalation itself is the
//! best knowledge we'll ever have).
//!
//! Pinned entries are immune to the TTL and reload at startup. The
//! operator escape hatch is a rules-level driver-mode pin (which
//! overrides this table entirely) or deleting `driver_modes.json`.
use openpxe_core::DriverMode; use openpxe_core::DriverMode;
use parking_lot::Mutex; use parking_lot::Mutex;
use std::collections::HashMap; use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
/// Multiple DISCOVERs within this window belong to the *same* boot (DHCP /// Multiple DISCOVERs within this window belong to the *same* boot (DHCP
@@ -26,13 +45,14 @@ use std::time::{Duration, Instant};
/// DISCOVER). They must not be mistaken for a failed-and-retried boot. /// DISCOVER). They must not be mistaken for a failed-and-retried boot.
const SAME_BOOT_DEBOUNCE: Duration = Duration::from_secs(8); const SAME_BOOT_DEBOUNCE: Duration = Duration::from_secs(8);
/// Forget a MAC's state after this long with no activity, so a transient /// Forget an *unpinned* MAC's state after this long with no activity, so
/// escalation doesn't pin a client to Builtin forever and the map stays /// a transient mid-walk state doesn't linger and the map stays bounded.
/// bounded over a long-running deployment. /// Pinned (learned) entries are exempt — that's their whole point.
const ENTRY_TTL: Duration = Duration::from_mins(30); const ENTRY_TTL: Duration = Duration::from_mins(30);
/// Hard cap on tracked MACs. Past this we evict the least-recently-seen /// Hard cap on tracked MACs. Past this we evict the least-recently-seen
/// entry — escalation is best-effort, never a memory-growth vector. /// entry (unpinned first) — escalation is best-effort, never a
/// memory-growth vector.
const MAX_ENTRIES: usize = 4096; const MAX_ENTRIES: usize = 4096;
/// How often (at most) the whole map is swept for expired entries. /// How often (at most) the whole map is swept for expired entries.
@@ -49,6 +69,8 @@ struct Entry {
/// confirm it worked. A *new* boot arriving while this is still true means /// confirm it worked. A *new* boot arriving while this is still true means
/// the previous attempt failed and we should escalate. /// the previous attempt failed and we should escalate.
awaiting_confirm: bool, awaiting_confirm: bool,
/// Learned mode (v0.7.1): persisted to disk, exempt from the TTL.
pinned: bool,
last_seen: Instant, last_seen: Instant,
} }
@@ -72,14 +94,68 @@ impl Default for Inner {
#[derive(Debug, Default)] #[derive(Debug, Default)]
pub struct DriverEscalation { pub struct DriverEscalation {
inner: Mutex<Inner>, inner: Mutex<Inner>,
/// Persistence target for learned modes; `None` = ephemeral (tests).
path: Option<Arc<PathBuf>>,
} }
impl DriverEscalation { impl DriverEscalation {
/// Ephemeral instance (no persistence) — used by tests.
#[must_use] #[must_use]
pub fn new() -> Self { pub fn new() -> Self {
Self::default() Self::default()
} }
/// Instance backed by `<work_dir>/driver_modes.json`. Learned modes
/// from previous runs are reloaded as pinned entries; a missing or
/// corrupt file starts empty (same crash-cache policy as every other
/// store — a bad file must never block PXE).
#[must_use]
pub fn load_or_default(work_dir: &Path) -> Self {
let path = work_dir.join("driver_modes.json");
let mut map = HashMap::new();
if let Ok(text) = std::fs::read_to_string(&path) {
match serde_json::from_str::<HashMap<String, DriverMode>>(&text) {
Ok(loaded) => {
let now = Instant::now();
for (mac, mode) in loaded {
// Firmware is the default — persisting it would be
// noise; tolerate it in the file but don't track it.
if mode == DriverMode::Firmware {
continue;
}
map.insert(
mac,
Entry {
mode,
awaiting_confirm: false,
pinned: true,
last_seen: now,
},
);
}
tracing::info!(
target: "openpxe::dhcp",
learned = map.len(),
"loaded learned driver modes"
);
}
Err(e) => {
tracing::warn!(
target: "openpxe::dhcp",
"driver_modes.json present but unreadable ({e}); starting empty"
);
}
}
}
Self {
inner: Mutex::new(Inner {
map,
last_prune: Instant::now(),
}),
path: Some(Arc::new(path)),
}
}
/// Decide the driver mode for a firmware (PXEClient/HTTPClient) boot from /// Decide the driver mode for a firmware (PXEClient/HTTPClient) boot from
/// `mac`. `primary` is true for the main DHCP DISCOVER (:67) and false for /// `mac`. `primary` is true for the main DHCP DISCOVER (:67) and false for
/// the PXE Boot Server query (:4011); only the primary path drives /// the PXE Boot Server query (:4011); only the primary path drives
@@ -91,28 +167,32 @@ impl DriverEscalation {
/// Record that `mac` completed the iPXE handoff (a DISCOVER carrying the /// Record that `mac` completed the iPXE handoff (a DISCOVER carrying the
/// `iPXE` user-class). The mode we last served worked, so stop awaiting /// `iPXE` user-class). The mode we last served worked, so stop awaiting
/// confirmation and keep it sticky for next time. /// confirmation, keep it sticky, and — for non-default modes — pin it to
/// disk so the machine never re-walks the ladder (v0.7.1).
pub fn mark_ipxe_success(&self, mac: &str) { pub fn mark_ipxe_success(&self, mac: &str) {
self.confirm_at(mac, Instant::now()); self.confirm_at(mac, Instant::now());
} }
fn decide_at(&self, mac: &str, primary: bool, now: Instant) -> DriverMode { fn decide_at(&self, mac: &str, primary: bool, now: Instant) -> DriverMode {
let (mode, snapshot) = {
let mut g = self.inner.lock(); let mut g = self.inner.lock();
if now.duration_since(g.last_prune) >= PRUNE_INTERVAL { if now.duration_since(g.last_prune) >= PRUNE_INTERVAL {
g.map g.map
.retain(|_, e| now.duration_since(e.last_seen) < ENTRY_TTL); .retain(|_, e| e.pinned || now.duration_since(e.last_seen) < ENTRY_TTL);
g.last_prune = now; g.last_prune = now;
} }
// Inline staleness check: a MAC whose entry outlived the TTL starts // Inline staleness check: an unpinned MAC whose entry outlived
// fresh even when the amortized sweep above hasn't caught it yet. // the TTL starts fresh even when the amortized sweep above
// hasn't caught it yet. Pinned entries never go stale.
if g.map if g.map
.get(mac) .get(mac)
.is_some_and(|e| now.duration_since(e.last_seen) >= ENTRY_TTL) .is_some_and(|e| !e.pinned && now.duration_since(e.last_seen) >= ENTRY_TTL)
{ {
g.map.remove(mac); g.map.remove(mac);
} }
match g.map.get_mut(mac) { let mut newly_pinned = false;
let mode = match g.map.get_mut(mac) {
None => { None => {
g.map.insert( g.map.insert(
mac.to_owned(), mac.to_owned(),
@@ -120,6 +200,7 @@ impl DriverEscalation {
mode: DriverMode::Firmware, mode: DriverMode::Firmware,
// Only the primary DISCOVER opens a confirmation window. // Only the primary DISCOVER opens a confirmation window.
awaiting_confirm: primary, awaiting_confirm: primary,
pinned: false,
last_seen: now, last_seen: now,
}, },
); );
@@ -131,42 +212,105 @@ impl DriverEscalation {
Some(entry) => { Some(entry) => {
let recent = now.duration_since(entry.last_seen) < SAME_BOOT_DEBOUNCE; let recent = now.duration_since(entry.last_seen) < SAME_BOOT_DEBOUNCE;
if primary && !recent { if primary && !recent {
// A genuinely new boot. If the previous attempt was never // A genuinely new boot. If the previous attempt was
// confirmed, the build we served failed → climb one rung: // never confirmed, the build we served failed → climb
// Firmware (firmware NIC stack) → Builtin (iPXE's own // one rung: Firmware (firmware NIC stack) → Builtin
// drivers) → Shim (signed shim+GRUB, v0.7.0 — covers // (iPXE's own drivers) → Shim (signed shim+GRUB
// Secure Boot firmware that downloads our unsigned iPXE // covers Secure Boot firmware that downloads our
// but refuses to execute it). Shim is terminal: a MAC // unsigned iPXE but refuses to execute it). Shim is
// there stays until its entry TTLs out and resets. // terminal and pins to disk: SB machines never emit
// an iPXE handoff from the signed menu, so reaching
// the rung *is* the durable knowledge.
if entry.awaiting_confirm { if entry.awaiting_confirm {
entry.mode = match entry.mode { entry.mode = match entry.mode {
DriverMode::Firmware => DriverMode::Builtin, DriverMode::Firmware => DriverMode::Builtin,
DriverMode::Builtin | DriverMode::Shim => DriverMode::Shim, DriverMode::Builtin | DriverMode::Shim => DriverMode::Shim,
}; };
if entry.mode == DriverMode::Shim && !entry.pinned {
entry.pinned = true;
newly_pinned = true;
}
} }
entry.awaiting_confirm = true; entry.awaiting_confirm = true;
} }
entry.last_seen = now; entry.last_seen = now;
entry.mode entry.mode
} }
};
(mode, newly_pinned.then(|| pinned_snapshot(&g.map)))
};
if let Some(s) = snapshot {
self.persist(&s);
} }
mode
} }
fn confirm_at(&self, mac: &str, now: Instant) { fn confirm_at(&self, mac: &str, now: Instant) {
let snapshot = {
let mut g = self.inner.lock(); let mut g = self.inner.lock();
if let Some(e) = g.map.get_mut(mac) { let Some(e) = g.map.get_mut(mac) else {
return;
};
e.awaiting_confirm = false; e.awaiting_confirm = false;
e.last_seen = now; e.last_seen = now;
// A proven non-default mode is worth remembering forever —
// the machine demonstrably can't use the default path.
if e.mode != DriverMode::Firmware && !e.pinned {
e.pinned = true;
Some(pinned_snapshot(&g.map))
} else {
None
}
};
if let Some(s) = snapshot {
self.persist(&s);
}
}
/// Best-effort atomic write of the learned-mode table. No-op for
/// ephemeral instances. Failure logs and moves on — persistence is an
/// optimization, never a correctness requirement.
fn persist(&self, snapshot: &HashMap<String, DriverMode>) {
let Some(path) = &self.path else { return };
let body = match serde_json::to_vec_pretty(snapshot) {
Ok(b) => b,
Err(e) => {
tracing::warn!(target: "openpxe::dhcp", "serialize driver_modes.json: {e}");
return;
}
};
if let Some(parent) = path.parent() {
let _ = std::fs::create_dir_all(parent);
}
let tmp = path.with_extension("json.tmp");
if let Err(e) = std::fs::write(&tmp, body) {
tracing::warn!(target: "openpxe::dhcp", "write driver_modes.json tmp: {e}");
return;
}
if let Err(e) = std::fs::rename(&tmp, path.as_path()) {
tracing::warn!(target: "openpxe::dhcp", "rename driver_modes.json: {e}");
} }
} }
} }
fn pinned_snapshot(map: &HashMap<String, Entry>) -> HashMap<String, DriverMode> {
map.iter()
.filter(|(_, e)| e.pinned)
.map(|(k, e)| (k.clone(), e.mode))
.collect()
}
fn evict_oldest(map: &mut HashMap<String, Entry>) { fn evict_oldest(map: &mut HashMap<String, Entry>) {
if let Some(oldest) = map // Prefer evicting an unpinned entry; only touch learned modes when
.iter() // the whole table is pinned (4096 learned machines — at that point
// the operator has bigger questions than our memory bound).
let pick = |pinned: bool| {
map.iter()
.filter(|(_, e)| e.pinned == pinned)
.min_by_key(|(_, e)| e.last_seen) .min_by_key(|(_, e)| e.last_seen)
.map(|(k, _)| k.clone()) .map(|(k, _)| k.clone())
{ };
if let Some(oldest) = pick(false).or_else(|| pick(true)) {
map.remove(&oldest); map.remove(&oldest);
} }
} }
@@ -174,6 +318,7 @@ fn evict_oldest(map: &mut HashMap<String, Entry>) {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use tempfile::tempdir;
#[test] #[test]
fn firmware_first_then_escalates_on_unconfirmed_retry() { fn firmware_first_then_escalates_on_unconfirmed_retry() {
@@ -250,7 +395,7 @@ mod tests {
} }
#[test] #[test]
fn stale_entry_is_forgotten_and_resets_to_firmware() { fn stale_unpinned_entry_is_forgotten_and_resets_to_firmware() {
let e = DriverEscalation::new(); let e = DriverEscalation::new();
let t0 = Instant::now(); let t0 = Instant::now();
assert_eq!(e.decide_at("dd", true, t0), DriverMode::Firmware); assert_eq!(e.decide_at("dd", true, t0), DriverMode::Firmware);
@@ -258,8 +403,92 @@ mod tests {
e.decide_at("dd", true, t0 + Duration::from_mins(1)), e.decide_at("dd", true, t0 + Duration::from_mins(1)),
DriverMode::Builtin DriverMode::Builtin
); );
// After the TTL with no activity the entry is pruned → fresh firmware. // After the TTL with no activity the (unpinned) Builtin walk is
// pruned → fresh firmware. (A *confirmed* Builtin would be pinned
// and survive — see learned_builtin_survives_ttl.)
let later = t0 + Duration::from_mins(1) + ENTRY_TTL + Duration::from_secs(1); let later = t0 + Duration::from_mins(1) + ENTRY_TTL + Duration::from_secs(1);
assert_eq!(e.decide_at("dd", true, later), DriverMode::Firmware); assert_eq!(e.decide_at("dd", true, later), DriverMode::Firmware);
} }
#[test]
fn shim_pin_survives_ttl() {
// v0.7.1: reaching the Shim rung is durable knowledge — the
// machine must NOT re-walk the ladder after an idle period.
let e = DriverEscalation::new();
let t0 = Instant::now();
let _ = e.decide_at("ff", true, t0);
let _ = e.decide_at("ff", true, t0 + Duration::from_mins(1));
assert_eq!(
e.decide_at("ff", true, t0 + Duration::from_mins(2)),
DriverMode::Shim
);
let much_later = t0 + Duration::from_mins(2) + ENTRY_TTL + Duration::from_mins(5);
assert_eq!(e.decide_at("ff", true, much_later), DriverMode::Shim);
}
#[test]
fn learned_builtin_survives_ttl() {
let e = DriverEscalation::new();
let t0 = Instant::now();
let _ = e.decide_at("gg", true, t0);
assert_eq!(
e.decide_at("gg", true, t0 + Duration::from_mins(1)),
DriverMode::Builtin
);
// The handoff confirms Builtin → pinned.
e.confirm_at("gg", t0 + Duration::from_secs(61));
let much_later = t0 + ENTRY_TTL + Duration::from_mins(10);
assert_eq!(e.decide_at("gg", true, much_later), DriverMode::Builtin);
}
#[test]
fn learned_modes_persist_across_restart() {
let dir = tempdir().unwrap();
let t0 = Instant::now();
{
let e = DriverEscalation::load_or_default(dir.path());
// Walk one MAC to Shim (pins on escalation)...
let _ = e.decide_at("aa:01", true, t0);
let _ = e.decide_at("aa:01", true, t0 + Duration::from_mins(1));
assert_eq!(
e.decide_at("aa:01", true, t0 + Duration::from_mins(2)),
DriverMode::Shim
);
// ...and another to a confirmed Builtin (pins on handoff).
let _ = e.decide_at("aa:02", true, t0);
let _ = e.decide_at("aa:02", true, t0 + Duration::from_mins(1));
e.confirm_at("aa:02", t0 + Duration::from_secs(61));
}
// "Restart": a fresh instance from the same work_dir knows both.
let e2 = DriverEscalation::load_or_default(dir.path());
assert_eq!(e2.decide_at("aa:01", true, t0), DriverMode::Shim);
assert_eq!(e2.decide_at("aa:02", true, t0), DriverMode::Builtin);
// Unlearned MACs still start at the default.
assert_eq!(e2.decide_at("aa:03", true, t0), DriverMode::Firmware);
}
#[test]
fn corrupt_persistence_file_starts_empty() {
let dir = tempdir().unwrap();
std::fs::write(dir.path().join("driver_modes.json"), b"{broken").unwrap();
let e = DriverEscalation::load_or_default(dir.path());
assert_eq!(
e.decide_at("aa:bb", true, Instant::now()),
DriverMode::Firmware
);
}
#[test]
fn confirmed_firmware_is_not_persisted() {
// The default mode is never written — the file only carries
// exceptions, so a healthy fleet leaves it absent/empty.
let dir = tempdir().unwrap();
let t0 = Instant::now();
{
let e = DriverEscalation::load_or_default(dir.path());
let _ = e.decide_at("aa:09", true, t0);
e.confirm_at("aa:09", t0 + Duration::from_secs(2));
}
assert!(!dir.path().join("driver_modes.json").exists());
}
} }
+27 -7
View File
@@ -5,7 +5,9 @@ use crate::escalation::DriverEscalation;
use crate::reply::{build_reply, decide, BootDirective, ReplyContext}; use crate::reply::{build_reply, decide, BootDirective, ReplyContext};
use dhcproto::v4::{DhcpOption, Message, OptionCode}; use dhcproto::v4::{DhcpOption, Message, OptionCode};
use dhcproto::{Decodable, Decoder, Encodable, Encoder}; use dhcproto::{Decodable, Decoder, Encodable, Encoder};
use openpxe_core::{ClientArch, ClientEvent, ClientRegistry, DriverMode, FirmwareClass}; use openpxe_core::{
BootRulesStore, ClientArch, ClientEvent, ClientRegistry, DriverMode, FirmwareClass,
};
use socket2::{Domain, Protocol, Socket, Type}; use socket2::{Domain, Protocol, Socket, Type};
use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}; use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4};
use std::sync::Arc; use std::sync::Arc;
@@ -19,12 +21,19 @@ pub struct DhcpProxyServer {
public_base_url: String, public_base_url: String,
clients: Arc<ClientRegistry>, clients: Arc<ClientRegistry>,
metrics: openpxe_core::Metrics, metrics: openpxe_core::Metrics,
/// Automatic per-MAC NIC driver-mode escalation (v0.6.1). Shared across /// Automatic per-MAC NIC driver-mode escalation (v0.6.1; persistent
/// the :67 and :4011 listener tasks via the server `Arc`. /// learned modes since v0.7.1). Shared across the :67 and :4011
/// listener tasks via the server `Arc`. Built by the caller so the
/// persistence path comes from the configured work dir.
escalation: DriverEscalation, escalation: DriverEscalation,
/// v0.7.1: boot rules — consulted for an operator driver-mode pin
/// (e.g. "this OUI is all Secure Boot → serve shim immediately")
/// before the automatic escalation ladder.
rules: BootRulesStore,
} }
impl DhcpProxyServer { impl DhcpProxyServer {
#[allow(clippy::too_many_arguments)]
pub fn new( pub fn new(
bind: IpAddr, bind: IpAddr,
dhcp_port: u16, dhcp_port: u16,
@@ -33,6 +42,8 @@ impl DhcpProxyServer {
public_base_url: String, public_base_url: String,
clients: Arc<ClientRegistry>, clients: Arc<ClientRegistry>,
metrics: openpxe_core::Metrics, metrics: openpxe_core::Metrics,
escalation: DriverEscalation,
rules: BootRulesStore,
) -> Self { ) -> Self {
Self { Self {
bind, bind,
@@ -42,7 +53,8 @@ impl DhcpProxyServer {
public_base_url, public_base_url,
clients, clients,
metrics, metrics,
escalation: DriverEscalation::new(), escalation,
rules,
} }
} }
@@ -142,9 +154,17 @@ impl DhcpProxyServer {
self.escalation.mark_ipxe_success(&mac); self.escalation.mark_ipxe_success(&mac);
DriverMode::Firmware // unused: this path serves the HTTP script DriverMode::Firmware // unused: this path serves the HTTP script
} }
FirmwareClass::PxeClient | FirmwareClass::HttpClient => self FirmwareClass::PxeClient | FirmwareClass::HttpClient => {
.escalation // v0.7.1: an operator rule pin wins over (and bypasses)
.mode_for_firmware_attempt(&mac, label == "67"), // the automatic escalation ladder — known Secure-Boot
// fleets boot the signed chain on the very first cycle.
if let Some(pinned) = self.rules.driver_mode_hint(&mac, Some(arch.as_str())) {
pinned
} else {
self.escalation
.mode_for_firmware_attempt(&mac, label == "67")
}
}
// Unreachable: FirmwareClass::Other returned above. // Unreachable: FirmwareClass::Other returned above.
FirmwareClass::Other => DriverMode::Firmware, FirmwareClass::Other => DriverMode::Firmware,
}; };
+36
View File
@@ -35,6 +35,24 @@ pub fn render_grub_menu(isos: &[IsoMeta], base_url: &str) -> String {
let dev = grub_http_device(base); let dev = grub_http_device(base);
let mut s = String::new(); let mut s = String::new();
let _ = writeln!(s, "# OpenPXE — Secure Boot menu (signed shim+GRUB chain)"); let _ = writeln!(s, "# OpenPXE — Secure Boot menu (signed shim+GRUB chain)");
// v0.7.1: before showing the limited signed menu, try to hand the
// boot back to full iPXE *in this same boot cycle*. With Secure Boot
// OFF the chainload succeeds and the client gets the complete iPXE
// feature set (sanboot, wimboot, the full menu) despite having been
// escalated here. With Secure Boot ON, shim's verifier refuses the
// unsigned image INLINE — no reboot, no failed cycle — and execution
// falls through to the signed menu below. The all-drivers build is
// used because a MAC only lands here after the firmware-net build
// already failed once.
let _ = writeln!(s, "if [ \"$grub_cpu\" = \"arm64\" ]; then");
let _ = writeln!(s, " set openpxe_ipxe=ipxe-arm64.efi");
let _ = writeln!(s, "else");
let _ = writeln!(s, " set openpxe_ipxe=ipxe.efi");
let _ = writeln!(s, "fi");
let _ = writeln!(s, "if chainloader {dev}/ipxe/$openpxe_ipxe ; then");
let _ = writeln!(s, " boot");
let _ = writeln!(s, "fi");
let _ = writeln!(s);
let _ = writeln!(s, "set timeout=30"); let _ = writeln!(s, "set timeout=30");
let _ = writeln!(s, "set default=0"); let _ = writeln!(s, "set default=0");
let _ = writeln!(s); let _ = writeln!(s);
@@ -139,6 +157,24 @@ mod tests {
assert!(cfg.contains("Boot from local disk"), "{cfg}"); assert!(cfg.contains("Boot from local disk"), "{cfg}");
} }
#[test]
fn config_tries_ipxe_chainload_before_menu() {
// v0.7.1: SB-off machines recover full iPXE in the same boot;
// SB-on machines fail the chainload inline and reach the menu.
let cfg = render_grub_menu(&[linux_iso()], "http://10.0.0.5:8080");
let chain_pos = cfg
.find("if chainloader (http,10.0.0.5:8080)/ipxe/$openpxe_ipxe ; then")
.expect("chainload attempt missing");
let menu_pos = cfg.find("menuentry").expect("menu missing");
assert!(
chain_pos < menu_pos,
"chainload must precede the menu:\n{cfg}"
);
// Arch-conditional binary selection via GRUB's $grub_cpu.
assert!(cfg.contains("set openpxe_ipxe=ipxe-arm64.efi"), "{cfg}");
assert!(cfg.contains("set openpxe_ipxe=ipxe.efi"), "{cfg}");
}
#[test] #[test]
fn sanboot_and_wimboot_entries_are_omitted() { fn sanboot_and_wimboot_entries_are_omitted() {
let mut iso = linux_iso(); let mut iso = linux_iso();
+20
View File
@@ -2751,3 +2751,23 @@ async fn arch_selective_rule_ignores_other_arches() {
assert_eq!(s, StatusCode::OK); assert_eq!(s, StatusCode::OK);
assert!(String::from_utf8_lossy(&b).contains("boot rule -> fake-alpine-linux")); assert!(String::from_utf8_lossy(&b).contains("boot rule -> fake-alpine-linux"));
} }
#[tokio::test]
async fn boot_rule_driver_mode_pin_round_trips_via_api() {
// v0.7.1: a rule may pin only a boot binary (no target) — the API
// must persist and return it for the DHCP proxy to consult.
let (state, _dir) = build_state().await;
let app = build_router(state.clone());
let cfg = r#"{"rules":[{"mac_prefix":"aa:bb:cc","arch":"","target":"","driver_mode":"shim","enabled":true,"note":"SB rack"}],"webhook_url":""}"#;
let (s, _) = put_json(&app, "/api/boot-rules", cfg).await;
assert_eq!(s, StatusCode::NO_CONTENT);
let (s, b) = get(&app, "/api/boot-rules").await;
assert_eq!(s, StatusCode::OK);
let v: serde_json::Value = serde_json::from_slice(&b).unwrap();
assert_eq!(v["rules"][0]["driver_mode"], "shim");
// And the store the DHCP proxy shares resolves the pin.
assert_eq!(
state.boot_rules.driver_mode_hint("aa:bb:cc:00:00:07", None),
Some(openpxe_core::DriverMode::Shim)
);
}
+9 -1
View File
@@ -184,6 +184,10 @@ async fn main() -> anyhow::Result<()> {
"network info" "network info"
); );
// v0.7.1: boot rules are shared between the HTTP layer (target rules,
// webhook, the editor API) and the DHCP proxy (driver-mode pins).
let boot_rules = openpxe_core::BootRulesStore::load_or_default(&config.paths.work_dir);
let state = AppState { let state = AppState {
iso_store: iso_store.clone(), iso_store: iso_store.clone(),
clients: clients.clone(), clients: clients.clone(),
@@ -191,7 +195,7 @@ async fn main() -> anyhow::Result<()> {
queue: queue.clone(), queue: queue.clone(),
hosts: hosts.clone(), hosts: hosts.clone(),
boot_log: boot_log.clone(), boot_log: boot_log.clone(),
boot_rules: openpxe_core::BootRulesStore::load_or_default(&config.paths.work_dir), boot_rules: boot_rules.clone(),
boot_tokens: openpxe_core::BootTokens::new(), boot_tokens: openpxe_core::BootTokens::new(),
branding: branding.clone(), branding: branding.clone(),
pxe_bg_cache: openpxe_http_api::state::PxeBgCache::default(), pxe_bg_cache: openpxe_http_api::state::PxeBgCache::default(),
@@ -267,6 +271,10 @@ async fn main() -> anyhow::Result<()> {
public_base_url.clone(), public_base_url.clone(),
clients.clone(), clients.clone(),
metrics.clone(), metrics.clone(),
// v0.7.1: learned driver modes persist next to the other
// state files, so a machine walks the ladder once *ever*.
openpxe_dhcp_proxy::DriverEscalation::load_or_default(&config.paths.work_dir),
boot_rules.clone(),
); );
tokio::spawn(s.run()) tokio::spawn(s.run())
} }
+17 -5
View File
@@ -209,6 +209,13 @@
['', 'any arch'], ['bios', 'BIOS'], ['uefi-x64', 'UEFI x64'], ['', 'any arch'], ['bios', 'BIOS'], ['uefi-x64', 'UEFI x64'],
['uefi-ia32', 'UEFI IA32'], ['uefi-arm64', 'UEFI ARM64'], ['uefi-ia32', 'UEFI IA32'], ['uefi-arm64', 'UEFI ARM64'],
]; ];
// v0.7.1: optional first-boot binary pin. "Auto" lets the
// escalation ladder learn per machine; pinning skips the learning
// walk entirely (e.g. a rack known to run Secure Boot → shim).
const modeChoices = [
['', 'auto (learn)'], ['firmware', 'Firmware NIC'],
['builtin', 'iPXE drivers'], ['shim', 'Secure Boot (shim)'],
];
const rules = (cfg.rules || []).map(r => Object.assign({}, r)); const rules = (cfg.rules || []).map(r => Object.assign({}, r));
const tbody = el('tbody', {}); const tbody = el('tbody', {});
const msg = el('div', {class:'msg'}); const msg = el('div', {class:'msg'});
@@ -224,8 +231,8 @@
const redraw = () => { const redraw = () => {
tbody.innerHTML = ''; tbody.innerHTML = '';
if (!rules.length) { if (!rules.length) {
tbody.appendChild(el('tr', {}, el('td', {colspan:'6', class:'empty', style:'padding:14px'}, tbody.appendChild(el('tr', {}, el('td', {colspan:'7', class:'empty', style:'padding:14px'},
'No rules. Add one to route whole groups of machines (an OUI, an architecture) to a target.'))); 'No rules. Add one to route whole groups of machines (an OUI, an architecture) to a target — or to pin a boot binary (e.g. Secure Boot racks → shim, zero failed cycles).')));
} }
rules.forEach((r, i) => { rules.forEach((r, i) => {
const macIn = el('input', {type:'text', spellcheck:'false', placeholder:'aa:bb:cc (prefix)', const macIn = el('input', {type:'text', spellcheck:'false', placeholder:'aa:bb:cc (prefix)',
@@ -235,6 +242,9 @@
el('option', Object.assign({value: v}, v === (r.arch || '') ? {selected:''} : {}), label))); el('option', Object.assign({value: v}, v === (r.arch || '') ? {selected:''} : {}), label)));
const tgtSel = targetSelect(r.target || ''); const tgtSel = targetSelect(r.target || '');
tgtSel.onchange = e => { r.target = e.target.value; }; tgtSel.onchange = e => { r.target = e.target.value; };
const modeSel = el('select', {onchange: e => { r.driver_mode = e.target.value; }},
modeChoices.map(([v, label]) =>
el('option', Object.assign({value: v}, v === (r.driver_mode || '') ? {selected:''} : {}), label)));
const noteIn = el('input', {type:'text', placeholder:'note', const noteIn = el('input', {type:'text', placeholder:'note',
value: r.note || '', oninput: e => { r.note = e.target.value; }}); value: r.note || '', oninput: e => { r.note = e.target.value; }});
const enabled = el('input', {type:'checkbox', onchange: e => { r.enabled = e.target.checked; }}); const enabled = el('input', {type:'checkbox', onchange: e => { r.enabled = e.target.checked; }});
@@ -243,6 +253,7 @@
el('td', {}, macIn), el('td', {}, macIn),
el('td', {}, archSel), el('td', {}, archSel),
el('td', {}, tgtSel), el('td', {}, tgtSel),
el('td', {}, modeSel),
el('td', {}, noteIn), el('td', {}, noteIn),
el('td', {style:'text-align:center'}, enabled), el('td', {style:'text-align:center'}, enabled),
el('td', {style:'text-align:right'}, el('td', {style:'text-align:right'},
@@ -257,8 +268,9 @@
redraw(); redraw();
}}, '+ Add rule'); }}, '+ Add rule');
const saveBtn = el('button', {onclick: async () => { const saveBtn = el('button', {onclick: async () => {
const bad = rules.find(r => r.enabled !== false && !r.target); // A rule needs at least one effect: a target or a boot-binary pin.
if (bad) { msg.textContent = 'Every enabled rule needs a target.'; msg.className = 'msg err'; return; } const bad = rules.find(r => r.enabled !== false && !r.target && !r.driver_mode);
if (bad) { msg.textContent = 'Every enabled rule needs a target or a boot-binary pin.'; msg.className = 'msg err'; return; }
const r = await putJSON('/api/boot-rules', {rules, webhook_url: webhookInput.value.trim()}); const r = await putJSON('/api/boot-rules', {rules, webhook_url: webhookInput.value.trim()});
if (r.ok) { msg.textContent = 'Saved.'; msg.className = 'msg ok'; } if (r.ok) { msg.textContent = 'Saved.'; msg.className = 'msg ok'; }
else { msg.textContent = 'Save failed: ' + await r.text(); msg.className = 'msg err'; } else { msg.textContent = 'Save failed: ' + await r.text(); msg.className = 'msg err'; }
@@ -273,7 +285,7 @@
el('table', {}, [ el('table', {}, [
el('thead', {}, el('tr', {}, [ el('thead', {}, el('tr', {}, [
el('th',{},'MAC prefix'), el('th',{},'Arch'), el('th',{},'Target'), el('th',{},'MAC prefix'), el('th',{},'Arch'), el('th',{},'Target'),
el('th',{},'Note'), el('th',{},'On'), el('th',{},''), el('th',{},'Boot binary'), el('th',{},'Note'), el('th',{},'On'), el('th',{},''),
])), ])),
tbody, tbody,
]), ]),