v0.7.1: walk the ladder once ever — persistent learned modes, rule pins, same-boot iPXE recovery
Answers the operational question 'can a machine try all three boot binaries in one go?' The protocol can't carry three NBPs in one cycle (one boot file per DHCP round, the Secure-Boot refusal happens after handoff with no error report, and the broken-NIC case specifically needs the firmware itself to load builtin-driver iPXE — GRUB's network rides the same broken firmware stack). What we CAN do is make the walk a once-per-machine-ever event and give operators a way to skip it: - Learned driver modes persist (<work_dir>/driver_modes.json). A MAC that reaches the Shim rung, or confirms an iPXE handoff at Builtin, is pinned to disk: immune to the 30-min TTL, reloaded at startup. The file only carries exceptions — a healthy fleet never writes it. Corrupt file starts empty (standard crash-cache policy). - Boot rules gain an optional driver_mode pin (auto/firmware/builtin/ shim), consulted by the DHCP proxy BEFORE the escalation ladder: 'this OUI is a Secure Boot rack -> serve shim immediately' = zero failed cycles. Mode-only rules coexist with target rules (a pin doesn't shadow a later target match). Editor column on Hosts tab. - grub.cfg now tries to chainload all-drivers iPXE before showing the signed menu: with SB off the chainload succeeds and the client gets the full iPXE feature set back in the SAME boot (self-healing for mis-escalations, and the handoff then pins the working mode); with SB on, shim's verifier refuses it inline — no reboot — and the signed menu appears. DhcpProxyServer now takes the escalation table + rules store from main (persistence path comes from the configured work dir). Validation: clippy clean, fmt clean, 299 workspace tests green (+9: persistence round-trip across restart, Shim pin survives TTL, learned Builtin survives TTL, corrupt-file recovery, default-mode-never- persisted, rule-pin matching incl. unknown-mode tolerance and pin/target coexistence, GRUB chainload-before-menu ordering, API round-trip of the driver_mode field). Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Miles Ward
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
3a32d65fb7
commit
29040e8a5a
@@ -41,7 +41,16 @@ pub struct BootRule {
|
||||
pub arch: String,
|
||||
/// Boot entry id (a `BootEntry::id`) or reserved menu name
|
||||
/// (`_local`, `_queue`, …) to chain to when this rule matches.
|
||||
/// May be empty for a rule that only pins a driver mode.
|
||||
#[serde(default)]
|
||||
pub target: String,
|
||||
/// v0.7.1: optional first-boot binary pin — `""` (auto: let the
|
||||
/// escalation ladder decide), `"firmware"`, `"builtin"`, or
|
||||
/// `"shim"`. Lets an operator declare "this rack is all Secure
|
||||
/// Boot → serve the signed chain immediately", skipping the
|
||||
/// learn-by-failing walk entirely for known fleets.
|
||||
#[serde(default)]
|
||||
pub driver_mode: String,
|
||||
/// Rules can be parked without deleting them.
|
||||
#[serde(default = "default_true")]
|
||||
pub enabled: bool,
|
||||
@@ -111,6 +120,7 @@ impl BootRulesStore {
|
||||
r.mac_prefix = normalize_mac(&r.mac_prefix);
|
||||
r.arch = r.arch.trim().to_ascii_lowercase();
|
||||
r.target = r.target.trim().to_string();
|
||||
r.driver_mode = r.driver_mode.trim().to_ascii_lowercase();
|
||||
r.note = r.note.trim().to_string();
|
||||
}
|
||||
cfg.webhook_url = cfg.webhook_url.trim().to_string();
|
||||
@@ -134,10 +144,40 @@ impl BootRulesStore {
|
||||
/// passed one along, `None` otherwise (older chains).
|
||||
#[must_use]
|
||||
pub fn match_target(&self, mac: &str, arch: Option<&str>) -> Option<String> {
|
||||
self.first_match(mac, arch, |r| {
|
||||
(!r.target.is_empty()).then(|| r.target.clone())
|
||||
})
|
||||
}
|
||||
|
||||
/// v0.7.1: first enabled rule that pins a driver mode for `(mac,
|
||||
/// arch)`. Consulted by the DHCP proxy *before* the automatic
|
||||
/// escalation ladder — an operator who knows a rack is all Secure
|
||||
/// Boot pins it to `shim` and those machines never walk the ladder.
|
||||
/// Unknown mode strings are ignored (forward compatibility).
|
||||
#[must_use]
|
||||
pub fn driver_mode_hint(&self, mac: &str, arch: Option<&str>) -> Option<crate::DriverMode> {
|
||||
self.first_match(mac, arch, |r| match r.driver_mode.as_str() {
|
||||
"firmware" => Some(crate::DriverMode::Firmware),
|
||||
"builtin" => Some(crate::DriverMode::Builtin),
|
||||
"shim" => Some(crate::DriverMode::Shim),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Shared rule-matching walk: returns the first `extract` result from
|
||||
/// an enabled rule whose selectors match. Rules that match but yield
|
||||
/// `None` from `extract` (e.g. no target set, or no driver mode set)
|
||||
/// don't stop the walk — target rules and mode-pin rules coexist.
|
||||
fn first_match<T>(
|
||||
&self,
|
||||
mac: &str,
|
||||
arch: Option<&str>,
|
||||
extract: impl Fn(&BootRule) -> Option<T>,
|
||||
) -> Option<T> {
|
||||
let mac = normalize_mac(mac);
|
||||
let g = self.inner.read();
|
||||
for r in &g.rules {
|
||||
if !r.enabled || r.target.is_empty() {
|
||||
if !r.enabled {
|
||||
continue;
|
||||
}
|
||||
if !r.mac_prefix.is_empty() && !mac.starts_with(r.mac_prefix.as_str()) {
|
||||
@@ -151,7 +191,9 @@ impl BootRulesStore {
|
||||
_ => continue,
|
||||
}
|
||||
}
|
||||
return Some(r.target.clone());
|
||||
if let Some(v) = extract(r) {
|
||||
return Some(v);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
@@ -189,11 +231,54 @@ mod tests {
|
||||
mac_prefix: mac_prefix.into(),
|
||||
arch: arch.into(),
|
||||
target: target.into(),
|
||||
driver_mode: String::new(),
|
||||
enabled: true,
|
||||
note: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn driver_mode_hint_pins_known_modes_and_ignores_unknown() {
|
||||
let dir = tempdir().unwrap();
|
||||
let s = BootRulesStore::load_or_default(dir.path());
|
||||
let mut sb_rack = rule("aa:bb:cc", "", "");
|
||||
sb_rack.driver_mode = "SHIM".into(); // normalized on replace
|
||||
let mut weird = rule("11:22:33", "", "");
|
||||
weird.driver_mode = "quantum".into(); // unknown → ignored
|
||||
s.replace(BootRulesConfig {
|
||||
rules: vec![sb_rack, weird],
|
||||
webhook_url: String::new(),
|
||||
});
|
||||
assert_eq!(
|
||||
s.driver_mode_hint("aa:bb:cc:00:00:01", None),
|
||||
Some(crate::DriverMode::Shim)
|
||||
);
|
||||
assert_eq!(s.driver_mode_hint("11:22:33:00:00:01", None), None);
|
||||
assert_eq!(s.driver_mode_hint("99:99:99:00:00:01", None), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mode_pin_rule_does_not_shadow_later_target_rule() {
|
||||
// A mode-only rule and a target rule can both apply to the same
|
||||
// client: the mode pin must not consume the target walk.
|
||||
let dir = tempdir().unwrap();
|
||||
let s = BootRulesStore::load_or_default(dir.path());
|
||||
let mut pin = rule("aa:bb", "", "");
|
||||
pin.driver_mode = "builtin".into();
|
||||
s.replace(BootRulesConfig {
|
||||
rules: vec![pin, rule("aa:bb", "", "rack-image")],
|
||||
webhook_url: String::new(),
|
||||
});
|
||||
assert_eq!(
|
||||
s.driver_mode_hint("aa:bb:00:00:00:01", None),
|
||||
Some(crate::DriverMode::Builtin)
|
||||
);
|
||||
assert_eq!(
|
||||
s.match_target("aa:bb:00:00:00:01", None).as_deref(),
|
||||
Some("rack-image")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_config_matches_nothing() {
|
||||
let dir = tempdir().unwrap();
|
||||
|
||||
@@ -19,3 +19,8 @@ thiserror.workspace = true
|
||||
anyhow.workspace = true
|
||||
bytes.workspace = true
|
||||
parking_lot.workspace = true
|
||||
# v0.7.1: learned driver modes persist to <work_dir>/driver_modes.json.
|
||||
serde_json.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.12"
|
||||
|
||||
@@ -1,24 +1,43 @@
|
||||
//! Automatic per-MAC NIC driver-mode escalation (v0.6.1).
|
||||
//! Automatic per-MAC boot-binary escalation (v0.6.1, extended v0.7.x).
|
||||
//!
|
||||
//! OpenPXE serves the firmware-net iPXE build (`snponly`/`undionly`) by
|
||||
//! default — it's the most reliable choice for chainloading because the
|
||||
//! firmware just proved its network works by downloading the NBP. A minority
|
||||
//! of NICs have a missing or buggy firmware UNDI/SNP stack; those clients
|
||||
//! TFTP the binary fine, but then iPXE can't bring the link up, so the
|
||||
//! tell-tale second DHCP DISCOVER carrying the `iPXE` user-class never arrives
|
||||
//! and the machine eventually re-PXE-boots.
|
||||
//! firmware just proved its network works by downloading the NBP. Two
|
||||
//! classes of machine can't run it:
|
||||
//!
|
||||
//! We detect exactly that: a *fresh* firmware DISCOVER from a MAC whose
|
||||
//! previous firmware attempt was never confirmed by an iPXE handoff means the
|
||||
//! firmware-net build failed → escalate that MAC to [`DriverMode::Builtin`]
|
||||
//! (iPXE's own NIC drivers). The decision is sticky — once a MAC settles on a
|
||||
//! mode that completes the handoff, later boots go straight to it. There is no
|
||||
//! operator toggle; it just works, and the default (firmware) path is
|
||||
//! unchanged so hardware that already boots never regresses.
|
||||
//! * a minority of NICs have a missing or buggy firmware UNDI/SNP stack —
|
||||
//! they TFTP the binary fine but iPXE can't bring the link up;
|
||||
//! * Secure-Boot firmware downloads it fine but refuses to *execute* an
|
||||
//! unsigned image.
|
||||
//!
|
||||
//! Both look identical from here: the tell-tale second DHCP DISCOVER
|
||||
//! carrying the `iPXE` user-class never arrives and the machine
|
||||
//! re-PXE-boots. So a fresh firmware DISCOVER from a MAC whose previous
|
||||
//! attempt was never confirmed climbs one rung:
|
||||
//! `Firmware → Builtin → Shim` (the signed shim+GRUB chain). The decision
|
||||
//! is sticky; there is no operator toggle; the default path is unchanged
|
||||
//! so hardware that already boots never regresses.
|
||||
//!
|
||||
//! v0.7.1 — **learned modes persist**. Walking the ladder costs one or
|
||||
//! two failed boot cycles, so a machine should pay it once *ever*, not
|
||||
//! once per idle window or server restart. Two events pin a MAC's mode
|
||||
//! to disk (`<work_dir>/driver_modes.json`):
|
||||
//!
|
||||
//! * a confirmed iPXE handoff at a non-default mode (Builtin proved to
|
||||
//! work — also Shim, via the GRUB→iPXE same-boot chainload);
|
||||
//! * reaching the terminal Shim rung (Secure-Boot machines never produce
|
||||
//! an iPXE handoff from the signed menu, so escalation itself is the
|
||||
//! best knowledge we'll ever have).
|
||||
//!
|
||||
//! Pinned entries are immune to the TTL and reload at startup. The
|
||||
//! operator escape hatch is a rules-level driver-mode pin (which
|
||||
//! overrides this table entirely) or deleting `driver_modes.json`.
|
||||
|
||||
use openpxe_core::DriverMode;
|
||||
use parking_lot::Mutex;
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Multiple DISCOVERs within this window belong to the *same* boot (DHCP
|
||||
@@ -26,13 +45,14 @@ use std::time::{Duration, Instant};
|
||||
/// DISCOVER). They must not be mistaken for a failed-and-retried boot.
|
||||
const SAME_BOOT_DEBOUNCE: Duration = Duration::from_secs(8);
|
||||
|
||||
/// Forget a MAC's state after this long with no activity, so a transient
|
||||
/// escalation doesn't pin a client to Builtin forever and the map stays
|
||||
/// bounded over a long-running deployment.
|
||||
/// Forget an *unpinned* MAC's state after this long with no activity, so
|
||||
/// a transient mid-walk state doesn't linger and the map stays bounded.
|
||||
/// Pinned (learned) entries are exempt — that's their whole point.
|
||||
const ENTRY_TTL: Duration = Duration::from_mins(30);
|
||||
|
||||
/// Hard cap on tracked MACs. Past this we evict the least-recently-seen
|
||||
/// entry — escalation is best-effort, never a memory-growth vector.
|
||||
/// entry (unpinned first) — escalation is best-effort, never a
|
||||
/// memory-growth vector.
|
||||
const MAX_ENTRIES: usize = 4096;
|
||||
|
||||
/// How often (at most) the whole map is swept for expired entries.
|
||||
@@ -49,6 +69,8 @@ struct Entry {
|
||||
/// confirm it worked. A *new* boot arriving while this is still true means
|
||||
/// the previous attempt failed and we should escalate.
|
||||
awaiting_confirm: bool,
|
||||
/// Learned mode (v0.7.1): persisted to disk, exempt from the TTL.
|
||||
pinned: bool,
|
||||
last_seen: Instant,
|
||||
}
|
||||
|
||||
@@ -72,14 +94,68 @@ impl Default for Inner {
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DriverEscalation {
|
||||
inner: Mutex<Inner>,
|
||||
/// Persistence target for learned modes; `None` = ephemeral (tests).
|
||||
path: Option<Arc<PathBuf>>,
|
||||
}
|
||||
|
||||
impl DriverEscalation {
|
||||
/// Ephemeral instance (no persistence) — used by tests.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Instance backed by `<work_dir>/driver_modes.json`. Learned modes
|
||||
/// from previous runs are reloaded as pinned entries; a missing or
|
||||
/// corrupt file starts empty (same crash-cache policy as every other
|
||||
/// store — a bad file must never block PXE).
|
||||
#[must_use]
|
||||
pub fn load_or_default(work_dir: &Path) -> Self {
|
||||
let path = work_dir.join("driver_modes.json");
|
||||
let mut map = HashMap::new();
|
||||
if let Ok(text) = std::fs::read_to_string(&path) {
|
||||
match serde_json::from_str::<HashMap<String, DriverMode>>(&text) {
|
||||
Ok(loaded) => {
|
||||
let now = Instant::now();
|
||||
for (mac, mode) in loaded {
|
||||
// Firmware is the default — persisting it would be
|
||||
// noise; tolerate it in the file but don't track it.
|
||||
if mode == DriverMode::Firmware {
|
||||
continue;
|
||||
}
|
||||
map.insert(
|
||||
mac,
|
||||
Entry {
|
||||
mode,
|
||||
awaiting_confirm: false,
|
||||
pinned: true,
|
||||
last_seen: now,
|
||||
},
|
||||
);
|
||||
}
|
||||
tracing::info!(
|
||||
target: "openpxe::dhcp",
|
||||
learned = map.len(),
|
||||
"loaded learned driver modes"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "openpxe::dhcp",
|
||||
"driver_modes.json present but unreadable ({e}); starting empty"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Self {
|
||||
inner: Mutex::new(Inner {
|
||||
map,
|
||||
last_prune: Instant::now(),
|
||||
}),
|
||||
path: Some(Arc::new(path)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Decide the driver mode for a firmware (PXEClient/HTTPClient) boot from
|
||||
/// `mac`. `primary` is true for the main DHCP DISCOVER (:67) and false for
|
||||
/// the PXE Boot Server query (:4011); only the primary path drives
|
||||
@@ -91,82 +167,150 @@ impl DriverEscalation {
|
||||
|
||||
/// Record that `mac` completed the iPXE handoff (a DISCOVER carrying the
|
||||
/// `iPXE` user-class). The mode we last served worked, so stop awaiting
|
||||
/// confirmation and keep it sticky for next time.
|
||||
/// confirmation, keep it sticky, and — for non-default modes — pin it to
|
||||
/// disk so the machine never re-walks the ladder (v0.7.1).
|
||||
pub fn mark_ipxe_success(&self, mac: &str) {
|
||||
self.confirm_at(mac, Instant::now());
|
||||
}
|
||||
|
||||
fn decide_at(&self, mac: &str, primary: bool, now: Instant) -> DriverMode {
|
||||
let mut g = self.inner.lock();
|
||||
if now.duration_since(g.last_prune) >= PRUNE_INTERVAL {
|
||||
g.map
|
||||
.retain(|_, e| now.duration_since(e.last_seen) < ENTRY_TTL);
|
||||
g.last_prune = now;
|
||||
}
|
||||
// Inline staleness check: a MAC whose entry outlived the TTL starts
|
||||
// fresh even when the amortized sweep above hasn't caught it yet.
|
||||
if g.map
|
||||
.get(mac)
|
||||
.is_some_and(|e| now.duration_since(e.last_seen) >= ENTRY_TTL)
|
||||
{
|
||||
g.map.remove(mac);
|
||||
}
|
||||
let (mode, snapshot) = {
|
||||
let mut g = self.inner.lock();
|
||||
if now.duration_since(g.last_prune) >= PRUNE_INTERVAL {
|
||||
g.map
|
||||
.retain(|_, e| e.pinned || now.duration_since(e.last_seen) < ENTRY_TTL);
|
||||
g.last_prune = now;
|
||||
}
|
||||
// Inline staleness check: an unpinned MAC whose entry outlived
|
||||
// the TTL starts fresh even when the amortized sweep above
|
||||
// hasn't caught it yet. Pinned entries never go stale.
|
||||
if g.map
|
||||
.get(mac)
|
||||
.is_some_and(|e| !e.pinned && now.duration_since(e.last_seen) >= ENTRY_TTL)
|
||||
{
|
||||
g.map.remove(mac);
|
||||
}
|
||||
|
||||
match g.map.get_mut(mac) {
|
||||
None => {
|
||||
g.map.insert(
|
||||
mac.to_owned(),
|
||||
Entry {
|
||||
mode: DriverMode::Firmware,
|
||||
// Only the primary DISCOVER opens a confirmation window.
|
||||
awaiting_confirm: primary,
|
||||
last_seen: now,
|
||||
},
|
||||
);
|
||||
if g.map.len() > MAX_ENTRIES {
|
||||
evict_oldest(&mut g.map);
|
||||
}
|
||||
DriverMode::Firmware
|
||||
}
|
||||
Some(entry) => {
|
||||
let recent = now.duration_since(entry.last_seen) < SAME_BOOT_DEBOUNCE;
|
||||
if primary && !recent {
|
||||
// A genuinely new boot. If the previous attempt was never
|
||||
// confirmed, the build we served failed → climb one rung:
|
||||
// Firmware (firmware NIC stack) → Builtin (iPXE's own
|
||||
// drivers) → Shim (signed shim+GRUB, v0.7.0 — covers
|
||||
// Secure Boot firmware that downloads our unsigned iPXE
|
||||
// but refuses to execute it). Shim is terminal: a MAC
|
||||
// there stays until its entry TTLs out and resets.
|
||||
if entry.awaiting_confirm {
|
||||
entry.mode = match entry.mode {
|
||||
DriverMode::Firmware => DriverMode::Builtin,
|
||||
DriverMode::Builtin | DriverMode::Shim => DriverMode::Shim,
|
||||
};
|
||||
let mut newly_pinned = false;
|
||||
let mode = match g.map.get_mut(mac) {
|
||||
None => {
|
||||
g.map.insert(
|
||||
mac.to_owned(),
|
||||
Entry {
|
||||
mode: DriverMode::Firmware,
|
||||
// Only the primary DISCOVER opens a confirmation window.
|
||||
awaiting_confirm: primary,
|
||||
pinned: false,
|
||||
last_seen: now,
|
||||
},
|
||||
);
|
||||
if g.map.len() > MAX_ENTRIES {
|
||||
evict_oldest(&mut g.map);
|
||||
}
|
||||
entry.awaiting_confirm = true;
|
||||
DriverMode::Firmware
|
||||
}
|
||||
entry.last_seen = now;
|
||||
entry.mode
|
||||
}
|
||||
Some(entry) => {
|
||||
let recent = now.duration_since(entry.last_seen) < SAME_BOOT_DEBOUNCE;
|
||||
if primary && !recent {
|
||||
// A genuinely new boot. If the previous attempt was
|
||||
// never confirmed, the build we served failed → climb
|
||||
// one rung: Firmware (firmware NIC stack) → Builtin
|
||||
// (iPXE's own drivers) → Shim (signed shim+GRUB —
|
||||
// covers Secure Boot firmware that downloads our
|
||||
// unsigned iPXE but refuses to execute it). Shim is
|
||||
// terminal and pins to disk: SB machines never emit
|
||||
// an iPXE handoff from the signed menu, so reaching
|
||||
// the rung *is* the durable knowledge.
|
||||
if entry.awaiting_confirm {
|
||||
entry.mode = match entry.mode {
|
||||
DriverMode::Firmware => DriverMode::Builtin,
|
||||
DriverMode::Builtin | DriverMode::Shim => DriverMode::Shim,
|
||||
};
|
||||
if entry.mode == DriverMode::Shim && !entry.pinned {
|
||||
entry.pinned = true;
|
||||
newly_pinned = true;
|
||||
}
|
||||
}
|
||||
entry.awaiting_confirm = true;
|
||||
}
|
||||
entry.last_seen = now;
|
||||
entry.mode
|
||||
}
|
||||
};
|
||||
(mode, newly_pinned.then(|| pinned_snapshot(&g.map)))
|
||||
};
|
||||
if let Some(s) = snapshot {
|
||||
self.persist(&s);
|
||||
}
|
||||
mode
|
||||
}
|
||||
|
||||
fn confirm_at(&self, mac: &str, now: Instant) {
|
||||
let mut g = self.inner.lock();
|
||||
if let Some(e) = g.map.get_mut(mac) {
|
||||
let snapshot = {
|
||||
let mut g = self.inner.lock();
|
||||
let Some(e) = g.map.get_mut(mac) else {
|
||||
return;
|
||||
};
|
||||
e.awaiting_confirm = false;
|
||||
e.last_seen = now;
|
||||
// A proven non-default mode is worth remembering forever —
|
||||
// the machine demonstrably can't use the default path.
|
||||
if e.mode != DriverMode::Firmware && !e.pinned {
|
||||
e.pinned = true;
|
||||
Some(pinned_snapshot(&g.map))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some(s) = snapshot {
|
||||
self.persist(&s);
|
||||
}
|
||||
}
|
||||
|
||||
/// Best-effort atomic write of the learned-mode table. No-op for
|
||||
/// ephemeral instances. Failure logs and moves on — persistence is an
|
||||
/// optimization, never a correctness requirement.
|
||||
fn persist(&self, snapshot: &HashMap<String, DriverMode>) {
|
||||
let Some(path) = &self.path else { return };
|
||||
let body = match serde_json::to_vec_pretty(snapshot) {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
tracing::warn!(target: "openpxe::dhcp", "serialize driver_modes.json: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
if let Some(parent) = path.parent() {
|
||||
let _ = std::fs::create_dir_all(parent);
|
||||
}
|
||||
let tmp = path.with_extension("json.tmp");
|
||||
if let Err(e) = std::fs::write(&tmp, body) {
|
||||
tracing::warn!(target: "openpxe::dhcp", "write driver_modes.json tmp: {e}");
|
||||
return;
|
||||
}
|
||||
if let Err(e) = std::fs::rename(&tmp, path.as_path()) {
|
||||
tracing::warn!(target: "openpxe::dhcp", "rename driver_modes.json: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn pinned_snapshot(map: &HashMap<String, Entry>) -> HashMap<String, DriverMode> {
|
||||
map.iter()
|
||||
.filter(|(_, e)| e.pinned)
|
||||
.map(|(k, e)| (k.clone(), e.mode))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn evict_oldest(map: &mut HashMap<String, Entry>) {
|
||||
if let Some(oldest) = map
|
||||
.iter()
|
||||
.min_by_key(|(_, e)| e.last_seen)
|
||||
.map(|(k, _)| k.clone())
|
||||
{
|
||||
// Prefer evicting an unpinned entry; only touch learned modes when
|
||||
// the whole table is pinned (4096 learned machines — at that point
|
||||
// the operator has bigger questions than our memory bound).
|
||||
let pick = |pinned: bool| {
|
||||
map.iter()
|
||||
.filter(|(_, e)| e.pinned == pinned)
|
||||
.min_by_key(|(_, e)| e.last_seen)
|
||||
.map(|(k, _)| k.clone())
|
||||
};
|
||||
if let Some(oldest) = pick(false).or_else(|| pick(true)) {
|
||||
map.remove(&oldest);
|
||||
}
|
||||
}
|
||||
@@ -174,6 +318,7 @@ fn evict_oldest(map: &mut HashMap<String, Entry>) {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn firmware_first_then_escalates_on_unconfirmed_retry() {
|
||||
@@ -250,7 +395,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_entry_is_forgotten_and_resets_to_firmware() {
|
||||
fn stale_unpinned_entry_is_forgotten_and_resets_to_firmware() {
|
||||
let e = DriverEscalation::new();
|
||||
let t0 = Instant::now();
|
||||
assert_eq!(e.decide_at("dd", true, t0), DriverMode::Firmware);
|
||||
@@ -258,8 +403,92 @@ mod tests {
|
||||
e.decide_at("dd", true, t0 + Duration::from_mins(1)),
|
||||
DriverMode::Builtin
|
||||
);
|
||||
// After the TTL with no activity the entry is pruned → fresh firmware.
|
||||
// After the TTL with no activity the (unpinned) Builtin walk is
|
||||
// pruned → fresh firmware. (A *confirmed* Builtin would be pinned
|
||||
// and survive — see learned_builtin_survives_ttl.)
|
||||
let later = t0 + Duration::from_mins(1) + ENTRY_TTL + Duration::from_secs(1);
|
||||
assert_eq!(e.decide_at("dd", true, later), DriverMode::Firmware);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shim_pin_survives_ttl() {
|
||||
// v0.7.1: reaching the Shim rung is durable knowledge — the
|
||||
// machine must NOT re-walk the ladder after an idle period.
|
||||
let e = DriverEscalation::new();
|
||||
let t0 = Instant::now();
|
||||
let _ = e.decide_at("ff", true, t0);
|
||||
let _ = e.decide_at("ff", true, t0 + Duration::from_mins(1));
|
||||
assert_eq!(
|
||||
e.decide_at("ff", true, t0 + Duration::from_mins(2)),
|
||||
DriverMode::Shim
|
||||
);
|
||||
let much_later = t0 + Duration::from_mins(2) + ENTRY_TTL + Duration::from_mins(5);
|
||||
assert_eq!(e.decide_at("ff", true, much_later), DriverMode::Shim);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn learned_builtin_survives_ttl() {
|
||||
let e = DriverEscalation::new();
|
||||
let t0 = Instant::now();
|
||||
let _ = e.decide_at("gg", true, t0);
|
||||
assert_eq!(
|
||||
e.decide_at("gg", true, t0 + Duration::from_mins(1)),
|
||||
DriverMode::Builtin
|
||||
);
|
||||
// The handoff confirms Builtin → pinned.
|
||||
e.confirm_at("gg", t0 + Duration::from_secs(61));
|
||||
let much_later = t0 + ENTRY_TTL + Duration::from_mins(10);
|
||||
assert_eq!(e.decide_at("gg", true, much_later), DriverMode::Builtin);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn learned_modes_persist_across_restart() {
|
||||
let dir = tempdir().unwrap();
|
||||
let t0 = Instant::now();
|
||||
{
|
||||
let e = DriverEscalation::load_or_default(dir.path());
|
||||
// Walk one MAC to Shim (pins on escalation)...
|
||||
let _ = e.decide_at("aa:01", true, t0);
|
||||
let _ = e.decide_at("aa:01", true, t0 + Duration::from_mins(1));
|
||||
assert_eq!(
|
||||
e.decide_at("aa:01", true, t0 + Duration::from_mins(2)),
|
||||
DriverMode::Shim
|
||||
);
|
||||
// ...and another to a confirmed Builtin (pins on handoff).
|
||||
let _ = e.decide_at("aa:02", true, t0);
|
||||
let _ = e.decide_at("aa:02", true, t0 + Duration::from_mins(1));
|
||||
e.confirm_at("aa:02", t0 + Duration::from_secs(61));
|
||||
}
|
||||
// "Restart": a fresh instance from the same work_dir knows both.
|
||||
let e2 = DriverEscalation::load_or_default(dir.path());
|
||||
assert_eq!(e2.decide_at("aa:01", true, t0), DriverMode::Shim);
|
||||
assert_eq!(e2.decide_at("aa:02", true, t0), DriverMode::Builtin);
|
||||
// Unlearned MACs still start at the default.
|
||||
assert_eq!(e2.decide_at("aa:03", true, t0), DriverMode::Firmware);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_persistence_file_starts_empty() {
|
||||
let dir = tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("driver_modes.json"), b"{broken").unwrap();
|
||||
let e = DriverEscalation::load_or_default(dir.path());
|
||||
assert_eq!(
|
||||
e.decide_at("aa:bb", true, Instant::now()),
|
||||
DriverMode::Firmware
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn confirmed_firmware_is_not_persisted() {
|
||||
// The default mode is never written — the file only carries
|
||||
// exceptions, so a healthy fleet leaves it absent/empty.
|
||||
let dir = tempdir().unwrap();
|
||||
let t0 = Instant::now();
|
||||
{
|
||||
let e = DriverEscalation::load_or_default(dir.path());
|
||||
let _ = e.decide_at("aa:09", true, t0);
|
||||
e.confirm_at("aa:09", t0 + Duration::from_secs(2));
|
||||
}
|
||||
assert!(!dir.path().join("driver_modes.json").exists());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,9 @@ use crate::escalation::DriverEscalation;
|
||||
use crate::reply::{build_reply, decide, BootDirective, ReplyContext};
|
||||
use dhcproto::v4::{DhcpOption, Message, OptionCode};
|
||||
use dhcproto::{Decodable, Decoder, Encodable, Encoder};
|
||||
use openpxe_core::{ClientArch, ClientEvent, ClientRegistry, DriverMode, FirmwareClass};
|
||||
use openpxe_core::{
|
||||
BootRulesStore, ClientArch, ClientEvent, ClientRegistry, DriverMode, FirmwareClass,
|
||||
};
|
||||
use socket2::{Domain, Protocol, Socket, Type};
|
||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4};
|
||||
use std::sync::Arc;
|
||||
@@ -19,12 +21,19 @@ pub struct DhcpProxyServer {
|
||||
public_base_url: String,
|
||||
clients: Arc<ClientRegistry>,
|
||||
metrics: openpxe_core::Metrics,
|
||||
/// Automatic per-MAC NIC driver-mode escalation (v0.6.1). Shared across
|
||||
/// the :67 and :4011 listener tasks via the server `Arc`.
|
||||
/// Automatic per-MAC NIC driver-mode escalation (v0.6.1; persistent
|
||||
/// learned modes since v0.7.1). Shared across the :67 and :4011
|
||||
/// listener tasks via the server `Arc`. Built by the caller so the
|
||||
/// persistence path comes from the configured work dir.
|
||||
escalation: DriverEscalation,
|
||||
/// v0.7.1: boot rules — consulted for an operator driver-mode pin
|
||||
/// (e.g. "this OUI is all Secure Boot → serve shim immediately")
|
||||
/// before the automatic escalation ladder.
|
||||
rules: BootRulesStore,
|
||||
}
|
||||
|
||||
impl DhcpProxyServer {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
bind: IpAddr,
|
||||
dhcp_port: u16,
|
||||
@@ -33,6 +42,8 @@ impl DhcpProxyServer {
|
||||
public_base_url: String,
|
||||
clients: Arc<ClientRegistry>,
|
||||
metrics: openpxe_core::Metrics,
|
||||
escalation: DriverEscalation,
|
||||
rules: BootRulesStore,
|
||||
) -> Self {
|
||||
Self {
|
||||
bind,
|
||||
@@ -42,7 +53,8 @@ impl DhcpProxyServer {
|
||||
public_base_url,
|
||||
clients,
|
||||
metrics,
|
||||
escalation: DriverEscalation::new(),
|
||||
escalation,
|
||||
rules,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -142,9 +154,17 @@ impl DhcpProxyServer {
|
||||
self.escalation.mark_ipxe_success(&mac);
|
||||
DriverMode::Firmware // unused: this path serves the HTTP script
|
||||
}
|
||||
FirmwareClass::PxeClient | FirmwareClass::HttpClient => self
|
||||
.escalation
|
||||
.mode_for_firmware_attempt(&mac, label == "67"),
|
||||
FirmwareClass::PxeClient | FirmwareClass::HttpClient => {
|
||||
// v0.7.1: an operator rule pin wins over (and bypasses)
|
||||
// the automatic escalation ladder — known Secure-Boot
|
||||
// fleets boot the signed chain on the very first cycle.
|
||||
if let Some(pinned) = self.rules.driver_mode_hint(&mac, Some(arch.as_str())) {
|
||||
pinned
|
||||
} else {
|
||||
self.escalation
|
||||
.mode_for_firmware_attempt(&mac, label == "67")
|
||||
}
|
||||
}
|
||||
// Unreachable: FirmwareClass::Other returned above.
|
||||
FirmwareClass::Other => DriverMode::Firmware,
|
||||
};
|
||||
|
||||
@@ -35,6 +35,24 @@ pub fn render_grub_menu(isos: &[IsoMeta], base_url: &str) -> String {
|
||||
let dev = grub_http_device(base);
|
||||
let mut s = String::new();
|
||||
let _ = writeln!(s, "# OpenPXE — Secure Boot menu (signed shim+GRUB chain)");
|
||||
// v0.7.1: before showing the limited signed menu, try to hand the
|
||||
// boot back to full iPXE *in this same boot cycle*. With Secure Boot
|
||||
// OFF the chainload succeeds and the client gets the complete iPXE
|
||||
// feature set (sanboot, wimboot, the full menu) despite having been
|
||||
// escalated here. With Secure Boot ON, shim's verifier refuses the
|
||||
// unsigned image INLINE — no reboot, no failed cycle — and execution
|
||||
// falls through to the signed menu below. The all-drivers build is
|
||||
// used because a MAC only lands here after the firmware-net build
|
||||
// already failed once.
|
||||
let _ = writeln!(s, "if [ \"$grub_cpu\" = \"arm64\" ]; then");
|
||||
let _ = writeln!(s, " set openpxe_ipxe=ipxe-arm64.efi");
|
||||
let _ = writeln!(s, "else");
|
||||
let _ = writeln!(s, " set openpxe_ipxe=ipxe.efi");
|
||||
let _ = writeln!(s, "fi");
|
||||
let _ = writeln!(s, "if chainloader {dev}/ipxe/$openpxe_ipxe ; then");
|
||||
let _ = writeln!(s, " boot");
|
||||
let _ = writeln!(s, "fi");
|
||||
let _ = writeln!(s);
|
||||
let _ = writeln!(s, "set timeout=30");
|
||||
let _ = writeln!(s, "set default=0");
|
||||
let _ = writeln!(s);
|
||||
@@ -139,6 +157,24 @@ mod tests {
|
||||
assert!(cfg.contains("Boot from local disk"), "{cfg}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_tries_ipxe_chainload_before_menu() {
|
||||
// v0.7.1: SB-off machines recover full iPXE in the same boot;
|
||||
// SB-on machines fail the chainload inline and reach the menu.
|
||||
let cfg = render_grub_menu(&[linux_iso()], "http://10.0.0.5:8080");
|
||||
let chain_pos = cfg
|
||||
.find("if chainloader (http,10.0.0.5:8080)/ipxe/$openpxe_ipxe ; then")
|
||||
.expect("chainload attempt missing");
|
||||
let menu_pos = cfg.find("menuentry").expect("menu missing");
|
||||
assert!(
|
||||
chain_pos < menu_pos,
|
||||
"chainload must precede the menu:\n{cfg}"
|
||||
);
|
||||
// Arch-conditional binary selection via GRUB's $grub_cpu.
|
||||
assert!(cfg.contains("set openpxe_ipxe=ipxe-arm64.efi"), "{cfg}");
|
||||
assert!(cfg.contains("set openpxe_ipxe=ipxe.efi"), "{cfg}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanboot_and_wimboot_entries_are_omitted() {
|
||||
let mut iso = linux_iso();
|
||||
|
||||
@@ -2751,3 +2751,23 @@ async fn arch_selective_rule_ignores_other_arches() {
|
||||
assert_eq!(s, StatusCode::OK);
|
||||
assert!(String::from_utf8_lossy(&b).contains("boot rule -> fake-alpine-linux"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn boot_rule_driver_mode_pin_round_trips_via_api() {
|
||||
// v0.7.1: a rule may pin only a boot binary (no target) — the API
|
||||
// must persist and return it for the DHCP proxy to consult.
|
||||
let (state, _dir) = build_state().await;
|
||||
let app = build_router(state.clone());
|
||||
let cfg = r#"{"rules":[{"mac_prefix":"aa:bb:cc","arch":"","target":"","driver_mode":"shim","enabled":true,"note":"SB rack"}],"webhook_url":""}"#;
|
||||
let (s, _) = put_json(&app, "/api/boot-rules", cfg).await;
|
||||
assert_eq!(s, StatusCode::NO_CONTENT);
|
||||
let (s, b) = get(&app, "/api/boot-rules").await;
|
||||
assert_eq!(s, StatusCode::OK);
|
||||
let v: serde_json::Value = serde_json::from_slice(&b).unwrap();
|
||||
assert_eq!(v["rules"][0]["driver_mode"], "shim");
|
||||
// And the store the DHCP proxy shares resolves the pin.
|
||||
assert_eq!(
|
||||
state.boot_rules.driver_mode_hint("aa:bb:cc:00:00:07", None),
|
||||
Some(openpxe_core::DriverMode::Shim)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -184,6 +184,10 @@ async fn main() -> anyhow::Result<()> {
|
||||
"network info"
|
||||
);
|
||||
|
||||
// v0.7.1: boot rules are shared between the HTTP layer (target rules,
|
||||
// webhook, the editor API) and the DHCP proxy (driver-mode pins).
|
||||
let boot_rules = openpxe_core::BootRulesStore::load_or_default(&config.paths.work_dir);
|
||||
|
||||
let state = AppState {
|
||||
iso_store: iso_store.clone(),
|
||||
clients: clients.clone(),
|
||||
@@ -191,7 +195,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
queue: queue.clone(),
|
||||
hosts: hosts.clone(),
|
||||
boot_log: boot_log.clone(),
|
||||
boot_rules: openpxe_core::BootRulesStore::load_or_default(&config.paths.work_dir),
|
||||
boot_rules: boot_rules.clone(),
|
||||
boot_tokens: openpxe_core::BootTokens::new(),
|
||||
branding: branding.clone(),
|
||||
pxe_bg_cache: openpxe_http_api::state::PxeBgCache::default(),
|
||||
@@ -267,6 +271,10 @@ async fn main() -> anyhow::Result<()> {
|
||||
public_base_url.clone(),
|
||||
clients.clone(),
|
||||
metrics.clone(),
|
||||
// v0.7.1: learned driver modes persist next to the other
|
||||
// state files, so a machine walks the ladder once *ever*.
|
||||
openpxe_dhcp_proxy::DriverEscalation::load_or_default(&config.paths.work_dir),
|
||||
boot_rules.clone(),
|
||||
);
|
||||
tokio::spawn(s.run())
|
||||
}
|
||||
|
||||
+17
-5
@@ -209,6 +209,13 @@
|
||||
['', 'any arch'], ['bios', 'BIOS'], ['uefi-x64', 'UEFI x64'],
|
||||
['uefi-ia32', 'UEFI IA32'], ['uefi-arm64', 'UEFI ARM64'],
|
||||
];
|
||||
// v0.7.1: optional first-boot binary pin. "Auto" lets the
|
||||
// escalation ladder learn per machine; pinning skips the learning
|
||||
// walk entirely (e.g. a rack known to run Secure Boot → shim).
|
||||
const modeChoices = [
|
||||
['', 'auto (learn)'], ['firmware', 'Firmware NIC'],
|
||||
['builtin', 'iPXE drivers'], ['shim', 'Secure Boot (shim)'],
|
||||
];
|
||||
const rules = (cfg.rules || []).map(r => Object.assign({}, r));
|
||||
const tbody = el('tbody', {});
|
||||
const msg = el('div', {class:'msg'});
|
||||
@@ -224,8 +231,8 @@
|
||||
const redraw = () => {
|
||||
tbody.innerHTML = '';
|
||||
if (!rules.length) {
|
||||
tbody.appendChild(el('tr', {}, el('td', {colspan:'6', class:'empty', style:'padding:14px'},
|
||||
'No rules. Add one to route whole groups of machines (an OUI, an architecture) to a target.')));
|
||||
tbody.appendChild(el('tr', {}, el('td', {colspan:'7', class:'empty', style:'padding:14px'},
|
||||
'No rules. Add one to route whole groups of machines (an OUI, an architecture) to a target — or to pin a boot binary (e.g. Secure Boot racks → shim, zero failed cycles).')));
|
||||
}
|
||||
rules.forEach((r, i) => {
|
||||
const macIn = el('input', {type:'text', spellcheck:'false', placeholder:'aa:bb:cc (prefix)',
|
||||
@@ -235,6 +242,9 @@
|
||||
el('option', Object.assign({value: v}, v === (r.arch || '') ? {selected:''} : {}), label)));
|
||||
const tgtSel = targetSelect(r.target || '');
|
||||
tgtSel.onchange = e => { r.target = e.target.value; };
|
||||
const modeSel = el('select', {onchange: e => { r.driver_mode = e.target.value; }},
|
||||
modeChoices.map(([v, label]) =>
|
||||
el('option', Object.assign({value: v}, v === (r.driver_mode || '') ? {selected:''} : {}), label)));
|
||||
const noteIn = el('input', {type:'text', placeholder:'note',
|
||||
value: r.note || '', oninput: e => { r.note = e.target.value; }});
|
||||
const enabled = el('input', {type:'checkbox', onchange: e => { r.enabled = e.target.checked; }});
|
||||
@@ -243,6 +253,7 @@
|
||||
el('td', {}, macIn),
|
||||
el('td', {}, archSel),
|
||||
el('td', {}, tgtSel),
|
||||
el('td', {}, modeSel),
|
||||
el('td', {}, noteIn),
|
||||
el('td', {style:'text-align:center'}, enabled),
|
||||
el('td', {style:'text-align:right'},
|
||||
@@ -257,8 +268,9 @@
|
||||
redraw();
|
||||
}}, '+ Add rule');
|
||||
const saveBtn = el('button', {onclick: async () => {
|
||||
const bad = rules.find(r => r.enabled !== false && !r.target);
|
||||
if (bad) { msg.textContent = 'Every enabled rule needs a target.'; msg.className = 'msg err'; return; }
|
||||
// A rule needs at least one effect: a target or a boot-binary pin.
|
||||
const bad = rules.find(r => r.enabled !== false && !r.target && !r.driver_mode);
|
||||
if (bad) { msg.textContent = 'Every enabled rule needs a target or a boot-binary pin.'; msg.className = 'msg err'; return; }
|
||||
const r = await putJSON('/api/boot-rules', {rules, webhook_url: webhookInput.value.trim()});
|
||||
if (r.ok) { msg.textContent = 'Saved.'; msg.className = 'msg ok'; }
|
||||
else { msg.textContent = 'Save failed: ' + await r.text(); msg.className = 'msg err'; }
|
||||
@@ -273,7 +285,7 @@
|
||||
el('table', {}, [
|
||||
el('thead', {}, el('tr', {}, [
|
||||
el('th',{},'MAC prefix'), el('th',{},'Arch'), el('th',{},'Target'),
|
||||
el('th',{},'Note'), el('th',{},'On'), el('th',{},''),
|
||||
el('th',{},'Boot binary'), el('th',{},'Note'), el('th',{},'On'), el('th',{},''),
|
||||
])),
|
||||
tbody,
|
||||
]),
|
||||
|
||||
Reference in New Issue
Block a user