//! Saved-generation-bound restart preparation, execution, and publication. //! //! This module coordinates the service job, runtime observation and comparison, //! durable verification bookkeeping, and applied state publication. Low-level //! service and runtime adapters remain separate. use std::time::{Duration, Instant}; use crate::backup::{self, BackupReason, HistoricalApplyStatus, SavedBytes}; use crate::config::ResolvedSettings; use crate::domain::{PublicKey, Snapshot}; use crate::persistence::{BackupComplete, PersistedState, PersistenceError, SavedStore}; use crate::runtime::transport::TransportError; use crate::runtime::{self, RuntimeObservation, SavedPeerProjection, TechnicalMatch}; use crate::service::{self, RestartDeadline, RestartExecution, RestartOutcome}; use crate::vault::VaultBundle; use crate::wireguard::{ConfigDocument, SectionKind}; use time::OffsetDateTime; use time::format_description::well_known::Rfc3339; use super::{AppError, Application, clone_saved, historical_apply_from_saved, same_saved_bytes}; /// Read-only, saved-generation-bound preview used by the explicit restart confirmation flow. pub struct RestartPreview { saved: SavedBytes, interface: String, unit: String, timeout: Duration, projection: SavedPeerProjection, expected_clients: Option, has_draft: bool, } impl RestartPreview { pub fn interface(&self) -> &str { &self.interface } pub fn systemd_unit(&self) -> &str { &self.unit } pub const fn has_draft(&self) -> bool { self.has_draft } } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum RestartChoice { Cancel, RestartSavedState, DiscardDraftAndRestart, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum RestartStage { Cancelled, Preparation, Backup, Service, RuntimeRead, Verification, Persistence, Complete, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum RestartFailureKind { InvalidOrStaleSavedGeneration, UnsafeSaveConfig, UnsupportedSavedShape, BackupFailed, DraftDiscardFailed, ServiceFailed, RuntimeReadFailed, RuntimeNoDevice, RuntimePermissionDenied, RuntimeMalformed, RuntimeTransport, RuntimeTimeout, RuntimeLimitExceeded, RuntimeMismatch, VerificationPersistenceFailed, } /// Secret-free explanation for saved configuration that cannot be used as a /// restart verification target. The identity is a public WireGuard key. #[derive(Clone, Debug, Eq, PartialEq)] pub struct RestartDiagnostic { pub identity: Option, pub section_line: Option, pub reason: &'static str, } #[derive(Clone, Debug, Eq, PartialEq)] pub struct RestartReport { pub stage: RestartStage, pub failure: Option, pub service_failure: Option, pub service_phase: Option, pub systemd_unit: Option, pub backup_path: Option, pub restart_submitted: bool, pub restart_may_still_complete: bool, pub manual_recovery_required: bool, } pub(super) struct RestartAttempt { pub(super) id: u64, pub(super) saved: SavedBytes, } pub(super) struct PreparedRestart { pub(super) attempt_id: u64, pub(super) installed: SavedBytes, pub(super) settings: ResolvedSettings, pub(super) projection: SavedPeerProjection, pub(super) expected_clients: Option, pub(super) backup_path: std::path::PathBuf, pub(super) deadline: service::RestartDeadline, } pub(super) fn preview_restart(application: &Application) -> Result { let installed = application.store.read_saved()?; if !same_saved_bytes(&installed, &application.saved) { return Err(AppError::SavedGenerationChanged); } let settings = application.store.resolved_settings_for(&installed)?; let document = ConfigDocument::parse(&installed.server_config)?; if document .sections() .iter() .filter(|section| section.kind == SectionKind::Interface) .count() != 1 { return Err(AppError::InvalidSavedState); } if document.has_unsafe_save_config() { return Err(AppError::UnsafeSaveConfig); } let (projection, expected_clients) = restart_projection_from_document(&document).map_err(AppError::UnsafeSavedPeers)?; let expected_clients = expected_clients.filter(|snapshot| snapshot == application.draft.states.saved()); Ok(RestartPreview { saved: clone_saved(&installed), interface: settings.interface_name, unit: settings.systemd_unit, timeout: settings.restart_timeout, projection, expected_clients, has_draft: application.has_draft(), }) } /// Validate the reviewed preview and preserve the established preparation /// order: finalize the pre-restart backup before discarding any draft, then bind /// the one-use attempt and initialize the single deadline before service work. pub(super) fn prepare_confirmed_restart( application: &mut Application, preview: RestartPreview, choice: RestartChoice, now: &impl Fn() -> Instant, ) -> Result { if choice == RestartChoice::Cancel { return Err(report( RestartStage::Cancelled, None, None, false, false, false, )); } application.active_restart_attempt = None; if application.has_draft() != preview.has_draft || (preview.has_draft && choice != RestartChoice::DiscardDraftAndRestart) || (!preview.has_draft && choice == RestartChoice::DiscardDraftAndRestart) { return Err(report( RestartStage::Preparation, Some(RestartFailureKind::InvalidOrStaleSavedGeneration), None, false, false, false, )); } let installed = match application.store.read_saved() { Ok(saved) if same_saved_bytes(&saved, &preview.saved) && same_saved_bytes(&saved, &application.saved) => { saved } _ => { application.active_restart_attempt = None; return Err(report( RestartStage::Preparation, Some(RestartFailureKind::InvalidOrStaleSavedGeneration), None, false, false, false, )); } }; let settings = match application.store.resolved_settings_for(&installed) { Ok(settings) => settings, Err(_) => return Err(preparation_failure()), }; let document = match ConfigDocument::parse(&installed.server_config) { Ok(document) => document, Err(_) => return Err(preparation_failure()), }; if document .sections() .iter() .filter(|section| section.kind == SectionKind::Interface) .count() != 1 { return Err(preparation_failure()); } if document.has_unsafe_save_config() { return Err(report( RestartStage::Preparation, Some(RestartFailureKind::UnsafeSaveConfig), None, false, false, false, )); } let (projection, expected_clients) = match restart_projection_from_document(&document) { Ok(projection) => projection, Err(_) => { return Err(report( RestartStage::Preparation, Some(RestartFailureKind::UnsupportedSavedShape), None, false, false, false, )); } }; let expected_clients = expected_clients.filter(|snapshot| snapshot == application.draft.states.saved()); if settings.interface_name != preview.interface || settings.systemd_unit != preview.unit || settings.restart_timeout != preview.timeout || projection != preview.projection || expected_clients != preview.expected_clients { return Err(preparation_failure()); } let historical_apply = match historical_apply_from_saved(&installed) { Ok(status) => status, Err(_) => return Err(preparation_failure()), }; let backup_path = match backup::create_archive( application.store.archives_directory(), BackupReason::PreRestart, historical_apply, &installed, ) { Ok(path) => path, Err(_) => { application.active_restart_attempt = None; return Err(report( RestartStage::Backup, Some(RestartFailureKind::BackupFailed), None, false, false, false, )); } }; let id = application.next_restart_attempt; let Some(next_id) = id.checked_add(1) else { return Err(report( RestartStage::Preparation, Some(RestartFailureKind::InvalidOrStaleSavedGeneration), Some(backup_path), false, false, false, )); }; application.next_restart_attempt = next_id; application.active_restart_attempt = Some(id); if choice == RestartChoice::DiscardDraftAndRestart && application.discard_draft().is_err() { application.active_restart_attempt = None; return Err(report( RestartStage::Preparation, Some(RestartFailureKind::DraftDiscardFailed), Some(backup_path), false, false, false, )); } application.draft.states.runtime_knowledge_unknown(); let deadline = service::RestartDeadline::new(now(), settings.restart_timeout); Ok(PreparedRestart { attempt_id: id, installed, settings, projection, expected_clients, backup_path, deadline, }) } /// Run the confirmed restart scenario after preparation has completed. The /// backup is already durable before this function is entered; exactly one /// shared deadline bounds the service job and the subsequent runtime read. pub(super) fn execute_confirmed_restart( application: &mut Application, prepared: PreparedRestart, service: impl FnOnce(&str, RestartDeadline) -> RestartExecution, runtime_read: impl FnOnce(&str, Duration) -> Result, install: impl FnOnce(&SavedStore, &SavedBytes, BackupComplete) -> Result<(), PersistenceError>, now: impl Fn() -> Instant, ) -> RestartReport { let PreparedRestart { attempt_id, installed, settings, projection, expected_clients, backup_path, deadline, } = prepared; let execution = service(&settings.systemd_unit, deadline); let job = execution.outcome; let submitted = execution.submitted; let service_phase = execution.phase; if job != RestartOutcome::Succeeded { application.active_restart_attempt = None; let may_complete = submitted && !matches!( job, RestartOutcome::Failed( service::RestartFailure::JobFailed | service::RestartFailure::JobDependency | service::RestartFailure::JobTimedOut | service::RestartFailure::JobCanceled | service::RestartFailure::JobSkipped | service::RestartFailure::JobResultUnknown ) ); let mut report = report( RestartStage::Service, Some(RestartFailureKind::ServiceFailed), Some(backup_path), submitted, may_complete, submitted, ); report.service_failure = match job { RestartOutcome::Failed(failure) => Some(failure), RestartOutcome::Waiting | RestartOutcome::Succeeded => None, }; report.service_phase = Some(service_phase); report.systemd_unit = Some(settings.systemd_unit); return report; } let remaining = deadline.remaining_at(now()); if remaining.is_zero() { application.active_restart_attempt = None; return report( RestartStage::RuntimeRead, Some(RestartFailureKind::RuntimeTimeout), Some(backup_path), true, false, true, ); } let observation = match runtime_read(&settings.interface_name, remaining) { Ok(observation) => observation, Err(error) => { application.active_restart_attempt = None; return report( RestartStage::RuntimeRead, Some(match error { TransportError::NoDevice => RestartFailureKind::RuntimeNoDevice, TransportError::PermissionDenied => RestartFailureKind::RuntimePermissionDenied, TransportError::Malformed => RestartFailureKind::RuntimeMalformed, TransportError::Transport => RestartFailureKind::RuntimeTransport, TransportError::Timeout => RestartFailureKind::RuntimeTimeout, TransportError::LimitExceeded => RestartFailureKind::RuntimeLimitExceeded, }), Some(backup_path), true, false, true, ); } }; if deadline.is_expired_at(now()) { application.active_restart_attempt = None; return report( RestartStage::Verification, Some(RestartFailureKind::RuntimeTimeout), Some(backup_path), true, false, true, ); } if runtime::compare_peers(&projection, &settings.interface_name, &observation) != TechnicalMatch::Match { application.active_restart_attempt = None; return report( RestartStage::Verification, Some(RestartFailureKind::RuntimeMismatch), Some(backup_path), true, false, true, ); } let Some(verified_at) = OffsetDateTime::now_utc() .replace_nanosecond(0) .ok() .and_then(|timestamp| timestamp.format(&Rfc3339).ok()) else { application.active_restart_attempt = None; return report( RestartStage::Persistence, Some(RestartFailureKind::VerificationPersistenceFailed), Some(backup_path), true, false, true, ); }; let attempt = RestartAttempt { id: attempt_id, saved: clone_saved(&installed), }; if mark_verified_with_attempt(application, attempt, expected_clients, verified_at, install) .is_err() { application.active_restart_attempt = None; return report( RestartStage::Persistence, Some(RestartFailureKind::VerificationPersistenceFailed), Some(backup_path), true, false, true, ); } report( RestartStage::Complete, None, Some(backup_path), true, false, false, ) } /// Persist verified apply knowledge using the active restart's one-use attempt /// and exact saved generation. Session publication follows durable success. pub(super) fn mark_verified_with_attempt( application: &mut Application, attempt: RestartAttempt, observed_management_snapshot: Option, verified_at_utc: String, install: impl FnOnce(&SavedStore, &SavedBytes, BackupComplete) -> Result<(), PersistenceError>, ) -> Result<(), AppError> { if application.active_restart_attempt != Some(attempt.id) { return Err(AppError::PreRestartBackupRequired); } let installed = application.store.read_saved()?; if !same_saved_bytes(&installed, &application.saved) || !same_saved_bytes(&installed, &attempt.saved) { return Err(AppError::SavedGenerationChanged); } let mut next_states = application.draft.states.clone(); next_states.prepare_technically_verified_result(observed_management_snapshot)?; let mut persisted = match &installed.internal_state { Some(bytes) => PersistedState::from_toml(bytes)?, None => PersistedState::new( next_states .reserved_addresses() .iter() .map(ToString::to_string), None, )?, }; persisted.reserved_addresses = next_states .reserved_addresses() .iter() .map(ToString::to_string) .collect(); let historical_apply = HistoricalApplyStatus::Verified { verified_at_utc: verified_at_utc.clone(), }; persisted.last_verified_at_utc = Some(verified_at_utc); let mut proposed = installed; proposed.internal_state = Some(persisted.to_toml()?); application.store.validate_saved(&proposed)?; if let Err(error) = install(&application.store, &proposed, BackupComplete(())) { application .first_delivery .invalidate_active_authorizations(); match application.store.read_saved() { Ok(current) => { let document = ConfigDocument::parse(¤t.server_config).ok(); let vault = current .vault .as_deref() .and_then(|bytes| std::str::from_utf8(bytes).ok()) .and_then(|text| VaultBundle::from_toml(text).ok()); if let Some(document) = document { application .first_delivery .retain_matching(&document, vault.as_ref()); } else { application.first_delivery.clear(); } } Err(_) => application.first_delivery.clear(), } return Err(error.into()); } application.saved = proposed; application.draft.states = next_states; application.verified_saved_projection = Some(SavedPeerProjection::from_document( &application.draft.document, )); application.historical_apply = historical_apply; application.active_restart_attempt = None; Ok(()) } fn preparation_failure() -> RestartReport { report( RestartStage::Preparation, Some(RestartFailureKind::InvalidOrStaleSavedGeneration), None, false, false, false, ) } pub(super) fn report( stage: RestartStage, failure: Option, backup_path: Option, restart_submitted: bool, restart_may_still_complete: bool, manual_recovery_required: bool, ) -> RestartReport { RestartReport { stage, failure, service_failure: None, service_phase: None, systemd_unit: None, backup_path, restart_submitted, restart_may_still_complete, manual_recovery_required, } } pub(super) fn restart_projection_from_document( document: &ConfigDocument, ) -> Result<(SavedPeerProjection, Option), Vec> { use crate::wireguard::{DocumentIssue, PeerIssue}; let mut diagnostics = Vec::new(); for issue in document.issues() { let (line, reason) = match issue { DocumentIssue::MalformedSectionHeader { line } => (*line, "malformed section header"), DocumentIssue::DirectiveOutsideSection { line } => { (*line, "directive appears outside a section") } DocumentIssue::MalformedDirective { line } => (*line, "malformed directive"), DocumentIssue::ManagedMetadata(_) => continue, }; diagnostics.push(RestartDiagnostic { identity: None, section_line: Some(line), reason, }); } for peer in document.peers() { let identity = peer .identity .or_else(|| peer.public_key_candidates.first().copied()); for issue in &peer.issues { let reason = match issue { PeerIssue::MissingPublicKey => Some("peer is missing a public key"), PeerIssue::MalformedPublicKey { .. } => Some("peer has a malformed public key"), PeerIssue::DuplicatePublicKeyField { .. } => { Some("peer has duplicate public key fields") } PeerIssue::DuplicatePeerIdentity { .. } => { Some("public key is used by multiple peer sections") } PeerIssue::MalformedAllowedIp { .. } => { Some("peer has a malformed AllowedIPs item") } PeerIssue::UnsupportedAllowedIpShape { .. } | PeerIssue::MissingIpv4HostRoute => { None } }; if let Some(reason) = reason { diagnostics.push(RestartDiagnostic { identity, section_line: Some(peer.section_line), reason, }); } } } let projection = SavedPeerProjection::from_document(document); if (projection.document_unverifiable || !projection.unverifiable_peer_indices.is_empty() || !projection.duplicate_public_keys.is_empty()) && diagnostics.is_empty() { diagnostics.push(RestartDiagnostic { identity: None, section_line: None, reason: "saved peer set is not uniquely verifiable", }); } if !diagnostics.is_empty() { return Err(diagnostics); } let mut clients = Vec::with_capacity(document.peers().len()); for peer in document.peers() { let Some(identity) = peer.identity else { return Err(vec![RestartDiagnostic { identity: None, section_line: Some(peer.section_line), reason: "peer identity is unavailable", }]); }; let Ok(client) = document.project_client(&identity) else { clients.clear(); break; }; clients.push(client); } let management_snapshot = (clients.len() == document.peers().len()) .then(|| Snapshot::new(clients).ok()) .flatten(); Ok((projection, management_snapshot)) }