Implement explicit restart verification scenario

src/app/restart.rs

@@ -0,0 +1,670 @@
+//! Saved-generation-bound restart preparation, execution, and publication.
+//!
+//! This module coordinates the service job, runtime observation and comparison,
+//! durable verification bookkeeping, and applied state publication. Low-level
+//! service and runtime adapters remain separate.
+
+use std::time::{Duration, Instant};
+
+use crate::backup::{self, BackupReason, HistoricalApplyStatus, SavedBytes};
+use crate::config::ResolvedSettings;
+use crate::domain::{PublicKey, Snapshot};
+use crate::persistence::{BackupComplete, PersistedState, PersistenceError, SavedStore};
+use crate::runtime::transport::TransportError;
+use crate::runtime::{self, RuntimeObservation, SavedPeerProjection, TechnicalMatch};
+use crate::service::{self, RestartDeadline, RestartExecution, RestartOutcome};
+use crate::vault::VaultBundle;
+use crate::wireguard::{ConfigDocument, SectionKind};
+use time::OffsetDateTime;
+use time::format_description::well_known::Rfc3339;
+
+use super::{AppError, Application, clone_saved, historical_apply_from_saved, same_saved_bytes};
+
+/// Read-only, saved-generation-bound restart preview for a future confirmation UI.
+pub struct RestartPreview {
+    saved: SavedBytes,
+    interface: String,
+    unit: String,
+    timeout: Duration,
+    projection: SavedPeerProjection,
+    expected_clients: Option<Snapshot>,
+    has_draft: bool,
+}
+
+impl RestartPreview {
+    pub fn interface(&self) -> &str {
+        &self.interface
+    }
+
+    pub fn systemd_unit(&self) -> &str {
+        &self.unit
+    }
+
+    pub const fn has_draft(&self) -> bool {
+        self.has_draft
+    }
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum RestartChoice {
+    Cancel,
+    RestartSavedState,
+    DiscardDraftAndRestart,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum RestartStage {
+    Cancelled,
+    Preparation,
+    Backup,
+    Service,
+    RuntimeRead,
+    Verification,
+    Persistence,
+    Complete,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum RestartFailureKind {
+    InvalidOrStaleSavedGeneration,
+    UnsafeSaveConfig,
+    UnsupportedSavedShape,
+    BackupFailed,
+    DraftDiscardFailed,
+    ServiceFailed,
+    RuntimeReadFailed,
+    RuntimeNoDevice,
+    RuntimePermissionDenied,
+    RuntimeMalformed,
+    RuntimeTransport,
+    RuntimeTimeout,
+    RuntimeLimitExceeded,
+    RuntimeMismatch,
+    VerificationPersistenceFailed,
+}
+
+/// Secret-free explanation for saved configuration that cannot be used as a
+/// restart verification target. The identity is a public WireGuard key.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct RestartDiagnostic {
+    pub identity: Option<PublicKey>,
+    pub section_line: Option<usize>,
+    pub reason: &'static str,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct RestartReport {
+    pub stage: RestartStage,
+    pub failure: Option<RestartFailureKind>,
+    pub service_failure: Option<service::RestartFailure>,
+    pub service_phase: Option<service::RestartPhase>,
+    pub systemd_unit: Option<String>,
+    pub backup_path: Option<std::path::PathBuf>,
+    pub restart_submitted: bool,
+    pub restart_may_still_complete: bool,
+    pub manual_recovery_required: bool,
+}
+
+pub(super) struct RestartAttempt {
+    pub(super) id: u64,
+    pub(super) saved: SavedBytes,
+}
+
+pub(super) struct PreparedRestart {
+    pub(super) attempt_id: u64,
+    pub(super) installed: SavedBytes,
+    pub(super) settings: ResolvedSettings,
+    pub(super) projection: SavedPeerProjection,
+    pub(super) expected_clients: Option<Snapshot>,
+    pub(super) backup_path: std::path::PathBuf,
+    pub(super) deadline: service::RestartDeadline,
+}
+
+pub(super) fn preview_restart(application: &Application) -> Result<RestartPreview, AppError> {
+    let installed = application.store.read_saved()?;
+    if !same_saved_bytes(&installed, &application.saved) {
+        return Err(AppError::SavedGenerationChanged);
+    }
+    let settings = application.store.resolved_settings_for(&installed)?;
+    let document = ConfigDocument::parse(&installed.server_config)?;
+    if document
+        .sections()
+        .iter()
+        .filter(|section| section.kind == SectionKind::Interface)
+        .count()
+        != 1
+    {
+        return Err(AppError::InvalidSavedState);
+    }
+    if document.has_unsafe_save_config() {
+        return Err(AppError::UnsafeSaveConfig);
+    }
+    let (projection, expected_clients) =
+        restart_projection_from_document(&document).map_err(AppError::UnsafeSavedPeers)?;
+    let expected_clients =
+        expected_clients.filter(|snapshot| snapshot == application.draft.states.saved());
+    Ok(RestartPreview {
+        saved: clone_saved(&installed),
+        interface: settings.interface_name,
+        unit: settings.systemd_unit,
+        timeout: settings.restart_timeout,
+        projection,
+        expected_clients,
+        has_draft: application.has_draft(),
+    })
+}
+
+/// Validate the reviewed preview and preserve the established preparation
+/// order: finalize the pre-restart backup before discarding any draft, then bind
+/// the one-use attempt and initialize the single deadline before service work.
+pub(super) fn prepare_confirmed_restart(
+    application: &mut Application,
+    preview: RestartPreview,
+    choice: RestartChoice,
+    now: &impl Fn() -> Instant,
+) -> Result<PreparedRestart, RestartReport> {
+    if choice == RestartChoice::Cancel {
+        return Err(report(
+            RestartStage::Cancelled,
+            None,
+            None,
+            false,
+            false,
+            false,
+        ));
+    }
+    application.active_restart_attempt = None;
+    if application.has_draft() != preview.has_draft
+        || (preview.has_draft && choice != RestartChoice::DiscardDraftAndRestart)
+        || (!preview.has_draft && choice == RestartChoice::DiscardDraftAndRestart)
+    {
+        return Err(report(
+            RestartStage::Preparation,
+            Some(RestartFailureKind::InvalidOrStaleSavedGeneration),
+            None,
+            false,
+            false,
+            false,
+        ));
+    }
+    let installed = match application.store.read_saved() {
+        Ok(saved)
+            if same_saved_bytes(&saved, &preview.saved)
+                && same_saved_bytes(&saved, &application.saved) =>
+        {
+            saved
+        }
+        _ => {
+            application.active_restart_attempt = None;
+            return Err(report(
+                RestartStage::Preparation,
+                Some(RestartFailureKind::InvalidOrStaleSavedGeneration),
+                None,
+                false,
+                false,
+                false,
+            ));
+        }
+    };
+    let settings = match application.store.resolved_settings_for(&installed) {
+        Ok(settings) => settings,
+        Err(_) => return Err(preparation_failure()),
+    };
+    let document = match ConfigDocument::parse(&installed.server_config) {
+        Ok(document) => document,
+        Err(_) => return Err(preparation_failure()),
+    };
+    if document
+        .sections()
+        .iter()
+        .filter(|section| section.kind == SectionKind::Interface)
+        .count()
+        != 1
+    {
+        return Err(preparation_failure());
+    }
+    if document.has_unsafe_save_config() {
+        return Err(report(
+            RestartStage::Preparation,
+            Some(RestartFailureKind::UnsafeSaveConfig),
+            None,
+            false,
+            false,
+            false,
+        ));
+    }
+    let (projection, expected_clients) = match restart_projection_from_document(&document) {
+        Ok(projection) => projection,
+        Err(_) => {
+            return Err(report(
+                RestartStage::Preparation,
+                Some(RestartFailureKind::UnsupportedSavedShape),
+                None,
+                false,
+                false,
+                false,
+            ));
+        }
+    };
+    let expected_clients =
+        expected_clients.filter(|snapshot| snapshot == application.draft.states.saved());
+    if settings.interface_name != preview.interface
+        || settings.systemd_unit != preview.unit
+        || settings.restart_timeout != preview.timeout
+        || projection != preview.projection
+        || expected_clients != preview.expected_clients
+    {
+        return Err(preparation_failure());
+    }
+    let historical_apply = match historical_apply_from_saved(&installed) {
+        Ok(status) => status,
+        Err(_) => return Err(preparation_failure()),
+    };
+    let backup_path = match backup::create_archive(
+        application.store.archives_directory(),
+        BackupReason::PreRestart,
+        historical_apply,
+        &installed,
+    ) {
+        Ok(path) => path,
+        Err(_) => {
+            application.active_restart_attempt = None;
+            return Err(report(
+                RestartStage::Backup,
+                Some(RestartFailureKind::BackupFailed),
+                None,
+                false,
+                false,
+                false,
+            ));
+        }
+    };
+
+    let id = application.next_restart_attempt;
+    let Some(next_id) = id.checked_add(1) else {
+        return Err(report(
+            RestartStage::Preparation,
+            Some(RestartFailureKind::InvalidOrStaleSavedGeneration),
+            Some(backup_path),
+            false,
+            false,
+            false,
+        ));
+    };
+    application.next_restart_attempt = next_id;
+    application.active_restart_attempt = Some(id);
+    if choice == RestartChoice::DiscardDraftAndRestart && application.discard_draft().is_err() {
+        application.active_restart_attempt = None;
+        return Err(report(
+            RestartStage::Preparation,
+            Some(RestartFailureKind::DraftDiscardFailed),
+            Some(backup_path),
+            false,
+            false,
+            false,
+        ));
+    }
+
+    application.draft.states.runtime_knowledge_unknown();
+    let deadline = service::RestartDeadline::new(now(), settings.restart_timeout);
+    Ok(PreparedRestart {
+        attempt_id: id,
+        installed,
+        settings,
+        projection,
+        expected_clients,
+        backup_path,
+        deadline,
+    })
+}
+
+/// Run the confirmed restart scenario after preparation has completed. The
+/// backup is already durable before this function is entered; exactly one
+/// shared deadline bounds the service job and the subsequent runtime read.
+pub(super) fn execute_confirmed_restart(
+    application: &mut Application,
+    prepared: PreparedRestart,
+    service: impl FnOnce(&str, RestartDeadline) -> RestartExecution,
+    runtime_read: impl FnOnce(&str, Duration) -> Result<RuntimeObservation, TransportError>,
+    install: impl FnOnce(&SavedStore, &SavedBytes, BackupComplete) -> Result<(), PersistenceError>,
+    now: impl Fn() -> Instant,
+) -> RestartReport {
+    let PreparedRestart {
+        attempt_id,
+        installed,
+        settings,
+        projection,
+        expected_clients,
+        backup_path,
+        deadline,
+    } = prepared;
+    let execution = service(&settings.systemd_unit, deadline);
+    let job = execution.outcome;
+    let submitted = execution.submitted;
+    let service_phase = execution.phase;
+    if job != RestartOutcome::Succeeded {
+        application.active_restart_attempt = None;
+        let may_complete = submitted
+            && !matches!(
+                job,
+                RestartOutcome::Failed(
+                    service::RestartFailure::JobFailed
+                        | service::RestartFailure::JobDependency
+                        | service::RestartFailure::JobTimedOut
+                        | service::RestartFailure::JobCanceled
+                        | service::RestartFailure::JobSkipped
+                        | service::RestartFailure::JobResultUnknown
+                )
+            );
+        let mut report = report(
+            RestartStage::Service,
+            Some(RestartFailureKind::ServiceFailed),
+            Some(backup_path),
+            submitted,
+            may_complete,
+            submitted,
+        );
+        report.service_failure = match job {
+            RestartOutcome::Failed(failure) => Some(failure),
+            RestartOutcome::Waiting | RestartOutcome::Succeeded => None,
+        };
+        report.service_phase = Some(service_phase);
+        report.systemd_unit = Some(settings.systemd_unit);
+        return report;
+    }
+    let remaining = deadline.remaining_at(now());
+    if remaining.is_zero() {
+        application.active_restart_attempt = None;
+        return report(
+            RestartStage::RuntimeRead,
+            Some(RestartFailureKind::RuntimeTimeout),
+            Some(backup_path),
+            true,
+            false,
+            true,
+        );
+    }
+    let observation = match runtime_read(&settings.interface_name, remaining) {
+        Ok(observation) => observation,
+        Err(error) => {
+            application.active_restart_attempt = None;
+            return report(
+                RestartStage::RuntimeRead,
+                Some(match error {
+                    TransportError::NoDevice => RestartFailureKind::RuntimeNoDevice,
+                    TransportError::PermissionDenied => RestartFailureKind::RuntimePermissionDenied,
+                    TransportError::Malformed => RestartFailureKind::RuntimeMalformed,
+                    TransportError::Transport => RestartFailureKind::RuntimeTransport,
+                    TransportError::Timeout => RestartFailureKind::RuntimeTimeout,
+                    TransportError::LimitExceeded => RestartFailureKind::RuntimeLimitExceeded,
+                }),
+                Some(backup_path),
+                true,
+                false,
+                true,
+            );
+        }
+    };
+    if deadline.is_expired_at(now()) {
+        application.active_restart_attempt = None;
+        return report(
+            RestartStage::Verification,
+            Some(RestartFailureKind::RuntimeTimeout),
+            Some(backup_path),
+            true,
+            false,
+            true,
+        );
+    }
+    if runtime::compare_peers(&projection, &settings.interface_name, &observation)
+        != TechnicalMatch::Match
+    {
+        application.active_restart_attempt = None;
+        return report(
+            RestartStage::Verification,
+            Some(RestartFailureKind::RuntimeMismatch),
+            Some(backup_path),
+            true,
+            false,
+            true,
+        );
+    }
+    let Some(verified_at) = OffsetDateTime::now_utc()
+        .replace_nanosecond(0)
+        .ok()
+        .and_then(|timestamp| timestamp.format(&Rfc3339).ok())
+    else {
+        application.active_restart_attempt = None;
+        return report(
+            RestartStage::Persistence,
+            Some(RestartFailureKind::VerificationPersistenceFailed),
+            Some(backup_path),
+            true,
+            false,
+            true,
+        );
+    };
+    let attempt = RestartAttempt {
+        id: attempt_id,
+        saved: clone_saved(&installed),
+    };
+    if mark_verified_with_attempt(application, attempt, expected_clients, verified_at, install)
+        .is_err()
+    {
+        application.active_restart_attempt = None;
+        return report(
+            RestartStage::Persistence,
+            Some(RestartFailureKind::VerificationPersistenceFailed),
+            Some(backup_path),
+            true,
+            false,
+            true,
+        );
+    }
+    report(
+        RestartStage::Complete,
+        None,
+        Some(backup_path),
+        true,
+        false,
+        false,
+    )
+}
+
+/// Persist verified apply knowledge using the active restart's one-use attempt
+/// and exact saved generation. Session publication follows durable success.
+pub(super) fn mark_verified_with_attempt(
+    application: &mut Application,
+    attempt: RestartAttempt,
+    observed_management_snapshot: Option<Snapshot>,
+    verified_at_utc: String,
+    install: impl FnOnce(&SavedStore, &SavedBytes, BackupComplete) -> Result<(), PersistenceError>,
+) -> Result<(), AppError> {
+    if application.active_restart_attempt != Some(attempt.id) {
+        return Err(AppError::PreRestartBackupRequired);
+    }
+    let installed = application.store.read_saved()?;
+    if !same_saved_bytes(&installed, &application.saved)
+        || !same_saved_bytes(&installed, &attempt.saved)
+    {
+        return Err(AppError::SavedGenerationChanged);
+    }
+    let mut next_states = application.draft.states.clone();
+    next_states.prepare_technically_verified_result(observed_management_snapshot)?;
+    let mut persisted = match &installed.internal_state {
+        Some(bytes) => PersistedState::from_toml(bytes)?,
+        None => PersistedState::new(
+            next_states
+                .reserved_addresses()
+                .iter()
+                .map(ToString::to_string),
+            None,
+        )?,
+    };
+    persisted.reserved_addresses = next_states
+        .reserved_addresses()
+        .iter()
+        .map(ToString::to_string)
+        .collect();
+    let historical_apply = HistoricalApplyStatus::Verified {
+        verified_at_utc: verified_at_utc.clone(),
+    };
+    persisted.last_verified_at_utc = Some(verified_at_utc);
+    let mut proposed = installed;
+    proposed.internal_state = Some(persisted.to_toml()?);
+    application.store.validate_saved(&proposed)?;
+    if let Err(error) = install(&application.store, &proposed, BackupComplete(())) {
+        application
+            .first_delivery
+            .invalidate_active_authorizations();
+        match application.store.read_saved() {
+            Ok(current) => {
+                let document = ConfigDocument::parse(&current.server_config).ok();
+                let vault = current
+                    .vault
+                    .as_deref()
+                    .and_then(|bytes| std::str::from_utf8(bytes).ok())
+                    .and_then(|text| VaultBundle::from_toml(text).ok());
+                if let Some(document) = document {
+                    application
+                        .first_delivery
+                        .retain_matching(&document, vault.as_ref());
+                } else {
+                    application.first_delivery.clear();
+                }
+            }
+            Err(_) => application.first_delivery.clear(),
+        }
+        return Err(error.into());
+    }
+    application.saved = proposed;
+    application.draft.states = next_states;
+    application.verified_saved_projection = Some(SavedPeerProjection::from_document(
+        &application.draft.document,
+    ));
+    application.historical_apply = historical_apply;
+    application.active_restart_attempt = None;
+    Ok(())
+}
+
+fn preparation_failure() -> RestartReport {
+    report(
+        RestartStage::Preparation,
+        Some(RestartFailureKind::InvalidOrStaleSavedGeneration),
+        None,
+        false,
+        false,
+        false,
+    )
+}
+
+pub(super) fn report(
+    stage: RestartStage,
+    failure: Option<RestartFailureKind>,
+    backup_path: Option<std::path::PathBuf>,
+    restart_submitted: bool,
+    restart_may_still_complete: bool,
+    manual_recovery_required: bool,
+) -> RestartReport {
+    RestartReport {
+        stage,
+        failure,
+        service_failure: None,
+        service_phase: None,
+        systemd_unit: None,
+        backup_path,
+        restart_submitted,
+        restart_may_still_complete,
+        manual_recovery_required,
+    }
+}
+
+pub(super) fn restart_projection_from_document(
+    document: &ConfigDocument,
+) -> Result<(SavedPeerProjection, Option<Snapshot>), Vec<RestartDiagnostic>> {
+    use crate::wireguard::{DocumentIssue, PeerIssue};
+
+    let mut diagnostics = Vec::new();
+    for issue in document.issues() {
+        let (line, reason) = match issue {
+            DocumentIssue::MalformedSectionHeader { line } => (*line, "malformed section header"),
+            DocumentIssue::DirectiveOutsideSection { line } => {
+                (*line, "directive appears outside a section")
+            }
+            DocumentIssue::MalformedDirective { line } => (*line, "malformed directive"),
+            DocumentIssue::ManagedMetadata(_) => continue,
+        };
+        diagnostics.push(RestartDiagnostic {
+            identity: None,
+            section_line: Some(line),
+            reason,
+        });
+    }
+
+    for peer in document.peers() {
+        let identity = peer
+            .identity
+            .or_else(|| peer.public_key_candidates.first().copied());
+        for issue in &peer.issues {
+            let reason = match issue {
+                PeerIssue::MissingPublicKey => Some("peer is missing a public key"),
+                PeerIssue::MalformedPublicKey { .. } => Some("peer has a malformed public key"),
+                PeerIssue::DuplicatePublicKeyField { .. } => {
+                    Some("peer has duplicate public key fields")
+                }
+                PeerIssue::DuplicatePeerIdentity { .. } => {
+                    Some("public key is used by multiple peer sections")
+                }
+                PeerIssue::MalformedAllowedIp { .. } => {
+                    Some("peer has a malformed AllowedIPs item")
+                }
+                PeerIssue::UnsupportedAllowedIpShape { .. } | PeerIssue::MissingIpv4HostRoute => {
+                    None
+                }
+            };
+            if let Some(reason) = reason {
+                diagnostics.push(RestartDiagnostic {
+                    identity,
+                    section_line: Some(peer.section_line),
+                    reason,
+                });
+            }
+        }
+    }
+
+    let projection = SavedPeerProjection::from_document(document);
+    if (projection.document_unverifiable
+        || !projection.unverifiable_peer_indices.is_empty()
+        || !projection.duplicate_public_keys.is_empty())
+        && diagnostics.is_empty()
+    {
+        diagnostics.push(RestartDiagnostic {
+            identity: None,
+            section_line: None,
+            reason: "saved peer set is not uniquely verifiable",
+        });
+    }
+    if !diagnostics.is_empty() {
+        return Err(diagnostics);
+    }
+
+    let mut clients = Vec::with_capacity(document.peers().len());
+    for peer in document.peers() {
+        let Some(identity) = peer.identity else {
+            return Err(vec![RestartDiagnostic {
+                identity: None,
+                section_line: Some(peer.section_line),
+                reason: "peer identity is unavailable",
+            }]);
+        };
+        let Ok(client) = document.project_client(&identity) else {
+            clients.clear();
+            break;
+        };
+        clients.push(client);
+    }
+    let management_snapshot = (clients.len() == document.peers().len())
+        .then(|| Snapshot::new(clients).ok())
+        .flatten();
+    Ok((projection, management_snapshot))
+}