wireguard-admin/src/app/restart.rs

//! Saved-generation-bound restart preparation, execution, and publication.
//!
//! This module coordinates the service job, runtime observation and comparison,
//! durable verification bookkeeping, and applied state publication. Low-level
//! service and runtime adapters remain separate.

use std::time::{Duration, Instant};

use crate::backup::{self, BackupReason, HistoricalApplyStatus, SavedBytes};
use crate::config::ResolvedSettings;
use crate::domain::{PublicKey, Snapshot};
use crate::persistence::{BackupComplete, PersistedState, PersistenceError, SavedStore};
use crate::runtime::transport::TransportError;
use crate::runtime::{self, RuntimeObservation, SavedPeerProjection, TechnicalMatch};
use crate::service::{self, RestartDeadline, RestartExecution, RestartOutcome};
use crate::vault::VaultBundle;
use crate::wireguard::{ConfigDocument, SectionKind};
use time::OffsetDateTime;
use time::format_description::well_known::Rfc3339;

use super::{AppError, Application, clone_saved, historical_apply_from_saved, same_saved_bytes};

/// Read-only, saved-generation-bound preview used by the explicit restart confirmation flow.
pub struct RestartPreview {
    saved: SavedBytes,
    interface: String,
    unit: String,
    timeout: Duration,
    projection: SavedPeerProjection,
    expected_clients: Option<Snapshot>,
    has_draft: bool,
}

impl RestartPreview {
    pub fn interface(&self) -> &str {
        &self.interface
    }

    pub fn systemd_unit(&self) -> &str {
        &self.unit
    }

    pub const fn has_draft(&self) -> bool {
        self.has_draft
    }
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum RestartChoice {
    Cancel,
    RestartSavedState,
    DiscardDraftAndRestart,
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum RestartStage {
    Cancelled,
    Preparation,
    Backup,
    Service,
    RuntimeRead,
    Verification,
    Persistence,
    Complete,
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum RestartFailureKind {
    InvalidOrStaleSavedGeneration,
    UnsafeSaveConfig,
    UnsupportedSavedShape,
    BackupFailed,
    DraftDiscardFailed,
    ServiceFailed,
    RuntimeReadFailed,
    RuntimeNoDevice,
    RuntimePermissionDenied,
    RuntimeMalformed,
    RuntimeTransport,
    RuntimeTimeout,
    RuntimeLimitExceeded,
    RuntimeMismatch,
    VerificationPersistenceFailed,
}

/// Secret-free explanation for saved configuration that cannot be used as a
/// restart verification target. The identity is a public WireGuard key.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct RestartDiagnostic {
    pub identity: Option<PublicKey>,
    pub section_line: Option<usize>,
    pub reason: &'static str,
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct RestartReport {
    pub stage: RestartStage,
    pub failure: Option<RestartFailureKind>,
    pub service_failure: Option<service::RestartFailure>,
    pub service_phase: Option<service::RestartPhase>,
    pub systemd_unit: Option<String>,
    pub backup_path: Option<std::path::PathBuf>,
    pub restart_submitted: bool,
    pub restart_may_still_complete: bool,
    pub manual_recovery_required: bool,
}

pub(super) struct RestartAttempt {
    pub(super) id: u64,
    pub(super) saved: SavedBytes,
}

pub(super) struct PreparedRestart {
    pub(super) attempt_id: u64,
    pub(super) installed: SavedBytes,
    pub(super) settings: ResolvedSettings,
    pub(super) projection: SavedPeerProjection,
    pub(super) expected_clients: Option<Snapshot>,
    pub(super) backup_path: std::path::PathBuf,
    pub(super) deadline: service::RestartDeadline,
}

pub(super) fn preview_restart(application: &Application) -> Result<RestartPreview, AppError> {
    let installed = application.store.read_saved()?;
    if !same_saved_bytes(&installed, &application.saved) {
        return Err(AppError::SavedGenerationChanged);
    }
    let settings = application.store.resolved_settings_for(&installed)?;
    let document = ConfigDocument::parse(&installed.server_config)?;
    if document
        .sections()
        .iter()
        .filter(|section| section.kind == SectionKind::Interface)
        .count()
        != 1
    {
        return Err(AppError::InvalidSavedState);
    }
    if document.has_unsafe_save_config() {
        return Err(AppError::UnsafeSaveConfig);
    }
    let (projection, expected_clients) =
        restart_projection_from_document(&document).map_err(AppError::UnsafeSavedPeers)?;
    let expected_clients =
        expected_clients.filter(|snapshot| snapshot == application.draft.states.saved());
    Ok(RestartPreview {
        saved: clone_saved(&installed),
        interface: settings.interface_name,
        unit: settings.systemd_unit,
        timeout: settings.restart_timeout,
        projection,
        expected_clients,
        has_draft: application.has_draft(),
    })
}

/// Validate the reviewed preview and preserve the established preparation
/// order: finalize the pre-restart backup before discarding any draft, then bind
/// the one-use attempt and initialize the single deadline before service work.
pub(super) fn prepare_confirmed_restart(
    application: &mut Application,
    preview: RestartPreview,
    choice: RestartChoice,
    now: &impl Fn() -> Instant,
) -> Result<PreparedRestart, RestartReport> {
    if choice == RestartChoice::Cancel {
        return Err(report(
            RestartStage::Cancelled,
            None,
            None,
            false,
            false,
            false,
        ));
    }
    application.active_restart_attempt = None;
    if application.has_draft() != preview.has_draft
        || (preview.has_draft && choice != RestartChoice::DiscardDraftAndRestart)
        || (!preview.has_draft && choice == RestartChoice::DiscardDraftAndRestart)
    {
        return Err(report(
            RestartStage::Preparation,
            Some(RestartFailureKind::InvalidOrStaleSavedGeneration),
            None,
            false,
            false,
            false,
        ));
    }
    let installed = match application.store.read_saved() {
        Ok(saved)
            if same_saved_bytes(&saved, &preview.saved)
                && same_saved_bytes(&saved, &application.saved) =>
        {
            saved
        }
        _ => {
            application.active_restart_attempt = None;
            return Err(report(
                RestartStage::Preparation,
                Some(RestartFailureKind::InvalidOrStaleSavedGeneration),
                None,
                false,
                false,
                false,
            ));
        }
    };
    let settings = match application.store.resolved_settings_for(&installed) {
        Ok(settings) => settings,
        Err(_) => return Err(preparation_failure()),
    };
    let document = match ConfigDocument::parse(&installed.server_config) {
        Ok(document) => document,
        Err(_) => return Err(preparation_failure()),
    };
    if document
        .sections()
        .iter()
        .filter(|section| section.kind == SectionKind::Interface)
        .count()
        != 1
    {
        return Err(preparation_failure());
    }
    if document.has_unsafe_save_config() {
        return Err(report(
            RestartStage::Preparation,
            Some(RestartFailureKind::UnsafeSaveConfig),
            None,
            false,
            false,
            false,
        ));
    }
    let (projection, expected_clients) = match restart_projection_from_document(&document) {
        Ok(projection) => projection,
        Err(_) => {
            return Err(report(
                RestartStage::Preparation,
                Some(RestartFailureKind::UnsupportedSavedShape),
                None,
                false,
                false,
                false,
            ));
        }
    };
    let expected_clients =
        expected_clients.filter(|snapshot| snapshot == application.draft.states.saved());
    if settings.interface_name != preview.interface
        || settings.systemd_unit != preview.unit
        || settings.restart_timeout != preview.timeout
        || projection != preview.projection
        || expected_clients != preview.expected_clients
    {
        return Err(preparation_failure());
    }
    let historical_apply = match historical_apply_from_saved(&installed) {
        Ok(status) => status,
        Err(_) => return Err(preparation_failure()),
    };
    let backup_path = match backup::create_archive(
        application.store.archives_directory(),
        BackupReason::PreRestart,
        historical_apply,
        &installed,
    ) {
        Ok(path) => path,
        Err(_) => {
            application.active_restart_attempt = None;
            return Err(report(
                RestartStage::Backup,
                Some(RestartFailureKind::BackupFailed),
                None,
                false,
                false,
                false,
            ));
        }
    };

    let id = application.next_restart_attempt;
    let Some(next_id) = id.checked_add(1) else {
        return Err(report(
            RestartStage::Preparation,
            Some(RestartFailureKind::InvalidOrStaleSavedGeneration),
            Some(backup_path),
            false,
            false,
            false,
        ));
    };
    application.next_restart_attempt = next_id;
    application.active_restart_attempt = Some(id);
    if choice == RestartChoice::DiscardDraftAndRestart && application.discard_draft().is_err() {
        application.active_restart_attempt = None;
        return Err(report(
            RestartStage::Preparation,
            Some(RestartFailureKind::DraftDiscardFailed),
            Some(backup_path),
            false,
            false,
            false,
        ));
    }

    application.draft.states.runtime_knowledge_unknown();
    let deadline = service::RestartDeadline::new(now(), settings.restart_timeout);
    Ok(PreparedRestart {
        attempt_id: id,
        installed,
        settings,
        projection,
        expected_clients,
        backup_path,
        deadline,
    })
}

/// Run the confirmed restart scenario after preparation has completed. The
/// backup is already durable before this function is entered; exactly one
/// shared deadline bounds the service job and the subsequent runtime read.
pub(super) fn execute_confirmed_restart(
    application: &mut Application,
    prepared: PreparedRestart,
    service: impl FnOnce(&str, RestartDeadline) -> RestartExecution,
    runtime_read: impl FnOnce(&str, Duration) -> Result<RuntimeObservation, TransportError>,
    install: impl FnOnce(&SavedStore, &SavedBytes, BackupComplete) -> Result<(), PersistenceError>,
    now: impl Fn() -> Instant,
) -> RestartReport {
    let PreparedRestart {
        attempt_id,
        installed,
        settings,
        projection,
        expected_clients,
        backup_path,
        deadline,
    } = prepared;
    let execution = service(&settings.systemd_unit, deadline);
    let job = execution.outcome;
    let submitted = execution.submitted;
    let service_phase = execution.phase;
    if job != RestartOutcome::Succeeded {
        application.active_restart_attempt = None;
        let may_complete = submitted
            && !matches!(
                job,
                RestartOutcome::Failed(
                    service::RestartFailure::JobFailed
                        | service::RestartFailure::JobDependency
                        | service::RestartFailure::JobTimedOut
                        | service::RestartFailure::JobCanceled
                        | service::RestartFailure::JobSkipped
                        | service::RestartFailure::JobResultUnknown
                )
            );
        let mut report = report(
            RestartStage::Service,
            Some(RestartFailureKind::ServiceFailed),
            Some(backup_path),
            submitted,
            may_complete,
            submitted,
        );
        report.service_failure = match job {
            RestartOutcome::Failed(failure) => Some(failure),
            RestartOutcome::Waiting | RestartOutcome::Succeeded => None,
        };
        report.service_phase = Some(service_phase);
        report.systemd_unit = Some(settings.systemd_unit);
        return report;
    }
    let remaining = deadline.remaining_at(now());
    if remaining.is_zero() {
        application.active_restart_attempt = None;
        return report(
            RestartStage::RuntimeRead,
            Some(RestartFailureKind::RuntimeTimeout),
            Some(backup_path),
            true,
            false,
            true,
        );
    }
    let observation = match runtime_read(&settings.interface_name, remaining) {
        Ok(observation) => observation,
        Err(error) => {
            application.active_restart_attempt = None;
            return report(
                RestartStage::RuntimeRead,
                Some(match error {
                    TransportError::NoDevice => RestartFailureKind::RuntimeNoDevice,
                    TransportError::PermissionDenied => RestartFailureKind::RuntimePermissionDenied,
                    TransportError::Malformed => RestartFailureKind::RuntimeMalformed,
                    TransportError::Transport => RestartFailureKind::RuntimeTransport,
                    TransportError::Timeout => RestartFailureKind::RuntimeTimeout,
                    TransportError::LimitExceeded => RestartFailureKind::RuntimeLimitExceeded,
                }),
                Some(backup_path),
                true,
                false,
                true,
            );
        }
    };
    if deadline.is_expired_at(now()) {
        application.active_restart_attempt = None;
        return report(
            RestartStage::Verification,
            Some(RestartFailureKind::RuntimeTimeout),
            Some(backup_path),
            true,
            false,
            true,
        );
    }
    if runtime::compare_peers(&projection, &settings.interface_name, &observation)
        != TechnicalMatch::Match
    {
        application.active_restart_attempt = None;
        return report(
            RestartStage::Verification,
            Some(RestartFailureKind::RuntimeMismatch),
            Some(backup_path),
            true,
            false,
            true,
        );
    }
    let Some(verified_at) = OffsetDateTime::now_utc()
        .replace_nanosecond(0)
        .ok()
        .and_then(|timestamp| timestamp.format(&Rfc3339).ok())
    else {
        application.active_restart_attempt = None;
        return report(
            RestartStage::Persistence,
            Some(RestartFailureKind::VerificationPersistenceFailed),
            Some(backup_path),
            true,
            false,
            true,
        );
    };
    let attempt = RestartAttempt {
        id: attempt_id,
        saved: clone_saved(&installed),
    };
    if mark_verified_with_attempt(application, attempt, expected_clients, verified_at, install)
        .is_err()
    {
        application.active_restart_attempt = None;
        return report(
            RestartStage::Persistence,
            Some(RestartFailureKind::VerificationPersistenceFailed),
            Some(backup_path),
            true,
            false,
            true,
        );
    }
    report(
        RestartStage::Complete,
        None,
        Some(backup_path),
        true,
        false,
        false,
    )
}

/// Persist verified apply knowledge using the active restart's one-use attempt
/// and exact saved generation. Session publication follows durable success.
pub(super) fn mark_verified_with_attempt(
    application: &mut Application,
    attempt: RestartAttempt,
    observed_management_snapshot: Option<Snapshot>,
    verified_at_utc: String,
    install: impl FnOnce(&SavedStore, &SavedBytes, BackupComplete) -> Result<(), PersistenceError>,
) -> Result<(), AppError> {
    if application.active_restart_attempt != Some(attempt.id) {
        return Err(AppError::PreRestartBackupRequired);
    }
    let installed = application.store.read_saved()?;
    if !same_saved_bytes(&installed, &application.saved)
        || !same_saved_bytes(&installed, &attempt.saved)
    {
        return Err(AppError::SavedGenerationChanged);
    }
    let mut next_states = application.draft.states.clone();
    next_states.prepare_technically_verified_result(observed_management_snapshot)?;
    let mut persisted = match &installed.internal_state {
        Some(bytes) => PersistedState::from_toml(bytes)?,
        None => PersistedState::new(
            next_states
                .reserved_addresses()
                .iter()
                .map(ToString::to_string),
            None,
        )?,
    };
    persisted.reserved_addresses = next_states
        .reserved_addresses()
        .iter()
        .map(ToString::to_string)
        .collect();
    let historical_apply = HistoricalApplyStatus::Verified {
        verified_at_utc: verified_at_utc.clone(),
    };
    persisted.last_verified_at_utc = Some(verified_at_utc);
    let mut proposed = installed;
    proposed.internal_state = Some(persisted.to_toml()?);
    application.store.validate_saved(&proposed)?;
    if let Err(error) = install(&application.store, &proposed, BackupComplete(())) {
        application
            .first_delivery
            .invalidate_active_authorizations();
        match application.store.read_saved() {
            Ok(current) => {
                let document = ConfigDocument::parse(&current.server_config).ok();
                let vault = current
                    .vault
                    .as_deref()
                    .and_then(|bytes| std::str::from_utf8(bytes).ok())
                    .and_then(|text| VaultBundle::from_toml(text).ok());
                if let Some(document) = document {
                    application
                        .first_delivery
                        .retain_matching(&document, vault.as_ref());
                } else {
                    application.first_delivery.clear();
                }
            }
            Err(_) => application.first_delivery.clear(),
        }
        return Err(error.into());
    }
    application.saved = proposed;
    application.draft.states = next_states;
    application.verified_saved_projection = Some(SavedPeerProjection::from_document(
        &application.draft.document,
    ));
    application.historical_apply = historical_apply;
    application.active_restart_attempt = None;
    Ok(())
}

fn preparation_failure() -> RestartReport {
    report(
        RestartStage::Preparation,
        Some(RestartFailureKind::InvalidOrStaleSavedGeneration),
        None,
        false,
        false,
        false,
    )
}

pub(super) fn report(
    stage: RestartStage,
    failure: Option<RestartFailureKind>,
    backup_path: Option<std::path::PathBuf>,
    restart_submitted: bool,
    restart_may_still_complete: bool,
    manual_recovery_required: bool,
) -> RestartReport {
    RestartReport {
        stage,
        failure,
        service_failure: None,
        service_phase: None,
        systemd_unit: None,
        backup_path,
        restart_submitted,
        restart_may_still_complete,
        manual_recovery_required,
    }
}

pub(super) fn restart_projection_from_document(
    document: &ConfigDocument,
) -> Result<(SavedPeerProjection, Option<Snapshot>), Vec<RestartDiagnostic>> {
    use crate::wireguard::{DocumentIssue, PeerIssue};

    let mut diagnostics = Vec::new();
    for issue in document.issues() {
        let (line, reason) = match issue {
            DocumentIssue::MalformedSectionHeader { line } => (*line, "malformed section header"),
            DocumentIssue::DirectiveOutsideSection { line } => {
                (*line, "directive appears outside a section")
            }
            DocumentIssue::MalformedDirective { line } => (*line, "malformed directive"),
            DocumentIssue::ManagedMetadata(_) => continue,
        };
        diagnostics.push(RestartDiagnostic {
            identity: None,
            section_line: Some(line),
            reason,
        });
    }

    for peer in document.peers() {
        let identity = peer
            .identity
            .or_else(|| peer.public_key_candidates.first().copied());
        for issue in &peer.issues {
            let reason = match issue {
                PeerIssue::MissingPublicKey => Some("peer is missing a public key"),
                PeerIssue::MalformedPublicKey { .. } => Some("peer has a malformed public key"),
                PeerIssue::DuplicatePublicKeyField { .. } => {
                    Some("peer has duplicate public key fields")
                }
                PeerIssue::DuplicatePeerIdentity { .. } => {
                    Some("public key is used by multiple peer sections")
                }
                PeerIssue::MalformedAllowedIp { .. } => {
                    Some("peer has a malformed AllowedIPs item")
                }
                PeerIssue::UnsupportedAllowedIpShape { .. } | PeerIssue::MissingIpv4HostRoute => {
                    None
                }
            };
            if let Some(reason) = reason {
                diagnostics.push(RestartDiagnostic {
                    identity,
                    section_line: Some(peer.section_line),
                    reason,
                });
            }
        }
    }

    let projection = SavedPeerProjection::from_document(document);
    if (projection.document_unverifiable
        || !projection.unverifiable_peer_indices.is_empty()
        || !projection.duplicate_public_keys.is_empty())
        && diagnostics.is_empty()
    {
        diagnostics.push(RestartDiagnostic {
            identity: None,
            section_line: None,
            reason: "saved peer set is not uniquely verifiable",
        });
    }
    if !diagnostics.is_empty() {
        return Err(diagnostics);
    }

    let mut clients = Vec::with_capacity(document.peers().len());
    for peer in document.peers() {
        let Some(identity) = peer.identity else {
            return Err(vec![RestartDiagnostic {
                identity: None,
                section_line: Some(peer.section_line),
                reason: "peer identity is unavailable",
            }]);
        };
        let Ok(client) = document.project_client(&identity) else {
            clients.clear();
            break;
        };
        clients.push(client);
    }
    let management_snapshot = (clients.len() == document.peers().len())
        .then(|| Snapshot::new(clients).ok())
        .flatten();
    Ok((projection, management_snapshot))
}