Add runtime observation and systemd restart adapter

This commit is contained in:
2026-10-20 23:01:43 +03:00
parent 5418841b92
commit 6ec593c783
18 changed files with 6001 additions and 66 deletions
+680
View File
@@ -0,0 +1,680 @@
//! Bounded asynchronous systemd D-Bus restart adapter.
//!
//! zbus's `async-io` backend owns its socket-driving thread. This module uses
//! a small standard-library executor to poll each caller-side future only
//! until the shared monotonic deadline; it does not use zbus's blocking signal
//! iterator, whose `next` call has no application deadline.
use std::future::{Future, poll_fn};
use std::pin::{Pin, pin};
use std::sync::Arc;
use std::task::{Context, Poll, Wake, Waker};
use std::thread;
use std::time::Instant;
use zbus::export::futures_core::Stream;
use zbus::message::Type;
use zbus::{Connection, MatchRule, MessageStream, Proxy};
use super::{
JobResult, RestartDeadline, RestartExecution, RestartFailure, RestartJobTracker,
RestartOutcome, ServiceEvent,
};
const SYSTEMD_NAME: &str = "org.freedesktop.systemd1";
const SYSTEMD_PATH: &str = "/org/freedesktop/systemd1";
const SYSTEMD_MANAGER: &str = "org.freedesktop.systemd1.Manager";
const DBUS_NAME: &str = "org.freedesktop.DBus";
const DBUS_INTERFACE: &str = "org.freedesktop.DBus";
/// Resolve the configured unit's primary ID, request a restart of exactly
/// `configured_unit`, and wait for its matching systemd `JobRemoved` result.
/// The caller owns backup ordering and subsequent runtime verification, and
/// must pass the same deadline to those stages.
/// Restart a configured unit and retain whether a timed-out request was submitted.
pub(crate) fn restart_systemd_unit_report(
configured_unit: &str,
deadline: RestartDeadline,
) -> RestartExecution {
restart_with_transport_report(&mut SystemdBus::default(), configured_unit, deadline)
}
trait RestartTransport {
fn connect(&mut self, deadline: RestartDeadline) -> Result<(), RestartFailure>;
fn subscribe(&mut self, deadline: RestartDeadline) -> Result<(), RestartFailure>;
fn resolve_primary_unit(
&mut self,
configured_unit: &str,
deadline: RestartDeadline,
) -> Result<String, RestartFailure>;
fn restart_unit(
&mut self,
unit: &str,
deadline: RestartDeadline,
) -> Result<String, RestartFailure>;
fn next_event(&mut self, deadline: RestartDeadline) -> Result<ServiceEvent, RestartFailure>;
}
#[cfg(test)]
fn restart_with_transport(
transport: &mut impl RestartTransport,
configured_unit: &str,
deadline: RestartDeadline,
) -> RestartOutcome {
restart_with_transport_report(transport, configured_unit, deadline).outcome
}
fn restart_with_transport_report(
transport: &mut impl RestartTransport,
configured_unit: &str,
deadline: RestartDeadline,
) -> RestartExecution {
if configured_unit.is_empty() || configured_unit.as_bytes().contains(&0) {
return execution(RestartOutcome::Failed(RestartFailure::RequestFailed), false);
}
if let Err(failure) = transport.connect(deadline) {
return execution(RestartOutcome::Failed(failure), false);
}
if deadline.is_expired_at(Instant::now()) {
return execution(
RestartOutcome::Failed(RestartFailure::WaitTimedOutMayStillComplete),
false,
);
}
if let Err(failure) = transport.subscribe(deadline) {
return execution(RestartOutcome::Failed(failure), false);
}
if deadline.is_expired_at(Instant::now()) {
return execution(
RestartOutcome::Failed(RestartFailure::WaitTimedOutMayStillComplete),
false,
);
}
let primary_unit = match transport.resolve_primary_unit(configured_unit, deadline) {
Ok(unit) if !unit.is_empty() => unit,
Ok(_) => {
return execution(
RestartOutcome::Failed(RestartFailure::UnitResolutionFailed),
false,
);
}
Err(failure) => return execution(RestartOutcome::Failed(failure), false),
};
if deadline.is_expired_at(Instant::now()) {
return execution(
RestartOutcome::Failed(RestartFailure::WaitTimedOutMayStillComplete),
false,
);
}
let job_path = match transport.restart_unit(configured_unit, deadline) {
Ok(path) => path,
Err(failure) => {
let submitted = matches!(
failure,
RestartFailure::WaitTimedOutMayStillComplete | RestartFailure::BusDisconnected
);
return execution(RestartOutcome::Failed(failure), submitted);
}
};
let mut tracker = RestartJobTracker::new(primary_unit, deadline);
let mut outcome = tracker.job_submitted(job_path, Instant::now());
while outcome == RestartOutcome::Waiting {
let event = match transport.next_event(deadline) {
Ok(event) => event,
Err(failure) => return execution(RestartOutcome::Failed(failure), true),
};
outcome = tracker.observe(event, Instant::now());
}
execution(outcome, true)
}
fn execution(outcome: RestartOutcome, submitted: bool) -> RestartExecution {
RestartExecution { outcome, submitted }
}
#[derive(Default)]
struct SystemdBus {
connection: Option<Connection>,
manager: Option<Proxy<'static>>,
job_events: Option<zbus::proxy::SignalStream<'static>>,
owner_events: Option<zbus::MessageStream>,
}
impl RestartTransport for SystemdBus {
fn connect(&mut self, deadline: RestartDeadline) -> Result<(), RestartFailure> {
let connection = drive(Connection::system(), deadline)?.map_err(map_bus_error)?;
let manager = drive(
Proxy::new_owned(
connection.clone(),
SYSTEMD_NAME,
SYSTEMD_PATH,
SYSTEMD_MANAGER,
),
deadline,
)?
.map_err(map_bus_error)?;
self.manager = Some(manager);
self.connection = Some(connection);
Ok(())
}
fn subscribe(&mut self, deadline: RestartDeadline) -> Result<(), RestartFailure> {
let manager = self
.manager
.as_ref()
.ok_or(RestartFailure::SubscriptionFailed)?;
let job_events = drive(manager.receive_signal("JobRemoved"), deadline)?
.map_err(|_| RestartFailure::SubscriptionFailed)?;
let owner_rule = MatchRule::builder()
.msg_type(Type::Signal)
.sender(DBUS_NAME)
.map_err(|_| RestartFailure::SubscriptionFailed)?
.interface(DBUS_INTERFACE)
.map_err(|_| RestartFailure::SubscriptionFailed)?
.member("NameOwnerChanged")
.map_err(|_| RestartFailure::SubscriptionFailed)?
.add_arg(SYSTEMD_NAME)
.map_err(|_| RestartFailure::SubscriptionFailed)?
.build();
let owner_events = drive(
MessageStream::for_match_rule(
owner_rule,
self.connection
.as_ref()
.ok_or(RestartFailure::SubscriptionFailed)?,
Some(1),
),
deadline,
)?
.map_err(|_| RestartFailure::SubscriptionFailed)?;
// Both signal matches are installed before Subscribe and RestartUnit.
self.job_events = Some(job_events);
self.owner_events = Some(owner_events);
drive(manager.call::<_, _, ()>("Subscribe", &()), deadline)?.map_err(map_bus_error)
}
fn resolve_primary_unit(
&mut self,
configured_unit: &str,
deadline: RestartDeadline,
) -> Result<String, RestartFailure> {
let manager = self
.manager
.as_ref()
.ok_or(RestartFailure::UnitResolutionFailed)?;
let connection = self
.connection
.as_ref()
.ok_or(RestartFailure::BusDisconnected)?;
let unit_path: zbus::zvariant::OwnedObjectPath =
drive(manager.call("GetUnit", &(configured_unit,)), deadline)?
.map_err(map_unit_resolution_error)?;
let unit = drive(
Proxy::new_owned(
connection.clone(),
SYSTEMD_NAME,
unit_path,
"org.freedesktop.systemd1.Unit",
),
deadline,
)?
.map_err(map_unit_resolution_error)?;
drive(unit.get_property::<String>("Id"), deadline)?.map_err(map_unit_resolution_error)
}
fn restart_unit(
&mut self,
unit: &str,
deadline: RestartDeadline,
) -> Result<String, RestartFailure> {
let manager = self.manager.as_ref().ok_or(RestartFailure::RequestFailed)?;
let connection = self
.connection
.as_ref()
.ok_or(RestartFailure::BusDisconnected)?;
let owner_events = self
.owner_events
.as_mut()
.ok_or(RestartFailure::BusDisconnected)?;
let request_body = (unit, "replace");
let request = manager.call("RestartUnit", &request_body);
let job_path: zbus::zvariant::OwnedObjectPath = drive(
async {
let mut request = pin!(request);
let mut closed = pin!(connection.closed());
poll_fn(|cx| {
if connection.is_closed() {
return Poll::Ready(Err(RestartFailure::BusDisconnected));
}
if let Poll::Ready(event) = Stream::poll_next(Pin::new(&mut *owner_events), cx)
{
return Poll::Ready(match event {
None => Err(RestartFailure::BusDisconnected),
Some(Err(error)) => Err(map_bus_error(error)),
Some(Ok(_)) => Err(RestartFailure::ManagerOwnerChanged),
});
}
if let Poll::Ready(result) = Future::poll(request.as_mut(), cx) {
return Poll::Ready(result.map_err(map_bus_error));
}
if Future::poll(closed.as_mut(), cx).is_ready() {
return Poll::Ready(Err(RestartFailure::BusDisconnected));
}
Poll::Pending
})
.await
},
deadline,
)??;
Ok(job_path.to_string())
}
fn next_event(&mut self, deadline: RestartDeadline) -> Result<ServiceEvent, RestartFailure> {
let connection = self
.connection
.as_ref()
.ok_or(RestartFailure::BusDisconnected)?;
let job_events = self
.job_events
.as_mut()
.ok_or(RestartFailure::BusDisconnected)?;
let owner_events = self
.owner_events
.as_mut()
.ok_or(RestartFailure::BusDisconnected)?;
drive(
async {
let mut closed = pin!(connection.closed());
poll_fn(|cx| {
if connection.is_closed() {
return Poll::Ready(Ok(ServiceEvent::Disconnected));
}
if let Poll::Ready(message) =
Stream::poll_next(Pin::new(&mut *owner_events), cx)
{
return Poll::Ready(match message {
None => Ok(ServiceEvent::Disconnected),
Some(Err(error)) => Err(map_bus_error(error)),
Some(Ok(message)) => {
let decoded: Result<(String, String, String), _> =
message.body().deserialize();
match decoded {
Ok((name, _, _)) if name == SYSTEMD_NAME => {
Ok(ServiceEvent::ManagerOwnerChanged)
}
Ok(_) => Ok(ServiceEvent::JobRemoved {
job_path: String::new(),
unit: String::new(),
result: JobResult::Other,
}),
Err(_) => Err(RestartFailure::RequestFailed),
}
}
});
}
if let Poll::Ready(message) = Stream::poll_next(Pin::new(&mut *job_events), cx)
{
return Poll::Ready(match message {
None => Ok(ServiceEvent::Disconnected),
Some(message) => {
let decoded: Result<
(u32, zbus::zvariant::OwnedObjectPath, String, String),
_,
> = message.body().deserialize();
match decoded {
Ok((_, path, unit, result)) => Ok(ServiceEvent::JobRemoved {
job_path: path.to_string(),
unit,
result: map_job_result(&result),
}),
Err(_) => Err(RestartFailure::RequestFailed),
}
}
});
}
if Future::poll(closed.as_mut(), cx).is_ready() {
return Poll::Ready(Ok(ServiceEvent::Disconnected));
}
Poll::Pending
})
.await
},
deadline,
)?
}
}
fn map_job_result(result: &str) -> JobResult {
match result {
"done" => JobResult::Done,
"failed" => JobResult::Failed,
"dependency" => JobResult::Dependency,
"timeout" => JobResult::Timeout,
"canceled" => JobResult::Canceled,
"skipped" => JobResult::Skipped,
_ => JobResult::Other,
}
}
fn map_bus_error(error: zbus::Error) -> RestartFailure {
match error {
zbus::Error::MethodError(name, _, _)
if name.as_str() == "org.freedesktop.DBus.Error.AccessDenied"
|| name.as_str() == "org.freedesktop.systemd1.AccessDenied" =>
{
RestartFailure::PermissionDenied
}
zbus::Error::InputOutput(_) | zbus::Error::Handshake(_) => RestartFailure::BusDisconnected,
_ => RestartFailure::RequestFailed,
}
}
fn map_unit_resolution_error(error: zbus::Error) -> RestartFailure {
match map_bus_error(error) {
RestartFailure::PermissionDenied => RestartFailure::PermissionDenied,
RestartFailure::BusDisconnected => RestartFailure::BusDisconnected,
_ => RestartFailure::UnitResolutionFailed,
}
}
struct ThreadWake(thread::Thread);
impl Wake for ThreadWake {
fn wake(self: Arc<Self>) {
self.0.unpark();
}
fn wake_by_ref(self: &Arc<Self>) {
self.0.unpark();
}
}
/// Poll a future on this thread, parking only for the remaining operation
/// budget. zbus's async-io connection executor continues driving socket I/O
/// independently and wakes this thread when the future can make progress.
fn drive<F: Future>(future: F, deadline: RestartDeadline) -> Result<F::Output, RestartFailure> {
let waker = Waker::from(Arc::new(ThreadWake(thread::current())));
let mut context = Context::from_waker(&waker);
let mut future = pin!(future);
loop {
if deadline.is_expired_at(Instant::now()) {
return Err(RestartFailure::WaitTimedOutMayStillComplete);
}
match Future::poll(future.as_mut(), &mut context) {
Poll::Ready(output) => return Ok(output),
Poll::Pending => {
let remaining = deadline.remaining_at(Instant::now());
if remaining.is_zero() {
return Err(RestartFailure::WaitTimedOutMayStillComplete);
}
thread::park_timeout(remaining);
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::VecDeque;
use std::time::Duration;
struct FakeBus {
calls: Vec<&'static str>,
deadlines: Vec<(&'static str, RestartDeadline)>,
events: VecDeque<ServiceEvent>,
failure: Option<(&'static str, RestartFailure)>,
configured_unit: &'static str,
job_path: &'static str,
primary_unit: &'static str,
}
impl Default for FakeBus {
fn default() -> Self {
Self {
calls: Vec::new(),
deadlines: Vec::new(),
events: VecDeque::new(),
failure: None,
configured_unit: "wg-quick@wg0.service",
job_path: "",
primary_unit: "wg-quick@wg0.service",
}
}
}
impl FakeBus {
fn stage(&mut self, stage: &'static str) -> Result<(), RestartFailure> {
self.calls.push(stage);
match self.failure {
Some((failed_stage, failure)) if failed_stage == stage => Err(failure),
_ => Ok(()),
}
}
}
impl RestartTransport for FakeBus {
fn connect(&mut self, deadline: RestartDeadline) -> Result<(), RestartFailure> {
self.deadlines.push(("connect", deadline));
self.stage("connect")
}
fn subscribe(&mut self, deadline: RestartDeadline) -> Result<(), RestartFailure> {
self.deadlines.push(("subscribe", deadline));
self.stage("subscribe")
}
fn resolve_primary_unit(
&mut self,
configured_unit: &str,
deadline: RestartDeadline,
) -> Result<String, RestartFailure> {
assert_eq!(configured_unit, self.configured_unit);
self.deadlines.push(("resolve", deadline));
self.stage("resolve")?;
Ok(self.primary_unit.to_owned())
}
fn restart_unit(
&mut self,
unit: &str,
deadline: RestartDeadline,
) -> Result<String, RestartFailure> {
assert_eq!(unit, self.configured_unit);
self.deadlines.push(("restart", deadline));
self.stage("restart")?;
Ok(self.job_path.to_owned())
}
fn next_event(
&mut self,
deadline: RestartDeadline,
) -> Result<ServiceEvent, RestartFailure> {
self.deadlines.push(("event", deadline));
self.stage("event")?;
self.events
.pop_front()
.ok_or(RestartFailure::BusDisconnected)
}
}
fn deadline() -> RestartDeadline {
RestartDeadline::new(Instant::now(), Duration::from_secs(2))
}
fn removed(path: &str, unit: &str, result: JobResult) -> ServiceEvent {
ServiceEvent::JobRemoved {
job_path: path.to_owned(),
unit: unit.to_owned(),
result,
}
}
#[test]
fn subscribes_before_restart_and_uses_returned_job_path() {
let mut fake = FakeBus {
job_path: "/job/actual",
primary_unit: "wg-quick@wg0.service",
events: VecDeque::from([
removed("/job/other", "wg-quick@wg0.service", JobResult::Done),
removed("/job/actual", "wg-quick@wg0.service", JobResult::Done),
]),
..FakeBus::default()
};
assert_eq!(
restart_with_transport(&mut fake, "wg-quick@wg0.service", deadline()),
RestartOutcome::Succeeded
);
assert_eq!(
fake.calls,
[
"connect",
"subscribe",
"resolve",
"restart",
"event",
"event"
]
);
}
#[test]
fn fast_completion_and_failure_results_are_processed_without_a_bus() {
let mut fast = FakeBus {
job_path: "/job/fast",
primary_unit: "wg-quick@wg0.service",
events: VecDeque::from([removed(
"/job/fast",
"wg-quick@wg0.service",
JobResult::Done,
)]),
..FakeBus::default()
};
assert_eq!(
restart_with_transport(&mut fast, "wg-quick@wg0.service", deadline()),
RestartOutcome::Succeeded
);
let mut failed = FakeBus {
job_path: "/job/failed",
primary_unit: "wg-quick@wg0.service",
events: VecDeque::from([removed(
"/job/failed",
"wg-quick@wg0.service",
JobResult::Canceled,
)]),
..FakeBus::default()
};
assert_eq!(
restart_with_transport(&mut failed, "wg-quick@wg0.service", deadline()),
RestartOutcome::Failed(RestartFailure::JobCanceled)
);
}
#[test]
fn setup_and_permission_failures_stop_before_restart() {
for (stage, failure) in [
("connect", RestartFailure::BusDisconnected),
("subscribe", RestartFailure::SubscriptionFailed),
("resolve", RestartFailure::UnitResolutionFailed),
("restart", RestartFailure::PermissionDenied),
] {
let mut fake = FakeBus {
job_path: "/job/never",
primary_unit: "wg-quick@wg0.service",
failure: Some((stage, failure)),
..FakeBus::default()
};
assert_eq!(
restart_with_transport(&mut fake, "wg-quick@wg0.service", deadline()),
RestartOutcome::Failed(failure)
);
if stage != "restart" {
assert!(!fake.calls.contains(&"restart"));
}
}
}
#[test]
fn resolves_alias_before_restart_and_uses_one_shared_deadline() {
let deadline = deadline();
let mut fake = FakeBus {
job_path: "/job/alias",
configured_unit: "wireguard-vpn.service",
primary_unit: "wg-quick@wg0.service",
events: VecDeque::from([removed(
"/job/alias",
"wg-quick@wg0.service",
JobResult::Done,
)]),
..FakeBus::default()
};
assert_eq!(
restart_with_transport(&mut fake, "wireguard-vpn.service", deadline),
RestartOutcome::Succeeded
);
assert_eq!(
fake.calls,
["connect", "subscribe", "resolve", "restart", "event"]
);
assert!(fake.deadlines.iter().all(|(_, passed)| *passed == deadline));
}
#[test]
fn expired_shared_deadline_stops_between_setup_and_submission() {
let mut fake = FakeBus {
job_path: "/job/never",
..FakeBus::default()
};
let expired = RestartDeadline::new(Instant::now(), Duration::ZERO);
assert_eq!(
restart_with_transport(&mut fake, "wg-quick@wg0.service", expired),
RestartOutcome::Failed(RestartFailure::WaitTimedOutMayStillComplete)
);
assert_eq!(fake.calls, ["connect"]);
}
#[test]
fn result_mapping_accepts_only_the_documented_done_value() {
assert_eq!(map_job_result("done"), JobResult::Done);
for value in ["failed", "dependency", "timeout", "canceled", "skipped"] {
assert_ne!(map_job_result(value), JobResult::Done);
}
assert_eq!(map_job_result("future-result"), JobResult::Other);
}
#[test]
fn std_wake_park_driver_rejects_an_already_expired_future() {
let expired = RestartDeadline::new(Instant::now(), Duration::ZERO);
assert_eq!(
drive(std::future::pending::<()>(), expired),
Err(RestartFailure::WaitTimedOutMayStillComplete)
);
}
#[test]
fn owner_change_and_disconnect_fail_the_attempt() {
for (event, failure) in [
(
ServiceEvent::ManagerOwnerChanged,
RestartFailure::ManagerOwnerChanged,
),
(ServiceEvent::Disconnected, RestartFailure::BusDisconnected),
] {
let mut fake = FakeBus {
job_path: "/job/actual",
primary_unit: "wg-quick@wg0.service",
events: VecDeque::from([event]),
..FakeBus::default()
};
assert_eq!(
restart_with_transport(&mut fake, "wg-quick@wg0.service", deadline()),
RestartOutcome::Failed(failure)
);
}
}
}