"""Crash auto-restart back-off policy (pure, GUI-independent).""" from __future__ import annotations import math from dataclasses import dataclass @dataclass(frozen=True, slots=True) class RestartPolicy: """Decide when to relaunch a crashed pipeline — retry forever with capped back-off. This is an unattended appliance, so the pipeline never permanently gives up. The wait between restart attempts grows with the number of consecutive failures (so a persistently broken pipeline is not hammered) but is capped at ``max_interval_s``, and the failure streak resets to zero once genuine data flows again. A burst of crash signals within the current back-off window collapses to a single restart. """ min_interval_s: float = 3.0 max_interval_s: float = 60.0 backoff_factor: float = 2.0 def backoff_for(self, consecutive_failures: int) -> float: """Return the seconds to wait before the next restart for this failure streak. ``consecutive_failures`` is the number of restarts already attempted without a recovery: 0 → ``min_interval_s``, then each additional failure multiplies the wait by ``backoff_factor``, capped at ``max_interval_s``. """ if consecutive_failures <= 0: return self.min_interval_s # Beyond this many doublings the wait is always capped; clamp the exponent so a # long streak cannot overflow ``factor ** n``. max_exponent = max(1, math.ceil(math.log(self.max_interval_s / self.min_interval_s, self.backoff_factor))) exponent = min(int(consecutive_failures), max_exponent) return min(self.min_interval_s * (self.backoff_factor ** exponent), self.max_interval_s) def should_restart_now( self, *, now_s: float, last_restart_s: float, consecutive_failures: int, ) -> bool: """Return whether enough back-off has elapsed since the last restart to retry.""" return now_s - last_restart_s >= self.backoff_for(consecutive_failures)