48 lines
2.0 KiB
Python
48 lines
2.0 KiB
Python
"""Crash auto-restart back-off policy (pure, GUI-independent)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
from dataclasses import dataclass
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class RestartPolicy:
|
|
"""Decide when to relaunch a crashed pipeline — retry forever with capped back-off.
|
|
|
|
This is an unattended appliance, so the pipeline never permanently gives up. The
|
|
wait between restart attempts grows with the number of consecutive failures (so a
|
|
persistently broken pipeline is not hammered) but is capped at ``max_interval_s``,
|
|
and the failure streak resets to zero once genuine data flows again. A burst of
|
|
crash signals within the current back-off window collapses to a single restart.
|
|
"""
|
|
|
|
min_interval_s: float = 3.0
|
|
max_interval_s: float = 60.0
|
|
backoff_factor: float = 2.0
|
|
|
|
def backoff_for(self, consecutive_failures: int) -> float:
|
|
"""Return the seconds to wait before the next restart for this failure streak.
|
|
|
|
``consecutive_failures`` is the number of restarts already attempted without a
|
|
recovery: 0 → ``min_interval_s``, then each additional failure multiplies the
|
|
wait by ``backoff_factor``, capped at ``max_interval_s``.
|
|
"""
|
|
if consecutive_failures <= 0:
|
|
return self.min_interval_s
|
|
# Beyond this many doublings the wait is always capped; clamp the exponent so a
|
|
# long streak cannot overflow ``factor ** n``.
|
|
max_exponent = max(1, math.ceil(math.log(self.max_interval_s / self.min_interval_s, self.backoff_factor)))
|
|
exponent = min(int(consecutive_failures), max_exponent)
|
|
return min(self.min_interval_s * (self.backoff_factor ** exponent), self.max_interval_s)
|
|
|
|
def should_restart_now(
|
|
self,
|
|
*,
|
|
now_s: float,
|
|
last_restart_s: float,
|
|
consecutive_failures: int,
|
|
) -> bool:
|
|
"""Return whether enough back-off has elapsed since the last restart to retry."""
|
|
return now_s - last_restart_s >= self.backoff_for(consecutive_failures)
|