303 lines
14 KiB
Python
303 lines
14 KiB
Python
"""Orchestration & recovery tests.
|
|
|
|
Pins the agreed semantics:
|
|
* crash auto-restart retries FOREVER with capped exponential back-off (never gives
|
|
up — unattended appliance); the failure streak resets when data flows again;
|
|
* config writes are atomic — a serialization failure leaves no partial output and
|
|
never corrupts an existing config;
|
|
* the acquisition producer command is selected by radar.model;
|
|
* process exit reports classify clean vs unexpected exits correctly.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import signal
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
from python_app.models.run_config_schema import RunConfigModel
|
|
from python_app.tests.test_reader_recovery import pipeline_fixture
|
|
from python_app.orchestration.config_writer import ConfigWriter
|
|
from python_app.orchestration.process_supervisor import ProcessExitReport, ProcessSupervisor
|
|
from python_app.orchestration.restart_policy import RestartPolicy
|
|
|
|
|
|
class RestartPolicyTest(unittest.TestCase):
|
|
def test_backoff_grows_and_caps(self) -> None:
|
|
policy = RestartPolicy(min_interval_s=3.0, max_interval_s=60.0, backoff_factor=2.0)
|
|
self.assertEqual(policy.backoff_for(0), 3.0)
|
|
self.assertEqual(policy.backoff_for(1), 6.0)
|
|
self.assertEqual(policy.backoff_for(2), 12.0)
|
|
self.assertEqual(policy.backoff_for(3), 24.0)
|
|
self.assertEqual(policy.backoff_for(100), 60.0) # capped
|
|
|
|
def test_never_gives_up_even_after_huge_streak(self) -> None:
|
|
# No attempt cap: a long failure streak still yields a finite, capped wait.
|
|
policy = RestartPolicy()
|
|
self.assertEqual(policy.backoff_for(10_000), policy.max_interval_s)
|
|
|
|
def test_should_restart_respects_backoff_window(self) -> None:
|
|
policy = RestartPolicy(min_interval_s=3.0, max_interval_s=60.0)
|
|
# First failure (streak 0): needs >= 3s since last restart.
|
|
self.assertFalse(policy.should_restart_now(now_s=102.0, last_restart_s=100.0, consecutive_failures=0))
|
|
self.assertTrue(policy.should_restart_now(now_s=103.0, last_restart_s=100.0, consecutive_failures=0))
|
|
# After 2 failures the window is 12s.
|
|
self.assertFalse(policy.should_restart_now(now_s=111.0, last_restart_s=100.0, consecutive_failures=2))
|
|
self.assertTrue(policy.should_restart_now(now_s=112.0, last_restart_s=100.0, consecutive_failures=2))
|
|
|
|
def test_first_restart_is_immediate(self) -> None:
|
|
# last_restart defaults to 0.0 in PipelineController, so the very first crash restarts now.
|
|
self.assertTrue(RestartPolicy().should_restart_now(now_s=5_000.0, last_restart_s=0.0, consecutive_failures=0))
|
|
|
|
|
|
class PipelineRecoveryPollingTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.pipeline, self.supervisor = pipeline_fixture(self)
|
|
self.pipeline.should_run = True
|
|
|
|
def test_missing_children_keep_retrying_without_fresh_exit_reports(self) -> None:
|
|
with mock.patch.object(self.pipeline, "_recover_pipeline_after_crash") as recover:
|
|
for pids in ({"data_processor": 1, "data_preprocessor": 2}, {}):
|
|
with self.subTest(pids=pids):
|
|
recover.reset_mock()
|
|
self.supervisor.pids.return_value = pids
|
|
for _ in range(3):
|
|
self.pipeline.poll()
|
|
self.assertEqual(recover.call_count, 3)
|
|
|
|
def test_failed_restarts_retry_after_backoff_without_new_exit_reports(self) -> None:
|
|
self.supervisor.start.side_effect = RuntimeError("device unavailable")
|
|
self.pipeline.active_config = RunConfigModel()
|
|
self.pipeline.active_config_path = Path("run_config.json")
|
|
with mock.patch("python_app.gui.controllers.pipeline_controller.time.monotonic",
|
|
side_effect=[100.0, 101.0, 106.0]):
|
|
for _ in range(3):
|
|
self.pipeline.poll()
|
|
self.assertEqual(self.supervisor.start.call_count, 2)
|
|
self.assertEqual(self.pipeline._restart_count, 2)
|
|
self.assertTrue(self.pipeline.should_run)
|
|
|
|
def test_healthy_pipeline_does_not_restart(self) -> None:
|
|
self.supervisor.pids.return_value = {"data_processor": 1, "data_preprocessor": 2, "acquisition": 3}
|
|
self.pipeline.poll()
|
|
self.supervisor.start.assert_not_called()
|
|
|
|
def test_explicit_stop_does_not_restart_missing_children(self) -> None:
|
|
self.pipeline.stop()
|
|
self.pipeline.poll()
|
|
self.supervisor.start.assert_not_called()
|
|
self.assertFalse(self.pipeline.should_run)
|
|
self.assertEqual(self.pipeline.stop_generation, 1)
|
|
|
|
|
|
class ConfigWriterTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self._dir = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self._dir.cleanup)
|
|
self.root = Path(self._dir.name)
|
|
self.writer = ConfigWriter(self.root / "runtime")
|
|
|
|
def test_writes_loadable_config_and_leaves_no_tmp(self) -> None:
|
|
out = self.root / "run_config.json"
|
|
self.writer.write(RunConfigModel(), out)
|
|
self.assertTrue(out.exists())
|
|
self.assertFalse(out.with_suffix(".json.tmp").exists())
|
|
RunConfigModel.from_dict(json.loads(out.read_text())) # round-trips back through the schema
|
|
|
|
def test_nan_fails_loudly_without_corrupting_existing(self) -> None:
|
|
out = self.root / "run_config.json"
|
|
self.writer.write(RunConfigModel(), out)
|
|
original = out.read_text()
|
|
|
|
broken = RunConfigModel()
|
|
broken.radar.sweep.if_bandwidth_hz = float("nan")
|
|
with self.assertRaises(ValueError): # allow_nan=False
|
|
self.writer.write(broken, out)
|
|
|
|
self.assertEqual(out.read_text(), original) # existing config untouched
|
|
self.assertFalse(out.with_suffix(".json.tmp").exists()) # no half-written tmp left
|
|
|
|
|
|
class RadarModelCommandTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self._dir = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self._dir.cleanup)
|
|
self.root = Path(self._dir.name)
|
|
self.supervisor = ProcessSupervisor(self.root)
|
|
|
|
def _config_with_model(self, model: str, driver_mode: str = "native") -> Path:
|
|
path = self.root / "cfg.json"
|
|
path.write_text(json.dumps({"radar": {"model": model, "driver_mode": driver_mode}}))
|
|
return path
|
|
|
|
|
|
def test_null_replay_directory_does_not_select_replay(self) -> None:
|
|
path = self._config_with_model("librevna")
|
|
path.write_text(json.dumps({"radar": {"replay": {"dataset_dir": None}}}))
|
|
self.assertEqual(self.supervisor._acquisition_command(path)[:3],
|
|
[sys.executable, "-m", "python_app.scripts.matrix_raw_producer"])
|
|
|
|
def test_replay_overrides_every_model(self) -> None:
|
|
for model in ("librevna", "librevna_multi", "kamil_adc"):
|
|
path = self._config_with_model(model)
|
|
path.write_text(json.dumps({"radar": {"model": model, "replay": {"dataset_dir": "capture"}}}))
|
|
self.assertEqual(self.supervisor._acquisition_command(path)[:3],
|
|
[sys.executable, "-m", "python_app.scripts.replay_raw_producer"])
|
|
|
|
def test_matrix_producer_for_multi(self) -> None:
|
|
cmd = self.supervisor._acquisition_command(self._config_with_model("librevna_multi"))
|
|
self.assertEqual(cmd[:3], [sys.executable, "-m", "python_app.scripts.matrix_raw_producer"])
|
|
|
|
def test_kamil_producer_for_kamil_adc(self) -> None:
|
|
cmd = self.supervisor._acquisition_command(self._config_with_model("kamil_adc"))
|
|
self.assertEqual(cmd[:3], [sys.executable, "-m", "python_app.scripts.kamil_adc_raw_producer"])
|
|
|
|
|
|
def test_single_and_multi_use_python_in_both_modes(self) -> None:
|
|
for model in ("librevna", "librevna_multi"):
|
|
for mode in ("native", "mock"):
|
|
command = self.supervisor._acquisition_command(self._config_with_model(model, mode))
|
|
self.assertEqual(command[:3], [sys.executable, "-m", "python_app.scripts.matrix_raw_producer"])
|
|
|
|
def test_omitted_driver_mode_uses_the_default_simulator(self) -> None:
|
|
path = self._config_with_model("librevna")
|
|
path.write_text(json.dumps({"radar": {"model": "librevna"}}))
|
|
self.assertEqual(self.supervisor._acquisition_command(path)[2], "python_app.scripts.matrix_raw_producer")
|
|
|
|
def test_missing_or_null_radar_fields_keep_schema_defaults(self) -> None:
|
|
path = self.root / "defaults.json"
|
|
for payload in ({}, {"radar": None}, {"radar": {}}, {"radar": {"model": None}},
|
|
{"radar": {"driver_mode": None}}, {"radar": {"driver_mode": "mock"}}):
|
|
with self.subTest(payload=payload):
|
|
path.write_text(json.dumps(payload))
|
|
self.assertEqual(self.supervisor._acquisition_command(path)[:3],
|
|
[sys.executable, "-m", "python_app.scripts.matrix_raw_producer"])
|
|
|
|
|
|
class ProcessExitReportTest(unittest.TestCase):
|
|
@staticmethod
|
|
def _report(*, code: int, clean: bool) -> ProcessExitReport:
|
|
return ProcessExitReport(
|
|
name="data_processor",
|
|
command=["x"],
|
|
working_directory=Path("/tmp"),
|
|
return_code=code,
|
|
stdout_path=Path("/nonexistent.out"),
|
|
stderr_path=Path("/nonexistent.err"),
|
|
expected_clean_exit=clean,
|
|
)
|
|
|
|
def test_clean_exit_is_info(self) -> None:
|
|
report = self._report(code=0, clean=True)
|
|
self.assertEqual(report.level, "INFO")
|
|
self.assertIn("completed normally", report.format())
|
|
|
|
def test_nonzero_exit_is_error(self) -> None:
|
|
report = self._report(code=3, clean=False)
|
|
self.assertEqual(report.level, "ERROR")
|
|
self.assertIn("exited with code 3", report.format())
|
|
|
|
def test_unexpected_zero_exit_is_error(self) -> None:
|
|
report = self._report(code=0, clean=False)
|
|
self.assertEqual(report.level, "ERROR")
|
|
self.assertIn("exited unexpectedly with code 0", report.format())
|
|
|
|
|
|
class SupervisorLifecycleTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self._dir = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self._dir.cleanup)
|
|
self.supervisor = ProcessSupervisor(Path(self._dir.name))
|
|
self.addCleanup(self.supervisor.stop_all)
|
|
|
|
def _await_reports(self, timeout_s: float = 3.0) -> list[ProcessExitReport]:
|
|
deadline = time.monotonic() + timeout_s
|
|
while time.monotonic() < deadline:
|
|
reports = self.supervisor.collect_exit_reports()
|
|
if reports:
|
|
return reports
|
|
time.sleep(0.02)
|
|
return []
|
|
|
|
def test_spawn_alive_then_stop(self) -> None:
|
|
self.supervisor._spawn("data_preprocessor",
|
|
[sys.executable, "-c", "import time; time.sleep(30)"],
|
|
allow_clean_exit=False)
|
|
self.assertTrue(self.supervisor.is_running())
|
|
self.assertIn("data_preprocessor", self.supervisor.pids())
|
|
self.supervisor.stop()
|
|
self.assertFalse(self.supervisor.is_running())
|
|
|
|
def test_unexpected_exit_reported_as_error(self) -> None:
|
|
self.supervisor._spawn("data_processor",
|
|
[sys.executable, "-c", "import sys; sys.exit(3)"],
|
|
allow_clean_exit=False)
|
|
reports = self._await_reports()
|
|
self.assertEqual(len(reports), 1)
|
|
self.assertEqual(reports[0].return_code, 3)
|
|
self.assertEqual(reports[0].level, "ERROR")
|
|
|
|
def test_clean_exit_reported_as_info(self) -> None:
|
|
self.supervisor._spawn("acquisition",
|
|
[sys.executable, "-c", "import sys; sys.exit(0)"],
|
|
allow_clean_exit=True)
|
|
reports = self._await_reports()
|
|
self.assertEqual(len(reports), 1)
|
|
self.assertTrue(reports[0].expected_clean_exit)
|
|
self.assertEqual(reports[0].level, "INFO")
|
|
|
|
def test_stale_pid_guard_rejects_unrelated_processes(self) -> None:
|
|
# The reap guard must never kill a recycled PID that is not one of our binaries.
|
|
self.assertFalse(self.supervisor._is_stale_pipeline_pid(2_000_000_000)) # no such pid
|
|
self.assertFalse(self.supervisor._is_stale_pipeline_pid(os.getpid())) # the test runner
|
|
|
|
def test_stale_pid_guard_matches_executable_or_module_arguments(self) -> None:
|
|
processor = str(self.supervisor._project_root / "build/bin/data_processor")
|
|
cases = (
|
|
(f"{processor}\x00--config\x00config.json\x00", True),
|
|
("python\x00-m\x00python_app.scripts.matrix_raw_producer\x00", True),
|
|
(f"bash\x00-c\x00echo {processor}\x00", False),
|
|
("python\x00-c\x00print('python_app.scripts.matrix_raw_producer')\x00", False),
|
|
("/other/project/build/bin/data_processor\x00", False),
|
|
)
|
|
for cmdline, expected in cases:
|
|
with (
|
|
self.subTest(cmdline=cmdline),
|
|
mock.patch.object(Path, "read_bytes", return_value=cmdline.encode()),
|
|
):
|
|
self.assertEqual(self.supervisor._is_stale_pipeline_pid(123), expected)
|
|
|
|
def test_signal_group_does_not_kill_a_non_leaders_group(self) -> None:
|
|
with (
|
|
mock.patch.object(os, "getpgid", return_value=456),
|
|
mock.patch.object(os, "killpg") as group_kill,
|
|
mock.patch.object(os, "kill") as kill,
|
|
):
|
|
self.supervisor._signal_group(123, signal.SIGTERM)
|
|
group_kill.assert_not_called()
|
|
kill.assert_called_once_with(123, signal.SIGTERM)
|
|
|
|
def test_collector_reaper_does_not_match_command_text(self) -> None:
|
|
entries = [Path("/proc/123"), Path("/proc/456")]
|
|
cmdlines = [
|
|
b"bash\x00-c\x00echo kamil_adc_collector\x00",
|
|
b"/project/build/bin/kamil_adc_collector\x00--config\x00config.json\x00",
|
|
]
|
|
with (
|
|
mock.patch.object(Path, "iterdir", return_value=entries),
|
|
mock.patch.object(Path, "read_bytes", side_effect=cmdlines),
|
|
mock.patch.object(ProcessSupervisor, "_signal_group") as signal_group,
|
|
):
|
|
self.supervisor._kill_kamil_collectors()
|
|
signal_group.assert_called_once_with(456, signal.SIGKILL)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|