"""Orchestration & recovery tests. Pins the agreed semantics: * crash auto-restart retries FOREVER with capped exponential back-off (never gives up — unattended appliance); the failure streak resets when data flows again; * config writes are atomic — a serialization failure leaves no partial output and never corrupts an existing config; * the acquisition producer command is selected by radar.model; * process exit reports classify clean vs unexpected exits correctly. """ from __future__ import annotations import json import os import signal import sys import tempfile import time import unittest from pathlib import Path from unittest import mock from python_app.models.run_config_schema import RunConfigModel from python_app.tests.test_reader_recovery import pipeline_fixture from python_app.orchestration.config_writer import ConfigWriter from python_app.orchestration.process_supervisor import ProcessExitReport, ProcessSupervisor from python_app.orchestration.restart_policy import RestartPolicy class RestartPolicyTest(unittest.TestCase): def test_backoff_grows_and_caps(self) -> None: policy = RestartPolicy(min_interval_s=3.0, max_interval_s=60.0, backoff_factor=2.0) self.assertEqual(policy.backoff_for(0), 3.0) self.assertEqual(policy.backoff_for(1), 6.0) self.assertEqual(policy.backoff_for(2), 12.0) self.assertEqual(policy.backoff_for(3), 24.0) self.assertEqual(policy.backoff_for(100), 60.0) # capped def test_never_gives_up_even_after_huge_streak(self) -> None: # No attempt cap: a long failure streak still yields a finite, capped wait. policy = RestartPolicy() self.assertEqual(policy.backoff_for(10_000), policy.max_interval_s) def test_should_restart_respects_backoff_window(self) -> None: policy = RestartPolicy(min_interval_s=3.0, max_interval_s=60.0) # First failure (streak 0): needs >= 3s since last restart. self.assertFalse(policy.should_restart_now(now_s=102.0, last_restart_s=100.0, consecutive_failures=0)) self.assertTrue(policy.should_restart_now(now_s=103.0, last_restart_s=100.0, consecutive_failures=0)) # After 2 failures the window is 12s. self.assertFalse(policy.should_restart_now(now_s=111.0, last_restart_s=100.0, consecutive_failures=2)) self.assertTrue(policy.should_restart_now(now_s=112.0, last_restart_s=100.0, consecutive_failures=2)) def test_first_restart_is_immediate(self) -> None: # last_restart defaults to 0.0 in PipelineController, so the very first crash restarts now. self.assertTrue(RestartPolicy().should_restart_now(now_s=5_000.0, last_restart_s=0.0, consecutive_failures=0)) class PipelineRecoveryPollingTest(unittest.TestCase): def setUp(self) -> None: self.pipeline, self.supervisor = pipeline_fixture(self) self.pipeline.should_run = True def test_missing_children_keep_retrying_without_fresh_exit_reports(self) -> None: with mock.patch.object(self.pipeline, "_recover_pipeline_after_crash") as recover: for pids in ({"data_processor": 1, "data_preprocessor": 2}, {}): with self.subTest(pids=pids): recover.reset_mock() self.supervisor.pids.return_value = pids for _ in range(3): self.pipeline.poll() self.assertEqual(recover.call_count, 3) def test_failed_restarts_retry_after_backoff_without_new_exit_reports(self) -> None: self.supervisor.start.side_effect = RuntimeError("device unavailable") self.pipeline.active_config = RunConfigModel() self.pipeline.active_config_path = Path("run_config.json") with mock.patch("python_app.gui.controllers.pipeline_controller.time.monotonic", side_effect=[100.0, 101.0, 106.0]): for _ in range(3): self.pipeline.poll() self.assertEqual(self.supervisor.start.call_count, 2) self.assertEqual(self.pipeline._restart_count, 2) self.assertTrue(self.pipeline.should_run) def test_healthy_pipeline_does_not_restart(self) -> None: self.supervisor.pids.return_value = {"data_processor": 1, "data_preprocessor": 2, "acquisition": 3} self.pipeline.poll() self.supervisor.start.assert_not_called() def test_explicit_stop_does_not_restart_missing_children(self) -> None: self.pipeline.stop() self.pipeline.poll() self.supervisor.start.assert_not_called() self.assertFalse(self.pipeline.should_run) self.assertEqual(self.pipeline.stop_generation, 1) class ConfigWriterTest(unittest.TestCase): def setUp(self) -> None: self._dir = tempfile.TemporaryDirectory() self.addCleanup(self._dir.cleanup) self.root = Path(self._dir.name) self.writer = ConfigWriter(self.root / "runtime") def test_writes_loadable_config_and_leaves_no_tmp(self) -> None: out = self.root / "run_config.json" self.writer.write(RunConfigModel(), out) self.assertTrue(out.exists()) self.assertFalse(out.with_suffix(".json.tmp").exists()) RunConfigModel.from_dict(json.loads(out.read_text())) # round-trips back through the schema def test_nan_fails_loudly_without_corrupting_existing(self) -> None: out = self.root / "run_config.json" self.writer.write(RunConfigModel(), out) original = out.read_text() broken = RunConfigModel() broken.radar.sweep.if_bandwidth_hz = float("nan") with self.assertRaises(ValueError): # allow_nan=False self.writer.write(broken, out) self.assertEqual(out.read_text(), original) # existing config untouched self.assertFalse(out.with_suffix(".json.tmp").exists()) # no half-written tmp left class RadarModelCommandTest(unittest.TestCase): def setUp(self) -> None: self._dir = tempfile.TemporaryDirectory() self.addCleanup(self._dir.cleanup) self.root = Path(self._dir.name) self.supervisor = ProcessSupervisor(self.root) def _config_with_model(self, model: str, driver_mode: str = "native") -> Path: path = self.root / "cfg.json" path.write_text(json.dumps({"radar": {"model": model, "driver_mode": driver_mode}})) return path def test_matrix_producer_for_multi(self) -> None: cmd = self.supervisor._acquisition_command(self._config_with_model("librevna_multi")) self.assertEqual(cmd[:3], [sys.executable, "-m", "python_app.scripts.matrix_raw_producer"]) def test_kamil_producer_for_kamil_adc(self) -> None: cmd = self.supervisor._acquisition_command(self._config_with_model("kamil_adc")) self.assertEqual(cmd[:3], [sys.executable, "-m", "python_app.scripts.kamil_adc_raw_producer"]) def test_single_and_multi_use_python_in_both_modes(self) -> None: for model in ("librevna", "librevna_multi"): for mode in ("native", "mock"): command = self.supervisor._acquisition_command(self._config_with_model(model, mode)) self.assertEqual(command[:3], [sys.executable, "-m", "python_app.scripts.matrix_raw_producer"]) def test_omitted_driver_mode_uses_the_default_simulator(self) -> None: path = self._config_with_model("librevna") path.write_text(json.dumps({"radar": {"model": "librevna"}})) self.assertEqual(self.supervisor._acquisition_command(path)[2], "python_app.scripts.matrix_raw_producer") def test_missing_or_null_radar_fields_keep_schema_defaults(self) -> None: path = self.root / "defaults.json" for payload in ({}, {"radar": None}, {"radar": {}}, {"radar": {"model": None}}, {"radar": {"driver_mode": None}}, {"radar": {"driver_mode": "mock"}}): with self.subTest(payload=payload): path.write_text(json.dumps(payload)) self.assertEqual(self.supervisor._acquisition_command(path)[:3], [sys.executable, "-m", "python_app.scripts.matrix_raw_producer"]) class ProcessExitReportTest(unittest.TestCase): @staticmethod def _report(*, code: int, clean: bool) -> ProcessExitReport: return ProcessExitReport( name="data_processor", command=["x"], working_directory=Path("/tmp"), return_code=code, stdout_path=Path("/nonexistent.out"), stderr_path=Path("/nonexistent.err"), expected_clean_exit=clean, ) def test_clean_exit_is_info(self) -> None: report = self._report(code=0, clean=True) self.assertEqual(report.level, "INFO") self.assertIn("completed normally", report.format()) def test_nonzero_exit_is_error(self) -> None: report = self._report(code=3, clean=False) self.assertEqual(report.level, "ERROR") self.assertIn("exited with code 3", report.format()) def test_unexpected_zero_exit_is_error(self) -> None: report = self._report(code=0, clean=False) self.assertEqual(report.level, "ERROR") self.assertIn("exited unexpectedly with code 0", report.format()) class SupervisorLifecycleTest(unittest.TestCase): def setUp(self) -> None: self._dir = tempfile.TemporaryDirectory() self.addCleanup(self._dir.cleanup) self.supervisor = ProcessSupervisor(Path(self._dir.name)) self.addCleanup(self.supervisor.stop_all) def _await_reports(self, timeout_s: float = 3.0) -> list[ProcessExitReport]: deadline = time.monotonic() + timeout_s while time.monotonic() < deadline: reports = self.supervisor.collect_exit_reports() if reports: return reports time.sleep(0.02) return [] def test_spawn_alive_then_stop(self) -> None: self.supervisor._spawn("data_preprocessor", [sys.executable, "-c", "import time; time.sleep(30)"], allow_clean_exit=False) self.assertTrue(self.supervisor.is_running()) self.assertIn("data_preprocessor", self.supervisor.pids()) self.supervisor.stop() self.assertFalse(self.supervisor.is_running()) def test_unexpected_exit_reported_as_error(self) -> None: self.supervisor._spawn("data_processor", [sys.executable, "-c", "import sys; sys.exit(3)"], allow_clean_exit=False) reports = self._await_reports() self.assertEqual(len(reports), 1) self.assertEqual(reports[0].return_code, 3) self.assertEqual(reports[0].level, "ERROR") def test_clean_exit_reported_as_info(self) -> None: self.supervisor._spawn("acquisition", [sys.executable, "-c", "import sys; sys.exit(0)"], allow_clean_exit=True) reports = self._await_reports() self.assertEqual(len(reports), 1) self.assertTrue(reports[0].expected_clean_exit) self.assertEqual(reports[0].level, "INFO") def test_stale_pid_guard_rejects_unrelated_processes(self) -> None: # The reap guard must never kill a recycled PID that is not one of our binaries. self.assertFalse(self.supervisor._is_stale_pipeline_pid(2_000_000_000)) # no such pid self.assertFalse(self.supervisor._is_stale_pipeline_pid(os.getpid())) # the test runner def test_stale_pid_guard_matches_executable_or_module_arguments(self) -> None: processor = str(self.supervisor._project_root / "build/bin/data_processor") cases = ( (f"{processor}\x00--config\x00config.json\x00", True), ("python\x00-m\x00python_app.scripts.matrix_raw_producer\x00", True), (f"bash\x00-c\x00echo {processor}\x00", False), ("python\x00-c\x00print('python_app.scripts.matrix_raw_producer')\x00", False), ("/other/project/build/bin/data_processor\x00", False), ) for cmdline, expected in cases: with ( self.subTest(cmdline=cmdline), mock.patch.object(Path, "read_bytes", return_value=cmdline.encode()), ): self.assertEqual(self.supervisor._is_stale_pipeline_pid(123), expected) def test_signal_group_does_not_kill_a_non_leaders_group(self) -> None: with ( mock.patch.object(os, "getpgid", return_value=456), mock.patch.object(os, "killpg") as group_kill, mock.patch.object(os, "kill") as kill, ): self.supervisor._signal_group(123, signal.SIGTERM) group_kill.assert_not_called() kill.assert_called_once_with(123, signal.SIGTERM) def test_collector_reaper_does_not_match_command_text(self) -> None: entries = [Path("/proc/123"), Path("/proc/456")] cmdlines = [ b"bash\x00-c\x00echo kamil_adc_collector\x00", b"/project/build/bin/kamil_adc_collector\x00--config\x00config.json\x00", ] with ( mock.patch.object(Path, "iterdir", return_value=entries), mock.patch.object(Path, "read_bytes", side_effect=cmdlines), mock.patch.object(ProcessSupervisor, "_signal_group") as signal_group, ): self.supervisor._kill_kamil_collectors() signal_group.assert_called_once_with(456, signal.SIGKILL) if __name__ == "__main__": unittest.main()