web UI added and refactoring done
This commit is contained in:
Ayzen
@@ -0,0 +1,197 @@
|
||||
"""Orchestration & recovery tests.
|
||||
|
||||
Pins the agreed semantics:
|
||||
* crash auto-restart retries FOREVER with capped exponential back-off (never gives
|
||||
up — unattended appliance); the failure streak resets when data flows again;
|
||||
* config writes are atomic — a serialization failure leaves no partial output and
|
||||
never corrupts an existing config;
|
||||
* the acquisition producer command is selected by radar.model;
|
||||
* process exit reports classify clean vs unexpected exits correctly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from python_app.models.run_config_model import RunConfigModel
|
||||
from python_app.orchestration.config_writer import ConfigWriter
|
||||
from python_app.orchestration.process_supervisor import ProcessExitReport, ProcessSupervisor
|
||||
from python_app.orchestration.restart_policy import RestartPolicy
|
||||
|
||||
|
||||
class RestartPolicyTest(unittest.TestCase):
|
||||
def test_backoff_grows_and_caps(self) -> None:
|
||||
policy = RestartPolicy(min_interval_s=3.0, max_interval_s=60.0, backoff_factor=2.0)
|
||||
self.assertEqual(policy.backoff_for(0), 3.0)
|
||||
self.assertEqual(policy.backoff_for(1), 6.0)
|
||||
self.assertEqual(policy.backoff_for(2), 12.0)
|
||||
self.assertEqual(policy.backoff_for(3), 24.0)
|
||||
self.assertEqual(policy.backoff_for(100), 60.0) # capped
|
||||
|
||||
def test_never_gives_up_even_after_huge_streak(self) -> None:
|
||||
# No attempt cap: a long failure streak still yields a finite, capped wait.
|
||||
policy = RestartPolicy()
|
||||
self.assertEqual(policy.backoff_for(10_000), policy.max_interval_s)
|
||||
|
||||
def test_should_restart_respects_backoff_window(self) -> None:
|
||||
policy = RestartPolicy(min_interval_s=3.0, max_interval_s=60.0)
|
||||
# First failure (streak 0): needs >= 3s since last restart.
|
||||
self.assertFalse(policy.should_restart_now(now_s=102.0, last_restart_s=100.0, consecutive_failures=0))
|
||||
self.assertTrue(policy.should_restart_now(now_s=103.0, last_restart_s=100.0, consecutive_failures=0))
|
||||
# After 2 failures the window is 12s.
|
||||
self.assertFalse(policy.should_restart_now(now_s=111.0, last_restart_s=100.0, consecutive_failures=2))
|
||||
self.assertTrue(policy.should_restart_now(now_s=112.0, last_restart_s=100.0, consecutive_failures=2))
|
||||
|
||||
def test_first_restart_is_immediate(self) -> None:
|
||||
# last_restart defaults to 0.0 in the mixin, so the very first crash restarts now.
|
||||
self.assertTrue(RestartPolicy().should_restart_now(now_s=5_000.0, last_restart_s=0.0, consecutive_failures=0))
|
||||
|
||||
|
||||
class ConfigWriterTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._dir = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self._dir.cleanup)
|
||||
self.root = Path(self._dir.name)
|
||||
self.writer = ConfigWriter(self.root / "runtime")
|
||||
|
||||
def test_writes_loadable_config_and_leaves_no_tmp(self) -> None:
|
||||
out = self.root / "run_config.json"
|
||||
self.writer.write(RunConfigModel(), out)
|
||||
self.assertTrue(out.exists())
|
||||
self.assertFalse(out.with_suffix(".json.tmp").exists())
|
||||
RunConfigModel.from_dict(json.loads(out.read_text())) # round-trips back through the schema
|
||||
|
||||
def test_nan_fails_loudly_without_corrupting_existing(self) -> None:
|
||||
out = self.root / "run_config.json"
|
||||
self.writer.write(RunConfigModel(), out)
|
||||
original = out.read_text()
|
||||
|
||||
broken = RunConfigModel()
|
||||
broken.radar.sweep.if_bandwidth_hz = float("nan")
|
||||
with self.assertRaises(ValueError): # allow_nan=False
|
||||
self.writer.write(broken, out)
|
||||
|
||||
self.assertEqual(out.read_text(), original) # existing config untouched
|
||||
self.assertFalse(out.with_suffix(".json.tmp").exists()) # no half-written tmp left
|
||||
|
||||
|
||||
class RadarModelCommandTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._dir = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self._dir.cleanup)
|
||||
self.root = Path(self._dir.name)
|
||||
self.supervisor = ProcessSupervisor(self.root)
|
||||
|
||||
def _config_with_model(self, model: str) -> Path:
|
||||
path = self.root / "cfg.json"
|
||||
path.write_text(json.dumps({"radar": {"model": model}}))
|
||||
return path
|
||||
|
||||
def test_read_radar_model(self) -> None:
|
||||
self.assertEqual(ProcessSupervisor._read_radar_model(self._config_with_model("kamil_adc")), "kamil_adc")
|
||||
|
||||
def test_read_radar_model_defaults_on_malformed(self) -> None:
|
||||
path = self.root / "bad.json"
|
||||
path.write_text("[]") # not an object
|
||||
self.assertEqual(ProcessSupervisor._read_radar_model(path), "librevna")
|
||||
|
||||
def test_matrix_producer_for_multi_and_sn9000(self) -> None:
|
||||
for model in ("librevna_multi", "sn9000"):
|
||||
cmd = self.supervisor._acquisition_command(self._config_with_model(model))
|
||||
self.assertEqual(cmd[:3], [sys.executable, "-m", "python_app.scripts.matrix_raw_producer"])
|
||||
|
||||
def test_kamil_producer_for_kamil_adc(self) -> None:
|
||||
cmd = self.supervisor._acquisition_command(self._config_with_model("kamil_adc"))
|
||||
self.assertEqual(cmd[:3], [sys.executable, "-m", "python_app.scripts.kamil_adc_raw_producer"])
|
||||
|
||||
def test_native_orchestrator_for_librevna(self) -> None:
|
||||
cmd = self.supervisor._acquisition_command(self._config_with_model("librevna"))
|
||||
self.assertTrue(cmd[0].endswith("build/bin/sweep_orchestrator"))
|
||||
|
||||
|
||||
class ProcessExitReportTest(unittest.TestCase):
|
||||
@staticmethod
|
||||
def _report(*, code: int, clean: bool) -> ProcessExitReport:
|
||||
return ProcessExitReport(
|
||||
name="data_processor",
|
||||
command=["x"],
|
||||
working_directory=Path("/tmp"),
|
||||
return_code=code,
|
||||
stdout_path=Path("/nonexistent.out"),
|
||||
stderr_path=Path("/nonexistent.err"),
|
||||
expected_clean_exit=clean,
|
||||
)
|
||||
|
||||
def test_clean_exit_is_info(self) -> None:
|
||||
report = self._report(code=0, clean=True)
|
||||
self.assertEqual(report.level, "INFO")
|
||||
self.assertIn("completed normally", report.format())
|
||||
|
||||
def test_nonzero_exit_is_error(self) -> None:
|
||||
report = self._report(code=3, clean=False)
|
||||
self.assertEqual(report.level, "ERROR")
|
||||
self.assertIn("exited with code 3", report.format())
|
||||
|
||||
def test_unexpected_zero_exit_is_error(self) -> None:
|
||||
report = self._report(code=0, clean=False)
|
||||
self.assertEqual(report.level, "ERROR")
|
||||
self.assertIn("exited unexpectedly with code 0", report.format())
|
||||
|
||||
|
||||
class SupervisorLifecycleTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._dir = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self._dir.cleanup)
|
||||
self.supervisor = ProcessSupervisor(Path(self._dir.name))
|
||||
self.addCleanup(self.supervisor.stop_all)
|
||||
|
||||
def _await_reports(self, timeout_s: float = 3.0) -> list[ProcessExitReport]:
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
reports = self.supervisor.collect_exit_reports()
|
||||
if reports:
|
||||
return reports
|
||||
time.sleep(0.02)
|
||||
return []
|
||||
|
||||
def test_spawn_alive_then_stop(self) -> None:
|
||||
self.supervisor._spawn("data_preprocessor",
|
||||
[sys.executable, "-c", "import time; time.sleep(30)"],
|
||||
allow_clean_exit=False)
|
||||
self.assertTrue(self.supervisor.is_running())
|
||||
self.assertIn("data_preprocessor", self.supervisor.pids())
|
||||
self.supervisor.stop()
|
||||
self.assertFalse(self.supervisor.is_running())
|
||||
|
||||
def test_unexpected_exit_reported_as_error(self) -> None:
|
||||
self.supervisor._spawn("data_processor",
|
||||
[sys.executable, "-c", "import sys; sys.exit(3)"],
|
||||
allow_clean_exit=False)
|
||||
reports = self._await_reports()
|
||||
self.assertEqual(len(reports), 1)
|
||||
self.assertEqual(reports[0].return_code, 3)
|
||||
self.assertEqual(reports[0].level, "ERROR")
|
||||
|
||||
def test_clean_exit_reported_as_info(self) -> None:
|
||||
self.supervisor._spawn("sweep_orchestrator",
|
||||
[sys.executable, "-c", "import sys; sys.exit(0)"],
|
||||
allow_clean_exit=True)
|
||||
reports = self._await_reports()
|
||||
self.assertEqual(len(reports), 1)
|
||||
self.assertTrue(reports[0].expected_clean_exit)
|
||||
self.assertEqual(reports[0].level, "INFO")
|
||||
|
||||
def test_stale_pid_guard_rejects_unrelated_processes(self) -> None:
|
||||
# The reap guard must never kill a recycled PID that is not one of our binaries.
|
||||
self.assertFalse(self.supervisor._is_stale_pipeline_pid(2_000_000_000)) # no such pid
|
||||
self.assertFalse(self.supervisor._is_stale_pipeline_pid(os.getpid())) # the test runner
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user