177 lines
9.6 KiB
Python
177 lines
9.6 KiB
Python
"""Reader recovery requires data, rather than an open mapping or an empty poll."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from contextlib import ExitStack
|
|
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
from PyQt6.QtWidgets import QApplication
|
|
|
|
from python_app.gui.controllers.history_controller import HistoryController
|
|
from python_app.gui.controllers.pipeline_controller import PipelineController
|
|
from python_app.gui.controllers.plot_controller import PlotController
|
|
from python_app.gui.controllers.recording_controller import RecordingController
|
|
from python_app.models.dataset_model import ResultCollection, SweepCollection
|
|
from python_app.models.gui_profile_schema import GuiProcessingStateModel
|
|
from python_app.models.run_config_schema import RunConfigModel
|
|
from python_app.models.run_config_schema import GprModel
|
|
from python_app.orchestration.config_writer import ConfigWriter
|
|
from python_app.orchestration.live_processing_config import ProcessingLiveConfig
|
|
from python_app.orchestration.process_supervisor import ProcessSupervisor
|
|
from python_app.orchestration.shm.ring_reader import ShmRingReader
|
|
from python_app.storage.npz.store import NpzStore
|
|
|
|
_app: QApplication | None = None
|
|
|
|
|
|
def pipeline_fixture(test: unittest.TestCase) -> tuple[PipelineController, mock.Mock]:
|
|
"""Use real controllers, isolating only processes and external GUI callbacks."""
|
|
cleanup = ExitStack()
|
|
test.addCleanup(cleanup.close)
|
|
global _app
|
|
instance = QApplication.instance()
|
|
_app = instance if isinstance(instance, QApplication) else QApplication([])
|
|
root = Path(cleanup.enter_context(tempfile.TemporaryDirectory()))
|
|
store, history = NpzStore(root / "data"), HistoryController()
|
|
plots = PlotController(history, GuiProcessingStateModel, GprModel, lambda: False, mock.Mock())
|
|
test.addCleanup(plots.stack.close)
|
|
recording = RecordingController(store, {}, mock.Mock(), mock.Mock(), mock.Mock())
|
|
supervisor = mock.Mock(spec=ProcessSupervisor)
|
|
supervisor.collect_exit_reports.return_value = []
|
|
supervisor.pids.return_value = {}
|
|
supervisor.is_running.return_value = False
|
|
supervisor.is_processor_running.return_value = False
|
|
with mock.patch("python_app.gui.controllers.pipeline_controller.ProcessSupervisor", return_value=supervisor):
|
|
pipeline = PipelineController(root, store, ConfigWriter(root / "runtime"), history, plots, recording,
|
|
build_config=RunConfigModel, live_config=ProcessingLiveConfig, publish_live=mock.Mock(),
|
|
capture_guard=lambda: None, refresh_radar_limits=lambda: False, status=mock.Mock(),
|
|
history_changed=mock.Mock(), poll_ui=mock.Mock(), show_exception=mock.Mock(), deferred_close=mock.Mock())
|
|
test.addCleanup(pipeline.timer.stop)
|
|
test.addCleanup(pipeline.replay.close)
|
|
return pipeline, supervisor
|
|
|
|
|
|
def _reader(name: str) -> mock.Mock:
|
|
reader = mock.Mock(spec=ShmRingReader)
|
|
reader.name = name
|
|
reader.pop_raw_collection.return_value = None
|
|
reader.pop_preprocessed_collection.return_value = None
|
|
reader.pop_result_collection.return_value = None
|
|
return reader
|
|
|
|
|
|
class ReaderRecoveryTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
cleanup = ExitStack()
|
|
self.addCleanup(cleanup.close)
|
|
self.pipeline, self.supervisor = pipeline_fixture(self)
|
|
cleanup.enter_context(mock.patch.multiple(self.pipeline, _READER_ERROR_RECONNECT_AT=2,
|
|
_READER_ERROR_RELOG_EVERY=3, _READER_ERROR_STOP_AT=5))
|
|
self.pipeline.max_pop_per_poll = 1
|
|
self.pipeline._raw_reader = self.raw = _reader("/raw")
|
|
self.pipeline._pre_reader = self.pre = _reader("/preprocessed")
|
|
self.pipeline._result_reader = self.result = _reader("/results")
|
|
self.stop = cleanup.enter_context(mock.patch.object(self.pipeline, "stop_all"))
|
|
self.draw = cleanup.enter_context(mock.patch.object(self.pipeline.plots, "draw"))
|
|
self.error = cleanup.enter_context(mock.patch.object(self.pipeline, "_show_exception"))
|
|
self.status = cleanup.enter_context(mock.patch.object(self.pipeline, "_status"))
|
|
self.open_reader = cleanup.enter_context(mock.patch(
|
|
"python_app.gui.controllers.pipeline_controller.ShmRingReader", side_effect=_reader))
|
|
|
|
def fail_poll(self, error: Exception) -> None:
|
|
assert self.pipeline._raw_reader is not None
|
|
with mock.patch.object(self.pipeline._raw_reader, "pop_raw_collection", side_effect=error):
|
|
self.pipeline.poll()
|
|
|
|
def test_successful_reconnect_and_empty_polls_preserve_the_failure_streak(self) -> None:
|
|
self.fail_poll(ValueError("corrupt frame"))
|
|
self.open_reader.assert_not_called()
|
|
self.fail_poll(ValueError("corrupt frame"))
|
|
self.assertEqual(self.open_reader.call_args_list, [mock.call("/raw"), mock.call("/preprocessed"), mock.call("/results")])
|
|
for reader in (self.raw, self.pre, self.result):
|
|
reader.close.assert_called_once()
|
|
for _ in range(3):
|
|
self.pipeline.poll()
|
|
self.assertEqual(self.pipeline._reader_error_repeat_count, 2)
|
|
self.assertEqual(self.pipeline._last_reader_error_signature, ("ValueError", "corrupt frame"))
|
|
self.stop.assert_not_called()
|
|
|
|
def test_alternating_error_signatures_reach_stop_despite_empty_polls(self) -> None:
|
|
for count in range(1, 6):
|
|
self.fail_poll(ValueError("bad size") if count % 2 else RuntimeError("bad magic"))
|
|
if count < 5:
|
|
self.stop.assert_not_called()
|
|
self.pipeline.poll()
|
|
self.assertEqual(self.pipeline._reader_error_repeat_count, count)
|
|
self.stop.assert_called_once()
|
|
self.assertEqual(self.open_reader.call_count, 3)
|
|
self.assertEqual(self.pipeline._reader_error_repeat_count, 0)
|
|
self.assertIsNone(self.pipeline._last_reader_error_signature)
|
|
self.status.assert_called_with("Status: error")
|
|
|
|
def test_failed_reconnect_does_not_cancel_escalation(self) -> None:
|
|
self.open_reader.side_effect = OSError("mapping unavailable")
|
|
for _ in range(5):
|
|
self.fail_poll(RuntimeError("closed mapping"))
|
|
self.open_reader.assert_called_once_with("/raw")
|
|
self.stop.assert_called_once()
|
|
|
|
def test_only_a_received_collection_resets_the_streak_for_each_stage(self) -> None:
|
|
stages = ((self.raw.pop_raw_collection, self.pipeline.history.raw, SweepCollection(1, 100)),
|
|
(self.pre.pop_preprocessed_collection, self.pipeline.history.preprocessed, SweepCollection(1, 110)),
|
|
(self.result.pop_result_collection, self.pipeline.history.results, ResultCollection(1, 110)))
|
|
for pop, history, collection in stages:
|
|
with self.subTest(collection=type(collection).__name__, stage=pop):
|
|
self.pipeline._reader_error_repeat_count = 4
|
|
self.pipeline._last_reader_error_signature = ("ValueError", "corrupt frame")
|
|
pop.return_value = collection
|
|
self.pipeline.poll()
|
|
pop.return_value = None
|
|
self.assertEqual(list(history), [collection])
|
|
self.assertEqual(self.pipeline._reader_error_repeat_count, 0)
|
|
self.assertIsNone(self.pipeline._last_reader_error_signature)
|
|
self.fail_poll(ValueError("another corrupt frame"))
|
|
self.assertEqual(self.pipeline._reader_error_repeat_count, 1)
|
|
self.stop.assert_not_called()
|
|
|
|
def test_identical_failures_are_periodically_logged_without_resetting_the_streak(self) -> None:
|
|
with self.assertLogs("python_app.gui.controllers.pipeline_controller", level="ERROR") as logs:
|
|
for _ in range(3):
|
|
self.fail_poll(ValueError("corrupt frame"))
|
|
self.assertEqual(self.pipeline._reader_error_repeat_count, 3)
|
|
self.assertEqual([record.getMessage() for record in logs.records], [
|
|
"Reader poll failed", "Reader poll still failing (repeat #3)"])
|
|
|
|
def test_read_failure_skips_display_and_uses_reader_recovery(self) -> None:
|
|
self.result.pop_result_collection.return_value = ResultCollection(1, 110)
|
|
self.fail_poll(ValueError("corrupt frame"))
|
|
self.assertEqual(self.pipeline._reader_error_repeat_count, 1)
|
|
self.draw.assert_not_called()
|
|
self.error.assert_not_called()
|
|
self.stop.assert_not_called()
|
|
|
|
def test_preprocess_change_rejects_late_results_while_processor_stays_alive(self) -> None:
|
|
for restart in (False, True):
|
|
with self.subTest(restart_processor=restart), mock.patch.object(self.pipeline, "publish_live") as publish:
|
|
old, fresh = [ResultCollection(1, start, capture_start_ns=start) for start in (100, 201)]
|
|
self.pipeline.history.results.append(old)
|
|
self.pipeline._processor_signature = ("old processor",)
|
|
self.supervisor.is_processor_running.return_value = True
|
|
with mock.patch("python_app.gui.controllers.history_controller.time.monotonic_ns", return_value=200):
|
|
self.pipeline.preprocessing_changed(restart_processor=restart)
|
|
self.result.pop_result_collection.side_effect = [old, fresh]
|
|
self.assertIsNone(self.pipeline.read_results())
|
|
self.assertFalse(self.pipeline.history.results)
|
|
self.assertIs(self.pipeline.read_results(), fresh)
|
|
publish.assert_called_once()
|
|
self.assertEqual(self.pipeline._processor_signature, None if restart else ("old processor",))
|
|
self.supervisor.stop_all.assert_not_called()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|