awe/radar_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rework radar system

Browse Source
This commit is contained in:
Ayzen
2026-10-06 12:32:26 +03:00
parent 552cdc688a
commit 5bb86b7165
299 changed files with 13537 additions and 27763 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python_app/tests/test_bscan_replay.py
+195 -315
View File
@@ -1,366 +1,246 @@
"""Re-feeding retained sweeps to the processor to recompute the whole B-scan.
The processor keeps only ~50 preprocessed sweeps of its own, so a live settings edit
used to refresh just the newest 50 columns. The GUI holds up to 1000 of them and hands
them back through the processor's input ring, which only works if the bytes the GUI
writes are exactly the ones the C++ side expects — that wire-format contract is what
the first test pins, without needing the pipeline running.
The guard tests pin the other half: the GUI must never write into that ring while
`data_preprocessor` owns it.
"""
"""Full-history replay uses exact acquisition keys and owns its ring lifecycle."""
from __future__ import annotations
from contextlib import suppress
from dataclasses import replace
import json
import os
import struct
from pathlib import Path
import unittest
from unittest.mock import Mock, patch
import numpy as np
os.environ.setdefault("QT_QPA_PLATFORM", "offscreen")
from PyQt6.QtCore import QSignalBlocker # noqa: E402
from PyQt6.QtWidgets import QApplication # noqa: E402
from PyQt6.QtWidgets import QApplication
from python_app.gui.app_window import AppWindow # noqa: E402
from python_app.tests.gui_test_support import isolated_window
from python_app.gui.runtime.history import collection_key # noqa: E402
from python_app.models.dataset_model import ( # noqa: E402
ComboKey,
ResultBlock,
ResultCollection,
ResultPayload,
SweepCollection,
TraceData,
)
from python_app.orchestration.shm import ShmRingWriter # noqa: E402
from python_app.orchestration.shm.ring_reader import ShmRingReader # noqa: E402
from python_app.storage.npz.serialize import PREPROC_MAGIC, serialize_trace_collection # noqa: E402
# Mirrors kPreprocessedCollectionMagic in common_cpp/ipc/src/shared_types.cpp. If this
# ever drifts, the processor silently rejects everything the GUI re-feeds.
_CPP_PREPROCESSED_MAGIC = 0x32525050
from python_app.gui.controllers.bscan_replay import BscanReplay
from python_app.gui.controllers.history_controller import HistoryController
from python_app.models.collection_identity import collection_key
from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection, TraceData
from python_app.models.run_config_schema import RingEndpointModel
from python_app.orchestration.shm.decoder import PREPROC_MAGIC, RESULT_MAGIC, decode_trace_collection
from python_app.orchestration.shm.ring_reader import ShmRingReader
from python_app.orchestration.shm.ring_writer import ShmRingWriter
from python_app.storage.npz.serialize import serialize_trace_collection
_app: QApplication | None = None
_window: AppWindow | None = None
def setUpModule() -> None:
global _app, _window
_app = QApplication.instance() or QApplication([])
_window = isolated_window(unittest.addModuleCleanup)
def tearDownModule() -> None:
if _window is not None:
_window.close()
global _app
instance = QApplication.instance()
_app = instance if isinstance(instance, QApplication) else QApplication([])
def _sweep(collection_id: int, *, combos: int = 3, points: int = 64) -> SweepCollection:
traces = [
TraceData(
combo=ComboKey(input=index, output=index + 1),
frequency_hz=np.linspace(1e9, 8e9, points, dtype=np.float32),
s11=(np.arange(points) + index).astype(np.complex64) + 0.5j,
s21=(np.arange(points) * 2 + index).astype(np.complex64) - 0.25j,
)
for index in range(combos)
]
return SweepCollection(
collection_id=collection_id,
monotonic_ns=collection_id * 1_000_000 + 7,
traces=traces,
capture_start_ns=collection_id * 10,
capture_end_ns=collection_id * 10 + 5,
)
def _bscan_result(collection_id: int, *, points: int = 32) -> ResultCollection:
payload = ResultPayload(
processing_name="bscan",
kind=1,
frequency_hz=np.linspace(0.0, 1.0, points, dtype=np.float32),
trace=(np.arange(points) + collection_id).astype(np.complex64),
)
return ResultCollection(
collection_id=collection_id,
monotonic_ns=collection_id,
blocks=[ResultBlock(combo=ComboKey(input=0, output=0), payloads=[payload])],
)
class PreprocessedWireFormatTest(unittest.TestCase):
"""The bytes the GUI re-feeds must be the ones the C++ processor decodes."""
def setUp(self) -> None:
self.ring_name = f"/radar_test_{self._testMethodName}"
with suppress(OSError):
(Path("/dev/shm") / self.ring_name[1:]).unlink()
self.writer = ShmRingWriter(self.ring_name, 8, 1 << 20)
self.reader = ShmRingReader(self.ring_name)
self.addCleanup(self._cleanup)
def _cleanup(self) -> None:
with suppress(Exception):
self.reader.close()
with suppress(Exception):
self.writer.close()
with suppress(OSError):
(Path("/dev/shm") / self.ring_name[1:]).unlink()
def _assert_same(self, original: SweepCollection, decoded: SweepCollection) -> None:
self.assertEqual(decoded.collection_id, original.collection_id)
self.assertEqual(decoded.monotonic_ns, original.monotonic_ns)
self.assertEqual(decoded.capture_start_ns, original.capture_start_ns)
self.assertEqual(decoded.capture_end_ns, original.capture_end_ns)
self.assertEqual(len(decoded.traces), len(original.traces))
for left, right in zip(original.traces, decoded.traces, strict=True):
self.assertEqual((left.combo.input, left.combo.output), (right.combo.input, right.combo.output))
np.testing.assert_array_equal(left.frequency_hz, right.frequency_hz)
np.testing.assert_array_equal(left.s11, right.s11)
np.testing.assert_array_equal(left.s21, right.s21)
def test_magic_matches_the_cpp_decoder(self) -> None:
payload = serialize_trace_collection(_sweep(1), PREPROC_MAGIC)
self.assertEqual(int.from_bytes(payload[:4], "little"), _CPP_PREPROCESSED_MAGIC)
def test_round_trip_preserves_every_field(self) -> None:
original = _sweep(42)
self.assertTrue(self.writer.push(serialize_trace_collection(original, PREPROC_MAGIC)))
decoded = self.reader.pop_preprocessed_collection()
self.assertIsNotNone(decoded)
assert decoded is not None
self._assert_same(original, decoded)
def test_batch_keeps_order(self) -> None:
originals = [_sweep(cid) for cid in range(100, 105)]
for collection in originals:
self.assertTrue(self.writer.push(serialize_trace_collection(collection, PREPROC_MAGIC)))
decoded = []
while (collection := self.reader.pop_preprocessed_collection()) is not None:
decoded.append(collection)
self.assertEqual([c.collection_id for c in decoded], [c.collection_id for c in originals])
for left, right in zip(originals, decoded, strict=True):
self._assert_same(left, right)
def test_overflow_drops_oldest(self) -> None:
# Why the replay pushes in chunks instead of one burst: an undrained ring
# silently overwrites, which would punch holes into the very image we are
# trying to make coherent.
capacity = 8
for cid in range(200, 200 + capacity + 4):
self.writer.push(serialize_trace_collection(_sweep(cid), PREPROC_MAGIC))
survived = []
while (collection := self.reader.pop_preprocessed_collection()) is not None:
survived.append(collection.collection_id)
self.assertEqual(len(survived), capacity)
self.assertEqual(survived[-1], 200 + capacity + 3)
traces = [TraceData(
combo=ComboKey(input=index, output=index + 1),
frequency_hz=np.linspace(1e9, 8e9, points, dtype=np.float32),
s21=(np.arange(points) * 2 + index).astype(np.complex64) - 0.25j,
capture_start_ns=collection_id * 10 + index,
capture_end_ns=collection_id * 10 + index + 1,
) for index in range(combos)]
return SweepCollection(collection_id, collection_id * 1_000_000 + 7, traces,
capture_start_ns=collection_id * 10, capture_end_ns=collection_id * 10 + 5)
def _result_of(sweep: SweepCollection) -> ResultCollection:
"""The processor's result of a sweep: it keeps the sweep's id and timestamp."""
return replace(_bscan_result(sweep.collection_id), monotonic_ns=sweep.monotonic_ns)
return ResultCollection(sweep.collection_id, sweep.monotonic_ns + 100,
capture_start_ns=sweep.capture_start_ns, capture_end_ns=sweep.capture_end_ns, settings_revision=7)
class _FakeProcessor:
"""Both ends of the processor during a feed-back.
Each pushed sweep comes back as a result, after whatever the results ring already
held when the feed-back started.
"""
slot_size_bytes = 1 << 20
class _ProcessorReader(ShmRingReader):
"""Boundary stand-in: decode pushed wire bytes and queue processor results."""
def __init__(self, sweeps: list[SweepCollection], already_published: list[ResultCollection]) -> None:
self._unprocessed = list(sweeps)
self._published = list(already_published)
def __init__(self) -> None:
self.published: list[ResultCollection] = []
self.received: list[SweepCollection] = []
self.wait_once = False
def push(self, _payload: bytes) -> bool:
self._published.append(_result_of(self._unprocessed.pop(0)))
@property
def capacity(self) -> int:
return 50
def push(self, payload: bytes) -> bool:
sweep = decode_trace_collection(payload, PREPROC_MAGIC)
self.received.append(sweep)
self.published.append(_result_of(sweep))
return True
def pop_result_collection(self) -> ResultCollection | None:
return self._published.pop(0) if self._published else None
def close(self) -> None:
return None
if self.wait_once:
self.wait_once = False
return None
return self.published.pop(0) if self.published else None
class _IdleResultReader:
"""Stands in for a connected results reader that simply has nothing to hand out."""
def pop_result_collection(self):
return None
def close(self) -> None:
return None
class ReplayGuardTest(unittest.TestCase):
"""The GUI must refuse to write into a ring `data_preprocessor` still owns."""
class ReplayTest(unittest.TestCase):
def setUp(self) -> None:
self.w = _window
# The 50 ms ring poll and the debounce would both run against this half-faked
# window while the test drives it by hand; park them for the duration.
self.w._timer.stop()
self.w._bscan_reprocess_timer.stop()
self.addCleanup(self.w._timer.start)
self.addCleanup(self.w._bscan_reprocess_timer.stop)
self._original_is_running = self.w._supervisor.is_running
self._original_is_processor_running = self.w._supervisor.is_processor_running
self._original_mode = self.w._processing_mode.currentText()
# Changing the mode fires the live-settings handler, which would kick off the
# very replay these tests are inspecting.
with QSignalBlocker(self.w._processing_mode):
self.w._processing_mode.setCurrentText("bscan")
self.w._supervisor.is_running = lambda: False
self.w._supervisor.is_processor_running = lambda: True
self.w._result_reader = _IdleResultReader()
def tearDown(self) -> None:
self.w._supervisor.is_running = self._original_is_running
self.w._supervisor.is_processor_running = self._original_is_processor_running
with QSignalBlocker(self.w._processing_mode):
self.w._processing_mode.setCurrentText(self._original_mode)
self.w._result_reader = None
def test_refuses_while_acquisition_runs(self) -> None:
self.w._supervisor.is_running = lambda: True
self.assertFalse(self.w._can_reprocess_history())
def test_refuses_when_processor_is_down(self) -> None:
self.w._supervisor.is_processor_running = lambda: False
self.assertFalse(self.w._can_reprocess_history())
def test_refuses_outside_bscan_mode(self) -> None:
self.w._processing_mode.setCurrentText("pass_through")
self.assertFalse(self.w._can_reprocess_history())
def test_refuses_without_a_results_reader(self) -> None:
self.w._result_reader = None
self.assertFalse(self.w._can_reprocess_history())
def test_allows_when_stopped_with_a_live_processor(self) -> None:
self.assertTrue(self.w._can_reprocess_history())
def test_replayed_results_are_rendered_instead_of_runtime_history(self) -> None:
"""Regression: 300 re-sent, 300 recovered, only 145 drawn.
`_pre_history` and `_result_history` come from two rings that drop
independently, so the re-sent sweeps only partly overlap the recorded results.
The non-overlapping ones get appended to the deque under old ids, leaving its
newest `window` entries a mix of freshly and stale-processed frames. Rendering
from the replayed set instead sidesteps the whole problem.
"""
window = 30
replayed = [_bscan_result(cid) for cid in range(9000, 9000 + window)]
self.w._result_history.clear()
# Stand-in for a runtime history whose ids barely overlap the replayed ones.
self.w._result_history.extend(_bscan_result(cid) for cid in range(1, 200))
self.addCleanup(self.w._result_history.clear)
self.w._bscan_replay_results = replayed
self.addCleanup(self.w._discard_bscan_replay_results)
self.w._bscan_history_window.setValue(window)
self.w._bscan_render_signature = None
self.w._sync_bscan_history_from_results()
self.assertEqual(len(self.w._bscan_history_by_combo[(0, 0)]), window)
def test_feed_back_renders_its_sweeps_in_acquisition_order(self) -> None:
"""Regression: after switching the displayed combo the columns came out rotated.
The processor was still replaying its own sweeps when the GUI fed its history
back, and the GUI took the first results to arrive for its own, in arrival order.
"""
sweeps = [_sweep(cid) for cid in range(1, 11)]
self.w._pre_history.clear()
self.w._pre_history.extend(sweeps)
self.addCleanup(self.w._pre_history.clear)
self.w._result_history.clear()
self.addCleanup(self.w._result_history.clear)
with QSignalBlocker(self.w._bscan_history_window):
self.w._bscan_history_window.setValue(len(sweeps))
# Already in the results ring: the tail of the processor's own replay and a
# result of a sweep that is not being fed back.
processor = _FakeProcessor(sweeps, [_result_of(sweep) for sweep in sweeps[6:]] + [_bscan_result(999)])
self.w._replay_ring_writer = processor
self.addCleanup(setattr, self.w, "_replay_ring_writer", None)
self.w._result_reader = processor
self.addCleanup(self.w._discard_bscan_replay_results)
self.assertTrue(self.w._reprocess_history_through_processor())
self.assertEqual(
[collection_key(collection) for collection in self.w._bscan_replay_results],
[collection_key(sweep) for sweep in sweeps],
self.history = HistoryController()
self.warnings: list[str] = []
self.replay = BscanReplay(self.history, lambda: None, self.warning, lambda _text: None)
self.addCleanup(self.replay.close)
self.ring = RingEndpointModel("/unused_test_replay", 50, 1 << 20)
self.processor = _ProcessorReader()
self.writer = Mock(spec=ShmRingWriter, push=self.processor.push, slot_size_bytes=1 << 20)
writer_patch = patch(
"python_app.gui.controllers.bscan_replay.ShmRingWriter", return_value=self.writer,
)
self.assertIn(999, [collection.collection_id for collection in self.w._result_history])
self.writer_factory = writer_patch.start()
self.addCleanup(writer_patch.stop)
def test_changing_the_displayed_combo_recomputes_nothing(self) -> None:
# Every combo's B-scan is cached already; recomputing raced the processor (above).
writes: list[dict] = []
self.w._write_live_processing_config = lambda **kwargs: writes.append(kwargs)
self.addCleanup(delattr, self.w, "_write_live_processing_config")
def warning(self, text: str, *, details: str | None = None, once_key: str | None = None) -> None:
self.warnings.append(text)
self.w._single_combo_output.editingFinished.emit()
def run_replay(self) -> bool:
return self.replay.run(window=300, revision=7, ring=self.ring, reader=self.processor,
continue_replay=lambda: True, pump_events=lambda _pause: None)
self.assertEqual(writes, [])
self.assertFalse(self.w._bscan_reprocess_timer.isActive())
def test_result_ring_capacity_bounds_each_replay_burst(self) -> None:
name = f"/radar_replay_small_result_{os.getpid()}"
self.addCleanup((Path("/dev/shm") / name[1:]).unlink, missing_ok=True)
results = ShmRingWriter(name, 2, 256)
self.addCleanup(results.close)
reader = ShmRingReader(name)
self.addCleanup(reader.close)
def test_processor_reprocesses_only_while_the_gui_cannot(self) -> None:
# Its own replay would duplicate the feed-back and race it for the results ring.
def written_flag() -> bool:
self.w._write_live_processing_config()
config = json.loads(self.w._live_config_writer.path.read_text(encoding="utf-8"))
return bool(config["reprocess_current_result"])
def process(payload: bytes) -> bool:
sweep = decode_trace_collection(payload, PREPROC_MAGIC)
return results.push(struct.pack("<I6QfII", RESULT_MAGIC, sweep.collection_id,
sweep.monotonic_ns, 0, 7, sweep.capture_start_ns,
sweep.capture_end_ns, 0., 0, 0))
self.assertFalse(written_flag())
self.w._supervisor.is_running = lambda: True
self.assertTrue(written_flag())
self.writer.push = Mock(side_effect=process)
self.history.preprocessed.extend(_sweep(cid) for cid in range(1, 7))
self.assertTrue(self.replay.run(
window=6, revision=7, ring=self.ring, reader=reader,
continue_replay=lambda: True, pump_events=lambda _pause: None,
))
self.assertEqual([item.collection_id for item in self.history.replay_results], list(range(1, 7)))
self.assertEqual(self.warnings, [])
def test_discarding_replay_falls_back_to_runtime_history(self) -> None:
self.w._bscan_replay_results = [_bscan_result(1)]
self.w._discard_bscan_replay_results()
self.assertEqual(self.w._bscan_replay_results, [])
self.assertIsNone(self.w._bscan_render_signature)
def test_replays_full_history_in_capture_order_despite_independent_ring_drops(self) -> None:
sweeps = [_sweep(cid) for cid in range(1, 301)]
self.history.preprocessed.extend(sweeps)
self.history.results.extend(_result_of(_sweep(cid)) for cid in range(300, 450))
# Old processor replay arrives before the GUI's own results, including a
# restarted collection id whose capture key differs from the requested one.
unrelated = _result_of(replace(_sweep(1), capture_start_ns=9000, capture_end_ns=9005))
stale_tail = replace(_result_of(sweeps[-1]), settings_revision=6)
self.processor.published = [stale_tail, *[_result_of(sweep) for sweep in sweeps[6:10]], unrelated]
self.assertTrue(self.run_replay())
source = self.history.bscan_source()
self.assertEqual([collection_key(item) for item in source], [collection_key(item) for item in sweeps])
self.assertEqual(len(self.processor.received), 300)
self.assertTrue(all(item.settings_revision == 7 for item in source))
self.assertIsNot(source[-1], stale_tail)
self.assertIn(unrelated, self.history.results)
self.writer_factory.assert_called_once_with(self.ring.name, self.ring.capacity, self.ring.slot_size_bytes)
def test_replay_refuses_to_re_enter_itself(self) -> None:
# Pumping the event loop mid-replay can fire the debounce again; a nested burst
# would share the ring and corrupt the result accounting.
self.w._pre_history.clear()
self.w._pre_history.extend(_sweep(cid) for cid in range(1, 6))
self.addCleanup(self.w._pre_history.clear)
def test_refuses_without_reader_or_active_ring_geometry(self) -> None:
self.history.preprocessed.append(_sweep(1))
self.assertFalse(self.replay.run(window=1, revision=7, ring=self.ring, reader=None,
continue_replay=lambda: True, pump_events=lambda _pause: None))
self.assertFalse(self.replay.run(window=1, revision=7, ring=None, reader=self.processor,
continue_replay=lambda: True, pump_events=lambda _pause: None))
self.writer_factory.assert_not_called()
self.assertIn("geometry is unknown", self.warnings[-1])
self.w._bscan_replay_active = True
self.addCleanup(setattr, self.w, "_bscan_replay_active", False)
def test_nested_replay_is_refused(self) -> None:
self.history.preprocessed.append(_sweep(1))
self.processor.wait_once = True
nested: list[bool] = []
self.assertFalse(self.w._reprocess_history_through_processor())
def pump(_pause: float) -> None:
nested.append(self.run_replay())
def test_replay_refuses_without_an_active_run_config(self) -> None:
# A GUI restarted against a still-running processor does not know the ring
# geometry, and ShmRingWriter would recreate the segment underneath it.
self.w._pre_history.clear()
self.w._pre_history.extend(_sweep(cid) for cid in range(1, 6))
self.addCleanup(self.w._pre_history.clear)
self.assertTrue(self.replay.run(window=1, revision=7, ring=self.ring, reader=self.processor,
continue_replay=lambda: True, pump_events=pump))
self.assertEqual(nested, [False])
self.assertFalse(self.replay.active)
previous = getattr(self.w, "_active_run_config", None)
self.w._active_run_config = None
self.addCleanup(setattr, self.w, "_active_run_config", previous)
def test_close_discard_or_settings_edit_during_pump_cancels_current_replay(self) -> None:
self.history.preprocessed.extend(_sweep(cid) for cid in range(1, 25))
for action in ("close", "discard", "schedule"):
with self.subTest(action=action):
self.processor.published.clear()
self.processor.received.clear()
self.processor.wait_once = True
self.assertIsNone(self.w._ensure_replay_ring_writer())
self.assertFalse(self.w._reprocess_history_through_processor())
def pump(_pause: float) -> None:
if action == "schedule":
self.replay.schedule(True)
elif action == "close":
self.replay.close()
else:
self.replay.discard()
self.assertFalse(self.run_replay())
self.assertFalse(self.replay.run(
window=24, revision=7, ring=self.ring, reader=self.processor,
continue_replay=lambda: True, pump_events=pump,
))
self.assertLess(len(self.processor.received), 24)
self.assertEqual(list(self.history.results), [])
self.assertEqual(self.history.replay_results, [])
self.assertFalse(self.replay.active)
self.assertEqual(self.replay.timer.isActive(), action == "schedule")
def test_runtime_guard_is_rechecked_before_io_and_after_event_pumping(self) -> None:
self.history.preprocessed.extend(_sweep(cid) for cid in range(1, 25))
for stop_at in ("push", "pop", "pump"):
with self.subTest(stop_at=stop_at):
self.processor.published.clear()
self.processor.received.clear()
self.processor.wait_once = stop_at == "pump"
running = True
def guard() -> bool:
count = len(self.processor.received)
return running and not (stop_at == "push" and count >= 1
or stop_at == "pop" and count >= 12)
def pump(_pause: float) -> None:
nonlocal running
running = False
self.assertFalse(self.replay.run(
window=24, revision=7, ring=self.ring, reader=self.processor,
continue_replay=guard, pump_events=pump,
))
self.assertEqual(len(self.processor.received), 1 if stop_at == "push" else 12)
self.assertEqual(list(self.history.results), [])
self.assertEqual(self.history.replay_results, [])
self.assertFalse(self.replay.active)
def test_small_ring_is_drained_before_another_burst(self) -> None:
self.ring.capacity = 2
self.history.preprocessed.extend(_sweep(cid) for cid in range(1, 16))
def push(payload: bytes) -> bool:
self.assertLess(len(self.processor.published), self.ring.capacity)
return self.processor.push(payload)
self.writer.push = push
self.assertTrue(self.run_replay())
self.assertEqual(len(self.history.replay_results), 15)
def test_discard_and_close_release_replay_source_timer_and_writer(self) -> None:
self.history.preprocessed.append(_sweep(1))
self.assertTrue(self.run_replay())
self.replay.discard()
self.assertEqual(self.history.bscan_source(), list(self.history.results))
self.replay.schedule(False)
self.assertFalse(self.replay.timer.isActive())
self.replay.schedule(True)
self.assertTrue(self.replay.timer.isActive())
self.replay.close()
self.replay.close()
self.assertFalse(self.replay.timer.isActive())
self.writer.close.assert_called_once()
if __name__ == "__main__":