"""The processor's history stays the acquisition, in order, when the GUI feeds sweeps back. To recompute the whole B-scan the GUI pushes its retained sweeps into the processor's input ring again (BscanReplay). The processor used to append them to its own history as if they were new, so its next replay published every sweep two or three times and out of order, and the B-scan came out with its columns rotated. Runs the real `data_processor` on private rings; skipped when it has not been built. """ from __future__ import annotations import json import os import signal import subprocess import tempfile import time import unittest from pathlib import Path import numpy as np from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData from python_app.orchestration.live_processing_config import ProcessingLiveConfig, ProcessingLiveConfigWriter from python_app.orchestration.shm.ring_writer import ShmRingWriter from python_app.orchestration.shm.ring_reader import ShmRingReader from python_app.orchestration.shm.decoder import PREPROC_MAGIC from python_app.storage.npz.serialize import serialize_trace_collection _ROOT = Path(__file__).resolve().parents[2] _PROCESSOR = _ROOT / "build/bin/data_processor" _SWEEPS = 5 _TIMEOUT_S = 5.0 def _sweep(collection_id: int, monotonic_ns: int) -> SweepCollection: points = 64 trace = TraceData( combo=ComboKey(input=0, output=0), frequency_hz=np.linspace(1e9, 6e9, points, dtype=np.float32), s21=np.full(points, collection_id, dtype=np.complex64), capture_start_ns=monotonic_ns, capture_end_ns=monotonic_ns + 1000, ) return SweepCollection(collection_id=collection_id, monotonic_ns=monotonic_ns, traces=[trace], capture_start_ns=monotonic_ns, capture_end_ns=monotonic_ns + 1000) @unittest.skipUnless(_PROCESSOR.exists(), "data_processor is not built (run make)") class _ProcessorTestCase(unittest.TestCase): result_slot_size_bytes: int | None = None idle_sleep_ms: int | None = None def setUp(self) -> None: workdir = Path(tempfile.mkdtemp(prefix="processor_history_")) config = json.loads((_ROOT / "run_config.json").read_text(encoding="utf-8")) for name, ring in config["rings"].items(): ring["name"] = f"/test_processor_history_{os.getpid()}_{name}" if self.result_slot_size_bytes is not None: config["rings"]["results"]["slot_size_bytes"] = self.result_slot_size_bytes if self.idle_sleep_ms is not None: config["run"]["idle_sleep_ms"] = self.idle_sleep_ms config["run"]["combos"] = [{"input": 0, "output": 0}] config["run"]["processing_live_config_path"] = str(workdir / "processing_live.json") config["run"]["locator_server"]["enabled"] = False config_path = workdir / "run_config.json" config_path.write_text(json.dumps(config), encoding="utf-8") self._live = ProcessingLiveConfig(processor_mode="bscan") self._live_writer = ProcessingLiveConfigWriter(workdir / "processing_live.json") self._live_writer.write(self._live) rings = config["rings"] shm_files = [Path("/dev/shm" + ring["name"]) for ring in rings.values()] self._stderr_path = workdir / "processor.log" stderr_log = self._stderr_path.open("w") self.addCleanup(stderr_log.close) processor = subprocess.Popen( [str(_PROCESSOR), "--config", str(config_path)], cwd=_ROOT, stdout=subprocess.DEVNULL, stderr=stderr_log, ) def cleanup() -> None: processor.terminate() processor.wait(_TIMEOUT_S) for path in shm_files: path.unlink(missing_ok=True) for path in workdir.iterdir(): path.unlink() workdir.rmdir() self.addCleanup(cleanup) self._processor = processor # The processor creates both rings at start-up, the results ring last. self._results = ShmRingReader(rings["results"]["name"], open_timeout_s=_TIMEOUT_S) self.addCleanup(self._results.close) ring = rings["preprocessed"] self._input = ShmRingWriter(ring["name"], int(ring["capacity"]), int(ring["slot_size_bytes"])) self.addCleanup(self._input.close) def _push(self, sweeps: list[SweepCollection]) -> None: for sweep in sweeps: self.assertTrue(self._input.push(serialize_trace_collection(sweep, PREPROC_MAGIC))) def _published_ids(self, count: int) -> list[int]: """Ids of the next results: `count` of them plus any that follow shortly after.""" ids: list[int] = [] deadline = time.monotonic() + _TIMEOUT_S while time.monotonic() < deadline: collection = self._results.pop_result_collection() if collection is None: time.sleep(0.01) continue ids.append(int(collection.collection_id)) if len(ids) == count: deadline = min(deadline, time.monotonic() + 0.3) return ids def _result_for_revision(self, revision: int, collection_id: int | None = None): deadline = time.monotonic() + _TIMEOUT_S while time.monotonic() < deadline: result = self._results.pop_result_collection() if (result is not None and result.settings_revision == revision and (collection_id is None or result.collection_id == collection_id)): return result time.sleep(0.01) self.fail(f"No processor result for settings revision {revision}: {self._stderr_path.read_text()}") class FedBackSweepsTest(_ProcessorTestCase): def test_remove_targets_exact_sweep_with_reused_ids_and_preserves_missing_targets(self) -> None: start_ns = time.monotonic_ns() sweeps = [_sweep(cid, start_ns + index * 1_000_000) for index, cid in enumerate((7, 8, 7, 9, 8))] self._push(sweeps) self.assertEqual(self._published_ids(5), [7, 8, 7, 9, 8]) targets = [ ({"collection_id": 7, "capture_start_ns": sweeps[2].capture_start_ns}, [7, 8, 9, 8]), ({"collection_id": 8, "capture_start_ns": sweeps[1].capture_start_ns}, [7, 9, 8]), ({"collection_id": 7, "capture_start_ns": start_ns + 20_000_000}, [7, 9, 8]), ] for sequence, (target, expected) in enumerate(targets, start=1): with self.subTest(target=target): self._live.settings_revision = sequence self._live.history_removed.append(target) self._live_writer.write(self._live) self.assertEqual(self._published_ids(len(expected)), expected) def test_removed_target_arriving_after_command_cannot_reenter_history(self) -> None: now = time.monotonic_ns() seed, target, following = [_sweep(cid, now + cid * 1_000_000) for cid in (1, 2, 3)] self._push([seed]) self.assertEqual(self._published_ids(1), [1]) self._live.settings_revision = 1 self._live.history_removed = [dict(collection_id=target.collection_id, capture_start_ns=target.capture_start_ns)] self._live_writer.write(self._live) self.assertEqual(self._published_ids(1), [1]) # Confirms the command was applied. self._push([target, following]) self.assertEqual(self._published_ids(1), [3]) self._live.settings_revision = 2 self._live_writer.write(self._live) self.assertEqual(self._published_ids(2), [1, 3]) def test_result_preserves_capture_window_for_missing_tap_counterparts(self) -> None: sweep = _sweep(1, time.monotonic_ns()) sweep.capture_start_ns = (1 << 63) + 987654321 sweep.capture_end_ns = sweep.capture_start_ns + 123456789 self._push([sweep]) result = self._result_for_revision(0) self.assertEqual(result.capture_start_ns, sweep.capture_start_ns) self.assertEqual(result.capture_end_ns, sweep.capture_end_ns) def test_fed_back_sweeps_are_processed_but_not_retained(self) -> None: start_ns = time.monotonic_ns() sweeps = [_sweep(index, start_ns + index * 1_000_000) for index in range(1, _SWEEPS + 1)] expected = [sweep.collection_id for sweep in sweeps] self._push(sweeps) self.assertEqual(self._published_ids(_SWEEPS), expected) # The GUI feeds them back: each is processed again under its own id. self._push(sweeps) self.assertEqual(self._published_ids(_SWEEPS), expected) # A settings change makes the processor replay its history: each sweep once, in order. self._live.bscan_gain = 2.0 self._live_writer.write(self._live) self.assertEqual(self._published_ids(_SWEEPS), expected) def test_live_geometry_recomputes_retained_gpr_without_restarting(self) -> None: self._live.processor_mode = "gpr" self._live.gpr_input_positions = [0] self._live.gpr_output_positions = [0] self._live.gpr_background_subtract_enabled = False self._live.gpr_speed_m_s = 0.75 self._live.tx_geometry = [{"output_pos": 0, "x_m": 0.0, "y_m": 0.0, "z_m": 0.0}] self._live.rx_geometry = [{"input_pos": 0, "x_m": 0.0, "y_m": 0.0, "z_m": 0.0}] self._live.settings_revision = (1 << 63) + 987654321 self._live_writer.write(self._live) self._push([_sweep(1, time.monotonic_ns())]) original = self._result_for_revision(self._live.settings_revision) self.assertEqual(original.speed_m_s, 0.75) original_image = next(payload for payload in original.collection_payloads if payload.processing_name == "gpr_accumulator") # A malformed live file retains the last accepted geometry and revision. accepted_revision = self._live.settings_revision self._live.settings_revision += 1 self._live.gpr_speed_m_s = 1.25 self._live.tx_geometry.append(dict(self._live.tx_geometry[0])) self._live_writer.write(self._live) self._push([_sweep(2, time.monotonic_ns())]) retained = self._result_for_revision(accepted_revision, collection_id=2) self.assertEqual(retained.speed_m_s, 0.75) retained_image = next(payload for payload in retained.collection_payloads if payload.processing_name == "gpr_accumulator") np.testing.assert_array_equal(retained_image.image_x_axis, original_image.image_x_axis) self._live.tx_geometry = self._live.tx_geometry[:1] self._live.tx_geometry[0]["x_m"] = 2.0 self._live.rx_geometry[0]["x_m"] = 2.0 self._live.gpr_speed_m_s = 2.5 self._live.settings_revision += 1 self._live_writer.write(self._live) updated = self._result_for_revision(self._live.settings_revision) self.assertEqual(updated.speed_m_s, 2.5) updated_image = next(payload for payload in updated.collection_payloads if payload.processing_name == "gpr_accumulator") self.assertEqual(updated.collection_id, retained.collection_id) self.assertEqual(updated.capture_start_ns, retained.capture_start_ns) np.testing.assert_allclose(updated_image.image_x_axis, original_image.image_x_axis + 2.0, atol=1e-6) self.assertIsNone(self._processor.poll()) class PendingInputHistoryTest(_ProcessorTestCase): idle_sleep_ms = 1000 def test_clear_history_discards_already_queued_input_and_accepts_future_sweeps(self) -> None: now = time.monotonic_ns() sweeps = [_sweep(cid, now + cid * 1_000_000) for cid in (1, 2, 3, 4)] self._push(sweeps[:1]) self.assertEqual(self._published_ids(1), [1]) # The seed has completed and the processor is in its configured idle wait. # Freeze the consumer while preparing a backlog and the Clear command. # No further data is produced until that backlog has been consumed. self._processor.send_signal(signal.SIGSTOP) os.waitpid(self._processor.pid, os.WUNTRACED) try: self._push(sweeps[1:3]) self._live.history_clear_before_ns = time.monotonic_ns() self._live.settings_revision = 1 self._live_writer.write(self._live) finally: self._processor.send_signal(signal.SIGCONT) deadline = time.monotonic() + _TIMEOUT_S while self._input._read_u64(32) != self._input._read_u64(24): self.assertLess(time.monotonic(), deadline, self._stderr_path.read_text()) time.sleep(.01) delayed = _sweep(2, time.monotonic_ns()) delayed.capture_start_ns, delayed.capture_end_ns = sweeps[1].capture_start_ns, sweeps[1].capture_end_ns self._push([delayed, _sweep(4, time.monotonic_ns())]) self.assertEqual(self._published_ids(1), [4]) self._live.settings_revision = 2 self._live_writer.write(self._live) self.assertEqual(self._published_ids(1), [4]) def test_multiple_deletions_survive_coalesced_live_updates_and_queued_replays(self) -> None: now = time.monotonic_ns() sweeps = [_sweep(cid, now + cid * 1_000_000) for cid in (1, 2, 3, 4)] self._push(sweeps[:3]) self.assertEqual(self._published_ids(3), [1, 2, 3]) self._processor.send_signal(signal.SIGSTOP) os.waitpid(self._processor.pid, os.WUNTRACED) try: for sweep in reversed(sweeps[1:3]): self._live.history_removed.append(dict(collection_id=sweep.collection_id, capture_start_ns=sweep.capture_start_ns)) self._live.settings_revision += 1 self._live_writer.write(self._live) # A normal settings write follows both clicks before the processor wakes. self._live.bscan_gain = 2. self._live.settings_revision += 1 self._live_writer.write(self._live) self._push(sweeps[1:]) finally: self._processor.send_signal(signal.SIGCONT) self.assertEqual(self._published_ids(2), [1, 4]) self._live.settings_revision += 1 self._live_writer.write(self._live) self.assertEqual(self._published_ids(2), [1, 4]) class OversizedResultsTest(_ProcessorTestCase): result_slot_size_bytes = 4096 def test_valid_mode_recovers_after_an_oversized_result(self) -> None: self._live.processor_mode = "gpr" self._live.gpr_input_positions = [0] self._live.gpr_output_positions = [0] self._live.gpr_background_subtract_enabled = False self._live.settings_revision = 17 self._live_writer.write(self._live) self._push([_sweep(1, time.monotonic_ns())]) deadline = time.monotonic() + _TIMEOUT_S while "requires" not in self._stderr_path.read_text(): self.assertIsNone(self._processor.poll(), self._stderr_path.read_text()) self.assertLess(time.monotonic(), deadline, self._stderr_path.read_text()) time.sleep(0.01) self.assertIn("increase rings.results.slot_size_bytes", self._stderr_path.read_text()) self._live.processor_mode = "pass_through" self._live.settings_revision = 18 self._live_writer.write(self._live) self.assertEqual(self._result_for_revision(18).collection_id, 1) self._push([_sweep(2, time.monotonic_ns())]) self.assertEqual(self._result_for_revision(18, collection_id=2).collection_id, 2) self.assertIsNone(self._processor.poll()) class Sar3dRecoveryTest(_ProcessorTestCase): def test_sar3d_failure_recovers_without_restarting(self) -> None: self._live.processor_mode = "sar3d" self._live.gpr_input_positions = [0] self._live.gpr_output_positions = [0] self._live.gpr_background_subtract_enabled = False self._live.gpr_min_depth_m = 2.0 self._live.gpr_max_depth_m = 5.0 self._live.gpr_speed_m_s = 0.0 self._live.gpr_start_freq_mhz = 1500.0 self._live.gpr_stop_freq_mhz = 6000.0 self._live.sar3d_voxel_m = 0.4 self._live.sar3d_frame_radius = 0 self._live.tx_geometry = [{"output_pos": 0, "x_m": -0.3, "y_m": 0.0, "z_m": 0.0}] self._live.rx_geometry = [{"input_pos": 0, "x_m": 0.2, "y_m": 0.4, "z_m": 0.0}] self._live.settings_revision = 31 self._live_writer.write(self._live) sweep = _sweep(1, time.monotonic_ns()) # df=25 MHz gives a 6 m unambiguous one-way range. The history tests' # 64-point sweep aliases before this test's 2..5 m depth gate. sweep.traces[0].frequency_hz = np.linspace(1e9, 6e9, 201, dtype=np.float32) sweep.traces[0].s21 = np.exp(-4j * np.pi * sweep.traces[0].frequency_hz * 4.0 / 3e8).astype(np.complex64) self._push([sweep]) def ready_images(result): payloads = {item.processing_name: item for item in result.collection_payloads} self.assertEqual(payloads["sar3d_status"].table[0, 0], 0) images = [payloads[name].image for name in ("sar3d_front", "sar3d_side", "sar3d_top", "sar3d_oblique")] for image in images: self.assertTrue(np.all(np.isfinite(image))) self.assertGreater(float(image.max()), 0.0) self.assertIn("sar3d_peak", payloads) return images first = self._result_for_revision(31) status = next(item for item in first.collection_payloads if item.processing_name == "sar3d_status") if status.table[0, 0] == 3: self.assertIn("CUDA", bytes(map(int, status.table[0, 1:])).decode("utf-8")) self.assertFalse(any(item.processing_name in {"sar3d_front", "sar3d_side", "sar3d_top", "sar3d_oblique"} for item in first.collection_payloads)) self._push([_sweep(2, sweep.monotonic_ns + 1_000_000)]) repeated = self._result_for_revision(31, collection_id=2) repeated_status = next(item for item in repeated.collection_payloads if item.processing_name == "sar3d_status") np.testing.assert_array_equal(status.table, repeated_status.table) self._live.processor_mode = "pass_through" self._live.settings_revision = 34 self._live_writer.write(self._live) recovered = self._result_for_revision(34, collection_id=2) self.assertEqual(len(recovered.blocks), 1) self.assertIsNone(self._processor.poll()) return original = ready_images(first) self._live.gpr_start_freq_mhz = 8000.0 self._live.gpr_stop_freq_mhz = 9000.0 self._live.settings_revision = 32 self._live_writer.write(self._live) failed = self._result_for_revision(32) status = next(item for item in failed.collection_payloads if item.processing_name == "sar3d_status") self.assertEqual(status.table[0, 0], 2) self.assertIn("frequency points", bytes(map(int, status.table[0, 1:])).decode("utf-8")) self.assertIsNone(self._processor.poll()) self._live.gpr_start_freq_mhz = 1500.0 self._live.gpr_stop_freq_mhz = 6000.0 self._live.settings_revision = 33 self._live_writer.write(self._live) recovered = self._result_for_revision(33) self.assertEqual(recovered.collection_id, sweep.collection_id) for before, after in zip(original, ready_images(recovered)): np.testing.assert_array_equal(before, after) self.assertIsNone(self._processor.poll()) if __name__ == "__main__": unittest.main()