keep sweeps in acquisition order through history replays

The processor appended sweeps the GUI fed back to its own history, so its next replay
published them twice and out of order, and the B-scan came out with rotated columns.
Fed-back sweeps now stay out of that history, results are kept in timestamp order,
picking the displayed combo only redraws and only new sweeps are recorded to disk.

python_app/tests/test_bscan_history_window.py

@@ -23,6 +23,8 @@ os.environ.setdefault("QT_QPA_PLATFORM", "offscreen")
 from PyQt6.QtWidgets import QApplication  # noqa: E402
 
 from python_app.gui.app_window import AppWindow  # noqa: E402
+from collections import deque  # noqa: E402
+
 from python_app.gui.runtime.history import record_result_history  # noqa: E402
 from python_app.models.dataset_model import (  # noqa: E402
     ComboKey,
@@ -167,5 +169,61 @@ class BscanDisplayWindowTest(unittest.TestCase):
         self.assertEqual(len(self.w._bscan_history_by_combo[(0, 0)]), 50)
 
 
+class ResultHistoryOrderTest(unittest.TestCase):
+    """The result history stays in acquisition order whatever order results arrive in.
+
+    The processor reprocessing its retained sweeps also delivers sweeps whose first
+    result the GUI missed (the results ring overwrites unread slots). Appended, such a
+    sweep showed up as the newest B-scan column, and was recorded to disk once more.
+    """
+
+    @staticmethod
+    def _ids(history: deque[ResultCollection]) -> list[int]:
+        return [int(collection.collection_id) for collection in history]
+
+    def test_a_new_sweep_is_appended(self) -> None:
+        history = deque(_bscan_result(cid) for cid in (1, 2))
+        self.assertTrue(record_result_history(history, _bscan_result(3)))
+        self.assertEqual(self._ids(history), [1, 2, 3])
+
+    def test_a_late_result_takes_its_place_in_acquisition_order(self) -> None:
+        history = deque(_bscan_result(cid) for cid in (1, 2, 4, 5))
+        self.assertFalse(record_result_history(history, _bscan_result(3)))
+        self.assertEqual(self._ids(history), [1, 2, 3, 4, 5])
+
+    def test_a_reprocessed_result_replaces_its_entry(self) -> None:
+        history = deque(_bscan_result(cid) for cid in (1, 2, 3))
+        reprocessed = _bscan_result(2)
+        self.assertFalse(record_result_history(history, reprocessed))
+        self.assertEqual(self._ids(history), [1, 2, 3])
+        self.assertIs(history[1], reprocessed)
+
+    def test_a_full_history_drops_its_oldest_entry(self) -> None:
+        history = deque((_bscan_result(cid) for cid in (1, 2, 4, 5)), maxlen=4)
+        record_result_history(history, _bscan_result(3))
+        self.assertEqual(self._ids(history), [2, 3, 4, 5])
+        record_result_history(history, _bscan_result(1))   # older than all it keeps
+        self.assertEqual(self._ids(history), [2, 3, 4, 5])
+        record_result_history(history, _bscan_result(6))
+        self.assertEqual(self._ids(history), [3, 4, 5, 6])
+
+    def test_only_new_sweeps_are_recorded_to_disk(self) -> None:
+        window = _window
+        arrivals = [_bscan_result(cid) for cid in (2, 3, 2, 1)]   # 2 reprocessed, 1 late
+        reader = type("Reader", (), {"pop_result_collection": lambda _self: arrivals.pop(0) if arrivals else None})()
+        recorded: list[int] = []
+        window._result_reader = reader
+        self.addCleanup(setattr, window, "_result_reader", None)
+        window._record_collection = lambda stage, collection: recorded.append(int(collection.collection_id))
+        self.addCleanup(delattr, window, "_record_collection")
+        window._result_history.clear()
+        self.addCleanup(window._result_history.clear)
+
+        window._read_all_results()
+
+        self.assertEqual(recorded, [2, 3])
+        self.assertEqual(self._ids(window._result_history), [1, 2, 3])
+
+
 if __name__ == "__main__":
     unittest.main()

python_app/tests/test_bscan_replay.py

@@ -13,6 +13,8 @@ The guard tests pin the other half: the GUI must never write into that ring whil
 from __future__ import annotations
 
 from contextlib import suppress
+from dataclasses import replace
+import json
 import os
 from pathlib import Path
 import unittest
@@ -24,6 +26,7 @@ from PyQt6.QtCore import QSignalBlocker  # noqa: E402
 from PyQt6.QtWidgets import QApplication  # noqa: E402
 
 from python_app.gui.app_window import AppWindow  # noqa: E402
+from python_app.gui.runtime.history import collection_key  # noqa: E402
 from python_app.models.dataset_model import (  # noqa: E402
     ComboKey,
     ResultBlock,
@@ -161,6 +164,35 @@ class PreprocessedWireFormatTest(unittest.TestCase):
         self.assertEqual(survived[-1], 200 + capacity + 3)
 
 
+def _result_of(sweep: SweepCollection) -> ResultCollection:
+    """The processor's result of a sweep: it keeps the sweep's id and timestamp."""
+    return replace(_bscan_result(sweep.collection_id), monotonic_ns=sweep.monotonic_ns)
+
+
+class _FakeProcessor:
+    """Both ends of the processor during a feed-back.
+
+    Each pushed sweep comes back as a result, after whatever the results ring already
+    held when the feed-back started.
+    """
+
+    slot_size_bytes = 1 << 20
+
+    def __init__(self, sweeps: list[SweepCollection], already_published: list[ResultCollection]) -> None:
+        self._unprocessed = list(sweeps)
+        self._published = list(already_published)
+
+    def push(self, _payload: bytes) -> bool:
+        self._published.append(_result_of(self._unprocessed.pop(0)))
+        return True
+
+    def pop_result_collection(self) -> ResultCollection | None:
+        return self._published.pop(0) if self._published else None
+
+    def close(self) -> None:
+        return None
+
+
 class _IdleResultReader:
     """Stands in for a connected results reader that simply has nothing to hand out."""
 
@@ -245,6 +277,58 @@ class ReplayGuardTest(unittest.TestCase):
 
         self.assertEqual(len(self.w._bscan_history_by_combo[(0, 0)]), window)
 
+    def test_feed_back_renders_its_sweeps_in_acquisition_order(self) -> None:
+        """Regression: after switching the displayed combo the columns came out rotated.
+
+        The processor was still replaying its own sweeps when the GUI fed its history
+        back, and the GUI took the first results to arrive for its own, in arrival order.
+        """
+        sweeps = [_sweep(cid) for cid in range(1, 11)]
+        self.w._pre_history.clear()
+        self.w._pre_history.extend(sweeps)
+        self.addCleanup(self.w._pre_history.clear)
+        self.w._result_history.clear()
+        self.addCleanup(self.w._result_history.clear)
+        with QSignalBlocker(self.w._bscan_history_window):
+            self.w._bscan_history_window.setValue(len(sweeps))
+        # Already in the results ring: the tail of the processor's own replay and a
+        # result of a sweep that is not being fed back.
+        processor = _FakeProcessor(sweeps, [_result_of(sweep) for sweep in sweeps[6:]] + [_bscan_result(999)])
+        self.w._replay_ring_writer = processor
+        self.addCleanup(setattr, self.w, "_replay_ring_writer", None)
+        self.w._result_reader = processor
+        self.addCleanup(self.w._discard_bscan_replay_results)
+
+        self.assertTrue(self.w._reprocess_history_through_processor())
+
+        self.assertEqual(
+            [collection_key(collection) for collection in self.w._bscan_replay_results],
+            [collection_key(sweep) for sweep in sweeps],
+        )
+        self.assertIn(999, [collection.collection_id for collection in self.w._result_history])
+
+    def test_changing_the_displayed_combo_recomputes_nothing(self) -> None:
+        # Every combo's B-scan is cached already; recomputing raced the processor (above).
+        writes: list[dict] = []
+        self.w._write_live_processing_config = lambda **kwargs: writes.append(kwargs)
+        self.addCleanup(delattr, self.w, "_write_live_processing_config")
+
+        self.w._single_combo_output.editingFinished.emit()
+
+        self.assertEqual(writes, [])
+        self.assertFalse(self.w._bscan_reprocess_timer.isActive())
+
+    def test_processor_reprocesses_only_while_the_gui_cannot(self) -> None:
+        # Its own replay would duplicate the feed-back and race it for the results ring.
+        def written_flag() -> bool:
+            self.w._write_live_processing_config()
+            config = json.loads(self.w._live_config_writer.path.read_text(encoding="utf-8"))
+            return bool(config["reprocess_current_result"])
+
+        self.assertFalse(written_flag())
+        self.w._supervisor.is_running = lambda: True
+        self.assertTrue(written_flag())
+
     def test_discarding_replay_falls_back_to_runtime_history(self) -> None:
         self.w._bscan_replay_results = [_bscan_result(1)]
         self.w._discard_bscan_replay_results()

python_app/tests/test_processor_history.py

@@ -0,0 +1,124 @@
+"""The processor's history stays the acquisition, in order, when the GUI feeds sweeps back.
+
+To recompute the whole B-scan the GUI pushes its retained sweeps into the processor's
+input ring again (AppWindowBscanReplayMixin). The processor used to append them to its
+own history as if they were new, so its next replay published every sweep two or three
+times and out of order, and the B-scan came out with its columns rotated.
+
+Runs the real `data_processor` on private rings; skipped when it has not been built.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import tempfile
+import time
+import unittest
+from pathlib import Path
+
+import numpy as np
+
+from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData
+from python_app.orchestration.live_processing_config import ProcessingLiveConfig, ProcessingLiveConfigWriter
+from python_app.orchestration.shm import ShmRingWriter
+from python_app.orchestration.shm.ring_reader import ShmRingReader
+from python_app.storage.npz.serialize import PREPROC_MAGIC, serialize_trace_collection
+
+_ROOT = Path(__file__).resolve().parents[2]
+_PROCESSOR = _ROOT / "build/bin/data_processor"
+_SWEEPS = 5
+_TIMEOUT_S = 5.0
+
+
+def _sweep(collection_id: int, monotonic_ns: int) -> SweepCollection:
+    points = 64
+    trace = TraceData(
+        combo=ComboKey(input=0, output=0),
+        frequency_hz=np.linspace(1e9, 6e9, points, dtype=np.float32),
+        s11=np.zeros(points, dtype=np.complex64),
+        s21=np.full(points, collection_id, dtype=np.complex64),
+    )
+    return SweepCollection(collection_id=collection_id, monotonic_ns=monotonic_ns, traces=[trace])
+
+
+@unittest.skipUnless(_PROCESSOR.exists(), "data_processor is not built (run make)")
+class FedBackSweepsTest(unittest.TestCase):
+    def setUp(self) -> None:
+        workdir = Path(tempfile.mkdtemp(prefix="processor_history_"))
+        config = json.loads((_ROOT / "run_config.json").read_text(encoding="utf-8"))
+        for name, ring in config["rings"].items():
+            ring["name"] = f"/test_processor_history_{os.getpid()}_{name}"
+        config["run"]["combos"] = [{"input": 0, "output": 0}]
+        config["run"]["processing_live_config_path"] = str(workdir / "processing_live.json")
+        config["run"]["locator_server"]["enabled"] = False
+        config_path = workdir / "run_config.json"
+        config_path.write_text(json.dumps(config), encoding="utf-8")
+        self._live = ProcessingLiveConfig(processor_mode="bscan")
+        self._live_writer = ProcessingLiveConfigWriter(workdir / "processing_live.json")
+        self._live_writer.write(self._live)
+
+        rings = config["rings"]
+        shm_files = [Path("/dev/shm" + ring["name"]) for ring in rings.values()]
+        processor = subprocess.Popen(
+            [str(_PROCESSOR), "--config", str(config_path)],
+            cwd=_ROOT,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+
+        def cleanup() -> None:
+            processor.terminate()
+            processor.wait(_TIMEOUT_S)
+            for path in shm_files:
+                path.unlink(missing_ok=True)
+            for path in workdir.iterdir():
+                path.unlink()
+            workdir.rmdir()
+
+        self.addCleanup(cleanup)
+        # The processor creates both rings at start-up, the results ring last.
+        self._results = ShmRingReader(rings["results"]["name"], open_timeout_s=_TIMEOUT_S)
+        self.addCleanup(self._results.close)
+        ring = rings["preprocessed"]
+        self._input = ShmRingWriter(ring["name"], int(ring["capacity"]), int(ring["slot_size_bytes"]))
+        self.addCleanup(self._input.close)
+
+    def _push(self, sweeps: list[SweepCollection]) -> None:
+        for sweep in sweeps:
+            self.assertTrue(self._input.push(serialize_trace_collection(sweep, PREPROC_MAGIC)))
+
+    def _published_ids(self, count: int) -> list[int]:
+        """Ids of the next results: `count` of them plus any that follow shortly after."""
+        ids: list[int] = []
+        deadline = time.monotonic() + _TIMEOUT_S
+        while time.monotonic() < deadline:
+            collection = self._results.pop_result_collection()
+            if collection is None:
+                time.sleep(0.01)
+                continue
+            ids.append(int(collection.collection_id))
+            if len(ids) == count:
+                deadline = min(deadline, time.monotonic() + 0.3)
+        return ids
+
+    def test_fed_back_sweeps_are_processed_but_not_retained(self) -> None:
+        start_ns = time.monotonic_ns()
+        sweeps = [_sweep(index, start_ns + index * 1_000_000) for index in range(1, _SWEEPS + 1)]
+        expected = [sweep.collection_id for sweep in sweeps]
+        self._push(sweeps)
+        self.assertEqual(self._published_ids(_SWEEPS), expected)
+
+        # The GUI feeds them back: each is processed again under its own id.
+        self._push(sweeps)
+        self.assertEqual(self._published_ids(_SWEEPS), expected)
+
+        # A settings change makes the processor replay its history: each sweep once, in order.
+        self._live.bscan_gain = 2.0
+        self._live_writer.write(self._live)
+        self.assertEqual(self._published_ids(_SWEEPS), expected)
+
+
+if __name__ == "__main__":
+    unittest.main()