411 lines
20 KiB
Python
411 lines
20 KiB
Python
"""The processor's history stays the acquisition, in order, when the GUI feeds sweeps back.
|
|
|
|
To recompute the whole B-scan the GUI pushes its retained sweeps into the processor's
|
|
input ring again (BscanReplay). The processor used to append them to its
|
|
own history as if they were new, so its next replay published every sweep two or three
|
|
times and out of order, and the B-scan came out with its columns rotated.
|
|
|
|
Runs the real `data_processor` on private rings; skipped when it has not been built.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
|
|
from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData
|
|
from python_app.orchestration.live_processing_config import ProcessingLiveConfig, ProcessingLiveConfigWriter
|
|
from python_app.orchestration.shm.ring_writer import ShmRingWriter
|
|
from python_app.orchestration.shm.ring_reader import ShmRingReader
|
|
from python_app.orchestration.shm.decoder import PREPROC_MAGIC
|
|
from python_app.storage.npz.serialize import serialize_trace_collection
|
|
|
|
_ROOT = Path(__file__).resolve().parents[2]
|
|
_PROCESSOR = _ROOT / "build/bin/data_processor"
|
|
_SWEEPS = 5
|
|
_TIMEOUT_S = 5.0
|
|
|
|
|
|
def _sweep(collection_id: int, monotonic_ns: int) -> SweepCollection:
|
|
points = 64
|
|
trace = TraceData(
|
|
combo=ComboKey(input=0, output=0),
|
|
frequency_hz=np.linspace(1e9, 6e9, points, dtype=np.float32),
|
|
s21=np.full(points, collection_id, dtype=np.complex64),
|
|
capture_start_ns=monotonic_ns,
|
|
capture_end_ns=monotonic_ns + 1000,
|
|
)
|
|
return SweepCollection(collection_id=collection_id, monotonic_ns=monotonic_ns, traces=[trace],
|
|
capture_start_ns=monotonic_ns, capture_end_ns=monotonic_ns + 1000)
|
|
|
|
|
|
@unittest.skipUnless(_PROCESSOR.exists(), "data_processor is not built (run make)")
|
|
class _ProcessorTestCase(unittest.TestCase):
|
|
result_slot_size_bytes: int | None = None
|
|
idle_sleep_ms: int | None = None
|
|
|
|
def setUp(self) -> None:
|
|
workdir = Path(tempfile.mkdtemp(prefix="processor_history_"))
|
|
config = json.loads((_ROOT / "run_config.json").read_text(encoding="utf-8"))
|
|
for name, ring in config["rings"].items():
|
|
ring["name"] = f"/test_processor_history_{os.getpid()}_{name}"
|
|
if self.result_slot_size_bytes is not None:
|
|
config["rings"]["results"]["slot_size_bytes"] = self.result_slot_size_bytes
|
|
if self.idle_sleep_ms is not None:
|
|
config["run"]["idle_sleep_ms"] = self.idle_sleep_ms
|
|
config["run"]["combos"] = [{"input": 0, "output": 0}]
|
|
config["run"]["processing_live_config_path"] = str(workdir / "processing_live.json")
|
|
config["run"]["locator_server"]["enabled"] = False
|
|
config_path = workdir / "run_config.json"
|
|
config_path.write_text(json.dumps(config), encoding="utf-8")
|
|
self._live = ProcessingLiveConfig(processor_mode="bscan")
|
|
self._live_writer = ProcessingLiveConfigWriter(workdir / "processing_live.json")
|
|
self._live_writer.write(self._live)
|
|
|
|
rings = config["rings"]
|
|
shm_files = [Path("/dev/shm" + ring["name"]) for ring in rings.values()]
|
|
self._stderr_path = workdir / "processor.log"
|
|
stderr_log = self._stderr_path.open("w")
|
|
self.addCleanup(stderr_log.close)
|
|
processor = subprocess.Popen(
|
|
[str(_PROCESSOR), "--config", str(config_path)],
|
|
cwd=_ROOT,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=stderr_log,
|
|
)
|
|
|
|
def cleanup() -> None:
|
|
processor.terminate()
|
|
processor.wait(_TIMEOUT_S)
|
|
for path in shm_files:
|
|
path.unlink(missing_ok=True)
|
|
for path in workdir.iterdir():
|
|
path.unlink()
|
|
workdir.rmdir()
|
|
|
|
self.addCleanup(cleanup)
|
|
self._processor = processor
|
|
# The processor creates both rings at start-up, the results ring last.
|
|
self._results = ShmRingReader(rings["results"]["name"], open_timeout_s=_TIMEOUT_S)
|
|
self.addCleanup(self._results.close)
|
|
ring = rings["preprocessed"]
|
|
self._input = ShmRingWriter(ring["name"], int(ring["capacity"]), int(ring["slot_size_bytes"]))
|
|
self.addCleanup(self._input.close)
|
|
|
|
def _push(self, sweeps: list[SweepCollection]) -> None:
|
|
for sweep in sweeps:
|
|
self.assertTrue(self._input.push(serialize_trace_collection(sweep, PREPROC_MAGIC)))
|
|
|
|
def _published_ids(self, count: int) -> list[int]:
|
|
"""Ids of the next results: `count` of them plus any that follow shortly after."""
|
|
ids: list[int] = []
|
|
deadline = time.monotonic() + _TIMEOUT_S
|
|
while time.monotonic() < deadline:
|
|
collection = self._results.pop_result_collection()
|
|
if collection is None:
|
|
time.sleep(0.01)
|
|
continue
|
|
ids.append(int(collection.collection_id))
|
|
if len(ids) == count:
|
|
deadline = min(deadline, time.monotonic() + 0.3)
|
|
return ids
|
|
|
|
def _result_for_revision(self, revision: int, collection_id: int | None = None):
|
|
deadline = time.monotonic() + _TIMEOUT_S
|
|
while time.monotonic() < deadline:
|
|
result = self._results.pop_result_collection()
|
|
if (result is not None and result.settings_revision == revision
|
|
and (collection_id is None or result.collection_id == collection_id)):
|
|
return result
|
|
time.sleep(0.01)
|
|
self.fail(f"No processor result for settings revision {revision}: {self._stderr_path.read_text()}")
|
|
|
|
|
|
class FedBackSweepsTest(_ProcessorTestCase):
|
|
def test_mean_history_is_not_limited_by_transport_ring_capacity(self) -> None:
|
|
self._live.processor_mode = "gpr"
|
|
self._live.gpr_input_positions = self._live.gpr_output_positions = [0]
|
|
self._live.gpr_background_subtract_enabled = True
|
|
self._live.gpr_background_mean_count = 60
|
|
self._live.gpr_min_depth_m, self._live.gpr_max_depth_m = .1, 1.5
|
|
self._live.gpr_bp_ground_depth_m = 0.
|
|
self._live_writer.write(self._live)
|
|
start = time.monotonic_ns()
|
|
for cid in range(1, 61):
|
|
sweep = _sweep(cid, start + cid*1_000_000)
|
|
sweep.traces[0].s21 *= (-1.)**np.arange(len(sweep.traces[0].s21))
|
|
self._push([sweep])
|
|
self._result_for_revision(0, cid)
|
|
current = _sweep(61, start + 61_000_000)
|
|
current.traces[0].s21[:] = 30.5 * (-1.)**np.arange(len(current.traces[0].s21))
|
|
self._push([current])
|
|
result = self._result_for_revision(0, 61)
|
|
image = next(payload.image for payload in result.collection_payloads if payload.processing_name == "gpr_accumulator")
|
|
self.assertFalse(np.any(image)) # Exact cancellation of the 60 preceding spectra (1…60).
|
|
|
|
def test_remove_targets_exact_sweep_with_reused_ids_and_preserves_missing_targets(self) -> None:
|
|
start_ns = time.monotonic_ns()
|
|
sweeps = [_sweep(cid, start_ns + index * 1_000_000)
|
|
for index, cid in enumerate((7, 8, 7, 9, 8))]
|
|
self._push(sweeps)
|
|
self.assertEqual(self._published_ids(5), [7, 8, 7, 9, 8])
|
|
targets = [
|
|
({"collection_id": 7, "capture_start_ns": sweeps[2].capture_start_ns}, [7, 8, 9, 8]),
|
|
({"collection_id": 8, "capture_start_ns": sweeps[1].capture_start_ns}, [7, 9, 8]),
|
|
({"collection_id": 7, "capture_start_ns": start_ns + 20_000_000}, [7, 9, 8]),
|
|
]
|
|
for sequence, (target, expected) in enumerate(targets, start=1):
|
|
with self.subTest(target=target):
|
|
self._live.settings_revision = sequence
|
|
self._live.history_removed.append(target)
|
|
self._live_writer.write(self._live)
|
|
self.assertEqual(self._published_ids(len(expected)), expected)
|
|
|
|
def test_removed_target_arriving_after_command_cannot_reenter_history(self) -> None:
|
|
now = time.monotonic_ns()
|
|
seed, target, following = [_sweep(cid, now + cid * 1_000_000) for cid in (1, 2, 3)]
|
|
self._push([seed])
|
|
self.assertEqual(self._published_ids(1), [1])
|
|
self._live.settings_revision = 1
|
|
self._live.history_removed = [dict(collection_id=target.collection_id, capture_start_ns=target.capture_start_ns)]
|
|
self._live_writer.write(self._live)
|
|
self.assertEqual(self._published_ids(1), [1]) # Confirms the command was applied.
|
|
self._push([target, following])
|
|
self.assertEqual(self._published_ids(1), [3])
|
|
self._live.settings_revision = 2
|
|
self._live_writer.write(self._live)
|
|
self.assertEqual(self._published_ids(2), [1, 3])
|
|
|
|
def test_result_preserves_capture_window_for_missing_tap_counterparts(self) -> None:
|
|
sweep = _sweep(1, time.monotonic_ns())
|
|
sweep.capture_start_ns = (1 << 63) + 987654321
|
|
sweep.capture_end_ns = sweep.capture_start_ns + 123456789
|
|
self._push([sweep])
|
|
result = self._result_for_revision(0)
|
|
self.assertEqual(result.capture_start_ns, sweep.capture_start_ns)
|
|
self.assertEqual(result.capture_end_ns, sweep.capture_end_ns)
|
|
|
|
def test_fed_back_sweeps_are_processed_but_not_retained(self) -> None:
|
|
start_ns = time.monotonic_ns()
|
|
sweeps = [_sweep(index, start_ns + index * 1_000_000) for index in range(1, _SWEEPS + 1)]
|
|
expected = [sweep.collection_id for sweep in sweeps]
|
|
self._push(sweeps)
|
|
self.assertEqual(self._published_ids(_SWEEPS), expected)
|
|
|
|
# The GUI feeds them back: each is processed again under its own id.
|
|
self._push(sweeps)
|
|
self.assertEqual(self._published_ids(_SWEEPS), expected)
|
|
|
|
# A settings change makes the processor replay its history: each sweep once, in order.
|
|
self._live.bscan_gain = 2.0
|
|
self._live_writer.write(self._live)
|
|
self.assertEqual(self._published_ids(_SWEEPS), expected)
|
|
|
|
def test_live_geometry_recomputes_retained_gpr_without_restarting(self) -> None:
|
|
self._live.processor_mode = "gpr"
|
|
self._live.gpr_input_positions = [0]
|
|
self._live.gpr_output_positions = [0]
|
|
self._live.gpr_background_subtract_enabled = False
|
|
self._live.gpr_speed_m_s = 0.75
|
|
self._live.tx_geometry = [{"output_pos": 0, "x_m": 0.0, "y_m": 0.0, "z_m": 0.0}]
|
|
self._live.rx_geometry = [{"input_pos": 0, "x_m": 0.0, "y_m": 0.0, "z_m": 0.0}]
|
|
self._live.settings_revision = (1 << 63) + 987654321
|
|
self._live_writer.write(self._live)
|
|
self._push([_sweep(1, time.monotonic_ns())])
|
|
original = self._result_for_revision(self._live.settings_revision)
|
|
self.assertEqual(original.speed_m_s, 0.75)
|
|
original_image = next(payload for payload in original.collection_payloads if payload.processing_name == "gpr_accumulator")
|
|
# A malformed live file retains the last accepted geometry and revision.
|
|
accepted_revision = self._live.settings_revision
|
|
self._live.settings_revision += 1
|
|
self._live.gpr_speed_m_s = 1.25
|
|
self._live.tx_geometry.append(dict(self._live.tx_geometry[0]))
|
|
self._live_writer.write(self._live)
|
|
self._push([_sweep(2, time.monotonic_ns())])
|
|
retained = self._result_for_revision(accepted_revision, collection_id=2)
|
|
self.assertEqual(retained.speed_m_s, 0.75)
|
|
retained_image = next(payload for payload in retained.collection_payloads if payload.processing_name == "gpr_accumulator")
|
|
np.testing.assert_array_equal(retained_image.image_x_axis, original_image.image_x_axis)
|
|
self._live.tx_geometry = self._live.tx_geometry[:1]
|
|
self._live.tx_geometry[0]["x_m"] = 2.0
|
|
self._live.rx_geometry[0]["x_m"] = 2.0
|
|
self._live.gpr_speed_m_s = 2.5
|
|
self._live.settings_revision += 1
|
|
self._live_writer.write(self._live)
|
|
updated = self._result_for_revision(self._live.settings_revision)
|
|
self.assertEqual(updated.speed_m_s, 2.5)
|
|
updated_image = next(payload for payload in updated.collection_payloads if payload.processing_name == "gpr_accumulator")
|
|
self.assertEqual(updated.collection_id, retained.collection_id)
|
|
self.assertEqual(updated.capture_start_ns, retained.capture_start_ns)
|
|
np.testing.assert_allclose(updated_image.image_x_axis, original_image.image_x_axis + 2.0, atol=1e-6)
|
|
self.assertIsNone(self._processor.poll())
|
|
|
|
|
|
class PendingInputHistoryTest(_ProcessorTestCase):
|
|
idle_sleep_ms = 1000
|
|
|
|
def test_clear_history_discards_already_queued_input_and_accepts_future_sweeps(self) -> None:
|
|
now = time.monotonic_ns()
|
|
sweeps = [_sweep(cid, now + cid * 1_000_000) for cid in (1, 2, 3, 4)]
|
|
self._push(sweeps[:1])
|
|
self.assertEqual(self._published_ids(1), [1])
|
|
# The seed has completed and the processor is in its configured idle wait.
|
|
# Freeze the consumer while preparing a backlog and the Clear command.
|
|
# No further data is produced until that backlog has been consumed.
|
|
self._processor.send_signal(signal.SIGSTOP)
|
|
os.waitpid(self._processor.pid, os.WUNTRACED)
|
|
try:
|
|
self._push(sweeps[1:3])
|
|
self._live.history_clear_before_ns = time.monotonic_ns()
|
|
self._live.settings_revision = 1
|
|
self._live_writer.write(self._live)
|
|
finally:
|
|
self._processor.send_signal(signal.SIGCONT)
|
|
deadline = time.monotonic() + _TIMEOUT_S
|
|
while self._input._read_u64(32) != self._input._read_u64(24):
|
|
self.assertLess(time.monotonic(), deadline, self._stderr_path.read_text())
|
|
time.sleep(.01)
|
|
delayed = _sweep(2, time.monotonic_ns())
|
|
delayed.capture_start_ns, delayed.capture_end_ns = sweeps[1].capture_start_ns, sweeps[1].capture_end_ns
|
|
self._push([delayed, _sweep(4, time.monotonic_ns())])
|
|
self.assertEqual(self._published_ids(1), [4])
|
|
self._live.settings_revision = 2
|
|
self._live_writer.write(self._live)
|
|
self.assertEqual(self._published_ids(1), [4])
|
|
|
|
def test_multiple_deletions_survive_coalesced_live_updates_and_queued_replays(self) -> None:
|
|
now = time.monotonic_ns()
|
|
sweeps = [_sweep(cid, now + cid * 1_000_000) for cid in (1, 2, 3, 4)]
|
|
self._push(sweeps[:3])
|
|
self.assertEqual(self._published_ids(3), [1, 2, 3])
|
|
self._processor.send_signal(signal.SIGSTOP)
|
|
os.waitpid(self._processor.pid, os.WUNTRACED)
|
|
try:
|
|
for sweep in reversed(sweeps[1:3]):
|
|
self._live.history_removed.append(dict(collection_id=sweep.collection_id,
|
|
capture_start_ns=sweep.capture_start_ns))
|
|
self._live.settings_revision += 1
|
|
self._live_writer.write(self._live)
|
|
# A normal settings write follows both clicks before the processor wakes.
|
|
self._live.bscan_gain = 2.
|
|
self._live.settings_revision += 1
|
|
self._live_writer.write(self._live)
|
|
self._push(sweeps[1:])
|
|
finally:
|
|
self._processor.send_signal(signal.SIGCONT)
|
|
self.assertEqual(self._published_ids(2), [1, 4])
|
|
self._live.settings_revision += 1
|
|
self._live_writer.write(self._live)
|
|
self.assertEqual(self._published_ids(2), [1, 4])
|
|
|
|
|
|
class OversizedResultsTest(_ProcessorTestCase):
|
|
result_slot_size_bytes = 4096
|
|
|
|
def test_valid_mode_recovers_after_an_oversized_result(self) -> None:
|
|
self._live.processor_mode = "gpr"
|
|
self._live.gpr_input_positions = [0]
|
|
self._live.gpr_output_positions = [0]
|
|
self._live.gpr_background_subtract_enabled = False
|
|
self._live.settings_revision = 17
|
|
self._live_writer.write(self._live)
|
|
self._push([_sweep(1, time.monotonic_ns())])
|
|
deadline = time.monotonic() + _TIMEOUT_S
|
|
while "requires" not in self._stderr_path.read_text():
|
|
self.assertIsNone(self._processor.poll(), self._stderr_path.read_text())
|
|
self.assertLess(time.monotonic(), deadline, self._stderr_path.read_text())
|
|
time.sleep(0.01)
|
|
self.assertIn("increase rings.results.slot_size_bytes", self._stderr_path.read_text())
|
|
self._live.processor_mode = "pass_through"
|
|
self._live.settings_revision = 18
|
|
self._live_writer.write(self._live)
|
|
self.assertEqual(self._result_for_revision(18).collection_id, 1)
|
|
self._push([_sweep(2, time.monotonic_ns())])
|
|
self.assertEqual(self._result_for_revision(18, collection_id=2).collection_id, 2)
|
|
self.assertIsNone(self._processor.poll())
|
|
|
|
|
|
class Sar3dRecoveryTest(_ProcessorTestCase):
|
|
def test_sar3d_failure_recovers_without_restarting(self) -> None:
|
|
self._live.processor_mode = "sar3d"
|
|
self._live.gpr_input_positions = [0]
|
|
self._live.gpr_output_positions = [0]
|
|
self._live.gpr_background_subtract_enabled = False
|
|
self._live.gpr_min_depth_m = 2.0
|
|
self._live.gpr_max_depth_m = 5.0
|
|
self._live.gpr_speed_m_s = 0.0
|
|
self._live.gpr_start_freq_mhz = 1500.0
|
|
self._live.gpr_stop_freq_mhz = 6000.0
|
|
self._live.sar3d_voxel_m = 0.4
|
|
self._live.sar3d_frame_radius = 0
|
|
self._live.tx_geometry = [{"output_pos": 0, "x_m": -0.3, "y_m": 0.0, "z_m": 0.0}]
|
|
self._live.rx_geometry = [{"input_pos": 0, "x_m": 0.2, "y_m": 0.4, "z_m": 0.0}]
|
|
self._live.settings_revision = 31
|
|
self._live_writer.write(self._live)
|
|
sweep = _sweep(1, time.monotonic_ns())
|
|
# df=25 MHz gives a 6 m unambiguous one-way range. The history tests'
|
|
# 64-point sweep aliases before this test's 2..5 m depth gate.
|
|
sweep.traces[0].frequency_hz = np.linspace(1e9, 6e9, 201, dtype=np.float32)
|
|
sweep.traces[0].s21 = np.exp(-4j * np.pi * sweep.traces[0].frequency_hz * 4.0 / 3e8).astype(np.complex64)
|
|
self._push([sweep])
|
|
|
|
def ready_images(result):
|
|
payloads = {item.processing_name: item for item in result.collection_payloads}
|
|
self.assertEqual(payloads["sar3d_status"].table[0, 0], 0)
|
|
images = [payloads[name].image for name in ("sar3d_front", "sar3d_side", "sar3d_top", "sar3d_oblique")]
|
|
for image in images:
|
|
self.assertTrue(np.all(np.isfinite(image)))
|
|
self.assertGreater(float(image.max()), 0.0)
|
|
self.assertIn("sar3d_peak", payloads)
|
|
return images
|
|
|
|
first = self._result_for_revision(31)
|
|
status = next(item for item in first.collection_payloads if item.processing_name == "sar3d_status")
|
|
if status.table[0, 0] == 3:
|
|
self.assertIn("CUDA", bytes(map(int, status.table[0, 1:])).decode("utf-8"))
|
|
self.assertFalse(any(item.processing_name in {"sar3d_front", "sar3d_side", "sar3d_top", "sar3d_oblique"}
|
|
for item in first.collection_payloads))
|
|
self._push([_sweep(2, sweep.monotonic_ns + 1_000_000)])
|
|
repeated = self._result_for_revision(31, collection_id=2)
|
|
repeated_status = next(item for item in repeated.collection_payloads if item.processing_name == "sar3d_status")
|
|
np.testing.assert_array_equal(status.table, repeated_status.table)
|
|
self._live.processor_mode = "pass_through"
|
|
self._live.settings_revision = 34
|
|
self._live_writer.write(self._live)
|
|
recovered = self._result_for_revision(34, collection_id=2)
|
|
self.assertEqual(len(recovered.blocks), 1)
|
|
self.assertIsNone(self._processor.poll())
|
|
return
|
|
original = ready_images(first)
|
|
self._live.gpr_start_freq_mhz = 8000.0
|
|
self._live.gpr_stop_freq_mhz = 9000.0
|
|
self._live.settings_revision = 32
|
|
self._live_writer.write(self._live)
|
|
failed = self._result_for_revision(32)
|
|
status = next(item for item in failed.collection_payloads if item.processing_name == "sar3d_status")
|
|
self.assertEqual(status.table[0, 0], 2)
|
|
self.assertIn("frequency points", bytes(map(int, status.table[0, 1:])).decode("utf-8"))
|
|
self.assertIsNone(self._processor.poll())
|
|
|
|
self._live.gpr_start_freq_mhz = 1500.0
|
|
self._live.gpr_stop_freq_mhz = 6000.0
|
|
self._live.settings_revision = 33
|
|
self._live_writer.write(self._live)
|
|
recovered = self._result_for_revision(33)
|
|
self.assertEqual(recovered.collection_id, sweep.collection_id)
|
|
for before, after in zip(original, ready_images(recovered)):
|
|
np.testing.assert_array_equal(before, after)
|
|
self.assertIsNone(self._processor.poll())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|