431 lines
18 KiB
Python
431 lines
18 KiB
Python
"""IPC/SHM tests: encode/decode round-trips, corruption handling, ring semantics.
|
|
|
|
Pins the agreed contract:
|
|
* encode -> decode is lossless for raw/preprocessed traces and result payloads;
|
|
* a corrupt/truncated frame raises a single, catchable ValueError (never a bare
|
|
struct.error / UnicodeDecodeError);
|
|
* the ring is latest-wins: on overflow the oldest slot is overwritten and a slow
|
|
reader keeps the freshest frames;
|
|
* peek_latest returns the newest frame without consuming it;
|
|
* opening a missing/incompatible ring fails fast.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import struct
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from contextlib import suppress
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
import numpy as np
|
|
|
|
from python_app.models.dataset_model import (
|
|
ComboKey,
|
|
ResultBlock,
|
|
ResultCollection,
|
|
ResultPayload,
|
|
SweepCollection,
|
|
TraceData,
|
|
)
|
|
from python_app.orchestration.shm.decoder import (
|
|
PREPROC_MAGIC,
|
|
RAW_MAGIC,
|
|
RESULT_MAGIC,
|
|
decode_result_collection,
|
|
decode_trace_collection,
|
|
)
|
|
from python_app.orchestration.shm.ring_reader import ShmRingReader
|
|
from python_app.orchestration.shm.ring_writer import ShmRingWriter
|
|
from python_app.storage.npz.serialize import serialize_result_collection, serialize_trace_collection
|
|
|
|
|
|
def _trace(in_pos: int, out_pos: int, n: int, *, capture_ns: tuple[int, int] = (0, 0)) -> TraceData:
|
|
"""Build a trace with float32-exact data so round-trips compare exactly."""
|
|
freq = np.arange(n, dtype=np.float32) + 1.0
|
|
s11 = (np.arange(n, dtype=np.float32) + 0.5j * np.arange(n, dtype=np.float32)).astype(np.complex64)
|
|
s21 = (-np.arange(n, dtype=np.float32) + 2.0j * np.arange(n, dtype=np.float32)).astype(np.complex64)
|
|
return TraceData(
|
|
combo=ComboKey(input=in_pos, output=out_pos),
|
|
frequency_hz=freq,
|
|
s11=s11,
|
|
s21=s21,
|
|
capture_start_ns=capture_ns[0],
|
|
capture_end_ns=capture_ns[1],
|
|
)
|
|
|
|
|
|
class TraceCollectionRoundTripTest(unittest.TestCase):
|
|
def _assert_round_trips(self, magic: int) -> None:
|
|
collection = SweepCollection(
|
|
collection_id=7,
|
|
monotonic_ns=123,
|
|
traces=[_trace(0, 0, 4, capture_ns=(11, 13)), _trace(3, 1, 2, capture_ns=(15, 19))],
|
|
capture_start_ns=10,
|
|
capture_end_ns=20,
|
|
)
|
|
decoded = decode_trace_collection(serialize_trace_collection(collection, magic), magic)
|
|
self.assertEqual(decoded.collection_id, 7)
|
|
self.assertEqual(decoded.monotonic_ns, 123)
|
|
self.assertEqual((decoded.capture_start_ns, decoded.capture_end_ns), (10, 20))
|
|
self.assertEqual(len(decoded.traces), 2)
|
|
for original, got in zip(collection.traces, decoded.traces):
|
|
self.assertEqual((got.combo.input, got.combo.output), (original.combo.input, original.combo.output))
|
|
self.assertTrue(np.array_equal(got.frequency_hz, original.frequency_hz))
|
|
self.assertTrue(np.array_equal(got.s11, original.s11))
|
|
self.assertTrue(np.array_equal(got.s21, original.s21))
|
|
self.assertEqual(
|
|
(got.capture_start_ns, got.capture_end_ns),
|
|
(original.capture_start_ns, original.capture_end_ns),
|
|
)
|
|
|
|
def test_raw_round_trips(self) -> None:
|
|
self._assert_round_trips(RAW_MAGIC)
|
|
|
|
def test_preprocessed_round_trips(self) -> None:
|
|
self._assert_round_trips(PREPROC_MAGIC)
|
|
|
|
def test_empty_traces_round_trip(self) -> None:
|
|
collection = SweepCollection(collection_id=1, monotonic_ns=2, traces=[])
|
|
decoded = decode_trace_collection(serialize_trace_collection(collection, RAW_MAGIC), RAW_MAGIC)
|
|
self.assertEqual(decoded.traces, [])
|
|
|
|
def test_payload_without_per_trace_window_table_still_decodes(self) -> None:
|
|
# A producer built before per-trace timing stops after the collection
|
|
# window; its traces must still decode, with the timestamps left at zero.
|
|
collection = SweepCollection(
|
|
collection_id=4,
|
|
monotonic_ns=5,
|
|
traces=[_trace(1, 0, 3, capture_ns=(7, 9))],
|
|
capture_start_ns=6,
|
|
capture_end_ns=10,
|
|
)
|
|
full = serialize_trace_collection(collection, RAW_MAGIC)
|
|
legacy = full[: -(4 + 16 * len(collection.traces))]
|
|
|
|
decoded = decode_trace_collection(legacy, RAW_MAGIC)
|
|
self.assertEqual((decoded.capture_start_ns, decoded.capture_end_ns), (6, 10))
|
|
self.assertEqual(len(decoded.traces), 1)
|
|
self.assertEqual((decoded.traces[0].capture_start_ns, decoded.traces[0].capture_end_ns), (0, 0))
|
|
|
|
def test_per_trace_window_count_mismatch_is_rejected(self) -> None:
|
|
collection = SweepCollection(
|
|
collection_id=4, monotonic_ns=5, traces=[_trace(1, 0, 3, capture_ns=(7, 9))]
|
|
)
|
|
payload = serialize_trace_collection(collection, RAW_MAGIC)
|
|
# Overwrite the window-table count (u32 before the single 16-byte pair).
|
|
corrupt = payload[:-20] + struct.pack("<I", 2) + payload[-16:]
|
|
with self.assertRaises(ValueError):
|
|
decode_trace_collection(corrupt, RAW_MAGIC)
|
|
|
|
|
|
class ResultCollectionRoundTripTest(unittest.TestCase):
|
|
def test_empty_tables_and_images_retain_their_declared_dimensions(self) -> None:
|
|
for shape in ((0, 3), (2, 0), (0, 0)):
|
|
for kind in (3, 4):
|
|
with self.subTest(shape=shape, kind=kind):
|
|
payload = ResultPayload('empty', kind, table=np.empty(shape, dtype=np.float32),
|
|
image=np.empty(shape, dtype=np.float32),
|
|
image_x_axis=np.zeros(shape[1], dtype=np.float32),
|
|
image_y_axis=np.zeros(shape[0], dtype=np.float32))
|
|
if kind == 4 and shape == (2, 0):
|
|
with self.assertRaises(ValueError):
|
|
serialize_result_collection(ResultCollection(1, 2, collection_payloads=[payload]))
|
|
continue
|
|
result = decode_result_collection(serialize_result_collection(
|
|
ResultCollection(1, 2, collection_payloads=[payload])))
|
|
decoded = result.collection_payloads[0]
|
|
self.assertEqual((decoded.image if kind == 3 else decoded.table).shape, shape)
|
|
|
|
def test_multidimensional_trace_cannot_be_serialized_as_a_flat_trace(self) -> None:
|
|
payload = ResultPayload('bad', 1, frequency_hz=np.ones((1, 2)), trace=np.ones((1, 2)))
|
|
with self.assertRaises(ValueError):
|
|
serialize_result_collection(ResultCollection(1, 2, collection_payloads=[payload]))
|
|
|
|
def test_all_payload_kinds_round_trip(self) -> None:
|
|
image = np.arange(6, dtype=np.float32).reshape((2, 3))
|
|
table = np.array([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], dtype=np.float32)
|
|
collection = ResultCollection(
|
|
collection_id=9,
|
|
monotonic_ns=42,
|
|
processing_duration_ns=1000,
|
|
collection_payloads=[
|
|
ResultPayload(
|
|
processing_name="gpr_accumulator", kind=3,
|
|
image_x_axis=np.array([0.0, 1.0, 2.0], dtype=np.float32),
|
|
image_y_axis=np.array([0.0, 1.0], dtype=np.float32),
|
|
image=image,
|
|
),
|
|
ResultPayload(processing_name="gpr_points", kind=4, table=table),
|
|
],
|
|
blocks=[
|
|
ResultBlock(combo=ComboKey(input=1, output=0), payloads=[
|
|
ResultPayload(
|
|
processing_name="bscan", kind=1,
|
|
frequency_hz=np.array([1.0, 2.0], dtype=np.float32),
|
|
trace=np.array([1 + 1j, 2 - 2j], dtype=np.complex64),
|
|
),
|
|
ResultPayload(processing_name="snr", kind=2, scalar_value=2.5),
|
|
]),
|
|
],
|
|
)
|
|
decoded = decode_result_collection(serialize_result_collection(collection))
|
|
self.assertEqual((decoded.collection_id, decoded.monotonic_ns, decoded.processing_duration_ns), (9, 42, 1000))
|
|
acc, points = decoded.collection_payloads
|
|
self.assertEqual(acc.processing_name, "gpr_accumulator")
|
|
self.assertTrue(np.array_equal(acc.image, image))
|
|
self.assertTrue(np.array_equal(points.table, table))
|
|
block = decoded.blocks[0]
|
|
self.assertEqual((block.combo.input, block.combo.output), (1, 0))
|
|
self.assertTrue(np.array_equal(block.payloads[0].trace, np.array([1 + 1j, 2 - 2j], dtype=np.complex64)))
|
|
self.assertAlmostEqual(block.payloads[1].scalar_value, 2.5)
|
|
|
|
|
|
class CorruptFrameTest(unittest.TestCase):
|
|
"""Any corruption surfaces as ValueError (the single catchable contract)."""
|
|
|
|
def _valid_trace_bytes(self) -> bytes:
|
|
return serialize_trace_collection(
|
|
SweepCollection(collection_id=1, monotonic_ns=1, traces=[_trace(0, 0, 3)]), RAW_MAGIC
|
|
)
|
|
|
|
def test_bad_magic(self) -> None:
|
|
corrupt = b"\x00\x00\x00\x00" + self._valid_trace_bytes()[4:]
|
|
with self.assertRaises(ValueError):
|
|
decode_trace_collection(corrupt, RAW_MAGIC)
|
|
|
|
def test_truncated_buffer(self) -> None:
|
|
with self.assertRaises(ValueError):
|
|
decode_trace_collection(self._valid_trace_bytes()[:18], RAW_MAGIC)
|
|
|
|
def test_absurd_trace_count(self) -> None:
|
|
# Claims 1e6 traces but supplies none -> the first trace read runs off the end.
|
|
corrupt = struct.pack("<IQQI", RAW_MAGIC, 1, 1, 1_000_000)
|
|
with self.assertRaises(ValueError):
|
|
decode_trace_collection(corrupt, RAW_MAGIC)
|
|
|
|
def test_unsupported_payload_kind(self) -> None:
|
|
corrupt = struct.pack("<IQQQII", RESULT_MAGIC, 1, 1, 0, 1, 0) + struct.pack("<BH", 99, 0)
|
|
with self.assertRaises(ValueError):
|
|
decode_result_collection(corrupt)
|
|
|
|
def test_invalid_utf8_name(self) -> None:
|
|
corrupt = struct.pack("<IQQQII", RESULT_MAGIC, 1, 1, 0, 1, 0) + struct.pack("<BH", 2, 1) + b"\xff"
|
|
with self.assertRaises(ValueError):
|
|
decode_result_collection(corrupt)
|
|
|
|
|
|
class _RingTestCase(unittest.TestCase):
|
|
"""Base case that creates/cleans named /dev/shm rings."""
|
|
|
|
def _ring_name(self, suffix: str = "") -> str:
|
|
return f"/radar_test_{self._testMethodName}{suffix}"
|
|
|
|
def _writer(self, name: str, capacity: int, slot_size: int) -> ShmRingWriter:
|
|
with suppress(OSError):
|
|
(Path("/dev/shm") / name[1:]).unlink() # drop a leftover from a crashed run
|
|
writer = ShmRingWriter(name, capacity, slot_size)
|
|
self.addCleanup(self._cleanup, name, writer)
|
|
return writer
|
|
|
|
def _reader(self, name: str, **kw: object) -> ShmRingReader:
|
|
reader = ShmRingReader(name, **kw)
|
|
self.addCleanup(self._safe_close, reader)
|
|
return reader
|
|
|
|
@staticmethod
|
|
def _safe_close(obj: object) -> None:
|
|
with suppress(Exception):
|
|
obj.close() # type: ignore[attr-defined]
|
|
|
|
@staticmethod
|
|
def _cleanup(name: str, writer: ShmRingWriter) -> None:
|
|
with suppress(Exception):
|
|
writer.close()
|
|
with suppress(OSError):
|
|
(Path("/dev/shm") / name[1:]).unlink()
|
|
|
|
|
|
class RingRoundTripTest(_RingTestCase):
|
|
def test_fifo_round_trip(self) -> None:
|
|
name = self._ring_name()
|
|
writer = self._writer(name, capacity=8, slot_size=64)
|
|
reader = self._reader(name)
|
|
payloads = [f"frame{i}".encode() for i in range(5)]
|
|
for p in payloads:
|
|
self.assertTrue(writer.push(p))
|
|
self.assertEqual([reader.pop_payload() for _ in payloads], payloads)
|
|
self.assertIsNone(reader.pop_payload()) # empty afterwards
|
|
|
|
def test_oversized_payload_rejected(self) -> None:
|
|
writer = self._writer(self._ring_name(), capacity=4, slot_size=8)
|
|
self.assertFalse(writer.push(b"x" * 9)) # larger than the slot
|
|
|
|
|
|
class RingOverflowTest(_RingTestCase):
|
|
def test_reader_cannot_rewind_cursor_after_producer_laps_during_commit(self) -> None:
|
|
from python_app.orchestration.shm import atomics
|
|
|
|
name = self._ring_name()
|
|
writer = self._writer(name, capacity=4, slot_size=64)
|
|
reader = self._reader(name)
|
|
writer.push(b"f0")
|
|
advance = atomics.advance
|
|
|
|
def lap_before_reader_commit(buffer, offset, desired):
|
|
if buffer is reader._mmap and desired == 1:
|
|
for i in range(1, 6):
|
|
writer.push(f"f{i}".encode())
|
|
advance(buffer, offset, desired)
|
|
|
|
with mock.patch.object(atomics, "advance", side_effect=lap_before_reader_commit):
|
|
self.assertEqual(reader.pop_payload(), b"f0")
|
|
self.assertEqual(reader._read_u64(32), 2)
|
|
self.assertEqual([reader.pop_payload() for _ in range(4)], [b"f2", b"f3", b"f4", b"f5"])
|
|
|
|
def test_latest_wins_drops_oldest(self) -> None:
|
|
name = self._ring_name()
|
|
writer = self._writer(name, capacity=4, slot_size=64)
|
|
reader = self._reader(name)
|
|
for i in range(7): # 3 more than capacity
|
|
self.assertTrue(writer.push(f"f{i}".encode()))
|
|
# The 4 newest survive; the 3 oldest were overwritten.
|
|
survivors = []
|
|
while (item := reader.pop_payload()) is not None:
|
|
survivors.append(item)
|
|
self.assertEqual(survivors, [b"f3", b"f4", b"f5", b"f6"])
|
|
|
|
|
|
class PeekLatestTest(_RingTestCase):
|
|
def test_peek_rejects_a_slot_while_it_is_being_overwritten(self) -> None:
|
|
name = self._ring_name()
|
|
writer = self._writer(name, capacity=1, slot_size=8)
|
|
reader = self._reader(name)
|
|
writer.push(b"aaaaaaaa")
|
|
observations = []
|
|
write_size = writer._write_u32
|
|
|
|
def overwrite_in_progress(offset: int, value: int) -> None:
|
|
write_size(offset, value)
|
|
if offset == 64:
|
|
writer._mmap[80:84] = b"bbbb"
|
|
observations.append(reader.peek_latest_payload())
|
|
|
|
with mock.patch.object(writer, "_write_u32", side_effect=overwrite_in_progress):
|
|
writer.push(b"bbbbbbbb")
|
|
|
|
self.assertEqual(observations, [None])
|
|
self.assertEqual(reader.peek_latest_payload(), b"bbbbbbbb")
|
|
|
|
def test_peek_is_latest_and_non_consuming(self) -> None:
|
|
name = self._ring_name()
|
|
writer = self._writer(name, capacity=8, slot_size=64)
|
|
reader = self._reader(name)
|
|
self.assertIsNone(reader.peek_latest_payload()) # nothing published yet
|
|
for i in range(3):
|
|
writer.push(f"f{i}".encode())
|
|
self.assertEqual(reader.peek_latest_payload(), b"f2") # newest
|
|
self.assertEqual(reader.peek_latest_payload(), b"f2") # stable, not consumed
|
|
self.assertEqual(reader.pop_payload(), b"f0") # consumer cursor untouched
|
|
writer.push(b"f3")
|
|
self.assertEqual(reader.peek_latest_payload(), b"f3") # follows the newest
|
|
|
|
|
|
class RingOpenTest(_RingTestCase):
|
|
def test_invalid_geometry_and_names_fail_before_opening_files(self) -> None:
|
|
for capacity, slot in ((True, 8), (1.5, 8), (1, 9), (2 ** 32, 8)):
|
|
with self.subTest(capacity=capacity, slot=slot), self.assertRaises(ValueError):
|
|
ShmRingWriter(self._ring_name(), capacity, slot)
|
|
for name in ("/", "/../tmp/ring", "/nested/ring", "/bad\x00ring"):
|
|
with self.subTest(name=name):
|
|
with self.assertRaises(ValueError):
|
|
ShmRingWriter(name, 1, 8)
|
|
with self.assertRaises(ValueError):
|
|
ShmRingReader(name, open_timeout_s=0)
|
|
|
|
def test_failed_mapping_closes_opened_file(self) -> None:
|
|
from python_app.orchestration.shm import ring_writer
|
|
|
|
name = self._ring_name()
|
|
opened = []
|
|
fdopen = ring_writer.os.fdopen
|
|
|
|
def track_fd(fd, *args, **kwargs):
|
|
handle = fdopen(fd, *args, **kwargs)
|
|
opened.append(handle)
|
|
return handle
|
|
|
|
self.addCleanup(lambda: (Path('/dev/shm') / name[1:]).unlink(missing_ok=True))
|
|
with mock.patch.object(ring_writer.os, 'fdopen', side_effect=track_fd), \
|
|
mock.patch.object(ring_writer.mmap, 'mmap', side_effect=OSError('mapping failed')), \
|
|
self.assertRaises(OSError):
|
|
ShmRingWriter(name, 1, 8)
|
|
self.assertTrue(opened[0].closed)
|
|
|
|
def test_owner_replaces_wrong_size_without_truncating_existing_mappings(self) -> None:
|
|
name = self._ring_name()
|
|
writer = self._writer(name, capacity=8, slot_size=64)
|
|
reader = self._reader(name)
|
|
writer.push(b"old frame")
|
|
path = Path("/dev/shm") / name[1:]
|
|
old_inode = path.stat().st_ino
|
|
|
|
replacement = ShmRingWriter(name, capacity=1, slot_size_bytes=8)
|
|
self.addCleanup(self._safe_close, replacement)
|
|
|
|
self.assertNotEqual(path.stat().st_ino, old_inode)
|
|
self.assertEqual(reader.pop_payload(), b"old frame")
|
|
self.assertTrue(replacement.push(b"new"))
|
|
self.assertEqual(self._reader(name).pop_payload(), b"new")
|
|
|
|
def test_waits_for_created_but_unallocated_ring(self) -> None:
|
|
name = self._ring_name()
|
|
path = Path("/dev/shm") / name[1:]
|
|
path.touch()
|
|
self.addCleanup(lambda: path.unlink(missing_ok=True))
|
|
writers = []
|
|
|
|
def initialize() -> None:
|
|
time.sleep(0.02)
|
|
writers.append(ShmRingWriter(name, capacity=1, slot_size_bytes=8))
|
|
|
|
creator = threading.Thread(target=initialize)
|
|
creator.start()
|
|
|
|
def cleanup_creator() -> None:
|
|
creator.join()
|
|
for writer in writers:
|
|
writer.close()
|
|
|
|
self.addCleanup(cleanup_creator)
|
|
reader = self._reader(name)
|
|
creator.join()
|
|
|
|
self.assertEqual(reader.capacity, 1)
|
|
self.assertEqual(reader.slot_size_bytes, 8)
|
|
|
|
def test_header_geometry_outside_mapping_is_rejected(self) -> None:
|
|
name = self._ring_name()
|
|
writer = self._writer(name, capacity=1, slot_size=8)
|
|
writer._write_u32(12, 100)
|
|
with self.assertRaisesRegex(RuntimeError, "geometry mismatch"):
|
|
ShmRingReader(name)
|
|
|
|
def test_missing_ring_fails_fast(self) -> None:
|
|
with self.assertRaises(FileNotFoundError):
|
|
ShmRingReader("/radar_test_definitely_missing", open_timeout_s=0.1, open_poll_s=0.02)
|
|
|
|
def test_incompatible_header_rejected(self) -> None:
|
|
name = "/radar_test_bad_header"
|
|
path = Path("/dev/shm") / name[1:]
|
|
path.write_bytes(b"\x00" * 128) # right size, wrong magic
|
|
self.addCleanup(lambda: path.unlink(missing_ok=True))
|
|
with self.assertRaises(RuntimeError):
|
|
ShmRingReader(name)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|