awe/radar_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rework radar system

Browse Source
This commit is contained in:
Ayzen
2026-10-06 12:32:26 +03:00
parent 552cdc688a
commit 5bb86b7165
299 changed files with 13537 additions and 27763 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python_app/tests/test_shm_ipc.py
+149 -119
View File
@@ -6,7 +6,6 @@ Pins the agreed contract:
struct.error / UnicodeDecodeError);
* the ring is latest-wins: on overflow the oldest slot is overwritten and a slow
reader keeps the freshest frames;
* peek_latest returns the newest frame without consuming it;
* opening a missing/incompatible ring fails fast.
"""
@@ -18,15 +17,13 @@ import time
import unittest
from contextlib import suppress
from pathlib import Path
from typing import cast
from unittest import mock
import numpy as np
from python_app.models.dataset_model import (
ComboKey,
ResultBlock,
ResultCollection,
ResultPayload,
SweepCollection,
TraceData,
)
@@ -39,18 +36,16 @@ from python_app.orchestration.shm.decoder import (
)
from python_app.orchestration.shm.ring_reader import ShmRingReader
from python_app.orchestration.shm.ring_writer import ShmRingWriter
from python_app.storage.npz.serialize import serialize_result_collection, serialize_trace_collection
from python_app.storage.npz.serialize import serialize_trace_collection
def _trace(in_pos: int, out_pos: int, n: int, *, capture_ns: tuple[int, int] = (0, 0)) -> TraceData:
"""Build a trace with float32-exact data so round-trips compare exactly."""
freq = np.arange(n, dtype=np.float32) + 1.0
s11 = (np.arange(n, dtype=np.float32) + 0.5j * np.arange(n, dtype=np.float32)).astype(np.complex64)
s21 = (-np.arange(n, dtype=np.float32) + 2.0j * np.arange(n, dtype=np.float32)).astype(np.complex64)
return TraceData(
combo=ComboKey(input=in_pos, output=out_pos),
frequency_hz=freq,
s11=s11,
s21=s21,
capture_start_ns=capture_ns[0],
capture_end_ns=capture_ns[1],
@@ -74,7 +69,6 @@ class TraceCollectionRoundTripTest(unittest.TestCase):
for original, got in zip(collection.traces, decoded.traces):
self.assertEqual((got.combo.input, got.combo.output), (original.combo.input, original.combo.output))
self.assertTrue(np.array_equal(got.frequency_hz, original.frequency_hz))
self.assertTrue(np.array_equal(got.s11, original.s11))
self.assertTrue(np.array_equal(got.s21, original.s21))
self.assertEqual(
(got.capture_start_ns, got.capture_end_ns),
@@ -87,28 +81,67 @@ class TraceCollectionRoundTripTest(unittest.TestCase):
def test_preprocessed_round_trips(self) -> None:
self._assert_round_trips(PREPROC_MAGIC)
def test_s21_only_wire_uses_literal_raw3_prp3_and_twelve_bytes_per_point(self) -> None:
collection = SweepCollection(7, 123, [_trace(1, 2, 3)], capture_start_ns=100, capture_end_ns=120)
for magic, label in ((RAW_MAGIC, b"RAW3"), (PREPROC_MAGIC, b"PRP3")):
with self.subTest(magic=label):
wire = serialize_trace_collection(collection, magic)
expected = label + struct.pack("<QQI", 7, 123, 1)
expected += struct.pack("<III3f6f", 1, 2, 3, 1., 2., 3., 0., 0., -1., 2., -2., 4.)
expected += struct.pack("<QQIQQ", 100, 120, 1, 0, 0)
self.assertEqual(wire, expected)
self.assertEqual(len(wire), 44 + 28 + 12 * 3)
def test_old_trace_magics_are_rejected(self) -> None:
collection = SweepCollection(1, 2, [_trace(0, 0, 2)])
for magic, old in ((RAW_MAGIC, b"RAW2"), (PREPROC_MAGIC, b"PPR2"), (PREPROC_MAGIC, b"PRP2")):
with self.subTest(old=old), self.assertRaisesRegex(ValueError, "magic"):
decode_trace_collection(old + serialize_trace_collection(collection, magic)[4:], magic)
def test_complex_components_are_decoded_without_arithmetic_changes(self) -> None:
words = [0x3f800000, 0x7fc01234, 0x80000000, 0xbf800000] # (1, NaN), (-0, -1)
samples = struct.pack("<4I", *words)
wire = b"RAW3" + struct.pack("<QQIIII2f", 1, 2, 1, 0, 0, 2, 10., 20.) + samples + struct.pack("<QQIQQ", 0, 0, 1, 0, 0)
decoded = decode_trace_collection(wire, RAW_MAGIC).traces[0]
self.assertEqual(decoded.s21.dtype, np.dtype("<c8"))
np.testing.assert_array_equal(decoded.s21.view("<u4"), words)
result_wire = struct.pack("<I6QfII", RESULT_MAGIC, 1, 2, 0, 0, 1, 2, 0., 1, 0)
result_wire += struct.pack("<BHI2f", 1, 0, 2, 10., 20.) + samples
trace = decode_result_collection(result_wire).collection_payloads[0].trace
np.testing.assert_array_equal(trace.view("<u4"), words)
def test_point_count_cannot_read_beyond_the_available_arrays(self) -> None:
for magic in (RAW_MAGIC, PREPROC_MAGIC):
for point_count in (3, 0xffffffff):
with self.subTest(magic=magic, points=point_count), self.assertRaisesRegex(ValueError, "truncated"):
wire = struct.pack("<IQQIIII6f", magic, 1, 2, 1, 0, 0, point_count, 1., 2., 3., 4., 5., 6.)
decode_trace_collection(wire, magic)
def test_empty_traces_round_trip(self) -> None:
collection = SweepCollection(collection_id=1, monotonic_ns=2, traces=[])
decoded = decode_trace_collection(serialize_trace_collection(collection, RAW_MAGIC), RAW_MAGIC)
self.assertEqual(decoded.traces, [])
def test_payload_without_per_trace_window_table_still_decodes(self) -> None:
# A producer built before per-trace timing stops after the collection
# window; its traces must still decode, with the timestamps left at zero.
collection = SweepCollection(
collection_id=4,
monotonic_ns=5,
traces=[_trace(1, 0, 3, capture_ns=(7, 9))],
capture_start_ns=6,
capture_end_ns=10,
)
full = serialize_trace_collection(collection, RAW_MAGIC)
legacy = full[: -(4 + 16 * len(collection.traces))]
def test_live_raw_requires_capture_window_but_calibration_bundles_do_not(self) -> None:
collection = SweepCollection(1, 100, [_trace(0, 0, 2)])
for start, end in ((0, 0), (0, 1), (10, 9)):
collection.capture_start_ns, collection.capture_end_ns = start, end
with self.subTest(window=(start, end)), self.assertRaisesRegex(ValueError, "capture window"):
serialize_trace_collection(collection, RAW_MAGIC, require_capture_window=True)
collection.capture_start_ns = collection.capture_end_ns = 10
serialize_trace_collection(collection, RAW_MAGIC, require_capture_window=True)
collection.capture_start_ns = collection.capture_end_ns = 0
decoded = decode_trace_collection(serialize_trace_collection(collection, RAW_MAGIC), RAW_MAGIC)
self.assertEqual((decoded.capture_start_ns, decoded.capture_end_ns), (0, 0))
decoded = decode_trace_collection(legacy, RAW_MAGIC)
self.assertEqual((decoded.capture_start_ns, decoded.capture_end_ns), (6, 10))
self.assertEqual(len(decoded.traces), 1)
self.assertEqual((decoded.traces[0].capture_start_ns, decoded.traces[0].capture_end_ns), (0, 0))
def test_timing_trailer_is_required(self) -> None:
collection = SweepCollection(4, 5, [_trace(1, 0, 3, capture_ns=(7, 9))],
capture_start_ns=6, capture_end_ns=10)
for magic in (RAW_MAGIC, PREPROC_MAGIC):
payload = serialize_trace_collection(collection, magic)
for removed_bytes in (20, 36):
with self.subTest(magic=magic, removed_bytes=removed_bytes), self.assertRaisesRegex(ValueError, "truncated"):
decode_trace_collection(payload[:-removed_bytes], magic)
def test_per_trace_window_count_mismatch_is_rejected(self) -> None:
collection = SweepCollection(
@@ -121,66 +154,54 @@ class TraceCollectionRoundTripTest(unittest.TestCase):
decode_trace_collection(corrupt, RAW_MAGIC)
class ResultCollectionRoundTripTest(unittest.TestCase):
class ResultCollectionDecodeTest(unittest.TestCase):
def test_empty_tables_and_images_retain_their_declared_dimensions(self) -> None:
for shape in ((0, 3), (2, 0), (0, 0)):
for rows, columns in ((0, 3), (2, 0), (0, 0)):
for kind in (3, 4):
with self.subTest(shape=shape, kind=kind):
payload = ResultPayload('empty', kind, table=np.empty(shape, dtype=np.float32),
image=np.empty(shape, dtype=np.float32),
image_x_axis=np.zeros(shape[1], dtype=np.float32),
image_y_axis=np.zeros(shape[0], dtype=np.float32))
if kind == 4 and shape == (2, 0):
with self.subTest(shape=(rows, columns), kind=kind):
wire = struct.pack("<I6QfII", RESULT_MAGIC, 1, 2, 0, 0, 1, 2, 0., 1, 0)
wire += struct.pack("<BHII", kind, 0, columns, rows)
if kind == 3:
wire += bytes(4 * (rows + columns)) # zero-valued axes
if kind == 4 and rows and not columns:
with self.assertRaises(ValueError):
serialize_result_collection(ResultCollection(1, 2, collection_payloads=[payload]))
continue
result = decode_result_collection(serialize_result_collection(
ResultCollection(1, 2, collection_payloads=[payload])))
decoded = result.collection_payloads[0]
self.assertEqual((decoded.image if kind == 3 else decoded.table).shape, shape)
decode_result_collection(wire)
else:
decoded = decode_result_collection(wire).collection_payloads[0]
self.assertEqual((decoded.image if kind == 3 else decoded.table).shape, (rows, columns))
def test_multidimensional_trace_cannot_be_serialized_as_a_flat_trace(self) -> None:
payload = ResultPayload('bad', 1, frequency_hz=np.ones((1, 2)), trace=np.ones((1, 2)))
with self.assertRaises(ValueError):
serialize_result_collection(ResultCollection(1, 2, collection_payloads=[payload]))
def test_all_payload_kinds_round_trip(self) -> None:
image = np.arange(6, dtype=np.float32).reshape((2, 3))
table = np.array([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], dtype=np.float32)
collection = ResultCollection(
collection_id=9,
monotonic_ns=42,
processing_duration_ns=1000,
collection_payloads=[
ResultPayload(
processing_name="gpr_accumulator", kind=3,
image_x_axis=np.array([0.0, 1.0, 2.0], dtype=np.float32),
image_y_axis=np.array([0.0, 1.0], dtype=np.float32),
image=image,
),
ResultPayload(processing_name="gpr_points", kind=4, table=table),
],
blocks=[
ResultBlock(combo=ComboKey(input=1, output=0), payloads=[
ResultPayload(
processing_name="bscan", kind=1,
frequency_hz=np.array([1.0, 2.0], dtype=np.float32),
trace=np.array([1 + 1j, 2 - 2j], dtype=np.complex64),
),
ResultPayload(processing_name="snr", kind=2, scalar_value=2.5),
]),
],
)
decoded = decode_result_collection(serialize_result_collection(collection))
def test_all_payload_kinds_decode_from_wire_format(self) -> None:
# A fixed frame independent of Python encoding preserves the C++ wire contract.
wire = struct.pack("<I6QfII", RESULT_MAGIC, 9, 42, 1000, 2**63 + 7, 2**60 + 3, 2**60 + 9, 1.25, 2, 1)
wire += struct.pack("<BH", 3, 15) + b"gpr_accumulator"
wire += struct.pack("<II11f", 3, 2, 0, 1, 2, 0, 1, 0, 1, 2, 3, 4, 5)
wire += struct.pack("<BH", 4, 10) + b"gpr_points"
wire += struct.pack("<II6f", 3, 2, 0, 1, 2, 3, 4, 5)
wire += struct.pack("<III", 1, 0, 2) # input, output, payload count
wire += struct.pack("<BH", 1, 5) + b"bscan"
wire += struct.pack("<I6f", 2, 1, 2, 1, 1, 2, -2)
wire += struct.pack("<BH", 2, 3) + b"snr" + struct.pack("<f", 2.5)
decoded = decode_result_collection(wire)
self.assertEqual((decoded.collection_id, decoded.monotonic_ns, decoded.processing_duration_ns), (9, 42, 1000))
acc, points = decoded.collection_payloads
self.assertEqual(acc.processing_name, "gpr_accumulator")
self.assertTrue(np.array_equal(acc.image, image))
self.assertTrue(np.array_equal(points.table, table))
self.assertEqual(wire[:4], b"RSL3")
self.assertEqual(struct.calcsize("<I6QfII"), 64)
self.assertEqual((decoded.settings_revision, decoded.capture_start_ns, decoded.capture_end_ns, decoded.speed_m_s),
(2**63 + 7, 2**60 + 3, 2**60 + 9, 1.25))
for invalid in (b"RSL2" + wire[4:], wire[:63], wire + b"\0"):
with self.subTest(invalid_size=len(invalid)), self.assertRaises(ValueError):
decode_result_collection(invalid)
image, table = decoded.collection_payloads
self.assertEqual(image.processing_name, "gpr_accumulator")
self.assertEqual(table.processing_name, "gpr_points")
np.testing.assert_array_equal(image.image, np.arange(6, dtype=np.float32).reshape(2, 3))
np.testing.assert_array_equal(image.image_x_axis, [0, 1, 2])
np.testing.assert_array_equal(image.image_y_axis, [0, 1])
np.testing.assert_array_equal(table.table, [[0, 1, 2], [3, 4, 5]])
block = decoded.blocks[0]
self.assertEqual((block.combo.input, block.combo.output), (1, 0))
self.assertTrue(np.array_equal(block.payloads[0].trace, np.array([1 + 1j, 2 - 2j], dtype=np.complex64)))
self.assertAlmostEqual(block.payloads[1].scalar_value, 2.5)
self.assertEqual(block.combo, ComboKey(1, 0))
np.testing.assert_array_equal(block.payloads[0].frequency_hz, [1, 2])
np.testing.assert_array_equal(block.payloads[0].trace, [1 + 1j, 2 - 2j])
self.assertEqual(block.payloads[1].scalar_value, 2.5)
class CorruptFrameTest(unittest.TestCase):
@@ -207,12 +228,12 @@ class CorruptFrameTest(unittest.TestCase):
decode_trace_collection(corrupt, RAW_MAGIC)
def test_unsupported_payload_kind(self) -> None:
corrupt = struct.pack("<IQQQII", RESULT_MAGIC, 1, 1, 0, 1, 0) + struct.pack("<BH", 99, 0)
corrupt = struct.pack("<I6QfII", RESULT_MAGIC, 1, 1, 0, 0, 1, 2, 0., 1, 0) + struct.pack("<BH", 99, 0)
with self.assertRaises(ValueError):
decode_result_collection(corrupt)
def test_invalid_utf8_name(self) -> None:
corrupt = struct.pack("<IQQQII", RESULT_MAGIC, 1, 1, 0, 1, 0) + struct.pack("<BH", 2, 1) + b"\xff"
corrupt = struct.pack("<I6QfII", RESULT_MAGIC, 1, 1, 0, 0, 1, 2, 0., 1, 0) + struct.pack("<BH", 2, 1) + b"\xff"
with self.assertRaises(ValueError):
decode_result_collection(corrupt)
@@ -230,15 +251,15 @@ class _RingTestCase(unittest.TestCase):
self.addCleanup(self._cleanup, name, writer)
return writer
def _reader(self, name: str, **kw: object) -> ShmRingReader:
def _reader(self, name: str, **kw: float) -> ShmRingReader:
reader = ShmRingReader(name, **kw)
self.addCleanup(self._safe_close, reader)
return reader
@staticmethod
def _safe_close(obj: object) -> None:
def _safe_close(obj: ShmRingReader | ShmRingWriter) -> None:
with suppress(Exception):
obj.close() # type: ignore[attr-defined]
obj.close()
@staticmethod
def _cleanup(name: str, writer: ShmRingWriter) -> None:
@@ -265,6 +286,27 @@ class RingRoundTripTest(_RingTestCase):
class RingOverflowTest(_RingTestCase):
def test_overwrite_during_read_discards_torn_payload(self) -> None:
from python_app.orchestration.shm import atomics
name = self._ring_name()
writer = self._writer(name, capacity=1, slot_size=8)
reader = self._reader(name)
writer.push(b"old")
fence = atomics.fence
overwritten = False
def overwrite_after_copy():
nonlocal overwritten
fence()
if not overwritten:
overwritten = True
writer.push(b"new")
with mock.patch.object(atomics, "fence", side_effect=overwrite_after_copy):
self.assertIsNone(reader.pop_payload())
self.assertEqual(reader.pop_payload(), b"new")
def test_reader_cannot_rewind_cursor_after_producer_laps_during_commit(self) -> None:
from python_app.orchestration.shm import atomics
@@ -298,46 +340,34 @@ class RingOverflowTest(_RingTestCase):
self.assertEqual(survivors, [b"f3", b"f4", b"f5", b"f6"])
class PeekLatestTest(_RingTestCase):
def test_peek_rejects_a_slot_while_it_is_being_overwritten(self) -> None:
name = self._ring_name()
writer = self._writer(name, capacity=1, slot_size=8)
reader = self._reader(name)
writer.push(b"aaaaaaaa")
observations = []
write_size = writer._write_u32
def overwrite_in_progress(offset: int, value: int) -> None:
write_size(offset, value)
if offset == 64:
writer._mmap[80:84] = b"bbbb"
observations.append(reader.peek_latest_payload())
with mock.patch.object(writer, "_write_u32", side_effect=overwrite_in_progress):
writer.push(b"bbbbbbbb")
self.assertEqual(observations, [None])
self.assertEqual(reader.peek_latest_payload(), b"bbbbbbbb")
def test_peek_is_latest_and_non_consuming(self) -> None:
name = self._ring_name()
writer = self._writer(name, capacity=8, slot_size=64)
reader = self._reader(name)
self.assertIsNone(reader.peek_latest_payload()) # nothing published yet
for i in range(3):
writer.push(f"f{i}".encode())
self.assertEqual(reader.peek_latest_payload(), b"f2") # newest
self.assertEqual(reader.peek_latest_payload(), b"f2") # stable, not consumed
self.assertEqual(reader.pop_payload(), b"f0") # consumer cursor untouched
writer.push(b"f3")
self.assertEqual(reader.peek_latest_payload(), b"f3") # follows the newest
class RingOpenTest(_RingTestCase):
def test_same_size_segment_is_reused_only_with_matching_header(self) -> None:
for corrupt_header in (False, True):
with self.subTest(corrupt_header=corrupt_header):
name = self._ring_name(str(corrupt_header))
writer = self._writer(name, capacity=1, slot_size=8)
reader = self._reader(name)
writer.push(b"old")
if corrupt_header:
writer._write_u32(8, 99)
path = Path("/dev/shm") / name[1:]
old_inode = path.stat().st_ino
replacement = ShmRingWriter(name, 1, 8)
self.addCleanup(self._safe_close, replacement)
new_reader = self._reader(name)
self.assertEqual(path.stat().st_ino == old_inode, not corrupt_header)
self.assertEqual(new_reader.pop_payload(), None if corrupt_header else b"old")
if corrupt_header:
self.assertEqual(reader.pop_payload(), b"old")
replacement.push(b"new")
self.assertEqual(new_reader.pop_payload(), b"new")
replacement.close()
replacement.close()
def test_invalid_geometry_and_names_fail_before_opening_files(self) -> None:
for capacity, slot in ((True, 8), (1.5, 8), (1, 9), (2 ** 32, 8)):
with self.subTest(capacity=capacity, slot=slot), self.assertRaises(ValueError):
ShmRingWriter(self._ring_name(), capacity, slot)
ShmRingWriter(self._ring_name(), cast(int, capacity), slot)
for name in ("/", "/../tmp/ring", "/nested/ring", "/bad\x00ring"):
with self.subTest(name=name):
with self.assertRaises(ValueError):