awe/radar_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rework radar system

Browse Source
This commit is contained in:
Ayzen
2026-10-06 12:32:26 +03:00
parent 552cdc688a
commit 5bb86b7165
299 changed files with 13537 additions and 27763 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python_app/orchestration/shm/__init__.py
+1 -23
View File
@@ -1,23 +1 @@
"""Shared-memory ring readers and payload decoders."""
from python_app.orchestration.shm.binary_cursor import ByteCursor
from python_app.orchestration.shm.decoder import (
PREPROC_MAGIC,
RAW_MAGIC,
RESULT_MAGIC,
decode_result_collection,
decode_trace_collection,
)
from python_app.orchestration.shm.ring_reader import ShmRingReader
from python_app.orchestration.shm.ring_writer import ShmRingWriter
__all__ = [
"ByteCursor",
"PREPROC_MAGIC",
"RAW_MAGIC",
"RESULT_MAGIC",
"ShmRingReader",
"ShmRingWriter",
"decode_result_collection",
"decode_trace_collection",
]
"""Shared-memory rings and payload decoding."""
python_app/orchestration/shm/decoder.py
+23 -30
View File
@@ -14,9 +14,9 @@ from python_app.models.dataset_model import (
)
from python_app.orchestration.shm.binary_cursor import ByteCursor
RAW_MAGIC = 0x32574152
PREPROC_MAGIC = 0x32525050
RESULT_MAGIC = 0x324C5352 # RSL2: adds processing_duration_ns after monotonic_ns
RAW_MAGIC = 0x33574152 # RAW3
PREPROC_MAGIC = 0x33505250 # PRP3
RESULT_MAGIC = 0x334C5352 # RSL3: typed acquisition and processing metadata
def decode_trace_collection(payload: bytes, expected_magic: int) -> SweepCollection:
@@ -39,40 +39,23 @@ def decode_trace_collection(payload: bytes, expected_magic: int) -> SweepCollect
freq_bytes = point_count * 4
freq = np.frombuffer(cursor.read_bytes(freq_bytes), dtype="<f4").astype(np.float32, copy=False)
interleaved_bytes = point_count * 8
s11_interleaved = np.frombuffer(cursor.read_bytes(interleaved_bytes), dtype="<f4")
s21_interleaved = np.frombuffer(cursor.read_bytes(interleaved_bytes), dtype="<f4")
s11 = (s11_interleaved[0::2] + 1j * s11_interleaved[1::2]).astype(np.complex64, copy=False)
s21 = (s21_interleaved[0::2] + 1j * s21_interleaved[1::2]).astype(np.complex64, copy=False)
s21 = np.frombuffer(cursor.read_bytes(point_count * 8), dtype="<c8")
traces.append(
TraceData(
combo=ComboKey(input=input_pos, output=output_pos),
frequency_hz=freq,
s11=s11,
s21=s21,
)
)
# Optional trailer, written after the trace blocks by newer producers: the
# collection capture window, then a per-trace window table. Both stages are
# optional so payloads from an older producer still decode (the timestamps
# simply stay zero).
capture_start_ns = 0
capture_end_ns = 0
if cursor.remaining_bytes() != 0:
if cursor.remaining_bytes() < 16:
raise ValueError("Truncated capture window in trace collection")
capture_start_ns = cursor.read_u64()
capture_end_ns = cursor.read_u64()
if cursor.remaining_bytes() != 0:
trace_time_count = cursor.read_u32()
if trace_time_count != len(traces):
raise ValueError("Per-trace capture window count does not match trace count")
for trace in traces:
trace.capture_start_ns = cursor.read_u64()
trace.capture_end_ns = cursor.read_u64()
capture_start_ns = cursor.read_u64()
capture_end_ns = cursor.read_u64()
if cursor.read_u32() != len(traces):
raise ValueError("Per-trace capture window count does not match trace count")
for trace in traces:
trace.capture_start_ns = cursor.read_u64()
trace.capture_end_ns = cursor.read_u64()
if cursor.remaining_bytes() != 0:
raise ValueError("Unexpected trailing bytes in trace collection")
@@ -98,8 +81,7 @@ def decode_result_collection(payload: bytes) -> ResultCollection:
if kind == 1:
point_count = cursor.read_u32()
freq = np.frombuffer(cursor.read_bytes(point_count * 4), dtype="<f4").astype(np.float32, copy=False)
interleaved = np.frombuffer(cursor.read_bytes(point_count * 8), dtype="<f4")
trace = (interleaved[0::2] + 1j * interleaved[1::2]).astype(np.complex64, copy=False)
trace = np.frombuffer(cursor.read_bytes(point_count * 8), dtype="<c8")
return ResultPayload(
processing_name=name,
kind=kind,
@@ -150,6 +132,10 @@ def decode_result_collection(payload: bytes) -> ResultCollection:
collection_id = cursor.read_u64()
monotonic_ns = cursor.read_u64()
processing_duration_ns = cursor.read_u64()
settings_revision = cursor.read_u64()
capture_start_ns = cursor.read_u64()
capture_end_ns = cursor.read_u64()
speed_m_s = cursor.read_f32()
collection_payload_count = cursor.read_u32()
block_count = cursor.read_u32()
@@ -174,10 +160,17 @@ def decode_result_collection(payload: bytes) -> ResultCollection:
)
)
if cursor.remaining_bytes() != 0:
raise ValueError("Unexpected trailing bytes in result collection")
return ResultCollection(
collection_id=collection_id,
monotonic_ns=monotonic_ns,
processing_duration_ns=processing_duration_ns,
settings_revision=settings_revision,
capture_start_ns=capture_start_ns,
capture_end_ns=capture_end_ns,
speed_m_s=speed_m_s,
collection_payloads=collection_payloads,
blocks=blocks,
)
python_app/orchestration/shm/ring_layout.py
+15
View File
@@ -0,0 +1,15 @@
"""Shared ring layout; keep in sync with common_cpp/ipc/src/shm_ring.cpp."""
from pathlib import Path
import struct
MAGIC = b"RDRRING2"
VERSION = 1
HEADER = struct.Struct("<8sIIIIQQQ16x")
SLOT_HEADER_SIZE = 16
def ring_path(name: str) -> Path:
if not name.startswith("/") or len(name) < 2 or "/" in name[1:] or "\x00" in name:
raise ValueError("ring_name must be '/' followed by a single nonempty filename")
return Path("/dev/shm") / name[1:]
python_app/orchestration/shm/ring_reader.py
+26 -72
View File
@@ -4,9 +4,7 @@ from __future__ import annotations
import logging
import mmap
from pathlib import Path
import time
from typing import Final
from python_app.models.dataset_model import ResultCollection, SweepCollection
from python_app.orchestration.shm import atomics
@@ -16,31 +14,23 @@ from python_app.orchestration.shm.decoder import (
decode_result_collection,
decode_trace_collection,
)
from python_app.orchestration.shm.ring_layout import HEADER, MAGIC, SLOT_HEADER_SIZE, VERSION, ring_path
logger = logging.getLogger(__name__)
_HEADER_SIZE: Final[int] = 64
_SLOT_HEADER_SIZE: Final[int] = 16
_MAGIC: Final[bytes] = b"RDRRING2"
_VERSION: Final[int] = 1
class ShmRingReader:
"""Read binary payloads from a lock-free ring in `/dev/shm`."""
def __init__(self, ring_name: str, open_timeout_s: float = 2.0, open_poll_s: float = 0.01) -> None:
"""Open and validate ring by name, for example `/radar_results`."""
if not ring_name.startswith("/") or len(ring_name) < 2 or "/" in ring_name[1:] or "\x00" in ring_name:
raise ValueError("ring_name must be '/' followed by a single nonempty filename")
self._ring_name = ring_name
self._path = Path("/dev/shm") / ring_name[1:]
self.name = ring_name
self._path = ring_path(ring_name)
self._wait_for_ring_file(timeout_s=open_timeout_s, poll_s=open_poll_s)
self._file = self._path.open("r+b", buffering=0)
self._mmap: mmap.mmap | None = None
with self._path.open("r+b", buffering=0) as file:
self._mmap = mmap.mmap(file.fileno(), 0)
try:
self._mmap = mmap.mmap(self._file.fileno(), 0)
self._validate_header_with_wait(timeout_s=1.0, poll_s=0.002)
except BaseException:
# A fail-fast open (absent/incompatible ring) must not leak the fd/mapping.
@@ -48,18 +38,15 @@ class ShmRingReader:
raise
logger.debug(
"Opened SHM ring reader %s (capacity=%d, slot_size=%d bytes)",
self._ring_name,
self.name,
self.capacity,
self.slot_size_bytes,
)
def close(self) -> None:
"""Close mmap and file handle."""
if self._mmap is not None:
self._mmap.close()
self._mmap = None
self._file.close()
logger.debug("Closed SHM ring reader %s", self._ring_name)
"""Close the mapping and its owned file descriptor; safe to repeat."""
self._mmap.close()
logger.debug("Closed SHM ring reader %s", self.name)
def pop_payload(self) -> bytes | None:
"""Read next payload from ring, or `None` if no unread payload exists."""
@@ -69,7 +56,7 @@ class ShmRingReader:
return None
index = read_seq % self.capacity
slot_offset = _HEADER_SIZE + index * (_SLOT_HEADER_SIZE + self.slot_size_bytes)
slot_offset = HEADER.size + index * (SLOT_HEADER_SIZE + self.slot_size_bytes)
# Seqlock read mirroring the C++ pop: a slot is valid for read_seq R only if
# its sequence equals R+1 and is unchanged across the payload copy (i.e. the
@@ -78,7 +65,7 @@ class ShmRingReader:
sequence = self._read_u64(slot_offset + 8)
if sequence != read_seq + 1:
# Producer overwrote this slot before we read it. Resync to latest.
logger.debug("Ring %s: slot lapped before read, resyncing to write_seq=%d", self._ring_name, write_seq)
logger.debug("Ring %s: slot lapped before read, resyncing to write_seq=%d", self.name, write_seq)
atomics.advance(self._mmap, 32, write_seq)
return None
@@ -88,21 +75,21 @@ class ShmRingReader:
if payload_size > self.slot_size_bytes:
logger.debug(
"Ring %s: payload_size %d exceeds slot %d, resyncing",
self._ring_name,
self.name,
payload_size,
self.slot_size_bytes,
)
atomics.advance(self._mmap, 32, write_seq)
return None
payload_offset = slot_offset + _SLOT_HEADER_SIZE
payload_offset = slot_offset + SLOT_HEADER_SIZE
payload = bytes(self._mmap[payload_offset : payload_offset + payload_size])
atomics.fence()
# Re-read the slot sequence after the copy; if it changed, the producer
# overwrote this slot mid-copy and the payload is torn — discard and resync.
if self._read_u64(slot_offset + 8) != read_seq + 1:
logger.debug("Ring %s: slot overwritten mid-copy, discarding torn payload", self._ring_name)
logger.debug("Ring %s: slot overwritten mid-copy, discarding torn payload", self.name)
atomics.advance(self._mmap, 32, write_seq)
return None
@@ -130,39 +117,6 @@ class ShmRingReader:
return None
return decode_result_collection(payload)
def peek_latest_payload(self) -> bytes | None:
"""Return the most recently published payload WITHOUT consuming it.
Reads the newest slot through the seqlock and never advances `read_seq`, so a
viewer can peek the freshest frame while the ring's real consumer keeps its own
cursor — the two coexist without stealing each other's payloads. Inherently
latest-wins: always the freshest published frame, or `None` when nothing has
been published yet or the slot is being overwritten at this instant.
"""
write_seq = self._read_u64(24)
if write_seq == 0:
return None
latest_seq = write_seq - 1
index = latest_seq % self.capacity
slot_offset = _HEADER_SIZE + index * (_SLOT_HEADER_SIZE + self.slot_size_bytes)
# Seqlock read with NO cursor advance: accept the slot only if its sequence
# equals the published value both before and after the copy (i.e. the producer
# did not lap this slot mid-read). Never touch read_seq, so the real consumer
# is undisturbed.
if self._read_u64(slot_offset + 8) != latest_seq + 1:
return None
payload_size = self._read_u32(slot_offset)
if payload_size > self.slot_size_bytes:
return None
payload_offset = slot_offset + _SLOT_HEADER_SIZE
payload = bytes(self._mmap[payload_offset : payload_offset + payload_size])
atomics.fence()
if self._read_u64(slot_offset + 8) != latest_seq + 1:
return None
return payload
def drop_all(self) -> int:
"""Mark all unread slots as consumed and return number of dropped payloads."""
write_seq = self._read_u64(24)
@@ -171,7 +125,7 @@ class ShmRingReader:
return 0
dropped = int(write_seq - read_seq)
atomics.advance(self._mmap, 32, write_seq)
logger.debug("Ring %s: dropped %d unread payload(s)", self._ring_name, dropped)
logger.debug("Ring %s: dropped %d unread payload(s)", self.name, dropped)
return dropped
@property
@@ -190,27 +144,27 @@ class ShmRingReader:
while True:
magic = self._mmap[:8]
version = self._read_u32(8)
if magic == _MAGIC and version == _VERSION:
if magic == MAGIC and version == VERSION:
capacity = self.capacity
slot_size = self.slot_size_bytes
expected_size = _HEADER_SIZE + capacity * (_SLOT_HEADER_SIZE + slot_size)
expected_size = HEADER.size + capacity * (SLOT_HEADER_SIZE + slot_size)
if capacity > 0 and slot_size > 0 and slot_size % 8 == 0 and expected_size == len(self._mmap):
return
if time.monotonic() >= deadline:
if magic != _MAGIC:
if magic != MAGIC:
got_magic = bytes(magic).hex()
expected_magic = _MAGIC.hex()
expected_magic = MAGIC.hex()
raise RuntimeError(
f"Shared memory ring magic mismatch for {self._ring_name}: "
f"Shared memory ring magic mismatch for {self.name}: "
f"got=0x{got_magic}, expected=0x{expected_magic}"
)
if version != _VERSION:
if version != VERSION:
raise RuntimeError(
f"Shared memory ring version mismatch for {self._ring_name}: "
f"got={version}, expected={_VERSION}"
f"Shared memory ring version mismatch for {self.name}: "
f"got={version}, expected={VERSION}"
)
raise RuntimeError(f"Shared memory ring geometry mismatch for {self._ring_name}")
raise RuntimeError(f"Shared memory ring geometry mismatch for {self.name}")
time.sleep(poll_s)
@@ -219,12 +173,12 @@ class ShmRingReader:
deadline = time.monotonic() + timeout_s
while True:
try:
if self._path.stat().st_size >= _HEADER_SIZE:
if self._path.stat().st_size >= HEADER.size:
return
except FileNotFoundError:
pass
if time.monotonic() >= deadline:
raise FileNotFoundError(f"Shared memory ring does not exist: {self._ring_name}")
raise FileNotFoundError(f"Shared memory ring does not exist: {self.name}")
time.sleep(poll_s)
def _read_u32(self, offset: int) -> int:
python_app/orchestration/shm/ring_writer.py
+55 -132
View File
@@ -5,117 +5,74 @@ from __future__ import annotations
import logging
import mmap
import os
from pathlib import Path
from typing import Final
from python_app.models.run_config_schema import RingEndpointModel
from python_app.models.run_config_validation import validate_ring_endpoint
from python_app.orchestration.shm import atomics
logger = logging.getLogger(__name__)
from python_app.orchestration.shm.ring_layout import HEADER, MAGIC, SLOT_HEADER_SIZE, VERSION, ring_path
_HEADER_SIZE: Final[int] = 64
_SLOT_HEADER_SIZE: Final[int] = 16
_MAGIC: Final[bytes] = b"RDRRING2"
_VERSION: Final[int] = 1
logger = logging.getLogger(__name__)
class ShmRingWriter:
"""Write binary payloads into the shared-memory ring used by C++ workers.
"""The sole producer owns the ring and replaces incompatible stale segments.
The writer is the sole *owner* of the rings it opens: there is exactly one
producer per ring (the acquisition producer for the raw/raw_tap rings). On a
geometry mismatch with a pre-existing segment (e.g. a stale ring left by a prior
run with a different sweep config), the owner unlinks and recreates the segment
from scratch rather than truncating in place or diverging silently — mirroring
the clean-shm-on-restart contract on the C++/deploy side (#13). A non-owner must
never recreate a ring; readers and C++ consumers only ever attach to an existing
one.
Replacement uses unlink plus a new inode, preserving existing readers' mappings.
Compatible segments retain their unread payloads and sequence counters.
"""
def __init__(self, ring_name: str, capacity: int, slot_size_bytes: int) -> None:
"""Open or create a POSIX SHM ring by name (as the ring owner)."""
if not ring_name.startswith("/") or len(ring_name) < 2 or "/" in ring_name[1:] or "\x00" in ring_name:
raise ValueError("ring_name must be '/' followed by a single nonempty filename")
self._path = ring_path(ring_name)
validate_ring_endpoint(RingEndpointModel(ring_name, capacity, slot_size_bytes))
self._ring_name = ring_name
self._capacity = int(capacity)
self._slot_size_bytes = int(slot_size_bytes)
self._mapped_size = _HEADER_SIZE + self._capacity * (_SLOT_HEADER_SIZE + self._slot_size_bytes)
self._path = Path("/dev/shm") / ring_name[1:]
self._file = self._mmap = None
self._capacity = capacity
self._slot_size_bytes = slot_size_bytes
self._mapped_size = HEADER.size + capacity * (SLOT_HEADER_SIZE + slot_size_bytes)
self._mmap = self._open_owned()
def _open_owned(self) -> mmap.mmap:
"""Reuse a compatible segment, otherwise replace it without truncating it."""
try:
self._open_owned()
except BaseException:
self.close()
raise
def _open_owned(self) -> None:
"""Open the ring, recreating it from scratch on a geometry/header mismatch.
As the single owner of this ring we may safely discard a stale segment: a
size or header mismatch means the existing segment belongs to an earlier,
incompatible run, so we unlink it and create a fresh one instead of mapping
an inconsistent layout.
"""
created = not self._path.exists()
fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o660)
self._file = os.fdopen(fd, "r+b", buffering=0)
if created or self._path.stat().st_size != self._mapped_size:
# Wrong-sized stale segment: drop it entirely and recreate, so the file
# and any future mapping agree on geometry instead of being truncated
# under a producer/consumer that still expects the old layout.
if not created:
logger.info("Ring %s: stale segment with wrong size, recreating", self._ring_name)
self._file.close()
self._unlink_if_present()
fd = os.open(self._path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o660)
self._file = os.fdopen(fd, "r+b", buffering=0)
self._file.truncate(self._mapped_size)
created = True
self._mmap = mmap.mmap(self._file.fileno(), self._mapped_size)
if created:
self._initialize_header()
logger.debug(
"Created SHM ring writer %s (capacity=%d, slot_size=%d bytes)",
self._ring_name,
self._capacity,
self._slot_size_bytes,
)
return
# Size matched but the header geometry/magic does not: the owner recreates
# rather than diverge. Unlink and reopen as a brand-new ring.
if not self._header_matches():
logger.warning("Ring %s: header/geometry mismatch on existing segment, recreating", self._ring_name)
self._mmap.close()
self._file.close()
self._unlink_if_present()
fd = os.open(self._path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o660)
self._file = os.fdopen(fd, "r+b", buffering=0)
self._file.truncate(self._mapped_size)
self._mmap = mmap.mmap(self._file.fileno(), self._mapped_size)
self._initialize_header()
else:
logger.debug("Reusing existing SHM ring writer %s", self._ring_name)
def _unlink_if_present(self) -> None:
"""Remove the backing /dev/shm file if it exists (owner-only operation)."""
try:
self._path.unlink()
with self._path.open("r+b", buffering=0) as file:
if os.fstat(file.fileno()).st_size == self._mapped_size:
mapping = mmap.mmap(file.fileno(), self._mapped_size)
try:
if self._header_matches(mapping):
return mapping
except BaseException:
mapping.close()
raise
mapping.close()
logger.warning("Ring %s: stale segment, recreating", self._ring_name)
except FileNotFoundError:
pass
self._path.unlink(missing_ok=True)
fd = os.open(self._path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o660)
with os.fdopen(fd, "r+b", buffering=0) as file:
file.truncate(self._mapped_size)
mapping = mmap.mmap(file.fileno(), self._mapped_size)
try:
# New slots are already zero-filled. Publish the version last so readers
# cannot accept a partially initialized header.
mapping[:HEADER.size] = HEADER.pack(MAGIC, 0, self._capacity, self._slot_size_bytes, 0, 0, 0, 0)
atomics.store(mapping, 8, VERSION, 4)
except BaseException:
mapping.close()
raise
return mapping
def _header_matches(self, mapping: mmap.mmap) -> bool:
return (
atomics.load(mapping, 8, 4) == VERSION
and mapping[:8] == MAGIC
and atomics.load(mapping, 12, 4) == self._capacity
and atomics.load(mapping, 16, 4) == self._slot_size_bytes
)
def close(self) -> None:
"""Close mmap and file handle."""
if self._mmap is not None:
self._mmap.close()
self._mmap = None
if self._file is not None:
self._file.close()
self._file = None
logger.debug("Closed SHM ring writer %s", self._ring_name)
"""Close the mapping and its owned file descriptor; safe to repeat."""
self._mmap.close()
def push(self, payload: bytes) -> bool:
"""Push one payload with overwrite-oldest semantics on overflow."""
@@ -137,11 +94,11 @@ class ShmRingWriter:
self._write_u64(40, self._read_u64(40) + 1)
index = write_seq % self._capacity
slot_offset = _HEADER_SIZE + index * (_SLOT_HEADER_SIZE + self._slot_size_bytes)
payload_offset = slot_offset + _SLOT_HEADER_SIZE
slot_offset = HEADER.size + index * (SLOT_HEADER_SIZE + self._slot_size_bytes)
payload_offset = slot_offset + SLOT_HEADER_SIZE
# Invalidate the previous publication before changing its bytes. A peek or
# a lapped consumer must not accept the old sequence during this overwrite.
# Invalidate the previous publication before changing its bytes. A lapped
# consumer must not accept the old sequence during this overwrite.
self._write_u64(slot_offset + 8, 0)
atomics.fence()
self._write_u32(slot_offset, len(payload))
@@ -151,50 +108,16 @@ class ShmRingWriter:
self._write_u64(24, write_seq + 1)
return True
@property
def ring_name(self) -> str:
"""Return the POSIX SHM ring name."""
return self._ring_name
@property
def slot_size_bytes(self) -> int:
"""Return maximum payload size per slot."""
return self._slot_size_bytes
def _initialize_header(self) -> None:
# A new ftruncate-extended segment is already zero-filled by the kernel;
# avoid allocating and copying another complete ring just to initialize it.
self._mmap[:_HEADER_SIZE] = b"\x00" * _HEADER_SIZE
self._mmap[:8] = _MAGIC
self._write_u32(8, _VERSION)
self._write_u32(12, self._capacity)
self._write_u32(16, self._slot_size_bytes)
self._write_u32(20, 0)
self._write_u64(24, 0)
self._write_u64(32, 0)
self._write_u64(40, 0)
def _header_matches(self) -> bool:
"""Return whether the existing segment's header matches this ring's geometry.
Used by the owner to decide whether a same-sized pre-existing segment can be
reused as-is or must be unlinked and recreated.
"""
return (
self._mmap[:8] == _MAGIC
and self._read_u32(8) == _VERSION
and self._read_u32(12) == self._capacity
and self._read_u32(16) == self._slot_size_bytes
)
def _read_u32(self, offset: int) -> int:
return atomics.load(self._mmap, offset, 4)
def _read_u64(self, offset: int) -> int:
return atomics.load(self._mmap, offset, 8)
def _write_u32(self, offset: int, value: int) -> None:
atomics.store(self._mmap, offset, int(value), 4)
atomics.store(self._mmap, offset, value, 4)
def _write_u64(self, offset: int, value: int) -> None:
atomics.store(self._mmap, offset, int(value), 8)
atomics.store(self._mmap, offset, value, 8)