init commit
This commit is contained in:
Ayzen
@@ -0,0 +1,21 @@
|
||||
"""Shared-memory ring readers and payload decoders."""
|
||||
|
||||
from python_app.orchestration.shm.binary_cursor import ByteCursor
|
||||
from python_app.orchestration.shm.decoder import (
|
||||
PREPROC_MAGIC,
|
||||
RAW_MAGIC,
|
||||
RESULT_MAGIC,
|
||||
decode_result_collection,
|
||||
decode_trace_collection,
|
||||
)
|
||||
from python_app.orchestration.shm.ring_reader import ShmRingReader
|
||||
|
||||
__all__ = [
|
||||
"ByteCursor",
|
||||
"PREPROC_MAGIC",
|
||||
"RAW_MAGIC",
|
||||
"RESULT_MAGIC",
|
||||
"ShmRingReader",
|
||||
"decode_result_collection",
|
||||
"decode_trace_collection",
|
||||
]
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Byte-wise cursor utilities for decoding binary ring payloads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
|
||||
|
||||
class ByteCursor:
|
||||
"""Read primitive values from bytes while tracking offset."""
|
||||
|
||||
def __init__(self, payload: bytes) -> None:
|
||||
"""Create cursor at start of payload."""
|
||||
self.payload = payload
|
||||
self.offset = 0
|
||||
|
||||
def read_u8(self) -> int:
|
||||
"""Read unsigned 8-bit integer."""
|
||||
value = struct.unpack_from("<B", self.payload, self.offset)[0]
|
||||
self.offset += 1
|
||||
return value
|
||||
|
||||
def read_u16(self) -> int:
|
||||
"""Read unsigned 16-bit integer."""
|
||||
value = struct.unpack_from("<H", self.payload, self.offset)[0]
|
||||
self.offset += 2
|
||||
return value
|
||||
|
||||
def read_u32(self) -> int:
|
||||
"""Read unsigned 32-bit integer."""
|
||||
value = struct.unpack_from("<I", self.payload, self.offset)[0]
|
||||
self.offset += 4
|
||||
return value
|
||||
|
||||
def read_u64(self) -> int:
|
||||
"""Read unsigned 64-bit integer."""
|
||||
value = struct.unpack_from("<Q", self.payload, self.offset)[0]
|
||||
self.offset += 8
|
||||
return value
|
||||
|
||||
def read_f32(self) -> float:
|
||||
"""Read 32-bit float."""
|
||||
value = struct.unpack_from("<f", self.payload, self.offset)[0]
|
||||
self.offset += 4
|
||||
return float(value)
|
||||
|
||||
def read_bytes(self, size: int) -> bytes:
|
||||
"""Read raw byte slice of fixed size."""
|
||||
data = self.payload[self.offset : self.offset + size]
|
||||
self.offset += size
|
||||
return data
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Binary decoders for raw/preprocessed/result payload collections."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
|
||||
from python_app.models.dataset_model import (
|
||||
ComboKey,
|
||||
ResultBlock,
|
||||
ResultCollection,
|
||||
ResultPayload,
|
||||
SweepCollection,
|
||||
TraceData,
|
||||
)
|
||||
from python_app.orchestration.shm.binary_cursor import ByteCursor
|
||||
|
||||
RAW_MAGIC = 0x31574152
|
||||
PREPROC_MAGIC = 0x31525050
|
||||
RESULT_MAGIC = 0x314C5352
|
||||
|
||||
|
||||
def decode_trace_collection(payload: bytes, expected_magic: int) -> SweepCollection:
|
||||
"""Decode one raw/preprocessed collection from binary payload."""
|
||||
cursor = ByteCursor(payload)
|
||||
magic = cursor.read_u32()
|
||||
if magic != expected_magic:
|
||||
raise ValueError("Unexpected trace collection magic")
|
||||
|
||||
collection_id = cursor.read_u64()
|
||||
monotonic_ns = cursor.read_u64()
|
||||
trace_count = cursor.read_u32()
|
||||
|
||||
traces: list[TraceData] = []
|
||||
for _ in range(trace_count):
|
||||
input_pos = cursor.read_u32()
|
||||
output_pos = cursor.read_u32()
|
||||
point_count = cursor.read_u32()
|
||||
|
||||
freq_bytes = point_count * 4
|
||||
freq = np.frombuffer(cursor.read_bytes(freq_bytes), dtype="<f4").astype(np.float32, copy=False)
|
||||
|
||||
interleaved_bytes = point_count * 8
|
||||
interleaved = np.frombuffer(cursor.read_bytes(interleaved_bytes), dtype="<f4")
|
||||
s21 = (interleaved[0::2] + 1j * interleaved[1::2]).astype(np.complex64, copy=False)
|
||||
|
||||
traces.append(
|
||||
TraceData(
|
||||
combo=ComboKey(input_pos=input_pos, output_pos=output_pos),
|
||||
frequency_hz=freq,
|
||||
s21=s21,
|
||||
)
|
||||
)
|
||||
|
||||
return SweepCollection(collection_id=collection_id, monotonic_ns=monotonic_ns, traces=traces)
|
||||
|
||||
|
||||
def decode_result_collection(payload: bytes) -> ResultCollection:
|
||||
"""Decode one processed result collection from binary payload."""
|
||||
cursor = ByteCursor(payload)
|
||||
magic = cursor.read_u32()
|
||||
if magic != RESULT_MAGIC:
|
||||
raise ValueError("Unexpected result collection magic")
|
||||
|
||||
collection_id = cursor.read_u64()
|
||||
monotonic_ns = cursor.read_u64()
|
||||
block_count = cursor.read_u32()
|
||||
|
||||
blocks: list[ResultBlock] = []
|
||||
for _ in range(block_count):
|
||||
input_pos = cursor.read_u32()
|
||||
output_pos = cursor.read_u32()
|
||||
payload_count = cursor.read_u32()
|
||||
|
||||
payloads: list[ResultPayload] = []
|
||||
for _ in range(payload_count):
|
||||
kind = cursor.read_u8()
|
||||
name_size = cursor.read_u16()
|
||||
name = cursor.read_bytes(name_size).decode("utf-8")
|
||||
|
||||
if kind == 1:
|
||||
point_count = cursor.read_u32()
|
||||
freq = np.frombuffer(cursor.read_bytes(point_count * 4), dtype="<f4").astype(np.float32, copy=False)
|
||||
interleaved = np.frombuffer(cursor.read_bytes(point_count * 8), dtype="<f4")
|
||||
trace = (interleaved[0::2] + 1j * interleaved[1::2]).astype(np.complex64, copy=False)
|
||||
payloads.append(
|
||||
ResultPayload(
|
||||
processing_name=name,
|
||||
kind=kind,
|
||||
frequency_hz=freq,
|
||||
trace=trace,
|
||||
)
|
||||
)
|
||||
elif kind == 2:
|
||||
scalar_value = cursor.read_f32()
|
||||
payloads.append(
|
||||
ResultPayload(
|
||||
processing_name=name,
|
||||
kind=kind,
|
||||
frequency_hz=np.array([], dtype=np.float32),
|
||||
trace=np.array([], dtype=np.complex64),
|
||||
scalar_value=scalar_value,
|
||||
)
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported result payload kind: {kind}")
|
||||
|
||||
blocks.append(
|
||||
ResultBlock(
|
||||
combo=ComboKey(input_pos=input_pos, output_pos=output_pos),
|
||||
payloads=payloads,
|
||||
)
|
||||
)
|
||||
|
||||
return ResultCollection(collection_id=collection_id, monotonic_ns=monotonic_ns, blocks=blocks)
|
||||
@@ -0,0 +1,150 @@
|
||||
"""POSIX shared-memory ring reader implementation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import mmap
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import time
|
||||
from typing import Final
|
||||
|
||||
from python_app.models.dataset_model import ResultCollection, SweepCollection
|
||||
from python_app.orchestration.shm.decoder import (
|
||||
PREPROC_MAGIC,
|
||||
RAW_MAGIC,
|
||||
decode_result_collection,
|
||||
decode_trace_collection,
|
||||
)
|
||||
|
||||
_HEADER_SIZE: Final[int] = 64
|
||||
_SLOT_HEADER_SIZE: Final[int] = 16
|
||||
_MAGIC: Final[bytes] = b"RDRRING2"
|
||||
_VERSION: Final[int] = 1
|
||||
|
||||
|
||||
class ShmRingReader:
|
||||
"""Read binary payloads from a lock-free ring in `/dev/shm`."""
|
||||
|
||||
def __init__(self, ring_name: str, open_timeout_s: float = 2.0, open_poll_s: float = 0.01) -> None:
|
||||
"""Open and validate ring by name, for example `/radar_results`."""
|
||||
if not ring_name.startswith("/"):
|
||||
raise ValueError("ring_name must start with '/'")
|
||||
|
||||
self._ring_name = ring_name
|
||||
self._path = Path("/dev/shm") / ring_name[1:]
|
||||
self._wait_for_ring_file(timeout_s=open_timeout_s, poll_s=open_poll_s)
|
||||
|
||||
self._file = self._path.open("r+b", buffering=0)
|
||||
self._mmap = mmap.mmap(self._file.fileno(), 0)
|
||||
self._validate_header_with_wait(timeout_s=1.0, poll_s=0.002)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close mmap and file handle."""
|
||||
self._mmap.close()
|
||||
self._file.close()
|
||||
|
||||
def pop_payload(self) -> bytes | None:
|
||||
"""Read next payload from ring, or `None` if no unread payload exists."""
|
||||
write_seq = self._read_u64(24)
|
||||
read_seq = self._read_u64(32)
|
||||
if read_seq >= write_seq:
|
||||
return None
|
||||
|
||||
index = read_seq % self.capacity
|
||||
slot_offset = _HEADER_SIZE + index * (_SLOT_HEADER_SIZE + self.slot_size_bytes)
|
||||
|
||||
payload_size = self._read_u32(slot_offset)
|
||||
sequence = self._read_u64(slot_offset + 8)
|
||||
if sequence != read_seq + 1:
|
||||
self._write_u64(32, write_seq)
|
||||
return None
|
||||
|
||||
payload_offset = slot_offset + _SLOT_HEADER_SIZE
|
||||
payload = self._mmap[payload_offset : payload_offset + payload_size]
|
||||
self._write_u64(32, read_seq + 1)
|
||||
return bytes(payload)
|
||||
|
||||
def pop_raw_collection(self) -> SweepCollection | None:
|
||||
"""Read next raw collection from ring."""
|
||||
payload = self.pop_payload()
|
||||
if payload is None:
|
||||
return None
|
||||
return decode_trace_collection(payload, RAW_MAGIC)
|
||||
|
||||
def pop_preprocessed_collection(self) -> SweepCollection | None:
|
||||
"""Read next preprocessed collection from ring."""
|
||||
payload = self.pop_payload()
|
||||
if payload is None:
|
||||
return None
|
||||
return decode_trace_collection(payload, PREPROC_MAGIC)
|
||||
|
||||
def pop_result_collection(self) -> ResultCollection | None:
|
||||
"""Read next processed result collection from ring."""
|
||||
payload = self.pop_payload()
|
||||
if payload is None:
|
||||
return None
|
||||
return decode_result_collection(payload)
|
||||
|
||||
def drop_all(self) -> int:
|
||||
"""Mark all unread slots as consumed and return number of dropped payloads."""
|
||||
write_seq = self._read_u64(24)
|
||||
read_seq = self._read_u64(32)
|
||||
if read_seq >= write_seq:
|
||||
return 0
|
||||
dropped = int(write_seq - read_seq)
|
||||
self._write_u64(32, write_seq)
|
||||
return dropped
|
||||
|
||||
@property
|
||||
def capacity(self) -> int:
|
||||
"""Number of slots in ring."""
|
||||
return self._read_u32(12)
|
||||
|
||||
@property
|
||||
def slot_size_bytes(self) -> int:
|
||||
"""Maximum payload size per slot."""
|
||||
return self._read_u32(16)
|
||||
|
||||
def _validate_header_with_wait(self, timeout_s: float, poll_s: float) -> None:
|
||||
"""Wait for ring header to contain expected magic and version."""
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while True:
|
||||
magic = self._mmap[:8]
|
||||
version = self._read_u32(8)
|
||||
if magic == _MAGIC and version == _VERSION:
|
||||
return
|
||||
|
||||
if time.monotonic() >= deadline:
|
||||
if magic != _MAGIC:
|
||||
got_magic = bytes(magic).hex()
|
||||
expected_magic = _MAGIC.hex()
|
||||
raise RuntimeError(
|
||||
f"Shared memory ring magic mismatch for {self._ring_name}: "
|
||||
f"got=0x{got_magic}, expected=0x{expected_magic}"
|
||||
)
|
||||
raise RuntimeError(
|
||||
f"Shared memory ring version mismatch for {self._ring_name}: "
|
||||
f"got={version}, expected={_VERSION}"
|
||||
)
|
||||
|
||||
time.sleep(poll_s)
|
||||
|
||||
def _wait_for_ring_file(self, timeout_s: float, poll_s: float) -> None:
|
||||
"""Wait until ring file appears in `/dev/shm`."""
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while not self._path.exists():
|
||||
if time.monotonic() >= deadline:
|
||||
raise FileNotFoundError(f"Shared memory ring does not exist: {self._ring_name}")
|
||||
time.sleep(poll_s)
|
||||
|
||||
def _read_u32(self, offset: int) -> int:
|
||||
"""Read little-endian u32 at mmap offset."""
|
||||
return struct.unpack_from("<I", self._mmap, offset)[0]
|
||||
|
||||
def _read_u64(self, offset: int) -> int:
|
||||
"""Read little-endian u64 at mmap offset."""
|
||||
return struct.unpack_from("<Q", self._mmap, offset)[0]
|
||||
|
||||
def _write_u64(self, offset: int, value: int) -> None:
|
||||
"""Write little-endian u64 at mmap offset."""
|
||||
struct.pack_into("<Q", self._mmap, offset, value)
|
||||
Reference in New Issue
Block a user