awe/radar_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rework radar system

Browse Source
This commit is contained in:
Ayzen
2026-10-06 12:32:26 +03:00
parent 552cdc688a
commit 5bb86b7165
299 changed files with 13537 additions and 27763 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python_app/storage/npz/serialize.py
+17 -106
View File
@@ -1,4 +1,4 @@
"""Binary payload serialization for sweep/result collections."""
"""Binary payload serialization for raw/preprocessed collections."""
from __future__ import annotations
@@ -7,25 +7,17 @@ from numbers import Integral
import numpy as np
from python_app.models.dataset_model import ResultCollection, SweepCollection
RAW_MAGIC = 0x32574152
PREPROC_MAGIC = 0x32525050
RESULT_MAGIC = 0x324C5352 # RSL2: adds processing_duration_ns after monotonic_ns
from python_app.models.dataset_model import SweepCollection
def _write_interleaved_complex(buffer: bytearray, values: np.ndarray) -> None:
"""Append complex64 array as interleaved float32 real/imag pairs."""
interleaved = np.empty(values.size * 2, dtype="<f4")
interleaved[0::2] = values.real.astype("<f4", copy=False)
interleaved[1::2] = values.imag.astype("<f4", copy=False)
buffer.extend(interleaved.tobytes())
def validate_trace_collection(collection: SweepCollection, *, require_finite: bool = False) -> None:
def validate_trace_collection(
collection: SweepCollection, *, require_finite: bool = False, require_capture_window: bool = False,
) -> None:
"""Reject ambiguous combinations and malformed traces before saving any files."""
for field in ("collection_id", "monotonic_ns", "capture_start_ns", "capture_end_ns"):
_validate_u64(getattr(collection, field), field)
if require_capture_window and not 0 < collection.capture_start_ns <= collection.capture_end_ns:
raise ValueError("Raw collection requires a positive, ordered capture window")
if require_finite and not collection.traces:
raise ValueError("Preprocess set must contain at least one trace")
seen: set[tuple[int, int]] = set()
@@ -43,11 +35,10 @@ def validate_trace_collection(collection: SweepCollection, *, require_finite: bo
seen.add(combo)
arrays = (
np.asarray(trace.frequency_hz, dtype=np.float32),
np.asarray(trace.s11, dtype=np.complex64),
np.asarray(trace.s21, dtype=np.complex64),
)
if any(array.ndim != 1 for array in arrays) or len({array.shape for array in arrays}) != 1:
raise ValueError(f"Trace {combo} frequency, S11 and S21 must be equally sized 1D arrays")
raise ValueError(f"Trace {combo} frequency and S21 must be equally sized 1D arrays")
if require_finite and (not arrays[0].size or any(not np.isfinite(array).all() for array in arrays)):
raise ValueError(f"Preprocess trace {combo} must contain finite, nonempty arrays")
@@ -57,28 +48,28 @@ def _validate_u64(value: int, field: str) -> None:
raise ValueError(f"{field} must be an unsigned 64-bit integer")
def serialize_trace_collection(collection: SweepCollection, magic: int) -> bytes:
def serialize_trace_collection(
collection: SweepCollection, magic: int, *, require_capture_window: bool = False,
) -> bytes:
"""Serialize one raw/preprocessed trace collection into ring-compatible binary format.
The trailer is appended after the trace blocks so older readers, which stop at
the last block, still decode the traces: first the collection capture window,
then a per-trace window table (one ``(start_ns, end_ns)`` pair per trace, in
trace order). See :func:`python_app.orchestration.shm.decoder.decode_trace_collection`
Each trace stores its frequency array followed by its complex S21 array, using
12 bytes per point. The trailer contains the collection capture window and a
per-trace window table (one ``(start_ns, end_ns)`` pair per trace, in trace
order). See :func:`python_app.orchestration.shm.decoder.decode_trace_collection`
and ``read_trace_collection`` in ``common_cpp/ipc/src/shared_types.cpp``.
"""
validate_trace_collection(collection)
validate_trace_collection(collection, require_capture_window=require_capture_window)
buffer = bytearray()
buffer.extend(struct.pack("<IQQI", magic, collection.collection_id, collection.monotonic_ns, len(collection.traces)))
for trace in collection.traces:
freq = np.asarray(trace.frequency_hz, dtype=np.float32)
s11 = np.asarray(trace.s11, dtype=np.complex64)
s21 = np.asarray(trace.s21, dtype=np.complex64)
buffer.extend(struct.pack("<III", trace.combo.input, trace.combo.output, int(freq.size)))
buffer.extend(freq.astype("<f4", copy=False).tobytes())
_write_interleaved_complex(buffer, s11)
_write_interleaved_complex(buffer, s21)
buffer.extend(s21.astype("<c8", copy=False).tobytes())
buffer.extend(
struct.pack(
@@ -93,83 +84,3 @@ def serialize_trace_collection(collection: SweepCollection, magic: int) -> bytes
struct.pack("<QQ", int(trace.capture_start_ns), int(trace.capture_end_ns))
)
return bytes(buffer)
def serialize_result_collection(collection: ResultCollection) -> bytes:
"""Serialize one processed collection with result blocks/payloads."""
def serialize_payload(buffer: bytearray, payload) -> None:
"""Append one payload in ring-compatible result format."""
name_bytes = payload.processing_name.encode("utf-8")
if len(name_bytes) > 0xFFFF:
raise ValueError("processing_name is too long")
buffer.extend(struct.pack("<BH", payload.kind, len(name_bytes)))
buffer.extend(name_bytes)
if payload.kind == 1:
freq = np.asarray(payload.frequency_hz, dtype=np.float32)
trace = np.asarray(payload.trace, dtype=np.complex64)
if freq.ndim != 1 or trace.ndim != 1 or freq.shape != trace.shape:
raise ValueError("Result trace frequency and values must be equally sized 1D arrays")
buffer.extend(struct.pack("<I", int(freq.size)))
buffer.extend(freq.astype("<f4", copy=False).tobytes())
_write_interleaved_complex(buffer, trace)
return
if payload.kind == 2:
buffer.extend(struct.pack("<f", float(payload.scalar_value)))
return
if payload.kind == 3:
image_x_axis = np.asarray(payload.image_x_axis, dtype=np.float32)
image_y_axis = np.asarray(payload.image_y_axis, dtype=np.float32)
image = np.asarray(payload.image, dtype=np.float32)
if image_x_axis.ndim != 1 or image_y_axis.ndim != 1 or image.shape != (image_y_axis.size, image_x_axis.size):
raise ValueError("Result image axis sizes must match image matrix shape")
buffer.extend(struct.pack("<II", int(image_x_axis.size), int(image_y_axis.size)))
buffer.extend(image_x_axis.astype("<f4", copy=False).tobytes())
buffer.extend(image_y_axis.astype("<f4", copy=False).tobytes())
buffer.extend(image.astype("<f4", copy=False).ravel(order="C").tobytes())
return
if payload.kind == 4:
table = np.asarray(payload.table, dtype=np.float32)
if table.ndim != 2:
raise ValueError("Result table payload must be a 2D matrix")
if table.shape[0] and not table.shape[1]:
raise ValueError("Result table with rows must have at least one column")
buffer.extend(struct.pack("<II", int(table.shape[1]), int(table.shape[0])))
buffer.extend(table.astype("<f4", copy=False).ravel(order="C").tobytes())
return
raise ValueError(f"Unsupported payload kind: {payload.kind}")
for field in ("collection_id", "monotonic_ns", "processing_duration_ns"):
_validate_u64(getattr(collection, field), field)
buffer = bytearray()
buffer.extend(
struct.pack(
"<IQQQII",
RESULT_MAGIC,
collection.collection_id,
collection.monotonic_ns,
int(collection.processing_duration_ns),
len(collection.collection_payloads),
len(collection.blocks),
)
)
for payload in collection.collection_payloads:
serialize_payload(buffer, payload)
for block in collection.blocks:
buffer.extend(struct.pack("<II", block.combo.input, block.combo.output))
buffer.extend(struct.pack("<I", len(block.payloads)))
for payload in block.payloads:
serialize_payload(buffer, payload)
return bytes(buffer)