176 lines
7.7 KiB
Python
176 lines
7.7 KiB
Python
"""Binary payload serialization for sweep/result collections."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import struct
|
|
from numbers import Integral
|
|
|
|
import numpy as np
|
|
|
|
from python_app.models.dataset_model import ResultCollection, SweepCollection
|
|
|
|
RAW_MAGIC = 0x32574152
|
|
PREPROC_MAGIC = 0x32525050
|
|
RESULT_MAGIC = 0x324C5352 # RSL2: adds processing_duration_ns after monotonic_ns
|
|
|
|
|
|
def _write_interleaved_complex(buffer: bytearray, values: np.ndarray) -> None:
|
|
"""Append complex64 array as interleaved float32 real/imag pairs."""
|
|
interleaved = np.empty(values.size * 2, dtype="<f4")
|
|
interleaved[0::2] = values.real.astype("<f4", copy=False)
|
|
interleaved[1::2] = values.imag.astype("<f4", copy=False)
|
|
buffer.extend(interleaved.tobytes())
|
|
|
|
|
|
def validate_trace_collection(collection: SweepCollection, *, require_finite: bool = False) -> None:
|
|
"""Reject ambiguous combinations and malformed traces before saving any files."""
|
|
for field in ("collection_id", "monotonic_ns", "capture_start_ns", "capture_end_ns"):
|
|
_validate_u64(getattr(collection, field), field)
|
|
if require_finite and not collection.traces:
|
|
raise ValueError("Preprocess set must contain at least one trace")
|
|
seen: set[tuple[int, int]] = set()
|
|
for trace in collection.traces:
|
|
_validate_u64(trace.capture_start_ns, "trace.capture_start_ns")
|
|
_validate_u64(trace.capture_end_ns, "trace.capture_end_ns")
|
|
if any(
|
|
isinstance(value, bool) or not isinstance(value, Integral) or not 0 <= value <= 0xFFFFFFFF
|
|
for value in (trace.combo.input, trace.combo.output)
|
|
):
|
|
raise ValueError("Trace input/output positions must be unsigned 32-bit integers")
|
|
combo = (int(trace.combo.input), int(trace.combo.output))
|
|
if combo in seen:
|
|
raise ValueError(f"Invalid or duplicate trace combination: {combo}")
|
|
seen.add(combo)
|
|
arrays = (
|
|
np.asarray(trace.frequency_hz, dtype=np.float32),
|
|
np.asarray(trace.s11, dtype=np.complex64),
|
|
np.asarray(trace.s21, dtype=np.complex64),
|
|
)
|
|
if any(array.ndim != 1 for array in arrays) or len({array.shape for array in arrays}) != 1:
|
|
raise ValueError(f"Trace {combo} frequency, S11 and S21 must be equally sized 1D arrays")
|
|
if require_finite and (not arrays[0].size or any(not np.isfinite(array).all() for array in arrays)):
|
|
raise ValueError(f"Preprocess trace {combo} must contain finite, nonempty arrays")
|
|
|
|
|
|
def _validate_u64(value: int, field: str) -> None:
|
|
if isinstance(value, bool) or not isinstance(value, Integral) or not 0 <= value < 1 << 64:
|
|
raise ValueError(f"{field} must be an unsigned 64-bit integer")
|
|
|
|
|
|
def serialize_trace_collection(collection: SweepCollection, magic: int) -> bytes:
|
|
"""Serialize one raw/preprocessed trace collection into ring-compatible binary format.
|
|
|
|
The trailer is appended after the trace blocks so older readers, which stop at
|
|
the last block, still decode the traces: first the collection capture window,
|
|
then a per-trace window table (one ``(start_ns, end_ns)`` pair per trace, in
|
|
trace order). See :func:`python_app.orchestration.shm.decoder.decode_trace_collection`
|
|
and ``read_trace_collection`` in ``common_cpp/ipc/src/shared_types.cpp``.
|
|
"""
|
|
validate_trace_collection(collection)
|
|
buffer = bytearray()
|
|
buffer.extend(struct.pack("<IQQI", magic, collection.collection_id, collection.monotonic_ns, len(collection.traces)))
|
|
|
|
for trace in collection.traces:
|
|
freq = np.asarray(trace.frequency_hz, dtype=np.float32)
|
|
s11 = np.asarray(trace.s11, dtype=np.complex64)
|
|
s21 = np.asarray(trace.s21, dtype=np.complex64)
|
|
buffer.extend(struct.pack("<III", trace.combo.input, trace.combo.output, int(freq.size)))
|
|
buffer.extend(freq.astype("<f4", copy=False).tobytes())
|
|
|
|
_write_interleaved_complex(buffer, s11)
|
|
_write_interleaved_complex(buffer, s21)
|
|
|
|
buffer.extend(
|
|
struct.pack(
|
|
"<QQ",
|
|
int(collection.capture_start_ns),
|
|
int(collection.capture_end_ns),
|
|
)
|
|
)
|
|
buffer.extend(struct.pack("<I", len(collection.traces)))
|
|
for trace in collection.traces:
|
|
buffer.extend(
|
|
struct.pack("<QQ", int(trace.capture_start_ns), int(trace.capture_end_ns))
|
|
)
|
|
return bytes(buffer)
|
|
|
|
|
|
def serialize_result_collection(collection: ResultCollection) -> bytes:
|
|
"""Serialize one processed collection with result blocks/payloads."""
|
|
|
|
def serialize_payload(buffer: bytearray, payload) -> None:
|
|
"""Append one payload in ring-compatible result format."""
|
|
name_bytes = payload.processing_name.encode("utf-8")
|
|
if len(name_bytes) > 0xFFFF:
|
|
raise ValueError("processing_name is too long")
|
|
|
|
buffer.extend(struct.pack("<BH", payload.kind, len(name_bytes)))
|
|
buffer.extend(name_bytes)
|
|
|
|
if payload.kind == 1:
|
|
freq = np.asarray(payload.frequency_hz, dtype=np.float32)
|
|
trace = np.asarray(payload.trace, dtype=np.complex64)
|
|
if freq.ndim != 1 or trace.ndim != 1 or freq.shape != trace.shape:
|
|
raise ValueError("Result trace frequency and values must be equally sized 1D arrays")
|
|
|
|
buffer.extend(struct.pack("<I", int(freq.size)))
|
|
buffer.extend(freq.astype("<f4", copy=False).tobytes())
|
|
|
|
_write_interleaved_complex(buffer, trace)
|
|
return
|
|
|
|
if payload.kind == 2:
|
|
buffer.extend(struct.pack("<f", float(payload.scalar_value)))
|
|
return
|
|
|
|
if payload.kind == 3:
|
|
image_x_axis = np.asarray(payload.image_x_axis, dtype=np.float32)
|
|
image_y_axis = np.asarray(payload.image_y_axis, dtype=np.float32)
|
|
image = np.asarray(payload.image, dtype=np.float32)
|
|
if image_x_axis.ndim != 1 or image_y_axis.ndim != 1 or image.shape != (image_y_axis.size, image_x_axis.size):
|
|
raise ValueError("Result image axis sizes must match image matrix shape")
|
|
buffer.extend(struct.pack("<II", int(image_x_axis.size), int(image_y_axis.size)))
|
|
buffer.extend(image_x_axis.astype("<f4", copy=False).tobytes())
|
|
buffer.extend(image_y_axis.astype("<f4", copy=False).tobytes())
|
|
buffer.extend(image.astype("<f4", copy=False).ravel(order="C").tobytes())
|
|
return
|
|
|
|
if payload.kind == 4:
|
|
table = np.asarray(payload.table, dtype=np.float32)
|
|
if table.ndim != 2:
|
|
raise ValueError("Result table payload must be a 2D matrix")
|
|
if table.shape[0] and not table.shape[1]:
|
|
raise ValueError("Result table with rows must have at least one column")
|
|
buffer.extend(struct.pack("<II", int(table.shape[1]), int(table.shape[0])))
|
|
buffer.extend(table.astype("<f4", copy=False).ravel(order="C").tobytes())
|
|
return
|
|
|
|
raise ValueError(f"Unsupported payload kind: {payload.kind}")
|
|
|
|
for field in ("collection_id", "monotonic_ns", "processing_duration_ns"):
|
|
_validate_u64(getattr(collection, field), field)
|
|
buffer = bytearray()
|
|
buffer.extend(
|
|
struct.pack(
|
|
"<IQQQII",
|
|
RESULT_MAGIC,
|
|
collection.collection_id,
|
|
collection.monotonic_ns,
|
|
int(collection.processing_duration_ns),
|
|
len(collection.collection_payloads),
|
|
len(collection.blocks),
|
|
)
|
|
)
|
|
|
|
for payload in collection.collection_payloads:
|
|
serialize_payload(buffer, payload)
|
|
|
|
for block in collection.blocks:
|
|
buffer.extend(struct.pack("<II", block.combo.input, block.combo.output))
|
|
buffer.extend(struct.pack("<I", len(block.payloads)))
|
|
|
|
for payload in block.payloads:
|
|
serialize_payload(buffer, payload)
|
|
|
|
return bytes(buffer)
|