Rework radar system
This commit is contained in:
Ayzen
@@ -1,13 +1 @@
|
||||
"""NPZ-based dataset storage implementation and serialization helpers."""
|
||||
|
||||
from python_app.storage.npz.paths import radar_key_from_config
|
||||
from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, RESULT_MAGIC
|
||||
from python_app.storage.npz.store import NpzStore
|
||||
|
||||
__all__ = [
|
||||
"NpzStore",
|
||||
"PREPROC_MAGIC",
|
||||
"RAW_MAGIC",
|
||||
"RESULT_MAGIC",
|
||||
"radar_key_from_config",
|
||||
]
|
||||
"""NPZ-based dataset storage."""
|
||||
|
||||
@@ -2,35 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Sequence
|
||||
|
||||
|
||||
def radar_key_from_config(
|
||||
model_name: str,
|
||||
serial: str,
|
||||
sweep_start_hz: float,
|
||||
sweep_stop_hz: float,
|
||||
sweep_points: int,
|
||||
ifbw_hz: float,
|
||||
power_dbm: float,
|
||||
extra_serials: Sequence[str] | None = None,
|
||||
) -> str:
|
||||
"""Build deterministic key for calibration/reference set lookup."""
|
||||
serial_parts = [serial or "no_serial"]
|
||||
if extra_serials:
|
||||
serial_parts.extend(str(value).strip() or "no_serial" for value in extra_serials)
|
||||
serial_part = "_".join(sanitize_path_component(value) for value in serial_parts)
|
||||
start_token = format_float_for_key(sweep_start_hz)
|
||||
stop_token = format_float_for_key(sweep_stop_hz)
|
||||
points_token = "adc" if model_name.strip().lower() == "kamil_adc" else str(int(sweep_points))
|
||||
ifbw_token = format_float_for_key(ifbw_hz)
|
||||
power_token = format_float_for_key(power_dbm)
|
||||
return (
|
||||
f"{model_name}_{serial_part}"
|
||||
f"_st{start_token}_sp{stop_token}"
|
||||
f"_p{points_token}_if{ifbw_token}_pw{power_token}"
|
||||
)
|
||||
from python_app.file_io import sanitize_path_component
|
||||
|
||||
|
||||
def radar_config_filename_prefix(
|
||||
@@ -73,18 +45,3 @@ def _frequency_unit(hz: float) -> tuple[str, float]:
|
||||
def collection_dir_name(index: int, collection_id: int, monotonic_ns: int) -> str:
|
||||
"""Build canonical collection directory name for snapshot stages."""
|
||||
return f"{index:04d}_id{int(collection_id)}_ns{int(monotonic_ns)}"
|
||||
|
||||
|
||||
def sanitize_path_component(name: str) -> str:
|
||||
"""Convert arbitrary string into filesystem-safe component."""
|
||||
cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", name.strip())
|
||||
cleaned = cleaned.strip("._")
|
||||
return cleaned or "snapshot"
|
||||
|
||||
|
||||
def format_float_for_key(value: float) -> str:
|
||||
"""Format float compactly for deterministic key serialization."""
|
||||
integer = int(round(value))
|
||||
if abs(value - float(integer)) < 1e-6:
|
||||
return str(integer)
|
||||
return f"{value:.6f}".rstrip("0").rstrip(".")
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Binary payload serialization for sweep/result collections."""
|
||||
"""Binary payload serialization for raw/preprocessed collections."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -7,25 +7,17 @@ from numbers import Integral
|
||||
|
||||
import numpy as np
|
||||
|
||||
from python_app.models.dataset_model import ResultCollection, SweepCollection
|
||||
|
||||
RAW_MAGIC = 0x32574152
|
||||
PREPROC_MAGIC = 0x32525050
|
||||
RESULT_MAGIC = 0x324C5352 # RSL2: adds processing_duration_ns after monotonic_ns
|
||||
from python_app.models.dataset_model import SweepCollection
|
||||
|
||||
|
||||
def _write_interleaved_complex(buffer: bytearray, values: np.ndarray) -> None:
|
||||
"""Append complex64 array as interleaved float32 real/imag pairs."""
|
||||
interleaved = np.empty(values.size * 2, dtype="<f4")
|
||||
interleaved[0::2] = values.real.astype("<f4", copy=False)
|
||||
interleaved[1::2] = values.imag.astype("<f4", copy=False)
|
||||
buffer.extend(interleaved.tobytes())
|
||||
|
||||
|
||||
def validate_trace_collection(collection: SweepCollection, *, require_finite: bool = False) -> None:
|
||||
def validate_trace_collection(
|
||||
collection: SweepCollection, *, require_finite: bool = False, require_capture_window: bool = False,
|
||||
) -> None:
|
||||
"""Reject ambiguous combinations and malformed traces before saving any files."""
|
||||
for field in ("collection_id", "monotonic_ns", "capture_start_ns", "capture_end_ns"):
|
||||
_validate_u64(getattr(collection, field), field)
|
||||
if require_capture_window and not 0 < collection.capture_start_ns <= collection.capture_end_ns:
|
||||
raise ValueError("Raw collection requires a positive, ordered capture window")
|
||||
if require_finite and not collection.traces:
|
||||
raise ValueError("Preprocess set must contain at least one trace")
|
||||
seen: set[tuple[int, int]] = set()
|
||||
@@ -43,11 +35,10 @@ def validate_trace_collection(collection: SweepCollection, *, require_finite: bo
|
||||
seen.add(combo)
|
||||
arrays = (
|
||||
np.asarray(trace.frequency_hz, dtype=np.float32),
|
||||
np.asarray(trace.s11, dtype=np.complex64),
|
||||
np.asarray(trace.s21, dtype=np.complex64),
|
||||
)
|
||||
if any(array.ndim != 1 for array in arrays) or len({array.shape for array in arrays}) != 1:
|
||||
raise ValueError(f"Trace {combo} frequency, S11 and S21 must be equally sized 1D arrays")
|
||||
raise ValueError(f"Trace {combo} frequency and S21 must be equally sized 1D arrays")
|
||||
if require_finite and (not arrays[0].size or any(not np.isfinite(array).all() for array in arrays)):
|
||||
raise ValueError(f"Preprocess trace {combo} must contain finite, nonempty arrays")
|
||||
|
||||
@@ -57,28 +48,28 @@ def _validate_u64(value: int, field: str) -> None:
|
||||
raise ValueError(f"{field} must be an unsigned 64-bit integer")
|
||||
|
||||
|
||||
def serialize_trace_collection(collection: SweepCollection, magic: int) -> bytes:
|
||||
def serialize_trace_collection(
|
||||
collection: SweepCollection, magic: int, *, require_capture_window: bool = False,
|
||||
) -> bytes:
|
||||
"""Serialize one raw/preprocessed trace collection into ring-compatible binary format.
|
||||
|
||||
The trailer is appended after the trace blocks so older readers, which stop at
|
||||
the last block, still decode the traces: first the collection capture window,
|
||||
then a per-trace window table (one ``(start_ns, end_ns)`` pair per trace, in
|
||||
trace order). See :func:`python_app.orchestration.shm.decoder.decode_trace_collection`
|
||||
Each trace stores its frequency array followed by its complex S21 array, using
|
||||
12 bytes per point. The trailer contains the collection capture window and a
|
||||
per-trace window table (one ``(start_ns, end_ns)`` pair per trace, in trace
|
||||
order). See :func:`python_app.orchestration.shm.decoder.decode_trace_collection`
|
||||
and ``read_trace_collection`` in ``common_cpp/ipc/src/shared_types.cpp``.
|
||||
"""
|
||||
validate_trace_collection(collection)
|
||||
validate_trace_collection(collection, require_capture_window=require_capture_window)
|
||||
buffer = bytearray()
|
||||
buffer.extend(struct.pack("<IQQI", magic, collection.collection_id, collection.monotonic_ns, len(collection.traces)))
|
||||
|
||||
for trace in collection.traces:
|
||||
freq = np.asarray(trace.frequency_hz, dtype=np.float32)
|
||||
s11 = np.asarray(trace.s11, dtype=np.complex64)
|
||||
s21 = np.asarray(trace.s21, dtype=np.complex64)
|
||||
buffer.extend(struct.pack("<III", trace.combo.input, trace.combo.output, int(freq.size)))
|
||||
buffer.extend(freq.astype("<f4", copy=False).tobytes())
|
||||
|
||||
_write_interleaved_complex(buffer, s11)
|
||||
_write_interleaved_complex(buffer, s21)
|
||||
buffer.extend(s21.astype("<c8", copy=False).tobytes())
|
||||
|
||||
buffer.extend(
|
||||
struct.pack(
|
||||
@@ -93,83 +84,3 @@ def serialize_trace_collection(collection: SweepCollection, magic: int) -> bytes
|
||||
struct.pack("<QQ", int(trace.capture_start_ns), int(trace.capture_end_ns))
|
||||
)
|
||||
return bytes(buffer)
|
||||
|
||||
|
||||
def serialize_result_collection(collection: ResultCollection) -> bytes:
|
||||
"""Serialize one processed collection with result blocks/payloads."""
|
||||
|
||||
def serialize_payload(buffer: bytearray, payload) -> None:
|
||||
"""Append one payload in ring-compatible result format."""
|
||||
name_bytes = payload.processing_name.encode("utf-8")
|
||||
if len(name_bytes) > 0xFFFF:
|
||||
raise ValueError("processing_name is too long")
|
||||
|
||||
buffer.extend(struct.pack("<BH", payload.kind, len(name_bytes)))
|
||||
buffer.extend(name_bytes)
|
||||
|
||||
if payload.kind == 1:
|
||||
freq = np.asarray(payload.frequency_hz, dtype=np.float32)
|
||||
trace = np.asarray(payload.trace, dtype=np.complex64)
|
||||
if freq.ndim != 1 or trace.ndim != 1 or freq.shape != trace.shape:
|
||||
raise ValueError("Result trace frequency and values must be equally sized 1D arrays")
|
||||
|
||||
buffer.extend(struct.pack("<I", int(freq.size)))
|
||||
buffer.extend(freq.astype("<f4", copy=False).tobytes())
|
||||
|
||||
_write_interleaved_complex(buffer, trace)
|
||||
return
|
||||
|
||||
if payload.kind == 2:
|
||||
buffer.extend(struct.pack("<f", float(payload.scalar_value)))
|
||||
return
|
||||
|
||||
if payload.kind == 3:
|
||||
image_x_axis = np.asarray(payload.image_x_axis, dtype=np.float32)
|
||||
image_y_axis = np.asarray(payload.image_y_axis, dtype=np.float32)
|
||||
image = np.asarray(payload.image, dtype=np.float32)
|
||||
if image_x_axis.ndim != 1 or image_y_axis.ndim != 1 or image.shape != (image_y_axis.size, image_x_axis.size):
|
||||
raise ValueError("Result image axis sizes must match image matrix shape")
|
||||
buffer.extend(struct.pack("<II", int(image_x_axis.size), int(image_y_axis.size)))
|
||||
buffer.extend(image_x_axis.astype("<f4", copy=False).tobytes())
|
||||
buffer.extend(image_y_axis.astype("<f4", copy=False).tobytes())
|
||||
buffer.extend(image.astype("<f4", copy=False).ravel(order="C").tobytes())
|
||||
return
|
||||
|
||||
if payload.kind == 4:
|
||||
table = np.asarray(payload.table, dtype=np.float32)
|
||||
if table.ndim != 2:
|
||||
raise ValueError("Result table payload must be a 2D matrix")
|
||||
if table.shape[0] and not table.shape[1]:
|
||||
raise ValueError("Result table with rows must have at least one column")
|
||||
buffer.extend(struct.pack("<II", int(table.shape[1]), int(table.shape[0])))
|
||||
buffer.extend(table.astype("<f4", copy=False).ravel(order="C").tobytes())
|
||||
return
|
||||
|
||||
raise ValueError(f"Unsupported payload kind: {payload.kind}")
|
||||
|
||||
for field in ("collection_id", "monotonic_ns", "processing_duration_ns"):
|
||||
_validate_u64(getattr(collection, field), field)
|
||||
buffer = bytearray()
|
||||
buffer.extend(
|
||||
struct.pack(
|
||||
"<IQQQII",
|
||||
RESULT_MAGIC,
|
||||
collection.collection_id,
|
||||
collection.monotonic_ns,
|
||||
int(collection.processing_duration_ns),
|
||||
len(collection.collection_payloads),
|
||||
len(collection.blocks),
|
||||
)
|
||||
)
|
||||
|
||||
for payload in collection.collection_payloads:
|
||||
serialize_payload(buffer, payload)
|
||||
|
||||
for block in collection.blocks:
|
||||
buffer.extend(struct.pack("<II", block.combo.input, block.combo.output))
|
||||
buffer.extend(struct.pack("<I", len(block.payloads)))
|
||||
|
||||
for payload in block.payloads:
|
||||
serialize_payload(buffer, payload)
|
||||
|
||||
return bytes(buffer)
|
||||
|
||||
@@ -1,20 +1,23 @@
|
||||
"""Snapshot selection and filesystem writers for runtime collection histories."""
|
||||
"""Atomic NumPy collection writers and processing settings for runtime snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from numbers import Integral
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import tempfile
|
||||
from uuid import uuid4
|
||||
from typing import Any, Iterator
|
||||
|
||||
import numpy as np
|
||||
|
||||
from python_app.file_io import atomic_write_json, rename_no_replace
|
||||
from python_app.models.dataset_model import ResultCollection, ResultPayload, SweepCollection
|
||||
from python_app.storage.npz.paths import collection_dir_name, sanitize_path_component
|
||||
from python_app.storage.npz.serialize import serialize_result_collection, serialize_trace_collection, validate_trace_collection
|
||||
from python_app.storage.npz.paths import collection_dir_name
|
||||
from python_app.file_io import sanitize_path_component
|
||||
from python_app.storage.npz.serialize import validate_trace_collection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -25,166 +28,34 @@ def atomic_snapshot_directory(destination: Path) -> Iterator[Path]:
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
if destination.exists():
|
||||
raise FileExistsError(f"Snapshot directory already exists: {destination}")
|
||||
staging = Path(tempfile.mkdtemp(prefix=f".{destination.name}.", dir=destination.parent))
|
||||
staging = destination.with_name(f".{destination.name}.{uuid4().hex}")
|
||||
staging.mkdir()
|
||||
try:
|
||||
yield staging
|
||||
if destination.exists():
|
||||
raise FileExistsError(f"Snapshot directory already exists: {destination}")
|
||||
staging.rename(destination)
|
||||
rename_no_replace(staging, destination)
|
||||
finally:
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging)
|
||||
|
||||
|
||||
def _matching_collection(
|
||||
history: list[SweepCollection], anchor: SweepCollection | ResultCollection,
|
||||
used: set[int], *, same_timestamp: bool,
|
||||
) -> SweepCollection | None:
|
||||
"""Match preprocessing identities exactly and raw sweeps by their capture window.
|
||||
|
||||
Producers stamp raw/preprocessed collections at different instants. Their capture
|
||||
window is copied unchanged; results keep the preprocessed timestamp. Old files
|
||||
without timing fall back to the latest preceding collection with the same id.
|
||||
"""
|
||||
candidates = [
|
||||
(index, item) for index, item in enumerate(history)
|
||||
if index not in used and int(item.collection_id) == int(anchor.collection_id)
|
||||
]
|
||||
timestamp = int(anchor.monotonic_ns)
|
||||
start_ns = int(getattr(anchor, "capture_start_ns", 0))
|
||||
end_ns = int(getattr(anchor, "capture_end_ns", 0))
|
||||
if same_timestamp and timestamp:
|
||||
candidates = [(index, item) for index, item in candidates if int(item.monotonic_ns) == timestamp]
|
||||
elif start_ns or end_ns:
|
||||
candidates = [
|
||||
(index, item) for index, item in candidates
|
||||
if (int(item.capture_start_ns), int(item.capture_end_ns)) == (start_ns, end_ns)
|
||||
]
|
||||
elif timestamp:
|
||||
candidates = [(index, item) for index, item in candidates if int(item.monotonic_ns) <= timestamp]
|
||||
if not candidates:
|
||||
return None
|
||||
index, item = max(candidates, key=lambda pair: (int(pair[1].monotonic_ns), pair[0]))
|
||||
used.add(index)
|
||||
return item
|
||||
|
||||
|
||||
def select_aligned_histories(
|
||||
raw_history: list[SweepCollection],
|
||||
preprocessed_history: list[SweepCollection],
|
||||
result_history: list[ResultCollection],
|
||||
last_n: int,
|
||||
*,
|
||||
include_unaligned: bool = False,
|
||||
) -> tuple[list[SweepCollection], list[SweepCollection], list[ResultCollection], dict[str, Any]]:
|
||||
"""Select aligned tails, or retain every stage independently for complete exports."""
|
||||
if last_n <= 0:
|
||||
raise ValueError("last_n must be > 0")
|
||||
if include_unaligned:
|
||||
return (
|
||||
raw_history[-last_n:], preprocessed_history[-last_n:], result_history[-last_n:],
|
||||
{"selection_mode": "independent_tail", "anchor_stage": "none",
|
||||
"selected_collection_ids": [int(item.collection_id) for item in result_history[-last_n:]],
|
||||
"aligned_key_count": 0},
|
||||
)
|
||||
|
||||
selected_raw: list[SweepCollection] = []
|
||||
selected_pre: list[SweepCollection] = []
|
||||
selected_results = result_history[-last_n:]
|
||||
used_raw: set[int] = set()
|
||||
used_pre: set[int] = set()
|
||||
if selected_results:
|
||||
anchors = selected_results
|
||||
anchor_stage = "results"
|
||||
for result in reversed(selected_results):
|
||||
pre = _matching_collection(preprocessed_history, result, used_pre, same_timestamp=True)
|
||||
raw = _matching_collection(raw_history, pre or result, used_raw, same_timestamp=False)
|
||||
if pre is not None:
|
||||
selected_pre.append(pre)
|
||||
if raw is not None:
|
||||
selected_raw.append(raw)
|
||||
selected_pre.reverse()
|
||||
selected_raw.reverse()
|
||||
elif preprocessed_history:
|
||||
selected_pre = preprocessed_history[-last_n:]
|
||||
anchors = selected_pre
|
||||
anchor_stage = "preprocessed"
|
||||
for pre in reversed(selected_pre):
|
||||
raw = _matching_collection(raw_history, pre, used_raw, same_timestamp=False)
|
||||
if raw is not None:
|
||||
selected_raw.append(raw)
|
||||
selected_raw.reverse()
|
||||
else:
|
||||
selected_raw = raw_history[-last_n:]
|
||||
anchors = selected_raw
|
||||
anchor_stage = "raw" if anchors else "none"
|
||||
|
||||
return selected_raw, selected_pre, selected_results, {
|
||||
"selection_mode": f"{anchor_stage}_tail_with_optional_alignment",
|
||||
"anchor_stage": anchor_stage,
|
||||
"selected_collection_ids": [int(item.collection_id) for item in anchors],
|
||||
"aligned_key_count": len(anchors),
|
||||
"raw_missing_count": len(anchors) - len(selected_raw),
|
||||
"preprocessed_missing_count": len(anchors) - len(selected_pre),
|
||||
"result_missing_count": len(anchors) - len(selected_results),
|
||||
}
|
||||
|
||||
|
||||
def save_trace_history_binary(stage_dir: Path, history: list[SweepCollection], magic: int) -> None:
|
||||
"""Write binary trace history with lightweight metadata sidecars."""
|
||||
stage_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.debug("Writing %d binary trace collection(s) to %s", len(history), stage_dir)
|
||||
for index, collection in enumerate(history):
|
||||
binary_path = stage_dir / f"{index:04d}.bin"
|
||||
metadata_path = stage_dir / f"{index:04d}.json"
|
||||
binary_path.write_bytes(serialize_trace_collection(collection, magic))
|
||||
metadata_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"collection_id": collection.collection_id,
|
||||
"monotonic_ns": collection.monotonic_ns,
|
||||
"capture_start_ns": int(collection.capture_start_ns),
|
||||
"capture_end_ns": int(collection.capture_end_ns),
|
||||
"trace_count": len(collection.traces),
|
||||
# Also in the .bin trailer; repeated here so per-combo timing is
|
||||
# readable without decoding the binary payload.
|
||||
"traces": [
|
||||
{
|
||||
"input": int(trace.combo.input),
|
||||
"output": int(trace.combo.output),
|
||||
"capture_start_ns": int(trace.capture_start_ns),
|
||||
"capture_end_ns": int(trace.capture_end_ns),
|
||||
}
|
||||
for trace in collection.traces
|
||||
],
|
||||
},
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def save_result_history_binary(stage_dir: Path, history: list[ResultCollection]) -> None:
|
||||
"""Write binary processed-result history with metadata sidecars."""
|
||||
stage_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.debug("Writing %d binary result collection(s) to %s", len(history), stage_dir)
|
||||
for index, collection in enumerate(history):
|
||||
binary_path = stage_dir / f"{index:04d}.bin"
|
||||
metadata_path = stage_dir / f"{index:04d}.json"
|
||||
binary_path.write_bytes(serialize_result_collection(collection))
|
||||
metadata_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"collection_id": collection.collection_id,
|
||||
"monotonic_ns": collection.monotonic_ns,
|
||||
"processing_duration_ns": int(collection.processing_duration_ns),
|
||||
"collection_payload_count": len(collection.collection_payloads),
|
||||
"block_count": len(collection.blocks),
|
||||
},
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
def save_processing_settings(directory: Path, settings: dict[int, dict] | None) -> None:
|
||||
"""Merge immutable revision settings before publishing results that refer to them."""
|
||||
if not settings:
|
||||
return
|
||||
path = directory / "processing_settings.json"
|
||||
saved = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
|
||||
if not isinstance(saved, dict):
|
||||
raise ValueError("Saved processing settings must be a JSON object")
|
||||
updated = dict(saved)
|
||||
for revision, value in settings.items():
|
||||
if isinstance(revision, bool) or not isinstance(revision, Integral) or not 0 <= revision < 1 << 64 or not isinstance(value, dict):
|
||||
raise ValueError("Processing settings require uint64 revision keys and JSON objects")
|
||||
key = str(revision)
|
||||
if key in saved and saved[key] != value:
|
||||
raise ValueError(f"Processing revision {revision} already has different settings")
|
||||
updated[key] = value
|
||||
if updated != saved:
|
||||
atomic_write_json(path, updated, durable=True)
|
||||
|
||||
|
||||
def save_trace_history_numpy(
|
||||
@@ -202,47 +73,43 @@ def save_trace_history_numpy(
|
||||
logger.debug("Writing %d NumPy trace collection(s) to %s", len(history), stage_dir)
|
||||
for index, collection in enumerate(history, start=index_offset):
|
||||
collection_dir = stage_dir / collection_dir_name(index, collection.collection_id, collection.monotonic_ns)
|
||||
collection_dir.mkdir(parents=True, exist_ok=False)
|
||||
with atomic_snapshot_directory(collection_dir) as collection_dir:
|
||||
traces_meta: list[dict[str, int | str]] = []
|
||||
for trace in collection.traces:
|
||||
tag = f"i{trace.combo.input}_o{trace.combo.output}"
|
||||
freq = np.asarray(trace.frequency_hz, dtype=np.float32)
|
||||
s21 = np.asarray(trace.s21, dtype=np.complex64)
|
||||
np.save(collection_dir / f"{tag}_freq.npy", freq)
|
||||
np.save(collection_dir / f"{tag}_s21.npy", s21)
|
||||
traces_meta.append(
|
||||
{
|
||||
"input": int(trace.combo.input),
|
||||
"output": int(trace.combo.output),
|
||||
"points": int(freq.size),
|
||||
# When each combo was measured, which in a switched matrix is
|
||||
# spread across the collection window rather than aligned with it.
|
||||
"capture_start_ns": int(trace.capture_start_ns),
|
||||
"capture_end_ns": int(trace.capture_end_ns),
|
||||
"freq_file": f"{tag}_freq.npy",
|
||||
"s21_file": f"{tag}_s21.npy",
|
||||
}
|
||||
)
|
||||
|
||||
traces_meta: list[dict[str, int | str]] = []
|
||||
for trace in collection.traces:
|
||||
tag = f"i{trace.combo.input}_o{trace.combo.output}"
|
||||
freq = np.asarray(trace.frequency_hz, dtype=np.float32)
|
||||
s11 = np.asarray(trace.s11, dtype=np.complex64)
|
||||
s21 = np.asarray(trace.s21, dtype=np.complex64)
|
||||
np.save(collection_dir / f"{tag}_freq.npy", freq)
|
||||
np.save(collection_dir / f"{tag}_s11.npy", s11)
|
||||
np.save(collection_dir / f"{tag}_s21.npy", s21)
|
||||
traces_meta.append(
|
||||
{
|
||||
"input": int(trace.combo.input),
|
||||
"output": int(trace.combo.output),
|
||||
"points": int(freq.size),
|
||||
# When each combo was measured, which in a switched matrix is
|
||||
# spread across the collection window rather than aligned with it.
|
||||
"capture_start_ns": int(trace.capture_start_ns),
|
||||
"capture_end_ns": int(trace.capture_end_ns),
|
||||
"freq_file": f"{tag}_freq.npy",
|
||||
"s11_file": f"{tag}_s11.npy",
|
||||
"s21_file": f"{tag}_s21.npy",
|
||||
}
|
||||
(collection_dir / "meta.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"collection_id": int(collection.collection_id),
|
||||
"monotonic_ns": int(collection.monotonic_ns),
|
||||
"capture_start_ns": int(collection.capture_start_ns),
|
||||
"capture_end_ns": int(collection.capture_end_ns),
|
||||
"trace_count": len(collection.traces),
|
||||
"traces": traces_meta,
|
||||
},
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
(collection_dir / "meta.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"collection_id": int(collection.collection_id),
|
||||
"monotonic_ns": int(collection.monotonic_ns),
|
||||
"capture_start_ns": int(collection.capture_start_ns),
|
||||
"capture_end_ns": int(collection.capture_end_ns),
|
||||
"trace_count": len(collection.traces),
|
||||
"traces": traces_meta,
|
||||
},
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _save_result_payload(directory: Path, base_name: str, payload: ResultPayload) -> dict[str, Any]:
|
||||
"""Persist every supported payload kind identically at collection and combo scope."""
|
||||
@@ -293,35 +160,39 @@ def save_result_history_numpy(
|
||||
logger.debug("Writing %d NumPy result collection(s) to %s", len(history), stage_dir)
|
||||
for index, collection in enumerate(history, start=index_offset):
|
||||
collection_dir = stage_dir / collection_dir_name(index, collection.collection_id, collection.monotonic_ns)
|
||||
collection_dir.mkdir(parents=True, exist_ok=False)
|
||||
collection_payload_meta = []
|
||||
for payload_index, payload in enumerate(collection.collection_payloads):
|
||||
name = sanitize_path_component(payload.processing_name or "processor")
|
||||
base_name = f"collection_{payload_index:03d}_{name}_kind{payload.kind}"
|
||||
collection_payload_meta.append(_save_result_payload(collection_dir, base_name, payload))
|
||||
|
||||
blocks_meta = []
|
||||
for block_index, block in enumerate(collection.blocks):
|
||||
block_dir = collection_dir / f"block_{block_index:03d}_i{block.combo.input}_o{block.combo.output}"
|
||||
block_dir.mkdir(parents=True, exist_ok=False)
|
||||
payload_meta = []
|
||||
for payload_index, payload in enumerate(block.payloads):
|
||||
with atomic_snapshot_directory(collection_dir) as collection_dir:
|
||||
collection_payload_meta = []
|
||||
for payload_index, payload in enumerate(collection.collection_payloads):
|
||||
name = sanitize_path_component(payload.processing_name or "processor")
|
||||
base_name = f"{payload_index:03d}_{name}_kind{payload.kind}"
|
||||
payload_meta.append(_save_result_payload(block_dir, base_name, payload))
|
||||
blocks_meta.append({
|
||||
"input": int(block.combo.input), "output": int(block.combo.output),
|
||||
"payload_count": len(block.payloads), "dir": block_dir.name, "payloads": payload_meta,
|
||||
})
|
||||
base_name = f"collection_{payload_index:03d}_{name}_kind{payload.kind}"
|
||||
collection_payload_meta.append(_save_result_payload(collection_dir, base_name, payload))
|
||||
|
||||
(collection_dir / "meta.json").write_text(
|
||||
json.dumps({
|
||||
"collection_id": int(collection.collection_id),
|
||||
"monotonic_ns": int(collection.monotonic_ns),
|
||||
"processing_duration_ns": int(collection.processing_duration_ns),
|
||||
"collection_payload_count": len(collection.collection_payloads),
|
||||
"collection_payloads": collection_payload_meta,
|
||||
"block_count": len(collection.blocks), "blocks": blocks_meta,
|
||||
}, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
blocks_meta = []
|
||||
for block_index, block in enumerate(collection.blocks):
|
||||
block_dir = collection_dir / f"block_{block_index:03d}_i{block.combo.input}_o{block.combo.output}"
|
||||
block_dir.mkdir(parents=True, exist_ok=False)
|
||||
payload_meta = []
|
||||
for payload_index, payload in enumerate(block.payloads):
|
||||
name = sanitize_path_component(payload.processing_name or "processor")
|
||||
base_name = f"{payload_index:03d}_{name}_kind{payload.kind}"
|
||||
payload_meta.append(_save_result_payload(block_dir, base_name, payload))
|
||||
blocks_meta.append({
|
||||
"input": int(block.combo.input), "output": int(block.combo.output),
|
||||
"payload_count": len(block.payloads), "dir": block_dir.name, "payloads": payload_meta,
|
||||
})
|
||||
|
||||
(collection_dir / "meta.json").write_text(
|
||||
json.dumps({
|
||||
"collection_id": int(collection.collection_id),
|
||||
"monotonic_ns": int(collection.monotonic_ns),
|
||||
"processing_duration_ns": int(collection.processing_duration_ns),
|
||||
"settings_revision": int(collection.settings_revision),
|
||||
"capture_start_ns": int(collection.capture_start_ns),
|
||||
"capture_end_ns": int(collection.capture_end_ns),
|
||||
"speed_m_s": float(collection.speed_m_s),
|
||||
"collection_payload_count": len(collection.collection_payloads),
|
||||
"collection_payloads": collection_payload_meta,
|
||||
"block_count": len(collection.blocks), "blocks": blocks_meta,
|
||||
}, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
+109
-315
@@ -1,4 +1,4 @@
|
||||
"""Concrete :class:`StoreApi` implementation backed by NPZ files."""
|
||||
"""Preprocess sets and datasets backed by NPZ files."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -9,26 +9,24 @@ import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import tempfile
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
from zipfile import BadZipFile
|
||||
|
||||
import numpy as np
|
||||
|
||||
from python_app.file_io import make_directory, rename_no_replace, sync_directory, temporary_file
|
||||
from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection, TraceData
|
||||
from python_app.models.run_config_model import RunConfigModel
|
||||
from python_app.storage.npz.paths import radar_key_from_config, sanitize_path_component
|
||||
from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, serialize_trace_collection, validate_trace_collection
|
||||
from python_app.file_io import sanitize_path_component
|
||||
from python_app.orchestration.shm.decoder import RAW_MAGIC
|
||||
from python_app.storage.npz.serialize import serialize_trace_collection, validate_trace_collection
|
||||
from python_app.storage.npz.snapshot_numpy import (
|
||||
save_result_history_binary,
|
||||
save_result_history_numpy,
|
||||
save_trace_history_binary,
|
||||
save_trace_history_numpy,
|
||||
select_aligned_histories,
|
||||
save_processing_settings,
|
||||
atomic_snapshot_directory,
|
||||
)
|
||||
from python_app.storage.npz.vna_history_json import build_vna_history_payload
|
||||
from python_app.storage.store_api import StoreApi
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_SET_METADATA_KEY = "metadata_json"
|
||||
@@ -44,22 +42,11 @@ def _compose_snapshot_stem(name: str, name_prefix: str) -> str:
|
||||
|
||||
|
||||
class SnapshotStreamWriter:
|
||||
"""Append received raw/preprocessed/result collections to a snapshot dir in chunks.
|
||||
|
||||
A streaming alternative to :meth:`NpzStore.save_runtime_snapshot_numpy` for long
|
||||
recordings: the caller flushes small chunks as measurements arrive and frees them,
|
||||
so memory stays flat instead of buffering every measurement. The on-disk layout is
|
||||
identical (``raw``/``preprocessed``/``results`` subtrees), and per-collection
|
||||
directory indices continue across chunks via a running offset per stage.
|
||||
"""
|
||||
|
||||
__slots__ = ("_dir", "_raw_written", "_preprocessed_written", "_result_written")
|
||||
"""Publish complete collections incrementally, retaining them if a later write fails."""
|
||||
|
||||
def __init__(self, snapshot_dir: Path) -> None:
|
||||
self._dir = snapshot_dir
|
||||
self._raw_written = 0
|
||||
self._preprocessed_written = 0
|
||||
self._result_written = 0
|
||||
self._counts = {"raw": 0, "preprocessed": 0, "results": 0}
|
||||
|
||||
@property
|
||||
def directory(self) -> Path:
|
||||
@@ -67,66 +54,50 @@ class SnapshotStreamWriter:
|
||||
|
||||
@property
|
||||
def result_count(self) -> int:
|
||||
"""Number of result collections written to disk so far."""
|
||||
return self._result_written
|
||||
return self._counts["results"]
|
||||
|
||||
def append(
|
||||
self,
|
||||
raw: list[SweepCollection],
|
||||
preprocessed: list[SweepCollection],
|
||||
results: list[ResultCollection],
|
||||
*, processing_settings: dict[int, dict] | None = None,
|
||||
) -> None:
|
||||
"""Write a chunk without requiring equal stage lengths, continuing each index."""
|
||||
# Stage a whole chunk before publishing complete collection directories. A
|
||||
# failed write leaves earlier chunks intact and the next retry uses the same indices.
|
||||
chunk_dir = Path(tempfile.mkdtemp(prefix=".chunk-", dir=self._dir))
|
||||
published: list[Path] = []
|
||||
try:
|
||||
save_trace_history_numpy(chunk_dir / "raw", raw, index_offset=self._raw_written)
|
||||
save_trace_history_numpy(
|
||||
chunk_dir / "preprocessed", preprocessed, index_offset=self._preprocessed_written
|
||||
)
|
||||
save_result_history_numpy(chunk_dir / "results", results, index_offset=self._result_written)
|
||||
for stage in ("raw", "preprocessed", "results"):
|
||||
stage_dir = self._dir / stage
|
||||
stage_dir.mkdir(exist_ok=True)
|
||||
for source in sorted((chunk_dir / stage).iterdir()):
|
||||
destination = stage_dir / source.name
|
||||
if destination.exists():
|
||||
raise FileExistsError(f"Collection directory already exists: {destination}")
|
||||
source.rename(destination)
|
||||
published.append(destination)
|
||||
except BaseException:
|
||||
for destination in published:
|
||||
shutil.rmtree(destination)
|
||||
raise
|
||||
finally:
|
||||
shutil.rmtree(chunk_dir)
|
||||
self._raw_written += len(raw)
|
||||
self._preprocessed_written += len(preprocessed)
|
||||
self._result_written += len(results)
|
||||
"""Append a chunk; after failure, only the unwritten collections may be retried."""
|
||||
save_processing_settings(self._dir, processing_settings)
|
||||
for stage, history in (("raw", raw), ("preprocessed", preprocessed)):
|
||||
stage_dir = self._dir / stage
|
||||
stage_dir.mkdir(exist_ok=True)
|
||||
for collection in history:
|
||||
save_trace_history_numpy(stage_dir, [collection], index_offset=self._counts[stage])
|
||||
self._counts[stage] += 1
|
||||
stage_dir = self._dir / "results"
|
||||
stage_dir.mkdir(exist_ok=True)
|
||||
for result in results:
|
||||
save_result_history_numpy(stage_dir, [result], index_offset=self.result_count)
|
||||
self._counts["results"] += 1
|
||||
|
||||
|
||||
class NpzStore(StoreApi):
|
||||
class NpzStore:
|
||||
"""Persist preprocess sets and runtime snapshots using NumPy files."""
|
||||
|
||||
def __init__(self, root_dir: Path) -> None:
|
||||
"""Create store rooted at `root_dir`."""
|
||||
self._root_dir = root_dir
|
||||
self._root_dir.mkdir(parents=True, exist_ok=True)
|
||||
make_directory(self._root_dir, durable=True)
|
||||
logger.debug("NpzStore rooted at %s", self._root_dir)
|
||||
|
||||
@staticmethod
|
||||
def _vna_json_output_dir(output_root_dir: Path, output_stem: str) -> Path:
|
||||
"""Return per-export directory for VNA history JSON files."""
|
||||
return output_root_dir / output_stem
|
||||
|
||||
def save_set(self, kind: str, radar_key: str, set_name: str, collection: SweepCollection) -> None:
|
||||
"""Persist named preprocess set as NPZ and metadata JSON."""
|
||||
def save_set(
|
||||
self, kind: str, radar_key: str, set_name: str, collection: SweepCollection,
|
||||
*, metadata: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""Persist a named set with optional JSON metadata embedded in the NPZ."""
|
||||
self._validate_set_name(set_name)
|
||||
validate_trace_collection(collection, require_finite=True)
|
||||
if metadata is not None and not isinstance(metadata, dict):
|
||||
raise ValueError("Set user metadata must be a JSON object")
|
||||
set_dir = self._set_dir(kind, radar_key)
|
||||
set_dir.mkdir(parents=True, exist_ok=True)
|
||||
make_directory(set_dir, durable=True)
|
||||
|
||||
npz_path = set_dir / f"{set_name}.npz"
|
||||
meta_path = set_dir / f"{set_name}.json"
|
||||
@@ -137,10 +108,8 @@ class NpzStore(StoreApi):
|
||||
for trace in collection.traces:
|
||||
suffix = f"i{trace.combo.input}_o{trace.combo.output}"
|
||||
freq_key = f"freq_{suffix}"
|
||||
s11_key = f"s11_{suffix}"
|
||||
s21_key = f"s21_{suffix}"
|
||||
payload[freq_key] = np.asarray(trace.frequency_hz, dtype=np.float32)
|
||||
payload[s11_key] = np.asarray(trace.s11, dtype=np.complex64)
|
||||
payload[s21_key] = np.asarray(trace.s21, dtype=np.complex64)
|
||||
combo_records.append(
|
||||
{
|
||||
@@ -149,7 +118,6 @@ class NpzStore(StoreApi):
|
||||
"capture_start_ns": int(trace.capture_start_ns),
|
||||
"capture_end_ns": int(trace.capture_end_ns),
|
||||
"freq_key": freq_key,
|
||||
"s11_key": s11_key,
|
||||
"s21_key": s21_key,
|
||||
}
|
||||
)
|
||||
@@ -160,24 +128,30 @@ class NpzStore(StoreApi):
|
||||
"capture_start_ns": int(collection.capture_start_ns),
|
||||
"capture_end_ns": int(collection.capture_end_ns),
|
||||
"combos": combo_records,
|
||||
"metadata": {} if metadata is None else metadata,
|
||||
}
|
||||
|
||||
# Metadata in the NPZ makes a single atomic replacement authoritative. The
|
||||
# JSON remains a readable sidecar and legacy NPZ/JSON pairs remain loadable.
|
||||
metadata_json = json.dumps(meta, indent=2)
|
||||
metadata_json = json.dumps(meta, indent=2, allow_nan=False)
|
||||
payload[_SET_METADATA_KEY] = np.asarray(metadata_json)
|
||||
npz_tmp = meta_tmp = backup_path = None
|
||||
archive_replaced = False
|
||||
try:
|
||||
npz_tmp = self._temporary_file(npz_path)
|
||||
meta_tmp = self._temporary_file(meta_path)
|
||||
backup_path = self._temporary_file(npz_path) if npz_path.exists() else None
|
||||
npz_tmp = temporary_file(npz_path)
|
||||
meta_tmp = temporary_file(meta_path)
|
||||
backup_path = temporary_file(npz_path) if npz_path.exists() else None
|
||||
if backup_path is not None:
|
||||
backup_path.unlink()
|
||||
os.link(npz_path, backup_path)
|
||||
with npz_tmp.open("wb") as npz_file:
|
||||
np.savez(npz_file, **payload)
|
||||
meta_tmp.write_text(metadata_json, encoding="utf-8")
|
||||
npz_file.flush()
|
||||
os.fsync(npz_file.fileno())
|
||||
with meta_tmp.open("w", encoding="utf-8") as meta_file:
|
||||
meta_file.write(metadata_json)
|
||||
meta_file.flush()
|
||||
os.fsync(meta_file.fileno())
|
||||
npz_tmp.replace(npz_path)
|
||||
archive_replaced = True
|
||||
meta_tmp.replace(meta_path)
|
||||
@@ -193,6 +167,7 @@ class NpzStore(StoreApi):
|
||||
logger.exception("Set rollback failed; previous archive retained at %s", backup_path)
|
||||
backup_path = None # retain the recovery copy rather than deleting it below
|
||||
raise
|
||||
sync_directory(set_dir)
|
||||
raise
|
||||
finally:
|
||||
for tmp_path in (npz_tmp, meta_tmp, backup_path):
|
||||
@@ -200,6 +175,7 @@ class NpzStore(StoreApi):
|
||||
continue
|
||||
with suppress(OSError):
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
sync_directory(set_dir)
|
||||
logger.info(
|
||||
"Saved set %s/%s/%s (%d traces) to %s", kind, radar_key, set_name, len(combo_records), npz_path
|
||||
)
|
||||
@@ -209,7 +185,8 @@ class NpzStore(StoreApi):
|
||||
self._validate_set_name(set_name)
|
||||
for collection in collections.values():
|
||||
validate_trace_collection(collection, require_finite=True)
|
||||
backup_dir = Path(tempfile.mkdtemp(prefix=".sets-", dir=self._root_dir))
|
||||
backup_dir = self._root_dir / f".sets-{uuid4().hex}"
|
||||
backup_dir.mkdir()
|
||||
backups: dict[Path, Path | None] = {}
|
||||
retain_backups = False
|
||||
try:
|
||||
@@ -236,6 +213,12 @@ class NpzStore(StoreApi):
|
||||
except OSError:
|
||||
rollback_failed = True
|
||||
logger.exception("Failed to restore previous set file %s", destination)
|
||||
for directory in {path.parent for path in backups if path.parent.exists()}:
|
||||
try:
|
||||
sync_directory(directory)
|
||||
except OSError:
|
||||
rollback_failed = True
|
||||
logger.exception("Failed to sync restored set directory %s", directory)
|
||||
retain_backups = rollback_failed
|
||||
if retain_backups:
|
||||
raise RuntimeError(
|
||||
@@ -250,6 +233,12 @@ class NpzStore(StoreApi):
|
||||
|
||||
def load_set(self, kind: str, radar_key: str, set_name: str) -> SweepCollection:
|
||||
"""Load named preprocess set from NPZ representation."""
|
||||
return self.load_set_with_metadata(kind, radar_key, set_name)[0]
|
||||
|
||||
def load_set_with_metadata(
|
||||
self, kind: str, radar_key: str, set_name: str,
|
||||
) -> tuple[SweepCollection, dict[str, Any]]:
|
||||
"""Load a named set and its user metadata; older sets have empty metadata."""
|
||||
self._validate_set_name(set_name)
|
||||
set_dir = self._set_dir(kind, radar_key)
|
||||
npz_path = set_dir / f"{set_name}.npz"
|
||||
@@ -269,13 +258,16 @@ class NpzStore(StoreApi):
|
||||
)
|
||||
if not isinstance(meta, dict):
|
||||
raise ValueError("Set metadata must be a JSON object")
|
||||
metadata = meta.get("metadata", {})
|
||||
if not isinstance(metadata, dict):
|
||||
raise ValueError("Set user metadata must be a JSON object")
|
||||
json.dumps(metadata, allow_nan=False)
|
||||
traces: list[TraceData] = []
|
||||
for combo in meta["combos"]:
|
||||
traces.append(
|
||||
TraceData(
|
||||
combo=ComboKey(input=combo["input"], output=combo["output"]),
|
||||
frequency_hz=np.asarray(arrays[combo["freq_key"]], dtype=np.float32),
|
||||
s11=np.asarray(arrays[combo["s11_key"]], dtype=np.complex64),
|
||||
s21=np.asarray(arrays[combo["s21_key"]], dtype=np.complex64),
|
||||
capture_start_ns=combo.get("capture_start_ns", 0),
|
||||
capture_end_ns=combo.get("capture_end_ns", 0),
|
||||
@@ -291,7 +283,7 @@ class NpzStore(StoreApi):
|
||||
)
|
||||
validate_trace_collection(collection, require_finite=True)
|
||||
logger.debug("Loaded set %s (%d traces)", set_label, len(traces))
|
||||
return collection
|
||||
return collection, metadata
|
||||
except (OSError, BadZipFile, EOFError, KeyError, TypeError, ValueError) as exc:
|
||||
logger.exception("Corrupted preprocess set %s", set_label)
|
||||
raise RuntimeError(f"Corrupted preprocess set {set_label}: {exc}") from exc
|
||||
@@ -301,34 +293,10 @@ class NpzStore(StoreApi):
|
||||
set_dir = self._set_dir(kind, radar_key)
|
||||
if not set_dir.exists():
|
||||
return []
|
||||
if kind == "s21_object_calibration":
|
||||
return sorted(path.name for path in set_dir.iterdir()
|
||||
if not path.name.startswith(".") and (path / "cal_meta.json").is_file())
|
||||
return sorted(path.stem for path in set_dir.glob("*.npz"))
|
||||
|
||||
def save_object_calibration(self, radar_key: str, set_name: str, calibration: dict[str, Any]) -> None:
|
||||
"""Save an object calibration in its shared archive format without overwriting a set."""
|
||||
from python_app.workflows.mimo_calibration import save_calibration
|
||||
|
||||
self._validate_set_name(set_name)
|
||||
destination = self._set_dir("s21_object_calibration", radar_key) / set_name
|
||||
if destination.exists():
|
||||
raise FileExistsError(f"Object calibration already exists: {set_name}")
|
||||
save_calibration(calibration, destination)
|
||||
|
||||
def load_object_calibration(self, radar_key: str, set_name: str) -> dict[str, Any]:
|
||||
"""Load and validate a saved object calibration, retaining all estimation metadata."""
|
||||
from python_app.workflows.mimo_calibration import load_calibration
|
||||
|
||||
self._validate_set_name(set_name)
|
||||
return load_calibration(self._set_dir("s21_object_calibration", radar_key) / set_name)
|
||||
|
||||
def has_combo_coverage(self, kind: str, radar_key: str, set_name: str, combos: list[ComboKey]) -> bool:
|
||||
"""Validate that named set covers all required switch combinations."""
|
||||
if kind == "s21_object_calibration":
|
||||
calibration = self.load_object_calibration(radar_key, set_name)
|
||||
return all(combo.input in calibration["rx_input_pos"] and combo.output in calibration["tx_output_pos"]
|
||||
for combo in combos)
|
||||
collection = self.load_set(kind, radar_key, set_name)
|
||||
existing = {(trace.combo.input, trace.combo.output) for trace in collection.traces}
|
||||
required = {(combo.input, combo.output) for combo in combos}
|
||||
@@ -339,36 +307,11 @@ class NpzStore(StoreApi):
|
||||
collection = self.load_set(kind, radar_key, set_name)
|
||||
return self._export_collection_bundle(collection, output_path)
|
||||
|
||||
def export_object_calibration_bundle(
|
||||
self, radar_key: str, set_name: str, output_path: Path, config: RunConfigModel,
|
||||
) -> Path:
|
||||
"""Resample pair multipliers on the through-calibration grids used by the acquisition."""
|
||||
from python_app.workflows.mimo_calibration import calibration_on_grid, validate_for_config
|
||||
|
||||
calibration = self.load_object_calibration(radar_key, set_name)
|
||||
validate_for_config(calibration, config.to_dict())
|
||||
tx_indices = {int(port): index for index, port in enumerate(calibration["tx_output_pos"])}
|
||||
rx_indices = {int(port): index for index, port in enumerate(calibration["rx_input_pos"])}
|
||||
source = self.load_set("s21_calibration", radar_key, config.preprocess.s21.calibration.set_name)
|
||||
by_combo = {trace.combo: trace for trace in source.traces}
|
||||
grids: dict[bytes, np.ndarray] = {}
|
||||
traces: list[TraceData] = []
|
||||
for combo in config.combos:
|
||||
key = ComboKey(input=combo.input, output=combo.output)
|
||||
trace = by_combo[key]
|
||||
frequency = trace.frequency_hz
|
||||
grid_key = frequency.tobytes()
|
||||
if grid_key not in grids:
|
||||
grids[grid_key] = calibration_on_grid(calibration, frequency)
|
||||
correction = grids[grid_key][tx_indices[key.output], rx_indices[key.input]]
|
||||
traces.append(TraceData(key, frequency, np.zeros(frequency.size, np.complex64), correction))
|
||||
return self._export_collection_bundle(SweepCollection(0, 0, traces), output_path)
|
||||
|
||||
def _export_collection_bundle(self, collection: SweepCollection, output_path: Path) -> Path:
|
||||
"""Publish a complete binary trace bundle using one atomic replacement."""
|
||||
validate_trace_collection(collection, require_finite=True)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = self._temporary_file(output_path)
|
||||
temporary = temporary_file(output_path)
|
||||
try:
|
||||
temporary.write_bytes(serialize_trace_collection(collection, RAW_MAGIC))
|
||||
temporary.replace(output_path)
|
||||
@@ -377,28 +320,6 @@ class NpzStore(StoreApi):
|
||||
logger.info("Exported %d traces to bundle %s", len(collection.traces), output_path)
|
||||
return output_path
|
||||
|
||||
def save_runtime_snapshot(
|
||||
self,
|
||||
output_dir: Path,
|
||||
raw_history: list[SweepCollection],
|
||||
preprocessed_history: list[SweepCollection],
|
||||
result_history: list[ResultCollection],
|
||||
last_n: int,
|
||||
) -> Path:
|
||||
"""Save historical runtime collections using binary on-disk format."""
|
||||
if last_n <= 0:
|
||||
raise ValueError("last_n must be > 0")
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
|
||||
snapshot_dir = output_dir / f"snapshot_{timestamp}"
|
||||
with atomic_snapshot_directory(snapshot_dir) as staging:
|
||||
save_trace_history_binary(staging / "raw", raw_history[-last_n:], RAW_MAGIC)
|
||||
save_trace_history_binary(staging / "preprocessed", preprocessed_history[-last_n:], PREPROC_MAGIC)
|
||||
save_result_history_binary(staging / "results", result_history[-last_n:])
|
||||
logger.info("Saved binary runtime snapshot (last_n=%d) to %s", last_n, snapshot_dir)
|
||||
return snapshot_dir
|
||||
|
||||
def snapshot_directory(
|
||||
self, output_root_dir: Path, snapshot_name: str, *, name_prefix: str = ""
|
||||
) -> Path:
|
||||
@@ -435,126 +356,19 @@ class NpzStore(StoreApi):
|
||||
raw_history: list[SweepCollection],
|
||||
preprocessed_history: list[SweepCollection],
|
||||
result_history: list[ResultCollection],
|
||||
last_n: int,
|
||||
*,
|
||||
name_prefix: str = "",
|
||||
include_unaligned: bool = False,
|
||||
) -> tuple[Path, dict[str, Any]]:
|
||||
"""Save historical runtime collections in NumPy tree format."""
|
||||
if last_n <= 0:
|
||||
raise ValueError("last_n must be > 0")
|
||||
|
||||
snapshot_stem = _compose_snapshot_stem(snapshot_name, name_prefix)
|
||||
|
||||
output_root_dir.mkdir(parents=True, exist_ok=True)
|
||||
snapshot_dir = output_root_dir / snapshot_stem
|
||||
if snapshot_dir.exists():
|
||||
raise FileExistsError(f"Snapshot directory already exists: {snapshot_dir}")
|
||||
|
||||
selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories(
|
||||
raw_history,
|
||||
preprocessed_history,
|
||||
result_history,
|
||||
last_n,
|
||||
include_unaligned=include_unaligned,
|
||||
)
|
||||
|
||||
if selection_summary.get("anchor_stage") == "results":
|
||||
expected = min(int(last_n), len(result_history))
|
||||
if len(selected_results) != expected:
|
||||
raise RuntimeError(
|
||||
"Snapshot results selection invariant failed: "
|
||||
f"expected={expected}, selected={len(selected_results)}"
|
||||
)
|
||||
|
||||
processing_settings: dict[int, dict] | None = None,
|
||||
) -> Path:
|
||||
"""Save exactly the supplied collections and their processing revisions."""
|
||||
snapshot_dir = self.snapshot_directory(output_root_dir, snapshot_name, name_prefix=name_prefix)
|
||||
with atomic_snapshot_directory(snapshot_dir) as staging:
|
||||
save_trace_history_numpy(staging / "raw", selected_raw)
|
||||
save_trace_history_numpy(staging / "preprocessed", selected_preprocessed)
|
||||
save_result_history_numpy(staging / "results", selected_results)
|
||||
|
||||
selection_summary["raw_count"] = len(selected_raw)
|
||||
selection_summary["preprocessed_count"] = len(selected_preprocessed)
|
||||
selection_summary["result_count"] = len(selected_results)
|
||||
selection_summary["snapshot_stem"] = snapshot_stem
|
||||
selection_summary["snapshot_dir"] = str(snapshot_dir)
|
||||
logger.info(
|
||||
"Saved NumPy runtime snapshot to %s (raw=%d preprocessed=%d results=%d, anchor=%s)",
|
||||
snapshot_dir,
|
||||
len(selected_raw),
|
||||
len(selected_preprocessed),
|
||||
len(selected_results),
|
||||
selection_summary.get("anchor_stage"),
|
||||
)
|
||||
return snapshot_dir, selection_summary
|
||||
|
||||
def save_runtime_vna_history_json(
|
||||
self,
|
||||
output_root_dir: Path,
|
||||
output_name: str,
|
||||
raw_history: list[SweepCollection],
|
||||
preprocessed_history: list[SweepCollection],
|
||||
result_history: list[ResultCollection],
|
||||
last_n: int,
|
||||
*,
|
||||
input_index: int = 0,
|
||||
output_index: int = 0,
|
||||
channel: str = "s21",
|
||||
primary_stage: str = "preprocessed",
|
||||
include_unaligned: bool = False,
|
||||
) -> tuple[Path, dict[str, Any]]:
|
||||
"""Save runtime history as vna_system-compatible JSON file."""
|
||||
if last_n <= 0:
|
||||
raise ValueError("last_n must be > 0")
|
||||
|
||||
output_stem = output_name.strip() or datetime.now(timezone.utc).strftime("snapshot_%Y%m%d_%H%M%S")
|
||||
output_stem = sanitize_path_component(output_stem)
|
||||
|
||||
output_root_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_dir = self._vna_json_output_dir(output_root_dir, output_stem)
|
||||
output_path = output_dir / f"{output_stem}_vna_bscan_history.json"
|
||||
|
||||
selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories(
|
||||
raw_history,
|
||||
preprocessed_history,
|
||||
result_history,
|
||||
last_n,
|
||||
include_unaligned=include_unaligned,
|
||||
)
|
||||
payload = build_vna_history_payload(
|
||||
selected_raw,
|
||||
selected_preprocessed,
|
||||
selected_results,
|
||||
input_index=input_index,
|
||||
output_index=output_index,
|
||||
channel=channel,
|
||||
primary_stage=primary_stage,
|
||||
)
|
||||
with atomic_snapshot_directory(output_dir) as staging:
|
||||
(staging / output_path.name).write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
summary: dict[str, Any] = dict(selection_summary)
|
||||
summary["raw_count"] = len(selected_raw)
|
||||
summary["preprocessed_count"] = len(selected_preprocessed)
|
||||
summary["result_count"] = len(selected_results)
|
||||
summary["raw_record_count"] = int(payload.get("raw_record_count", 0))
|
||||
summary["preprocessed_record_count"] = int(payload.get("preprocessed_record_count", 0))
|
||||
summary["sweep_count"] = len(payload.get("sweep_history", []))
|
||||
summary["input_index"] = int(input_index)
|
||||
summary["output_index"] = int(output_index)
|
||||
summary["channel"] = str(channel)
|
||||
summary["primary_stage"] = str(primary_stage)
|
||||
summary["output_stem"] = output_stem
|
||||
summary["output_dir"] = str(output_dir)
|
||||
summary["output_path"] = str(output_path)
|
||||
logger.info(
|
||||
"Saved VNA history JSON to %s (input=%d output=%d channel=%s sweeps=%d)",
|
||||
output_path,
|
||||
int(input_index),
|
||||
int(output_index),
|
||||
channel,
|
||||
summary["sweep_count"],
|
||||
)
|
||||
return output_path, summary
|
||||
save_processing_settings(staging, processing_settings)
|
||||
save_trace_history_numpy(staging / "raw", raw_history)
|
||||
save_trace_history_numpy(staging / "preprocessed", preprocessed_history)
|
||||
save_result_history_numpy(staging / "results", result_history)
|
||||
logger.info("Saved NumPy runtime snapshot to %s", snapshot_dir)
|
||||
return snapshot_dir
|
||||
|
||||
def save_runtime_vna_history_json_batch(
|
||||
self,
|
||||
@@ -563,33 +377,19 @@ class NpzStore(StoreApi):
|
||||
raw_history: list[SweepCollection],
|
||||
preprocessed_history: list[SweepCollection],
|
||||
result_history: list[ResultCollection],
|
||||
last_n: int,
|
||||
*,
|
||||
channel: str = "s21",
|
||||
primary_stage: str = "preprocessed",
|
||||
name_prefix: str = "",
|
||||
include_unaligned: bool = False,
|
||||
) -> tuple[list[Path], dict[str, Any]]:
|
||||
) -> list[Path]:
|
||||
"""Save one runtime VNA-history JSON per available combo."""
|
||||
if last_n <= 0:
|
||||
raise ValueError("last_n must be > 0")
|
||||
|
||||
output_stem = _compose_snapshot_stem(output_name, name_prefix)
|
||||
output_root_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_dir = self._vna_json_output_dir(output_root_dir, output_stem)
|
||||
|
||||
selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories(
|
||||
raw_history,
|
||||
preprocessed_history,
|
||||
result_history,
|
||||
last_n,
|
||||
include_unaligned=include_unaligned,
|
||||
)
|
||||
output_dir = output_root_dir / output_stem
|
||||
|
||||
combos = sorted(
|
||||
{
|
||||
(int(trace.combo.input), int(trace.combo.output))
|
||||
for collection in [*selected_raw, *selected_preprocessed]
|
||||
for collection in [*raw_history, *preprocessed_history]
|
||||
for trace in collection.traces
|
||||
}
|
||||
)
|
||||
@@ -601,45 +401,51 @@ class NpzStore(StoreApi):
|
||||
payloads: list[dict[str, Any]] = []
|
||||
for input_index, output_index in combos:
|
||||
output_path = output_dir / (
|
||||
f"{output_stem}_i{input_index}_o{output_index}_{channel}_vna_bscan_history.json"
|
||||
f"{output_stem}_i{input_index}_o{output_index}_s21_vna_bscan_history.json"
|
||||
)
|
||||
if output_path.exists():
|
||||
raise FileExistsError(f"Output JSON file already exists: {output_path}")
|
||||
payloads.append(
|
||||
build_vna_history_payload(
|
||||
selected_raw,
|
||||
selected_preprocessed,
|
||||
selected_results,
|
||||
raw_history,
|
||||
preprocessed_history,
|
||||
result_history,
|
||||
input_index=input_index,
|
||||
output_index=output_index,
|
||||
channel=channel,
|
||||
primary_stage=primary_stage,
|
||||
)
|
||||
)
|
||||
output_paths.append(output_path)
|
||||
|
||||
with atomic_snapshot_directory(output_dir) as staging:
|
||||
output_existed = output_dir.exists()
|
||||
output_dir.mkdir(exist_ok=True)
|
||||
published: list[Path] = []
|
||||
try:
|
||||
# Publish complete files without overwriting a concurrent export.
|
||||
for output_path, payload in zip(output_paths, payloads, strict=True):
|
||||
(staging / output_path.name).write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
temporary = temporary_file(output_path)
|
||||
try:
|
||||
with temporary.open("w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=False, indent=2, allow_nan=False)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
rename_no_replace(temporary, output_path)
|
||||
published.append(output_path)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
sync_directory(output_dir)
|
||||
except BaseException:
|
||||
for path in published:
|
||||
path.unlink()
|
||||
if not output_existed:
|
||||
with suppress(OSError):
|
||||
output_dir.rmdir()
|
||||
raise
|
||||
|
||||
summary: dict[str, Any] = dict(selection_summary)
|
||||
summary["raw_count"] = len(selected_raw)
|
||||
summary["preprocessed_count"] = len(selected_preprocessed)
|
||||
summary["result_count"] = len(selected_results)
|
||||
summary["preprocessed_record_count"] = int(sum(int(payload.get("preprocessed_record_count", 0)) for payload in payloads))
|
||||
summary["combo_count"] = len(combos)
|
||||
summary["combos"] = [list(combo) for combo in combos]
|
||||
summary["channel"] = str(channel)
|
||||
summary["primary_stage"] = str(primary_stage)
|
||||
summary["output_stem"] = output_stem
|
||||
summary["output_dir"] = str(output_dir)
|
||||
summary["output_paths"] = [str(path) for path in output_paths]
|
||||
logger.info(
|
||||
"Saved %d VNA history JSON file(s) to %s (channel=%s)", len(output_paths), output_dir, channel
|
||||
"Saved %d S21 VNA history JSON file(s) to %s", len(output_paths), output_dir
|
||||
)
|
||||
return output_paths, summary
|
||||
return output_paths
|
||||
|
||||
def _set_dir(self, kind: str, radar_key: str) -> Path:
|
||||
"""Return directory for set kind and radar key."""
|
||||
@@ -653,15 +459,6 @@ class NpzStore(StoreApi):
|
||||
if not value or value in {".", ".."} or Path(value).name != value or "\\" in value:
|
||||
raise ValueError(f"Invalid set path component: {value!r}")
|
||||
|
||||
@staticmethod
|
||||
def _temporary_file(destination: Path) -> Path:
|
||||
"""Create a unique temporary file beside its atomic-replacement destination."""
|
||||
with tempfile.NamedTemporaryFile(
|
||||
dir=destination.parent, prefix=f".{destination.name}.", suffix=".tmp", delete=False
|
||||
) as handle:
|
||||
path = Path(handle.name)
|
||||
return path
|
||||
|
||||
def preview_png_dir(self, kind: str, set_name: str, radar_key: str) -> Path:
|
||||
"""Return directory for preview PNGs of one set/radar variant.
|
||||
|
||||
@@ -671,6 +468,3 @@ class NpzStore(StoreApi):
|
||||
filesystem-safe by construction.
|
||||
"""
|
||||
return self._root_dir / "preview_png" / kind / sanitize_path_component(set_name) / radar_key
|
||||
|
||||
|
||||
__all__ = ["NpzStore", "radar_key_from_config"]
|
||||
|
||||
@@ -5,11 +5,14 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import logging
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from python_app.models.dataset_model import ResultCollection, SweepCollection, TraceData
|
||||
from python_app.models.collection_identity import index_collections
|
||||
from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection, TraceData
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -18,30 +21,15 @@ logger = logging.getLogger(__name__)
|
||||
class TraceRecord:
|
||||
"""One raw/preprocessed trace selected for vna_history export."""
|
||||
|
||||
stage: str
|
||||
collection_id: int
|
||||
monotonic_ns: int
|
||||
stage_index: int
|
||||
frequency_hz: np.ndarray
|
||||
samples: np.ndarray
|
||||
# End of this trace's own sweep, or 0 when the producer reported no per-trace
|
||||
# timing. Preferred over the collection timestamp for the exported sweep time:
|
||||
# in a switched matrix each combo is measured at a different instant.
|
||||
# Collection window identifies the acquisition across pipeline stages.
|
||||
capture_end_ns: int = 0
|
||||
capture_start_ns: int = 0
|
||||
|
||||
|
||||
def _normalize_channel(channel: str) -> str:
|
||||
normalized = str(channel).strip().lower()
|
||||
if normalized not in {"s21", "s11"}:
|
||||
raise ValueError("channel must be either 's21' or 's11'")
|
||||
return normalized
|
||||
|
||||
|
||||
def _select_trace_samples(trace: TraceData, channel: str) -> np.ndarray:
|
||||
if channel == "s11":
|
||||
return np.asarray(trace.s11, dtype=np.complex128).reshape(-1)
|
||||
return np.asarray(trace.s21, dtype=np.complex128).reshape(-1)
|
||||
trace_capture_end_ns: int = 0
|
||||
|
||||
|
||||
def _pick_trace(collection: SweepCollection, input_index: int, output_index: int) -> TraceData | None:
|
||||
@@ -57,7 +45,6 @@ def _build_stage_records(
|
||||
*,
|
||||
input_index: int,
|
||||
output_index: int,
|
||||
channel: str,
|
||||
) -> list[TraceRecord]:
|
||||
records: list[TraceRecord] = []
|
||||
for stage_index, collection in enumerate(history):
|
||||
@@ -66,11 +53,11 @@ def _build_stage_records(
|
||||
continue
|
||||
|
||||
frequency_hz = np.asarray(trace.frequency_hz, dtype=np.float64).reshape(-1)
|
||||
samples = _select_trace_samples(trace, channel)
|
||||
samples = np.asarray(trace.s21, dtype=np.complex128).reshape(-1)
|
||||
if frequency_hz.shape != samples.shape:
|
||||
raise ValueError(
|
||||
f"Shape mismatch in {stage} stage for collection_id={collection.collection_id}: "
|
||||
f"freq{frequency_hz.shape} vs {channel}{samples.shape}"
|
||||
f"freq{frequency_hz.shape} vs S21{samples.shape}"
|
||||
)
|
||||
if frequency_hz.size == 0:
|
||||
continue
|
||||
@@ -83,14 +70,14 @@ def _build_stage_records(
|
||||
|
||||
records.append(
|
||||
TraceRecord(
|
||||
stage=stage,
|
||||
collection_id=int(collection.collection_id),
|
||||
monotonic_ns=int(collection.monotonic_ns),
|
||||
stage_index=int(stage_index),
|
||||
frequency_hz=frequency_hz,
|
||||
samples=samples,
|
||||
capture_end_ns=int(trace.capture_end_ns),
|
||||
capture_start_ns=int(trace.capture_start_ns),
|
||||
capture_end_ns=int(collection.capture_end_ns),
|
||||
capture_start_ns=int(collection.capture_start_ns),
|
||||
trace_capture_end_ns=int(trace.capture_end_ns),
|
||||
)
|
||||
)
|
||||
return records
|
||||
@@ -104,69 +91,59 @@ def _build_sweep_history(
|
||||
raw_records: list[TraceRecord],
|
||||
preprocessed_records: list[TraceRecord],
|
||||
*,
|
||||
channel: str,
|
||||
primary_stage: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
# Runtime identities are exact. Only archives with no collection window need
|
||||
# the old monotonic/index fallback; never merge their repeated IDs in a dict.
|
||||
raw_index = index_collections(record for record in raw_records if record.capture_start_ns)
|
||||
pre_index = index_collections(record for record in preprocessed_records if record.capture_start_ns)
|
||||
pairs = [(raw_index.get(key), pre_index.get(key)) for key in raw_index.keys() | pre_index.keys()]
|
||||
primary = preprocessed_records if primary_stage == "preprocessed" else raw_records
|
||||
secondary = raw_records if primary_stage == "preprocessed" else preprocessed_records
|
||||
used: set[int] = set()
|
||||
pairs: list[tuple[TraceRecord, TraceRecord | None]] = []
|
||||
secondary = [record for record in secondary if not record.capture_start_ns]
|
||||
for base in primary:
|
||||
candidates = [
|
||||
(index, record) for index, record in enumerate(secondary)
|
||||
if index not in used and record.collection_id == base.collection_id
|
||||
]
|
||||
if base.capture_start_ns or base.capture_end_ns:
|
||||
candidates = [
|
||||
(index, record) for index, record in candidates
|
||||
if (record.capture_start_ns, record.capture_end_ns) == (base.capture_start_ns, base.capture_end_ns)
|
||||
]
|
||||
elif base.monotonic_ns:
|
||||
# Raw is produced before preprocessing; never pair it to a newer run.
|
||||
candidates = [
|
||||
(index, record) for index, record in candidates
|
||||
if (record.monotonic_ns <= base.monotonic_ns if primary_stage == "preprocessed"
|
||||
else record.monotonic_ns >= base.monotonic_ns)
|
||||
]
|
||||
match = None
|
||||
if candidates:
|
||||
index, match = min(candidates, key=lambda pair: abs(pair[1].monotonic_ns - base.monotonic_ns))
|
||||
used.add(index)
|
||||
pairs.append((base, match))
|
||||
# A tap can deliver a sweep while another stage drops it or is still processing.
|
||||
# Preserve those records as well instead of silently discarding the extra stage.
|
||||
pairs.extend((record, None) for index, record in enumerate(secondary) if index not in used)
|
||||
pairs.sort(key=lambda pair: (pair[0].monotonic_ns, pair[0].stage_index))
|
||||
|
||||
history: list[dict[str, Any]] = []
|
||||
for fallback_index, (base, match) in enumerate(pairs):
|
||||
raw = base if base.stage == "raw" else match
|
||||
pre = base if base.stage == "preprocessed" else match
|
||||
sweep_source = raw or pre
|
||||
calibrated_source = pre or raw
|
||||
if sweep_source is None or calibrated_source is None:
|
||||
if base.capture_start_ns:
|
||||
continue
|
||||
candidates = [record for record in secondary if record.collection_id == base.collection_id and (
|
||||
not base.monotonic_ns or
|
||||
(record.monotonic_ns <= base.monotonic_ns if primary_stage == "preprocessed"
|
||||
else record.monotonic_ns >= base.monotonic_ns)
|
||||
)]
|
||||
match = min(candidates, key=lambda record: (
|
||||
abs(record.monotonic_ns - base.monotonic_ns), abs(record.stage_index - base.stage_index),
|
||||
)) if candidates else None
|
||||
if match is not None:
|
||||
secondary = [record for record in secondary if record is not match]
|
||||
pairs.append((match, base) if primary_stage == "preprocessed" else (base, match))
|
||||
pairs.extend((record, None) if primary_stage == "preprocessed" else (None, record) for record in secondary)
|
||||
|
||||
def source(pair: tuple[TraceRecord | None, TraceRecord | None]) -> TraceRecord:
|
||||
raw, pre = pair
|
||||
base = (pre or raw) if primary_stage == "preprocessed" else (raw or pre)
|
||||
assert base is not None
|
||||
return base
|
||||
|
||||
pairs.sort(key=lambda pair: (source(pair).capture_start_ns or source(pair).monotonic_ns,
|
||||
source(pair).stage_index))
|
||||
history: list[dict[str, Any]] = []
|
||||
for fallback_index, (raw, pre) in enumerate(pairs):
|
||||
base = source((raw, pre))
|
||||
start_freq_hz = float(base.frequency_hz[0])
|
||||
stop_freq_hz = float(base.frequency_hz[-1])
|
||||
# Prefer the exported trace's own sweep time: with a switching matrix the
|
||||
# combos of one collection are measured milliseconds apart, so the
|
||||
# collection timestamp misplaces every combo but the last.
|
||||
if base.capture_end_ns > 0:
|
||||
timestamp_sec = float(base.capture_end_ns) / 1_000_000_000.0
|
||||
elif base.monotonic_ns > 0:
|
||||
timestamp_sec = float(base.monotonic_ns) / 1_000_000_000.0
|
||||
else:
|
||||
timestamp_sec = float(fallback_index)
|
||||
timestamp_ns = base.trace_capture_end_ns or base.capture_end_ns or base.monotonic_ns
|
||||
timestamp_sec = timestamp_ns / 1_000_000_000.0 if timestamp_ns else float(fallback_index)
|
||||
|
||||
history.append(
|
||||
{
|
||||
"timestamp": timestamp_sec,
|
||||
"sweep_points": _complex_to_points(sweep_source.samples),
|
||||
"calibrated_points": _complex_to_points(calibrated_source.samples),
|
||||
"sweep_points": _complex_to_points(raw.samples) if raw is not None else [],
|
||||
"calibrated_points": _complex_to_points(pre.samples) if pre is not None else [],
|
||||
"reference_points": [],
|
||||
"vna_config": {
|
||||
"mode": channel,
|
||||
"mode": "s21",
|
||||
"start_freq": start_freq_hz,
|
||||
"stop_freq": stop_freq_hz,
|
||||
"points": int(base.frequency_hz.size),
|
||||
@@ -180,30 +157,18 @@ def _stage_alignment_warning(pre_history: list[SweepCollection], result_history:
|
||||
if not pre_history or not result_history:
|
||||
return None
|
||||
|
||||
common_size = min(len(pre_history), len(result_history))
|
||||
if common_size <= 0:
|
||||
def keys(history: list[SweepCollection] | list[ResultCollection]) -> set[tuple[int, int, int]]:
|
||||
return {(item.collection_id, item.capture_start_ns or item.monotonic_ns,
|
||||
-1 if item.capture_start_ns else index) for index, item in enumerate(history)}
|
||||
|
||||
pre_keys, result_keys = keys(pre_history), keys(result_history)
|
||||
missing_results = len(pre_keys - result_keys)
|
||||
missing_pre = len(result_keys - pre_keys)
|
||||
if not missing_results and not missing_pre:
|
||||
return None
|
||||
|
||||
mismatches = 0
|
||||
first_mismatch: tuple[int, SweepCollection, ResultCollection] | None = None
|
||||
for index in range(common_size):
|
||||
pre = pre_history[index]
|
||||
result = result_history[index]
|
||||
if int(pre.collection_id) != int(result.collection_id) or int(pre.monotonic_ns) != int(result.monotonic_ns):
|
||||
mismatches += 1
|
||||
if first_mismatch is None:
|
||||
first_mismatch = (index, pre, result)
|
||||
|
||||
if mismatches == 0 or first_mismatch is None:
|
||||
return None
|
||||
|
||||
index, pre, result = first_mismatch
|
||||
return (
|
||||
"WARNING: snapshot stages are not fully aligned (preprocessed vs results). "
|
||||
f"Mismatches={mismatches}/{common_size}. "
|
||||
f"First mismatch at index={index}: "
|
||||
f"pre=(id={int(pre.collection_id)},ns={int(pre.monotonic_ns)}) vs "
|
||||
f"results=(id={int(result.collection_id)},ns={int(result.monotonic_ns)}). "
|
||||
f"Sweeps without results={missing_results}; results without preprocessed sweeps={missing_pre}. "
|
||||
"Export uses preprocessed traces; loaded view in vna_system may differ from "
|
||||
"radar_system on-screen replayed results."
|
||||
)
|
||||
@@ -216,11 +181,9 @@ def build_vna_history_payload(
|
||||
*,
|
||||
input_index: int,
|
||||
output_index: int,
|
||||
channel: str = "s21",
|
||||
primary_stage: str = "preprocessed",
|
||||
) -> dict[str, Any]:
|
||||
"""Build vna_system-compatible history JSON payload from runtime histories."""
|
||||
channel = _normalize_channel(channel)
|
||||
if primary_stage not in {"preprocessed", "raw"}:
|
||||
raise ValueError("primary_stage must be either 'preprocessed' or 'raw'")
|
||||
|
||||
@@ -229,14 +192,12 @@ def build_vna_history_payload(
|
||||
raw_history,
|
||||
input_index=input_index,
|
||||
output_index=output_index,
|
||||
channel=channel,
|
||||
)
|
||||
preprocessed_records = _build_stage_records(
|
||||
"preprocessed",
|
||||
preprocessed_history,
|
||||
input_index=input_index,
|
||||
output_index=output_index,
|
||||
channel=channel,
|
||||
)
|
||||
if not raw_records and not preprocessed_records:
|
||||
raise ValueError(
|
||||
@@ -247,7 +208,6 @@ def build_vna_history_payload(
|
||||
sweep_history = _build_sweep_history(
|
||||
raw_records,
|
||||
preprocessed_records,
|
||||
channel=channel,
|
||||
primary_stage=primary_stage,
|
||||
)
|
||||
if not sweep_history:
|
||||
@@ -255,12 +215,12 @@ def build_vna_history_payload(
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"format": "vna-system-history-v1",
|
||||
"converter": "python_app/scripts/convert_snapshot_to_vna_history.py",
|
||||
"converter": "python_app/storage/npz/vna_history_json.py",
|
||||
"converted_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"source_snapshot_dir": "<runtime_history>",
|
||||
"input_index": int(input_index),
|
||||
"output_index": int(output_index),
|
||||
"channel": channel,
|
||||
"channel": "s21",
|
||||
"primary_stage": str(primary_stage),
|
||||
"raw_record_count": len(raw_records),
|
||||
"preprocessed_record_count": len(preprocessed_records),
|
||||
@@ -285,3 +245,55 @@ def build_vna_history_payload(
|
||||
len(sweep_history),
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def collection_directories(stage_dir: Path) -> list[Path]:
|
||||
"""Order a snapshot stage numerically, including indices beyond four digits."""
|
||||
def order(path: Path) -> tuple[int, str]:
|
||||
prefix = path.name.split("_", 1)[0]
|
||||
return (int(prefix) if prefix.isdigit() else 0, path.name)
|
||||
|
||||
return sorted((path for path in stage_dir.iterdir() if path.is_dir()), key=order) if stage_dir.exists() else []
|
||||
|
||||
|
||||
def load_vna_history_snapshot(
|
||||
snapshot_dir: Path, *, input_index: int, output_index: int, primary_stage: str = "preprocessed",
|
||||
) -> dict[str, Any]:
|
||||
"""Load snapshot identities and selected S21 arrays, then use the runtime exporter."""
|
||||
if not snapshot_dir.is_dir():
|
||||
raise FileNotFoundError(f"Snapshot directory not found: {snapshot_dir}")
|
||||
histories: dict[str, list[SweepCollection]] = {"raw": [], "preprocessed": []}
|
||||
results: list[ResultCollection] = []
|
||||
for stage in ("raw", "preprocessed", "results"):
|
||||
for directory in collection_directories(snapshot_dir / stage):
|
||||
meta_path = directory / "meta.json"
|
||||
if not meta_path.exists():
|
||||
continue
|
||||
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
if not isinstance(meta, dict):
|
||||
raise ValueError(f"JSON root must be object: {meta_path}")
|
||||
cid, ns = int(meta.get("collection_id", -1)), int(meta.get("monotonic_ns", 0))
|
||||
start, end = int(meta.get("capture_start_ns", 0)), int(meta.get("capture_end_ns", 0))
|
||||
if stage == "results":
|
||||
results.append(ResultCollection(cid, ns, capture_start_ns=start, capture_end_ns=end))
|
||||
continue
|
||||
collection = SweepCollection(cid, ns, capture_start_ns=start, capture_end_ns=end)
|
||||
for trace in meta.get("traces", []):
|
||||
if (int(trace.get("input", -1)), int(trace.get("output", -1))) != (input_index, output_index):
|
||||
continue
|
||||
if not trace.get("freq_file") or not trace.get("s21_file"):
|
||||
continue
|
||||
collection.traces.append(TraceData(
|
||||
ComboKey(input_index, output_index), np.load(directory / trace["freq_file"]),
|
||||
np.load(directory / trace["s21_file"]),
|
||||
capture_start_ns=int(trace.get("capture_start_ns", 0)),
|
||||
capture_end_ns=int(trace.get("capture_end_ns", 0)),
|
||||
))
|
||||
break
|
||||
histories[stage].append(collection)
|
||||
payload = build_vna_history_payload(
|
||||
histories["raw"], histories["preprocessed"], results,
|
||||
input_index=input_index, output_index=output_index, primary_stage=primary_stage,
|
||||
)
|
||||
payload["source_snapshot_dir"] = str(snapshot_dir)
|
||||
return payload
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
"""Facade exports for NPZ-based storage implementation."""
|
||||
|
||||
from python_app.storage.npz.paths import (
|
||||
collection_dir_name as _collection_dir_name,
|
||||
format_float_for_key as _format_float_for_key,
|
||||
radar_key_from_config,
|
||||
sanitize_path_component as _sanitize_path_component,
|
||||
)
|
||||
from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, RESULT_MAGIC
|
||||
from python_app.storage.npz.store import NpzStore
|
||||
|
||||
__all__ = [
|
||||
"NpzStore",
|
||||
"PREPROC_MAGIC",
|
||||
"RAW_MAGIC",
|
||||
"RESULT_MAGIC",
|
||||
"_collection_dir_name",
|
||||
"_format_float_for_key",
|
||||
"_sanitize_path_component",
|
||||
"radar_key_from_config",
|
||||
]
|
||||
@@ -1,37 +0,0 @@
|
||||
"""Abstract storage API for preprocess sets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
|
||||
from python_app.models.dataset_model import ComboKey, SweepCollection
|
||||
|
||||
|
||||
class StoreApi(ABC):
|
||||
"""Storage contract used by workflows and GUI code."""
|
||||
|
||||
@abstractmethod
|
||||
def save_set(self, kind: str, radar_key: str, set_name: str, collection: SweepCollection) -> None:
|
||||
"""Persist a named set."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def load_set(self, kind: str, radar_key: str, set_name: str) -> SweepCollection:
|
||||
"""Load a named set."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def list_sets(self, kind: str, radar_key: str) -> list[str]:
|
||||
"""List set names for kind/radar key."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def has_combo_coverage(self, kind: str, radar_key: str, set_name: str, combos: list[ComboKey]) -> bool:
|
||||
"""Check whether stored set covers requested switch combinations."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def export_set_bundle(self, kind: str, radar_key: str, set_name: str, output_path: Path) -> Path:
|
||||
"""Export set into binary bundle file for C++ preprocessing."""
|
||||
raise NotImplementedError
|
||||
Reference in New Issue
Block a user