awe/radar_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor radar system

Browse Source
This commit is contained in:
Ayzen
2026-10-05 16:46:28 +03:00
parent 515049d2ed
commit 8f097dfa2e
179 changed files with 8212 additions and 6676 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python_app/storage/npz/serialize.py
+45 -14
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import struct
from numbers import Integral
import numpy as np
@@ -21,6 +22,41 @@ def _write_interleaved_complex(buffer: bytearray, values: np.ndarray) -> None:
buffer.extend(interleaved.tobytes())
def validate_trace_collection(collection: SweepCollection, *, require_finite: bool = False) -> None:
"""Reject ambiguous combinations and malformed traces before saving any files."""
for field in ("collection_id", "monotonic_ns", "capture_start_ns", "capture_end_ns"):
_validate_u64(getattr(collection, field), field)
if require_finite and not collection.traces:
raise ValueError("Preprocess set must contain at least one trace")
seen: set[tuple[int, int]] = set()
for trace in collection.traces:
_validate_u64(trace.capture_start_ns, "trace.capture_start_ns")
_validate_u64(trace.capture_end_ns, "trace.capture_end_ns")
if any(
isinstance(value, bool) or not isinstance(value, Integral) or not 0 <= value <= 0xFFFFFFFF
for value in (trace.combo.input, trace.combo.output)
):
raise ValueError("Trace input/output positions must be unsigned 32-bit integers")
combo = (int(trace.combo.input), int(trace.combo.output))
if combo in seen:
raise ValueError(f"Invalid or duplicate trace combination: {combo}")
seen.add(combo)
arrays = (
np.asarray(trace.frequency_hz, dtype=np.float32),
np.asarray(trace.s11, dtype=np.complex64),
np.asarray(trace.s21, dtype=np.complex64),
)
if any(array.ndim != 1 for array in arrays) or len({array.shape for array in arrays}) != 1:
raise ValueError(f"Trace {combo} frequency, S11 and S21 must be equally sized 1D arrays")
if require_finite and (not arrays[0].size or any(not np.isfinite(array).all() for array in arrays)):
raise ValueError(f"Preprocess trace {combo} must contain finite, nonempty arrays")
def _validate_u64(value: int, field: str) -> None:
if isinstance(value, bool) or not isinstance(value, Integral) or not 0 <= value < 1 << 64:
raise ValueError(f"{field} must be an unsigned 64-bit integer")
def serialize_trace_collection(collection: SweepCollection, magic: int) -> bytes:
"""Serialize one raw/preprocessed trace collection into ring-compatible binary format.
@@ -30,6 +66,7 @@ def serialize_trace_collection(collection: SweepCollection, magic: int) -> bytes
trace order). See :func:`python_app.orchestration.shm.decoder.decode_trace_collection`
and ``read_trace_collection`` in ``common_cpp/ipc/src/shared_types.cpp``.
"""
validate_trace_collection(collection)
buffer = bytearray()
buffer.extend(struct.pack("<IQQI", magic, collection.collection_id, collection.monotonic_ns, len(collection.traces)))
@@ -37,11 +74,6 @@ def serialize_trace_collection(collection: SweepCollection, magic: int) -> bytes
freq = np.asarray(trace.frequency_hz, dtype=np.float32)
s11 = np.asarray(trace.s11, dtype=np.complex64)
s21 = np.asarray(trace.s21, dtype=np.complex64)
if freq.size != s11.size:
raise ValueError("Trace frequency and S11 sizes must match")
if freq.size != s21.size:
raise ValueError("Trace frequency and S21 sizes must match")
buffer.extend(struct.pack("<III", trace.combo.input, trace.combo.output, int(freq.size)))
buffer.extend(freq.astype("<f4", copy=False).tobytes())
@@ -78,16 +110,13 @@ def serialize_result_collection(collection: ResultCollection) -> bytes:
if payload.kind == 1:
freq = np.asarray(payload.frequency_hz, dtype=np.float32)
trace = np.asarray(payload.trace, dtype=np.complex64)
if freq.size != trace.size:
raise ValueError("Result trace frequency and values sizes must match")
if freq.ndim != 1 or trace.ndim != 1 or freq.shape != trace.shape:
raise ValueError("Result trace frequency and values must be equally sized 1D arrays")
buffer.extend(struct.pack("<I", int(freq.size)))
buffer.extend(freq.astype("<f4", copy=False).tobytes())
interleaved = np.empty(freq.size * 2, dtype="<f4")
interleaved[0::2] = trace.real.astype("<f4", copy=False)
interleaved[1::2] = trace.imag.astype("<f4", copy=False)
buffer.extend(interleaved.tobytes())
_write_interleaved_complex(buffer, trace)
return
if payload.kind == 2:
@@ -98,9 +127,7 @@ def serialize_result_collection(collection: ResultCollection) -> bytes:
image_x_axis = np.asarray(payload.image_x_axis, dtype=np.float32)
image_y_axis = np.asarray(payload.image_y_axis, dtype=np.float32)
image = np.asarray(payload.image, dtype=np.float32)
if image.ndim != 2:
raise ValueError("Result image payload must be a 2D matrix")
if image.shape != (image_y_axis.size, image_x_axis.size):
if image_x_axis.ndim != 1 or image_y_axis.ndim != 1 or image.shape != (image_y_axis.size, image_x_axis.size):
raise ValueError("Result image axis sizes must match image matrix shape")
buffer.extend(struct.pack("<II", int(image_x_axis.size), int(image_y_axis.size)))
buffer.extend(image_x_axis.astype("<f4", copy=False).tobytes())
@@ -112,12 +139,16 @@ def serialize_result_collection(collection: ResultCollection) -> bytes:
table = np.asarray(payload.table, dtype=np.float32)
if table.ndim != 2:
raise ValueError("Result table payload must be a 2D matrix")
if table.shape[0] and not table.shape[1]:
raise ValueError("Result table with rows must have at least one column")
buffer.extend(struct.pack("<II", int(table.shape[1]), int(table.shape[0])))
buffer.extend(table.astype("<f4", copy=False).ravel(order="C").tobytes())
return
raise ValueError(f"Unsupported payload kind: {payload.kind}")
for field in ("collection_id", "monotonic_ns", "processing_duration_ns"):
_validate_u64(getattr(collection, field), field)
buffer = bytearray()
buffer.extend(
struct.pack(
python_app/storage/npz/snapshot_numpy.py
+170 -258
View File
@@ -4,18 +4,69 @@ from __future__ import annotations
import json
import logging
from contextlib import contextmanager
from pathlib import Path
from typing import Any, TypeVar
import shutil
import tempfile
from typing import Any, Iterator
import numpy as np
from python_app.models.dataset_model import ResultCollection, SweepCollection
from python_app.models.dataset_model import ResultCollection, ResultPayload, SweepCollection
from python_app.storage.npz.paths import collection_dir_name, sanitize_path_component
from python_app.storage.npz.serialize import serialize_result_collection, serialize_trace_collection
from python_app.storage.npz.serialize import serialize_result_collection, serialize_trace_collection, validate_trace_collection
logger = logging.getLogger(__name__)
TCollection = TypeVar("TCollection")
@contextmanager
def atomic_snapshot_directory(destination: Path) -> Iterator[Path]:
"""Publish a complete export directory; discard staging files after a failed write."""
destination.parent.mkdir(parents=True, exist_ok=True)
if destination.exists():
raise FileExistsError(f"Snapshot directory already exists: {destination}")
staging = Path(tempfile.mkdtemp(prefix=f".{destination.name}.", dir=destination.parent))
try:
yield staging
if destination.exists():
raise FileExistsError(f"Snapshot directory already exists: {destination}")
staging.rename(destination)
finally:
if staging.exists():
shutil.rmtree(staging)
def _matching_collection(
history: list[SweepCollection], anchor: SweepCollection | ResultCollection,
used: set[int], *, same_timestamp: bool,
) -> SweepCollection | None:
"""Match preprocessing identities exactly and raw sweeps by their capture window.
Producers stamp raw/preprocessed collections at different instants. Their capture
window is copied unchanged; results keep the preprocessed timestamp. Old files
without timing fall back to the latest preceding collection with the same id.
"""
candidates = [
(index, item) for index, item in enumerate(history)
if index not in used and int(item.collection_id) == int(anchor.collection_id)
]
timestamp = int(anchor.monotonic_ns)
start_ns = int(getattr(anchor, "capture_start_ns", 0))
end_ns = int(getattr(anchor, "capture_end_ns", 0))
if same_timestamp and timestamp:
candidates = [(index, item) for index, item in candidates if int(item.monotonic_ns) == timestamp]
elif start_ns or end_ns:
candidates = [
(index, item) for index, item in candidates
if (int(item.capture_start_ns), int(item.capture_end_ns)) == (start_ns, end_ns)
]
elif timestamp:
candidates = [(index, item) for index, item in candidates if int(item.monotonic_ns) <= timestamp]
if not candidates:
return None
index, item = max(candidates, key=lambda pair: (int(pair[1].monotonic_ns), pair[0]))
used.add(index)
return item
def select_aligned_histories(
@@ -23,89 +74,60 @@ def select_aligned_histories(
preprocessed_history: list[SweepCollection],
result_history: list[ResultCollection],
last_n: int,
*,
include_unaligned: bool = False,
) -> tuple[list[SweepCollection], list[SweepCollection], list[ResultCollection], dict[str, Any]]:
"""Select aligned tails with `results` as preferred anchor stage."""
raw_index, raw_pos = _index_by_collection_sequence(raw_history)
pre_index, pre_pos = _index_by_collection_sequence(preprocessed_history)
result_index, result_pos = _index_by_collection_sequence(result_history)
if result_index:
ordered_result_keys = sorted(result_index, key=lambda key: result_pos[key])
selected_keys = ordered_result_keys[-last_n:]
selected_raw = [raw_index[key] for key in selected_keys if key in raw_index]
selected_pre = [pre_index[key] for key in selected_keys if key in pre_index]
selected_results = [result_index[key] for key in selected_keys]
raw_missing = len(selected_keys) - len(selected_raw)
pre_missing = len(selected_keys) - len(selected_pre)
"""Select aligned tails, or retain every stage independently for complete exports."""
if last_n <= 0:
raise ValueError("last_n must be > 0")
if include_unaligned:
return (
selected_raw,
selected_pre,
selected_results,
{
"selection_mode": "result_tail_with_optional_alignment",
"anchor_stage": "results",
"selected_collection_ids": [int(key[0]) for key in selected_keys],
"aligned_key_count": len(selected_keys),
"raw_missing_count": raw_missing,
"preprocessed_missing_count": pre_missing,
},
raw_history[-last_n:], preprocessed_history[-last_n:], result_history[-last_n:],
{"selection_mode": "independent_tail", "anchor_stage": "none",
"selected_collection_ids": [int(item.collection_id) for item in result_history[-last_n:]],
"aligned_key_count": 0},
)
if raw_index:
ordered_raw_keys = sorted(raw_index, key=lambda key: raw_pos[key])
selected_keys = ordered_raw_keys[-last_n:]
selected_raw = [raw_index[key] for key in selected_keys]
selected_pre = [pre_index[key] for key in selected_keys if key in pre_index]
selected_results = [result_index[key] for key in selected_keys if key in result_index]
pre_missing = len(selected_keys) - len(selected_pre)
result_missing = len(selected_keys) - len(selected_results)
return (
selected_raw,
selected_pre,
selected_results,
{
"selection_mode": "raw_tail_with_optional_alignment",
"anchor_stage": "raw",
"selected_collection_ids": [int(key[0]) for key in selected_keys],
"aligned_key_count": len(selected_keys),
"preprocessed_missing_count": pre_missing,
"result_missing_count": result_missing,
},
)
selected_raw: list[SweepCollection] = []
selected_pre: list[SweepCollection] = []
selected_results = result_history[-last_n:]
used_raw: set[int] = set()
used_pre: set[int] = set()
if selected_results:
anchors = selected_results
anchor_stage = "results"
for result in reversed(selected_results):
pre = _matching_collection(preprocessed_history, result, used_pre, same_timestamp=True)
raw = _matching_collection(raw_history, pre or result, used_raw, same_timestamp=False)
if pre is not None:
selected_pre.append(pre)
if raw is not None:
selected_raw.append(raw)
selected_pre.reverse()
selected_raw.reverse()
elif preprocessed_history:
selected_pre = preprocessed_history[-last_n:]
anchors = selected_pre
anchor_stage = "preprocessed"
for pre in reversed(selected_pre):
raw = _matching_collection(raw_history, pre, used_raw, same_timestamp=False)
if raw is not None:
selected_raw.append(raw)
selected_raw.reverse()
else:
selected_raw = raw_history[-last_n:]
anchors = selected_raw
anchor_stage = "raw" if anchors else "none"
if pre_index:
ordered_pre_keys = sorted(pre_index, key=lambda key: pre_pos[key])
selected_keys = ordered_pre_keys[-last_n:]
selected_raw = [raw_index[key] for key in selected_keys if key in raw_index]
selected_pre = [pre_index[key] for key in selected_keys]
selected_results = [result_index[key] for key in selected_keys if key in result_index]
raw_missing = len(selected_keys) - len(selected_raw)
result_missing = len(selected_keys) - len(selected_results)
return (
selected_raw,
selected_pre,
selected_results,
{
"selection_mode": "preprocessed_tail_with_optional_alignment",
"anchor_stage": "preprocessed",
"selected_collection_ids": [int(key[0]) for key in selected_keys],
"aligned_key_count": len(selected_keys),
"raw_missing_count": raw_missing,
"result_missing_count": result_missing,
},
)
return (
raw_history[-last_n:],
preprocessed_history[-last_n:],
result_history[-last_n:],
{
"selection_mode": "independent_tail",
"anchor_stage": "none",
"selected_collection_ids": [],
"aligned_key_count": 0,
},
)
return selected_raw, selected_pre, selected_results, {
"selection_mode": f"{anchor_stage}_tail_with_optional_alignment",
"anchor_stage": anchor_stage,
"selected_collection_ids": [int(item.collection_id) for item in anchors],
"aligned_key_count": len(anchors),
"raw_missing_count": len(anchors) - len(selected_raw),
"preprocessed_missing_count": len(anchors) - len(selected_pre),
"result_missing_count": len(anchors) - len(selected_results),
}
def save_trace_history_binary(stage_dir: Path, history: list[SweepCollection], magic: int) -> None:
@@ -155,6 +177,7 @@ def save_result_history_binary(stage_dir: Path, history: list[ResultCollection])
{
"collection_id": collection.collection_id,
"monotonic_ns": collection.monotonic_ns,
"processing_duration_ns": int(collection.processing_duration_ns),
"collection_payload_count": len(collection.collection_payloads),
"block_count": len(collection.blocks),
},
@@ -173,6 +196,8 @@ def save_trace_history_numpy(
streaming recorder can append successive chunks into the same stage directory
without colliding or restarting the index.
"""
for collection in history:
validate_trace_collection(collection)
stage_dir.mkdir(parents=True, exist_ok=True)
logger.debug("Writing %d NumPy trace collection(s) to %s", len(history), stage_dir)
for index, collection in enumerate(history, start=index_offset):
@@ -219,197 +244,84 @@ def save_trace_history_numpy(
)
def _save_result_payload(directory: Path, base_name: str, payload: ResultPayload) -> dict[str, Any]:
"""Persist every supported payload kind identically at collection and combo scope."""
meta: dict[str, Any] = {"kind": int(payload.kind), "name": payload.processing_name}
def save_array(field: str, array: np.ndarray) -> None:
filename = f"{base_name}_{field}.npy"
np.save(directory / filename, array, allow_pickle=False)
meta[f"{field}_file"] = filename
if payload.kind == 1:
freq = np.asarray(payload.frequency_hz, dtype=np.float32)
trace = np.asarray(payload.trace, dtype=np.complex64)
if freq.ndim != 1 or trace.ndim != 1 or freq.shape != trace.shape:
raise ValueError("Result trace frequency and values must be equally sized 1D arrays")
save_array("freq", freq)
save_array("trace", trace)
meta["points"] = int(freq.size)
elif payload.kind == 2:
save_array("scalar", np.asarray([payload.scalar_value], dtype=np.float32))
meta["scalar_value"] = float(payload.scalar_value)
elif payload.kind == 3:
x_axis = np.asarray(payload.image_x_axis, dtype=np.float32)
y_axis = np.asarray(payload.image_y_axis, dtype=np.float32)
image = np.asarray(payload.image, dtype=np.float32)
if x_axis.ndim != 1 or y_axis.ndim != 1 or image.shape != (y_axis.size, x_axis.size):
raise ValueError("Result image axis sizes must match its 2D matrix shape")
save_array("x_axis", x_axis)
save_array("y_axis", y_axis)
save_array("image", image)
meta.update(x_points=int(x_axis.size), y_points=int(y_axis.size))
elif payload.kind == 4:
table = np.asarray(payload.table, dtype=np.float32)
if table.ndim != 2:
raise ValueError("Result table payload must be a 2D matrix")
save_array("table", table)
meta.update(rows=int(table.shape[0]), columns=int(table.shape[1]))
else:
raise ValueError(f"Unsupported payload kind: {payload.kind}")
return meta
def save_result_history_numpy(
stage_dir: Path, history: list[ResultCollection], *, index_offset: int = 0
) -> None:
"""Write processed result collections as a NumPy directory tree.
``index_offset`` continues the per-collection directory numbering across calls (see
:func:`save_trace_history_numpy`) so streamed chunks append cleanly.
"""
"""Write result payloads, preserving per-stage indices across streaming chunks."""
stage_dir.mkdir(parents=True, exist_ok=True)
logger.debug("Writing %d NumPy result collection(s) to %s", len(history), stage_dir)
for index, collection in enumerate(history, start=index_offset):
collection_dir = stage_dir / collection_dir_name(index, collection.collection_id, collection.monotonic_ns)
collection_dir.mkdir(parents=True, exist_ok=False)
collection_payload_meta: list[dict[str, int | str | float]] = []
collection_payload_meta = []
for payload_index, payload in enumerate(collection.collection_payloads):
safe_name = sanitize_path_component(payload.processing_name or "processor")
base_name = f"collection_{payload_index:03d}_{safe_name}_kind{payload.kind}"
if payload.kind == 1:
freq = np.asarray(payload.frequency_hz, dtype=np.float32)
trace = np.asarray(payload.trace, dtype=np.complex64)
np.save(collection_dir / f"{base_name}_freq.npy", freq)
np.save(collection_dir / f"{base_name}_trace.npy", trace)
collection_payload_meta.append(
{
"kind": int(payload.kind),
"name": payload.processing_name,
"points": int(freq.size),
"freq_file": f"{base_name}_freq.npy",
"trace_file": f"{base_name}_trace.npy",
}
)
elif payload.kind == 2:
scalar = np.asarray([float(payload.scalar_value)], dtype=np.float32)
np.save(collection_dir / f"{base_name}_scalar.npy", scalar)
collection_payload_meta.append(
{
"kind": int(payload.kind),
"name": payload.processing_name,
"scalar_file": f"{base_name}_scalar.npy",
"scalar_value": float(payload.scalar_value),
}
)
elif payload.kind == 3:
image_x_axis = np.asarray(payload.image_x_axis, dtype=np.float32)
image_y_axis = np.asarray(payload.image_y_axis, dtype=np.float32)
image = np.asarray(payload.image, dtype=np.float32)
np.save(collection_dir / f"{base_name}_x_axis.npy", image_x_axis)
np.save(collection_dir / f"{base_name}_y_axis.npy", image_y_axis)
np.save(collection_dir / f"{base_name}_image.npy", image)
collection_payload_meta.append(
{
"kind": int(payload.kind),
"name": payload.processing_name,
"x_points": int(image_x_axis.size),
"y_points": int(image_y_axis.size),
"x_axis_file": f"{base_name}_x_axis.npy",
"y_axis_file": f"{base_name}_y_axis.npy",
"image_file": f"{base_name}_image.npy",
}
)
elif payload.kind == 4:
table = np.asarray(payload.table, dtype=np.float32)
np.save(collection_dir / f"{base_name}_table.npy", table)
collection_payload_meta.append(
{
"kind": int(payload.kind),
"name": payload.processing_name,
"rows": int(table.shape[0]) if table.ndim == 2 else 0,
"columns": int(table.shape[1]) if table.ndim == 2 else 0,
"table_file": f"{base_name}_table.npy",
}
)
name = sanitize_path_component(payload.processing_name or "processor")
base_name = f"collection_{payload_index:03d}_{name}_kind{payload.kind}"
collection_payload_meta.append(_save_result_payload(collection_dir, base_name, payload))
blocks_meta: list[dict[str, int | str | list[dict[str, int | str | float]]]] = []
blocks_meta = []
for block_index, block in enumerate(collection.blocks):
block_dir = collection_dir / f"block_{block_index:03d}_i{block.combo.input}_o{block.combo.output}"
block_dir.mkdir(parents=True, exist_ok=False)
payload_meta: list[dict[str, int | str | float]] = []
payload_meta = []
for payload_index, payload in enumerate(block.payloads):
safe_name = sanitize_path_component(payload.processing_name or "processor")
base_name = f"{payload_index:03d}_{safe_name}_kind{payload.kind}"
if payload.kind == 1:
freq = np.asarray(payload.frequency_hz, dtype=np.float32)
trace = np.asarray(payload.trace, dtype=np.complex64)
np.save(block_dir / f"{base_name}_freq.npy", freq)
np.save(block_dir / f"{base_name}_trace.npy", trace)
payload_meta.append(
{
"kind": int(payload.kind),
"name": payload.processing_name,
"points": int(freq.size),
"freq_file": f"{base_name}_freq.npy",
"trace_file": f"{base_name}_trace.npy",
}
)
elif payload.kind == 2:
scalar = np.asarray([float(payload.scalar_value)], dtype=np.float32)
np.save(block_dir / f"{base_name}_scalar.npy", scalar)
payload_meta.append(
{
"kind": int(payload.kind),
"name": payload.processing_name,
"scalar_file": f"{base_name}_scalar.npy",
"scalar_value": float(payload.scalar_value),
}
)
elif payload.kind == 3:
image_x_axis = np.asarray(payload.image_x_axis, dtype=np.float32)
image_y_axis = np.asarray(payload.image_y_axis, dtype=np.float32)
image = np.asarray(payload.image, dtype=np.float32)
np.save(block_dir / f"{base_name}_x_axis.npy", image_x_axis)
np.save(block_dir / f"{base_name}_y_axis.npy", image_y_axis)
np.save(block_dir / f"{base_name}_image.npy", image)
payload_meta.append(
{
"kind": int(payload.kind),
"name": payload.processing_name,
"x_points": int(image_x_axis.size),
"y_points": int(image_y_axis.size),
"x_axis_file": f"{base_name}_x_axis.npy",
"y_axis_file": f"{base_name}_y_axis.npy",
"image_file": f"{base_name}_image.npy",
}
)
elif payload.kind == 4:
table = np.asarray(payload.table, dtype=np.float32)
np.save(block_dir / f"{base_name}_table.npy", table)
payload_meta.append(
{
"kind": int(payload.kind),
"name": payload.processing_name,
"rows": int(table.shape[0]) if table.ndim == 2 else 0,
"columns": int(table.shape[1]) if table.ndim == 2 else 0,
"table_file": f"{base_name}_table.npy",
}
)
blocks_meta.append(
{
"input": int(block.combo.input),
"output": int(block.combo.output),
"payload_count": len(block.payloads),
"dir": block_dir.name,
"payloads": payload_meta,
}
)
name = sanitize_path_component(payload.processing_name or "processor")
base_name = f"{payload_index:03d}_{name}_kind{payload.kind}"
payload_meta.append(_save_result_payload(block_dir, base_name, payload))
blocks_meta.append({
"input": int(block.combo.input), "output": int(block.combo.output),
"payload_count": len(block.payloads), "dir": block_dir.name, "payloads": payload_meta,
})
(collection_dir / "meta.json").write_text(
json.dumps(
{
"collection_id": int(collection.collection_id),
"monotonic_ns": int(collection.monotonic_ns),
"collection_payload_count": len(collection.collection_payloads),
"collection_payloads": collection_payload_meta,
"block_count": len(collection.blocks),
"blocks": blocks_meta,
},
indent=2,
),
json.dumps({
"collection_id": int(collection.collection_id),
"monotonic_ns": int(collection.monotonic_ns),
"processing_duration_ns": int(collection.processing_duration_ns),
"collection_payload_count": len(collection.collection_payloads),
"collection_payloads": collection_payload_meta,
"block_count": len(collection.blocks), "blocks": blocks_meta,
}, indent=2),
encoding="utf-8",
)
def validate_snapshot_name(snapshot_name: str) -> str:
"""Normalize snapshot directory stem."""
return sanitize_path_component(snapshot_name)
def _index_by_collection_sequence(
history: list[TCollection],
) -> tuple[dict[tuple[int, int], TCollection], dict[tuple[int, int], int]]:
"""Index collections by latest-first `(collection_id, occurrence)` preserving history position.
Occurrence is counted from the tail (newest item is occurrence `0` for its
collection id). This avoids cross-run misalignment when collection ids are
reused after restarts and one stage keeps longer history than another.
"""
indexed: dict[tuple[int, int], TCollection] = {}
positions: dict[tuple[int, int], int] = {}
seen_count_from_tail_by_id: dict[int, int] = {}
# Walk from newest to oldest so occurrence=0 always means the most recent
# instance for the given collection id.
for index in range(len(history) - 1, -1, -1):
item = history[index]
collection_id = int(getattr(item, "collection_id"))
occurrence = seen_count_from_tail_by_id.get(collection_id, 0)
key = (collection_id, occurrence)
indexed[key] = item
positions[key] = index
seen_count_from_tail_by_id[collection_id] = occurrence + 1
return indexed, positions
python_app/storage/npz/store.py
+182 -63
View File
@@ -6,25 +6,31 @@ from contextlib import suppress
from datetime import datetime, timezone
import json
import logging
import os
from pathlib import Path
import shutil
import tempfile
from typing import Any
from zipfile import BadZipFile
import numpy as np
from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection, TraceData
from python_app.storage.npz.paths import radar_key_from_config, sanitize_path_component
from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, serialize_trace_collection
from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, serialize_trace_collection, validate_trace_collection
from python_app.storage.npz.snapshot_numpy import (
save_result_history_binary,
save_result_history_numpy,
save_trace_history_binary,
save_trace_history_numpy,
select_aligned_histories,
atomic_snapshot_directory,
)
from python_app.storage.npz.vna_history_json import build_vna_history_payload
from python_app.storage.store_api import StoreApi
logger = logging.getLogger(__name__)
_SET_METADATA_KEY = "metadata_json"
def _compose_snapshot_stem(name: str, name_prefix: str) -> str:
@@ -37,7 +43,7 @@ def _compose_snapshot_stem(name: str, name_prefix: str) -> str:
class SnapshotStreamWriter:
"""Append aligned raw/preprocessed/result collections to a snapshot dir in chunks.
"""Append received raw/preprocessed/result collections to a snapshot dir in chunks.
A streaming alternative to :meth:`NpzStore.save_runtime_snapshot_numpy` for long
recordings: the caller flushes small chunks as measurements arrive and frees them,
@@ -69,12 +75,32 @@ class SnapshotStreamWriter:
preprocessed: list[SweepCollection],
results: list[ResultCollection],
) -> None:
"""Write one chunk of (already aligned) collections, continuing each stage's index."""
save_trace_history_numpy(self._dir / "raw", raw, index_offset=self._raw_written)
save_trace_history_numpy(
self._dir / "preprocessed", preprocessed, index_offset=self._preprocessed_written
)
save_result_history_numpy(self._dir / "results", results, index_offset=self._result_written)
"""Write a chunk without requiring equal stage lengths, continuing each index."""
# Stage a whole chunk before publishing complete collection directories. A
# failed write leaves earlier chunks intact and the next retry uses the same indices.
chunk_dir = Path(tempfile.mkdtemp(prefix=".chunk-", dir=self._dir))
published: list[Path] = []
try:
save_trace_history_numpy(chunk_dir / "raw", raw, index_offset=self._raw_written)
save_trace_history_numpy(
chunk_dir / "preprocessed", preprocessed, index_offset=self._preprocessed_written
)
save_result_history_numpy(chunk_dir / "results", results, index_offset=self._result_written)
for stage in ("raw", "preprocessed", "results"):
stage_dir = self._dir / stage
stage_dir.mkdir(exist_ok=True)
for source in sorted((chunk_dir / stage).iterdir()):
destination = stage_dir / source.name
if destination.exists():
raise FileExistsError(f"Collection directory already exists: {destination}")
source.rename(destination)
published.append(destination)
except BaseException:
for destination in published:
shutil.rmtree(destination)
raise
finally:
shutil.rmtree(chunk_dir)
self._raw_written += len(raw)
self._preprocessed_written += len(preprocessed)
self._result_written += len(results)
@@ -96,13 +122,15 @@ class NpzStore(StoreApi):
def save_set(self, kind: str, radar_key: str, set_name: str, collection: SweepCollection) -> None:
"""Persist named preprocess set as NPZ and metadata JSON."""
self._validate_set_name(set_name)
validate_trace_collection(collection, require_finite=True)
set_dir = self._set_dir(kind, radar_key)
set_dir.mkdir(parents=True, exist_ok=True)
npz_path = set_dir / f"{set_name}.npz"
meta_path = set_dir / f"{set_name}.json"
payload: dict[str, np.ndarray] = {}
payload: dict[str, Any] = {}
combo_records: list[dict[str, str | int]] = []
for trace in collection.traces:
@@ -133,67 +161,137 @@ class NpzStore(StoreApi):
"combos": combo_records,
}
# Write both files to temporary paths first, then atomically rename so a
# crash never leaves an .npz without its meta (or vice versa).
npz_tmp = npz_path.with_name(npz_path.name + ".tmp")
meta_tmp = meta_path.with_name(meta_path.name + ".tmp")
# Metadata in the NPZ makes a single atomic replacement authoritative. The
# JSON remains a readable sidecar and legacy NPZ/JSON pairs remain loadable.
metadata_json = json.dumps(meta, indent=2)
payload[_SET_METADATA_KEY] = np.asarray(metadata_json)
npz_tmp = meta_tmp = backup_path = None
archive_replaced = False
try:
npz_tmp = self._temporary_file(npz_path)
meta_tmp = self._temporary_file(meta_path)
backup_path = self._temporary_file(npz_path) if npz_path.exists() else None
if backup_path is not None:
backup_path.unlink()
os.link(npz_path, backup_path)
with npz_tmp.open("wb") as npz_file:
np.savez(npz_file, **payload)
meta_tmp.write_text(json.dumps(meta, indent=2), encoding="utf-8")
meta_tmp.write_text(metadata_json, encoding="utf-8")
npz_tmp.replace(npz_path)
archive_replaced = True
meta_tmp.replace(meta_path)
except BaseException:
logger.exception("Failed to save set %s/%s/%s; rolling back temp files", kind, radar_key, set_name)
for tmp_path in (npz_tmp, meta_tmp):
if archive_replaced:
if backup_path is None:
npz_path.unlink(missing_ok=True)
else:
try:
backup_path.replace(npz_path)
except OSError:
logger.exception("Set rollback failed; previous archive retained at %s", backup_path)
backup_path = None # retain the recovery copy rather than deleting it below
raise
raise
finally:
for tmp_path in (npz_tmp, meta_tmp, backup_path):
if tmp_path is None:
continue
with suppress(OSError):
tmp_path.unlink(missing_ok=True)
raise
logger.info(
"Saved set %s/%s/%s (%d traces) to %s", kind, radar_key, set_name, len(combo_records), npz_path
)
def save_sets(self, radar_key: str, set_name: str, collections: dict[str, SweepCollection]) -> None:
"""Save related sets together, restoring their previous files if any save fails."""
self._validate_set_name(set_name)
for collection in collections.values():
validate_trace_collection(collection, require_finite=True)
backup_dir = Path(tempfile.mkdtemp(prefix=".sets-", dir=self._root_dir))
backups: dict[Path, Path | None] = {}
retain_backups = False
try:
for kind in collections:
set_dir = self._set_dir(kind, radar_key)
for extension in ("npz", "json"):
destination = set_dir / f"{set_name}.{extension}"
backup = backup_dir / f"{kind}.{extension}" if destination.exists() else None
if backup is not None:
os.link(destination, backup)
backups[destination] = backup
try:
for kind, collection in collections.items():
self.save_set(kind, radar_key, set_name, collection)
except BaseException as save_error:
retain_backups = True # preserve recovery files if rollback itself is interrupted
rollback_failed = False
for destination, backup in backups.items():
try:
if backup is None:
destination.unlink(missing_ok=True)
else:
backup.replace(destination)
except OSError:
rollback_failed = True
logger.exception("Failed to restore previous set file %s", destination)
retain_backups = rollback_failed
if retain_backups:
raise RuntimeError(
f"Saving set group '{set_name}' failed and rollback was incomplete; "
f"previous files are retained at {backup_dir}"
) from save_error
raise
finally:
if not retain_backups:
with suppress(OSError):
shutil.rmtree(backup_dir)
def load_set(self, kind: str, radar_key: str, set_name: str) -> SweepCollection:
"""Load named preprocess set from NPZ representation."""
self._validate_set_name(set_name)
set_dir = self._set_dir(kind, radar_key)
npz_path = set_dir / f"{set_name}.npz"
meta_path = set_dir / f"{set_name}.json"
if not npz_path.exists() or not meta_path.exists():
if not npz_path.exists():
logger.error("Missing set files for %s/%s/%s", kind, radar_key, set_name)
raise FileNotFoundError(f"Missing set files for {kind}/{radar_key}/{set_name}")
set_label = f"{kind}/{radar_key}/{set_name}"
try:
meta = json.loads(meta_path.read_text(encoding="utf-8"))
arrays = np.load(npz_path, allow_pickle=False)
traces: list[TraceData] = []
for combo in meta["combos"]:
freq = np.asarray(arrays[combo["freq_key"]], dtype=np.float32)
s11 = np.asarray(arrays[combo["s11_key"]], dtype=np.complex64)
s21 = np.asarray(arrays[combo["s21_key"]], dtype=np.complex64)
traces.append(
TraceData(
combo=ComboKey(input=int(combo["input"]), output=int(combo["output"])),
frequency_hz=freq,
s11=s11,
s21=s21,
capture_start_ns=int(combo.get("capture_start_ns", 0)),
capture_end_ns=int(combo.get("capture_end_ns", 0)),
)
with np.load(npz_path, allow_pickle=False) as arrays:
meta = json.loads(
str(arrays[_SET_METADATA_KEY].item())
if _SET_METADATA_KEY in arrays
else meta_path.read_text(encoding="utf-8")
)
if not isinstance(meta, dict):
raise ValueError("Set metadata must be a JSON object")
traces: list[TraceData] = []
for combo in meta["combos"]:
traces.append(
TraceData(
combo=ComboKey(input=combo["input"], output=combo["output"]),
frequency_hz=np.asarray(arrays[combo["freq_key"]], dtype=np.float32),
s11=np.asarray(arrays[combo["s11_key"]], dtype=np.complex64),
s21=np.asarray(arrays[combo["s21_key"]], dtype=np.complex64),
capture_start_ns=combo.get("capture_start_ns", 0),
capture_end_ns=combo.get("capture_end_ns", 0),
)
)
collection = SweepCollection(
collection_id=int(meta["collection_id"]),
monotonic_ns=int(meta["monotonic_ns"]),
collection_id=meta["collection_id"],
monotonic_ns=meta["monotonic_ns"],
traces=traces,
capture_start_ns=int(meta.get("capture_start_ns", 0)),
capture_end_ns=int(meta.get("capture_end_ns", 0)),
capture_start_ns=meta.get("capture_start_ns", 0),
capture_end_ns=meta.get("capture_end_ns", 0),
)
validate_trace_collection(collection, require_finite=True)
logger.debug("Loaded set %s (%d traces)", set_label, len(traces))
return collection
except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:
except (OSError, BadZipFile, EOFError, KeyError, TypeError, ValueError) as exc:
logger.exception("Corrupted preprocess set %s", set_label)
raise RuntimeError(f"Corrupted preprocess set {set_label}: {exc}") from exc
@@ -215,7 +313,12 @@ class NpzStore(StoreApi):
"""Export named set as binary collection bundle for C++ preprocessing stage."""
collection = self.load_set(kind, radar_key, set_name)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(serialize_trace_collection(collection, RAW_MAGIC))
temporary = self._temporary_file(output_path)
try:
temporary.write_bytes(serialize_trace_collection(collection, RAW_MAGIC))
temporary.replace(output_path)
finally:
temporary.unlink(missing_ok=True)
logger.info("Exported set %s/%s/%s bundle to %s", kind, radar_key, set_name, output_path)
return output_path
@@ -234,11 +337,10 @@ class NpzStore(StoreApi):
output_dir.mkdir(parents=True, exist_ok=True)
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
snapshot_dir = output_dir / f"snapshot_{timestamp}"
snapshot_dir.mkdir(parents=True, exist_ok=True)
save_trace_history_binary(snapshot_dir / "raw", raw_history[-last_n:], RAW_MAGIC)
save_trace_history_binary(snapshot_dir / "preprocessed", preprocessed_history[-last_n:], PREPROC_MAGIC)
save_result_history_binary(snapshot_dir / "results", result_history[-last_n:])
with atomic_snapshot_directory(snapshot_dir) as staging:
save_trace_history_binary(staging / "raw", raw_history[-last_n:], RAW_MAGIC)
save_trace_history_binary(staging / "preprocessed", preprocessed_history[-last_n:], PREPROC_MAGIC)
save_result_history_binary(staging / "results", result_history[-last_n:])
logger.info("Saved binary runtime snapshot (last_n=%d) to %s", last_n, snapshot_dir)
return snapshot_dir
@@ -281,6 +383,7 @@ class NpzStore(StoreApi):
last_n: int,
*,
name_prefix: str = "",
include_unaligned: bool = False,
) -> tuple[Path, dict[str, Any]]:
"""Save historical runtime collections in NumPy tree format."""
if last_n <= 0:
@@ -292,19 +395,15 @@ class NpzStore(StoreApi):
snapshot_dir = output_root_dir / snapshot_stem
if snapshot_dir.exists():
raise FileExistsError(f"Snapshot directory already exists: {snapshot_dir}")
snapshot_dir.mkdir(parents=True, exist_ok=False)
selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories(
raw_history,
preprocessed_history,
result_history,
last_n,
include_unaligned=include_unaligned,
)
save_trace_history_numpy(snapshot_dir / "raw", selected_raw)
save_trace_history_numpy(snapshot_dir / "preprocessed", selected_preprocessed)
save_result_history_numpy(snapshot_dir / "results", selected_results)
if selection_summary.get("anchor_stage") == "results":
expected = min(int(last_n), len(result_history))
if len(selected_results) != expected:
@@ -313,6 +412,11 @@ class NpzStore(StoreApi):
f"expected={expected}, selected={len(selected_results)}"
)
with atomic_snapshot_directory(snapshot_dir) as staging:
save_trace_history_numpy(staging / "raw", selected_raw)
save_trace_history_numpy(staging / "preprocessed", selected_preprocessed)
save_result_history_numpy(staging / "results", selected_results)
selection_summary["raw_count"] = len(selected_raw)
selection_summary["preprocessed_count"] = len(selected_preprocessed)
selection_summary["result_count"] = len(selected_results)
@@ -341,6 +445,7 @@ class NpzStore(StoreApi):
output_index: int = 0,
channel: str = "s21",
primary_stage: str = "preprocessed",
include_unaligned: bool = False,
) -> tuple[Path, dict[str, Any]]:
"""Save runtime history as vna_system-compatible JSON file."""
if last_n <= 0:
@@ -351,16 +456,14 @@ class NpzStore(StoreApi):
output_root_dir.mkdir(parents=True, exist_ok=True)
output_dir = self._vna_json_output_dir(output_root_dir, output_stem)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = output_dir / f"{output_stem}_vna_bscan_history.json"
if output_path.exists():
raise FileExistsError(f"Output JSON file already exists: {output_path}")
selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories(
raw_history,
preprocessed_history,
result_history,
last_n,
include_unaligned=include_unaligned,
)
payload = build_vna_history_payload(
selected_raw,
@@ -371,10 +474,8 @@ class NpzStore(StoreApi):
channel=channel,
primary_stage=primary_stage,
)
output_path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2),
encoding="utf-8",
)
with atomic_snapshot_directory(output_dir) as staging:
(staging / output_path.name).write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
summary: dict[str, Any] = dict(selection_summary)
summary["raw_count"] = len(selected_raw)
@@ -412,6 +513,7 @@ class NpzStore(StoreApi):
channel: str = "s21",
primary_stage: str = "preprocessed",
name_prefix: str = "",
include_unaligned: bool = False,
) -> tuple[list[Path], dict[str, Any]]:
"""Save one runtime VNA-history JSON per available combo."""
if last_n <= 0:
@@ -420,13 +522,13 @@ class NpzStore(StoreApi):
output_stem = _compose_snapshot_stem(output_name, name_prefix)
output_root_dir.mkdir(parents=True, exist_ok=True)
output_dir = self._vna_json_output_dir(output_root_dir, output_stem)
output_dir.mkdir(parents=True, exist_ok=True)
selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories(
raw_history,
preprocessed_history,
result_history,
last_n,
include_unaligned=include_unaligned,
)
combos = sorted(
@@ -461,11 +563,11 @@ class NpzStore(StoreApi):
)
output_paths.append(output_path)
for output_path, payload in zip(output_paths, payloads, strict=True):
output_path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2),
encoding="utf-8",
)
with atomic_snapshot_directory(output_dir) as staging:
for output_path, payload in zip(output_paths, payloads, strict=True):
(staging / output_path.name).write_text(
json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8"
)
summary: dict[str, Any] = dict(selection_summary)
summary["raw_count"] = len(selected_raw)
@@ -486,8 +588,25 @@ class NpzStore(StoreApi):
def _set_dir(self, kind: str, radar_key: str) -> Path:
"""Return directory for set kind and radar key."""
self._validate_set_name(kind)
self._validate_set_name(radar_key)
return self._root_dir / kind / radar_key
@staticmethod
def _validate_set_name(value: str) -> None:
"""Keep caller-provided set components inside their storage directory."""
if not value or value in {".", ".."} or Path(value).name != value or "\\" in value:
raise ValueError(f"Invalid set path component: {value!r}")
@staticmethod
def _temporary_file(destination: Path) -> Path:
"""Create a unique temporary file beside its atomic-replacement destination."""
with tempfile.NamedTemporaryFile(
dir=destination.parent, prefix=f".{destination.name}.", suffix=".tmp", delete=False
) as handle:
path = Path(handle.name)
return path
def preview_png_dir(self, kind: str, set_name: str, radar_key: str) -> Path:
"""Return directory for preview PNGs of one set/radar variant.
python_app/storage/npz/vna_history_json.py
+35 -28
View File
@@ -2,7 +2,6 @@
from __future__ import annotations
from collections import defaultdict
from dataclasses import dataclass
from datetime import datetime, timezone
import logging
@@ -29,6 +28,7 @@ class TraceRecord:
# timing. Preferred over the collection timestamp for the exported sweep time:
# in a switched matrix each combo is measured at a different instant.
capture_end_ns: int = 0
capture_start_ns: int = 0
def _normalize_channel(channel: str) -> str:
@@ -90,24 +90,12 @@ def _build_stage_records(
frequency_hz=frequency_hz,
samples=samples,
capture_end_ns=int(trace.capture_end_ns),
capture_start_ns=int(trace.capture_start_ns),
)
)
return records
def _index_by_collection_occurrence(records: list[TraceRecord]) -> tuple[dict[tuple[int, int], TraceRecord], list[tuple[int, int]]]:
counters: defaultdict[int, int] = defaultdict(int)
record_map: dict[tuple[int, int], TraceRecord] = {}
order: list[tuple[int, int]] = []
for record in records:
occurrence = counters[record.collection_id]
counters[record.collection_id] += 1
key = (record.collection_id, occurrence)
record_map[key] = record
order.append(key)
return record_map, order
def _complex_to_points(values: np.ndarray) -> list[list[float]]:
return [[float(value.real), float(value.imag)] for value in values]
@@ -119,22 +107,41 @@ def _build_sweep_history(
channel: str,
primary_stage: str,
) -> list[dict[str, Any]]:
raw_map, raw_order = _index_by_collection_occurrence(raw_records)
pre_map, pre_order = _index_by_collection_occurrence(preprocessed_records)
if primary_stage == "preprocessed":
primary_order = pre_order or raw_order
else:
primary_order = raw_order or pre_order
primary = preprocessed_records if primary_stage == "preprocessed" else raw_records
secondary = raw_records if primary_stage == "preprocessed" else preprocessed_records
used: set[int] = set()
pairs: list[tuple[TraceRecord, TraceRecord | None]] = []
for base in primary:
candidates = [
(index, record) for index, record in enumerate(secondary)
if index not in used and record.collection_id == base.collection_id
]
if base.capture_start_ns or base.capture_end_ns:
candidates = [
(index, record) for index, record in candidates
if (record.capture_start_ns, record.capture_end_ns) == (base.capture_start_ns, base.capture_end_ns)
]
elif base.monotonic_ns:
# Raw is produced before preprocessing; never pair it to a newer run.
candidates = [
(index, record) for index, record in candidates
if (record.monotonic_ns <= base.monotonic_ns if primary_stage == "preprocessed"
else record.monotonic_ns >= base.monotonic_ns)
]
match = None
if candidates:
index, match = min(candidates, key=lambda pair: abs(pair[1].monotonic_ns - base.monotonic_ns))
used.add(index)
pairs.append((base, match))
# A tap can deliver a sweep while another stage drops it or is still processing.
# Preserve those records as well instead of silently discarding the extra stage.
pairs.extend((record, None) for index, record in enumerate(secondary) if index not in used)
pairs.sort(key=lambda pair: (pair[0].monotonic_ns, pair[0].stage_index))
history: list[dict[str, Any]] = []
for fallback_index, key in enumerate(primary_order):
raw = raw_map.get(key)
pre = pre_map.get(key)
base = pre or raw
if base is None:
continue
for fallback_index, (base, match) in enumerate(pairs):
raw = base if base.stage == "raw" else match
pre = base if base.stage == "preprocessed" else match
sweep_source = raw or pre
calibrated_source = pre or raw
if sweep_source is None or calibrated_source is None: