awe/radar_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor radar system

Browse Source
This commit is contained in:
Ayzen
2026-10-05 16:46:28 +03:00
parent 515049d2ed
commit 8f097dfa2e
179 changed files with 8212 additions and 6676 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python_app/storage/npz/store.py
+182 -63
View File
@@ -6,25 +6,31 @@ from contextlib import suppress
from datetime import datetime, timezone
import json
import logging
import os
from pathlib import Path
import shutil
import tempfile
from typing import Any
from zipfile import BadZipFile
import numpy as np
from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection, TraceData
from python_app.storage.npz.paths import radar_key_from_config, sanitize_path_component
from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, serialize_trace_collection
from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, serialize_trace_collection, validate_trace_collection
from python_app.storage.npz.snapshot_numpy import (
save_result_history_binary,
save_result_history_numpy,
save_trace_history_binary,
save_trace_history_numpy,
select_aligned_histories,
atomic_snapshot_directory,
)
from python_app.storage.npz.vna_history_json import build_vna_history_payload
from python_app.storage.store_api import StoreApi
logger = logging.getLogger(__name__)
_SET_METADATA_KEY = "metadata_json"
def _compose_snapshot_stem(name: str, name_prefix: str) -> str:
@@ -37,7 +43,7 @@ def _compose_snapshot_stem(name: str, name_prefix: str) -> str:
class SnapshotStreamWriter:
"""Append aligned raw/preprocessed/result collections to a snapshot dir in chunks.
"""Append received raw/preprocessed/result collections to a snapshot dir in chunks.
A streaming alternative to :meth:`NpzStore.save_runtime_snapshot_numpy` for long
recordings: the caller flushes small chunks as measurements arrive and frees them,
@@ -69,12 +75,32 @@ class SnapshotStreamWriter:
preprocessed: list[SweepCollection],
results: list[ResultCollection],
) -> None:
"""Write one chunk of (already aligned) collections, continuing each stage's index."""
save_trace_history_numpy(self._dir / "raw", raw, index_offset=self._raw_written)
save_trace_history_numpy(
self._dir / "preprocessed", preprocessed, index_offset=self._preprocessed_written
)
save_result_history_numpy(self._dir / "results", results, index_offset=self._result_written)
"""Write a chunk without requiring equal stage lengths, continuing each index."""
# Stage a whole chunk before publishing complete collection directories. A
# failed write leaves earlier chunks intact and the next retry uses the same indices.
chunk_dir = Path(tempfile.mkdtemp(prefix=".chunk-", dir=self._dir))
published: list[Path] = []
try:
save_trace_history_numpy(chunk_dir / "raw", raw, index_offset=self._raw_written)
save_trace_history_numpy(
chunk_dir / "preprocessed", preprocessed, index_offset=self._preprocessed_written
)
save_result_history_numpy(chunk_dir / "results", results, index_offset=self._result_written)
for stage in ("raw", "preprocessed", "results"):
stage_dir = self._dir / stage
stage_dir.mkdir(exist_ok=True)
for source in sorted((chunk_dir / stage).iterdir()):
destination = stage_dir / source.name
if destination.exists():
raise FileExistsError(f"Collection directory already exists: {destination}")
source.rename(destination)
published.append(destination)
except BaseException:
for destination in published:
shutil.rmtree(destination)
raise
finally:
shutil.rmtree(chunk_dir)
self._raw_written += len(raw)
self._preprocessed_written += len(preprocessed)
self._result_written += len(results)
@@ -96,13 +122,15 @@ class NpzStore(StoreApi):
def save_set(self, kind: str, radar_key: str, set_name: str, collection: SweepCollection) -> None:
"""Persist named preprocess set as NPZ and metadata JSON."""
self._validate_set_name(set_name)
validate_trace_collection(collection, require_finite=True)
set_dir = self._set_dir(kind, radar_key)
set_dir.mkdir(parents=True, exist_ok=True)
npz_path = set_dir / f"{set_name}.npz"
meta_path = set_dir / f"{set_name}.json"
payload: dict[str, np.ndarray] = {}
payload: dict[str, Any] = {}
combo_records: list[dict[str, str | int]] = []
for trace in collection.traces:
@@ -133,67 +161,137 @@ class NpzStore(StoreApi):
"combos": combo_records,
}
# Write both files to temporary paths first, then atomically rename so a
# crash never leaves an .npz without its meta (or vice versa).
npz_tmp = npz_path.with_name(npz_path.name + ".tmp")
meta_tmp = meta_path.with_name(meta_path.name + ".tmp")
# Metadata in the NPZ makes a single atomic replacement authoritative. The
# JSON remains a readable sidecar and legacy NPZ/JSON pairs remain loadable.
metadata_json = json.dumps(meta, indent=2)
payload[_SET_METADATA_KEY] = np.asarray(metadata_json)
npz_tmp = meta_tmp = backup_path = None
archive_replaced = False
try:
npz_tmp = self._temporary_file(npz_path)
meta_tmp = self._temporary_file(meta_path)
backup_path = self._temporary_file(npz_path) if npz_path.exists() else None
if backup_path is not None:
backup_path.unlink()
os.link(npz_path, backup_path)
with npz_tmp.open("wb") as npz_file:
np.savez(npz_file, **payload)
meta_tmp.write_text(json.dumps(meta, indent=2), encoding="utf-8")
meta_tmp.write_text(metadata_json, encoding="utf-8")
npz_tmp.replace(npz_path)
archive_replaced = True
meta_tmp.replace(meta_path)
except BaseException:
logger.exception("Failed to save set %s/%s/%s; rolling back temp files", kind, radar_key, set_name)
for tmp_path in (npz_tmp, meta_tmp):
if archive_replaced:
if backup_path is None:
npz_path.unlink(missing_ok=True)
else:
try:
backup_path.replace(npz_path)
except OSError:
logger.exception("Set rollback failed; previous archive retained at %s", backup_path)
backup_path = None # retain the recovery copy rather than deleting it below
raise
raise
finally:
for tmp_path in (npz_tmp, meta_tmp, backup_path):
if tmp_path is None:
continue
with suppress(OSError):
tmp_path.unlink(missing_ok=True)
raise
logger.info(
"Saved set %s/%s/%s (%d traces) to %s", kind, radar_key, set_name, len(combo_records), npz_path
)
def save_sets(self, radar_key: str, set_name: str, collections: dict[str, SweepCollection]) -> None:
"""Save related sets together, restoring their previous files if any save fails."""
self._validate_set_name(set_name)
for collection in collections.values():
validate_trace_collection(collection, require_finite=True)
backup_dir = Path(tempfile.mkdtemp(prefix=".sets-", dir=self._root_dir))
backups: dict[Path, Path | None] = {}
retain_backups = False
try:
for kind in collections:
set_dir = self._set_dir(kind, radar_key)
for extension in ("npz", "json"):
destination = set_dir / f"{set_name}.{extension}"
backup = backup_dir / f"{kind}.{extension}" if destination.exists() else None
if backup is not None:
os.link(destination, backup)
backups[destination] = backup
try:
for kind, collection in collections.items():
self.save_set(kind, radar_key, set_name, collection)
except BaseException as save_error:
retain_backups = True # preserve recovery files if rollback itself is interrupted
rollback_failed = False
for destination, backup in backups.items():
try:
if backup is None:
destination.unlink(missing_ok=True)
else:
backup.replace(destination)
except OSError:
rollback_failed = True
logger.exception("Failed to restore previous set file %s", destination)
retain_backups = rollback_failed
if retain_backups:
raise RuntimeError(
f"Saving set group '{set_name}' failed and rollback was incomplete; "
f"previous files are retained at {backup_dir}"
) from save_error
raise
finally:
if not retain_backups:
with suppress(OSError):
shutil.rmtree(backup_dir)
def load_set(self, kind: str, radar_key: str, set_name: str) -> SweepCollection:
"""Load named preprocess set from NPZ representation."""
self._validate_set_name(set_name)
set_dir = self._set_dir(kind, radar_key)
npz_path = set_dir / f"{set_name}.npz"
meta_path = set_dir / f"{set_name}.json"
if not npz_path.exists() or not meta_path.exists():
if not npz_path.exists():
logger.error("Missing set files for %s/%s/%s", kind, radar_key, set_name)
raise FileNotFoundError(f"Missing set files for {kind}/{radar_key}/{set_name}")
set_label = f"{kind}/{radar_key}/{set_name}"
try:
meta = json.loads(meta_path.read_text(encoding="utf-8"))
arrays = np.load(npz_path, allow_pickle=False)
traces: list[TraceData] = []
for combo in meta["combos"]:
freq = np.asarray(arrays[combo["freq_key"]], dtype=np.float32)
s11 = np.asarray(arrays[combo["s11_key"]], dtype=np.complex64)
s21 = np.asarray(arrays[combo["s21_key"]], dtype=np.complex64)
traces.append(
TraceData(
combo=ComboKey(input=int(combo["input"]), output=int(combo["output"])),
frequency_hz=freq,
s11=s11,
s21=s21,
capture_start_ns=int(combo.get("capture_start_ns", 0)),
capture_end_ns=int(combo.get("capture_end_ns", 0)),
)
with np.load(npz_path, allow_pickle=False) as arrays:
meta = json.loads(
str(arrays[_SET_METADATA_KEY].item())
if _SET_METADATA_KEY in arrays
else meta_path.read_text(encoding="utf-8")
)
if not isinstance(meta, dict):
raise ValueError("Set metadata must be a JSON object")
traces: list[TraceData] = []
for combo in meta["combos"]:
traces.append(
TraceData(
combo=ComboKey(input=combo["input"], output=combo["output"]),
frequency_hz=np.asarray(arrays[combo["freq_key"]], dtype=np.float32),
s11=np.asarray(arrays[combo["s11_key"]], dtype=np.complex64),
s21=np.asarray(arrays[combo["s21_key"]], dtype=np.complex64),
capture_start_ns=combo.get("capture_start_ns", 0),
capture_end_ns=combo.get("capture_end_ns", 0),
)
)
collection = SweepCollection(
collection_id=int(meta["collection_id"]),
monotonic_ns=int(meta["monotonic_ns"]),
collection_id=meta["collection_id"],
monotonic_ns=meta["monotonic_ns"],
traces=traces,
capture_start_ns=int(meta.get("capture_start_ns", 0)),
capture_end_ns=int(meta.get("capture_end_ns", 0)),
capture_start_ns=meta.get("capture_start_ns", 0),
capture_end_ns=meta.get("capture_end_ns", 0),
)
validate_trace_collection(collection, require_finite=True)
logger.debug("Loaded set %s (%d traces)", set_label, len(traces))
return collection
except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:
except (OSError, BadZipFile, EOFError, KeyError, TypeError, ValueError) as exc:
logger.exception("Corrupted preprocess set %s", set_label)
raise RuntimeError(f"Corrupted preprocess set {set_label}: {exc}") from exc
@@ -215,7 +313,12 @@ class NpzStore(StoreApi):
"""Export named set as binary collection bundle for C++ preprocessing stage."""
collection = self.load_set(kind, radar_key, set_name)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(serialize_trace_collection(collection, RAW_MAGIC))
temporary = self._temporary_file(output_path)
try:
temporary.write_bytes(serialize_trace_collection(collection, RAW_MAGIC))
temporary.replace(output_path)
finally:
temporary.unlink(missing_ok=True)
logger.info("Exported set %s/%s/%s bundle to %s", kind, radar_key, set_name, output_path)
return output_path
@@ -234,11 +337,10 @@ class NpzStore(StoreApi):
output_dir.mkdir(parents=True, exist_ok=True)
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
snapshot_dir = output_dir / f"snapshot_{timestamp}"
snapshot_dir.mkdir(parents=True, exist_ok=True)
save_trace_history_binary(snapshot_dir / "raw", raw_history[-last_n:], RAW_MAGIC)
save_trace_history_binary(snapshot_dir / "preprocessed", preprocessed_history[-last_n:], PREPROC_MAGIC)
save_result_history_binary(snapshot_dir / "results", result_history[-last_n:])
with atomic_snapshot_directory(snapshot_dir) as staging:
save_trace_history_binary(staging / "raw", raw_history[-last_n:], RAW_MAGIC)
save_trace_history_binary(staging / "preprocessed", preprocessed_history[-last_n:], PREPROC_MAGIC)
save_result_history_binary(staging / "results", result_history[-last_n:])
logger.info("Saved binary runtime snapshot (last_n=%d) to %s", last_n, snapshot_dir)
return snapshot_dir
@@ -281,6 +383,7 @@ class NpzStore(StoreApi):
last_n: int,
*,
name_prefix: str = "",
include_unaligned: bool = False,
) -> tuple[Path, dict[str, Any]]:
"""Save historical runtime collections in NumPy tree format."""
if last_n <= 0:
@@ -292,19 +395,15 @@ class NpzStore(StoreApi):
snapshot_dir = output_root_dir / snapshot_stem
if snapshot_dir.exists():
raise FileExistsError(f"Snapshot directory already exists: {snapshot_dir}")
snapshot_dir.mkdir(parents=True, exist_ok=False)
selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories(
raw_history,
preprocessed_history,
result_history,
last_n,
include_unaligned=include_unaligned,
)
save_trace_history_numpy(snapshot_dir / "raw", selected_raw)
save_trace_history_numpy(snapshot_dir / "preprocessed", selected_preprocessed)
save_result_history_numpy(snapshot_dir / "results", selected_results)
if selection_summary.get("anchor_stage") == "results":
expected = min(int(last_n), len(result_history))
if len(selected_results) != expected:
@@ -313,6 +412,11 @@ class NpzStore(StoreApi):
f"expected={expected}, selected={len(selected_results)}"
)
with atomic_snapshot_directory(snapshot_dir) as staging:
save_trace_history_numpy(staging / "raw", selected_raw)
save_trace_history_numpy(staging / "preprocessed", selected_preprocessed)
save_result_history_numpy(staging / "results", selected_results)
selection_summary["raw_count"] = len(selected_raw)
selection_summary["preprocessed_count"] = len(selected_preprocessed)
selection_summary["result_count"] = len(selected_results)
@@ -341,6 +445,7 @@ class NpzStore(StoreApi):
output_index: int = 0,
channel: str = "s21",
primary_stage: str = "preprocessed",
include_unaligned: bool = False,
) -> tuple[Path, dict[str, Any]]:
"""Save runtime history as vna_system-compatible JSON file."""
if last_n <= 0:
@@ -351,16 +456,14 @@ class NpzStore(StoreApi):
output_root_dir.mkdir(parents=True, exist_ok=True)
output_dir = self._vna_json_output_dir(output_root_dir, output_stem)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = output_dir / f"{output_stem}_vna_bscan_history.json"
if output_path.exists():
raise FileExistsError(f"Output JSON file already exists: {output_path}")
selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories(
raw_history,
preprocessed_history,
result_history,
last_n,
include_unaligned=include_unaligned,
)
payload = build_vna_history_payload(
selected_raw,
@@ -371,10 +474,8 @@ class NpzStore(StoreApi):
channel=channel,
primary_stage=primary_stage,
)
output_path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2),
encoding="utf-8",
)
with atomic_snapshot_directory(output_dir) as staging:
(staging / output_path.name).write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
summary: dict[str, Any] = dict(selection_summary)
summary["raw_count"] = len(selected_raw)
@@ -412,6 +513,7 @@ class NpzStore(StoreApi):
channel: str = "s21",
primary_stage: str = "preprocessed",
name_prefix: str = "",
include_unaligned: bool = False,
) -> tuple[list[Path], dict[str, Any]]:
"""Save one runtime VNA-history JSON per available combo."""
if last_n <= 0:
@@ -420,13 +522,13 @@ class NpzStore(StoreApi):
output_stem = _compose_snapshot_stem(output_name, name_prefix)
output_root_dir.mkdir(parents=True, exist_ok=True)
output_dir = self._vna_json_output_dir(output_root_dir, output_stem)
output_dir.mkdir(parents=True, exist_ok=True)
selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories(
raw_history,
preprocessed_history,
result_history,
last_n,
include_unaligned=include_unaligned,
)
combos = sorted(
@@ -461,11 +563,11 @@ class NpzStore(StoreApi):
)
output_paths.append(output_path)
for output_path, payload in zip(output_paths, payloads, strict=True):
output_path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2),
encoding="utf-8",
)
with atomic_snapshot_directory(output_dir) as staging:
for output_path, payload in zip(output_paths, payloads, strict=True):
(staging / output_path.name).write_text(
json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8"
)
summary: dict[str, Any] = dict(selection_summary)
summary["raw_count"] = len(selected_raw)
@@ -486,8 +588,25 @@ class NpzStore(StoreApi):
def _set_dir(self, kind: str, radar_key: str) -> Path:
"""Return directory for set kind and radar key."""
self._validate_set_name(kind)
self._validate_set_name(radar_key)
return self._root_dir / kind / radar_key
@staticmethod
def _validate_set_name(value: str) -> None:
"""Keep caller-provided set components inside their storage directory."""
if not value or value in {".", ".."} or Path(value).name != value or "\\" in value:
raise ValueError(f"Invalid set path component: {value!r}")
@staticmethod
def _temporary_file(destination: Path) -> Path:
"""Create a unique temporary file beside its atomic-replacement destination."""
with tempfile.NamedTemporaryFile(
dir=destination.parent, prefix=f".{destination.name}.", suffix=".tmp", delete=False
) as handle:
path = Path(handle.name)
return path
def preview_png_dir(self, kind: str, set_name: str, radar_key: str) -> Path:
"""Return directory for preview PNGs of one set/radar variant.