awe/radar_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improved logging

Browse Source
This commit is contained in:
Ayzen
2026-06-06 00:52:52 +03:00
parent af6005d68f
commit aea49f6128
65 changed files with 1206 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python_app/storage/npz/paths.py
+4 -9
View File
@@ -21,11 +21,11 @@ def radar_key_from_config(
if extra_serials:
serial_parts.extend(str(value).strip() or "no_serial" for value in extra_serials)
serial_part = "_".join(sanitize_path_component(value) for value in serial_parts)
start_token = _format_float_for_key(sweep_start_hz)
stop_token = _format_float_for_key(sweep_stop_hz)
start_token = format_float_for_key(sweep_start_hz)
stop_token = format_float_for_key(sweep_stop_hz)
points_token = "adc" if model_name.strip().lower() == "kamil_adc" else str(int(sweep_points))
ifbw_token = _format_float_for_key(ifbw_hz)
power_token = _format_float_for_key(power_dbm)
ifbw_token = format_float_for_key(ifbw_hz)
power_token = format_float_for_key(power_dbm)
return (
f"{model_name}_{serial_part}"
f"_st{start_token}_sp{stop_token}"
@@ -51,8 +51,3 @@ def format_float_for_key(value: float) -> str:
if abs(value - float(integer)) < 1e-6:
return str(integer)
return f"{value:.6f}".rstrip("0").rstrip(".")
def _format_float_for_key(value: float) -> str:
"""Private alias preserved for internal compatibility."""
return format_float_for_key(value)
python_app/storage/npz/snapshot_numpy.py
+7
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import Any, TypeVar
@@ -12,6 +13,8 @@ from python_app.models.dataset_model import ResultCollection, SweepCollection
from python_app.storage.npz.paths import collection_dir_name, sanitize_path_component
from python_app.storage.npz.serialize import serialize_result_collection, serialize_trace_collection
logger = logging.getLogger(__name__)
TCollection = TypeVar("TCollection")
@@ -108,6 +111,7 @@ def select_aligned_histories(
def save_trace_history_binary(stage_dir: Path, history: list[SweepCollection], magic: int) -> None:
"""Write binary trace history with lightweight metadata sidecars."""
stage_dir.mkdir(parents=True, exist_ok=True)
logger.debug("Writing %d binary trace collection(s) to %s", len(history), stage_dir)
for index, collection in enumerate(history):
binary_path = stage_dir / f"{index:04d}.bin"
metadata_path = stage_dir / f"{index:04d}.json"
@@ -130,6 +134,7 @@ def save_trace_history_binary(stage_dir: Path, history: list[SweepCollection], m
def save_result_history_binary(stage_dir: Path, history: list[ResultCollection]) -> None:
"""Write binary processed-result history with metadata sidecars."""
stage_dir.mkdir(parents=True, exist_ok=True)
logger.debug("Writing %d binary result collection(s) to %s", len(history), stage_dir)
for index, collection in enumerate(history):
binary_path = stage_dir / f"{index:04d}.bin"
metadata_path = stage_dir / f"{index:04d}.json"
@@ -151,6 +156,7 @@ def save_result_history_binary(stage_dir: Path, history: list[ResultCollection])
def save_trace_history_numpy(stage_dir: Path, history: list[SweepCollection]) -> None:
"""Write raw/preprocessed collections as NumPy directory tree."""
stage_dir.mkdir(parents=True, exist_ok=True)
logger.debug("Writing %d NumPy trace collection(s) to %s", len(history), stage_dir)
for index, collection in enumerate(history):
collection_dir = stage_dir / collection_dir_name(index, collection.collection_id, collection.monotonic_ns)
collection_dir.mkdir(parents=True, exist_ok=False)
@@ -194,6 +200,7 @@ def save_trace_history_numpy(stage_dir: Path, history: list[SweepCollection]) ->
def save_result_history_numpy(stage_dir: Path, history: list[ResultCollection]) -> None:
"""Write processed result collections as NumPy directory tree."""
stage_dir.mkdir(parents=True, exist_ok=True)
logger.debug("Writing %d NumPy result collection(s) to %s", len(history), stage_dir)
for index, collection in enumerate(history):
collection_dir = stage_dir / collection_dir_name(index, collection.collection_id, collection.monotonic_ns)
collection_dir.mkdir(parents=True, exist_ok=False)
python_app/storage/npz/store.py
+35 -1
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
from contextlib import suppress
from datetime import datetime, timezone
import json
import logging
from pathlib import Path
from typing import Any
@@ -23,6 +24,8 @@ from python_app.storage.npz.snapshot_numpy import (
from python_app.storage.npz.vna_history_json import build_vna_history_payload
from python_app.storage.store_api import StoreApi
logger = logging.getLogger(__name__)
class NpzStore(StoreApi):
"""Persist preprocess sets and runtime snapshots using NumPy files."""
@@ -31,6 +34,7 @@ class NpzStore(StoreApi):
"""Create store rooted at `root_dir`."""
self._root_dir = root_dir
self._root_dir.mkdir(parents=True, exist_ok=True)
logger.debug("NpzStore rooted at %s", self._root_dir)
@staticmethod
def _vna_json_output_dir(output_root_dir: Path, output_stem: str) -> Path:
@@ -85,10 +89,14 @@ class NpzStore(StoreApi):
npz_tmp.replace(npz_path)
meta_tmp.replace(meta_path)
except BaseException:
logger.exception("Failed to save set %s/%s/%s; rolling back temp files", kind, radar_key, set_name)
for tmp_path in (npz_tmp, meta_tmp):
with suppress(OSError):
tmp_path.unlink(missing_ok=True)
raise
logger.info(
"Saved set %s/%s/%s (%d traces) to %s", kind, radar_key, set_name, len(combo_records), npz_path
)
def load_set(self, kind: str, radar_key: str, set_name: str) -> SweepCollection:
"""Load named preprocess set from NPZ representation."""
@@ -97,6 +105,7 @@ class NpzStore(StoreApi):
meta_path = set_dir / f"{set_name}.json"
if not npz_path.exists() or not meta_path.exists():
logger.error("Missing set files for %s/%s/%s", kind, radar_key, set_name)
raise FileNotFoundError(f"Missing set files for {kind}/{radar_key}/{set_name}")
set_label = f"{kind}/{radar_key}/{set_name}"
@@ -118,14 +127,17 @@ class NpzStore(StoreApi):
)
)
return SweepCollection(
collection = SweepCollection(
collection_id=int(meta["collection_id"]),
monotonic_ns=int(meta["monotonic_ns"]),
traces=traces,
capture_start_ns=int(meta.get("capture_start_ns", 0)),
capture_end_ns=int(meta.get("capture_end_ns", 0)),
)
logger.debug("Loaded set %s (%d traces)", set_label, len(traces))
return collection
except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:
logger.exception("Corrupted preprocess set %s", set_label)
raise RuntimeError(f"Corrupted preprocess set {set_label}: {exc}") from exc
def list_sets(self, kind: str, radar_key: str) -> list[str]:
@@ -147,6 +159,7 @@ class NpzStore(StoreApi):
collection = self.load_set(kind, radar_key, set_name)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(serialize_trace_collection(collection, RAW_MAGIC))
logger.info("Exported set %s/%s/%s bundle to %s", kind, radar_key, set_name, output_path)
return output_path
def save_runtime_snapshot(
@@ -169,6 +182,7 @@ class NpzStore(StoreApi):
save_trace_history_binary(snapshot_dir / "raw", raw_history[-last_n:], RAW_MAGIC)
save_trace_history_binary(snapshot_dir / "preprocessed", preprocessed_history[-last_n:], PREPROC_MAGIC)
save_result_history_binary(snapshot_dir / "results", result_history[-last_n:])
logger.info("Saved binary runtime snapshot (last_n=%d) to %s", last_n, snapshot_dir)
return snapshot_dir
def save_runtime_snapshot_numpy(
@@ -217,6 +231,14 @@ class NpzStore(StoreApi):
selection_summary["result_count"] = len(selected_results)
selection_summary["snapshot_stem"] = snapshot_stem
selection_summary["snapshot_dir"] = str(snapshot_dir)
logger.info(
"Saved NumPy runtime snapshot to %s (raw=%d preprocessed=%d results=%d, anchor=%s)",
snapshot_dir,
len(selected_raw),
len(selected_preprocessed),
len(selected_results),
selection_summary.get("anchor_stage"),
)
return snapshot_dir, selection_summary
def save_runtime_vna_history_json(
@@ -281,6 +303,14 @@ class NpzStore(StoreApi):
summary["output_stem"] = output_stem
summary["output_dir"] = str(output_dir)
summary["output_path"] = str(output_path)
logger.info(
"Saved VNA history JSON to %s (input=%d output=%d channel=%s sweeps=%d)",
output_path,
int(input_index),
int(output_index),
channel,
summary["sweep_count"],
)
return output_path, summary
def save_runtime_vna_history_json_batch(
@@ -320,6 +350,7 @@ class NpzStore(StoreApi):
}
)
if not combos:
logger.warning("No raw/preprocessed combos found in runtime history for VNA JSON batch export")
raise ValueError("No matching raw/preprocessed traces were found in runtime history for any combo.")
output_paths: list[Path] = []
@@ -361,6 +392,9 @@ class NpzStore(StoreApi):
summary["output_stem"] = output_stem
summary["output_dir"] = str(output_dir)
summary["output_paths"] = [str(path) for path in output_paths]
logger.info(
"Saved %d VNA history JSON file(s) to %s (channel=%s)", len(output_paths), output_dir, channel
)
return output_paths, summary
def _set_dir(self, kind: str, radar_key: str) -> Path:
python_app/storage/npz/vna_history_json.py
+16
View File
@@ -5,12 +5,15 @@ from __future__ import annotations
from collections import defaultdict
from dataclasses import dataclass
from datetime import datetime, timezone
import logging
from typing import Any
import numpy as np
from python_app.models.dataset_model import ResultCollection, SweepCollection, TraceData
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class TraceRecord:
@@ -247,5 +250,18 @@ def build_vna_history_payload(
alignment_warning = _stage_alignment_warning(preprocessed_history, result_history)
if alignment_warning is not None:
payload["alignment_warning"] = alignment_warning
logger.warning(
"VNA history export (input=%d output=%d): preprocessed/results stages not fully aligned",
input_index,
output_index,
)
logger.debug(
"Built VNA history payload: input=%d output=%d raw=%d preprocessed=%d sweeps=%d",
input_index,
output_index,
len(raw_records),
len(preprocessed_records),
len(sweep_history),
)
return payload