added json save button and remove history button. Added logging to save process
This commit is contained in:
Ayzen
@@ -21,21 +21,74 @@ def select_aligned_histories(
|
||||
result_history: list[ResultCollection],
|
||||
last_n: int,
|
||||
) -> tuple[list[SweepCollection], list[SweepCollection], list[ResultCollection], dict[str, Any]]:
|
||||
"""Select history tails prioritizing currently displayed processed results."""
|
||||
raw_index, _ = _index_by_collection_sequence(raw_history)
|
||||
pre_index, _ = _index_by_collection_sequence(preprocessed_history)
|
||||
"""Select aligned tails with `results` as preferred anchor stage."""
|
||||
raw_index, raw_pos = _index_by_collection_sequence(raw_history)
|
||||
pre_index, pre_pos = _index_by_collection_sequence(preprocessed_history)
|
||||
result_index, result_pos = _index_by_collection_sequence(result_history)
|
||||
|
||||
if result_index:
|
||||
ordered_result_keys = sorted(result_index, key=lambda key: result_pos[key])
|
||||
selected_keys = ordered_result_keys[-last_n:]
|
||||
selected_raw = [raw_index[key] for key in selected_keys if key in raw_index]
|
||||
selected_pre = [pre_index[key] for key in selected_keys if key in pre_index]
|
||||
selected_results = [result_index[key] for key in selected_keys]
|
||||
raw_missing = len(selected_keys) - len(selected_raw)
|
||||
pre_missing = len(selected_keys) - len(selected_pre)
|
||||
return (
|
||||
[raw_index[key] for key in selected_keys if key in raw_index],
|
||||
[pre_index[key] for key in selected_keys if key in pre_index],
|
||||
[result_index[key] for key in selected_keys],
|
||||
selected_raw,
|
||||
selected_pre,
|
||||
selected_results,
|
||||
{
|
||||
"selection_mode": "result_tail_with_optional_alignment",
|
||||
"anchor_stage": "results",
|
||||
"selected_collection_ids": [int(key[0]) for key in selected_keys],
|
||||
"aligned_key_count": len(selected_keys),
|
||||
"raw_missing_count": raw_missing,
|
||||
"preprocessed_missing_count": pre_missing,
|
||||
},
|
||||
)
|
||||
|
||||
if raw_index:
|
||||
ordered_raw_keys = sorted(raw_index, key=lambda key: raw_pos[key])
|
||||
selected_keys = ordered_raw_keys[-last_n:]
|
||||
selected_raw = [raw_index[key] for key in selected_keys]
|
||||
selected_pre = [pre_index[key] for key in selected_keys if key in pre_index]
|
||||
selected_results = [result_index[key] for key in selected_keys if key in result_index]
|
||||
pre_missing = len(selected_keys) - len(selected_pre)
|
||||
result_missing = len(selected_keys) - len(selected_results)
|
||||
return (
|
||||
selected_raw,
|
||||
selected_pre,
|
||||
selected_results,
|
||||
{
|
||||
"selection_mode": "raw_tail_with_optional_alignment",
|
||||
"anchor_stage": "raw",
|
||||
"selected_collection_ids": [int(key[0]) for key in selected_keys],
|
||||
"aligned_key_count": len(selected_keys),
|
||||
"preprocessed_missing_count": pre_missing,
|
||||
"result_missing_count": result_missing,
|
||||
},
|
||||
)
|
||||
|
||||
if pre_index:
|
||||
ordered_pre_keys = sorted(pre_index, key=lambda key: pre_pos[key])
|
||||
selected_keys = ordered_pre_keys[-last_n:]
|
||||
selected_raw = [raw_index[key] for key in selected_keys if key in raw_index]
|
||||
selected_pre = [pre_index[key] for key in selected_keys]
|
||||
selected_results = [result_index[key] for key in selected_keys if key in result_index]
|
||||
raw_missing = len(selected_keys) - len(selected_raw)
|
||||
result_missing = len(selected_keys) - len(selected_results)
|
||||
return (
|
||||
selected_raw,
|
||||
selected_pre,
|
||||
selected_results,
|
||||
{
|
||||
"selection_mode": "preprocessed_tail_with_optional_alignment",
|
||||
"anchor_stage": "preprocessed",
|
||||
"selected_collection_ids": [int(key[0]) for key in selected_keys],
|
||||
"aligned_key_count": len(selected_keys),
|
||||
"raw_missing_count": raw_missing,
|
||||
"result_missing_count": result_missing,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -45,7 +98,9 @@ def select_aligned_histories(
|
||||
result_history[-last_n:],
|
||||
{
|
||||
"selection_mode": "independent_tail",
|
||||
"anchor_stage": "none",
|
||||
"selected_collection_ids": [],
|
||||
"aligned_key_count": 0,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ from python_app.storage.npz.snapshot_numpy import (
|
||||
save_trace_history_numpy,
|
||||
select_aligned_histories,
|
||||
)
|
||||
from python_app.storage.npz.vna_history_json import build_vna_history_payload
|
||||
from python_app.storage.store_api import StoreApi
|
||||
|
||||
|
||||
@@ -176,7 +177,12 @@ class NpzStore(StoreApi):
|
||||
{
|
||||
"format": "numpy-directory-v1",
|
||||
"selection_mode": selection_summary["selection_mode"],
|
||||
"anchor_stage": selection_summary.get("anchor_stage", "unknown"),
|
||||
"selected_collection_ids": selection_summary["selected_collection_ids"],
|
||||
"aligned_key_count": int(selection_summary.get("aligned_key_count", 0)),
|
||||
"raw_missing_count": int(selection_summary.get("raw_missing_count", 0)),
|
||||
"preprocessed_missing_count": int(selection_summary.get("preprocessed_missing_count", 0)),
|
||||
"result_missing_count": int(selection_summary.get("result_missing_count", 0)),
|
||||
"raw_collections": len(selected_raw),
|
||||
"preprocessed_collections": len(selected_preprocessed),
|
||||
"result_collections": len(selected_results),
|
||||
@@ -190,12 +196,77 @@ class NpzStore(StoreApi):
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
if selection_summary.get("anchor_stage") == "results":
|
||||
expected = min(int(last_n), len(result_history))
|
||||
if len(selected_results) != expected:
|
||||
raise RuntimeError(
|
||||
"Snapshot results selection invariant failed: "
|
||||
f"expected={expected}, selected={len(selected_results)}"
|
||||
)
|
||||
|
||||
selection_summary["raw_count"] = len(selected_raw)
|
||||
selection_summary["preprocessed_count"] = len(selected_preprocessed)
|
||||
selection_summary["result_count"] = len(selected_results)
|
||||
selection_summary["snapshot_dir"] = str(snapshot_dir)
|
||||
return snapshot_dir, selection_summary
|
||||
|
||||
def save_runtime_vna_history_json(
|
||||
self,
|
||||
output_root_dir: Path,
|
||||
output_name: str,
|
||||
raw_history: list[SweepCollection],
|
||||
preprocessed_history: list[SweepCollection],
|
||||
result_history: list[ResultCollection],
|
||||
last_n: int,
|
||||
*,
|
||||
input_index: int = 0,
|
||||
output_index: int = 0,
|
||||
primary_stage: str = "preprocessed",
|
||||
) -> tuple[Path, dict[str, Any]]:
|
||||
"""Save runtime history as vna_system-compatible JSON file."""
|
||||
if last_n <= 0:
|
||||
raise ValueError("last_n must be > 0")
|
||||
|
||||
output_stem = output_name.strip() or datetime.utcnow().strftime("snapshot_%Y%m%d_%H%M%S")
|
||||
output_stem = sanitize_path_component(output_stem)
|
||||
|
||||
output_root_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_path = output_root_dir / f"{output_stem}_vna_bscan_history.json"
|
||||
if output_path.exists():
|
||||
raise FileExistsError(f"Output JSON file already exists: {output_path}")
|
||||
|
||||
selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories(
|
||||
raw_history,
|
||||
preprocessed_history,
|
||||
result_history,
|
||||
last_n,
|
||||
)
|
||||
payload = build_vna_history_payload(
|
||||
selected_raw,
|
||||
selected_preprocessed,
|
||||
selected_results,
|
||||
input_index=input_index,
|
||||
output_index=output_index,
|
||||
primary_stage=primary_stage,
|
||||
)
|
||||
output_path.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
summary: dict[str, Any] = dict(selection_summary)
|
||||
summary["raw_count"] = len(selected_raw)
|
||||
summary["preprocessed_count"] = len(selected_preprocessed)
|
||||
summary["result_count"] = len(selected_results)
|
||||
summary["raw_record_count"] = int(payload.get("raw_record_count", 0))
|
||||
summary["preprocessed_record_count"] = int(payload.get("preprocessed_record_count", 0))
|
||||
summary["sweep_count"] = len(payload.get("sweep_history", []))
|
||||
summary["input_index"] = int(input_index)
|
||||
summary["output_index"] = int(output_index)
|
||||
summary["primary_stage"] = str(primary_stage)
|
||||
summary["output_path"] = str(output_path)
|
||||
return output_path, summary
|
||||
|
||||
def _set_dir(self, kind: str, radar_key: str) -> Path:
|
||||
"""Return directory for set kind and radar key."""
|
||||
return self._root_dir / kind / radar_key
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
"""Builders for vna_system-compatible sweep-history JSON payloads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from python_app.models.dataset_model import ResultCollection, SweepCollection, TraceData
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TraceRecord:
|
||||
"""One raw/preprocessed trace selected for vna_history export."""
|
||||
|
||||
stage: str
|
||||
collection_id: int
|
||||
monotonic_ns: int
|
||||
stage_index: int
|
||||
frequency_hz: np.ndarray
|
||||
s21: np.ndarray
|
||||
|
||||
|
||||
def _pick_trace(collection: SweepCollection, input_index: int, output_index: int) -> TraceData | None:
|
||||
for trace in collection.traces:
|
||||
if int(trace.combo.input_pos) == int(input_index) and int(trace.combo.output_pos) == int(output_index):
|
||||
return trace
|
||||
return None
|
||||
|
||||
|
||||
def _build_stage_records(
|
||||
stage: str,
|
||||
history: list[SweepCollection],
|
||||
*,
|
||||
input_index: int,
|
||||
output_index: int,
|
||||
) -> list[TraceRecord]:
|
||||
records: list[TraceRecord] = []
|
||||
for stage_index, collection in enumerate(history):
|
||||
trace = _pick_trace(collection, input_index, output_index)
|
||||
if trace is None:
|
||||
continue
|
||||
|
||||
frequency_hz = np.asarray(trace.frequency_hz, dtype=np.float64).reshape(-1)
|
||||
s21 = np.asarray(trace.s21, dtype=np.complex128).reshape(-1)
|
||||
if frequency_hz.shape != s21.shape:
|
||||
raise ValueError(
|
||||
f"Shape mismatch in {stage} stage for collection_id={collection.collection_id}: "
|
||||
f"freq{frequency_hz.shape} vs s21{s21.shape}"
|
||||
)
|
||||
if frequency_hz.size == 0:
|
||||
continue
|
||||
if not (
|
||||
np.isfinite(frequency_hz).all()
|
||||
and np.isfinite(np.real(s21)).all()
|
||||
and np.isfinite(np.imag(s21)).all()
|
||||
):
|
||||
raise ValueError(f"Non-finite values in {stage} stage for collection_id={collection.collection_id}")
|
||||
|
||||
records.append(
|
||||
TraceRecord(
|
||||
stage=stage,
|
||||
collection_id=int(collection.collection_id),
|
||||
monotonic_ns=int(collection.monotonic_ns),
|
||||
stage_index=int(stage_index),
|
||||
frequency_hz=frequency_hz,
|
||||
s21=s21,
|
||||
)
|
||||
)
|
||||
return records
|
||||
|
||||
|
||||
def _index_by_collection_occurrence(records: list[TraceRecord]) -> tuple[dict[tuple[int, int], TraceRecord], list[tuple[int, int]]]:
|
||||
counters: defaultdict[int, int] = defaultdict(int)
|
||||
record_map: dict[tuple[int, int], TraceRecord] = {}
|
||||
order: list[tuple[int, int]] = []
|
||||
for record in records:
|
||||
occurrence = counters[record.collection_id]
|
||||
counters[record.collection_id] += 1
|
||||
key = (record.collection_id, occurrence)
|
||||
record_map[key] = record
|
||||
order.append(key)
|
||||
return record_map, order
|
||||
|
||||
|
||||
def _complex_to_points(values: np.ndarray) -> list[list[float]]:
|
||||
return [[float(value.real), float(value.imag)] for value in values]
|
||||
|
||||
|
||||
def _build_sweep_history(
|
||||
raw_records: list[TraceRecord],
|
||||
preprocessed_records: list[TraceRecord],
|
||||
*,
|
||||
primary_stage: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
raw_map, raw_order = _index_by_collection_occurrence(raw_records)
|
||||
pre_map, pre_order = _index_by_collection_occurrence(preprocessed_records)
|
||||
|
||||
if primary_stage == "preprocessed":
|
||||
primary_order = pre_order or raw_order
|
||||
else:
|
||||
primary_order = raw_order or pre_order
|
||||
|
||||
history: list[dict[str, Any]] = []
|
||||
for fallback_index, key in enumerate(primary_order):
|
||||
raw = raw_map.get(key)
|
||||
pre = pre_map.get(key)
|
||||
base = pre or raw
|
||||
if base is None:
|
||||
continue
|
||||
|
||||
sweep_source = raw or pre
|
||||
calibrated_source = pre or raw
|
||||
if sweep_source is None or calibrated_source is None:
|
||||
continue
|
||||
|
||||
start_freq_hz = float(base.frequency_hz[0])
|
||||
stop_freq_hz = float(base.frequency_hz[-1])
|
||||
timestamp_sec = float(base.monotonic_ns) / 1_000_000_000.0 if base.monotonic_ns > 0 else float(fallback_index)
|
||||
|
||||
history.append(
|
||||
{
|
||||
"timestamp": timestamp_sec,
|
||||
"sweep_points": _complex_to_points(sweep_source.s21),
|
||||
"calibrated_points": _complex_to_points(calibrated_source.s21),
|
||||
"reference_points": [],
|
||||
"vna_config": {
|
||||
"mode": "s11",
|
||||
"start_freq": start_freq_hz,
|
||||
"stop_freq": stop_freq_hz,
|
||||
"points": int(base.frequency_hz.size),
|
||||
},
|
||||
}
|
||||
)
|
||||
return history
|
||||
|
||||
|
||||
def _stage_alignment_warning(pre_history: list[SweepCollection], result_history: list[ResultCollection]) -> str | None:
|
||||
if not pre_history or not result_history:
|
||||
return None
|
||||
|
||||
common_size = min(len(pre_history), len(result_history))
|
||||
if common_size <= 0:
|
||||
return None
|
||||
|
||||
mismatches = 0
|
||||
first_mismatch: tuple[int, SweepCollection, ResultCollection] | None = None
|
||||
for index in range(common_size):
|
||||
pre = pre_history[index]
|
||||
result = result_history[index]
|
||||
if int(pre.collection_id) != int(result.collection_id) or int(pre.monotonic_ns) != int(result.monotonic_ns):
|
||||
mismatches += 1
|
||||
if first_mismatch is None:
|
||||
first_mismatch = (index, pre, result)
|
||||
|
||||
if mismatches == 0 or first_mismatch is None:
|
||||
return None
|
||||
|
||||
index, pre, result = first_mismatch
|
||||
return (
|
||||
"WARNING: snapshot stages are not fully aligned (preprocessed vs results). "
|
||||
f"Mismatches={mismatches}/{common_size}. "
|
||||
f"First mismatch at index={index}: "
|
||||
f"pre=(id={int(pre.collection_id)},ns={int(pre.monotonic_ns)}) vs "
|
||||
f"results=(id={int(result.collection_id)},ns={int(result.monotonic_ns)}). "
|
||||
"Export uses preprocessed traces; loaded view in vna_system may differ from "
|
||||
"radar_system on-screen replayed results."
|
||||
)
|
||||
|
||||
|
||||
def build_vna_history_payload(
|
||||
raw_history: list[SweepCollection],
|
||||
preprocessed_history: list[SweepCollection],
|
||||
result_history: list[ResultCollection],
|
||||
*,
|
||||
input_index: int,
|
||||
output_index: int,
|
||||
primary_stage: str = "preprocessed",
|
||||
) -> dict[str, Any]:
|
||||
"""Build vna_system-compatible history JSON payload from runtime histories."""
|
||||
if primary_stage not in {"preprocessed", "raw"}:
|
||||
raise ValueError("primary_stage must be either 'preprocessed' or 'raw'")
|
||||
|
||||
raw_records = _build_stage_records(
|
||||
"raw",
|
||||
raw_history,
|
||||
input_index=input_index,
|
||||
output_index=output_index,
|
||||
)
|
||||
preprocessed_records = _build_stage_records(
|
||||
"preprocessed",
|
||||
preprocessed_history,
|
||||
input_index=input_index,
|
||||
output_index=output_index,
|
||||
)
|
||||
if not raw_records and not preprocessed_records:
|
||||
raise ValueError(
|
||||
"No matching raw/preprocessed traces were found in runtime history "
|
||||
f"for input={input_index}, output={output_index}."
|
||||
)
|
||||
|
||||
sweep_history = _build_sweep_history(
|
||||
raw_records,
|
||||
preprocessed_records,
|
||||
primary_stage=primary_stage,
|
||||
)
|
||||
if not sweep_history:
|
||||
raise ValueError("Conversion produced empty sweep_history.")
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"format": "vna-system-history-v1",
|
||||
"converter": "python_app/scripts/convert_snapshot_to_vna_history.py",
|
||||
"converted_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"source_snapshot_dir": "<runtime_history>",
|
||||
"input_index": int(input_index),
|
||||
"output_index": int(output_index),
|
||||
"primary_stage": str(primary_stage),
|
||||
"raw_record_count": len(raw_records),
|
||||
"preprocessed_record_count": len(preprocessed_records),
|
||||
"sweep_history": sweep_history,
|
||||
}
|
||||
|
||||
alignment_warning = _stage_alignment_warning(preprocessed_history, result_history)
|
||||
if alignment_warning is not None:
|
||||
payload["alignment_warning"] = alignment_warning
|
||||
|
||||
return payload
|
||||
|
||||
Reference in New Issue
Block a user