init commit

python_app/scripts/__init__.py

@@ -0,0 +1 @@
+"""Utility scripts for smoke checks, inspection, and manual diagnostics."""

python_app/scripts/check_snapshot_numpy.py

@@ -0,0 +1,298 @@
+"""Validate and visualize numpy-directory runtime snapshots."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+
+def _load_json(path: Path) -> dict[str, Any]:
+    """Load JSON object from file path."""
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(payload, dict):
+        raise ValueError(f"JSON root must be object: {path}")
+    return payload
+
+
+def _collection_dirs(stage_dir: Path) -> list[Path]:
+    """Return sorted list of collection directories for one stage."""
+    if not stage_dir.exists():
+        return []
+    return sorted([path for path in stage_dir.iterdir() if path.is_dir()], key=lambda path: path.name)
+
+
+def _first_trace_from_raw_or_pre(collection_dir: Path) -> tuple[np.ndarray, np.ndarray, str] | None:
+    """Return first trace payload from raw/preprocessed collection."""
+    traces = _all_traces_from_raw_or_pre(collection_dir)
+    if not traces:
+        return None
+    return traces[0]
+
+
+def _all_traces_from_raw_or_pre(collection_dir: Path) -> list[tuple[np.ndarray, np.ndarray, str]]:
+    """Load all traces from raw/preprocessed collection directory."""
+    meta = _load_json(collection_dir / "meta.json")
+    traces = meta.get("traces", [])
+    if not isinstance(traces, list):
+        raise ValueError(f"Invalid traces in {collection_dir / 'meta.json'}")
+
+    all_traces: list[tuple[np.ndarray, np.ndarray, str]] = []
+    for trace_meta in traces:
+        if not isinstance(trace_meta, dict):
+            raise ValueError(f"Invalid trace record in {collection_dir / 'meta.json'}")
+
+        freq_file = str(trace_meta.get("freq_file", ""))
+        s21_file = str(trace_meta.get("s21_file", ""))
+        freq = np.load(collection_dir / freq_file)
+        s21 = np.load(collection_dir / s21_file)
+        if freq.shape != s21.shape:
+            raise ValueError(f"Shape mismatch freq/s21 in {collection_dir}")
+        if not (np.isfinite(freq).all() and np.isfinite(np.real(s21)).all() and np.isfinite(np.imag(s21)).all()):
+            raise ValueError(f"Non-finite values in {collection_dir}")
+
+        label = f"i{int(trace_meta.get('input', 0))}_o{int(trace_meta.get('output', 0))}"
+        all_traces.append((np.asarray(freq, dtype=np.float64), np.asarray(s21, dtype=np.complex128), label))
+
+    return all_traces
+
+
+def _first_trace_from_results(collection_dir: Path) -> tuple[np.ndarray, np.ndarray, str] | None:
+    """Return first trace payload from results collection."""
+    traces = _all_traces_from_results(collection_dir)
+    if not traces:
+        return None
+    return traces[0]
+
+
+def _all_traces_from_results(collection_dir: Path) -> list[tuple[np.ndarray, np.ndarray, str]]:
+    """Load all trace-like payloads from results collection directory."""
+    meta = _load_json(collection_dir / "meta.json")
+    blocks = meta.get("blocks", [])
+    if not isinstance(blocks, list):
+        raise ValueError(f"Invalid blocks in {collection_dir / 'meta.json'}")
+
+    all_traces: list[tuple[np.ndarray, np.ndarray, str]] = []
+    for block in blocks:
+        if not isinstance(block, dict):
+            continue
+        block_dir_name = str(block.get("dir", ""))
+        block_dir = collection_dir / block_dir_name
+        payloads = block.get("payloads", [])
+        if not isinstance(payloads, list):
+            continue
+        for payload in payloads:
+            if not isinstance(payload, dict):
+                continue
+            kind = int(payload.get("kind", 0))
+            if kind != 1:
+                continue
+            freq_file = str(payload.get("freq_file", ""))
+            trace_file = str(payload.get("trace_file", ""))
+            freq = np.load(block_dir / freq_file)
+            trace = np.load(block_dir / trace_file)
+            if freq.shape != trace.shape:
+                raise ValueError(f"Shape mismatch freq/trace in {block_dir}")
+            if not (
+                np.isfinite(freq).all()
+                and np.isfinite(np.real(trace)).all()
+                and np.isfinite(np.imag(trace)).all()
+            ):
+                raise ValueError(f"Non-finite values in {block_dir}")
+
+            label = (
+                f"i{int(block.get('input', 0))}_o{int(block.get('output', 0))}_"
+                f"{str(payload.get('name', 'processor'))}"
+            )
+            all_traces.append((np.asarray(freq, dtype=np.float64), np.asarray(trace, dtype=np.complex128), label))
+
+    return all_traces
+
+
+def _validate_stage(stage_dir: Path, stage: str) -> tuple[list[int], list[tuple[np.ndarray, np.ndarray, str]]]:
+    """Validate one stage directory and collect representative traces."""
+    collection_ids: list[int] = []
+    traces: list[tuple[np.ndarray, np.ndarray, str]] = []
+    for collection_dir in _collection_dirs(stage_dir):
+        meta = _load_json(collection_dir / "meta.json")
+        collection_ids.append(int(meta.get("collection_id", -1)))
+        if stage in {"raw", "preprocessed"}:
+            trace = _first_trace_from_raw_or_pre(collection_dir)
+        else:
+            trace = _first_trace_from_results(collection_dir)
+        if trace is not None:
+            traces.append(trace)
+    return collection_ids, traces
+
+
+def _compare_two(
+    name: str,
+    first: tuple[np.ndarray, np.ndarray, str],
+    second: tuple[np.ndarray, np.ndarray, str],
+) -> None:
+    """Print numerical difference metrics for two traces."""
+    freq_a, data_a, label_a = first
+    freq_b, data_b, label_b = second
+    same_shape = freq_a.shape == freq_b.shape == data_a.shape == data_b.shape
+    if not same_shape:
+        print(f"[{name}] different shapes: {freq_a.shape}/{freq_b.shape} {data_a.shape}/{data_b.shape}")
+        return
+
+    are_equal = np.array_equal(data_a, data_b)
+    diff = data_a - data_b
+    max_abs_diff = float(np.max(np.abs(diff)))
+    l2_diff = float(np.linalg.norm(diff))
+    print(
+        f"[{name}] compare first two traces: {label_a} vs {label_b}, "
+        f"equal={are_equal}, max_abs_diff={max_abs_diff:.6g}, l2_diff={l2_diff:.6g}"
+    )
+
+
+def _plot_two(
+    stage: str,
+    first: tuple[np.ndarray, np.ndarray, str],
+    second: tuple[np.ndarray, np.ndarray, str],
+    output_dir: Path,
+) -> None:
+    """Plot magnitude comparison for two traces."""
+    try:
+        import matplotlib.pyplot as plt
+    except Exception as exc:  # noqa: BLE001
+        print(f"[{stage}] matplotlib is not available, plot skipped: {exc}")
+        return
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    freq_a, data_a, label_a = first
+    freq_b, data_b, label_b = second
+    if freq_a.shape != freq_b.shape or data_a.shape != data_b.shape:
+        print(f"[{stage}] shapes differ, plot skipped")
+        return
+
+    y_a = 20.0 * np.log10(np.maximum(np.abs(data_a), 1e-12))
+    y_b = 20.0 * np.log10(np.maximum(np.abs(data_b), 1e-12))
+
+    fig, ax = plt.subplots(figsize=(10, 4))
+    ax.plot(freq_a, y_a, linewidth=1.4, label=f"collection#1 {label_a}")
+    ax.plot(freq_b, y_b, linewidth=1.4, label=f"collection#2 {label_b}")
+    ax.set_title(f"{stage}: first two collections")
+    ax.set_xlabel("X axis")
+    ax.set_ylabel("Magnitude dB")
+    ax.grid(True, alpha=0.3)
+    ax.legend()
+    fig.tight_layout()
+
+    png_path = output_dir / f"{stage}_first_two.png"
+    fig.savefig(png_path, dpi=150)
+    plt.close(fig)
+    print(f"[{stage}] plot saved: {png_path}")
+
+
+def _plot_all_states_for_one_collection(stage: str, collection_dir: Path, output_dir: Path) -> None:
+    """Plot all switch-state traces available in one collection."""
+    try:
+        import matplotlib.pyplot as plt
+    except Exception as exc:  # noqa: BLE001
+        print(f"[{stage}] matplotlib is not available, all-states plot skipped: {exc}")
+        return
+
+    if stage in {"raw", "preprocessed"}:
+        traces = _all_traces_from_raw_or_pre(collection_dir)
+    else:
+        traces = _all_traces_from_results(collection_dir)
+
+    if not traces:
+        print(f"[{stage}] no trace payloads found in {collection_dir.name}, all-states plot skipped")
+        return
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    meta = _load_json(collection_dir / "meta.json")
+    collection_id = int(meta.get("collection_id", -1))
+
+    fig, ax = plt.subplots(figsize=(11, 5))
+    for freq, data, label in traces:
+        y = 20.0 * np.log10(np.maximum(np.abs(data), 1e-12))
+        ax.plot(freq, y, linewidth=1.2, label=label)
+    ax.set_title(f"{stage}: all switch states in one collection (id={collection_id})")
+    ax.set_xlabel("X axis")
+    ax.set_ylabel("Magnitude dB")
+    ax.grid(True, alpha=0.3)
+    ax.legend(fontsize=8, ncol=2)
+    fig.tight_layout()
+
+    png_path = output_dir / f"{stage}_all_states_one_collection.png"
+    fig.savefig(png_path, dpi=150)
+    plt.close(fig)
+    print(f"[{stage}] all-states plot saved: {png_path} (traces={len(traces)})")
+
+
+def _run(snapshot_dir: Path, output_dir: Path) -> None:
+    """Execute snapshot validation and plotting workflow."""
+    manifest_path = snapshot_dir / "manifest.json"
+    if manifest_path.exists():
+        manifest = _load_json(manifest_path)
+        print(
+            f"Snapshot: {snapshot_dir}\n"
+            f"selection_mode={manifest.get('selection_mode')} "
+            f"raw={manifest.get('raw_collections')} "
+            f"pre={manifest.get('preprocessed_collections')} "
+            f"res={manifest.get('result_collections')}"
+        )
+    else:
+        print(f"Snapshot: {snapshot_dir} (manifest.json is missing)")
+
+    stages = ("raw", "preprocessed", "results")
+    for stage in stages:
+        stage_dir = snapshot_dir / stage
+        collection_dirs = _collection_dirs(stage_dir)
+        collection_ids, traces = _validate_stage(stage_dir, stage)
+        duplicate_id_count = len(collection_ids) - len(set(collection_ids))
+        print(
+            f"[{stage}] collections={len(collection_ids)}, "
+            f"duplicate_ids={duplicate_id_count}, "
+            f"trace_samples={len(traces)}"
+        )
+        if len(traces) >= 2:
+            _compare_two(stage, traces[0], traces[1])
+            _plot_two(stage, traces[0], traces[1], output_dir)
+        else:
+            print(f"[{stage}] not enough trace-like collections to compare/plot (need >= 2)")
+
+        if collection_dirs:
+            _plot_all_states_for_one_collection(stage, collection_dirs[-1], output_dir)
+        else:
+            print(f"[{stage}] no collections for all-states plot")
+
+
+def main() -> None:
+    """CLI entrypoint."""
+    parser = argparse.ArgumentParser(description="Validate and visualize runtime numpy snapshot collections.")
+    parser.add_argument(
+        "snapshot_dir",
+        type=Path,
+        help="Path to snapshot directory (contains raw/preprocessed/results).",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        default=None,
+        help="Directory for output plots (default: <snapshot_dir>/inspection_plots).",
+    )
+    args = parser.parse_args()
+
+    snapshot_dir = args.snapshot_dir.expanduser().resolve()
+    if not snapshot_dir.exists():
+        raise FileNotFoundError(f"Snapshot directory not found: {snapshot_dir}")
+
+    output_dir = (
+        args.output_dir.expanduser().resolve()
+        if args.output_dir is not None
+        else snapshot_dir / "inspection_plots"
+    )
+    _run(snapshot_dir, output_dir)
+
+
+if __name__ == "__main__":
+    main()

python_app/scripts/convert_snapshot_to_vna_history.py

@@ -0,0 +1,353 @@
+"""Convert radar_system runtime snapshots into vna_system sweep-history JSON."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections import defaultdict
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+
+@dataclass(frozen=True)
+class TraceRecord:
+    """One raw/preprocessed trace extracted from a snapshot collection directory."""
+
+    stage: str
+    collection_id: int
+    monotonic_ns: int
+    stage_index: int
+    frequency_hz: np.ndarray
+    s21: np.ndarray
+
+
+@dataclass(frozen=True)
+class CollectionRef:
+    """Minimal collection identity descriptor for stage-alignment diagnostics."""
+
+    stage_index: int
+    collection_id: int
+    monotonic_ns: int
+
+
+def _load_json(path: Path) -> dict[str, Any]:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(payload, dict):
+        raise ValueError(f"JSON root must be object: {path}")
+    return payload
+
+
+def _collection_dirs(stage_dir: Path) -> list[Path]:
+    if not stage_dir.exists():
+        return []
+    return sorted([path for path in stage_dir.iterdir() if path.is_dir()], key=lambda path: path.name)
+
+
+def _parse_stage_index(name: str, fallback: int) -> int:
+    prefix = name.split("_", 1)[0]
+    return int(prefix) if prefix.isdigit() else fallback
+
+
+def _pick_trace_meta(meta: dict[str, Any], input_index: int, output_index: int) -> dict[str, Any] | None:
+    traces = meta.get("traces", [])
+    if not isinstance(traces, list):
+        return None
+    for trace in traces:
+        if not isinstance(trace, dict):
+            continue
+        if int(trace.get("input", -1)) == input_index and int(trace.get("output", -1)) == output_index:
+            return trace
+    return None
+
+
+def _load_stage_records(
+    snapshot_dir: Path,
+    stage: str,
+    *,
+    input_index: int,
+    output_index: int,
+) -> list[TraceRecord]:
+    stage_dir = snapshot_dir / stage
+    records: list[TraceRecord] = []
+
+    for fallback_idx, collection_dir in enumerate(_collection_dirs(stage_dir)):
+        meta_path = collection_dir / "meta.json"
+        if not meta_path.exists():
+            continue
+
+        meta = _load_json(meta_path)
+        trace_meta = _pick_trace_meta(meta, input_index, output_index)
+        if trace_meta is None:
+            continue
+
+        freq_file = str(trace_meta.get("freq_file", ""))
+        s21_file = str(trace_meta.get("s21_file", ""))
+        if not freq_file or not s21_file:
+            continue
+
+        frequency_hz = np.asarray(np.load(collection_dir / freq_file), dtype=np.float64).reshape(-1)
+        s21 = np.asarray(np.load(collection_dir / s21_file), dtype=np.complex128).reshape(-1)
+        if frequency_hz.shape != s21.shape:
+            raise ValueError(f"Shape mismatch in {collection_dir}: freq{frequency_hz.shape} vs s21{s21.shape}")
+        if frequency_hz.size == 0:
+            continue
+        if not (
+            np.isfinite(frequency_hz).all()
+            and np.isfinite(np.real(s21)).all()
+            and np.isfinite(np.imag(s21)).all()
+        ):
+            raise ValueError(f"Non-finite values in {collection_dir}")
+
+        records.append(
+            TraceRecord(
+                stage=stage,
+                collection_id=int(meta.get("collection_id", -1)),
+                monotonic_ns=int(meta.get("monotonic_ns", 0)),
+                stage_index=_parse_stage_index(collection_dir.name, fallback_idx),
+                frequency_hz=frequency_hz,
+                s21=s21,
+            )
+        )
+
+    return records
+
+
+def _load_stage_refs(snapshot_dir: Path, stage: str) -> list[CollectionRef]:
+    """Load `(index, collection_id, monotonic_ns)` for one snapshot stage."""
+    stage_dir = snapshot_dir / stage
+    refs: list[CollectionRef] = []
+
+    for fallback_idx, collection_dir in enumerate(_collection_dirs(stage_dir)):
+        meta_path = collection_dir / "meta.json"
+        if not meta_path.exists():
+            continue
+        meta = _load_json(meta_path)
+        refs.append(
+            CollectionRef(
+                stage_index=_parse_stage_index(collection_dir.name, fallback_idx),
+                collection_id=int(meta.get("collection_id", -1)),
+                monotonic_ns=int(meta.get("monotonic_ns", 0)),
+            )
+        )
+    return refs
+
+
+def _index_by_collection_occurrence(records: list[TraceRecord]) -> tuple[dict[tuple[int, int], TraceRecord], list[tuple[int, int]]]:
+    counters: defaultdict[int, int] = defaultdict(int)
+    record_map: dict[tuple[int, int], TraceRecord] = {}
+    order: list[tuple[int, int]] = []
+    for record in records:
+        occurrence = counters[record.collection_id]
+        counters[record.collection_id] += 1
+        key = (record.collection_id, occurrence)
+        record_map[key] = record
+        order.append(key)
+    return record_map, order
+
+
+def _complex_to_points(values: np.ndarray) -> list[list[float]]:
+    return [[float(v.real), float(v.imag)] for v in values]
+
+
+def _build_sweep_history(
+    raw_records: list[TraceRecord],
+    preprocessed_records: list[TraceRecord],
+    *,
+    primary_stage: str,
+) -> list[dict[str, Any]]:
+    raw_map, raw_order = _index_by_collection_occurrence(raw_records)
+    pre_map, pre_order = _index_by_collection_occurrence(preprocessed_records)
+
+    if primary_stage == "preprocessed":
+        primary_order = pre_order or raw_order
+    else:
+        primary_order = raw_order or pre_order
+
+    history: list[dict[str, Any]] = []
+    for fallback_index, key in enumerate(primary_order):
+        raw = raw_map.get(key)
+        pre = pre_map.get(key)
+        base = pre or raw
+        if base is None:
+            continue
+
+        # vna_system uses calibrated_data if present, otherwise sweep_data.
+        sweep_source = raw or pre
+        calibrated_source = pre or raw
+        if sweep_source is None or calibrated_source is None:
+            continue
+
+        start_freq_hz = float(base.frequency_hz[0])
+        stop_freq_hz = float(base.frequency_hz[-1])
+        timestamp_sec = float(base.monotonic_ns) / 1_000_000_000.0 if base.monotonic_ns > 0 else float(fallback_index)
+
+        history.append(
+            {
+                "timestamp": timestamp_sec,
+                "sweep_points": _complex_to_points(sweep_source.s21),
+                "calibrated_points": _complex_to_points(calibrated_source.s21),
+                "reference_points": [],
+                "vna_config": {
+                    "mode": "s11",
+                    "start_freq": start_freq_hz,
+                    "stop_freq": stop_freq_hz,
+                    "points": int(base.frequency_hz.size),
+                },
+            }
+        )
+
+    return history
+
+
+def _stage_alignment_warning(pre_refs: list[CollectionRef], result_refs: list[CollectionRef]) -> str | None:
+    """Return warning text when preprocessed/results stages are not identity-aligned."""
+    if not pre_refs or not result_refs:
+        return None
+
+    pre_by_index = {ref.stage_index: ref for ref in pre_refs}
+    result_by_index = {ref.stage_index: ref for ref in result_refs}
+    common_indices = sorted(set(pre_by_index) & set(result_by_index))
+    if not common_indices:
+        return None
+
+    mismatches = 0
+    first_mismatch: tuple[int, CollectionRef, CollectionRef] | None = None
+    for index in common_indices:
+        pre = pre_by_index[index]
+        result = result_by_index[index]
+        if pre.collection_id != result.collection_id or pre.monotonic_ns != result.monotonic_ns:
+            mismatches += 1
+            if first_mismatch is None:
+                first_mismatch = (index, pre, result)
+
+    if mismatches == 0:
+        return None
+
+    assert first_mismatch is not None
+    idx, pre, result = first_mismatch
+    return (
+        "WARNING: snapshot stages are not fully aligned (preprocessed vs results). "
+        f"Mismatches={mismatches}/{len(common_indices)}. "
+        f"First mismatch at index={idx}: "
+        f"pre=(id={pre.collection_id},ns={pre.monotonic_ns}) vs "
+        f"results=(id={result.collection_id},ns={result.monotonic_ns}). "
+        "Export uses preprocessed traces; loaded view in vna_system may differ from "
+        "radar_system on-screen replayed results."
+    )
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Convert radar_system snapshot (numpy-directory-v1) to a vna_system-compatible "
+            "history JSON file with `sweep_history`."
+        )
+    )
+    parser.add_argument("snapshot_dir", type=Path, help="Path to snapshot directory containing raw/preprocessed/results.")
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        default=None,
+        help="Output JSON path (default: <snapshot_dir>/vna_bscan_history.json).",
+    )
+    parser.add_argument("--input", dest="input_index", type=int, default=0, help="Input switch index to export.")
+    parser.add_argument("--output-index", dest="output_index", type=int, default=0, help="Output switch index to export.")
+    parser.add_argument(
+        "--primary-stage",
+        choices=("preprocessed", "raw"),
+        default="preprocessed",
+        help="Stage order to drive collection selection/alignment.",
+    )
+    parser.add_argument(
+        "--last-n",
+        type=int,
+        default=0,
+        help="Keep only the last N sweeps in output (0 means all available).",
+    )
+    return parser
+
+
+def main() -> None:
+    parser = _build_parser()
+    args = parser.parse_args()
+
+    snapshot_dir = args.snapshot_dir.expanduser().resolve()
+    if not snapshot_dir.exists():
+        raise FileNotFoundError(f"Snapshot directory not found: {snapshot_dir}")
+
+    output_path = (
+        args.output.expanduser().resolve()
+        if args.output is not None
+        else snapshot_dir / "vna_bscan_history.json"
+    )
+
+    raw_records = _load_stage_records(
+        snapshot_dir,
+        "raw",
+        input_index=args.input_index,
+        output_index=args.output_index,
+    )
+    preprocessed_records = _load_stage_records(
+        snapshot_dir,
+        "preprocessed",
+        input_index=args.input_index,
+        output_index=args.output_index,
+    )
+    if not raw_records and not preprocessed_records:
+        raise ValueError(
+            "No matching raw/preprocessed traces were found in snapshot "
+            f"for input={args.input_index}, output={args.output_index}."
+        )
+
+    sweep_history = _build_sweep_history(raw_records, preprocessed_records, primary_stage=args.primary_stage)
+    if args.last_n > 0:
+        sweep_history = sweep_history[-args.last_n :]
+    if not sweep_history:
+        raise ValueError("Conversion produced empty sweep_history.")
+
+    pre_refs = _load_stage_refs(snapshot_dir, "preprocessed")
+    result_refs = _load_stage_refs(snapshot_dir, "results")
+    alignment_warning = _stage_alignment_warning(pre_refs, result_refs)
+
+    manifest_path = snapshot_dir / "manifest.json"
+    manifest = _load_json(manifest_path) if manifest_path.exists() else {}
+
+    payload: dict[str, Any] = {
+        "format": "vna-system-history-v1",
+        "converter": "python_app/scripts/convert_snapshot_to_vna_history.py",
+        "converted_at_utc": datetime.now(timezone.utc).isoformat(),
+        "source_snapshot_dir": str(snapshot_dir),
+        "input_index": int(args.input_index),
+        "output_index": int(args.output_index),
+        "primary_stage": args.primary_stage,
+        "raw_record_count": len(raw_records),
+        "preprocessed_record_count": len(preprocessed_records),
+        "sweep_history": sweep_history,
+    }
+    if alignment_warning is not None:
+        payload["alignment_warning"] = alignment_warning
+    if manifest:
+        payload["snapshot_manifest"] = manifest
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+    print(
+        "Converted snapshot to vna_system history JSON:\n"
+        f"  input snapshot: {snapshot_dir}\n"
+        f"  output file:    {output_path}\n"
+        f"  sweeps written: {len(sweep_history)}\n"
+        f"  raw records:    {len(raw_records)}\n"
+        f"  pre records:    {len(preprocessed_records)}"
+    )
+    if alignment_warning is not None:
+        print(f"[convert-warning] {alignment_warning}")
+
+
+if __name__ == "__main__":
+    main()

python_app/scripts/hardware_raw_orchestrator_test.py

@@ -0,0 +1,360 @@
+"""Standalone GUI utility for inspecting raw orchestrator ring output."""
+
+from __future__ import annotations
+
+import argparse
+import ctypes
+import ctypes.util
+import json
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import time
+
+import numpy as np
+from PyQt6.QtCore import QTimer
+from PyQt6.QtWidgets import QApplication, QLabel, QMainWindow, QVBoxLayout, QWidget
+import pyqtgraph as pg
+
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from python_app.hardware_full.librevna_service import LibreVnaService
+from python_app.models.run_config_model import RadarSweepModel
+from python_app.orchestration.shm_reader import ShmRingReader
+
+
+def _shm_unlink(name: str) -> None:
+    """Best-effort unlink for POSIX shared-memory object."""
+    libc_name = ctypes.util.find_library("c")
+    if libc_name is None:
+        return
+
+    libc = ctypes.CDLL(libc_name, use_errno=True)
+    libc.shm_unlink.argtypes = [ctypes.c_char_p]
+    libc.shm_unlink.restype = ctypes.c_int
+
+    result = libc.shm_unlink(name.encode("utf-8"))
+    if result == 0:
+        return
+
+    err = ctypes.get_errno()
+    if err != 2:  # ENOENT
+        raise OSError(err, f"shm_unlink failed for {name}")
+
+
+def _read_raw_ring_name(config_path: Path) -> str:
+    """Extract raw ring name from run config."""
+    config = json.loads(config_path.read_text(encoding="utf-8"))
+    ring_name = config["rings"]["raw"]["name"]
+    if not isinstance(ring_name, str) or not ring_name.startswith("/"):
+        raise RuntimeError("Config rings.raw.name must be a POSIX shm name starting with '/'")
+    return ring_name
+
+
+def _read_native_summary(config_path: Path) -> str:
+    """Build short summary of radar/switch driver modes."""
+    config = json.loads(config_path.read_text(encoding="utf-8"))
+    radar_mode = config["radar"]["driver_mode"]
+    switches = config["switches"]
+    if "port1" in switches and "port2" in switches:
+        port1_mode = switches["port1"]["driver_mode"]
+        port2_mode = switches["port2"]["driver_mode"]
+    else:
+        port1_mode = switches["output"]["driver_mode"]
+        port2_mode = switches["input"]["driver_mode"]
+    return f"radar={radar_mode}, port1={port1_mode}, port2={port2_mode}"
+
+
+def _prepare_radar_if_needed(config_path: Path, *, strict: bool) -> str | None:
+    """Preconfigure native radar through Python service when requested."""
+    config = json.loads(config_path.read_text(encoding="utf-8"))
+    radar = config["radar"]
+    if radar["driver_mode"] != "native":
+        return "Radar pre-configuration skipped (mock mode)."
+
+    sweep = radar["sweep"]
+    sweep_model = RadarSweepModel(
+        start_hz=float(sweep["start_hz"]),
+        stop_hz=float(sweep["stop_hz"]),
+        points=int(sweep["points"]),
+        if_bandwidth_hz=float(sweep["if_bandwidth_hz"]),
+        power_dbm=float(sweep.get("stimulus_power_dbm", -10.0)),
+    )
+
+    radar_service = LibreVnaService(serial=radar.get("serial") or None)
+    if not radar_service.driver_available:
+        message = "LibreVNA Python driver is unavailable: skipping pre-configuration"
+        if strict:
+            raise RuntimeError(message)
+        return message
+
+    try:
+        radar_service.open()
+        radar_service.configure(sweep_model)
+        return "Radar pre-configuration completed."
+    except Exception as exc:
+        message = f"Radar pre-configuration failed ({exc})"
+        if strict:
+            raise
+        return f"{message}. Continuing with native C++ configuration."
+    finally:
+        radar_service.close()
+
+
+class RawOrchestratorViewer(QMainWindow):
+    """Qt window that runs sweep_orchestrator and plots raw collections."""
+
+    def __init__(self, config_path: Path, reset_ring: bool, prepare_radar: bool, strict_prepare: bool) -> None:
+        """Initialize viewer, optionally prepare radar, and start polling."""
+        super().__init__()
+
+        self._config_path = config_path
+        self._raw_ring_name = _read_raw_ring_name(config_path)
+        self._orchestrator_process: subprocess.Popen[str] | None = None
+        self._raw_reader: ShmRingReader | None = None
+
+        if reset_ring:
+            _shm_unlink(self._raw_ring_name)
+
+        self._build_ui()
+        if prepare_radar:
+            self._status.setText("Preparing radar...")
+            prepare_status = _prepare_radar_if_needed(self._config_path, strict=strict_prepare)
+            if prepare_status is not None:
+                self._status.setText(prepare_status)
+        self._start_orchestrator()
+        self._open_reader_or_fail()
+
+        self._timer = QTimer(self)
+        self._timer.setInterval(60)
+        self._timer.timeout.connect(self._poll)
+        self._timer.start()
+
+    def _build_ui(self) -> None:
+        """Build viewer widgets and raw trace plot."""
+        self.setWindowTitle("Raw Sweep Viewer (orchestrator)")
+        root = QWidget(self)
+        self.setCentralWidget(root)
+
+        layout = QVBoxLayout(root)
+
+        mode_summary = _read_native_summary(self._config_path)
+        self._status = QLabel(f"Starting... ({mode_summary})")
+        layout.addWidget(self._status)
+
+        self._plot = pg.PlotWidget(background="#101418")
+        self._plot.showGrid(x=True, y=True, alpha=0.2)
+        self._plot.setLabel("bottom", "Frequency", units="Hz")
+        self._plot.setLabel("left", "Magnitude", units="dB")
+        layout.addWidget(self._plot)
+
+        self.resize(1400, 900)
+
+    def _start_orchestrator(self) -> None:
+        """Start sweep orchestrator subprocess."""
+        binary = PROJECT_ROOT / "build/bin/sweep_orchestrator"
+        if not binary.exists():
+            raise RuntimeError(f"Missing binary: {binary}. Build first with 'make -j4'")
+
+        command = [str(binary), "--config", str(self._config_path)]
+        self._orchestrator_process = subprocess.Popen(
+            command,
+            cwd=PROJECT_ROOT,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+
+    def _open_reader_or_fail(self) -> None:
+        """Wait for raw ring readiness and open `ShmRingReader`."""
+        if self._orchestrator_process is None:
+            raise RuntimeError("Orchestrator process is not started")
+
+        shm_path = Path("/dev/shm") / self._raw_ring_name[1:]
+        deadline = time.monotonic() + 5.0
+        last_reader_error: str | None = None
+
+        while time.monotonic() < deadline:
+            return_code = self._orchestrator_process.poll()
+            if return_code is not None:
+                details = self._read_process_output(self._orchestrator_process)
+                raise RuntimeError(f"sweep_orchestrator exited with code {return_code}: {details}")
+
+            if shm_path.exists():
+                try:
+                    self._raw_reader = ShmRingReader(self._raw_ring_name)
+                    self._status.setText(f"Running: ring={self._raw_ring_name}")
+                    return
+                except RuntimeError as exc:
+                    last_reader_error = str(exc)
+
+            time.sleep(0.05)
+
+        if last_reader_error is not None:
+            raise RuntimeError(
+                f"Timed out waiting for raw ring header readiness: {self._raw_ring_name}; "
+                f"last error: {last_reader_error}"
+            )
+        raise RuntimeError(f"Timed out waiting for raw ring file: {shm_path}")
+
+    def _poll(self) -> None:
+        """Poll subprocess state and draw latest available raw collection."""
+        if self._orchestrator_process is None:
+            return
+
+        return_code = self._orchestrator_process.poll()
+        if return_code is not None:
+            details = self._read_process_output(self._orchestrator_process)
+            self._status.setText(f"Error: sweep_orchestrator exited ({return_code})")
+            raise RuntimeError(f"sweep_orchestrator exited with code {return_code}: {details}")
+
+        if self._raw_reader is None:
+            return
+
+        latest = None
+        for _ in range(16):
+            collection = self._raw_reader.pop_raw_collection()
+            if collection is None:
+                break
+            latest = collection
+
+        if latest is not None:
+            self._draw_collection(latest)
+
+    def _draw_collection(self, collection) -> None:
+        """Render all traces from one raw collection."""
+        self._plot.clear()
+        palette = [
+            "#4cc9f0",
+            "#f72585",
+            "#b8f2e6",
+            "#ffd166",
+            "#90be6d",
+            "#ff595e",
+            "#6a4c93",
+            "#1982c4",
+            "#ff9f1c",
+            "#2ec4b6",
+            "#e71d36",
+            "#a0c4ff",
+        ]
+
+        for idx, trace in enumerate(collection.traces):
+            magnitude_db = 20.0 * np.log10(np.maximum(np.abs(trace.s21), 1e-12))
+            label = f"input={trace.combo.input_pos}, output={trace.combo.output_pos}"
+            self._plot.plot(
+                trace.frequency_hz,
+                magnitude_db,
+                pen=pg.mkPen(palette[idx % len(palette)], width=1.6),
+                name=label,
+            )
+
+        self._status.setText(
+            f"Running: collection_id={collection.collection_id}, "
+            f"traces={len(collection.traces)}, ring={self._raw_ring_name}"
+        )
+
+    @staticmethod
+    def _read_process_output(process: subprocess.Popen[str]) -> str:
+        """Collect process stdout/stderr text for diagnostics."""
+        stdout = ""
+        stderr = ""
+        if process.stdout is not None:
+            stdout = process.stdout.read().strip()
+        if process.stderr is not None:
+            stderr = process.stderr.read().strip()
+
+        if stderr and stdout:
+            return f"{stderr}\nstdout:\n{stdout}"
+        if stderr:
+            return stderr
+        if stdout:
+            return f"stdout:\n{stdout}"
+        return "no output"
+
+    def closeEvent(self, event) -> None:  # noqa: N802
+        """Stop subprocess and close reader before window destruction."""
+        self._shutdown()
+        super().closeEvent(event)
+
+    def _shutdown(self) -> None:
+        """Close reader and terminate subprocess."""
+        if self._raw_reader is not None:
+            self._raw_reader.close()
+            self._raw_reader = None
+
+        process = self._orchestrator_process
+        self._orchestrator_process = None
+        if process is None:
+            return
+
+        if process.poll() is None:
+            process.terminate()
+            try:
+                process.wait(timeout=2.0)
+            except subprocess.TimeoutExpired:
+                process.kill()
+                process.wait(timeout=1.0)
+
+
+def main() -> int:
+    """CLI entrypoint for raw orchestrator viewer."""
+    parser = argparse.ArgumentParser(description="Run sweep_orchestrator and plot raw sweep collections")
+    parser.add_argument(
+        "--config",
+        type=Path,
+        default=PROJECT_ROOT / "run_config.json",
+        help="Path to run config JSON",
+    )
+    parser.add_argument(
+        "--no-reset-ring",
+        action="store_true",
+        help="Do not unlink existing raw ring name before starting",
+    )
+    parser.add_argument(
+        "--skip-radar-prepare",
+        action="store_true",
+        help="Skip Python pre-configuration of native radar before orchestrator start",
+    )
+    parser.add_argument(
+        "--strict-radar-prepare",
+        action="store_true",
+        help="Fail immediately if Python pre-configuration cannot run",
+    )
+    args = parser.parse_args()
+
+    config_path = args.config.resolve()
+    if not config_path.exists():
+        raise FileNotFoundError(f"Config file not found: {config_path}")
+
+    if os.geteuid() == 0 and os.environ.get("SUDO_USER"):
+        print(
+            "Warning: running GUI test via sudo can break Qt DBus/session integration. "
+            "Prefer regular user with USB/GPIO permissions."
+        )
+
+    app = QApplication(sys.argv)
+    viewer = RawOrchestratorViewer(
+        config_path=config_path,
+        reset_ring=not args.no_reset_ring,
+        prepare_radar=not args.skip_radar_prepare,
+        strict_prepare=args.strict_radar_prepare,
+    )
+    viewer.show()
+
+    def _sig_handler(_signum, _frame):
+        """Close viewer gracefully on process signals."""
+        viewer.close()
+
+    signal.signal(signal.SIGINT, _sig_handler)
+    signal.signal(signal.SIGTERM, _sig_handler)
+
+    return app.exec()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

python_app/scripts/manual_smoke_run.py

@@ -0,0 +1,152 @@
+"""Manual smoke scenario for end-to-end pipeline check without GUI."""
+
+from __future__ import annotations
+
+import argparse
+import ctypes
+import ctypes.util
+import os
+from pathlib import Path
+import sys
+import time
+
+import numpy as np
+
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData
+from python_app.models.run_config_model import RunConfigModel
+from python_app.orchestration.config_writer import ConfigWriter
+from python_app.orchestration.process_supervisor import ProcessSupervisor
+from python_app.orchestration.shm_reader import ShmRingReader
+from python_app.storage.npz_store import NpzStore, radar_key_from_config
+
+
+def _make_unique_ring_name(prefix: str) -> str:
+    """Build unique POSIX SHM ring name."""
+    stamp = time.monotonic_ns()
+    return f"/{prefix}_{os.getpid()}_{stamp}"
+
+
+def _shm_unlink(name: str) -> None:
+    """Best-effort unlink for POSIX shared-memory object."""
+    libc_name = ctypes.util.find_library("c")
+    if libc_name is None:
+        return
+
+    libc = ctypes.CDLL(libc_name, use_errno=True)
+    libc.shm_unlink.argtypes = [ctypes.c_char_p]
+    libc.shm_unlink.restype = ctypes.c_int
+
+    result = libc.shm_unlink(name.encode("utf-8"))
+    if result == 0:
+        return
+
+    err = ctypes.get_errno()
+    if err != 2:  # ENOENT
+        raise OSError(err, f"shm_unlink failed for {name}")
+
+
+def build_synthetic_collection(config: RunConfigModel, value_scale: float) -> SweepCollection:
+    """Build synthetic sweep collection for all configured switch combos."""
+    traces: list[TraceData] = []
+    combos = RunConfigModel.build_full_combos(config.input_switch.positions, config.output_switch.positions)
+
+    for combo in combos:
+        frequency_hz = np.linspace(
+            config.radar.sweep.start_hz,
+            config.radar.sweep.stop_hz,
+            config.radar.sweep.points,
+            dtype=np.float32,
+        )
+        phase = np.linspace(0.0, np.pi * 2.0, config.radar.sweep.points, dtype=np.float32)
+        s21 = value_scale * (np.cos(phase) + 1j * np.sin(phase)).astype(np.complex64)
+
+        traces.append(
+            TraceData(
+                combo=ComboKey(input_pos=combo.input, output_pos=combo.output),
+                frequency_hz=frequency_hz,
+                s21=s21,
+            )
+        )
+
+    return SweepCollection(collection_id=1, monotonic_ns=time.monotonic_ns(), traces=traces)
+
+
+def main() -> int:
+    """Run manual smoke-test pipeline scenario."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--duration", type=float, default=3.0)
+    args = parser.parse_args()
+
+    project_root = PROJECT_ROOT
+    store = NpzStore(project_root / "python_app/data")
+    config_writer = ConfigWriter(project_root / "python_app/runtime")
+    supervisor = ProcessSupervisor(project_root)
+
+    config = RunConfigModel.load_from_path(project_root / "run_config.json")
+    config.radar.driver_mode = "mock"
+    config.input_switch.driver_mode = "mock"
+    config.output_switch.driver_mode = "mock"
+    config.combos = RunConfigModel.build_full_combos(config.input_switch.positions, config.output_switch.positions)
+    config.rings.raw.name = _make_unique_ring_name("radar_raw_smoke")
+    config.rings.raw_tap.name = _make_unique_ring_name("radar_raw_tap_smoke")
+    config.rings.preprocessed.name = _make_unique_ring_name("radar_preprocessed_smoke")
+    config.rings.preprocessed_tap.name = _make_unique_ring_name("radar_preprocessed_tap_smoke")
+    config.rings.results.name = _make_unique_ring_name("radar_results_smoke")
+
+    radar_key = radar_key_from_config(
+        model_name=config.radar.model,
+        serial=config.radar.serial,
+        sweep_start_hz=config.radar.sweep.start_hz,
+        sweep_stop_hz=config.radar.sweep.stop_hz,
+        sweep_points=config.radar.sweep.points,
+        ifbw_hz=config.radar.sweep.if_bandwidth_hz,
+        power_dbm=config.radar.sweep.power_dbm,
+    )
+
+    calibration_set = build_synthetic_collection(config, value_scale=1.0)
+    reference_set = build_synthetic_collection(config, value_scale=0.3)
+
+    store.save_set("calibration", radar_key, "smoke_cal", calibration_set)
+    store.save_set("reference", radar_key, "smoke_ref", reference_set)
+
+    calibration_bundle, reference_bundle = config_writer.prepare_bundles(store, radar_key, "smoke_cal", "smoke_ref")
+    config.preprocess.calibration_set = "smoke_cal"
+    config.preprocess.reference_set = "smoke_ref"
+    config.preprocess.calibration_bundle_path = str(calibration_bundle)
+    config.preprocess.reference_bundle_path = str(reference_bundle)
+
+    config_path = config_writer.write(config, project_root / "python_app/runtime/run_config_smoke.json")
+
+    result_reader: ShmRingReader | None = None
+    try:
+        supervisor.start(config_path)
+        result_reader = ShmRingReader(config.rings.results.name)
+        deadline = time.monotonic() + args.duration
+        received = 0
+
+        while time.monotonic() < deadline:
+            result = result_reader.pop_result_collection() if result_reader is not None else None
+            if result is not None:
+                received += 1
+            time.sleep(0.02)
+
+        print(f"Received result collections: {received}")
+    finally:
+        supervisor.stop_all()
+        if result_reader is not None:
+            result_reader.close()
+        _shm_unlink(config.rings.raw.name)
+        _shm_unlink(config.rings.raw_tap.name)
+        _shm_unlink(config.rings.preprocessed.name)
+        _shm_unlink(config.rings.preprocessed_tap.name)
+        _shm_unlink(config.rings.results.name)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

python_app/scripts/vna_only_raw_test.py

@@ -0,0 +1,72 @@
+"""Minimal GUI tool for direct LibreVNA raw acquisition checks."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import signal
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from PyQt6.QtWidgets import QApplication
+
+from python_app.scripts.hardware_raw_orchestrator_test import RawOrchestratorViewer
+
+
+def main() -> int:
+    """Run standalone raw-viewer GUI against a selected run config."""
+    parser = argparse.ArgumentParser(
+        description="Intermediate test: native VNA acquisition with mock switch drivers"
+    )
+    parser.add_argument(
+        "--config",
+        type=Path,
+        default=PROJECT_ROOT / "run_config.json",
+        help="Path to run config JSON",
+    )
+    parser.add_argument(
+        "--no-reset-ring",
+        action="store_true",
+        help="Do not unlink existing raw ring name before starting",
+    )
+    parser.add_argument(
+        "--skip-radar-prepare",
+        action="store_true",
+        help="Skip Python pre-configuration of native radar before orchestrator start",
+    )
+    parser.add_argument(
+        "--strict-radar-prepare",
+        action="store_true",
+        help="Fail immediately if Python pre-configuration cannot run",
+    )
+    args = parser.parse_args()
+
+    config_path = args.config.resolve()
+    if not config_path.exists():
+        raise FileNotFoundError(f"Config file not found: {config_path}")
+
+    app = QApplication(sys.argv)
+    viewer = RawOrchestratorViewer(
+        config_path=config_path,
+        reset_ring=not args.no_reset_ring,
+        prepare_radar=not args.skip_radar_prepare,
+        strict_prepare=args.strict_radar_prepare,
+    )
+    viewer.setWindowTitle("Raw Sweep Viewer (VNA native + mock switches)")
+    viewer.show()
+
+    def _sig_handler(_signum, _frame):
+        """Close viewer gracefully on process signals."""
+        viewer.close()
+
+    signal.signal(signal.SIGINT, _sig_handler)
+    signal.signal(signal.SIGTERM, _sig_handler)
+
+    return app.exec()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())