Rework radar system
This commit is contained in:
Ayzen
@@ -1,301 +0,0 @@
|
||||
"""Validate and visualize numpy-directory runtime snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def _load_json(path: Path) -> dict[str, Any]:
|
||||
"""Load JSON object from file path."""
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"JSON root must be object: {path}")
|
||||
return payload
|
||||
|
||||
|
||||
def _collection_dirs(stage_dir: Path) -> list[Path]:
|
||||
"""Return sorted list of collection directories for one stage."""
|
||||
if not stage_dir.exists():
|
||||
return []
|
||||
return sorted([path for path in stage_dir.iterdir() if path.is_dir()], key=lambda path: path.name)
|
||||
|
||||
|
||||
def _first_trace_from_raw_or_pre(collection_dir: Path) -> tuple[np.ndarray, np.ndarray, str] | None:
|
||||
"""Return first trace payload from raw/preprocessed collection."""
|
||||
traces = _all_traces_from_raw_or_pre(collection_dir)
|
||||
if not traces:
|
||||
return None
|
||||
return traces[0]
|
||||
|
||||
|
||||
def _all_traces_from_raw_or_pre(collection_dir: Path) -> list[tuple[np.ndarray, np.ndarray, str]]:
|
||||
"""Load all traces from raw/preprocessed collection directory."""
|
||||
meta = _load_json(collection_dir / "meta.json")
|
||||
traces = meta.get("traces", [])
|
||||
if not isinstance(traces, list):
|
||||
raise ValueError(f"Invalid traces in {collection_dir / 'meta.json'}")
|
||||
|
||||
all_traces: list[tuple[np.ndarray, np.ndarray, str]] = []
|
||||
for trace_meta in traces:
|
||||
if not isinstance(trace_meta, dict):
|
||||
raise ValueError(f"Invalid trace record in {collection_dir / 'meta.json'}")
|
||||
|
||||
freq_file = str(trace_meta.get("freq_file", ""))
|
||||
s11_file = str(trace_meta.get("s11_file", ""))
|
||||
s21_file = str(trace_meta.get("s21_file", ""))
|
||||
freq = np.load(collection_dir / freq_file)
|
||||
s11 = np.load(collection_dir / s11_file)
|
||||
s21 = np.load(collection_dir / s21_file)
|
||||
if freq.shape != s11.shape:
|
||||
raise ValueError(f"Shape mismatch freq/s11 in {collection_dir}")
|
||||
if freq.shape != s21.shape:
|
||||
raise ValueError(f"Shape mismatch freq/s21 in {collection_dir}")
|
||||
if not (
|
||||
np.isfinite(freq).all()
|
||||
and np.isfinite(np.real(s11)).all()
|
||||
and np.isfinite(np.imag(s11)).all()
|
||||
and np.isfinite(np.real(s21)).all()
|
||||
and np.isfinite(np.imag(s21)).all()
|
||||
):
|
||||
raise ValueError(f"Non-finite values in {collection_dir}")
|
||||
|
||||
label = f"i{int(trace_meta.get('input', 0))}_o{int(trace_meta.get('output', 0))}"
|
||||
all_traces.append((np.asarray(freq, dtype=np.float64), np.asarray(s21, dtype=np.complex128), label))
|
||||
|
||||
return all_traces
|
||||
|
||||
|
||||
def _first_trace_from_results(collection_dir: Path) -> tuple[np.ndarray, np.ndarray, str] | None:
|
||||
"""Return first trace payload from results collection."""
|
||||
traces = _all_traces_from_results(collection_dir)
|
||||
if not traces:
|
||||
return None
|
||||
return traces[0]
|
||||
|
||||
|
||||
def _all_traces_from_results(collection_dir: Path) -> list[tuple[np.ndarray, np.ndarray, str]]:
|
||||
"""Load all trace-like payloads from results collection directory."""
|
||||
meta = _load_json(collection_dir / "meta.json")
|
||||
blocks = meta.get("blocks", [])
|
||||
if not isinstance(blocks, list):
|
||||
raise ValueError(f"Invalid blocks in {collection_dir / 'meta.json'}")
|
||||
|
||||
all_traces: list[tuple[np.ndarray, np.ndarray, str]] = []
|
||||
for block in blocks:
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
block_dir_name = str(block.get("dir", ""))
|
||||
block_dir = collection_dir / block_dir_name
|
||||
payloads = block.get("payloads", [])
|
||||
if not isinstance(payloads, list):
|
||||
continue
|
||||
for payload in payloads:
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
kind = int(payload.get("kind", 0))
|
||||
if kind != 1:
|
||||
continue
|
||||
freq_file = str(payload.get("freq_file", ""))
|
||||
trace_file = str(payload.get("trace_file", ""))
|
||||
freq = np.load(block_dir / freq_file)
|
||||
trace = np.load(block_dir / trace_file)
|
||||
if freq.shape != trace.shape:
|
||||
raise ValueError(f"Shape mismatch freq/trace in {block_dir}")
|
||||
if not (
|
||||
np.isfinite(freq).all()
|
||||
and np.isfinite(np.real(trace)).all()
|
||||
and np.isfinite(np.imag(trace)).all()
|
||||
):
|
||||
raise ValueError(f"Non-finite values in {block_dir}")
|
||||
|
||||
label = (
|
||||
f"i{int(block.get('input', 0))}_o{int(block.get('output', 0))}_"
|
||||
f"{str(payload.get('name', 'processor'))}"
|
||||
)
|
||||
all_traces.append((np.asarray(freq, dtype=np.float64), np.asarray(trace, dtype=np.complex128), label))
|
||||
|
||||
return all_traces
|
||||
|
||||
|
||||
def _validate_stage(stage_dir: Path, stage: str) -> tuple[list[int], list[tuple[np.ndarray, np.ndarray, str]]]:
|
||||
"""Validate one stage directory and collect representative traces."""
|
||||
collection_ids: list[int] = []
|
||||
traces: list[tuple[np.ndarray, np.ndarray, str]] = []
|
||||
for collection_dir in _collection_dirs(stage_dir):
|
||||
meta = _load_json(collection_dir / "meta.json")
|
||||
collection_ids.append(int(meta.get("collection_id", -1)))
|
||||
if stage in {"raw", "preprocessed"}:
|
||||
trace = _first_trace_from_raw_or_pre(collection_dir)
|
||||
else:
|
||||
trace = _first_trace_from_results(collection_dir)
|
||||
if trace is not None:
|
||||
traces.append(trace)
|
||||
return collection_ids, traces
|
||||
|
||||
|
||||
def _compare_two(
|
||||
name: str,
|
||||
first: tuple[np.ndarray, np.ndarray, str],
|
||||
second: tuple[np.ndarray, np.ndarray, str],
|
||||
) -> None:
|
||||
"""Print numerical difference metrics for two traces."""
|
||||
freq_a, data_a, label_a = first
|
||||
freq_b, data_b, label_b = second
|
||||
same_shape = freq_a.shape == freq_b.shape == data_a.shape == data_b.shape
|
||||
if not same_shape:
|
||||
print(f"[{name}] different shapes: {freq_a.shape}/{freq_b.shape} {data_a.shape}/{data_b.shape}")
|
||||
return
|
||||
|
||||
are_equal = np.array_equal(data_a, data_b)
|
||||
diff = data_a - data_b
|
||||
max_abs_diff = float(np.max(np.abs(diff)))
|
||||
l2_diff = float(np.linalg.norm(diff))
|
||||
print(
|
||||
f"[{name}] compare first two traces: {label_a} vs {label_b}, "
|
||||
f"equal={are_equal}, max_abs_diff={max_abs_diff:.6g}, l2_diff={l2_diff:.6g}"
|
||||
)
|
||||
|
||||
|
||||
def _plot_two(
|
||||
stage: str,
|
||||
first: tuple[np.ndarray, np.ndarray, str],
|
||||
second: tuple[np.ndarray, np.ndarray, str],
|
||||
output_dir: Path,
|
||||
) -> None:
|
||||
"""Plot magnitude comparison for two traces."""
|
||||
try:
|
||||
import matplotlib.pyplot as plt
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"[{stage}] matplotlib is not available, plot skipped: {exc}")
|
||||
return
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
freq_a, data_a, label_a = first
|
||||
freq_b, data_b, label_b = second
|
||||
if freq_a.shape != freq_b.shape or data_a.shape != data_b.shape:
|
||||
print(f"[{stage}] shapes differ, plot skipped")
|
||||
return
|
||||
|
||||
y_a = 20.0 * np.log10(np.maximum(np.abs(data_a), 1e-12))
|
||||
y_b = 20.0 * np.log10(np.maximum(np.abs(data_b), 1e-12))
|
||||
|
||||
fig, ax = plt.subplots(figsize=(10, 4))
|
||||
ax.plot(freq_a, y_a, linewidth=1.4, label=f"collection#1 {label_a}")
|
||||
ax.plot(freq_b, y_b, linewidth=1.4, label=f"collection#2 {label_b}")
|
||||
ax.set_title(f"{stage}: first two collections")
|
||||
ax.set_xlabel("X axis")
|
||||
ax.set_ylabel("Magnitude dB")
|
||||
ax.grid(True, alpha=0.3)
|
||||
ax.legend()
|
||||
fig.tight_layout()
|
||||
|
||||
png_path = output_dir / f"{stage}_first_two.png"
|
||||
fig.savefig(png_path, dpi=150)
|
||||
plt.close(fig)
|
||||
print(f"[{stage}] plot saved: {png_path}")
|
||||
|
||||
|
||||
def _plot_all_states_for_one_collection(stage: str, collection_dir: Path, output_dir: Path) -> None:
|
||||
"""Plot all switch-state traces available in one collection."""
|
||||
try:
|
||||
import matplotlib.pyplot as plt
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"[{stage}] matplotlib is not available, all-states plot skipped: {exc}")
|
||||
return
|
||||
|
||||
if stage in {"raw", "preprocessed"}:
|
||||
traces = _all_traces_from_raw_or_pre(collection_dir)
|
||||
else:
|
||||
traces = _all_traces_from_results(collection_dir)
|
||||
|
||||
if not traces:
|
||||
print(f"[{stage}] no trace payloads found in {collection_dir.name}, all-states plot skipped")
|
||||
return
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
meta = _load_json(collection_dir / "meta.json")
|
||||
collection_id = int(meta.get("collection_id", -1))
|
||||
|
||||
fig, ax = plt.subplots(figsize=(11, 5))
|
||||
for freq, data, label in traces:
|
||||
y = 20.0 * np.log10(np.maximum(np.abs(data), 1e-12))
|
||||
ax.plot(freq, y, linewidth=1.2, label=label)
|
||||
ax.set_title(f"{stage}: all switch states in one collection (id={collection_id})")
|
||||
ax.set_xlabel("X axis")
|
||||
ax.set_ylabel("Magnitude dB")
|
||||
ax.grid(True, alpha=0.3)
|
||||
ax.legend(fontsize=8, ncol=2)
|
||||
fig.tight_layout()
|
||||
|
||||
png_path = output_dir / f"{stage}_all_states_one_collection.png"
|
||||
fig.savefig(png_path, dpi=150)
|
||||
plt.close(fig)
|
||||
print(f"[{stage}] all-states plot saved: {png_path} (traces={len(traces)})")
|
||||
|
||||
|
||||
def _run(snapshot_dir: Path, output_dir: Path) -> None:
|
||||
"""Execute snapshot validation and plotting workflow."""
|
||||
config_profile_path = snapshot_dir / "config_profile.json"
|
||||
if config_profile_path.exists():
|
||||
print(f"Snapshot: {snapshot_dir}\nconfig_profile={config_profile_path.name}")
|
||||
else:
|
||||
print(f"Snapshot: {snapshot_dir}")
|
||||
|
||||
stages = ("raw", "preprocessed", "results")
|
||||
for stage in stages:
|
||||
stage_dir = snapshot_dir / stage
|
||||
collection_dirs = _collection_dirs(stage_dir)
|
||||
collection_ids, traces = _validate_stage(stage_dir, stage)
|
||||
duplicate_id_count = len(collection_ids) - len(set(collection_ids))
|
||||
print(
|
||||
f"[{stage}] collections={len(collection_ids)}, "
|
||||
f"duplicate_ids={duplicate_id_count}, "
|
||||
f"trace_samples={len(traces)}"
|
||||
)
|
||||
if len(traces) >= 2:
|
||||
_compare_two(stage, traces[0], traces[1])
|
||||
_plot_two(stage, traces[0], traces[1], output_dir)
|
||||
else:
|
||||
print(f"[{stage}] not enough trace-like collections to compare/plot (need >= 2)")
|
||||
|
||||
if collection_dirs:
|
||||
_plot_all_states_for_one_collection(stage, collection_dirs[-1], output_dir)
|
||||
else:
|
||||
print(f"[{stage}] no collections for all-states plot")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""CLI entrypoint."""
|
||||
parser = argparse.ArgumentParser(description="Validate and visualize runtime numpy snapshot collections.")
|
||||
parser.add_argument(
|
||||
"snapshot_dir",
|
||||
type=Path,
|
||||
help="Path to snapshot directory (contains raw/preprocessed/results).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Directory for output plots (default: <snapshot_dir>/inspection_plots).",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
snapshot_dir = args.snapshot_dir.expanduser().resolve()
|
||||
if not snapshot_dir.exists():
|
||||
raise FileNotFoundError(f"Snapshot directory not found: {snapshot_dir}")
|
||||
|
||||
output_dir = (
|
||||
args.output_dir.expanduser().resolve()
|
||||
if args.output_dir is not None
|
||||
else snapshot_dir / "inspection_plots"
|
||||
)
|
||||
_run(snapshot_dir, output_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,251 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Convert legacy preprocess set storage into the current two-channel format."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData
|
||||
from python_app.storage.npz_store import NpzStore
|
||||
|
||||
|
||||
LEGACY_KIND_MAP: dict[str, str] = {
|
||||
"calibration": "s21_calibration",
|
||||
"reference": "s21_reference",
|
||||
"s21_calibration": "s21_calibration",
|
||||
"s21_reference": "s21_reference",
|
||||
"s11_open": "s11_open",
|
||||
"s11_short": "s11_short",
|
||||
"s11_load": "s11_load",
|
||||
"s11_reference": "s11_reference",
|
||||
}
|
||||
|
||||
S21_ONLY_TARGET_KINDS = {"s21_calibration", "s21_reference"}
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
"""Return the argument parser for the legacy preprocess-set conversion CLI."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Convert old preprocess-set storage from a legacy python_app/data tree into the "
|
||||
"current format required by the new GUI/runtime."
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"source_data_dir",
|
||||
type=Path,
|
||||
help="Path to legacy python_app/data directory from the old project copy.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"output_data_dir",
|
||||
type=Path,
|
||||
help="Destination directory where converted sets will be written in the new format.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--overwrite",
|
||||
action="store_true",
|
||||
help="Allow overwriting already converted destination sets.",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def _load_json(path: Path) -> dict[str, Any]:
|
||||
"""Read a JSON file and return its top-level object, rejecting non-object roots."""
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"JSON root must be object: {path}")
|
||||
return payload
|
||||
|
||||
|
||||
def _read_combo_position(combo_payload: dict[str, Any], *, primary_key: str, alias_key: str) -> int:
|
||||
"""Return a switch position from a combo record, accepting the primary or alias key."""
|
||||
if primary_key in combo_payload:
|
||||
return int(combo_payload[primary_key])
|
||||
if alias_key in combo_payload:
|
||||
return int(combo_payload[alias_key])
|
||||
raise KeyError(f"Missing combo position field: {primary_key}/{alias_key}")
|
||||
|
||||
|
||||
def _combo_suffix(input_pos: int, output_pos: int) -> str:
|
||||
"""Return the per-combo array-name suffix (e.g. ``i0_o1``) used in legacy NPZ keys."""
|
||||
return f"i{input_pos}_o{output_pos}"
|
||||
|
||||
|
||||
def _load_array(arrays: Any, key: str, *, dtype: np.dtype[Any], label: str) -> np.ndarray:
|
||||
"""Return a flattened array of the given dtype from an NPZ mapping, by key."""
|
||||
if key not in arrays:
|
||||
raise KeyError(f"Missing {label} array '{key}' in NPZ archive")
|
||||
return np.asarray(arrays[key], dtype=dtype).reshape(-1)
|
||||
|
||||
|
||||
def _load_legacy_collection(meta_path: Path, npz_path: Path, *, target_kind: str) -> SweepCollection:
|
||||
"""Build a SweepCollection from a legacy meta/NPZ pair for the given target kind.
|
||||
|
||||
Reads per-combo frequency, S21 and (when present) S11 arrays. For S21-only target
|
||||
kinds a missing S11 is filled with zeros; for any other kind a missing S11 is an error.
|
||||
"""
|
||||
meta = _load_json(meta_path)
|
||||
combos_payload = meta.get("combos")
|
||||
if not isinstance(combos_payload, list):
|
||||
raise ValueError(f"Expected 'combos' list in {meta_path}")
|
||||
|
||||
with np.load(npz_path) as arrays:
|
||||
traces: list[TraceData] = []
|
||||
for combo_payload in combos_payload:
|
||||
if not isinstance(combo_payload, dict):
|
||||
raise ValueError(f"Expected combo object in {meta_path}")
|
||||
|
||||
input_pos = _read_combo_position(combo_payload, primary_key="input", alias_key="input_pos")
|
||||
output_pos = _read_combo_position(combo_payload, primary_key="output", alias_key="output_pos")
|
||||
suffix = _combo_suffix(input_pos, output_pos)
|
||||
|
||||
freq_key = str(combo_payload.get("freq_key") or f"freq_{suffix}")
|
||||
s21_key = str(combo_payload.get("s21_key") or f"s21_{suffix}")
|
||||
s11_key = str(combo_payload.get("s11_key") or f"s11_{suffix}")
|
||||
|
||||
frequency_hz = _load_array(arrays, freq_key, dtype=np.float32, label="frequency")
|
||||
s21 = _load_array(arrays, s21_key, dtype=np.complex64, label="S21")
|
||||
|
||||
if frequency_hz.shape != s21.shape:
|
||||
raise ValueError(f"Frequency/S21 shape mismatch in {npz_path}: {frequency_hz.shape} vs {s21.shape}")
|
||||
|
||||
if s11_key in arrays:
|
||||
s11 = _load_array(arrays, s11_key, dtype=np.complex64, label="S11")
|
||||
elif target_kind in S21_ONLY_TARGET_KINDS:
|
||||
s11 = np.zeros_like(s21, dtype=np.complex64)
|
||||
else:
|
||||
raise KeyError(
|
||||
f"Missing S11 array '{s11_key}' in {npz_path}; "
|
||||
f"cannot convert target kind '{target_kind}' without real S11 data"
|
||||
)
|
||||
|
||||
if frequency_hz.shape != s11.shape:
|
||||
raise ValueError(f"Frequency/S11 shape mismatch in {npz_path}: {frequency_hz.shape} vs {s11.shape}")
|
||||
|
||||
traces.append(
|
||||
TraceData(
|
||||
combo=ComboKey(input=input_pos, output=output_pos),
|
||||
frequency_hz=frequency_hz,
|
||||
s11=s11,
|
||||
s21=s21,
|
||||
)
|
||||
)
|
||||
|
||||
return SweepCollection(
|
||||
collection_id=int(meta.get("collection_id", 0)),
|
||||
monotonic_ns=int(meta.get("monotonic_ns", 0)),
|
||||
traces=traces,
|
||||
)
|
||||
|
||||
|
||||
def _convert_one_set(
|
||||
store: NpzStore,
|
||||
*,
|
||||
output_root: Path,
|
||||
source_kind: str,
|
||||
target_kind: str,
|
||||
radar_key: str,
|
||||
meta_path: Path,
|
||||
overwrite: bool,
|
||||
) -> None:
|
||||
"""Convert a single legacy set (meta + NPZ) and save it under the target kind.
|
||||
|
||||
Raises if the companion NPZ is missing, or if the destination already exists and
|
||||
``overwrite`` is False.
|
||||
"""
|
||||
set_name = meta_path.stem
|
||||
npz_path = meta_path.with_suffix(".npz")
|
||||
if not npz_path.exists():
|
||||
raise FileNotFoundError(f"Missing NPZ archive for set '{set_name}': {npz_path}")
|
||||
|
||||
target_dir = output_root / target_kind / radar_key
|
||||
target_json = target_dir / f"{set_name}.json"
|
||||
target_npz = target_dir / f"{set_name}.npz"
|
||||
if not overwrite and (target_json.exists() or target_npz.exists()):
|
||||
raise FileExistsError(f"Destination set already exists: {target_dir / set_name}")
|
||||
|
||||
collection = _load_legacy_collection(meta_path, npz_path, target_kind=target_kind)
|
||||
store.save_set(target_kind, radar_key, set_name, collection)
|
||||
print(
|
||||
f"[converted] {source_kind}/{radar_key}/{set_name} -> "
|
||||
f"{target_kind}/{radar_key}/{set_name} (traces={len(collection.traces)})"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Walk the legacy data tree, convert every recognized set, and print a summary.
|
||||
|
||||
Returns 0 on full success, 1 if any set failed to convert, or 2 if no convertible
|
||||
sets were found.
|
||||
"""
|
||||
parser = _build_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
source_root = args.source_data_dir.expanduser().resolve()
|
||||
output_root = args.output_data_dir.expanduser().resolve()
|
||||
|
||||
if not source_root.exists():
|
||||
raise FileNotFoundError(f"Source data directory does not exist: {source_root}")
|
||||
if source_root == output_root:
|
||||
raise ValueError("Source and output directories must be different")
|
||||
|
||||
store = NpzStore(output_root)
|
||||
converted_count = 0
|
||||
skipped_kind_count = 0
|
||||
error_messages: list[str] = []
|
||||
|
||||
for source_kind_dir in sorted(path for path in source_root.iterdir() if path.is_dir()):
|
||||
source_kind = source_kind_dir.name
|
||||
target_kind = LEGACY_KIND_MAP.get(source_kind)
|
||||
if target_kind is None:
|
||||
skipped_kind_count += 1
|
||||
print(f"[skip-kind] {source_kind_dir}")
|
||||
continue
|
||||
|
||||
for radar_key_dir in sorted(path for path in source_kind_dir.iterdir() if path.is_dir()):
|
||||
radar_key = radar_key_dir.name
|
||||
for meta_path in sorted(radar_key_dir.glob("*.json")):
|
||||
try:
|
||||
_convert_one_set(
|
||||
store,
|
||||
output_root=output_root,
|
||||
source_kind=source_kind,
|
||||
target_kind=target_kind,
|
||||
radar_key=radar_key,
|
||||
meta_path=meta_path,
|
||||
overwrite=bool(args.overwrite),
|
||||
)
|
||||
converted_count += 1
|
||||
except Exception as exc: # noqa: BLE001
|
||||
error_messages.append(f"{meta_path}: {exc}")
|
||||
print(f"[error] {meta_path}: {exc}")
|
||||
|
||||
print(
|
||||
"\nConversion summary:\n"
|
||||
f" source root: {source_root}\n"
|
||||
f" output root: {output_root}\n"
|
||||
f" converted sets: {converted_count}\n"
|
||||
f" skipped kinds: {skipped_kind_count}\n"
|
||||
f" errors: {len(error_messages)}"
|
||||
)
|
||||
|
||||
if error_messages:
|
||||
return 1
|
||||
if converted_count == 0:
|
||||
print("No convertible preprocess sets were found.")
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,409 +0,0 @@
|
||||
"""Convert manual LibreVNA S11/S21 CSV captures in prog_libre to vna history JSON."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from python_app.file_io import atomic_write_json
|
||||
|
||||
|
||||
def _real_imag_keys(trace_prefix: str) -> tuple[str, str]:
|
||||
"""Return the LibreVNA CSV column names for a trace's real and imaginary parts."""
|
||||
return f"{trace_prefix}_Real", f"{trace_prefix}_Imaginary"
|
||||
|
||||
|
||||
def _load_complex_trace(csv_path: Path, trace_prefix: str) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""Load one LibreVNA CSV and return its (frequency_hz, complex trace) arrays."""
|
||||
real_key, imag_key = _real_imag_keys(trace_prefix)
|
||||
frequencies: list[float] = []
|
||||
values: list[complex] = []
|
||||
|
||||
with csv_path.open(encoding="utf-8", newline="") as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
for row in reader:
|
||||
frequencies.append(float(row["Frequency"]))
|
||||
values.append(complex(float(row[real_key]), float(row[imag_key])))
|
||||
|
||||
frequency_hz = np.asarray(frequencies, dtype=np.float64)
|
||||
trace = np.asarray(values, dtype=np.complex128)
|
||||
if frequency_hz.size == 0 or trace.size == 0:
|
||||
raise ValueError(f"CSV has no points: {csv_path}")
|
||||
if frequency_hz.shape != trace.shape:
|
||||
raise ValueError(f"Frequency/trace size mismatch: {csv_path}")
|
||||
return frequency_hz, trace
|
||||
|
||||
|
||||
def _solve_one_port_osl(
|
||||
open_trace: np.ndarray,
|
||||
short_trace: np.ndarray,
|
||||
load_trace: np.ndarray,
|
||||
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
"""Solve ideal OSL one-port calibration coefficients."""
|
||||
directivity = load_trace
|
||||
open_delta = open_trace - directivity
|
||||
short_delta = short_trace - directivity
|
||||
denom = open_delta - short_delta
|
||||
|
||||
source_match = np.zeros_like(directivity)
|
||||
reflection_tracking = np.ones_like(directivity)
|
||||
|
||||
stable_mask = np.abs(denom) > 1e-18
|
||||
source_match[stable_mask] = (open_delta[stable_mask] + short_delta[stable_mask]) / denom[stable_mask]
|
||||
reflection_tracking[stable_mask] = open_delta[stable_mask] * (1.0 - source_match[stable_mask])
|
||||
return directivity, source_match, reflection_tracking
|
||||
|
||||
|
||||
def _apply_one_port_osl(
|
||||
measured_trace: np.ndarray,
|
||||
directivity: np.ndarray,
|
||||
source_match: np.ndarray,
|
||||
reflection_tracking: np.ndarray,
|
||||
) -> np.ndarray:
|
||||
"""Apply OSL one-port error correction to a measured S11 trace and return it."""
|
||||
numerator = measured_trace - directivity
|
||||
denominator = reflection_tracking + (source_match * numerator)
|
||||
|
||||
corrected = np.array(numerator, copy=True)
|
||||
stable_mask = np.abs(denominator) > 1e-18
|
||||
corrected[stable_mask] = numerator[stable_mask] / denominator[stable_mask]
|
||||
return corrected
|
||||
|
||||
|
||||
def _apply_through_calibration(measured_trace: np.ndarray, through_trace: np.ndarray) -> np.ndarray:
|
||||
"""Normalize a measured S21 trace by the through reference and return it."""
|
||||
corrected = np.array(measured_trace, copy=True)
|
||||
stable_mask = np.abs(through_trace) > 1e-18
|
||||
corrected[stable_mask] = measured_trace[stable_mask] / through_trace[stable_mask]
|
||||
return corrected
|
||||
|
||||
|
||||
def _complex_to_points(values: np.ndarray) -> list[list[float]]:
|
||||
"""Return complex samples as ``[real, imag]`` pairs for JSON serialization."""
|
||||
return [[float(value.real), float(value.imag)] for value in values]
|
||||
|
||||
|
||||
def _scan_file_sort_key(csv_path: Path) -> tuple[int, str]:
|
||||
"""Return a sort key ordering numeric scan filenames first, by integer value."""
|
||||
stem = csv_path.stem
|
||||
return (int(stem), stem) if stem.isdigit() else (10**9, stem)
|
||||
|
||||
|
||||
def _load_scan_series(folder: Path, trace_prefix: str) -> tuple[np.ndarray, list[tuple[str, np.ndarray]]]:
|
||||
"""Load every numbered scan CSV in a folder and return (frequency_hz, named traces).
|
||||
|
||||
All scans must share a common frequency axis; a mismatch raises ValueError.
|
||||
"""
|
||||
scan_paths = [
|
||||
path
|
||||
for path in sorted(folder.glob("*.csv"), key=_scan_file_sort_key)
|
||||
if path.stem.isdigit()
|
||||
]
|
||||
if not scan_paths:
|
||||
raise FileNotFoundError(f"No numbered scan CSV files found in {folder}")
|
||||
|
||||
base_frequency_hz: np.ndarray | None = None
|
||||
scans: list[tuple[str, np.ndarray]] = []
|
||||
for path in scan_paths:
|
||||
frequency_hz, trace = _load_complex_trace(path, trace_prefix)
|
||||
if base_frequency_hz is None:
|
||||
base_frequency_hz = frequency_hz
|
||||
elif not np.allclose(base_frequency_hz, frequency_hz, rtol=0.0, atol=1e-6):
|
||||
raise ValueError(f"Frequency axis mismatch in {path}")
|
||||
scans.append((path.name, trace))
|
||||
|
||||
assert base_frequency_hz is not None
|
||||
return base_frequency_hz, scans
|
||||
|
||||
|
||||
def _require_matching_frequency_axis(label: str, left: np.ndarray, right: np.ndarray) -> None:
|
||||
"""Raise ValueError (tagged with ``label``) unless two frequency axes match within tolerance."""
|
||||
if not np.allclose(left, right, rtol=0.0, atol=1e-6):
|
||||
raise ValueError(f"{label} frequency axes do not match")
|
||||
|
||||
|
||||
def _build_history_payload(
|
||||
*,
|
||||
source_dir: Path,
|
||||
mode: str,
|
||||
frequency_hz: np.ndarray,
|
||||
sweep_scans: list[tuple[str, np.ndarray]],
|
||||
calibrated_scans: list[tuple[str, np.ndarray]],
|
||||
reference_trace: np.ndarray,
|
||||
primary_stage: str,
|
||||
raw_record_count: int,
|
||||
preprocessed_record_count: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Assemble the vna-system history payload from paired sweep and calibrated scans.
|
||||
|
||||
Pairs each sweep scan with its calibrated counterpart by order (names must match),
|
||||
attaching the shared reference trace and sweep config to every history entry.
|
||||
"""
|
||||
if len(sweep_scans) != len(calibrated_scans):
|
||||
raise ValueError("Sweep/calibrated scan counts do not match")
|
||||
|
||||
sweep_history: list[dict[str, Any]] = []
|
||||
reference_points = _complex_to_points(reference_trace)
|
||||
for index, ((scan_name, sweep_trace), (cal_name, calibrated_trace)) in enumerate(
|
||||
zip(sweep_scans, calibrated_scans, strict=True)
|
||||
):
|
||||
if scan_name != cal_name:
|
||||
raise ValueError(f"Scan ordering mismatch: {scan_name} vs {cal_name}")
|
||||
|
||||
sweep_history.append(
|
||||
{
|
||||
"timestamp": float(index),
|
||||
"sweep_points": _complex_to_points(sweep_trace),
|
||||
"calibrated_points": _complex_to_points(calibrated_trace),
|
||||
"reference_points": reference_points,
|
||||
"vna_config": {
|
||||
"mode": mode,
|
||||
"start_freq": float(frequency_hz[0]),
|
||||
"stop_freq": float(frequency_hz[-1]),
|
||||
"points": int(frequency_hz.size),
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"format": "vna-system-history-v1",
|
||||
"converter": "python_app/scripts/convert_prog_libre_manual_to_vna_history.py",
|
||||
"converted_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"source_snapshot_dir": str(source_dir.resolve()),
|
||||
"input_index": 0,
|
||||
"output_index": 0,
|
||||
"primary_stage": primary_stage,
|
||||
"raw_record_count": int(raw_record_count),
|
||||
"preprocessed_record_count": int(preprocessed_record_count),
|
||||
"sweep_history": sweep_history,
|
||||
}
|
||||
|
||||
|
||||
def _write_payload(output_path: Path, payload: dict[str, Any]) -> None:
|
||||
"""Write a payload as indented UTF-8 JSON, creating parent directories as needed."""
|
||||
atomic_write_json(output_path, payload, durable=True)
|
||||
|
||||
|
||||
def _rmse(left: np.ndarray, right: np.ndarray) -> float:
|
||||
"""Return the root-mean-square magnitude difference between two complex traces."""
|
||||
return float(np.sqrt(np.mean(np.abs(left - right) ** 2)))
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
"""Return the argument parser for the prog_libre-to-vna-history conversion CLI."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Convert manual LibreVNA S11/S21 CSV captures in prog_libre to vna history JSON.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--calibration-dir",
|
||||
type=Path,
|
||||
default=Path("prog_libre/calibration"),
|
||||
help="Directory with calibration CSV files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--raw-dir",
|
||||
type=Path,
|
||||
default=Path("prog_libre/1-6000mhz_no-calibrated_libre"),
|
||||
help="Directory with uncalibrated scan CSV files and ref.csv.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--calibrated-dir",
|
||||
type=Path,
|
||||
default=Path("prog_libre/1-6000mhz_calibrated_libre"),
|
||||
help="Directory with already calibrated scan CSV files and ref.csv.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--raw-output",
|
||||
"--s11-raw-output",
|
||||
dest="s11_raw_output",
|
||||
type=Path,
|
||||
default=Path("prog_libre/1-6000mhz_no-calibrated_libre_s11_osl_p1_vna_bscan_history.json"),
|
||||
help="Output JSON for uncalibrated S11 scans after applying OSL calibration.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--calibrated-output",
|
||||
"--s11-calibrated-output",
|
||||
dest="s11_calibrated_output",
|
||||
type=Path,
|
||||
default=Path("prog_libre/1-6000mhz_calibrated_libre_s11_passthrough_vna_bscan_history.json"),
|
||||
help="Output JSON for already calibrated S11 scans without extra calibration.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--s21-raw-output",
|
||||
dest="s21_raw_output",
|
||||
type=Path,
|
||||
default=Path("prog_libre/1-6000mhz_no-calibrated_libre_s21_through_vna_bscan_history.json"),
|
||||
help="Output JSON for uncalibrated S21 scans after applying through calibration.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--s21-calibrated-output",
|
||||
dest="s21_calibrated_output",
|
||||
type=Path,
|
||||
default=Path("prog_libre/1-6000mhz_calibrated_libre_s21_passthrough_vna_bscan_history.json"),
|
||||
help="Output JSON for already calibrated S21 scans without extra calibration.",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Calibrate and convert the prog_libre S11/S21 CSV captures into history JSON files.
|
||||
|
||||
Builds OSL coefficients for S11 and a through reference for S21, applies them to the
|
||||
uncalibrated scans, writes raw and pre-calibrated history files for both channels, and
|
||||
prints RMSE diagnostics against the supplied calibrated folder.
|
||||
"""
|
||||
args = _build_parser().parse_args()
|
||||
|
||||
calibration_dir = args.calibration_dir.expanduser().resolve()
|
||||
raw_dir = args.raw_dir.expanduser().resolve()
|
||||
calibrated_dir = args.calibrated_dir.expanduser().resolve()
|
||||
|
||||
s11_raw_output = args.s11_raw_output.expanduser().resolve()
|
||||
s11_calibrated_output = args.s11_calibrated_output.expanduser().resolve()
|
||||
s21_raw_output = args.s21_raw_output.expanduser().resolve()
|
||||
s21_calibrated_output = args.s21_calibrated_output.expanduser().resolve()
|
||||
|
||||
s11_cal_frequency_hz, open_trace = _load_complex_trace(calibration_dir / "open_rfc18_p1.csv", "S11")
|
||||
short_frequency_hz, short_trace = _load_complex_trace(calibration_dir / "short_rfc18_p1.csv", "S11")
|
||||
load_frequency_hz, load_trace = _load_complex_trace(calibration_dir / "load_rfc18_p1.csv", "S11")
|
||||
_require_matching_frequency_axis("S11 calibration", s11_cal_frequency_hz, short_frequency_hz)
|
||||
_require_matching_frequency_axis("S11 calibration", s11_cal_frequency_hz, load_frequency_hz)
|
||||
|
||||
directivity, source_match, reflection_tracking = _solve_one_port_osl(open_trace, short_trace, load_trace)
|
||||
|
||||
s11_raw_frequency_hz, s11_raw_scans = _load_scan_series(raw_dir, "S11")
|
||||
s11_calibrated_frequency_hz, s11_passthrough_scans = _load_scan_series(calibrated_dir, "S11")
|
||||
_require_matching_frequency_axis("S11 raw vs calibration", s11_raw_frequency_hz, s11_cal_frequency_hz)
|
||||
_require_matching_frequency_axis("S11 calibrated vs calibration", s11_calibrated_frequency_hz, s11_cal_frequency_hz)
|
||||
|
||||
s11_raw_reference_frequency_hz, s11_raw_reference = _load_complex_trace(raw_dir / "ref.csv", "S11")
|
||||
s11_calibrated_reference_frequency_hz, s11_calibrated_reference = _load_complex_trace(calibrated_dir / "ref.csv", "S11")
|
||||
_require_matching_frequency_axis("S11 raw reference vs calibration", s11_raw_reference_frequency_hz, s11_cal_frequency_hz)
|
||||
_require_matching_frequency_axis(
|
||||
"S11 calibrated reference vs calibration",
|
||||
s11_calibrated_reference_frequency_hz,
|
||||
s11_cal_frequency_hz,
|
||||
)
|
||||
|
||||
s11_corrected_scans = [
|
||||
(
|
||||
scan_name,
|
||||
_apply_one_port_osl(trace, directivity, source_match, reflection_tracking),
|
||||
)
|
||||
for scan_name, trace in s11_raw_scans
|
||||
]
|
||||
s11_corrected_reference = _apply_one_port_osl(s11_raw_reference, directivity, source_match, reflection_tracking)
|
||||
|
||||
s11_raw_payload = _build_history_payload(
|
||||
source_dir=raw_dir,
|
||||
mode="s11",
|
||||
frequency_hz=s11_raw_frequency_hz,
|
||||
sweep_scans=s11_raw_scans,
|
||||
calibrated_scans=s11_corrected_scans,
|
||||
reference_trace=s11_corrected_reference,
|
||||
primary_stage="raw",
|
||||
raw_record_count=len(s11_raw_scans),
|
||||
preprocessed_record_count=len(s11_corrected_scans),
|
||||
)
|
||||
s11_calibrated_payload = _build_history_payload(
|
||||
source_dir=calibrated_dir,
|
||||
mode="s11",
|
||||
frequency_hz=s11_calibrated_frequency_hz,
|
||||
sweep_scans=s11_passthrough_scans,
|
||||
calibrated_scans=s11_passthrough_scans,
|
||||
reference_trace=s11_calibrated_reference,
|
||||
primary_stage="preprocessed",
|
||||
raw_record_count=0,
|
||||
preprocessed_record_count=len(s11_passthrough_scans),
|
||||
)
|
||||
|
||||
s21_cal_frequency_hz, through_trace = _load_complex_trace(calibration_dir / "through21_rfc18.csv", "S21")
|
||||
|
||||
s21_raw_frequency_hz, s21_raw_scans = _load_scan_series(raw_dir, "S21")
|
||||
s21_calibrated_frequency_hz, s21_passthrough_scans = _load_scan_series(calibrated_dir, "S21")
|
||||
_require_matching_frequency_axis("S21 raw vs calibration", s21_raw_frequency_hz, s21_cal_frequency_hz)
|
||||
_require_matching_frequency_axis("S21 calibrated vs calibration", s21_calibrated_frequency_hz, s21_cal_frequency_hz)
|
||||
|
||||
s21_raw_reference_frequency_hz, s21_raw_reference = _load_complex_trace(raw_dir / "ref.csv", "S21")
|
||||
s21_calibrated_reference_frequency_hz, s21_calibrated_reference = _load_complex_trace(calibrated_dir / "ref.csv", "S21")
|
||||
_require_matching_frequency_axis("S21 raw reference vs calibration", s21_raw_reference_frequency_hz, s21_cal_frequency_hz)
|
||||
_require_matching_frequency_axis(
|
||||
"S21 calibrated reference vs calibration",
|
||||
s21_calibrated_reference_frequency_hz,
|
||||
s21_cal_frequency_hz,
|
||||
)
|
||||
|
||||
s21_corrected_scans = [
|
||||
(
|
||||
scan_name,
|
||||
_apply_through_calibration(trace, through_trace),
|
||||
)
|
||||
for scan_name, trace in s21_raw_scans
|
||||
]
|
||||
s21_corrected_reference = _apply_through_calibration(s21_raw_reference, through_trace)
|
||||
|
||||
s21_raw_payload = _build_history_payload(
|
||||
source_dir=raw_dir,
|
||||
mode="s21",
|
||||
frequency_hz=s21_raw_frequency_hz,
|
||||
sweep_scans=s21_raw_scans,
|
||||
calibrated_scans=s21_corrected_scans,
|
||||
reference_trace=s21_corrected_reference,
|
||||
primary_stage="raw",
|
||||
raw_record_count=len(s21_raw_scans),
|
||||
preprocessed_record_count=len(s21_corrected_scans),
|
||||
)
|
||||
s21_calibrated_payload = _build_history_payload(
|
||||
source_dir=calibrated_dir,
|
||||
mode="s21",
|
||||
frequency_hz=s21_calibrated_frequency_hz,
|
||||
sweep_scans=s21_passthrough_scans,
|
||||
calibrated_scans=s21_passthrough_scans,
|
||||
reference_trace=s21_calibrated_reference,
|
||||
primary_stage="preprocessed",
|
||||
raw_record_count=0,
|
||||
preprocessed_record_count=len(s21_passthrough_scans),
|
||||
)
|
||||
|
||||
_write_payload(s11_raw_output, s11_raw_payload)
|
||||
_write_payload(s11_calibrated_output, s11_calibrated_payload)
|
||||
_write_payload(s21_raw_output, s21_raw_payload)
|
||||
_write_payload(s21_calibrated_output, s21_calibrated_payload)
|
||||
|
||||
s11_rmse_values = [
|
||||
_rmse(corrected_trace, passthrough_trace)
|
||||
for (_, corrected_trace), (_, passthrough_trace) in zip(s11_corrected_scans, s11_passthrough_scans, strict=True)
|
||||
]
|
||||
s21_rmse_values = [
|
||||
_rmse(corrected_trace, passthrough_trace)
|
||||
for (_, corrected_trace), (_, passthrough_trace) in zip(s21_corrected_scans, s21_passthrough_scans, strict=True)
|
||||
]
|
||||
s11_reference_rmse = _rmse(s11_corrected_reference, s11_calibrated_reference)
|
||||
s21_reference_rmse = _rmse(s21_corrected_reference, s21_calibrated_reference)
|
||||
|
||||
print(
|
||||
"Converted manual prog_libre captures to vna history JSON:\n"
|
||||
f" S11 raw output: {s11_raw_output}\n"
|
||||
f" S11 calibrated output: {s11_calibrated_output}\n"
|
||||
f" S21 raw output: {s21_raw_output}\n"
|
||||
f" S21 calibrated output: {s21_calibrated_output}\n"
|
||||
f" sweep count: {len(s11_raw_scans)}\n"
|
||||
f" points per sweep: {s11_raw_frequency_hz.size}\n"
|
||||
f" S11 calibration: p1 ideal OSL\n"
|
||||
f" S11 mean sweep RMSE vs provided calibrated folder: {float(np.mean(s11_rmse_values)):.6f}\n"
|
||||
f" S11 max sweep RMSE vs provided calibrated folder: {float(np.max(s11_rmse_values)):.6f}\n"
|
||||
f" S11 reference RMSE vs provided calibrated ref: {s11_reference_rmse:.6f}\n"
|
||||
f" S21 calibration: through21 complex division\n"
|
||||
f" S21 mean sweep RMSE vs provided calibrated folder: {float(np.mean(s21_rmse_values)):.6f}\n"
|
||||
f" S21 max sweep RMSE vs provided calibrated folder: {float(np.max(s21_rmse_values)):.6f}\n"
|
||||
f" S21 reference RMSE vs provided calibrated ref: {s21_reference_rmse:.6f}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,310 +0,0 @@
|
||||
"""Convert radar_system runtime snapshots into vna_system sweep-history JSON."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from python_app.file_io import atomic_write_json
|
||||
|
||||
|
||||
from python_app.storage.npz.vna_history_json import TraceRecord, _build_sweep_history, _normalize_channel
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CollectionRef:
|
||||
"""Minimal collection identity descriptor for stage-alignment diagnostics."""
|
||||
|
||||
stage_index: int
|
||||
collection_id: int
|
||||
monotonic_ns: int
|
||||
|
||||
|
||||
def _load_json(path: Path) -> dict[str, Any]:
|
||||
"""Read a JSON file and return its top-level object, rejecting non-object roots."""
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"JSON root must be object: {path}")
|
||||
return payload
|
||||
|
||||
|
||||
def _collection_dirs(stage_dir: Path) -> list[Path]:
|
||||
"""Return collection subdirectories in numeric recording order (empty if absent)."""
|
||||
if not stage_dir.exists():
|
||||
return []
|
||||
return sorted(
|
||||
(path for path in stage_dir.iterdir() if path.is_dir()),
|
||||
key=lambda path: (_parse_stage_index(path.name, 0), path.name),
|
||||
)
|
||||
|
||||
|
||||
def _parse_stage_index(name: str, fallback: int) -> int:
|
||||
"""Return the leading numeric prefix of a collection dir name, or ``fallback``."""
|
||||
prefix = name.split("_", 1)[0]
|
||||
return int(prefix) if prefix.isdigit() else fallback
|
||||
|
||||
|
||||
def _pick_trace_meta(meta: dict[str, Any], input_index: int, output_index: int) -> dict[str, Any] | None:
|
||||
"""Return the trace record matching the given input/output switch indices, or None."""
|
||||
traces = meta.get("traces", [])
|
||||
if not isinstance(traces, list):
|
||||
return None
|
||||
for trace in traces:
|
||||
if not isinstance(trace, dict):
|
||||
continue
|
||||
if int(trace.get("input", -1)) == input_index and int(trace.get("output", -1)) == output_index:
|
||||
return trace
|
||||
return None
|
||||
|
||||
|
||||
def _load_stage_records(
|
||||
snapshot_dir: Path,
|
||||
stage: str,
|
||||
*,
|
||||
input_index: int,
|
||||
output_index: int,
|
||||
channel: str,
|
||||
) -> list[TraceRecord]:
|
||||
"""Load TraceRecords for one stage and one switch combo, for the given channel.
|
||||
|
||||
Skips collections without a matching trace or array files; raises on shape mismatch
|
||||
or non-finite samples.
|
||||
"""
|
||||
stage_dir = snapshot_dir / stage
|
||||
records: list[TraceRecord] = []
|
||||
|
||||
for fallback_idx, collection_dir in enumerate(_collection_dirs(stage_dir)):
|
||||
meta_path = collection_dir / "meta.json"
|
||||
if not meta_path.exists():
|
||||
continue
|
||||
|
||||
meta = _load_json(meta_path)
|
||||
trace_meta = _pick_trace_meta(meta, input_index, output_index)
|
||||
if trace_meta is None:
|
||||
continue
|
||||
|
||||
freq_file = str(trace_meta.get("freq_file", ""))
|
||||
samples_file = str(trace_meta.get(f"{channel}_file", ""))
|
||||
if not freq_file or not samples_file:
|
||||
continue
|
||||
|
||||
frequency_hz = np.asarray(np.load(collection_dir / freq_file), dtype=np.float64).reshape(-1)
|
||||
samples = np.asarray(np.load(collection_dir / samples_file), dtype=np.complex128).reshape(-1)
|
||||
if frequency_hz.shape != samples.shape:
|
||||
raise ValueError(f"Shape mismatch in {collection_dir}: freq{frequency_hz.shape} vs {channel}{samples.shape}")
|
||||
if frequency_hz.size == 0:
|
||||
continue
|
||||
if not (
|
||||
np.isfinite(frequency_hz).all()
|
||||
and np.isfinite(np.real(samples)).all()
|
||||
and np.isfinite(np.imag(samples)).all()
|
||||
):
|
||||
raise ValueError(f"Non-finite values in {collection_dir}")
|
||||
|
||||
records.append(
|
||||
TraceRecord(
|
||||
stage=stage,
|
||||
collection_id=int(meta.get("collection_id", -1)),
|
||||
monotonic_ns=int(meta.get("monotonic_ns", 0)),
|
||||
stage_index=_parse_stage_index(collection_dir.name, fallback_idx),
|
||||
frequency_hz=frequency_hz,
|
||||
samples=samples,
|
||||
capture_end_ns=int(trace_meta.get("capture_end_ns", 0)),
|
||||
capture_start_ns=int(trace_meta.get("capture_start_ns", 0)),
|
||||
)
|
||||
)
|
||||
|
||||
return records
|
||||
|
||||
|
||||
def _load_stage_refs(snapshot_dir: Path, stage: str) -> list[CollectionRef]:
|
||||
"""Load `(index, collection_id, monotonic_ns)` for one snapshot stage."""
|
||||
stage_dir = snapshot_dir / stage
|
||||
refs: list[CollectionRef] = []
|
||||
|
||||
for fallback_idx, collection_dir in enumerate(_collection_dirs(stage_dir)):
|
||||
meta_path = collection_dir / "meta.json"
|
||||
if not meta_path.exists():
|
||||
continue
|
||||
meta = _load_json(meta_path)
|
||||
refs.append(
|
||||
CollectionRef(
|
||||
stage_index=_parse_stage_index(collection_dir.name, fallback_idx),
|
||||
collection_id=int(meta.get("collection_id", -1)),
|
||||
monotonic_ns=int(meta.get("monotonic_ns", 0)),
|
||||
)
|
||||
)
|
||||
return refs
|
||||
|
||||
|
||||
def _stage_alignment_warning(pre_refs: list[CollectionRef], result_refs: list[CollectionRef]) -> str | None:
|
||||
"""Return warning text when preprocessed/results stages are not identity-aligned."""
|
||||
if not pre_refs or not result_refs:
|
||||
return None
|
||||
|
||||
pre_by_index = {ref.stage_index: ref for ref in pre_refs}
|
||||
result_by_index = {ref.stage_index: ref for ref in result_refs}
|
||||
common_indices = sorted(set(pre_by_index) & set(result_by_index))
|
||||
if not common_indices:
|
||||
return None
|
||||
|
||||
mismatches = 0
|
||||
first_mismatch: tuple[int, CollectionRef, CollectionRef] | None = None
|
||||
for index in common_indices:
|
||||
pre = pre_by_index[index]
|
||||
result = result_by_index[index]
|
||||
if pre.collection_id != result.collection_id or pre.monotonic_ns != result.monotonic_ns:
|
||||
mismatches += 1
|
||||
if first_mismatch is None:
|
||||
first_mismatch = (index, pre, result)
|
||||
|
||||
if mismatches == 0:
|
||||
return None
|
||||
|
||||
assert first_mismatch is not None
|
||||
idx, pre, result = first_mismatch
|
||||
return (
|
||||
"WARNING: snapshot stages are not fully aligned (preprocessed vs results). "
|
||||
f"Mismatches={mismatches}/{len(common_indices)}. "
|
||||
f"First mismatch at index={idx}: "
|
||||
f"pre=(id={pre.collection_id},ns={pre.monotonic_ns}) vs "
|
||||
f"results=(id={result.collection_id},ns={result.monotonic_ns}). "
|
||||
"Export uses preprocessed traces; loaded view in vna_system may differ from "
|
||||
"radar_system on-screen replayed results."
|
||||
)
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
"""Return the argument parser for the snapshot-to-vna-history conversion CLI."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Convert radar_system snapshot (numpy-directory-v1) to a vna_system-compatible "
|
||||
"history JSON file with `sweep_history`."
|
||||
)
|
||||
)
|
||||
parser.add_argument("snapshot_dir", type=Path, help="Path to snapshot directory containing raw/preprocessed/results.")
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Output JSON path (default: <snapshot_dir>/vna_bscan_history.json).",
|
||||
)
|
||||
parser.add_argument("--input", dest="input_index", type=int, default=0, help="Input switch index to export.")
|
||||
parser.add_argument("--output-index", dest="output_index", type=int, default=0, help="Output switch index to export.")
|
||||
parser.add_argument(
|
||||
"--channel",
|
||||
choices=("s21", "s11"),
|
||||
default="s21",
|
||||
help="Trace channel to export into sweep/calibrated points.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--primary-stage",
|
||||
choices=("preprocessed", "raw"),
|
||||
default="preprocessed",
|
||||
help="Stage order to drive collection selection/alignment.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--last-n",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Keep only the last N sweeps in output (0 means all available).",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Convert one snapshot's chosen channel/combo into a vna-system history JSON file.
|
||||
|
||||
Loads raw and preprocessed traces, builds the sweep history (optionally trimmed to the
|
||||
last N sweeps), writes the output JSON, and prints a summary plus any stage-alignment
|
||||
warning.
|
||||
"""
|
||||
parser = _build_parser()
|
||||
args = parser.parse_args()
|
||||
channel = _normalize_channel(args.channel)
|
||||
|
||||
snapshot_dir = args.snapshot_dir.expanduser().resolve()
|
||||
if not snapshot_dir.exists():
|
||||
raise FileNotFoundError(f"Snapshot directory not found: {snapshot_dir}")
|
||||
|
||||
output_path = (
|
||||
args.output.expanduser().resolve()
|
||||
if args.output is not None
|
||||
else snapshot_dir / "vna_bscan_history.json"
|
||||
)
|
||||
|
||||
raw_records = _load_stage_records(
|
||||
snapshot_dir,
|
||||
"raw",
|
||||
input_index=args.input_index,
|
||||
output_index=args.output_index,
|
||||
channel=channel,
|
||||
)
|
||||
preprocessed_records = _load_stage_records(
|
||||
snapshot_dir,
|
||||
"preprocessed",
|
||||
input_index=args.input_index,
|
||||
output_index=args.output_index,
|
||||
channel=channel,
|
||||
)
|
||||
if not raw_records and not preprocessed_records:
|
||||
raise ValueError(
|
||||
"No matching raw/preprocessed traces were found in snapshot "
|
||||
f"for input={args.input_index}, output={args.output_index}."
|
||||
)
|
||||
|
||||
sweep_history = _build_sweep_history(
|
||||
raw_records,
|
||||
preprocessed_records,
|
||||
channel=channel,
|
||||
primary_stage=args.primary_stage,
|
||||
)
|
||||
if args.last_n > 0:
|
||||
sweep_history = sweep_history[-args.last_n :]
|
||||
if not sweep_history:
|
||||
raise ValueError("Conversion produced empty sweep_history.")
|
||||
|
||||
pre_refs = _load_stage_refs(snapshot_dir, "preprocessed")
|
||||
result_refs = _load_stage_refs(snapshot_dir, "results")
|
||||
alignment_warning = _stage_alignment_warning(pre_refs, result_refs)
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"format": "vna-system-history-v1",
|
||||
"converter": "python_app/scripts/convert_snapshot_to_vna_history.py",
|
||||
"converted_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"source_snapshot_dir": str(snapshot_dir),
|
||||
"input_index": int(args.input_index),
|
||||
"output_index": int(args.output_index),
|
||||
"channel": channel,
|
||||
"primary_stage": args.primary_stage,
|
||||
"raw_record_count": len(raw_records),
|
||||
"preprocessed_record_count": len(preprocessed_records),
|
||||
"sweep_history": sweep_history,
|
||||
}
|
||||
if alignment_warning is not None:
|
||||
payload["alignment_warning"] = alignment_warning
|
||||
|
||||
atomic_write_json(output_path, payload, durable=True)
|
||||
|
||||
print(
|
||||
"Converted snapshot to vna_system history JSON:\n"
|
||||
f" input snapshot: {snapshot_dir}\n"
|
||||
f" output file: {output_path}\n"
|
||||
f" channel: {channel}\n"
|
||||
f" sweeps written: {len(sweep_history)}\n"
|
||||
f" raw records: {len(raw_records)}\n"
|
||||
f" pre records: {len(preprocessed_records)}"
|
||||
)
|
||||
if alignment_warning is not None:
|
||||
print(f"[convert-warning] {alignment_warning}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,94 +0,0 @@
|
||||
"""CLI to create neutral S21 preprocess sets for a profile (headless).
|
||||
|
||||
Mirrors the GUI's "Create neutral sets" button: builds calibration = 1+0j and
|
||||
reference = 0 collections for every combo of the profile and saves them under
|
||||
the profile's radar key, so the data_preprocessor becomes a bit-exact
|
||||
pass-through. Needed for replay profiles whose recorded data is already
|
||||
preprocessed (see run_config_examples/run_config_replay_vna.example.json).
|
||||
|
||||
Usage:
|
||||
python -m python_app.scripts.create_neutral_preprocess_sets \
|
||||
--config run_config_examples/run_config_replay_vna.example.json \
|
||||
[--set-name neutral_replay] [--force]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from python_app.models.run_config_model import RunConfigModel
|
||||
from python_app.storage.npz_store import NpzStore, radar_key_from_config
|
||||
from python_app.workflows.kamil_adc_neutral_preprocess import (
|
||||
build_neutral_s21_sets,
|
||||
supports_neutral_preprocess_sets,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Create and save the neutral S21 calibration/reference pair."""
|
||||
parser = argparse.ArgumentParser(description="Create neutral S21 preprocess sets")
|
||||
parser.add_argument("--config", required=True, type=Path, help="Path to a run_config profile")
|
||||
parser.add_argument("--set-name", default="neutral_replay", help="Name for both saved sets")
|
||||
parser.add_argument(
|
||||
"--force", action="store_true", help="Overwrite the sets if they already exist"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
|
||||
|
||||
config = RunConfigModel.load_from_path(args.config)
|
||||
config.ensure_combos()
|
||||
if not supports_neutral_preprocess_sets(config):
|
||||
raise RuntimeError("Neutral S21 sets are not supported for this radar configuration")
|
||||
|
||||
radar_key = radar_key_from_config(
|
||||
model_name=config.radar.model,
|
||||
serial=config.radar.serial,
|
||||
sweep_start_hz=config.radar.sweep.start_hz,
|
||||
sweep_stop_hz=config.radar.sweep.stop_hz,
|
||||
sweep_points=config.radar.sweep.points,
|
||||
ifbw_hz=config.radar.sweep.if_bandwidth_hz,
|
||||
power_dbm=config.radar.sweep.power_dbm,
|
||||
extra_serials=config.radar_key_extra_parts() or None,
|
||||
)
|
||||
store = NpzStore(PROJECT_ROOT / "python_app/data")
|
||||
|
||||
kinds = ("s21_calibration", "s21_reference")
|
||||
existing = [
|
||||
kind
|
||||
for kind in kinds
|
||||
if args.set_name in store.list_sets(kind, radar_key)
|
||||
]
|
||||
if existing and not args.force:
|
||||
if len(existing) != len(kinds):
|
||||
missing = ", ".join(kind for kind in kinds if kind not in existing)
|
||||
raise RuntimeError(
|
||||
f"Neutral set pair '{args.set_name}' is incomplete: missing {missing}; "
|
||||
"use --force to replace both sets."
|
||||
)
|
||||
logger.info(
|
||||
"Sets '%s' already exist for %s (radar_key=%s); nothing to do (use --force to overwrite).",
|
||||
args.set_name, ", ".join(existing), radar_key,
|
||||
)
|
||||
return 0
|
||||
|
||||
calibration, reference = build_neutral_s21_sets(config)
|
||||
store.save_sets(radar_key, args.set_name, dict(zip(kinds, (calibration, reference))))
|
||||
logger.info(
|
||||
"Neutral S21 sets saved: set=%s, radar_key=%s, combos=%d, points=%d",
|
||||
args.set_name,
|
||||
radar_key,
|
||||
len(calibration.traces),
|
||||
int(calibration.traces[0].frequency_hz.size),
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,349 +0,0 @@
|
||||
"""Standalone GUI utility for inspecting raw orchestrator ring output."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
import ctypes.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
from PyQt6.QtCore import QTimer
|
||||
from PyQt6.QtWidgets import QApplication, QLabel, QMainWindow, QVBoxLayout, QWidget
|
||||
import pyqtgraph as pg
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from python_app.hardware_full.single_radar_service import create_single_radar_service
|
||||
from python_app.models.run_config_model import RunConfigModel
|
||||
from python_app.orchestration.shm_reader import ShmRingReader
|
||||
|
||||
|
||||
def _shm_unlink(name: str) -> None:
|
||||
"""Best-effort unlink for POSIX shared-memory object."""
|
||||
libc_name = ctypes.util.find_library("c")
|
||||
if libc_name is None:
|
||||
return
|
||||
|
||||
libc = ctypes.CDLL(libc_name, use_errno=True)
|
||||
libc.shm_unlink.argtypes = [ctypes.c_char_p]
|
||||
libc.shm_unlink.restype = ctypes.c_int
|
||||
|
||||
result = libc.shm_unlink(name.encode("utf-8"))
|
||||
if result == 0:
|
||||
return
|
||||
|
||||
err = ctypes.get_errno()
|
||||
if err != 2: # ENOENT
|
||||
raise OSError(err, f"shm_unlink failed for {name}")
|
||||
|
||||
|
||||
def _read_raw_ring_name(config_path: Path) -> str:
|
||||
"""Extract raw ring name from run config."""
|
||||
config = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
ring_name = config["rings"]["raw"]["name"]
|
||||
if not isinstance(ring_name, str) or not ring_name.startswith("/"):
|
||||
raise RuntimeError("Config rings.raw.name must be a POSIX shm name starting with '/'")
|
||||
return ring_name
|
||||
|
||||
|
||||
def _read_native_summary(config_path: Path) -> str:
|
||||
"""Build short summary of radar/switch driver modes."""
|
||||
config = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
radar_mode = config["radar"]["driver_mode"]
|
||||
switches = config["switches"]
|
||||
if "port1" in switches and "port2" in switches:
|
||||
port1_mode = switches["port1"]["driver_mode"]
|
||||
port2_mode = switches["port2"]["driver_mode"]
|
||||
else:
|
||||
port1_mode = switches["output"]["driver_mode"]
|
||||
port2_mode = switches["input"]["driver_mode"]
|
||||
return f"radar={radar_mode}, port1={port1_mode}, port2={port2_mode}"
|
||||
|
||||
|
||||
def _prepare_radar_if_needed(config_path: Path, *, strict: bool) -> str | None:
|
||||
"""Preconfigure native radar through Python service when requested."""
|
||||
config_payload = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
config = RunConfigModel.from_dict(config_payload)
|
||||
if config.radar.driver_mode != "native":
|
||||
return "Radar pre-configuration skipped (mock mode)."
|
||||
if config.is_matrix_radar:
|
||||
return "Radar pre-configuration skipped (matrix radar producer config)."
|
||||
|
||||
radar_service = None
|
||||
try:
|
||||
radar_service = create_single_radar_service(config)
|
||||
radar_service.open()
|
||||
radar_service.configure(config.radar.sweep)
|
||||
return "Radar pre-configuration completed."
|
||||
except Exception as exc:
|
||||
message = f"Radar pre-configuration failed ({exc})"
|
||||
if strict:
|
||||
raise
|
||||
return f"{message}. Continuing with native C++ configuration."
|
||||
finally:
|
||||
if radar_service is not None:
|
||||
radar_service.close()
|
||||
|
||||
|
||||
class RawOrchestratorViewer(QMainWindow):
|
||||
"""Qt window that runs sweep_orchestrator and plots raw collections."""
|
||||
|
||||
def __init__(self, config_path: Path, reset_ring: bool, prepare_radar: bool, strict_prepare: bool) -> None:
|
||||
"""Initialize viewer, optionally prepare radar, and start polling."""
|
||||
super().__init__()
|
||||
|
||||
self._config_path = config_path
|
||||
self._raw_ring_name = _read_raw_ring_name(config_path)
|
||||
self._orchestrator_process: subprocess.Popen[str] | None = None
|
||||
self._raw_reader: ShmRingReader | None = None
|
||||
|
||||
if reset_ring:
|
||||
_shm_unlink(self._raw_ring_name)
|
||||
|
||||
self._build_ui()
|
||||
if prepare_radar:
|
||||
self._status.setText("Preparing radar...")
|
||||
prepare_status = _prepare_radar_if_needed(self._config_path, strict=strict_prepare)
|
||||
if prepare_status is not None:
|
||||
self._status.setText(prepare_status)
|
||||
self._start_orchestrator()
|
||||
self._open_reader_or_fail()
|
||||
|
||||
self._timer = QTimer(self)
|
||||
self._timer.setInterval(60)
|
||||
self._timer.timeout.connect(self._poll)
|
||||
self._timer.start()
|
||||
|
||||
def _build_ui(self) -> None:
|
||||
"""Build viewer widgets and raw trace plot."""
|
||||
self.setWindowTitle("Raw Sweep Viewer (orchestrator)")
|
||||
root = QWidget(self)
|
||||
self.setCentralWidget(root)
|
||||
|
||||
layout = QVBoxLayout(root)
|
||||
|
||||
mode_summary = _read_native_summary(self._config_path)
|
||||
self._status = QLabel(f"Starting... ({mode_summary})")
|
||||
layout.addWidget(self._status)
|
||||
|
||||
self._plot = pg.PlotWidget(background="#101418")
|
||||
self._plot.showGrid(x=True, y=True, alpha=0.2)
|
||||
self._plot.setLabel("bottom", "Frequency", units="Hz")
|
||||
self._plot.setLabel("left", "Magnitude", units="dB")
|
||||
layout.addWidget(self._plot)
|
||||
|
||||
self.resize(1400, 900)
|
||||
|
||||
def _start_orchestrator(self) -> None:
|
||||
"""Start sweep orchestrator subprocess."""
|
||||
binary = PROJECT_ROOT / "build/bin/sweep_orchestrator"
|
||||
if not binary.exists():
|
||||
raise RuntimeError(f"Missing binary: {binary}. Build first with 'make -j4'")
|
||||
|
||||
command = [str(binary), "--config", str(self._config_path)]
|
||||
self._orchestrator_process = subprocess.Popen(
|
||||
command,
|
||||
cwd=PROJECT_ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
|
||||
def _open_reader_or_fail(self) -> None:
|
||||
"""Wait for raw ring readiness and open `ShmRingReader`."""
|
||||
if self._orchestrator_process is None:
|
||||
raise RuntimeError("Orchestrator process is not started")
|
||||
|
||||
shm_path = Path("/dev/shm") / self._raw_ring_name[1:]
|
||||
deadline = time.monotonic() + 5.0
|
||||
last_reader_error: str | None = None
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
return_code = self._orchestrator_process.poll()
|
||||
if return_code is not None:
|
||||
details = self._read_process_output(self._orchestrator_process)
|
||||
raise RuntimeError(f"sweep_orchestrator exited with code {return_code}: {details}")
|
||||
|
||||
if shm_path.exists():
|
||||
try:
|
||||
self._raw_reader = ShmRingReader(self._raw_ring_name)
|
||||
self._status.setText(f"Running: ring={self._raw_ring_name}")
|
||||
return
|
||||
except RuntimeError as exc:
|
||||
last_reader_error = str(exc)
|
||||
|
||||
time.sleep(0.05)
|
||||
|
||||
if last_reader_error is not None:
|
||||
raise RuntimeError(
|
||||
f"Timed out waiting for raw ring header readiness: {self._raw_ring_name}; "
|
||||
f"last error: {last_reader_error}"
|
||||
)
|
||||
raise RuntimeError(f"Timed out waiting for raw ring file: {shm_path}")
|
||||
|
||||
def _poll(self) -> None:
|
||||
"""Poll subprocess state and draw latest available raw collection."""
|
||||
if self._orchestrator_process is None:
|
||||
return
|
||||
|
||||
return_code = self._orchestrator_process.poll()
|
||||
if return_code is not None:
|
||||
details = self._read_process_output(self._orchestrator_process)
|
||||
self._status.setText(f"Error: sweep_orchestrator exited ({return_code})")
|
||||
raise RuntimeError(f"sweep_orchestrator exited with code {return_code}: {details}")
|
||||
|
||||
if self._raw_reader is None:
|
||||
return
|
||||
|
||||
latest = None
|
||||
for _ in range(16):
|
||||
collection = self._raw_reader.pop_raw_collection()
|
||||
if collection is None:
|
||||
break
|
||||
latest = collection
|
||||
|
||||
if latest is not None:
|
||||
self._draw_collection(latest)
|
||||
|
||||
def _draw_collection(self, collection) -> None:
|
||||
"""Render all traces from one raw collection."""
|
||||
self._plot.clear()
|
||||
palette = [
|
||||
"#4cc9f0",
|
||||
"#f72585",
|
||||
"#b8f2e6",
|
||||
"#ffd166",
|
||||
"#90be6d",
|
||||
"#ff595e",
|
||||
"#6a4c93",
|
||||
"#1982c4",
|
||||
"#ff9f1c",
|
||||
"#2ec4b6",
|
||||
"#e71d36",
|
||||
"#a0c4ff",
|
||||
]
|
||||
|
||||
for idx, trace in enumerate(collection.traces):
|
||||
magnitude_db = 20.0 * np.log10(np.maximum(np.abs(trace.s21), 1e-12))
|
||||
label = f"input={trace.combo.input}, output={trace.combo.output}"
|
||||
self._plot.plot(
|
||||
trace.frequency_hz,
|
||||
magnitude_db,
|
||||
pen=pg.mkPen(palette[idx % len(palette)], width=1.6),
|
||||
name=label,
|
||||
)
|
||||
|
||||
self._status.setText(
|
||||
f"Running: collection_id={collection.collection_id}, "
|
||||
f"traces={len(collection.traces)}, ring={self._raw_ring_name}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _read_process_output(process: subprocess.Popen[str]) -> str:
|
||||
"""Collect process stdout/stderr text for diagnostics."""
|
||||
stdout = ""
|
||||
stderr = ""
|
||||
if process.stdout is not None:
|
||||
stdout = process.stdout.read().strip()
|
||||
if process.stderr is not None:
|
||||
stderr = process.stderr.read().strip()
|
||||
|
||||
if stderr and stdout:
|
||||
return f"{stderr}\nstdout:\n{stdout}"
|
||||
if stderr:
|
||||
return stderr
|
||||
if stdout:
|
||||
return f"stdout:\n{stdout}"
|
||||
return "no output"
|
||||
|
||||
def closeEvent(self, event) -> None: # noqa: N802
|
||||
"""Stop subprocess and close reader before window destruction."""
|
||||
self._shutdown()
|
||||
super().closeEvent(event)
|
||||
|
||||
def _shutdown(self) -> None:
|
||||
"""Close reader and terminate subprocess."""
|
||||
if self._raw_reader is not None:
|
||||
self._raw_reader.close()
|
||||
self._raw_reader = None
|
||||
|
||||
process = self._orchestrator_process
|
||||
self._orchestrator_process = None
|
||||
if process is None:
|
||||
return
|
||||
|
||||
if process.poll() is None:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=2.0)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait(timeout=1.0)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""CLI entrypoint for raw orchestrator viewer."""
|
||||
parser = argparse.ArgumentParser(description="Run sweep_orchestrator and plot raw sweep collections")
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
type=Path,
|
||||
default=PROJECT_ROOT / "run_config.json",
|
||||
help="Path to run config JSON",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-reset-ring",
|
||||
action="store_true",
|
||||
help="Do not unlink existing raw ring name before starting",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-radar-prepare",
|
||||
action="store_true",
|
||||
help="Skip Python pre-configuration of native radar before orchestrator start",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strict-radar-prepare",
|
||||
action="store_true",
|
||||
help="Fail immediately if Python pre-configuration cannot run",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
config_path = args.config.resolve()
|
||||
if not config_path.exists():
|
||||
raise FileNotFoundError(f"Config file not found: {config_path}")
|
||||
|
||||
if os.geteuid() == 0 and os.environ.get("SUDO_USER"):
|
||||
print(
|
||||
"Warning: running GUI test via sudo can break Qt DBus/session integration. "
|
||||
"Prefer regular user with USB/GPIO permissions."
|
||||
)
|
||||
|
||||
app = QApplication(sys.argv)
|
||||
viewer = RawOrchestratorViewer(
|
||||
config_path=config_path,
|
||||
reset_ring=not args.no_reset_ring,
|
||||
prepare_radar=not args.skip_radar_prepare,
|
||||
strict_prepare=args.strict_radar_prepare,
|
||||
)
|
||||
viewer.show()
|
||||
|
||||
def _sig_handler(_signum, _frame):
|
||||
"""Close viewer gracefully on process signals."""
|
||||
viewer.close()
|
||||
|
||||
signal.signal(signal.SIGINT, _sig_handler)
|
||||
signal.signal(signal.SIGTERM, _sig_handler)
|
||||
|
||||
return app.exec()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,649 +0,0 @@
|
||||
"""Calibrate the Kamil ADC reference-phase frequency law from live sweeps.
|
||||
|
||||
Captures many raw sweeps from the collector and reports — and optionally writes
|
||||
back — the calibration the runtime uses:
|
||||
|
||||
* ``phase0_rad`` / ``phase1_rad`` — the unwrapped reference phase at the sweep
|
||||
start and stop, taken as the MEDIAN over all captured sweeps. ``freq0_hz`` /
|
||||
``freq1_hz`` (the known sweep endpoints) come from config and are kept as-is.
|
||||
* ``band.points`` — recommended as the median number of usable points landing
|
||||
inside ``[band.start_hz, band.stop_hz]``, so the fixed output grid matches the
|
||||
native density rather than inflating it.
|
||||
|
||||
It also reports how reliably the configured band is covered (sweeps that do not
|
||||
span it are rejected at runtime) and how many points the crop discards.
|
||||
|
||||
The collector is launched with the ``do8_freq_ref`` arguments regardless of what
|
||||
the on-disk config says, so the reference channel is always present. With
|
||||
``--apply`` the config is migrated to that collector/args and the calibrated
|
||||
anchors + recommended point count are written back.
|
||||
|
||||
Run on the Pi, e.g.::
|
||||
|
||||
.venv/bin/python -m python_app.scripts.kamil_adc_calibrate \
|
||||
--config run_config_kamil_adc.pi.json --sweeps 200 --apply
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from contextlib import suppress
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import statistics
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
from python_app.hardware_full.kamil_adc import KamilAdcService, apply_kamil_adc_laser_control
|
||||
from python_app.hardware_full.kamil_adc.processing import (
|
||||
KamilAdcProcessingParams,
|
||||
KamilAdcSweepProcessor,
|
||||
)
|
||||
from python_app.models.run_config_model import RunConfigModel
|
||||
|
||||
logger = logging.getLogger("kamil_adc_calibrate")
|
||||
|
||||
# The collector arguments that enable the DI8 reference overlay (mirrors the
|
||||
# kamil example config / run_do8_freq_ref.sh). Forced on so calibration always
|
||||
# sees the reference channel even if the on-disk config predates it.
|
||||
DO8_FREQ_REF_ARGS = [
|
||||
"profile:phase",
|
||||
"clock:internal",
|
||||
"internal_ref_hz:2000000",
|
||||
"start:di_syn2_rise",
|
||||
"stop:di_syn2_fall",
|
||||
"sample_clock_hz:max",
|
||||
"range:2",
|
||||
"duration_ms:100",
|
||||
"packet_limit:0",
|
||||
"do1_toggle_per_frame",
|
||||
"do1_pair_subtract_avg",
|
||||
"do8_freq_ref",
|
||||
"do8_cycle_period:8",
|
||||
]
|
||||
COLLECTOR_PATH = "build/bin/kamil_adc_collector"
|
||||
|
||||
# Hardware the daemon / a prior collector may be holding, and how to free it.
|
||||
# Mirrors what start.sh does before an interactive launch (stop the daemon, kill
|
||||
# an orphaned collector) so this tool can grab the L-Card E-502 + lasers too.
|
||||
RADAR_SERVICE_NAME = "radar.service"
|
||||
COLLECTOR_PROCESS_PATTERN = "kamil_adc_collector"
|
||||
# The E-502 is not reacquirable the instant it is released; wait before opening.
|
||||
_DEVICE_SETTLE_SECONDS = 8.0
|
||||
|
||||
|
||||
def _release_radar_hardware(*, settle_seconds: float = _DEVICE_SETTLE_SECONDS) -> None:
|
||||
"""Free the L-Card E-502 + lasers before we open our own collector.
|
||||
|
||||
Mirrors ``start.sh`` on an interactive launch: stop the headless
|
||||
``radar.service`` daemon (if active) so it releases the hardware, then kill any
|
||||
orphaned ``kamil_adc_collector`` a prior (e.g. SSH-killed) run left holding the
|
||||
device. Both steps are best-effort and non-fatal — the collector's own
|
||||
open-retry covers the residual settle time — this just removes the usual reason
|
||||
it never frees up. Passwordless ``sudo systemctl stop radar.service`` is
|
||||
provisioned in ``/etc/sudoers.d/radar``.
|
||||
"""
|
||||
freed = False
|
||||
|
||||
systemctl = shutil.which("systemctl")
|
||||
if systemctl is not None:
|
||||
is_active = subprocess.run(
|
||||
[systemctl, "is-active", "--quiet", RADAR_SERVICE_NAME],
|
||||
check=False,
|
||||
).returncode == 0
|
||||
if is_active:
|
||||
logger.info("Stopping %s so it releases the radar hardware...", RADAR_SERVICE_NAME)
|
||||
stopped = subprocess.run(
|
||||
["sudo", systemctl, "stop", RADAR_SERVICE_NAME], check=False
|
||||
).returncode == 0
|
||||
if stopped:
|
||||
freed = True
|
||||
else:
|
||||
logger.warning(
|
||||
"Could not stop %s (need passwordless sudo?); continuing anyway",
|
||||
RADAR_SERVICE_NAME,
|
||||
)
|
||||
|
||||
pkill = shutil.which("pkill")
|
||||
if pkill is not None:
|
||||
# pkill returns 0 when it matched & signalled at least one process.
|
||||
if subprocess.run(
|
||||
[pkill, "-9", "-f", COLLECTOR_PROCESS_PATTERN], check=False
|
||||
).returncode == 0:
|
||||
logger.info("Killed orphaned %s process(es) holding the device", COLLECTOR_PROCESS_PATTERN)
|
||||
freed = True
|
||||
|
||||
if freed:
|
||||
logger.info("Waiting %.0fs for the L-Card E-502 to settle...", settle_seconds)
|
||||
time.sleep(settle_seconds)
|
||||
|
||||
|
||||
def _open_with_retry(service: KamilAdcService, *, attempts: int = 4, delay_s: float = 8.0) -> None:
|
||||
"""Open the collector, retrying the transient E-502 device-busy after a close.
|
||||
|
||||
The L-Card device is not always reacquirable immediately after a previous
|
||||
collector released it; a short backoff lets it settle before the next try.
|
||||
"""
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
service.open()
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001 — device-busy is expected and retried
|
||||
logger.warning("Collector open failed (attempt %d/%d): %s", attempt, attempts, exc)
|
||||
with suppress(Exception):
|
||||
service.close()
|
||||
if attempt < attempts:
|
||||
time.sleep(delay_s)
|
||||
raise RuntimeError(f"Collector did not open after {attempts} attempts")
|
||||
|
||||
|
||||
def _capture_reference_phases(service: KamilAdcService, *, warmup: int, sweeps: int) -> list[np.ndarray]:
|
||||
"""Capture `sweeps` reference-phase arrays after discarding `warmup` sweeps."""
|
||||
for index in range(warmup):
|
||||
service.read_raw_sweep()
|
||||
if index == 0:
|
||||
logger.info("Warming up (%d sweeps) while the sweep settles...", warmup)
|
||||
|
||||
phases: list[np.ndarray] = []
|
||||
for index in range(sweeps):
|
||||
raw = service.read_raw_sweep()
|
||||
if raw.reference.size >= 2:
|
||||
phases.append(np.unwrap(np.angle(raw.reference.astype(np.complex128))))
|
||||
if (index + 1) % 50 == 0:
|
||||
logger.info("Captured %d/%d sweeps", index + 1, sweeps)
|
||||
return phases
|
||||
|
||||
|
||||
def _summarize(values: np.ndarray) -> str:
|
||||
"""Compact min / median / max summary for a 1-D array."""
|
||||
return f"min={np.min(values):.6g} median={np.median(values):.6g} max={np.max(values):.6g}"
|
||||
|
||||
|
||||
def _analyze(phases: list[np.ndarray], config: RunConfigModel) -> dict:
|
||||
"""Derive the calibration and band diagnostics from captured phase arrays."""
|
||||
kamil = config.radar.kamil_adc
|
||||
freq0, freq1 = kamil.phase_calibration.freq0_hz, kamil.phase_calibration.freq1_hz
|
||||
band_start, band_stop = kamil.band.start_hz, kamil.band.stop_hz
|
||||
|
||||
phase0 = float(statistics.median(float(phase[0]) for phase in phases))
|
||||
phase1 = float(statistics.median(float(phase[-1]) for phase in phases))
|
||||
if phase1 == phase0:
|
||||
raise RuntimeError("Degenerate calibration: median start and stop phases are equal")
|
||||
slope = (freq1 - freq0) / (phase1 - phase0)
|
||||
|
||||
total_points = np.array([phase.size for phase in phases], dtype=np.float64)
|
||||
f_starts, f_stops, in_band_counts, covers = [], [], [], []
|
||||
for phase in phases:
|
||||
freqs = freq0 + (phase - phase0) * slope
|
||||
lo, hi = float(np.min(freqs)), float(np.max(freqs))
|
||||
f_starts.append(lo)
|
||||
f_stops.append(hi)
|
||||
in_band_counts.append(int(np.count_nonzero((freqs >= band_start) & (freqs <= band_stop))))
|
||||
covers.append(lo <= band_start and hi >= band_stop)
|
||||
|
||||
in_band = np.array(in_band_counts, dtype=np.float64)
|
||||
f_start = np.array(f_starts)
|
||||
f_stop = np.array(f_stops)
|
||||
recommended_points = int(round(float(np.median(in_band))))
|
||||
# A band that ~95% of sweeps satisfy on each edge: start at the 95th percentile
|
||||
# of per-sweep start frequencies, stop at the 5th percentile of stop frequencies.
|
||||
rec_band_start = float(np.quantile(f_start, 0.95))
|
||||
rec_band_stop = float(np.quantile(f_stop, 0.05))
|
||||
rec_coverage = float(np.mean((f_start <= rec_band_start) & (f_stop >= rec_band_stop)))
|
||||
return {
|
||||
"sweeps": len(phases),
|
||||
"phase0_rad": phase0,
|
||||
"phase1_rad": phase1,
|
||||
"phase_span_rad": phase1 - phase0,
|
||||
"phase0_mad_rad": float(np.median(np.abs([float(p[0]) - phase0 for p in phases]))),
|
||||
"phase1_mad_rad": float(np.median(np.abs([float(p[-1]) - phase1 for p in phases]))),
|
||||
"freq0_hz": freq0,
|
||||
"freq1_hz": freq1,
|
||||
"band_start_hz": band_start,
|
||||
"band_stop_hz": band_stop,
|
||||
"total_points_median": float(np.median(total_points)),
|
||||
"in_band_points_median": float(np.median(in_band)),
|
||||
"recommended_points": recommended_points,
|
||||
"cropped_fraction": 1.0 - float(np.median(in_band)) / float(np.median(total_points)),
|
||||
"coverage_fraction": float(np.mean(covers)),
|
||||
"lower_ok_fraction": float(np.mean(f_start <= band_start)),
|
||||
"upper_ok_fraction": float(np.mean(f_stop >= band_stop)),
|
||||
"rec_band_start_hz": rec_band_start,
|
||||
"rec_band_stop_hz": rec_band_stop,
|
||||
"rec_coverage_fraction": rec_coverage,
|
||||
"f_start": f_start,
|
||||
"f_stop": f_stop,
|
||||
}
|
||||
|
||||
|
||||
def _report(result: dict) -> None:
|
||||
"""Print a human-readable calibration report."""
|
||||
print("\n" + "=" * 72)
|
||||
print(f"Kamil ADC calibration over {result['sweeps']} sweeps")
|
||||
print("=" * 72)
|
||||
print(
|
||||
f"phase0_rad = {result['phase0_rad']:.6f} (median start phase, MAD "
|
||||
f"{result['phase0_mad_rad']:.4f}) -> {result['freq0_hz'] / 1e9:.4f} GHz"
|
||||
)
|
||||
print(
|
||||
f"phase1_rad = {result['phase1_rad']:.6f} (median stop phase, MAD "
|
||||
f"{result['phase1_mad_rad']:.4f}) -> {result['freq1_hz'] / 1e9:.4f} GHz"
|
||||
)
|
||||
print(f"phase span = {result['phase_span_rad']:.4f} rad")
|
||||
print(
|
||||
f"points/sweep: total median={result['total_points_median']:.0f}, "
|
||||
f"in-band median={result['in_band_points_median']:.0f}"
|
||||
)
|
||||
print(f"recommended band.points = {result['recommended_points']}")
|
||||
print(
|
||||
f"band [{result['band_start_hz'] / 1e9:.3f}, {result['band_stop_hz'] / 1e9:.3f}] GHz: "
|
||||
f"covered by {result['coverage_fraction'] * 100:.1f}% of sweeps "
|
||||
f"(start<=lo: {result['lower_ok_fraction'] * 100:.1f}%, stop>=hi: {result['upper_ok_fraction'] * 100:.1f}%), "
|
||||
f"{result['cropped_fraction'] * 100:.1f}% of points cropped"
|
||||
)
|
||||
print(f"per-sweep start freq (GHz): {_summarize(result['f_start'] / 1e9)}")
|
||||
print(f"per-sweep stop freq (GHz): {_summarize(result['f_stop'] / 1e9)}")
|
||||
print(
|
||||
f"suggested band for ~95%/edge: [{result['rec_band_start_hz'] / 1e9:.3f}, "
|
||||
f"{result['rec_band_stop_hz'] / 1e9:.3f}] GHz -> covers "
|
||||
f"{result['rec_coverage_fraction'] * 100:.1f}% of sweeps"
|
||||
)
|
||||
if result["coverage_fraction"] < 0.95:
|
||||
print(
|
||||
"WARNING: many sweeps do not cover the configured band and would be rejected; "
|
||||
"consider the suggested band above (or longer laser settling if sweeps are partial)."
|
||||
)
|
||||
print("=" * 72 + "\n")
|
||||
|
||||
|
||||
_TWO_PI = 2.0 * np.pi
|
||||
|
||||
|
||||
def _capture_raw_sweeps(
|
||||
service: KamilAdcService, *, warmup: int, sweeps: int
|
||||
) -> list[tuple[np.ndarray, np.ndarray, np.ndarray]]:
|
||||
"""Capture full ``(steps, main, reference)`` arrays after discarding `warmup`.
|
||||
|
||||
Unlike :func:`_capture_reference_phases`, this keeps the complete complex main
|
||||
and reference samples (and their step indices) so the diagnosis can study the
|
||||
raw phase, amplitude, and the pass-through behaviour — not just the endpoints.
|
||||
"""
|
||||
for index in range(warmup):
|
||||
service.read_raw_sweep()
|
||||
if index == 0:
|
||||
logger.info("Warming up (%d sweeps) while the sweep settles...", warmup)
|
||||
|
||||
captured: list[tuple[np.ndarray, np.ndarray, np.ndarray]] = []
|
||||
for index in range(sweeps):
|
||||
raw = service.read_raw_sweep()
|
||||
if raw.reference.size >= 2 and raw.main.size == raw.reference.size:
|
||||
captured.append(
|
||||
(
|
||||
np.asarray(raw.steps),
|
||||
raw.main.astype(np.complex128),
|
||||
raw.reference.astype(np.complex128),
|
||||
)
|
||||
)
|
||||
if (index + 1) % 50 == 0:
|
||||
logger.info("Captured %d/%d sweeps", index + 1, sweeps)
|
||||
return captured
|
||||
|
||||
|
||||
def _diagnose(
|
||||
sweeps: list[tuple[np.ndarray, np.ndarray, np.ndarray]], config: RunConfigModel
|
||||
) -> dict:
|
||||
"""Characterise how the reference phase / frequency axis moves across sweeps.
|
||||
|
||||
The reference arm has a fixed electrical delay, so each sweep's unwrapped phase
|
||||
is a straight ramp; the sweep start frequency wandering shifts that ramp
|
||||
vertically (the *physical float*). When ``np.angle(ref[0])`` crosses the +/-pi
|
||||
cut, ``np.unwrap`` re-anchors the whole ramp one 2*pi turn away (the *spurious
|
||||
branch wrap*) — a ~``2*pi*slope`` jump of the entire frequency axis that wrecks
|
||||
the band mapping for that sweep.
|
||||
|
||||
This separates the two: it reconstructs the anchor phase per sweep, unwraps it
|
||||
*across* sweeps (the candidate fix), and reports how that removes discrete 2*pi
|
||||
steps while leaving the slow physical float intact. It also runs the live
|
||||
pass-through (``KamilAdcSweepProcessor``) to flag which sweeps actually come out
|
||||
distorted, and correlates those with the detected branch wraps.
|
||||
"""
|
||||
kamil = config.radar.kamil_adc
|
||||
cal = kamil.phase_calibration
|
||||
phase0, phase1 = float(cal.phase0_rad), float(cal.phase1_rad)
|
||||
freq0, freq1 = float(cal.freq0_hz), float(cal.freq1_hz)
|
||||
slope = (freq1 - freq0) / (phase1 - phase0) # Hz per rad
|
||||
band_start, band_stop = float(kamil.band.start_hz), float(kamil.band.stop_hz)
|
||||
|
||||
n = len(sweeps)
|
||||
anchor_phase = np.empty(n) # unwrap(angle(ref))[0] == angle(ref[0]) in (-pi, pi]
|
||||
stop_phase = np.empty(n) # unwrap(angle(ref))[-1]
|
||||
span_rad = np.empty(n) # stop_phase - anchor_phase (fixed-delay invariant)
|
||||
n_points = np.empty(n, dtype=np.int64)
|
||||
f_lo = np.empty(n) # min reconstructed frequency (config calibration)
|
||||
f_hi = np.empty(n) # max reconstructed frequency
|
||||
ref_amp_med = np.empty(n)
|
||||
|
||||
for i, (_steps, _main, reference) in enumerate(sweeps):
|
||||
phase = np.unwrap(np.angle(reference))
|
||||
anchor_phase[i] = phase[0]
|
||||
stop_phase[i] = phase[-1]
|
||||
span_rad[i] = phase[-1] - phase[0]
|
||||
n_points[i] = reference.size
|
||||
freqs = freq0 + (phase - phase0) * slope
|
||||
f_lo[i] = float(np.min(freqs))
|
||||
f_hi[i] = float(np.max(freqs))
|
||||
ref_amp_med[i] = float(np.median(np.abs(reference)))
|
||||
|
||||
# --- Candidate fix: unwrap the anchor phase ACROSS the sweep-time axis ------
|
||||
# The physical float moves the anchor smoothly; a branch wrap injects a +/-2*pi
|
||||
# step. Unwrapping along sweep index removes the discrete steps and keeps the
|
||||
# slow float. branch_turns is the integer turns each sweep was wrapped by.
|
||||
anchor_unwrapped = np.unwrap(anchor_phase)
|
||||
branch_turns = np.round((anchor_unwrapped - anchor_phase) / _TWO_PI).astype(np.int64)
|
||||
axis_shift = branch_turns * _TWO_PI * slope # Hz the whole axis was displaced
|
||||
f_lo_corr = f_lo + axis_shift
|
||||
f_hi_corr = f_hi + axis_shift
|
||||
|
||||
# Branch wraps: consecutive anchor steps above pi are spurious 2*pi jumps.
|
||||
anchor_step = np.diff(anchor_phase)
|
||||
wrap_indices = np.nonzero(np.abs(anchor_step) > np.pi)[0] + 1 # sweep index after wrap
|
||||
|
||||
covers_raw = (f_lo <= band_start) & (f_hi >= band_stop)
|
||||
covers_corr = (f_lo_corr <= band_start) & (f_hi_corr >= band_stop)
|
||||
|
||||
passthrough = _passthrough_flatness(sweeps, kamil)
|
||||
|
||||
return {
|
||||
"sweeps": n,
|
||||
"slope_hz_per_rad": slope,
|
||||
"axis_2pi_shift_hz": _TWO_PI * slope,
|
||||
"band_start_hz": band_start,
|
||||
"band_stop_hz": band_stop,
|
||||
"anchor_phase": anchor_phase,
|
||||
"anchor_unwrapped": anchor_unwrapped,
|
||||
"branch_turns": branch_turns,
|
||||
"n_branch_wraps": int(wrap_indices.size),
|
||||
"wrap_indices": wrap_indices,
|
||||
"span_rad": span_rad,
|
||||
"n_points": n_points,
|
||||
"f_lo": f_lo,
|
||||
"f_hi": f_hi,
|
||||
"f_lo_corr": f_lo_corr,
|
||||
"f_hi_corr": f_hi_corr,
|
||||
"axis_shift_hz": axis_shift,
|
||||
"ref_amp_med": ref_amp_med,
|
||||
"coverage_raw": float(np.mean(covers_raw)),
|
||||
"coverage_corr": float(np.mean(covers_corr)),
|
||||
# Largest sweep-to-sweep step of the *unwrapped* anchor: the fix is only
|
||||
# valid if this stays below pi (otherwise the cross-sweep unwrap is itself
|
||||
# ambiguous). Reported so we can confirm the float really is slow.
|
||||
"max_unwrapped_anchor_step_rad": float(np.max(np.abs(np.diff(anchor_unwrapped))))
|
||||
if n > 1
|
||||
else 0.0,
|
||||
**passthrough,
|
||||
}
|
||||
|
||||
|
||||
def _passthrough_flatness(
|
||||
sweeps: list[tuple[np.ndarray, np.ndarray, np.ndarray]], kamil
|
||||
) -> dict:
|
||||
"""Run the live processor and measure how flat each pass-through trace is.
|
||||
|
||||
Builds the same :class:`KamilAdcSweepProcessor` the runtime uses, processes
|
||||
every sweep onto the fixed grid, then measures each trace's deviation from a
|
||||
robust (median) reference trace. A branch-wrapped sweep lands on a frequency
|
||||
axis offset by ~one 2*pi turn, so after resampling it beats against the
|
||||
reference — showing up as a large phase-residual std. This is the observable
|
||||
symptom ("constant offset + oscillations") tied back to the raw-phase analysis.
|
||||
"""
|
||||
processor = KamilAdcSweepProcessor(KamilAdcProcessingParams.from_kamil_model(kamil))
|
||||
traces = [processor.process(main, reference) for _steps, main, reference in sweeps]
|
||||
passed = np.array([trace is not None for trace in traces])
|
||||
n = len(traces)
|
||||
phase_residual_std = np.full(n, np.nan)
|
||||
amp_residual_std = np.full(n, np.nan)
|
||||
|
||||
stack = np.array([trace for trace in traces if trace is not None], dtype=np.complex128)
|
||||
if stack.shape[0] >= 3:
|
||||
# Robust per-grid-point reference: median of real/imag over passing sweeps.
|
||||
reference_trace = np.median(stack.real, axis=0) + 1j * np.median(stack.imag, axis=0)
|
||||
safe_reference = np.where(np.abs(reference_trace) < 1e-12, 1.0 + 0j, reference_trace)
|
||||
passed_index = 0
|
||||
for i, trace in enumerate(traces):
|
||||
if trace is None:
|
||||
continue
|
||||
ratio = trace.astype(np.complex128) / safe_reference
|
||||
phase_residual_std[i] = float(np.std(np.angle(ratio)))
|
||||
amp_residual_std[i] = float(np.std(np.abs(ratio)))
|
||||
passed_index += 1
|
||||
|
||||
return {
|
||||
"passthrough_passed_fraction": float(np.mean(passed)),
|
||||
"passthrough_phase_residual_std": phase_residual_std,
|
||||
"passthrough_amp_residual_std": amp_residual_std,
|
||||
}
|
||||
|
||||
|
||||
def _report_diagnosis(result: dict) -> None:
|
||||
"""Print the detailed reference-phase / frequency-axis diagnosis."""
|
||||
n = result["sweeps"]
|
||||
print("\n" + "=" * 72)
|
||||
print(f"Kamil ADC reference-phase DIAGNOSIS over {n} sweeps")
|
||||
print("=" * 72)
|
||||
print(
|
||||
f"calibration slope = {result['slope_hz_per_rad'] / 1e6:.4f} MHz/rad\n"
|
||||
f"one 2*pi branch wrap = {result['axis_2pi_shift_hz'] / 1e6:.2f} MHz axis shift"
|
||||
)
|
||||
print("-" * 72)
|
||||
print(
|
||||
f"anchor phase angle(ref[0]): {_summarize(result['anchor_phase'])} rad\n"
|
||||
f"phase span (stop - start) : {_summarize(result['span_rad'])} rad "
|
||||
f"(std {np.std(result['span_rad']):.4f} -> fixed-delay {'OK' if np.std(result['span_rad']) < 1.0 else 'SUSPECT'})\n"
|
||||
f"points per sweep : {_summarize(result['n_points'].astype(float))}"
|
||||
)
|
||||
print("-" * 72)
|
||||
print(
|
||||
f"branch wraps detected : {result['n_branch_wraps']} "
|
||||
f"(at sweep indices {result['wrap_indices'].tolist()})"
|
||||
)
|
||||
print(f"branch turns range : {_summarize(result['branch_turns'].astype(float))} turns")
|
||||
print(
|
||||
f"max sweep-to-sweep step of UNWRAPPED anchor = "
|
||||
f"{result['max_unwrapped_anchor_step_rad']:.4f} rad "
|
||||
f"({'< pi: cross-sweep unwrap is unambiguous' if result['max_unwrapped_anchor_step_rad'] < np.pi else '>= pi: AMBIGUOUS, fix may misstep'})"
|
||||
)
|
||||
print("-" * 72)
|
||||
print("reconstructed band edges (GHz), config calibration:")
|
||||
print(f" raw start {_summarize(result['f_lo'] / 1e9)} stop {_summarize(result['f_hi'] / 1e9)}")
|
||||
print(f" corrected start {_summarize(result['f_lo_corr'] / 1e9)} stop {_summarize(result['f_hi_corr'] / 1e9)}")
|
||||
print(
|
||||
f"band [{result['band_start_hz'] / 1e9:.3f}, {result['band_stop_hz'] / 1e9:.3f}] GHz covered: "
|
||||
f"raw {result['coverage_raw'] * 100:.1f}% -> corrected {result['coverage_corr'] * 100:.1f}%"
|
||||
)
|
||||
print("-" * 72)
|
||||
phase_res = result["passthrough_phase_residual_std"]
|
||||
finite = phase_res[np.isfinite(phase_res)]
|
||||
print(f"pass-through: {result['passthrough_passed_fraction'] * 100:.1f}% of sweeps covered the band")
|
||||
if finite.size:
|
||||
print(f" phase-residual std vs median trace: {_summarize(finite)} rad")
|
||||
# Sweeps whose pass-through trace is far from flat (the visible glitches).
|
||||
bad = np.nonzero(phase_res > (np.median(finite) + 5.0 * (np.std(finite) + 1e-9)))[0]
|
||||
print(f" distorted sweeps (>5 sigma residual): {bad.tolist()}")
|
||||
print(f" do they coincide with branch wraps? wraps at {result['wrap_indices'].tolist()}")
|
||||
print("-" * 72)
|
||||
print(
|
||||
"HOW TO READ THIS:\n"
|
||||
" * phase span std small -> fixed reference delay confirmed (model holds).\n"
|
||||
" * 'branch wraps' are the suspected glitches: angle(ref[0]) crossing +/-pi.\n"
|
||||
" * If the UNWRAPPED-anchor max step stays < pi, the sweep-to-sweep float is\n"
|
||||
" slow enough that a CROSS-SWEEP unwrap can separate the real drift from the\n"
|
||||
" spurious 2*pi wrap. That is the candidate fix.\n"
|
||||
" * Expect: 'corrected' band edges move CONTINUOUSLY (no ~157 MHz steps) while\n"
|
||||
" 'raw' jumps by one wrap at each detected wrap index, and the distorted\n"
|
||||
" pass-through sweeps line up with those wrap indices."
|
||||
)
|
||||
print("=" * 72 + "\n")
|
||||
|
||||
|
||||
def _dump_diagnosis(
|
||||
path: Path,
|
||||
result: dict,
|
||||
sweeps: list[tuple[np.ndarray, np.ndarray, np.ndarray]],
|
||||
) -> None:
|
||||
"""Save per-sweep metrics and the full raw sweeps to an ``.npz`` for offline study."""
|
||||
arrays = {key: value for key, value in result.items() if isinstance(value, np.ndarray)}
|
||||
arrays["raw_steps"] = np.array([steps for steps, _m, _r in sweeps], dtype=object)
|
||||
arrays["raw_main"] = np.array([main for _s, main, _r in sweeps], dtype=object)
|
||||
arrays["raw_reference"] = np.array([reference for _s, _m, reference in sweeps], dtype=object)
|
||||
np.savez(path, **arrays)
|
||||
print(f"Saved diagnosis data ({len(sweeps)} sweeps) to {path}")
|
||||
|
||||
|
||||
def _plot_diagnosis(path: Path, result: dict) -> None:
|
||||
"""Render the key diagnostic plots to a PNG (no-op if matplotlib is missing)."""
|
||||
try:
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
except Exception as exc: # noqa: BLE001 — plotting is optional
|
||||
logger.warning("Plotting skipped (matplotlib unavailable: %s)", exc)
|
||||
return
|
||||
|
||||
sweep_index = np.arange(result["sweeps"])
|
||||
fig, axes = plt.subplots(3, 1, figsize=(11, 12), sharex=True)
|
||||
|
||||
axes[0].plot(sweep_index, result["anchor_phase"], ".", label="angle(ref[0]) raw (wrapped)")
|
||||
axes[0].plot(sweep_index, result["anchor_unwrapped"], "-", label="unwrapped across sweeps (fix)")
|
||||
for wrap in result["wrap_indices"]:
|
||||
axes[0].axvline(wrap, color="r", alpha=0.3)
|
||||
axes[0].set_ylabel("anchor phase [rad]")
|
||||
axes[0].legend(loc="best")
|
||||
axes[0].set_title("Reference anchor phase: raw wraps vs cross-sweep unwrap")
|
||||
|
||||
axes[1].plot(sweep_index, result["f_lo"] / 1e9, ".", label="start raw")
|
||||
axes[1].plot(sweep_index, result["f_lo_corr"] / 1e9, "-", label="start corrected")
|
||||
axes[1].plot(sweep_index, result["f_hi"] / 1e9, ".", label="stop raw")
|
||||
axes[1].plot(sweep_index, result["f_hi_corr"] / 1e9, "-", label="stop corrected")
|
||||
axes[1].axhline(result["band_start_hz"] / 1e9, color="k", ls="--", alpha=0.5)
|
||||
axes[1].axhline(result["band_stop_hz"] / 1e9, color="k", ls="--", alpha=0.5)
|
||||
axes[1].set_ylabel("reconstructed band edge [GHz]")
|
||||
axes[1].legend(loc="best")
|
||||
|
||||
axes[2].plot(sweep_index, result["passthrough_phase_residual_std"], ".", label="phase residual std")
|
||||
for wrap in result["wrap_indices"]:
|
||||
axes[2].axvline(wrap, color="r", alpha=0.3, label="_branch wrap")
|
||||
axes[2].set_ylabel("pass-through residual [rad]")
|
||||
axes[2].set_xlabel("sweep index")
|
||||
axes[2].legend(loc="best")
|
||||
|
||||
fig.tight_layout()
|
||||
fig.savefig(path, dpi=110)
|
||||
plt.close(fig)
|
||||
print(f"Saved diagnosis plot to {path}")
|
||||
|
||||
|
||||
def _apply(config_path: Path, result: dict) -> None:
|
||||
"""Write the migrated collector args + calibrated anchors + points to the config."""
|
||||
payload = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
kamil = payload.setdefault("radar", {}).setdefault("kamil_adc", {})
|
||||
kamil["executable_path"] = COLLECTOR_PATH
|
||||
kamil["args"] = list(DO8_FREQ_REF_ARGS)
|
||||
kamil["phase_calibration"] = {
|
||||
"phase0_rad": result["phase0_rad"],
|
||||
"freq0_hz": result["freq0_hz"],
|
||||
"phase1_rad": result["phase1_rad"],
|
||||
"freq1_hz": result["freq1_hz"],
|
||||
}
|
||||
kamil["band"] = {
|
||||
"start_hz": result["band_start_hz"],
|
||||
"stop_hz": result["band_stop_hz"],
|
||||
"points": result["recommended_points"],
|
||||
}
|
||||
config_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"Applied calibration to {config_path}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Calibrate Kamil ADC reference phase -> frequency")
|
||||
parser.add_argument("--config", required=True, type=Path, help="Path to the kamil_adc run config")
|
||||
parser.add_argument("--sweeps", type=int, default=200, help="Sweeps to average (default 200)")
|
||||
parser.add_argument("--warmup", type=int, default=10, help="Sweeps to discard first (default 10)")
|
||||
parser.add_argument("--apply", action="store_true", help="Write the calibration back to --config")
|
||||
parser.add_argument("--no-laser", action="store_true", help="Skip laser setup (already running)")
|
||||
parser.add_argument(
|
||||
"--no-release",
|
||||
action="store_true",
|
||||
help="Do not stop radar.service / kill orphaned collectors before opening",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--diagnose",
|
||||
action="store_true",
|
||||
help="Investigate reference-phase branch wraps instead of calibrating",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dump", type=Path, default=None, help="(--diagnose) save raw sweeps + metrics to this .npz"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--plot", type=Path, default=None, help="(--diagnose) render diagnostic plots to this .png"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
|
||||
config = RunConfigModel.load_from_path(args.config)
|
||||
if not config.is_kamil_adc:
|
||||
raise SystemExit("Config is not a kamil_adc profile")
|
||||
|
||||
# Force the reference-producing collector regardless of the on-disk config.
|
||||
config.radar.kamil_adc.executable_path = COLLECTOR_PATH
|
||||
config.radar.kamil_adc.args = list(DO8_FREQ_REF_ARGS)
|
||||
|
||||
# Free the device the headless daemon / an orphaned collector may be holding,
|
||||
# so opening our own collector does not fail with E-502 device-busy.
|
||||
if not args.no_release:
|
||||
_release_radar_hardware()
|
||||
|
||||
if not args.no_laser:
|
||||
logger.info("Applying laser control...")
|
||||
apply_kamil_adc_laser_control(config)
|
||||
|
||||
service = KamilAdcService(config)
|
||||
logger.info("Opening collector: %s", " ".join(service.command))
|
||||
_open_with_retry(service)
|
||||
|
||||
if args.diagnose:
|
||||
try:
|
||||
sweeps = _capture_raw_sweeps(service, warmup=args.warmup, sweeps=args.sweeps)
|
||||
finally:
|
||||
service.close()
|
||||
if len(sweeps) < max(2, args.sweeps // 2):
|
||||
raise SystemExit(f"Only {len(sweeps)} usable sweeps captured; check the reference signal")
|
||||
result = _diagnose(sweeps, config)
|
||||
_report_diagnosis(result)
|
||||
if args.dump is not None:
|
||||
_dump_diagnosis(args.dump, result, sweeps)
|
||||
if args.plot is not None:
|
||||
_plot_diagnosis(args.plot, result)
|
||||
return 0
|
||||
|
||||
try:
|
||||
phases = _capture_reference_phases(service, warmup=args.warmup, sweeps=args.sweeps)
|
||||
finally:
|
||||
service.close()
|
||||
|
||||
if len(phases) < max(2, args.sweeps // 2):
|
||||
raise SystemExit(f"Only {len(phases)} usable sweeps captured; check the reference signal")
|
||||
|
||||
result = _analyze(phases, config)
|
||||
_report(result)
|
||||
if args.apply:
|
||||
_apply(args.config, result)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -2,36 +2,21 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from contextlib import ExitStack, suppress
|
||||
from contextlib import ExitStack
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
from python_app.hardware_full.kamil_adc import KamilAdcService
|
||||
from python_app.hardware_full.switch_service import SwitchService
|
||||
from python_app.hardware_full.switch_service import SwitchService, select_and_settle
|
||||
from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData
|
||||
from python_app.models.run_config_model import RunConfigModel
|
||||
from python_app.orchestration.shm import ShmRingWriter
|
||||
from python_app.storage.npz.serialize import RAW_MAGIC, serialize_trace_collection
|
||||
from python_app.models.run_config_schema import RunConfigModel
|
||||
from python_app.scripts.producer_common import RawPublisher, close_devices, load_producer_config, retry_open
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The producer waits for the Kamil ADC collector forever: a device/collector that
|
||||
# is absent at boot or disappears mid-run must never kill the producer, only make
|
||||
# it wait. Reconnect uses a capped exponential backoff so a long absence does not
|
||||
# busy-spin, and every wait is interruptible by SIGINT/SIGTERM (stop_requested) for
|
||||
# a prompt clean exit. Mirrors matrix_raw_producer._open_radar_with_retry.
|
||||
_OPEN_RETRY_MIN_S = 1.0
|
||||
_OPEN_RETRY_MAX_S = 10.0
|
||||
# Throttle open-failure logging during a long wait so a permanently absent device
|
||||
# does not flood the process log: log the first failure, then every Nth attempt.
|
||||
_OPEN_RETRY_LOG_EVERY = 30
|
||||
|
||||
|
||||
def _open_radar_with_retry(
|
||||
config: RunConfigModel,
|
||||
@@ -40,33 +25,10 @@ def _open_radar_with_retry(
|
||||
output_switch: SwitchService | None,
|
||||
stop_requested: threading.Event,
|
||||
) -> bool:
|
||||
"""Open+configure the radar and both switches, retrying forever until stop.
|
||||
"""Reopen the ADC and Python-owned switches, cleaning partial opens before retry."""
|
||||
close_devices(input_switch, output_switch, radar)
|
||||
|
||||
Used for both the initial open and every in-loop reconnect, so a collector
|
||||
that is absent at boot or disappears mid-run never kills the producer — it just
|
||||
waits. Any partially-opened components are closed before each retry so a
|
||||
relaunched collector starts clean. Returns ``True`` once everything is open, or
|
||||
``False`` if a stop was requested before the device became available. Backoff is
|
||||
capped and every wait is interruptible by SIGTERM.
|
||||
|
||||
``input_switch``/``output_switch`` are ``None`` in switch-aware mode, where the
|
||||
collector owns the GPIO lines and the producer must not open them.
|
||||
"""
|
||||
# Tear down any prior open first: open()/switch.open() are idempotent no-ops
|
||||
# while still "open", so a mid-run reconnect must close them to force a fresh
|
||||
# collector relaunch and TTY re-attach.
|
||||
if input_switch is not None:
|
||||
with suppress(Exception):
|
||||
input_switch.close()
|
||||
if output_switch is not None:
|
||||
with suppress(Exception):
|
||||
output_switch.close()
|
||||
with suppress(Exception):
|
||||
radar.close()
|
||||
|
||||
attempt = 0
|
||||
delay = _OPEN_RETRY_MIN_S
|
||||
while not stop_requested.is_set():
|
||||
def attempt() -> KamilAdcService:
|
||||
try:
|
||||
radar.open(stop_event=stop_requested)
|
||||
radar.configure(config.radar.sweep)
|
||||
@@ -74,82 +36,28 @@ def _open_radar_with_retry(
|
||||
output_switch.open()
|
||||
if input_switch is not None:
|
||||
input_switch.open()
|
||||
except Exception as exc: # noqa: BLE001 — waiting for the device is the point
|
||||
# Drop any partial open (collector process, TTY reader, switches)
|
||||
# before the next attempt so the relaunch starts from a clean state.
|
||||
if input_switch is not None:
|
||||
with suppress(Exception):
|
||||
input_switch.close()
|
||||
if output_switch is not None:
|
||||
with suppress(Exception):
|
||||
output_switch.close()
|
||||
with suppress(Exception):
|
||||
radar.close()
|
||||
attempt += 1
|
||||
if attempt == 1 or attempt % _OPEN_RETRY_LOG_EVERY == 0:
|
||||
logger.warning(
|
||||
"Kamil ADC not available (attempt %d); retrying every up to %.0fs "
|
||||
"until the device is present: %s",
|
||||
attempt,
|
||||
_OPEN_RETRY_MAX_S,
|
||||
exc,
|
||||
)
|
||||
if stop_requested.wait(delay):
|
||||
return False
|
||||
delay = min(delay * 2.0, _OPEN_RETRY_MAX_S)
|
||||
continue
|
||||
except Exception:
|
||||
close_devices(input_switch, output_switch, radar)
|
||||
raise
|
||||
return radar
|
||||
|
||||
if attempt > 0:
|
||||
logger.info("Kamil ADC opened after %d attempt(s).", attempt + 1)
|
||||
return True
|
||||
|
||||
logger.debug("Stop requested before the Kamil ADC became available.")
|
||||
return False
|
||||
return retry_open(attempt, stop_requested, "Kamil ADC") is not None
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Run producer process until config or signal requests exit."""
|
||||
parser = argparse.ArgumentParser(description="Publish Kamil ADC raw sweeps to SHM rings")
|
||||
parser.add_argument("--config", required=True, type=Path, help="Path to run_config.json")
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
|
||||
stop_requested = threading.Event()
|
||||
|
||||
def request_stop(_signum: int, _frame: object) -> None:
|
||||
stop_requested.set()
|
||||
|
||||
signal.signal(signal.SIGINT, request_stop)
|
||||
signal.signal(signal.SIGTERM, request_stop)
|
||||
|
||||
config = RunConfigModel.load_from_path(args.config)
|
||||
config_path, config, stop_requested = load_producer_config("Publish Kamil ADC raw sweeps to SHM rings")
|
||||
if not config.is_kamil_adc:
|
||||
raise RuntimeError("kamil_adc_raw_producer requires radar.model='kamil_adc'")
|
||||
config.ensure_combos()
|
||||
logger.info(
|
||||
"Kamil ADC raw producer starting: config=%s, combos=%d, continuous=%s",
|
||||
args.config, len(config.combos), config.runtime.continuous,
|
||||
config_path, len(config.combos), config.runtime.continuous,
|
||||
)
|
||||
|
||||
radar = input_switch = output_switch = None
|
||||
cleanup = ExitStack()
|
||||
try:
|
||||
raw_writer = ShmRingWriter(
|
||||
config.rings.raw.name,
|
||||
config.rings.raw.capacity,
|
||||
config.rings.raw.slot_size_bytes,
|
||||
)
|
||||
cleanup.callback(raw_writer.close)
|
||||
raw_tap_writer = ShmRingWriter(
|
||||
config.rings.raw_tap.name,
|
||||
config.rings.raw_tap.capacity,
|
||||
config.rings.raw_tap.slot_size_bytes,
|
||||
)
|
||||
cleanup.callback(raw_tap_writer.close)
|
||||
logger.debug(
|
||||
"Opened SHM ring writers: raw=%s, raw_tap=%s",
|
||||
config.rings.raw.name, config.rings.raw_tap.name,
|
||||
)
|
||||
publisher = RawPublisher(config.rings, cleanup)
|
||||
# Switch-aware mode: with native switches the collector drives the RF switches
|
||||
# itself, in the hardware gap between sweeps, and tags each sweep with its combo
|
||||
# — no sweep lost at a switch boundary. The producer then only reads tagged
|
||||
@@ -162,7 +70,7 @@ def main() -> int:
|
||||
)
|
||||
radar = KamilAdcService(
|
||||
config,
|
||||
switch_config_path=str(args.config) if collector_driven else None,
|
||||
switch_config_path=str(config_path) if collector_driven else None,
|
||||
)
|
||||
input_switch = None if collector_driven else SwitchService.from_model(config.input_switch)
|
||||
output_switch = None if collector_driven else SwitchService.from_model(config.output_switch)
|
||||
@@ -175,7 +83,6 @@ def main() -> int:
|
||||
return 0 # asked to stop before a device became available
|
||||
|
||||
collection_id = 1
|
||||
publish_failures = 0
|
||||
while not stop_requested.is_set():
|
||||
collection_start = time.monotonic()
|
||||
capture_start_ns = time.monotonic_ns()
|
||||
@@ -193,9 +100,8 @@ def main() -> int:
|
||||
else:
|
||||
if input_switch is None or output_switch is None:
|
||||
raise RuntimeError("Producer-driven switches are not initialized")
|
||||
output_switch.switch_to(combo.output)
|
||||
input_switch.switch_to(combo.input)
|
||||
if config.runtime.settling_ms > 0 and stop_requested.wait(config.runtime.settling_ms / 1000.0):
|
||||
if not select_and_settle(output_switch, input_switch, combo.output, combo.input,
|
||||
config.runtime.settling_ms, wait=stop_requested.wait):
|
||||
break
|
||||
# Stamped after switching and settling so the window covers
|
||||
# the sweep alone, not the dead time before it.
|
||||
@@ -205,7 +111,6 @@ def main() -> int:
|
||||
TraceData(
|
||||
combo=ComboKey(input=combo.input, output=combo.output),
|
||||
frequency_hz=np.asarray(sweep.x, dtype=np.float32),
|
||||
s11=np.asarray(sweep.trace("s11"), dtype=np.complex64),
|
||||
s21=np.asarray(sweep.trace("s21"), dtype=np.complex64),
|
||||
capture_start_ns=sweep_start_ns,
|
||||
capture_end_ns=time.monotonic_ns(),
|
||||
@@ -244,18 +149,7 @@ def main() -> int:
|
||||
capture_start_ns=capture_start_ns,
|
||||
capture_end_ns=time.monotonic_ns(),
|
||||
)
|
||||
payload = serialize_trace_collection(collection, RAW_MAGIC)
|
||||
if raw_writer.push(payload):
|
||||
raw_tap_writer.push(payload) # best-effort GUI tap; never fatal
|
||||
else:
|
||||
# Oversized payload vs the ring slot is a persistent config error, not
|
||||
# a device fault: log (throttled) and skip rather than killing the producer.
|
||||
publish_failures += 1
|
||||
if publish_failures == 1 or publish_failures % 100 == 0:
|
||||
logger.error(
|
||||
"Raw payload %d B exceeds ring slot %d B; dropping collection %d (drops=%d)",
|
||||
len(payload), raw_writer.slot_size_bytes, collection_id, publish_failures,
|
||||
)
|
||||
publisher.publish(collection)
|
||||
|
||||
if not config.runtime.continuous:
|
||||
break
|
||||
@@ -264,15 +158,7 @@ def main() -> int:
|
||||
logger.info("Kamil ADC collection %d acquired in %.3f s", collection_id, collection_duration_s)
|
||||
collection_id += 1
|
||||
finally:
|
||||
if output_switch is not None:
|
||||
with suppress(Exception):
|
||||
output_switch.close()
|
||||
if input_switch is not None:
|
||||
with suppress(Exception):
|
||||
input_switch.close()
|
||||
if radar is not None:
|
||||
with suppress(Exception):
|
||||
radar.close()
|
||||
close_devices(output_switch, input_switch, radar)
|
||||
cleanup.close()
|
||||
|
||||
logger.info("Kamil ADC raw producer stopped")
|
||||
|
||||
@@ -36,7 +36,7 @@ from python_app.hardware_full.laser_control.monitoring import (
|
||||
ReadingWriter,
|
||||
resolve_period_s,
|
||||
)
|
||||
from python_app.models.run_config_model import RunConfigModel
|
||||
from python_app.models.run_config_schema import RunConfigModel
|
||||
|
||||
logger = logging.getLogger("laser_temp_monitor")
|
||||
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Standalone LibreVNA generator sweep driven by a local Python config."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from python_app.generator_sweep.config import resolve_generator_sweep_config
|
||||
from python_app.scripts.librevna_generator_sweep_config import CONFIG
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Load local generator config and execute the sweep."""
|
||||
resolved = resolve_generator_sweep_config(CONFIG)
|
||||
logging.basicConfig(
|
||||
level=resolved.log_level,
|
||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||||
)
|
||||
|
||||
from python_app.generator_sweep.runner import GeneratorSweepRunner
|
||||
|
||||
runner = GeneratorSweepRunner(CONFIG)
|
||||
runner.run()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,31 +0,0 @@
|
||||
"""Local editable config for ``librevna_generator_sweep.py``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from python_app.generator_sweep.config import GeneratorSweepConfig
|
||||
|
||||
|
||||
CONFIG = GeneratorSweepConfig(
|
||||
log_level="INFO", # DEBUG, INFO, WARNING, ERROR, or CRITICAL.
|
||||
serial=None, # None or exact serial string, for example "206930A15532".
|
||||
strict_protocol_version=14, # Exact value for the connected device: 14.
|
||||
start_hz=100_000_000.0, # Script constraint: > 0. Device reports frequency range 0.0 .. 6_000_000_000.0 Hz.
|
||||
stop_hz=6_000_000_000.0, # Script constraint: >= start_hz. Device reports frequency range 0.0 .. 6_000_000_000.0 Hz.
|
||||
step_hz=0, # > 0 to use step mode, or 0 to disable and use points mode.
|
||||
points=501, # 0 to disable, or >= 2 to use points mode; if start_hz == stop_hz then only 1 is allowed.
|
||||
hold_time_ms=50.0, # >= 0 ms. Float values like 0.5 are allowed.
|
||||
loop=True, # True or False.
|
||||
port=1, # Only 1 or 2.
|
||||
power_dbm=-10.0, # Device reports source power range -40.0 .. 0.0 dBm.
|
||||
amplitude_correction=False, # True = apply source amplitude calibration, False = use raw generator level.
|
||||
settle_timeout_ms=1_000, # > 0 ms.
|
||||
status_poll_ms=10.0, # > 0 ms. Float values like 0.2 are allowed.
|
||||
post_lock_delay_us=0, # >= 0 us.
|
||||
gpio_chip="/dev/gpiochip0", # Non-empty Linux GPIO chip path.
|
||||
sweep_start_pin=5, # BCM GPIO, >= 0, must differ from curr_step_pin and pwm_pin.
|
||||
curr_step_pin=6, # BCM GPIO, >= 0, must differ from sweep_start_pin and pwm_pin.
|
||||
curr_step_initial_level=0, # Only 0 or 1.
|
||||
pwm_pin=12, # Only 12, 13, 18, or 19 on Raspberry Pi 5.
|
||||
pwm_frequency_hz=2_000_000, # > 0 Hz.
|
||||
pwm_duty_cycle=0.5, # Range: 0.0 < value <= 1.0.
|
||||
)
|
||||
@@ -1,172 +0,0 @@
|
||||
"""Manual smoke scenario for end-to-end pipeline check without GUI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
import ctypes.util
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData
|
||||
from python_app.models.run_config_model import RunConfigModel
|
||||
from python_app.orchestration.config_writer import ConfigWriter
|
||||
from python_app.orchestration.process_supervisor import ProcessSupervisor
|
||||
from python_app.orchestration.shm_reader import ShmRingReader
|
||||
from python_app.storage.npz_store import NpzStore, radar_key_from_config
|
||||
|
||||
|
||||
def _make_unique_ring_name(prefix: str) -> str:
|
||||
"""Build unique POSIX SHM ring name."""
|
||||
stamp = time.monotonic_ns()
|
||||
return f"/{prefix}_{os.getpid()}_{stamp}"
|
||||
|
||||
|
||||
def _shm_unlink(name: str) -> None:
|
||||
"""Best-effort unlink for POSIX shared-memory object."""
|
||||
libc_name = ctypes.util.find_library("c")
|
||||
if libc_name is None:
|
||||
return
|
||||
|
||||
libc = ctypes.CDLL(libc_name, use_errno=True)
|
||||
libc.shm_unlink.argtypes = [ctypes.c_char_p]
|
||||
libc.shm_unlink.restype = ctypes.c_int
|
||||
|
||||
result = libc.shm_unlink(name.encode("utf-8"))
|
||||
if result == 0:
|
||||
return
|
||||
|
||||
err = ctypes.get_errno()
|
||||
if err != 2: # ENOENT
|
||||
raise OSError(err, f"shm_unlink failed for {name}")
|
||||
|
||||
|
||||
def build_synthetic_collection(
|
||||
config: RunConfigModel,
|
||||
value_scale: float,
|
||||
*,
|
||||
s11_scale: float,
|
||||
s11_phase_offset: float,
|
||||
) -> SweepCollection:
|
||||
"""Build synthetic sweep collection for all configured switch combos."""
|
||||
traces: list[TraceData] = []
|
||||
combos = RunConfigModel.build_full_combos(config.input_switch.positions, config.output_switch.positions)
|
||||
|
||||
for combo in combos:
|
||||
frequency_hz = np.linspace(
|
||||
config.radar.sweep.start_hz,
|
||||
config.radar.sweep.stop_hz,
|
||||
config.radar.sweep.points,
|
||||
dtype=np.float32,
|
||||
)
|
||||
phase = np.linspace(0.0, np.pi * 2.0, config.radar.sweep.points, dtype=np.float32)
|
||||
reflected_phase = (phase * 0.6) + s11_phase_offset
|
||||
s11 = (s11_scale * (np.cos(reflected_phase) + 1j * np.sin(reflected_phase))).astype(np.complex64)
|
||||
s21 = value_scale * (np.cos(phase) + 1j * np.sin(phase)).astype(np.complex64)
|
||||
|
||||
traces.append(
|
||||
TraceData(
|
||||
combo=ComboKey(input=combo.input, output=combo.output),
|
||||
frequency_hz=frequency_hz,
|
||||
s11=s11,
|
||||
s21=s21,
|
||||
)
|
||||
)
|
||||
|
||||
return SweepCollection(collection_id=1, monotonic_ns=time.monotonic_ns(), traces=traces)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Run manual smoke-test pipeline scenario."""
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--duration", type=float, default=3.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
project_root = PROJECT_ROOT
|
||||
store = NpzStore(project_root / "python_app/data")
|
||||
config_writer = ConfigWriter(project_root / "python_app/runtime")
|
||||
supervisor = ProcessSupervisor(project_root)
|
||||
|
||||
config = RunConfigModel.load_from_path(project_root / "run_config.json")
|
||||
config.radar.driver_mode = "mock"
|
||||
config.input_switch.driver_mode = "mock"
|
||||
config.output_switch.driver_mode = "mock"
|
||||
config.combos = RunConfigModel.build_full_combos(config.input_switch.positions, config.output_switch.positions)
|
||||
config.rings.raw.name = _make_unique_ring_name("radar_raw_smoke")
|
||||
config.rings.raw_tap.name = _make_unique_ring_name("radar_raw_tap_smoke")
|
||||
config.rings.preprocessed.name = _make_unique_ring_name("radar_preprocessed_smoke")
|
||||
config.rings.preprocessed_tap.name = _make_unique_ring_name("radar_preprocessed_tap_smoke")
|
||||
config.rings.results.name = _make_unique_ring_name("radar_results_smoke")
|
||||
|
||||
radar_key = radar_key_from_config(
|
||||
model_name=config.radar.model,
|
||||
serial=config.radar.serial,
|
||||
sweep_start_hz=config.radar.sweep.start_hz,
|
||||
sweep_stop_hz=config.radar.sweep.stop_hz,
|
||||
sweep_points=config.radar.sweep.points,
|
||||
ifbw_hz=config.radar.sweep.if_bandwidth_hz,
|
||||
power_dbm=config.radar.sweep.power_dbm,
|
||||
extra_serials=config.radar_key_extra_parts() or None,
|
||||
)
|
||||
|
||||
s21_calibration_set = build_synthetic_collection(config, value_scale=1.0, s11_scale=0.15, s11_phase_offset=0.4)
|
||||
s21_reference_set = build_synthetic_collection(config, value_scale=0.3, s11_scale=0.08, s11_phase_offset=0.9)
|
||||
s11_open_set = build_synthetic_collection(config, value_scale=0.85, s11_scale=0.95, s11_phase_offset=0.1)
|
||||
s11_short_set = build_synthetic_collection(config, value_scale=0.85, s11_scale=0.95, s11_phase_offset=3.2)
|
||||
s11_load_set = build_synthetic_collection(config, value_scale=0.85, s11_scale=0.05, s11_phase_offset=1.4)
|
||||
s11_reference_set = build_synthetic_collection(config, value_scale=0.3, s11_scale=0.18, s11_phase_offset=1.1)
|
||||
|
||||
store.save_set("s21_calibration", radar_key, "smoke_cal", s21_calibration_set)
|
||||
store.save_set("s21_reference", radar_key, "smoke_ref", s21_reference_set)
|
||||
store.save_set("s11_open", radar_key, "smoke_open", s11_open_set)
|
||||
store.save_set("s11_short", radar_key, "smoke_short", s11_short_set)
|
||||
store.save_set("s11_load", radar_key, "smoke_load", s11_load_set)
|
||||
store.save_set("s11_reference", radar_key, "smoke_s11_ref", s11_reference_set)
|
||||
|
||||
config.preprocess.s21.calibration.set_name = "smoke_cal"
|
||||
config.preprocess.s21.reference.set_name = "smoke_ref"
|
||||
config.preprocess.s11.calibration.open.set_name = "smoke_open"
|
||||
config.preprocess.s11.calibration.short.set_name = "smoke_short"
|
||||
config.preprocess.s11.calibration.load.set_name = "smoke_load"
|
||||
config.preprocess.s11.reference.set_name = "smoke_s11_ref"
|
||||
config_writer.prepare_preprocess_bundles(store, radar_key, config)
|
||||
|
||||
config_path = config_writer.write(config, project_root / "python_app/runtime/run_config_smoke.json")
|
||||
|
||||
result_reader: ShmRingReader | None = None
|
||||
try:
|
||||
supervisor.start(config_path)
|
||||
result_reader = ShmRingReader(config.rings.results.name)
|
||||
deadline = time.monotonic() + args.duration
|
||||
received = 0
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
result = result_reader.pop_result_collection() if result_reader is not None else None
|
||||
if result is not None:
|
||||
received += 1
|
||||
time.sleep(0.02)
|
||||
|
||||
print(f"Received result collections: {received}")
|
||||
finally:
|
||||
supervisor.stop_all()
|
||||
if result_reader is not None:
|
||||
result_reader.close()
|
||||
_shm_unlink(config.rings.raw.name)
|
||||
_shm_unlink(config.rings.raw_tap.name)
|
||||
_shm_unlink(config.rings.preprocessed.name)
|
||||
_shm_unlink(config.rings.preprocessed_tap.name)
|
||||
_shm_unlink(config.rings.results.name)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,142 +1,64 @@
|
||||
"""Raw producer for native matrix radars and the shared single/multi mock scene."""
|
||||
"""Publish LibreVNA single-device, matrix, or simulated collections to SHM."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from contextlib import ExitStack, suppress
|
||||
from pathlib import Path
|
||||
import signal
|
||||
from contextlib import ExitStack
|
||||
import threading
|
||||
import time
|
||||
|
||||
from python_app.hardware_full.matrix_radar_service import MatrixRadarService, create_acquisition_service
|
||||
from python_app.logging_setup import coerce_level
|
||||
from python_app.models.run_config_model import RunConfigModel
|
||||
from python_app.orchestration.shm import ShmRingWriter
|
||||
from python_app.storage.npz.serialize import RAW_MAGIC, serialize_trace_collection
|
||||
from python_app.hardware_full.matrix_radar_service import MatrixRadarService, create_matrix_radar_service
|
||||
from python_app.models.run_config_schema import RunConfigModel
|
||||
from python_app.scripts.producer_common import RawPublisher, close_devices, load_producer_config, retry_open
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The producer waits for the matrix radar forever: a device that is absent at boot
|
||||
# or disappears mid-run must never kill the producer, only make it wait. Reconnect
|
||||
# uses a capped exponential backoff so a long absence does not busy-spin, and every
|
||||
# wait is interruptible by SIGINT/SIGTERM (stop_requested) for a prompt clean exit.
|
||||
_OPEN_RETRY_MIN_S = 1.0
|
||||
_OPEN_RETRY_MAX_S = 10.0
|
||||
# Throttle open-failure logging during a long wait so a permanently absent device
|
||||
# does not flood the process log: log the first failure, then every Nth attempt.
|
||||
_OPEN_RETRY_LOG_EVERY = 30
|
||||
|
||||
|
||||
def _open_radar_with_retry(
|
||||
config: RunConfigModel,
|
||||
previous: MatrixRadarService | None,
|
||||
stop_requested: threading.Event,
|
||||
) -> MatrixRadarService | None:
|
||||
"""Open+configure the matrix radar, retrying forever until success or stop.
|
||||
"""Recreate a failed LibreVNA service and wait interruptibly for its return."""
|
||||
close_devices(previous)
|
||||
|
||||
Used for both the initial open and every in-loop reconnect, so a device that is
|
||||
absent at boot or disappears mid-run never kills the producer — it just waits.
|
||||
Returns the opened service, or ``None`` if a stop was requested before any device
|
||||
became available. Backoff is capped and every wait is interruptible by SIGTERM.
|
||||
"""
|
||||
if previous is not None:
|
||||
with suppress(Exception):
|
||||
previous.close()
|
||||
|
||||
attempt = 0
|
||||
delay = _OPEN_RETRY_MIN_S
|
||||
while not stop_requested.is_set():
|
||||
radar = create_acquisition_service(config)
|
||||
def attempt() -> MatrixRadarService:
|
||||
radar = create_matrix_radar_service(config)
|
||||
try:
|
||||
radar.open()
|
||||
radar.configure(config.radar.sweep)
|
||||
except Exception as exc: # noqa: BLE001 — waiting for the device is the point
|
||||
with suppress(Exception):
|
||||
radar.close() # drop any partial open before the next attempt
|
||||
attempt += 1
|
||||
if attempt == 1 or attempt % _OPEN_RETRY_LOG_EVERY == 0:
|
||||
logger.warning(
|
||||
"Matrix radar not available (attempt %d); retrying every up to %.0fs "
|
||||
"until the device is present: %s",
|
||||
attempt,
|
||||
_OPEN_RETRY_MAX_S,
|
||||
exc,
|
||||
)
|
||||
if stop_requested.wait(delay):
|
||||
with suppress(Exception):
|
||||
radar.close()
|
||||
return None
|
||||
delay = min(delay * 2.0, _OPEN_RETRY_MAX_S)
|
||||
continue
|
||||
|
||||
if attempt > 0:
|
||||
logger.info("Matrix radar opened after %d attempt(s).", attempt + 1)
|
||||
except Exception:
|
||||
close_devices(radar)
|
||||
raise
|
||||
return radar
|
||||
|
||||
logger.debug("Stop requested before any matrix radar became available.")
|
||||
return None
|
||||
return retry_open(attempt, stop_requested, "LibreVNA radar")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Run producer process until config or signal requests exit."""
|
||||
parser = argparse.ArgumentParser(description="Publish matrix-radar raw sweeps to SHM rings")
|
||||
parser.add_argument("--config", required=True, type=Path, help="Path to run_config.json")
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
|
||||
stop_requested = threading.Event()
|
||||
|
||||
def request_stop(_signum: int, _frame: object) -> None:
|
||||
stop_requested.set()
|
||||
|
||||
signal.signal(signal.SIGINT, request_stop)
|
||||
signal.signal(signal.SIGTERM, request_stop)
|
||||
|
||||
config = RunConfigModel.load_from_path(args.config)
|
||||
config.apply_device_model_constraints()
|
||||
# Honor the configured verbosity so the DEBUG switch/sweep timing trace can be
|
||||
# turned on from the profile instead of requiring a code edit. basicConfig above
|
||||
# only installed the handler; the package logger owns the level.
|
||||
logging.getLogger("python_app").setLevel(coerce_level(config.logging.level))
|
||||
config_path, config, stop_requested = load_producer_config("Publish LibreVNA raw sweeps to SHM rings")
|
||||
logger.info(
|
||||
"Matrix radar raw producer starting: config=%s, model=%s, continuous=%s",
|
||||
args.config, config.radar.model, config.runtime.continuous,
|
||||
"LibreVNA radar raw producer starting: config=%s, model=%s, continuous=%s",
|
||||
config_path, config.radar.model, config.runtime.continuous,
|
||||
)
|
||||
|
||||
radar: MatrixRadarService | None = None
|
||||
cleanup = ExitStack()
|
||||
try:
|
||||
raw_writer = ShmRingWriter(
|
||||
config.rings.raw.name,
|
||||
config.rings.raw.capacity,
|
||||
config.rings.raw.slot_size_bytes,
|
||||
)
|
||||
cleanup.callback(raw_writer.close)
|
||||
raw_tap_writer = ShmRingWriter(
|
||||
config.rings.raw_tap.name,
|
||||
config.rings.raw_tap.capacity,
|
||||
config.rings.raw_tap.slot_size_bytes,
|
||||
)
|
||||
cleanup.callback(raw_tap_writer.close)
|
||||
logger.debug(
|
||||
"Opened SHM ring writers: raw=%s, raw_tap=%s",
|
||||
config.rings.raw.name, config.rings.raw_tap.name,
|
||||
)
|
||||
publisher = RawPublisher(config.rings, cleanup)
|
||||
|
||||
radar = _open_radar_with_retry(config, previous=None, stop_requested=stop_requested)
|
||||
if radar is None:
|
||||
return 0 # asked to stop before a device became available
|
||||
collection_id = 1
|
||||
publish_failures = 0
|
||||
while not stop_requested.is_set():
|
||||
collection_start = time.monotonic()
|
||||
try:
|
||||
collection = radar.acquire_collection(collection_id=collection_id)
|
||||
except Exception as exc: # noqa: BLE001 — reconnect forever, never give up
|
||||
logger.warning(
|
||||
"Matrix radar acquisition failed; reconnecting and waiting for the device: %s",
|
||||
"LibreVNA radar acquisition failed; reconnecting and waiting for the device: %s",
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
@@ -145,36 +67,22 @@ def main() -> int:
|
||||
break # stop requested while waiting to reconnect
|
||||
continue
|
||||
|
||||
payload = serialize_trace_collection(collection, RAW_MAGIC)
|
||||
if raw_writer.push(payload):
|
||||
raw_tap_writer.push(payload) # best-effort GUI tap; never fatal
|
||||
else:
|
||||
# An oversized payload vs the ring slot is a persistent config error,
|
||||
# not a device fault: log it (throttled) and skip the collection rather
|
||||
# than letting a RuntimeError escape the loop and kill the producer.
|
||||
publish_failures += 1
|
||||
if publish_failures == 1 or publish_failures % 100 == 0:
|
||||
logger.error(
|
||||
"Raw payload %d B exceeds ring slot %d B; dropping collection %d (drops=%d)",
|
||||
len(payload), raw_writer.slot_size_bytes, collection_id, publish_failures,
|
||||
)
|
||||
publisher.publish(collection)
|
||||
if not config.runtime.continuous:
|
||||
break
|
||||
collection_duration_s = time.monotonic() - collection_start
|
||||
if collection_id == 1 or collection_id % 20 == 0 or collection_duration_s > 2.0:
|
||||
logger.info(
|
||||
"matrix radar collection %d acquired in %.3f s",
|
||||
"LibreVNA radar collection %d acquired in %.3f s",
|
||||
collection_id,
|
||||
collection_duration_s,
|
||||
)
|
||||
collection_id += 1
|
||||
finally:
|
||||
if radar is not None:
|
||||
with suppress(Exception):
|
||||
radar.close()
|
||||
close_devices(radar)
|
||||
cleanup.close()
|
||||
|
||||
logger.info("matrix radar raw producer stopped")
|
||||
logger.info("LibreVNA radar raw producer stopped")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ from python_app.hardware_full.librevna_multi_device_driver.models import (
|
||||
)
|
||||
from python_app.hardware_full.matrix_radar_service import create_matrix_radar_service
|
||||
from python_app.models.dataset_model import SweepCollection
|
||||
from python_app.models.run_config_model import RunConfigModel
|
||||
from python_app.models.run_config_schema import RunConfigModel
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -117,7 +117,7 @@ def main() -> int:
|
||||
config = RunConfigModel.load_from_path(args.config)
|
||||
config.apply_device_model_constraints()
|
||||
|
||||
if not config.is_matrix_radar:
|
||||
if not config.is_multi_device:
|
||||
raise SystemExit("нужен матричный radar.model (librevna_multi)")
|
||||
substages = config.matrix_stimulus_switch_positions
|
||||
outputs = config.output_switch.positions
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Shared CLI, interruptible device retry, and raw-ring publication."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections.abc import Callable
|
||||
from contextlib import ExitStack, suppress
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import threading
|
||||
from typing import Protocol, TypeVar
|
||||
|
||||
from python_app.logging_setup import coerce_level
|
||||
from python_app.models.dataset_model import SweepCollection
|
||||
from python_app.models.run_config_schema import RingEndpointModel, RingsModel, RunConfigModel
|
||||
from python_app.models.run_config_validation import validate_acquisition_config
|
||||
from python_app.orchestration.shm.decoder import RAW_MAGIC
|
||||
from python_app.orchestration.shm.ring_writer import ShmRingWriter
|
||||
from python_app.storage.npz.serialize import serialize_trace_collection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def load_producer_config(description: str) -> tuple[Path, RunConfigModel, threading.Event]:
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
parser.add_argument("--config", required=True, type=Path, help="Path to run_config.json")
|
||||
path = parser.parse_args().config
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
|
||||
stop_requested = threading.Event()
|
||||
|
||||
def request_stop(_signum: int, _frame: object) -> None:
|
||||
stop_requested.set()
|
||||
|
||||
signal.signal(signal.SIGINT, request_stop)
|
||||
signal.signal(signal.SIGTERM, request_stop)
|
||||
config = RunConfigModel.load_from_path(path)
|
||||
config.ensure_combos()
|
||||
validate_acquisition_config(config)
|
||||
logging.getLogger("python_app").setLevel(coerce_level(config.logging.level))
|
||||
return path, config, stop_requested
|
||||
|
||||
|
||||
class _Closable(Protocol):
|
||||
def close(self) -> None: ...
|
||||
|
||||
|
||||
def close_devices(*devices: _Closable | None) -> None:
|
||||
"""Attempt every hardware cleanup even if one disconnected device raises."""
|
||||
for device in devices:
|
||||
if device is not None:
|
||||
with suppress(Exception):
|
||||
device.close()
|
||||
|
||||
|
||||
def retry_open(
|
||||
attempt: Callable[[], T], stop_requested: threading.Event, device_name: str,
|
||||
) -> T | None:
|
||||
"""Retry an attempt that cleans up on failure, with capped, interruptible backoff."""
|
||||
failures = 0
|
||||
delay = 1.0
|
||||
while not stop_requested.is_set():
|
||||
try:
|
||||
device = attempt()
|
||||
except Exception as exc:
|
||||
failures += 1
|
||||
if failures == 1 or failures % 30 == 0:
|
||||
logger.warning("%s unavailable (attempt %d); retrying until present: %s", device_name, failures, exc)
|
||||
if stop_requested.wait(delay):
|
||||
break
|
||||
delay = min(delay * 2, 10.0)
|
||||
else:
|
||||
if failures:
|
||||
logger.info("%s opened after %d attempt(s)", device_name, failures + 1)
|
||||
return device
|
||||
return None
|
||||
|
||||
|
||||
class RawPublisher:
|
||||
"""Publish one validated frame to raw and, when enabled, its independent GUI tap."""
|
||||
|
||||
def __init__(self, rings: RingsModel, cleanup: ExitStack) -> None:
|
||||
def open_writer(endpoint: RingEndpointModel) -> ShmRingWriter:
|
||||
writer = ShmRingWriter(endpoint.name, endpoint.capacity, endpoint.slot_size_bytes)
|
||||
cleanup.callback(writer.close)
|
||||
return writer
|
||||
|
||||
self._raw = open_writer(rings.raw)
|
||||
self._tap = open_writer(rings.raw_tap) if rings.raw_tap.name else None
|
||||
self._dropped = 0
|
||||
|
||||
def publish(self, collection: SweepCollection) -> bool:
|
||||
payload = serialize_trace_collection(collection, RAW_MAGIC, require_capture_window=True)
|
||||
if not self._raw.push(payload):
|
||||
self._dropped += 1
|
||||
if self._dropped == 1 or self._dropped % 100 == 0:
|
||||
logger.error(
|
||||
"Raw payload %d B exceeds ring slot %d B; dropping collection %d (drops=%d)",
|
||||
len(payload), self._raw.slot_size_bytes, collection.collection_id, self._dropped,
|
||||
)
|
||||
return False
|
||||
if self._tap is not None:
|
||||
self._tap.push(payload) # independent latest-wins tap; never backpressure raw
|
||||
return True
|
||||
@@ -19,24 +19,20 @@ never see time jump backwards. ``collection_id`` increases forever.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from contextlib import ExitStack
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import re
|
||||
import signal
|
||||
import statistics
|
||||
import threading
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
from python_app.logging_setup import coerce_level
|
||||
from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData
|
||||
from python_app.models.run_config_model import RunConfigModel
|
||||
from python_app.orchestration.shm import ShmRingWriter
|
||||
from python_app.storage.npz.serialize import RAW_MAGIC, serialize_trace_collection
|
||||
from python_app.orchestration.shm.decoder import RAW_MAGIC
|
||||
from python_app.scripts.producer_common import RawPublisher, load_producer_config
|
||||
from python_app.storage.npz.serialize import serialize_trace_collection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -105,10 +101,8 @@ def _load_dataset(dataset_dir: Path, combos: list[ComboKey], expected_points: in
|
||||
)
|
||||
s21 = np.load(s21_path, allow_pickle=False).astype(np.complex64, copy=False)
|
||||
freq = np.load(freq_path, allow_pickle=False).astype(np.float32, copy=False)
|
||||
s11_path = folder / f"i{combo.input}_o{combo.output}_s11.npy"
|
||||
s11 = np.load(s11_path, allow_pickle=False).astype(np.complex64, copy=False) if s11_path.is_file() else np.zeros_like(s21)
|
||||
if freq.ndim != 1 or s21.shape != freq.shape or s11.shape != freq.shape:
|
||||
raise RuntimeError(f"Snapshot {name}: frequency, S11 and S21 must be equally sized 1D arrays")
|
||||
if freq.ndim != 1 or s21.shape != freq.shape:
|
||||
raise RuntimeError(f"Snapshot {name}: frequency and S21 must be equally sized 1D arrays")
|
||||
if expected_points and s21.size != expected_points:
|
||||
raise RuntimeError(
|
||||
f"{s21_path}: {s21.size} points, but radar.sweep.points = "
|
||||
@@ -119,7 +113,6 @@ def _load_dataset(dataset_dir: Path, combos: list[ComboKey], expected_points: in
|
||||
TraceData(
|
||||
combo=combo,
|
||||
frequency_hz=freq,
|
||||
s11=s11,
|
||||
s21=s21,
|
||||
capture_start_ns=window[0],
|
||||
capture_end_ns=window[1],
|
||||
@@ -157,7 +150,6 @@ def _build_collection(
|
||||
TraceData(
|
||||
combo=source.combo,
|
||||
frequency_hz=source.frequency_hz,
|
||||
s11=source.s11,
|
||||
s21=source.s21,
|
||||
capture_start_ns=capture[0],
|
||||
capture_end_ns=capture[1],
|
||||
@@ -174,22 +166,7 @@ def _build_collection(
|
||||
|
||||
def main() -> int:
|
||||
"""Replay the configured dataset into the raw SHM rings until stopped."""
|
||||
parser = argparse.ArgumentParser(description="Replay a recorded dataset to raw SHM rings")
|
||||
parser.add_argument("--config", required=True, type=Path, help="Path to run_config.json")
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
|
||||
stop_requested = threading.Event()
|
||||
|
||||
def request_stop(_signum: int, _frame: object) -> None:
|
||||
stop_requested.set()
|
||||
|
||||
signal.signal(signal.SIGINT, request_stop)
|
||||
signal.signal(signal.SIGTERM, request_stop)
|
||||
|
||||
config = RunConfigModel.load_from_path(args.config)
|
||||
config.apply_device_model_constraints()
|
||||
logging.getLogger("python_app").setLevel(coerce_level(config.logging.level))
|
||||
_, config, stop_requested = load_producer_config("Replay a recorded dataset to raw SHM rings")
|
||||
|
||||
replay = config.radar.replay
|
||||
if not replay.dataset_dir:
|
||||
@@ -211,23 +188,15 @@ def main() -> int:
|
||||
|
||||
cleanup = ExitStack()
|
||||
try:
|
||||
raw_writer = ShmRingWriter(
|
||||
config.rings.raw.name,
|
||||
config.rings.raw.capacity,
|
||||
config.rings.raw.slot_size_bytes,
|
||||
)
|
||||
cleanup.callback(raw_writer.close)
|
||||
raw_tap_writer = ShmRingWriter(
|
||||
config.rings.raw_tap.name,
|
||||
config.rings.raw_tap.capacity,
|
||||
config.rings.raw_tap.slot_size_bytes,
|
||||
)
|
||||
cleanup.callback(raw_tap_writer.close)
|
||||
publisher = RawPublisher(config.rings, cleanup)
|
||||
|
||||
payload_size = len(serialize_trace_collection(
|
||||
_build_collection(dataset, 0, 1, time.monotonic_ns()), RAW_MAGIC
|
||||
_build_collection(dataset, 0, 1, time.monotonic_ns()), RAW_MAGIC, require_capture_window=True,
|
||||
))
|
||||
for ring in (config.rings.raw, config.rings.raw_tap):
|
||||
for name in ("raw", "raw_tap", "preprocessed", "preprocessed_tap"):
|
||||
ring = getattr(config.rings, name)
|
||||
if name.endswith("_tap") and not ring.name:
|
||||
continue
|
||||
if ring.slot_size_bytes < payload_size:
|
||||
raise ValueError(f"Replay requires {payload_size} bytes per slot; ring {ring.name} has {ring.slot_size_bytes}")
|
||||
|
||||
@@ -257,10 +226,14 @@ def main() -> int:
|
||||
break
|
||||
|
||||
collection = _build_collection(dataset, index, collection_id, frame_ns)
|
||||
payload = serialize_trace_collection(collection, RAW_MAGIC)
|
||||
if not raw_writer.push(payload):
|
||||
# Acquisition identity uses real monotonic time, even at accelerated
|
||||
# replay rates. Keep the recorded duration for legacy GPR and the
|
||||
# virtual per-trace timeline for SAR aperture/motion reconstruction.
|
||||
duration_ns = collection.capture_end_ns - collection.capture_start_ns
|
||||
collection.capture_start_ns = time.monotonic_ns()
|
||||
collection.capture_end_ns = collection.capture_start_ns + duration_ns
|
||||
if not publisher.publish(collection):
|
||||
raise RuntimeError(f"Replay raw ring rejected collection {collection_id}")
|
||||
raw_tap_writer.push(payload)
|
||||
if collection_id == 1 or collection_id % 50 == 0:
|
||||
logger.info(
|
||||
"replay collection %d published (snapshot %d/%d, cycle %d)",
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
"""Minimal GUI tool for direct LibreVNA raw acquisition checks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import sys
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from PyQt6.QtWidgets import QApplication
|
||||
|
||||
from python_app.scripts.hardware_raw_orchestrator_test import RawOrchestratorViewer
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Run standalone raw-viewer GUI against a selected run config."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Intermediate test: native VNA acquisition with mock switch drivers"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
type=Path,
|
||||
default=PROJECT_ROOT / "run_config.json",
|
||||
help="Path to run config JSON",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-reset-ring",
|
||||
action="store_true",
|
||||
help="Do not unlink existing raw ring name before starting",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-radar-prepare",
|
||||
action="store_true",
|
||||
help="Skip Python pre-configuration of native radar before orchestrator start",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strict-radar-prepare",
|
||||
action="store_true",
|
||||
help="Fail immediately if Python pre-configuration cannot run",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
config_path = args.config.resolve()
|
||||
if not config_path.exists():
|
||||
raise FileNotFoundError(f"Config file not found: {config_path}")
|
||||
|
||||
app = QApplication(sys.argv)
|
||||
viewer = RawOrchestratorViewer(
|
||||
config_path=config_path,
|
||||
reset_ring=not args.no_reset_ring,
|
||||
prepare_radar=not args.skip_radar_prepare,
|
||||
strict_prepare=args.strict_radar_prepare,
|
||||
)
|
||||
viewer.setWindowTitle("Raw Sweep Viewer (VNA native + mock switches)")
|
||||
viewer.show()
|
||||
|
||||
def _sig_handler(_signum, _frame):
|
||||
"""Close viewer gracefully on process signals."""
|
||||
viewer.close()
|
||||
|
||||
signal.signal(signal.SIGINT, _sig_handler)
|
||||
signal.signal(signal.SIGTERM, _sig_handler)
|
||||
|
||||
return app.exec()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user