awe/radar_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor radar system

Browse Source
This commit is contained in:
Ayzen
2026-10-05 16:46:28 +03:00
parent 515049d2ed
commit 8f097dfa2e
179 changed files with 8212 additions and 6676 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python_app/scripts/convert_prog_libre_manual_to_vna_history.py
+3 -3
View File
@@ -5,12 +5,13 @@ from __future__ import annotations
import argparse
import csv
from datetime import datetime, timezone
import json
from pathlib import Path
from typing import Any
import numpy as np
from python_app.file_io import atomic_write_json
def _real_imag_keys(trace_prefix: str) -> tuple[str, str]:
"""Return the LibreVNA CSV column names for a trace's real and imaginary parts."""
@@ -185,8 +186,7 @@ def _build_history_payload(
def _write_payload(output_path: Path, payload: dict[str, Any]) -> None:
"""Write a payload as indented UTF-8 JSON, creating parent directories as needed."""
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
atomic_write_json(output_path, payload, durable=True)
def _rmse(left: np.ndarray, right: np.ndarray) -> float:
python_app/scripts/convert_snapshot_to_vna_history.py
+9 -113
View File
@@ -4,7 +4,6 @@ from __future__ import annotations
import argparse
import json
from collections import defaultdict
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
@@ -12,20 +11,10 @@ from typing import Any
import numpy as np
from python_app.file_io import atomic_write_json
@dataclass(frozen=True)
class TraceRecord:
"""One raw/preprocessed trace extracted from a snapshot collection directory."""
stage: str
collection_id: int
monotonic_ns: int
stage_index: int
frequency_hz: np.ndarray
samples: np.ndarray
# End of this trace's own sweep, from the snapshot's per-trace metadata; 0 for a
# snapshot recorded before per-trace timing existed.
capture_end_ns: int = 0
from python_app.storage.npz.vna_history_json import TraceRecord, _build_sweep_history, _normalize_channel
@dataclass(frozen=True)
@@ -46,10 +35,13 @@ def _load_json(path: Path) -> dict[str, Any]:
def _collection_dirs(stage_dir: Path) -> list[Path]:
"""Return the stage's collection subdirectories sorted by name (empty if absent)."""
"""Return collection subdirectories in numeric recording order (empty if absent)."""
if not stage_dir.exists():
return []
return sorted([path for path in stage_dir.iterdir() if path.is_dir()], key=lambda path: path.name)
return sorted(
(path for path in stage_dir.iterdir() if path.is_dir()),
key=lambda path: (_parse_stage_index(path.name, 0), path.name),
)
def _parse_stage_index(name: str, fallback: int) -> int:
@@ -71,14 +63,6 @@ def _pick_trace_meta(meta: dict[str, Any], input_index: int, output_index: int)
return None
def _normalize_channel(channel: str) -> str:
"""Return a lowercased channel name, accepting only ``s21`` or ``s11``."""
normalized = str(channel).strip().lower()
if normalized not in {"s21", "s11"}:
raise ValueError("channel must be either 's21' or 's11'")
return normalized
def _load_stage_records(
snapshot_dir: Path,
stage: str,
@@ -132,6 +116,7 @@ def _load_stage_records(
frequency_hz=frequency_hz,
samples=samples,
capture_end_ns=int(trace_meta.get("capture_end_ns", 0)),
capture_start_ns=int(trace_meta.get("capture_start_ns", 0)),
)
)
@@ -158,94 +143,6 @@ def _load_stage_refs(snapshot_dir: Path, stage: str) -> list[CollectionRef]:
return refs
def _index_by_collection_occurrence(records: list[TraceRecord]) -> tuple[dict[tuple[int, int], TraceRecord], list[tuple[int, int]]]:
"""Key records by (collection_id, occurrence) and return the map plus original order.
The occurrence counter disambiguates repeated collection ids, so raw and preprocessed
stages can be aligned slot-for-slot even when ids recur.
"""
counters: defaultdict[int, int] = defaultdict(int)
record_map: dict[tuple[int, int], TraceRecord] = {}
order: list[tuple[int, int]] = []
for record in records:
occurrence = counters[record.collection_id]
counters[record.collection_id] += 1
key = (record.collection_id, occurrence)
record_map[key] = record
order.append(key)
return record_map, order
def _complex_to_points(values: np.ndarray) -> list[list[float]]:
"""Return complex samples as ``[real, imag]`` pairs for JSON serialization."""
return [[float(v.real), float(v.imag)] for v in values]
def _build_sweep_history(
raw_records: list[TraceRecord],
preprocessed_records: list[TraceRecord],
*,
channel: str,
primary_stage: str,
) -> list[dict[str, Any]]:
"""Merge raw and preprocessed records into vna-system ``sweep_history`` entries.
Iterates collections in the primary stage's order, pairing each with its counterpart
in the other stage; the raw samples become ``sweep_points`` and the preprocessed
samples ``calibrated_points`` (falling back to whichever stage is present).
"""
raw_map, raw_order = _index_by_collection_occurrence(raw_records)
pre_map, pre_order = _index_by_collection_occurrence(preprocessed_records)
if primary_stage == "preprocessed":
primary_order = pre_order or raw_order
else:
primary_order = raw_order or pre_order
history: list[dict[str, Any]] = []
for fallback_index, key in enumerate(primary_order):
raw = raw_map.get(key)
pre = pre_map.get(key)
base = pre or raw
if base is None:
continue
# vna_system uses calibrated_data if present, otherwise sweep_data.
sweep_source = raw or pre
calibrated_source = pre or raw
if sweep_source is None or calibrated_source is None:
continue
start_freq_hz = float(base.frequency_hz[0])
stop_freq_hz = float(base.frequency_hz[-1])
# Prefer this trace's own sweep time: with a switching matrix the combos of
# one collection are measured milliseconds apart, so the collection
# timestamp misplaces every combo but the last.
if base.capture_end_ns > 0:
timestamp_sec = float(base.capture_end_ns) / 1_000_000_000.0
elif base.monotonic_ns > 0:
timestamp_sec = float(base.monotonic_ns) / 1_000_000_000.0
else:
timestamp_sec = float(fallback_index)
history.append(
{
"timestamp": timestamp_sec,
"sweep_points": _complex_to_points(sweep_source.samples),
"calibrated_points": _complex_to_points(calibrated_source.samples),
"reference_points": [],
"vna_config": {
"mode": channel,
"start_freq": start_freq_hz,
"stop_freq": stop_freq_hz,
"points": int(base.frequency_hz.size),
},
}
)
return history
def _stage_alignment_warning(pre_refs: list[CollectionRef], result_refs: list[CollectionRef]) -> str | None:
"""Return warning text when preprocessed/results stages are not identity-aligned."""
if not pre_refs or not result_refs:
@@ -394,8 +291,7 @@ def main() -> None:
if alignment_warning is not None:
payload["alignment_warning"] = alignment_warning
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
atomic_write_json(output_path, payload, durable=True)
print(
"Converted snapshot to vna_system history JSON:\n"
python_app/scripts/create_neutral_preprocess_sets.py
+10 -6
View File
@@ -45,9 +45,7 @@ def main() -> int:
config = RunConfigModel.load_from_path(args.config)
config.ensure_combos()
if not supports_neutral_preprocess_sets(config):
raise RuntimeError(
"Neutral S21 sets are available only for kamil_adc and librevna_multi"
)
raise RuntimeError("Neutral S21 sets are not supported for this radar configuration")
radar_key = radar_key_from_config(
model_name=config.radar.model,
@@ -61,12 +59,19 @@ def main() -> int:
)
store = NpzStore(PROJECT_ROOT / "python_app/data")
kinds = ("s21_calibration", "s21_reference")
existing = [
kind
for kind in ("s21_calibration", "s21_reference")
for kind in kinds
if args.set_name in store.list_sets(kind, radar_key)
]
if existing and not args.force:
if len(existing) != len(kinds):
missing = ", ".join(kind for kind in kinds if kind not in existing)
raise RuntimeError(
f"Neutral set pair '{args.set_name}' is incomplete: missing {missing}; "
"use --force to replace both sets."
)
logger.info(
"Sets '%s' already exist for %s (radar_key=%s); nothing to do (use --force to overwrite).",
args.set_name, ", ".join(existing), radar_key,
@@ -74,8 +79,7 @@ def main() -> int:
return 0
calibration, reference = build_neutral_s21_sets(config)
store.save_set("s21_calibration", radar_key, args.set_name, calibration)
store.save_set("s21_reference", radar_key, args.set_name, reference)
store.save_sets(radar_key, args.set_name, dict(zip(kinds, (calibration, reference))))
logger.info(
"Neutral S21 sets saved: set=%s, radar_key=%s, combos=%d, points=%d",
args.set_name,
python_app/scripts/hardware_raw_orchestrator_test.py
+4 -8
View File
@@ -78,14 +78,9 @@ def _prepare_radar_if_needed(config_path: Path, *, strict: bool) -> str | None:
if config.is_matrix_radar:
return "Radar pre-configuration skipped (matrix radar producer config)."
radar_service = create_single_radar_service(config)
if not getattr(radar_service, "driver_available", True):
message = "LibreVNA Python driver is unavailable: skipping pre-configuration"
if strict:
raise RuntimeError(message)
return message
radar_service = None
try:
radar_service = create_single_radar_service(config)
radar_service.open()
radar_service.configure(config.radar.sweep)
return "Radar pre-configuration completed."
@@ -95,7 +90,8 @@ def _prepare_radar_if_needed(config_path: Path, *, strict: bool) -> str | None:
raise
return f"{message}. Continuing with native C++ configuration."
finally:
radar_service.close()
if radar_service is not None:
radar_service.close()
class RawOrchestratorViewer(QMainWindow):
python_app/scripts/k209_remote_server.py
-161
View File
@@ -1,161 +0,0 @@
"""Run a TCP acquisition server for a locally connected Compact-M K209."""
from __future__ import annotations
import argparse
import socket
import socketserver
import sys
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from python_app.hardware_full.compact_m_k209_service import CompactMK209Service
from python_app.hardware_full.k209_remote_protocol import (
COMMAND_ACQUIRE,
COMMAND_CONFIGURE,
COMMAND_IDENTITY,
COMMAND_LIMITS,
CONFIG_STRUCT,
DEFAULT_REMOTE_PORT,
LIMITS_STRUCT,
STATUS_OK,
recv_exact,
send_error,
send_float32_array,
send_u32,
)
from python_app.models.run_config_model import RadarSweepModel
DEFAULT_RESOURCE = "TCPIP0::127.0.0.1::hislip0,4880::INSTR"
class K209RemoteRequestHandler(socketserver.StreamRequestHandler):
"""Handle one persistent K209 remote client connection."""
def setup(self) -> None:
"""Disable Nagle and open the local K209 VISA session for this connection."""
super().setup()
self.request.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.service = CompactMK209Service(
resource=self.server.resource,
timeout_ms=self.server.timeout_ms,
preset_on_open=False,
visa_library="@ivi",
)
self.service.open()
def finish(self) -> None:
"""Close the K209 VISA session when the client disconnects."""
try:
self.service.close()
finally:
super().finish()
def handle(self) -> None:
"""Serve command bytes from the client until EOF, reporting errors back inline."""
while True:
command = self.rfile.read(1)
if not command:
return
try:
if command == COMMAND_IDENTITY:
self._handle_identity()
elif command == COMMAND_LIMITS:
self._handle_limits()
elif command == COMMAND_CONFIGURE:
self._handle_configure()
elif command == COMMAND_ACQUIRE:
self._handle_acquire()
else:
raise RuntimeError(f"unsupported command byte {command!r}")
self.wfile.flush()
except Exception as exc: # noqa: BLE001
send_error(self.wfile, str(exc))
self.wfile.flush()
def _handle_identity(self) -> None:
"""Reply with the device identity string (length-prefixed UTF-8)."""
payload = self.service.query_identity().encode("utf-8")
self.wfile.write(STATUS_OK)
send_u32(self.wfile, len(payload))
self.wfile.write(payload)
def _handle_limits(self) -> None:
"""Reply with the device frequency/IFBW/power/point limits as a packed struct."""
limits = self.service.read_device_limits()
self.wfile.write(STATUS_OK)
self.wfile.write(
LIMITS_STRUCT.pack(
float(limits["min_frequency_hz"]),
float(limits["max_frequency_hz"]),
float(limits["min_ifbw_hz"]),
float(limits["max_ifbw_hz"]),
int(limits["max_points"]),
float(limits["min_power_dbm"]),
float(limits["max_power_dbm"]),
)
)
def _handle_configure(self) -> None:
"""Apply a sweep config from the client and reply with the frequency axis."""
start_hz, stop_hz, points, ifbw_hz, power_dbm = CONFIG_STRUCT.unpack(
recv_exact(self.rfile, CONFIG_STRUCT.size)
)
sweep = RadarSweepModel(
start_hz=start_hz,
stop_hz=stop_hz,
points=int(points),
if_bandwidth_hz=ifbw_hz,
power_dbm=power_dbm,
)
self.service.configure(sweep)
self.wfile.write(STATUS_OK)
send_u32(self.wfile, int(points))
send_float32_array(self.wfile, self.service.frequency_axis())
def _handle_acquire(self) -> None:
"""Acquire one interleaved sweep and reply with the S11 and S21 arrays."""
sweep = self.service.acquire_interleaved()
self.wfile.write(STATUS_OK)
send_u32(self.wfile, int(sweep.frequency_hz.size))
send_float32_array(self.wfile, sweep.s11_values)
send_float32_array(self.wfile, sweep.s21_values)
class K209RemoteServer(socketserver.TCPServer):
"""Single-client TCP server with K209 connection settings."""
allow_reuse_address = True
def __init__(self, server_address: tuple[str, int], resource: str, timeout_ms: int) -> None:
"""Store the VISA resource and timeout used by each accepted connection."""
self.resource = resource
self.timeout_ms = timeout_ms
super().__init__(server_address, K209RemoteRequestHandler)
def _parse_args() -> argparse.Namespace:
"""Parse command-line options for the K209 remote server."""
parser = argparse.ArgumentParser(description="Serve a locally connected Compact-M K209 over TCP.")
parser.add_argument("--host", default="0.0.0.0", help="Server bind address.")
parser.add_argument("--port", type=int, default=DEFAULT_REMOTE_PORT, help="Server TCP port.")
parser.add_argument("--resource", default=DEFAULT_RESOURCE, help="Local S2VNA VISA resource.")
parser.add_argument("--timeout-ms", type=int, default=20_000, help="K209 VISA timeout.")
return parser.parse_args()
def main() -> int:
"""Bind the K209 remote server and serve clients until interrupted."""
args = _parse_args()
with K209RemoteServer((args.host, args.port), resource=args.resource, timeout_ms=args.timeout_ms) as server:
print(f"K209 remote server listening on {args.host}:{args.port}")
print(f"Local S2VNA resource: {args.resource}")
server.serve_forever()
return 0
if __name__ == "__main__":
raise SystemExit(main())
python_app/scripts/k209_remote_smoke_test.py
-75
View File
@@ -1,75 +0,0 @@
"""Smoke test for a remote Compact-M K209 server."""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
import numpy as np
PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from python_app.hardware_full.k209_remote_protocol import DEFAULT_REMOTE_HOST, DEFAULT_REMOTE_PORT
from python_app.hardware_full.remote_compact_m_k209_service import RemoteCompactMK209Service
from python_app.models.run_config_model import RadarSweepModel
def _parse_args() -> argparse.Namespace:
"""Parse command-line options for the remote K209 smoke test."""
parser = argparse.ArgumentParser(description="Validate remote K209 connection and one sweep.")
parser.add_argument("--host", default=DEFAULT_REMOTE_HOST, help="K209 remote server host.")
parser.add_argument("--port", type=int, default=DEFAULT_REMOTE_PORT, help="K209 remote server port.")
parser.add_argument("--start-hz", type=float, default=10_000_000.0)
parser.add_argument("--stop-hz", type=float, default=100_000_000.0)
parser.add_argument("--points", type=int, default=11)
parser.add_argument("--ifbw-hz", type=float, default=10_000.0)
parser.add_argument("--power-dbm", type=float, default=-20.0)
return parser.parse_args()
def main() -> int:
"""Connect to a remote K209, run one sweep, and validate its shape and values.
Raises on an unexpected point count, non-finite samples, or a non-monotonic frequency
axis; prints a one-line summary on success.
"""
args = _parse_args()
sweep = RadarSweepModel(
start_hz=args.start_hz,
stop_hz=args.stop_hz,
points=args.points,
if_bandwidth_hz=args.ifbw_hz,
power_dbm=args.power_dbm,
)
service = RemoteCompactMK209Service(host=args.host, port=args.port)
try:
service.open()
print(f"K209 IDN: {service.query_identity()}")
service.configure(sweep)
result = service.acquire()
finally:
service.close()
s11 = result.trace("s11")
s21 = result.trace("s21")
if result.x.size != args.points or s11.size != args.points or s21.size != args.points:
raise RuntimeError("Remote K209 sweep returned an unexpected point count")
if not np.all(np.isfinite(result.x)) or not np.all(np.isfinite(s11)) or not np.all(np.isfinite(s21)):
raise RuntimeError("Remote K209 sweep contains non-finite values")
if not np.all(np.diff(result.x) >= 0):
raise RuntimeError("Remote K209 frequency axis is not monotonic")
print(
"Remote K209 sweep OK: "
f"points={args.points}, first_hz={result.x[0]:.3f}, last_hz={result.x[-1]:.3f}, "
f"mean_abs_s11={float(np.mean(np.abs(s11))):.6g}, "
f"mean_abs_s21={float(np.mean(np.abs(s21))):.6g}"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
python_app/scripts/k209_smoke_test.py
-99
View File
@@ -1,99 +0,0 @@
"""Standalone smoke test for Compact-M K209 VISA HiSLIP acquisition."""
from __future__ import annotations
import argparse
import numpy as np
from python_app.hardware_full.compact_m_k209_service import CompactMK209Service
from python_app.models.run_config_model import RadarSweepModel
def _parse_args() -> argparse.Namespace:
"""Parse command-line options for the local K209 VISA smoke test."""
parser = argparse.ArgumentParser(description="Acquire one K209 sweep through VISA HiSLIP")
parser.add_argument(
"--resource",
required=True,
help=(
"S2VNA VISA resource, e.g. TCPIP0::127.0.0.1::hislip0,4880::INSTR "
"when the USB-connected K209 is controlled by local S2VNA"
),
)
parser.add_argument("--start-hz", type=float, default=1_000_000.0)
parser.add_argument("--stop-hz", type=float, default=6_000_000_000.0)
parser.add_argument("--points", type=int, default=201)
parser.add_argument("--ifbw-hz", type=float, default=50_000.0)
parser.add_argument("--power-dbm", type=float, default=-10.0)
parser.add_argument("--timeout-ms", type=int, default=20_000)
parser.add_argument("--no-preset", action="store_true")
parser.add_argument(
"--visa-library",
default="@ivi",
help="PyVISA IVI backend specification, e.g. @ivi or /usr/lib/x86_64-linux-gnu/libvisa.so",
)
return parser.parse_args()
def _validate_result(result, expected_points: int) -> None:
"""Raise RuntimeError unless the sweep has the expected point count, monotonic axis, and finite S11/S21."""
if result.x.shape != (expected_points,):
raise RuntimeError(f"Unexpected frequency shape: {result.x.shape}")
s11 = result.trace("s11")
s21 = result.trace("s21")
if s11.shape != (expected_points,) or s21.shape != (expected_points,):
raise RuntimeError(f"Unexpected trace shapes: s11={s11.shape}, s21={s21.shape}")
if not np.all(np.isfinite(result.x)):
raise RuntimeError("Frequency axis contains non-finite values")
if np.any(np.diff(result.x) < 0.0):
raise RuntimeError("Frequency axis is not monotonic")
if not np.all(np.isfinite(s11.real)) or not np.all(np.isfinite(s11.imag)):
raise RuntimeError("S11 contains non-finite values")
if not np.all(np.isfinite(s21.real)) or not np.all(np.isfinite(s21.imag)):
raise RuntimeError("S21 contains non-finite values")
def main() -> int:
"""Acquire one sweep from a locally connected K209 and validate it end to end.
Opens the VISA session, configures the sweep, validates the result, checks the SCPI
error queue, and prints a summary; raises on any validation or SCPI failure.
"""
args = _parse_args()
sweep = RadarSweepModel(
start_hz=args.start_hz,
stop_hz=args.stop_hz,
points=args.points,
if_bandwidth_hz=args.ifbw_hz,
power_dbm=args.power_dbm,
)
service = CompactMK209Service(
resource=args.resource,
timeout_ms=args.timeout_ms,
preset_on_open=not args.no_preset,
visa_library=args.visa_library,
)
try:
service.open()
print(f"K209 IDN: {service.query_identity()}")
service.configure(sweep)
result = service.acquire()
_validate_result(result, args.points)
system_error = service.query_system_error()
if not system_error.startswith("0,"):
raise RuntimeError(f"K209 SCPI error after sweep: {system_error}")
print(
"K209 sweep OK: "
f"points={result.x.size}, first_hz={result.x[0]:.3f}, last_hz={result.x[-1]:.3f}, "
f"mean_abs_s11={np.mean(np.abs(result.trace('s11'))):.6g}, "
f"mean_abs_s21={np.mean(np.abs(result.trace('s21'))):.6g}"
)
finally:
service.close()
return 0
if __name__ == "__main__":
raise SystemExit(main())
python_app/scripts/k209_sweep_benchmark.py
-217
View File
@@ -1,217 +0,0 @@
"""Measure Compact-M K209 sweep acquisition throughput through VISA HiSLIP."""
from __future__ import annotations
from dataclasses import dataclass
import statistics
import time
import numpy as np
from python_app.hardware_full.compact_m_k209_service import CompactMK209Service
from python_app.models.run_config_model import RadarSweepModel
RESOURCE = "TCPIP0::127.0.0.1::hislip0,4880::INSTR"
VISA_LIBRARY = "@ivi"
START_HZ = 100_000_000.0
STOP_HZ = 6_000_000_000.0
POINTS = 1501
IFBW_HZ = 10_000.0
POWER_DBM = -20.0
TIMEOUT_MS = 20_000
WARMUP_SWEEPS = 3
TIMED_SWEEPS = 20
PRESET_ON_OPEN = False
# False measures the device/transport hot path: one synchronized TRIG:SING/*OPC?
# message plus S11/S21 REAL32 reads.
# True also includes public SweepResult construction and complex array conversion.
INCLUDE_RESULT_CONVERSION = False
@dataclass(frozen=True, slots=True)
class BenchmarkResult:
"""Timing summary for repeated sweep acquisition."""
durations_s: list[float]
points: int
@property
def total_s(self) -> float:
"""Return total timed acquisition duration."""
return sum(self.durations_s)
@property
def sweeps_per_s(self) -> float:
"""Return completed sweeps per second."""
return len(self.durations_s) / self.total_s
@property
def points_per_s(self) -> float:
"""Return measured sweep points per second."""
return (len(self.durations_s) * self.points) / self.total_s
@property
def binary_payload_mb_per_s(self) -> float:
"""Return S11+S21 binary payload throughput, excluding SCPI headers."""
payload_bytes = len(self.durations_s) * self.points * 2 * 2 * np.dtype(np.float32).itemsize
return payload_bytes / self.total_s / 1_000_000.0
def _validate_config() -> None:
"""Raise if any module-level benchmark constant is outside its valid range."""
if POINTS < 2:
raise ValueError("POINTS must be >= 2")
if WARMUP_SWEEPS < 0:
raise ValueError("WARMUP_SWEEPS must be >= 0")
if TIMED_SWEEPS <= 0:
raise ValueError("TIMED_SWEEPS must be > 0")
if IFBW_HZ <= 0.0:
raise ValueError("IFBW_HZ must be > 0")
if STOP_HZ < START_HZ:
raise ValueError("STOP_HZ must be >= START_HZ")
if TIMEOUT_MS <= 0:
raise ValueError("TIMEOUT_MS must be > 0")
def _validate_interleaved(raw, expected_points: int) -> None:
"""Raise unless a raw interleaved sweep has the expected shapes, monotonic axis, and finite values."""
if raw.frequency_hz.shape != (expected_points,):
raise RuntimeError(f"Unexpected frequency shape: {raw.frequency_hz.shape}")
if raw.s11_values.shape != (expected_points * 2,):
raise RuntimeError(f"Unexpected S11 shape: {raw.s11_values.shape}")
if raw.s21_values.shape != (expected_points * 2,):
raise RuntimeError(f"Unexpected S21 shape: {raw.s21_values.shape}")
if not np.all(np.isfinite(raw.frequency_hz)):
raise RuntimeError("Frequency axis contains non-finite values")
if np.any(np.diff(raw.frequency_hz) < 0.0):
raise RuntimeError("Frequency axis is not monotonic")
if not np.all(np.isfinite(raw.s11_values)):
raise RuntimeError("S11 contains non-finite values")
if not np.all(np.isfinite(raw.s21_values)):
raise RuntimeError("S21 contains non-finite values")
def _validate_result(result, expected_points: int) -> None:
"""Raise unless a converted SweepResult has the expected shapes and finite S11/S21."""
if result.x.shape != (expected_points,):
raise RuntimeError(f"Unexpected frequency shape: {result.x.shape}")
for name in ("s11", "s21"):
values = result.trace(name)
if values.shape != (expected_points,):
raise RuntimeError(f"Unexpected {name.upper()} shape: {values.shape}")
if not np.all(np.isfinite(values.real)) or not np.all(np.isfinite(values.imag)):
raise RuntimeError(f"{name.upper()} contains non-finite values")
def _percentile(values: list[float], percentile: float) -> float:
"""Return the nearest-rank percentile (0..1) of ``values``."""
sorted_values = sorted(values)
index = round((len(sorted_values) - 1) * percentile)
return sorted_values[index]
def _run_benchmark(service: CompactMK209Service, *, points: int, warmup: int, sweeps: int, convert: bool) -> BenchmarkResult:
"""Time ``sweeps`` acquisitions after ``warmup`` warm-up sweeps and return their durations.
When ``convert`` is True the timed path includes SweepResult construction; otherwise it
times the raw interleaved REAL32 acquisition. The first and last sweeps are validated.
"""
acquire = service.acquire if convert else service.acquire_interleaved
first = acquire()
if convert:
_validate_result(first, points)
else:
_validate_interleaved(first, points)
for _ in range(warmup):
acquire()
durations_s: list[float] = []
for _ in range(sweeps):
start_ns = time.perf_counter_ns()
last = acquire()
end_ns = time.perf_counter_ns()
durations_s.append((end_ns - start_ns) / 1_000_000_000.0)
if convert:
_validate_result(last, points)
else:
_validate_interleaved(last, points)
return BenchmarkResult(durations_s=durations_s, points=points)
def _print_limits(limits: dict[str, float | int]) -> None:
"""Print the device frequency/IFBW/power/point limits as a single human-readable line."""
print(
"K209 limits: "
f"frequency={limits['min_frequency_hz']:.0f}..{limits['max_frequency_hz']:.0f} Hz, "
f"IFBW={limits['min_ifbw_hz']:.0f}..{limits['max_ifbw_hz']:.0f} Hz, "
f"power={limits['min_power_dbm']:.1f}..{limits['max_power_dbm']:.1f} dBm, "
f"max_points={limits['max_points']}"
)
def main() -> int:
"""Run the K209 sweep-acquisition benchmark and print timing/throughput statistics."""
_validate_config()
sweep = RadarSweepModel(
start_hz=START_HZ,
stop_hz=STOP_HZ,
points=POINTS,
if_bandwidth_hz=IFBW_HZ,
power_dbm=POWER_DBM,
)
service = CompactMK209Service(
resource=RESOURCE,
timeout_ms=TIMEOUT_MS,
preset_on_open=PRESET_ON_OPEN,
visa_library=VISA_LIBRARY,
)
try:
service.open()
print(f"K209 IDN: {service.query_identity()}")
_print_limits(service.read_device_limits())
service.configure(sweep)
print(
"Benchmark settings: "
f"start_hz={START_HZ:.3f}, stop_hz={STOP_HZ:.3f}, "
f"points={POINTS}, ifbw_hz={IFBW_HZ:.3f}, power_dbm={POWER_DBM:.3f}, "
f"warmup={WARMUP_SWEEPS}, sweeps={TIMED_SWEEPS}, "
f"mode={'SweepResult' if INCLUDE_RESULT_CONVERSION else 'raw interleaved REAL32'}"
)
result = _run_benchmark(
service,
points=POINTS,
warmup=WARMUP_SWEEPS,
sweeps=TIMED_SWEEPS,
convert=INCLUDE_RESULT_CONVERSION,
)
system_error = service.query_system_error()
if not system_error.startswith("0,"):
raise RuntimeError(f"K209 SCPI error after benchmark: {system_error}")
finally:
service.close()
durations_ms = [value * 1_000.0 for value in result.durations_s]
print("Benchmark result:")
print(f" total_s={result.total_s:.6f}")
print(f" sweep_mean_ms={statistics.fmean(durations_ms):.3f}")
print(f" sweep_median_ms={statistics.median(durations_ms):.3f}")
print(f" sweep_min_ms={min(durations_ms):.3f}")
print(f" sweep_max_ms={max(durations_ms):.3f}")
print(f" sweep_p95_ms={_percentile(durations_ms, 0.95):.3f}")
print(f" sweeps_per_s={result.sweeps_per_s:.3f}")
print(f" points_per_s={result.points_per_s:.1f}")
print(f" s11_s21_payload_mb_per_s={result.binary_payload_mb_per_s:.3f}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
python_app/scripts/kamil_adc_raw_producer.py
+48 -42
View File
@@ -3,7 +3,7 @@
from __future__ import annotations
import argparse
from contextlib import suppress
from contextlib import ExitStack, suppress
import logging
from pathlib import Path
import signal
@@ -131,42 +131,46 @@ def main() -> int:
args.config, len(config.combos), config.runtime.continuous,
)
raw_writer = ShmRingWriter(
config.rings.raw.name,
config.rings.raw.capacity,
config.rings.raw.slot_size_bytes,
)
raw_tap_writer = ShmRingWriter(
config.rings.raw_tap.name,
config.rings.raw_tap.capacity,
config.rings.raw_tap.slot_size_bytes,
)
logger.debug(
"Opened SHM ring writers: raw=%s, raw_tap=%s",
config.rings.raw.name, config.rings.raw_tap.name,
)
# Switch-aware mode: with native switches the collector drives the RF switches
# itself, in the hardware gap between sweeps, and tags each sweep with its combo
# — no sweep lost at a switch boundary. The producer then only reads tagged
# sweeps and must not touch the GPIO lines the collector owns. With mock switches
# (dev/tests) we keep the Python-driven path, which is fine where speed and the
# in-gap timing do not matter.
collector_driven = (
config.input_switch.driver_mode == "native"
and config.output_switch.driver_mode == "native"
)
radar = KamilAdcService(
config,
switch_config_path=str(args.config) if collector_driven else None,
)
input_switch = None if collector_driven else SwitchService.from_model(config.input_switch)
output_switch = None if collector_driven else SwitchService.from_model(config.output_switch)
logger.info(
"Kamil ADC switch control: %s",
"collector-driven (in-gap, lossless)" if collector_driven else "producer-driven",
)
radar = input_switch = output_switch = None
cleanup = ExitStack()
try:
raw_writer = ShmRingWriter(
config.rings.raw.name,
config.rings.raw.capacity,
config.rings.raw.slot_size_bytes,
)
cleanup.callback(raw_writer.close)
raw_tap_writer = ShmRingWriter(
config.rings.raw_tap.name,
config.rings.raw_tap.capacity,
config.rings.raw_tap.slot_size_bytes,
)
cleanup.callback(raw_tap_writer.close)
logger.debug(
"Opened SHM ring writers: raw=%s, raw_tap=%s",
config.rings.raw.name, config.rings.raw_tap.name,
)
# Switch-aware mode: with native switches the collector drives the RF switches
# itself, in the hardware gap between sweeps, and tags each sweep with its combo
# — no sweep lost at a switch boundary. The producer then only reads tagged
# sweeps and must not touch the GPIO lines the collector owns. With mock switches
# (dev/tests) we keep the Python-driven path, which is fine where speed and the
# in-gap timing do not matter.
collector_driven = (
config.input_switch.driver_mode == "native"
and config.output_switch.driver_mode == "native"
)
radar = KamilAdcService(
config,
switch_config_path=str(args.config) if collector_driven else None,
)
input_switch = None if collector_driven else SwitchService.from_model(config.input_switch)
output_switch = None if collector_driven else SwitchService.from_model(config.output_switch)
logger.info(
"Kamil ADC switch control: %s",
"collector-driven (in-gap, lossless)" if collector_driven else "producer-driven",
)
if not _open_radar_with_retry(config, radar, input_switch, output_switch, stop_requested):
return 0 # asked to stop before a device became available
@@ -187,10 +191,12 @@ def main() -> int:
sweep_start_ns = time.monotonic_ns()
sweep = radar.acquire(combo=(combo.input, combo.output))
else:
if input_switch is None or output_switch is None:
raise RuntimeError("Producer-driven switches are not initialized")
output_switch.switch_to(combo.output)
input_switch.switch_to(combo.input)
if config.runtime.settling_ms > 0:
time.sleep(config.runtime.settling_ms / 1000.0)
if config.runtime.settling_ms > 0 and stop_requested.wait(config.runtime.settling_ms / 1000.0):
break
# Stamped after switching and settling so the window covers
# the sweep alone, not the dead time before it.
sweep_start_ns = time.monotonic_ns()
@@ -264,10 +270,10 @@ def main() -> int:
if input_switch is not None:
with suppress(Exception):
input_switch.close()
with suppress(Exception):
radar.close()
raw_tap_writer.close()
raw_writer.close()
if radar is not None:
with suppress(Exception):
radar.close()
cleanup.close()
logger.info("Kamil ADC raw producer stopped")
return 0
python_app/scripts/matrix_raw_producer.py
+23 -26
View File
@@ -1,16 +1,16 @@
"""Raw acquisition producer for matrix-mode radars (LibreVNA multi-device, SN9000)."""
"""Raw producer for native matrix radars and the shared single/multi mock scene."""
from __future__ import annotations
import argparse
import logging
from contextlib import suppress
from contextlib import ExitStack, suppress
from pathlib import Path
import signal
import threading
import time
from python_app.hardware_full.matrix_radar_service import MatrixRadarService, create_matrix_radar_service
from python_app.hardware_full.matrix_radar_service import MatrixRadarService, create_acquisition_service
from python_app.logging_setup import coerce_level
from python_app.models.run_config_model import RunConfigModel
from python_app.orchestration.shm import ShmRingWriter
@@ -48,7 +48,7 @@ def _open_radar_with_retry(
attempt = 0
delay = _OPEN_RETRY_MIN_S
while not stop_requested.is_set():
radar = create_matrix_radar_service(config)
radar = create_acquisition_service(config)
try:
radar.open()
radar.configure(config.radar.sweep)
@@ -100,33 +100,31 @@ def main() -> int:
# turned on from the profile instead of requiring a code edit. basicConfig above
# only installed the handler; the package logger owns the level.
logging.getLogger("python_app").setLevel(coerce_level(config.logging.level))
if not config.is_matrix_radar:
raise RuntimeError(
"matrix_raw_producer requires a matrix-mode radar.model "
"(librevna_multi or sn9000)"
)
logger.info(
"Matrix radar raw producer starting: config=%s, model=%s, continuous=%s",
args.config, config.radar.model, config.runtime.continuous,
)
raw_writer = ShmRingWriter(
config.rings.raw.name,
config.rings.raw.capacity,
config.rings.raw.slot_size_bytes,
)
raw_tap_writer = ShmRingWriter(
config.rings.raw_tap.name,
config.rings.raw_tap.capacity,
config.rings.raw_tap.slot_size_bytes,
)
logger.debug(
"Opened SHM ring writers: raw=%s, raw_tap=%s",
config.rings.raw.name, config.rings.raw_tap.name,
)
radar: MatrixRadarService | None = None
cleanup = ExitStack()
try:
raw_writer = ShmRingWriter(
config.rings.raw.name,
config.rings.raw.capacity,
config.rings.raw.slot_size_bytes,
)
cleanup.callback(raw_writer.close)
raw_tap_writer = ShmRingWriter(
config.rings.raw_tap.name,
config.rings.raw_tap.capacity,
config.rings.raw_tap.slot_size_bytes,
)
cleanup.callback(raw_tap_writer.close)
logger.debug(
"Opened SHM ring writers: raw=%s, raw_tap=%s",
config.rings.raw.name, config.rings.raw_tap.name,
)
radar = _open_radar_with_retry(config, previous=None, stop_requested=stop_requested)
if radar is None:
return 0 # asked to stop before a device became available
@@ -174,8 +172,7 @@ def main() -> int:
if radar is not None:
with suppress(Exception):
radar.close()
raw_tap_writer.close()
raw_writer.close()
cleanup.close()
logger.info("matrix radar raw producer stopped")
return 0
python_app/scripts/replay_raw_producer.py
+38 -33
View File
@@ -20,6 +20,7 @@ never see time jump backwards. ``collection_id`` increases forever.
from __future__ import annotations
import argparse
from contextlib import ExitStack
import json
import logging
from pathlib import Path
@@ -39,7 +40,7 @@ from python_app.storage.npz.serialize import RAW_MAGIC, serialize_trace_collecti
logger = logging.getLogger(__name__)
_SNAPSHOT_NAME = re.compile(r"^\d+_id\d+_ns(\d+)$")
_SNAPSHOT_NAME = re.compile(r"^(\d+)_id\d+_ns(\d+)$")
class ReplayDataset:
@@ -62,11 +63,11 @@ def _load_dataset(dataset_dir: Path, combos: list[ComboKey], expected_points: in
if not dataset_dir.is_dir():
raise RuntimeError(f"radar.replay.dataset_dir is not a directory: {dataset_dir}")
named: list[tuple[str, int]] = []
for entry in sorted(dataset_dir.iterdir()):
named: list[tuple[int, str, int]] = []
for entry in dataset_dir.iterdir():
match = _SNAPSHOT_NAME.match(entry.name)
if entry.is_dir() and match:
named.append((entry.name, int(match.group(1))))
named.append((int(match.group(1)), entry.name, int(match.group(2))))
if not named:
raise RuntimeError(
f"No snapshot folders (NNNN_idK_ns<ts>) found in {dataset_dir}"
@@ -74,7 +75,7 @@ def _load_dataset(dataset_dir: Path, combos: list[ComboKey], expected_points: in
snapshots: list[list[TraceData]] = []
timestamps_ns: list[int] = []
for name, timestamp_ns in named:
for _, name, timestamp_ns in sorted(named):
folder = dataset_dir / name
# Recorded per-trace capture windows from the snapshot's meta.json: the
# sar3d segmented timing model reconstructs the acquisition segments
@@ -102,10 +103,12 @@ def _load_dataset(dataset_dir: Path, combos: list[ComboKey], expected_points: in
f"input={combo.input} output={combo.output}: expected "
f"{s21_path.name} and {freq_path.name}. Check run.combos vs the dataset."
)
s21 = np.load(s21_path).astype(np.complex64, copy=False)
freq = np.load(freq_path).astype(np.float32, copy=False)
if s21.size != freq.size:
raise RuntimeError(f"{s21_path}: s21 size {s21.size} != freq size {freq.size}")
s21 = np.load(s21_path, allow_pickle=False).astype(np.complex64, copy=False)
freq = np.load(freq_path, allow_pickle=False).astype(np.float32, copy=False)
s11_path = folder / f"i{combo.input}_o{combo.output}_s11.npy"
s11 = np.load(s11_path, allow_pickle=False).astype(np.complex64, copy=False) if s11_path.is_file() else np.zeros_like(s21)
if freq.ndim != 1 or s21.shape != freq.shape or s11.shape != freq.shape:
raise RuntimeError(f"Snapshot {name}: frequency, S11 and S21 must be equally sized 1D arrays")
if expected_points and s21.size != expected_points:
raise RuntimeError(
f"{s21_path}: {s21.size} points, but radar.sweep.points = "
@@ -116,7 +119,7 @@ def _load_dataset(dataset_dir: Path, combos: list[ComboKey], expected_points: in
TraceData(
combo=combo,
frequency_hz=freq,
s11=np.zeros_like(s21),
s11=s11,
s21=s21,
capture_start_ns=window[0],
capture_end_ns=window[1],
@@ -206,25 +209,34 @@ def main() -> int:
dataset.median_gap_ns / 1e9, replay.loop, rate_scale,
)
raw_writer = ShmRingWriter(
config.rings.raw.name,
config.rings.raw.capacity,
config.rings.raw.slot_size_bytes,
)
raw_tap_writer = ShmRingWriter(
config.rings.raw_tap.name,
config.rings.raw_tap.capacity,
config.rings.raw_tap.slot_size_bytes,
)
cleanup = ExitStack()
try:
raw_writer = ShmRingWriter(
config.rings.raw.name,
config.rings.raw.capacity,
config.rings.raw.slot_size_bytes,
)
cleanup.callback(raw_writer.close)
raw_tap_writer = ShmRingWriter(
config.rings.raw_tap.name,
config.rings.raw_tap.capacity,
config.rings.raw_tap.slot_size_bytes,
)
cleanup.callback(raw_tap_writer.close)
payload_size = len(serialize_trace_collection(
_build_collection(dataset, 0, 1, time.monotonic_ns()), RAW_MAGIC
))
for ring in (config.rings.raw, config.rings.raw_tap):
if ring.slot_size_bytes < payload_size:
raise ValueError(f"Replay requires {payload_size} bytes per slot; ring {ring.name} has {ring.slot_size_bytes}")
base_ns = time.monotonic_ns()
wall_start_s = time.monotonic()
first_ts = dataset.timestamps_ns[0]
cycle_span_ns = dataset.timestamps_ns[-1] - first_ts + dataset.median_gap_ns
collection_id = 1
publish_failures = 0
cycle = 0
while not stop_requested.is_set():
for index in range(len(dataset)):
@@ -246,15 +258,9 @@ def main() -> int:
collection = _build_collection(dataset, index, collection_id, frame_ns)
payload = serialize_trace_collection(collection, RAW_MAGIC)
if raw_writer.push(payload):
raw_tap_writer.push(payload) # best-effort GUI tap; never fatal
else:
publish_failures += 1
if publish_failures == 1 or publish_failures % 100 == 0:
logger.error(
"Replay payload %d B exceeds ring slot %d B; dropping collection %d (drops=%d)",
len(payload), raw_writer.slot_size_bytes, collection_id, publish_failures,
)
if not raw_writer.push(payload):
raise RuntimeError(f"Replay raw ring rejected collection {collection_id}")
raw_tap_writer.push(payload)
if collection_id == 1 or collection_id % 50 == 0:
logger.info(
"replay collection %d published (snapshot %d/%d, cycle %d)",
@@ -268,8 +274,7 @@ def main() -> int:
break
cycle += 1
finally:
raw_tap_writer.close()
raw_writer.close()
cleanup.close()
logger.info("replay raw producer stopped")
return 0
python_app/scripts/sn9000_smoke_test.py
-131
View File
@@ -1,131 +0,0 @@
"""Standalone smoke test for the SN9000 multi-port VISA HiSLIP acquisition path."""
from __future__ import annotations
import argparse
import numpy as np
from python_app.hardware_full.sn9000_service import Sn9000Service
from python_app.models.dataset_model import SweepCollection
from python_app.models.run_config_model import RadarSweepModel, RunConfigModel
def _parse_args() -> argparse.Namespace:
"""Parse command-line options for the SN9000 VISA smoke test."""
parser = argparse.ArgumentParser(description="Acquire one SN9000 collection through VISA HiSLIP")
parser.add_argument(
"--host",
default="127.0.0.1",
help="SNVNA HiSLIP server host (default: 127.0.0.1)",
)
parser.add_argument(
"--port",
type=int,
default=4880,
help="SNVNA HiSLIP server TCP port (default: 4880)",
)
parser.add_argument("--start-hz", type=float, default=1_000_000.0)
parser.add_argument("--stop-hz", type=float, default=3_000_000_000.0)
parser.add_argument("--points", type=int, default=201)
parser.add_argument("--ifbw-hz", type=float, default=10_000.0)
parser.add_argument("--power-dbm", type=float, default=-10.0)
parser.add_argument("--timeout-ms", type=int, default=20_000)
parser.add_argument("--no-preset", action="store_true")
parser.add_argument(
"--visa-library",
default="@ivi",
help="PyVISA IVI backend specification, e.g. @ivi or /usr/lib/x86_64-linux-gnu/libvisa.so",
)
return parser.parse_args()
def _validate_collection(collection: SweepCollection, expected_points: int) -> None:
"""Raise unless a SN9000 collection has the expected traces, combos, shapes, and finite values."""
expected_traces = (
RunConfigModel.MULTI_DEVICE_INPUT_POSITIONS * RunConfigModel.MULTI_DEVICE_OUTPUT_POSITIONS
)
if len(collection.traces) != expected_traces:
raise RuntimeError(
f"SN9000 collection has {len(collection.traces)} traces, expected {expected_traces}"
)
reference_frequency = collection.traces[0].frequency_hz
if reference_frequency.shape != (expected_points,):
raise RuntimeError(f"Unexpected frequency shape: {reference_frequency.shape}")
if not np.all(np.isfinite(reference_frequency)):
raise RuntimeError("Frequency axis contains non-finite values")
if np.any(np.diff(reference_frequency) < 0.0):
raise RuntimeError("Frequency axis is not monotonic")
expected_combos = RunConfigModel.build_matrix_radar_virtual_combos()
for trace, expected_combo in zip(collection.traces, expected_combos, strict=True):
if (
int(trace.combo.input) != int(expected_combo.input)
or int(trace.combo.output) != int(expected_combo.output)
):
raise RuntimeError(
f"Trace combo mismatch: got ({trace.combo.input},{trace.combo.output}), "
f"expected ({expected_combo.input},{expected_combo.output})"
)
if trace.s11.shape != (expected_points,) or trace.s21.shape != (expected_points,):
raise RuntimeError(
f"Trace ({trace.combo.input},{trace.combo.output}) has shapes "
f"s11={trace.s11.shape}, s21={trace.s21.shape}"
)
if not np.all(np.isfinite(trace.s11.real)) or not np.all(np.isfinite(trace.s11.imag)):
raise RuntimeError(
f"Trace ({trace.combo.input},{trace.combo.output}) S11 contains non-finite values"
)
if not np.all(np.isfinite(trace.s21.real)) or not np.all(np.isfinite(trace.s21.imag)):
raise RuntimeError(
f"Trace ({trace.combo.input},{trace.combo.output}) S21 contains non-finite values"
)
def main() -> int:
"""Acquire one SN9000 collection and validate it end to end.
Opens the VISA session, configures the sweep, validates the collection, checks the SCPI
error queue, and prints a summary; raises on any validation or SCPI failure.
"""
args = _parse_args()
sweep = RadarSweepModel(
start_hz=args.start_hz,
stop_hz=args.stop_hz,
points=args.points,
if_bandwidth_hz=args.ifbw_hz,
power_dbm=args.power_dbm,
)
service = Sn9000Service(
host=args.host,
port=args.port,
timeout_ms=args.timeout_ms,
preset_on_open=not args.no_preset,
visa_library=args.visa_library,
)
try:
service.open()
print(f"SN9000 IDN: {service.query_identity()}")
service.configure(sweep)
collection = service.acquire_collection(collection_id=1)
_validate_collection(collection, args.points)
system_error = service.query_system_error()
if not system_error.startswith("0,"):
raise RuntimeError(f"SN9000 SCPI error after sweep: {system_error}")
mean_abs_s21 = np.mean([float(np.mean(np.abs(trace.s21))) for trace in collection.traces])
print(
"SN9000 collection OK: "
f"traces={len(collection.traces)}, points={args.points}, "
f"first_hz={collection.traces[0].frequency_hz[0]:.3f}, "
f"last_hz={collection.traces[0].frequency_hz[-1]:.3f}, "
f"mean_abs_s21={mean_abs_s21:.6g}"
)
finally:
service.close()
return 0
if __name__ == "__main__":
raise SystemExit(main())