improved logging

This commit is contained in:
Ayzen
2026-06-06 00:52:52 +03:00
parent af6005d68f
commit aea49f6128
65 changed files with 1206 additions and 240 deletions
@@ -34,6 +34,7 @@ S21_ONLY_TARGET_KINDS = {"s21_calibration", "s21_reference"}
def _build_parser() -> argparse.ArgumentParser:
"""Return the argument parser for the legacy preprocess-set conversion CLI."""
parser = argparse.ArgumentParser(
description=(
"Convert old preprocess-set storage from a legacy python_app/data tree into the "
@@ -59,6 +60,7 @@ def _build_parser() -> argparse.ArgumentParser:
def _load_json(path: Path) -> dict[str, Any]:
"""Read a JSON file and return its top-level object, rejecting non-object roots."""
payload = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(payload, dict):
raise ValueError(f"JSON root must be object: {path}")
@@ -66,6 +68,7 @@ def _load_json(path: Path) -> dict[str, Any]:
def _read_combo_position(combo_payload: dict[str, Any], *, primary_key: str, alias_key: str) -> int:
"""Return a switch position from a combo record, accepting the primary or alias key."""
if primary_key in combo_payload:
return int(combo_payload[primary_key])
if alias_key in combo_payload:
@@ -74,16 +77,23 @@ def _read_combo_position(combo_payload: dict[str, Any], *, primary_key: str, ali
def _combo_suffix(input_pos: int, output_pos: int) -> str:
"""Return the per-combo array-name suffix (e.g. ``i0_o1``) used in legacy NPZ keys."""
return f"i{input_pos}_o{output_pos}"
def _load_array(arrays: Any, key: str, *, dtype: np.dtype[Any], label: str) -> np.ndarray:
"""Return a flattened array of the given dtype from an NPZ mapping, by key."""
if key not in arrays:
raise KeyError(f"Missing {label} array '{key}' in NPZ archive")
return np.asarray(arrays[key], dtype=dtype).reshape(-1)
def _load_legacy_collection(meta_path: Path, npz_path: Path, *, target_kind: str) -> SweepCollection:
"""Build a SweepCollection from a legacy meta/NPZ pair for the given target kind.
Reads per-combo frequency, S21 and (when present) S11 arrays. For S21-only target
kinds a missing S11 is filled with zeros; for any other kind a missing S11 is an error.
"""
meta = _load_json(meta_path)
combos_payload = meta.get("combos")
if not isinstance(combos_payload, list):
@@ -148,6 +158,11 @@ def _convert_one_set(
meta_path: Path,
overwrite: bool,
) -> None:
"""Convert a single legacy set (meta + NPZ) and save it under the target kind.
Raises if the companion NPZ is missing, or if the destination already exists and
``overwrite`` is False.
"""
set_name = meta_path.stem
npz_path = meta_path.with_suffix(".npz")
if not npz_path.exists():
@@ -168,6 +183,11 @@ def _convert_one_set(
def main() -> int:
"""Walk the legacy data tree, convert every recognized set, and print a summary.
Returns 0 on full success, 1 if any set failed to convert, or 2 if no convertible
sets were found.
"""
parser = _build_parser()
args = parser.parse_args()
@@ -13,10 +13,12 @@ import numpy as np
def _real_imag_keys(trace_prefix: str) -> tuple[str, str]:
"""Return the LibreVNA CSV column names for a trace's real and imaginary parts."""
return f"{trace_prefix}_Real", f"{trace_prefix}_Imaginary"
def _load_complex_trace(csv_path: Path, trace_prefix: str) -> tuple[np.ndarray, np.ndarray]:
"""Load one LibreVNA CSV and return its (frequency_hz, complex trace) arrays."""
real_key, imag_key = _real_imag_keys(trace_prefix)
frequencies: list[float] = []
values: list[complex] = []
@@ -62,6 +64,7 @@ def _apply_one_port_osl(
source_match: np.ndarray,
reflection_tracking: np.ndarray,
) -> np.ndarray:
"""Apply OSL one-port error correction to a measured S11 trace and return it."""
numerator = measured_trace - directivity
denominator = reflection_tracking + (source_match * numerator)
@@ -72,6 +75,7 @@ def _apply_one_port_osl(
def _apply_through_calibration(measured_trace: np.ndarray, through_trace: np.ndarray) -> np.ndarray:
"""Normalize a measured S21 trace by the through reference and return it."""
corrected = np.array(measured_trace, copy=True)
stable_mask = np.abs(through_trace) > 1e-18
corrected[stable_mask] = measured_trace[stable_mask] / through_trace[stable_mask]
@@ -79,15 +83,21 @@ def _apply_through_calibration(measured_trace: np.ndarray, through_trace: np.nda
def _complex_to_points(values: np.ndarray) -> list[list[float]]:
"""Return complex samples as ``[real, imag]`` pairs for JSON serialization."""
return [[float(value.real), float(value.imag)] for value in values]
def _scan_file_sort_key(csv_path: Path) -> tuple[int, str]:
"""Return a sort key ordering numeric scan filenames first, by integer value."""
stem = csv_path.stem
return (int(stem), stem) if stem.isdigit() else (10**9, stem)
def _load_scan_series(folder: Path, trace_prefix: str) -> tuple[np.ndarray, list[tuple[str, np.ndarray]]]:
"""Load every numbered scan CSV in a folder and return (frequency_hz, named traces).
All scans must share a common frequency axis; a mismatch raises ValueError.
"""
scan_paths = [
path
for path in sorted(folder.glob("*.csv"), key=_scan_file_sort_key)
@@ -111,6 +121,7 @@ def _load_scan_series(folder: Path, trace_prefix: str) -> tuple[np.ndarray, list
def _require_matching_frequency_axis(label: str, left: np.ndarray, right: np.ndarray) -> None:
"""Raise ValueError (tagged with ``label``) unless two frequency axes match within tolerance."""
if not np.allclose(left, right, rtol=0.0, atol=1e-6):
raise ValueError(f"{label} frequency axes do not match")
@@ -127,6 +138,11 @@ def _build_history_payload(
raw_record_count: int,
preprocessed_record_count: int,
) -> dict[str, Any]:
"""Assemble the vna-system history payload from paired sweep and calibrated scans.
Pairs each sweep scan with its calibrated counterpart by order (names must match),
attaching the shared reference trace and sweep config to every history entry.
"""
if len(sweep_scans) != len(calibrated_scans):
raise ValueError("Sweep/calibrated scan counts do not match")
@@ -168,15 +184,18 @@ def _build_history_payload(
def _write_payload(output_path: Path, payload: dict[str, Any]) -> None:
"""Write a payload as indented UTF-8 JSON, creating parent directories as needed."""
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
def _rmse(left: np.ndarray, right: np.ndarray) -> float:
"""Return the root-mean-square magnitude difference between two complex traces."""
return float(np.sqrt(np.mean(np.abs(left - right) ** 2)))
def _build_parser() -> argparse.ArgumentParser:
"""Return the argument parser for the prog_libre-to-vna-history conversion CLI."""
parser = argparse.ArgumentParser(
description="Convert manual LibreVNA S11/S21 CSV captures in prog_libre to vna history JSON.",
)
@@ -232,6 +251,12 @@ def _build_parser() -> argparse.ArgumentParser:
def main() -> None:
"""Calibrate and convert the prog_libre S11/S21 CSV captures into history JSON files.
Builds OSL coefficients for S11 and a through reference for S21, applies them to the
uncalibrated scans, writes raw and pre-calibrated history files for both channels, and
prints RMSE diagnostics against the supplied calibrated folder.
"""
args = _build_parser().parse_args()
calibration_dir = args.calibration_dir.expanduser().resolve()
@@ -35,6 +35,7 @@ class CollectionRef:
def _load_json(path: Path) -> dict[str, Any]:
"""Read a JSON file and return its top-level object, rejecting non-object roots."""
payload = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(payload, dict):
raise ValueError(f"JSON root must be object: {path}")
@@ -42,17 +43,20 @@ def _load_json(path: Path) -> dict[str, Any]:
def _collection_dirs(stage_dir: Path) -> list[Path]:
"""Return the stage's collection subdirectories sorted by name (empty if absent)."""
if not stage_dir.exists():
return []
return sorted([path for path in stage_dir.iterdir() if path.is_dir()], key=lambda path: path.name)
def _parse_stage_index(name: str, fallback: int) -> int:
"""Return the leading numeric prefix of a collection dir name, or ``fallback``."""
prefix = name.split("_", 1)[0]
return int(prefix) if prefix.isdigit() else fallback
def _pick_trace_meta(meta: dict[str, Any], input_index: int, output_index: int) -> dict[str, Any] | None:
"""Return the trace record matching the given input/output switch indices, or None."""
traces = meta.get("traces", [])
if not isinstance(traces, list):
return None
@@ -65,6 +69,7 @@ def _pick_trace_meta(meta: dict[str, Any], input_index: int, output_index: int)
def _normalize_channel(channel: str) -> str:
"""Return a lowercased channel name, accepting only ``s21`` or ``s11``."""
normalized = str(channel).strip().lower()
if normalized not in {"s21", "s11"}:
raise ValueError("channel must be either 's21' or 's11'")
@@ -79,6 +84,11 @@ def _load_stage_records(
output_index: int,
channel: str,
) -> list[TraceRecord]:
"""Load TraceRecords for one stage and one switch combo, for the given channel.
Skips collections without a matching trace or array files; raises on shape mismatch
or non-finite samples.
"""
stage_dir = snapshot_dir / stage
records: list[TraceRecord] = []
@@ -145,6 +155,11 @@ def _load_stage_refs(snapshot_dir: Path, stage: str) -> list[CollectionRef]:
def _index_by_collection_occurrence(records: list[TraceRecord]) -> tuple[dict[tuple[int, int], TraceRecord], list[tuple[int, int]]]:
"""Key records by (collection_id, occurrence) and return the map plus original order.
The occurrence counter disambiguates repeated collection ids, so raw and preprocessed
stages can be aligned slot-for-slot even when ids recur.
"""
counters: defaultdict[int, int] = defaultdict(int)
record_map: dict[tuple[int, int], TraceRecord] = {}
order: list[tuple[int, int]] = []
@@ -158,6 +173,7 @@ def _index_by_collection_occurrence(records: list[TraceRecord]) -> tuple[dict[tu
def _complex_to_points(values: np.ndarray) -> list[list[float]]:
"""Return complex samples as ``[real, imag]`` pairs for JSON serialization."""
return [[float(v.real), float(v.imag)] for v in values]
@@ -168,6 +184,12 @@ def _build_sweep_history(
channel: str,
primary_stage: str,
) -> list[dict[str, Any]]:
"""Merge raw and preprocessed records into vna-system ``sweep_history`` entries.
Iterates collections in the primary stage's order, pairing each with its counterpart
in the other stage; the raw samples become ``sweep_points`` and the preprocessed
samples ``calibrated_points`` (falling back to whichever stage is present).
"""
raw_map, raw_order = _index_by_collection_occurrence(raw_records)
pre_map, pre_order = _index_by_collection_occurrence(preprocessed_records)
@@ -250,6 +272,7 @@ def _stage_alignment_warning(pre_refs: list[CollectionRef], result_refs: list[Co
def _build_parser() -> argparse.ArgumentParser:
"""Return the argument parser for the snapshot-to-vna-history conversion CLI."""
parser = argparse.ArgumentParser(
description=(
"Convert radar_system snapshot (numpy-directory-v1) to a vna_system-compatible "
@@ -288,6 +311,12 @@ def _build_parser() -> argparse.ArgumentParser:
def main() -> None:
"""Convert one snapshot's chosen channel/combo into a vna-system history JSON file.
Loads raw and preprocessed traces, builds the sweep history (optionally trimmed to the
last N sweeps), writes the output JSON, and prints a summary plus any stage-alignment
warning.
"""
parser = _build_parser()
args = parser.parse_args()
channel = _normalize_channel(args.channel)
+10
View File
@@ -36,6 +36,7 @@ class K209RemoteRequestHandler(socketserver.StreamRequestHandler):
"""Handle one persistent K209 remote client connection."""
def setup(self) -> None:
"""Disable Nagle and open the local K209 VISA session for this connection."""
super().setup()
self.request.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.service = CompactMK209Service(
@@ -47,12 +48,14 @@ class K209RemoteRequestHandler(socketserver.StreamRequestHandler):
self.service.open()
def finish(self) -> None:
"""Close the K209 VISA session when the client disconnects."""
try:
self.service.close()
finally:
super().finish()
def handle(self) -> None:
"""Serve command bytes from the client until EOF, reporting errors back inline."""
while True:
command = self.rfile.read(1)
if not command:
@@ -74,12 +77,14 @@ class K209RemoteRequestHandler(socketserver.StreamRequestHandler):
self.wfile.flush()
def _handle_identity(self) -> None:
"""Reply with the device identity string (length-prefixed UTF-8)."""
payload = self.service.query_identity().encode("utf-8")
self.wfile.write(STATUS_OK)
send_u32(self.wfile, len(payload))
self.wfile.write(payload)
def _handle_limits(self) -> None:
"""Reply with the device frequency/IFBW/power/point limits as a packed struct."""
limits = self.service.read_device_limits()
self.wfile.write(STATUS_OK)
self.wfile.write(
@@ -95,6 +100,7 @@ class K209RemoteRequestHandler(socketserver.StreamRequestHandler):
)
def _handle_configure(self) -> None:
"""Apply a sweep config from the client and reply with the frequency axis."""
start_hz, stop_hz, points, ifbw_hz, power_dbm = CONFIG_STRUCT.unpack(
recv_exact(self.rfile, CONFIG_STRUCT.size)
)
@@ -111,6 +117,7 @@ class K209RemoteRequestHandler(socketserver.StreamRequestHandler):
send_float32_array(self.wfile, self.service.frequency_axis())
def _handle_acquire(self) -> None:
"""Acquire one interleaved sweep and reply with the S11 and S21 arrays."""
sweep = self.service.acquire_interleaved()
self.wfile.write(STATUS_OK)
send_u32(self.wfile, int(sweep.frequency_hz.size))
@@ -124,12 +131,14 @@ class K209RemoteServer(socketserver.TCPServer):
allow_reuse_address = True
def __init__(self, server_address: tuple[str, int], resource: str, timeout_ms: int) -> None:
"""Store the VISA resource and timeout used by each accepted connection."""
self.resource = resource
self.timeout_ms = timeout_ms
super().__init__(server_address, K209RemoteRequestHandler)
def _parse_args() -> argparse.Namespace:
"""Parse command-line options for the K209 remote server."""
parser = argparse.ArgumentParser(description="Serve a locally connected Compact-M K209 over TCP.")
parser.add_argument("--host", default="0.0.0.0", help="Server bind address.")
parser.add_argument("--port", type=int, default=DEFAULT_REMOTE_PORT, help="Server TCP port.")
@@ -139,6 +148,7 @@ def _parse_args() -> argparse.Namespace:
def main() -> int:
"""Bind the K209 remote server and serve clients until interrupted."""
args = _parse_args()
with K209RemoteServer((args.host, args.port), resource=args.resource, timeout_ms=args.timeout_ms) as server:
print(f"K209 remote server listening on {args.host}:{args.port}")
@@ -18,6 +18,7 @@ from python_app.models.run_config_model import RadarSweepModel
def _parse_args() -> argparse.Namespace:
"""Parse command-line options for the remote K209 smoke test."""
parser = argparse.ArgumentParser(description="Validate remote K209 connection and one sweep.")
parser.add_argument("--host", default=DEFAULT_REMOTE_HOST, help="K209 remote server host.")
parser.add_argument("--port", type=int, default=DEFAULT_REMOTE_PORT, help="K209 remote server port.")
@@ -30,6 +31,11 @@ def _parse_args() -> argparse.Namespace:
def main() -> int:
"""Connect to a remote K209, run one sweep, and validate its shape and values.
Raises on an unexpected point count, non-finite samples, or a non-monotonic frequency
axis; prints a one-line summary on success.
"""
args = _parse_args()
sweep = RadarSweepModel(
start_hz=args.start_hz,
+7
View File
@@ -11,6 +11,7 @@ from python_app.models.run_config_model import RadarSweepModel
def _parse_args() -> argparse.Namespace:
"""Parse command-line options for the local K209 VISA smoke test."""
parser = argparse.ArgumentParser(description="Acquire one K209 sweep through VISA HiSLIP")
parser.add_argument(
"--resource",
@@ -36,6 +37,7 @@ def _parse_args() -> argparse.Namespace:
def _validate_result(result, expected_points: int) -> None:
"""Raise RuntimeError unless the sweep has the expected point count, monotonic axis, and finite S11/S21."""
if result.x.shape != (expected_points,):
raise RuntimeError(f"Unexpected frequency shape: {result.x.shape}")
s11 = result.trace("s11")
@@ -53,6 +55,11 @@ def _validate_result(result, expected_points: int) -> None:
def main() -> int:
"""Acquire one sweep from a locally connected K209 and validate it end to end.
Opens the VISA session, configures the sweep, validates the result, checks the SCPI
error queue, and prints a summary; raises on any validation or SCPI failure.
"""
args = _parse_args()
sweep = RadarSweepModel(
start_hz=args.start_hz,
@@ -62,6 +62,7 @@ class BenchmarkResult:
def _validate_config() -> None:
"""Raise if any module-level benchmark constant is outside its valid range."""
if POINTS < 2:
raise ValueError("POINTS must be >= 2")
if WARMUP_SWEEPS < 0:
@@ -77,6 +78,7 @@ def _validate_config() -> None:
def _validate_interleaved(raw, expected_points: int) -> None:
"""Raise unless a raw interleaved sweep has the expected shapes, monotonic axis, and finite values."""
if raw.frequency_hz.shape != (expected_points,):
raise RuntimeError(f"Unexpected frequency shape: {raw.frequency_hz.shape}")
if raw.s11_values.shape != (expected_points * 2,):
@@ -94,6 +96,7 @@ def _validate_interleaved(raw, expected_points: int) -> None:
def _validate_result(result, expected_points: int) -> None:
"""Raise unless a converted SweepResult has the expected shapes and finite S11/S21."""
if result.x.shape != (expected_points,):
raise RuntimeError(f"Unexpected frequency shape: {result.x.shape}")
for name in ("s11", "s21"):
@@ -105,12 +108,18 @@ def _validate_result(result, expected_points: int) -> None:
def _percentile(values: list[float], percentile: float) -> float:
"""Return the nearest-rank percentile (0..1) of ``values``."""
sorted_values = sorted(values)
index = round((len(sorted_values) - 1) * percentile)
return sorted_values[index]
def _run_benchmark(service: CompactMK209Service, *, points: int, warmup: int, sweeps: int, convert: bool) -> BenchmarkResult:
"""Time ``sweeps`` acquisitions after ``warmup`` warm-up sweeps and return their durations.
When ``convert`` is True the timed path includes SweepResult construction; otherwise it
times the raw interleaved REAL32 acquisition. The first and last sweeps are validated.
"""
acquire = service.acquire if convert else service.acquire_interleaved
first = acquire()
@@ -137,6 +146,7 @@ def _run_benchmark(service: CompactMK209Service, *, points: int, warmup: int, sw
def _print_limits(limits: dict[str, float | int]) -> None:
"""Print the device frequency/IFBW/power/point limits as a single human-readable line."""
print(
"K209 limits: "
f"frequency={limits['min_frequency_hz']:.0f}..{limits['max_frequency_hz']:.0f} Hz, "
@@ -147,6 +157,7 @@ def _print_limits(limits: dict[str, float | int]) -> None:
def main() -> int:
"""Run the K209 sweep-acquisition benchmark and print timing/throughput statistics."""
_validate_config()
sweep = RadarSweepModel(
@@ -94,6 +94,7 @@ def _open_radar_with_retry(
logger.info("Kamil ADC opened after %d attempt(s).", attempt + 1)
return True
logger.debug("Stop requested before the Kamil ADC became available.")
return False
@@ -116,6 +117,10 @@ def main() -> int:
if not config.is_kamil_adc:
raise RuntimeError("kamil_adc_raw_producer requires radar.model='kamil_adc'")
config.ensure_combos()
logger.info(
"Kamil ADC raw producer starting: config=%s, combos=%d, continuous=%s",
args.config, len(config.combos), config.runtime.continuous,
)
raw_writer = ShmRingWriter(
config.rings.raw.name,
@@ -127,6 +132,10 @@ def main() -> int:
config.rings.raw_tap.capacity,
config.rings.raw_tap.slot_size_bytes,
)
logger.debug(
"Opened SHM ring writers: raw=%s, raw_tap=%s",
config.rings.raw.name, config.rings.raw_tap.name,
)
radar = KamilAdcService(config)
input_switch = _switch_from_model(config.input_switch)
output_switch = _switch_from_model(config.output_switch)
@@ -74,6 +74,7 @@ def _open_radar_with_retry(
logger.info("Matrix radar opened after %d attempt(s).", attempt + 1)
return radar
logger.debug("Stop requested before any matrix radar became available.")
return None
@@ -99,6 +100,10 @@ def main() -> int:
"matrix_raw_producer requires a matrix-mode radar.model "
"(librevna_multi or sn9000)"
)
logger.info(
"Matrix radar raw producer starting: config=%s, model=%s, continuous=%s",
args.config, config.radar.model, config.runtime.continuous,
)
raw_writer = ShmRingWriter(
config.rings.raw.name,
@@ -110,6 +115,10 @@ def main() -> int:
config.rings.raw_tap.capacity,
config.rings.raw_tap.slot_size_bytes,
)
logger.debug(
"Opened SHM ring writers: raw=%s, raw_tap=%s",
config.rings.raw.name, config.rings.raw_tap.name,
)
radar: MatrixRadarService | None = None
try:
+7
View File
@@ -12,6 +12,7 @@ from python_app.models.run_config_model import RadarSweepModel, RunConfigModel
def _parse_args() -> argparse.Namespace:
"""Parse command-line options for the SN9000 VISA smoke test."""
parser = argparse.ArgumentParser(description="Acquire one SN9000 collection through VISA HiSLIP")
parser.add_argument(
"--host",
@@ -40,6 +41,7 @@ def _parse_args() -> argparse.Namespace:
def _validate_collection(collection: SweepCollection, expected_points: int) -> None:
"""Raise unless a SN9000 collection has the expected traces, combos, shapes, and finite values."""
expected_traces = (
RunConfigModel.MULTI_DEVICE_INPUT_POSITIONS * RunConfigModel.MULTI_DEVICE_OUTPUT_POSITIONS
)
@@ -82,6 +84,11 @@ def _validate_collection(collection: SweepCollection, expected_points: int) -> N
def main() -> int:
"""Acquire one SN9000 collection and validate it end to end.
Opens the VISA session, configures the sweep, validates the collection, checks the SCPI
error queue, and prints a summary; raises on any validation or SCPI failure.
"""
args = _parse_args()
sweep = RadarSweepModel(
start_hz=args.start_hz,