awe/radar_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improved logging

Browse Source
This commit is contained in:
Ayzen
2026-06-06 00:52:52 +03:00
parent af6005d68f
commit aea49f6128
65 changed files with 1206 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python_app/hardware_full/librevna_multi_device_driver/controller.py
+70 -1
View File
@@ -6,6 +6,7 @@ from collections.abc import Iterator, Sequence
from contextlib import suppress
from dataclasses import replace
from typing import Optional
import logging
import threading
import time
@@ -23,6 +24,8 @@ from python_app.hardware_full.librevna_multi_device_driver.protocol import (
)
from python_app.hardware_full.librevna_multi_device_driver.transport import LibreVnaUsbBulkConnection
logger = logging.getLogger(__name__)
class MultiDeviceVnaController:
"""Coordinate one master LibreVNA and receiver slave LibreVNAs."""
@@ -46,6 +49,13 @@ class MultiDeviceVnaController:
self._sweep_is_running = False
self._is_closed = False
logger.info(
"Opening multi-device controller (master=%s, slaves=%s, sync=%s, external_ref=%s)",
master_serial_number,
list(slave_serial_numbers),
self._synchronization_enabled,
self._force_external_reference,
)
try:
# Register each device the moment it opens so a partial open (e.g. a
# slave that fails after the master is up) is fully released by close().
@@ -57,9 +67,12 @@ class MultiDeviceVnaController:
self._slave_devices.append(connection)
self._all_devices.append(connection)
except Exception:
logger.exception("Failed to open multi-device controller; releasing devices")
self.close()
raise
logger.info("Multi-device controller ready (%d device(s) open)", len(self._all_devices))
def __enter__(self) -> MultiDeviceVnaController:
"""Return this controller as a context manager resource."""
return self
@@ -77,17 +90,20 @@ class MultiDeviceVnaController:
if self._is_closed:
return
logger.info("Closing multi-device controller (%d device(s))", len(self._all_devices))
self._is_closed = True
with suppress(Exception):
self._send_idle_to_all_devices()
for device_connection in self._all_devices:
with suppress(Exception):
device_connection.close()
logger.debug("Multi-device controller closed")
def stop_continuous_sweep(self) -> None:
"""Stop the currently running sweep without closing device transports."""
if self._is_closed:
return
logger.info("Stopping continuous sweep")
self._send_idle_to_all_devices()
def configure_continuous_sweep(
@@ -121,6 +137,7 @@ class MultiDeviceVnaController:
and self._last_applied_sweep_configuration == sweep_configuration
and self._last_master_stimulus_ports == stimulus_ports
):
logger.debug("Sweep configuration unchanged; keeping running sweep")
return
if self._sweep_is_running:
@@ -131,6 +148,15 @@ class MultiDeviceVnaController:
# so the new sweep starts on an empty queue.
self._drain_all_received_packets()
logger.info(
"Configuring continuous sweep: %d points %d-%d Hz, ifbw=%d Hz, power=%.2f dBm, ports=%s",
sweep_configuration.points,
sweep_configuration.start_hz,
sweep_configuration.stop_hz,
sweep_configuration.if_bandwidth,
sweep_configuration.power_dbm,
stimulus_ports,
)
self._configure_sweep_on_all_devices(
sweep_configuration,
master_stimulus_ports=stimulus_ports,
@@ -163,6 +189,7 @@ class MultiDeviceVnaController:
datapoint_timeout_seconds=datapoint_timeout_seconds,
)
except Exception:
logger.warning("Sweep cycle collection failed; idling all devices", exc_info=True)
self._send_idle_to_all_devices()
raise
@@ -179,6 +206,10 @@ class MultiDeviceVnaController:
timeout_seconds: float = 3.0,
retry_count: int = 1,
) -> None:
"""Send a packet and wait for its ACK, retrying up to ``retry_count`` times.
Re-raises the last error if every attempt fails to acknowledge in time.
"""
last_error: Exception | None = None
for _attempt_index in range(retry_count + 1):
device_connection.send_packet(packet_type, payload)
@@ -187,6 +218,14 @@ class MultiDeviceVnaController:
return
except Exception as exc: # noqa: BLE001
last_error = exc
logger.debug(
"No ACK for packet type %s from %s (attempt %d/%d): %s",
packet_type,
device_connection.serial_number,
_attempt_index + 1,
retry_count + 1,
exc,
)
assert last_error is not None
raise last_error
@@ -199,6 +238,11 @@ class MultiDeviceVnaController:
timeout_seconds: float = 3.0,
retry_count: int = 1,
) -> None:
"""Send a command and wait for its ACK, swallowing any failure.
Used on best-effort paths (e.g. idling devices during shutdown) where a
non-responsive device must not abort the operation.
"""
try:
self._send_command_and_wait_for_acknowledgement(
device_connection,
@@ -208,9 +252,15 @@ class MultiDeviceVnaController:
retry_count=retry_count,
)
except Exception:
pass
logger.debug(
"Best-effort command type %s to %s failed; ignoring",
packet_type,
device_connection.serial_number,
)
def _send_idle_to_all_devices(self) -> None:
"""Best-effort SET_IDLE to every device and mark the sweep as stopped."""
logger.debug("Sending SET_IDLE to %d device(s)", len(self._all_devices))
# SET_IDLE is a one-shot stop command. The ACK may be delayed only by the
# in-flight datapoint queue, which drains within a few hundred ms. A short,
# single-shot timeout keeps recovery snappy when one device stops responding
@@ -225,6 +275,12 @@ class MultiDeviceVnaController:
self._sweep_is_running = False
def _configure_reference_clocks(self) -> None:
"""Apply ReferenceSettings to every device and mark the reference configured."""
logger.debug(
"Configuring reference clocks on %d device(s) (external_ref=%s)",
len(self._all_devices),
self._force_external_reference,
)
for device_connection in self._all_devices:
# 1 s ACK timeout plus one retry caps worst-case at ~2 s per device
# so a stuck reference apply cannot stall recovery for minutes.
@@ -245,6 +301,11 @@ class MultiDeviceVnaController:
*,
master_stimulus_ports: tuple[int, ...],
) -> None:
"""Send SweepSettings to slaves then the master and mark the sweep running.
The master is configured last so receivers are armed before the master
begins driving the synchronized trigger.
"""
if self._master_device is None:
raise RuntimeError("Master device is not open")
@@ -270,8 +331,15 @@ class MultiDeviceVnaController:
self._last_applied_sweep_configuration = replace(sweep_configuration)
self._last_master_stimulus_ports = master_stimulus_ports
self._sweep_is_running = True
logger.debug("Sweep settings applied to all devices; sweep running")
def _drain_all_received_packets(self) -> None:
"""Empty every device's received-packet queue, in parallel for 2+ devices.
Concurrent draining keeps cross-device timing skew small so a hardware
cycle wrap cannot slip between per-device drains and desynchronize the
cycle counters.
"""
# Drain every device queue in parallel rather than one after another:
# serial drain leaves up to a few hundred microseconds of skew between
# the master and slave drain moments, which is enough room for a
@@ -299,6 +367,7 @@ class MultiDeviceVnaController:
@staticmethod
def _normalize_master_stimulus_ports(master_stimulus_ports: Sequence[int]) -> tuple[int, ...]:
"""Validate and return master stimulus ports as a tuple of ints (ports 1/2 only)."""
stimulus_ports = tuple(int(port) for port in master_stimulus_ports)
if not stimulus_ports:
raise ValueError("master_stimulus_ports must not be empty")
python_app/hardware_full/librevna_multi_device_driver/cycle_collection.py
+89
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
from collections.abc import Callable, Sequence
import logging
import queue
import threading
import time
@@ -21,6 +22,8 @@ from python_app.hardware_full.librevna_multi_device_driver.protocol import (
)
from python_app.hardware_full.librevna_multi_device_driver.transport import LibreVnaUsbBulkConnection
logger = logging.getLogger(__name__)
LIBREVNA_NATIVE_SWEEP_TIMEOUT_SECONDS = 1.5
# Hard upper bound on how long one full sweep cycle is allowed to take from
@@ -93,6 +96,14 @@ def collect_complete_running_sweep_cycles(
device_connection: LibreVnaUsbBulkConnection,
handle_datapoint: Callable[[ParsedVnaDatapoint], bool],
) -> None:
"""Read datapoints from one device until enough are consumed or a timeout fires.
Runs on a worker thread. Each datapoint is offered to ``handle_datapoint``,
which returns whether it was consumed; only consumed datapoints count toward
progress and refresh the no-progress timeout. On any timeout or transport
error the error is recorded and ``stop_collection_requested`` is set so the
other collector threads also stop.
"""
datapoints_received = 0
expected_datapoint_count = cycle_count * point_count
loop_start_timestamp = time.monotonic()
@@ -111,6 +122,13 @@ def collect_complete_running_sweep_cycles(
now = time.monotonic()
remaining_timeout_seconds = (last_consumed_timestamp + datapoint_timeout_seconds) - now
if remaining_timeout_seconds <= 0:
logger.warning(
"No usable datapoints from %s for %.1fs (received %d/%d); aborting collection",
device_connection.serial_number,
datapoint_timeout_seconds,
datapoints_received,
expected_datapoint_count,
)
collection_errors.append(
TimeoutError(
f"No usable datapoints from {device_connection.serial_number} for "
@@ -122,6 +140,12 @@ def collect_complete_running_sweep_cycles(
return
if not has_consumed_any_datapoint and (now - loop_start_timestamp) > cycle_start_guard_seconds:
logger.warning(
"Device %s streamed datapoints but never reached point_index=0 within %.1fs; "
"aborting collection",
device_connection.serial_number,
cycle_start_guard_seconds,
)
collection_errors.append(
TimeoutError(
f"Device {device_connection.serial_number} streamed datapoints but never "
@@ -138,6 +162,14 @@ def collect_complete_running_sweep_cycles(
# on for too long — this is the safety net the per-device timeout
# cannot provide by itself.
if (now - loop_start_timestamp) > _MAX_FULL_CYCLE_SECONDS:
logger.warning(
"Device %s did not finish a sweep cycle within %.1fs (received %d/%d); "
"aborting collection",
device_connection.serial_number,
_MAX_FULL_CYCLE_SECONDS,
datapoints_received,
expected_datapoint_count,
)
collection_errors.append(
TimeoutError(
f"Device {device_connection.serial_number} did not finish a sweep cycle "
@@ -157,10 +189,20 @@ def collect_complete_running_sweep_cycles(
return
if isinstance(exc, queue.Empty):
continue
logger.warning(
"Timed out receiving datapoint from %s; aborting collection: %s",
device_connection.serial_number,
exc,
)
collection_errors.append(exc)
stop_collection_requested.set()
return
except Exception as exc: # noqa: BLE001
logger.error(
"Error receiving datapoint from %s; aborting collection",
device_connection.serial_number,
exc_info=exc,
)
collection_errors.append(exc)
stop_collection_requested.set()
return
@@ -182,6 +224,13 @@ def collect_complete_running_sweep_cycles(
def build_cycle_tracking_handler(
cycle_aware_handler: Callable[[ParsedVnaDatapoint, int], None],
) -> Callable[[ParsedVnaDatapoint], bool]:
"""Wrap a cycle-aware handler with cross-device cycle tracking.
Returns a per-datapoint handler that anchors cycle 0 on the first
``point_index == 0`` seen, advances the cycle counter on each point-index
wrap, drops datapoints past ``cycle_count``, and reports whether each
datapoint was consumed.
"""
# The controller restarts the sweep before every collection, so the
# first packet each device emits is point 0 of a brand-new cycle 0.
# Anchoring cycle 0 on the first observed point_index==0 — instead of
@@ -198,6 +247,12 @@ def collect_complete_running_sweep_cycles(
}
def handle_datapoint(parsed_datapoint: ParsedVnaDatapoint) -> bool:
"""Track the cycle index for one datapoint and dispatch it to the handler.
Returns ``True`` when the datapoint was consumed (within ``cycle_count``)
and ``False`` when it was ignored (pre-sync straggler or past the last
requested cycle).
"""
current_point_index = parsed_datapoint.point_index
if not cycle_tracking_state["synchronized"]:
@@ -221,6 +276,12 @@ def collect_complete_running_sweep_cycles(
return handle_datapoint
def handle_master_datapoint(parsed_datapoint: ParsedVnaDatapoint, cycle_index: int) -> None:
"""Store the master device's frequency, reference, and reflection values.
Records the sweep-point frequency and, per active master stimulus port, the
reference receiver value and the matching reflection (S11/S22) into the
cycle/point measurement buffers.
"""
point_index = parsed_datapoint.point_index
frequencies_hz[point_index] = parsed_datapoint.frequency_hz
@@ -260,9 +321,16 @@ def collect_complete_running_sweep_cycles(
] = port_receiver_value
def build_slave_datapoint_handler(slave_index: int) -> Callable[[ParsedVnaDatapoint], bool]:
"""Build a cycle-tracking datapoint handler for the given slave device.
The slave's two receivers map to ports ``2*slave_index + 3`` and ``+ 4``,
producing forward S-parameters (e.g. S3x/S4x) for each active master
stimulus port.
"""
receiver_base_port = 2 * slave_index + 3
def handle_slave_datapoint(parsed_datapoint: ParsedVnaDatapoint, cycle_index: int) -> None:
"""Store this slave's forward receiver values into the measurement buffers."""
point_index = parsed_datapoint.point_index
for master_stimulus_port, stage_index in stage_by_master_port.items():
first_s_parameter_name = f"S{receiver_base_port}{master_stimulus_port}"
@@ -303,6 +371,13 @@ def collect_complete_running_sweep_cycles(
)
)
logger.debug(
"Collecting %d sweep cycle(s) of %d points from %d device(s) (datapoint_timeout=%.1fs)",
cycle_count,
point_count,
len(all_device_connections),
datapoint_timeout_seconds,
)
for collection_thread in collection_threads:
collection_thread.start()
@@ -321,6 +396,10 @@ def collect_complete_running_sweep_cycles(
collection_thread for collection_thread in collection_threads if collection_thread.is_alive()
]
if stalled_threads:
logger.warning(
"Collector thread(s) still alive after join; requesting stop again: %s",
", ".join(stalled_thread.name for stalled_thread in stalled_threads),
)
stop_collection_requested.set()
# Give them one more short window in case they were just slow to react.
secondary_deadline = time.monotonic() + 0.5
@@ -328,6 +407,10 @@ def collect_complete_running_sweep_cycles(
stalled_thread.join(timeout=max(0.0, secondary_deadline - time.monotonic()))
still_stalled = [stalled_thread for stalled_thread in stalled_threads if stalled_thread.is_alive()]
if still_stalled:
logger.error(
"Collector thread(s) failed to stop within the join deadline: %s",
", ".join(stalled_thread.name for stalled_thread in still_stalled),
)
collection_errors.append(
RuntimeError(
"Sweep collector thread(s) failed to stop within the join deadline: "
@@ -339,11 +422,17 @@ def collect_complete_running_sweep_cycles(
raise RuntimeError(f"Sweep collection failed: {collection_errors[0]}") from collection_errors[0]
if slave_device_connections and min(datapoint_counts_by_device_serial.values(), default=0) == 0:
logger.error(
"No datapoints from at least one device; hardware trigger sync did not start "
"(per-device counts: %s)",
datapoint_counts_by_device_serial,
)
raise RuntimeError(
"No datapoints received from at least one device; hardware trigger sync did not start. "
"Check Trigger Out/In loop and 10 MHz reference wiring."
)
logger.debug("Sweep cycle collection complete (per-device counts: %s)", datapoint_counts_by_device_serial)
return SweepMeasurementResult(
frequencies_hz=frequencies_hz,
s_parameters=calculate_last_cycle_s_parameters(
python_app/hardware_full/librevna_multi_device_driver/transport.py
+17 -1
View File
@@ -20,6 +20,11 @@ class LibreVnaUsbBulkConnection:
"""Minimal packet transport for one LibreVNA device."""
def __init__(self, serial_number: str) -> None:
"""Open the USB transport for ``serial_number`` and start receiving packets.
Raises ``ValueError`` when no serial number is supplied and propagates any
transport error raised while opening the device.
"""
if not serial_number:
raise ValueError("serial_number is required for multi-device acquisition")
self.serial_number = serial_number
@@ -32,10 +37,13 @@ class LibreVnaUsbBulkConnection:
on_disconnect=self._on_disconnect,
read_chunk_size=4096,
)
logger.debug("Opening LibreVNA USB connection (serial=%s)", serial_number)
self._transport.connect(serial=serial_number, timeout_s=2.0)
logger.info("LibreVNA USB connection ready (serial=%s)", serial_number)
def close(self) -> None:
"""Close USB resources."""
"""Disconnect the underlying USB transport and release its resources."""
logger.debug("Closing LibreVNA USB connection (serial=%s)", self.serial_number)
self._transport.disconnect()
def drain_received_packets(self) -> list[tuple[int, bytes]]:
@@ -83,6 +91,11 @@ class LibreVnaUsbBulkConnection:
raise RuntimeError(f"Device {self.serial_number} returned NACK")
def _on_data(self, chunk: bytes) -> None:
"""Decode a received USB chunk into frames and queue (type, payload) tuples.
Any decode failure is recorded as the fatal transport error so the next
send/receive call surfaces it to the caller.
"""
try:
packets = self._scanner.feed(chunk)
except Exception as exc: # noqa: BLE001
@@ -92,15 +105,18 @@ class LibreVnaUsbBulkConnection:
self._received_packets.put((int(packet.type), bytes(packet.payload)))
def _on_disconnect(self, exc: Exception) -> None:
"""Record an asynchronous transport disconnect as the fatal error."""
self._set_fatal_error(exc)
def _set_fatal_error(self, exc: Exception) -> None:
"""Store the first fatal transport error and log it; later errors are ignored."""
with self._fatal_lock:
if self._fatal_error is None:
logger.error("LibreVNA USB transport failed for %s: %s", self.serial_number, exc)
self._fatal_error = exc
def _raise_if_failed(self) -> None:
"""Re-raise the stored fatal transport error as ``RuntimeError`` if one exists."""
with self._fatal_lock:
if self._fatal_error is None:
return