338 lines
15 KiB
Python
338 lines
15 KiB
Python
"""High-level service for LibreVNA multi-device acquisition."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
import logging
|
|
import threading
|
|
import time
|
|
from typing import TYPE_CHECKING
|
|
|
|
import numpy as np
|
|
|
|
from python_app.hardware_full.librevna_multi_device_driver.cycle_collection import LIBREVNA_NATIVE_SWEEP_TIMEOUT_SECONDS
|
|
from python_app.hardware_full.librevna_multi_device_driver.exceptions import TransientCollectionError
|
|
from python_app.hardware_full.librevna_multi_device_driver.models import (
|
|
SweepConfiguration,
|
|
normalize_switch_substage_count,
|
|
)
|
|
from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData
|
|
from python_app.models.run_config_schema import RadarSweepModel, RunConfigModel
|
|
|
|
if TYPE_CHECKING:
|
|
from python_app.hardware_full.librevna_multi_device_driver.controller import MultiDeviceVnaController
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Backoff delays applied only BEFORE each reopen RETRY inside recover(); the first
|
|
# attempt runs immediately (no upfront sleep) so a transient stall recovers at
|
|
# once. Picked to give libusb time to re-enumerate a stuck device while staying
|
|
# short enough that a healthy reconnect feels instant. The total worst-case extra
|
|
# wait across all retries is the sum of these entries (1.75 s today).
|
|
_REOPEN_BACKOFF_SECONDS: tuple[float, ...] = (0.25, 0.5, 1.0)
|
|
|
|
# Hard ceiling on the wall-clock time a single acquire_collection() may spend in
|
|
# the recovery path (backoff sleeps + close()/open() cost across every retry).
|
|
# Bounding this keeps a shutdown that arrives mid-recovery comfortably under the
|
|
# systemd TimeoutStopSec so the unit is never SIGKILLed for hanging on exit.
|
|
_MAX_RECOVERY_WALL_SECONDS: float = 10.0
|
|
|
|
# How many times a TRANSIENT collection failure (dropped/incomplete datapoint,
|
|
# no-progress timeout, or cross-device cycle misalignment) is retried in place —
|
|
# re-arming the sweep and re-collecting WITHOUT a USB reopen — before escalating
|
|
# to the heavy close()/reopen recovery. Each retry costs ~one sweep period, so a
|
|
# handful absorbs ordinary glitches without the multi-second re-enumeration that
|
|
# previously caused the periodic freeze.
|
|
_MAX_TRANSIENT_COLLECTION_RETRIES: int = 2
|
|
|
|
@dataclass(slots=True)
|
|
class MultiDeviceLibreVnaService:
|
|
"""Acquire a virtual 2x4 switch matrix from synchronized LibreVNA devices."""
|
|
|
|
master_serial: str
|
|
slave_serials: list[str]
|
|
force_external_reference: bool = False
|
|
recovery_attempts: int = 3
|
|
# Stimulus switch positions stepped through WITHIN one frequency point. 1 keeps the
|
|
# stock 2-output matrix; 2 doubles the outputs (4) without a second sweep, at the
|
|
# cost of requiring the modified master bitstream and the trigger-loop controller.
|
|
switch_substage_count: int = 1
|
|
_controller: "MultiDeviceVnaController | None" = field(init=False, default=None, repr=False)
|
|
_sweep_configuration: SweepConfiguration | None = field(init=False, default=None, repr=False)
|
|
|
|
def __post_init__(self) -> None:
|
|
"""Validate static topology and backend selection."""
|
|
self.master_serial = str(self.master_serial).strip()
|
|
self.slave_serials = [str(value).strip() for value in self.slave_serials if str(value).strip()]
|
|
if len(self.slave_serials) != 2:
|
|
raise ValueError("LibreVNA multi-device mode requires exactly two slave serial numbers")
|
|
self.recovery_attempts = max(0, int(self.recovery_attempts))
|
|
|
|
self.switch_substage_count = normalize_switch_substage_count(self.switch_substage_count)
|
|
|
|
@property
|
|
def output_position_count(self) -> int:
|
|
"""Return how many stimulus outputs one acquisition produces."""
|
|
return RunConfigModel.MULTI_DEVICE_MASTER_PORTS * self.switch_substage_count
|
|
|
|
def open(self) -> None:
|
|
"""Open native device transports."""
|
|
if self._controller is not None:
|
|
return
|
|
try:
|
|
from python_app.hardware_full.librevna_multi_device_driver.controller import MultiDeviceVnaController
|
|
|
|
self._controller = MultiDeviceVnaController(
|
|
master_serial_number=self.master_serial,
|
|
slave_serial_numbers=self.slave_serials,
|
|
force_external_reference=self.force_external_reference,
|
|
switch_substage_count=self.switch_substage_count,
|
|
)
|
|
logger.info(
|
|
"Multi-device controller opened (master=%s, slaves=%s)",
|
|
self.master_serial, self.slave_serials,
|
|
)
|
|
except Exception as exc:
|
|
logger.warning("Multi-device open failed: %s", exc)
|
|
raise
|
|
|
|
def close(self) -> None:
|
|
"""Close native device transports; never raises.
|
|
|
|
Recovery loops rely on `close()` being safe to call on a half-open or
|
|
already-broken controller. We swallow any transport-level exception here
|
|
and just drop the reference so the next `open()` starts fresh.
|
|
"""
|
|
controller = self._controller
|
|
self._controller = None
|
|
if controller is None:
|
|
return
|
|
try:
|
|
controller.close()
|
|
except Exception as exc: # noqa: BLE001 — recovery path, never propagate
|
|
logger.warning("Multi-device close() ignored transport error: %s", exc)
|
|
|
|
def recover(
|
|
self,
|
|
*,
|
|
stop_event: threading.Event | None = None,
|
|
deadline_monotonic: float | None = None,
|
|
) -> None:
|
|
"""Reopen native device transports after a failed acquisition.
|
|
|
|
Tries several short backoffs so a transient USB stall does not kill the
|
|
producer on the very first retry. Raises the last error only after
|
|
every attempt failed — the outer acquisition loop is expected to count
|
|
these as recovery_attempts.
|
|
|
|
When `stop_event` is supplied the backoff waits on it instead of
|
|
sleeping, so a shutdown request aborts the loop immediately; an optional
|
|
`deadline_monotonic` caps the total wall-time spent here.
|
|
"""
|
|
self.close()
|
|
|
|
# Try to reopen immediately first, then back off only after a failure: a
|
|
# transient USB stall usually clears at once, so the common recovery should
|
|
# not pay an upfront sleep. The backoff delays apply only between retries.
|
|
reopen_delays = (0.0, *_REOPEN_BACKOFF_SECONDS)
|
|
last_error: Exception | None = None
|
|
for attempt_index, delay_s in enumerate(reopen_delays, start=1):
|
|
# Bail out the instant a stop is requested or the recovery budget is
|
|
# spent, rather than committing to another (re)open attempt.
|
|
if stop_event is not None and stop_event.is_set():
|
|
logger.info("Multi-device recover() aborted: stop requested")
|
|
return
|
|
if deadline_monotonic is not None and time.monotonic() >= deadline_monotonic:
|
|
logger.warning("Multi-device recover() aborted: recovery time budget exhausted")
|
|
break
|
|
# Interruptible backoff before retries; the first attempt has no delay.
|
|
if delay_s > 0.0:
|
|
if stop_event is not None:
|
|
if stop_event.wait(delay_s):
|
|
logger.info("Multi-device recover() aborted: stop requested")
|
|
return
|
|
else:
|
|
time.sleep(delay_s)
|
|
try:
|
|
self.open()
|
|
if self._controller is not None:
|
|
logger.info(
|
|
"Multi-device reopen succeeded on attempt %d/%d (after %.2fs)",
|
|
attempt_index,
|
|
len(reopen_delays),
|
|
delay_s,
|
|
)
|
|
return
|
|
except Exception as exc: # noqa: BLE001 — propagate only the last failure
|
|
last_error = exc
|
|
logger.warning(
|
|
"Multi-device reopen attempt %d/%d failed after %.2fs: %s",
|
|
attempt_index,
|
|
len(reopen_delays),
|
|
delay_s,
|
|
exc,
|
|
)
|
|
self.close() # tidy partially-opened state before next try
|
|
|
|
if last_error is not None:
|
|
raise last_error
|
|
raise RuntimeError("Multi-device recover() exhausted all reopen attempts")
|
|
|
|
def configure(self, sweep: RadarSweepModel) -> None:
|
|
"""Store sweep settings for subsequent full-matrix acquisitions."""
|
|
self._sweep_configuration = SweepConfiguration(
|
|
start_hz=int(round(float(sweep.start_hz))),
|
|
stop_hz=int(round(float(sweep.stop_hz))),
|
|
points=int(sweep.points),
|
|
if_bandwidth=int(round(float(sweep.if_bandwidth_hz))),
|
|
power_dbm=float(sweep.power_dbm),
|
|
dwell_us=int(sweep.dwell_us),
|
|
)
|
|
logger.debug(
|
|
"Multi-device configured: %s-%s Hz, %s points, IFBW=%s Hz, %s dBm",
|
|
self._sweep_configuration.start_hz,
|
|
self._sweep_configuration.stop_hz,
|
|
self._sweep_configuration.points,
|
|
self._sweep_configuration.if_bandwidth,
|
|
self._sweep_configuration.power_dbm,
|
|
)
|
|
|
|
def acquire_collection(
|
|
self,
|
|
collection_id: int = 1,
|
|
*,
|
|
combo: ComboKey | None = None,
|
|
stop_event: threading.Event | None = None,
|
|
) -> SweepCollection:
|
|
"""Acquire one complete virtual 2x4 matrix in canonical combo order.
|
|
|
|
When `stop_event` is supplied it is threaded into the recovery path so a
|
|
shutdown request aborts the backoff/retry loops promptly (default `None`
|
|
preserves the original blocking behavior).
|
|
"""
|
|
if self._sweep_configuration is None:
|
|
raise RuntimeError("Multi-device service is not configured")
|
|
return self._acquire_native_collection_with_recovery(collection_id, stop_event=stop_event)
|
|
|
|
def _acquire_native_collection_with_recovery(
|
|
self,
|
|
collection_id: int,
|
|
*,
|
|
stop_event: threading.Event | None = None,
|
|
) -> SweepCollection:
|
|
last_error: Exception | None = None
|
|
# Cap total recovery wall-time across all retries so a shutdown that
|
|
# lands mid-recovery stays well under the systemd TimeoutStopSec.
|
|
recovery_deadline = time.monotonic() + _MAX_RECOVERY_WALL_SECONDS
|
|
for attempt_index in range(self.recovery_attempts + 1):
|
|
try:
|
|
return self._acquire_native_collection_with_transient_retries(
|
|
collection_id, stop_event=stop_event
|
|
)
|
|
except Exception as exc: # noqa: BLE001
|
|
last_error = exc
|
|
if attempt_index >= self.recovery_attempts:
|
|
break
|
|
# Stop the moment shutdown is requested or the recovery budget is
|
|
# spent — do not start another reconnect we cannot finish in time.
|
|
if stop_event is not None and stop_event.is_set():
|
|
break
|
|
if time.monotonic() >= recovery_deadline:
|
|
logger.warning("multi-device recovery time budget exhausted; giving up")
|
|
break
|
|
logger.warning(
|
|
"multi-device acquisition failed, reconnecting devices (%d/%d): %s",
|
|
attempt_index + 1,
|
|
self.recovery_attempts,
|
|
exc,
|
|
exc_info=True,
|
|
)
|
|
# recover() may itself fail when libusb cannot re-enumerate the
|
|
# device fast enough; treat that as the same kind of recovery
|
|
# attempt and try again on the next loop iteration, so a
|
|
# transient USB hiccup cannot kill the whole producer.
|
|
try:
|
|
self.recover(stop_event=stop_event, deadline_monotonic=recovery_deadline)
|
|
except Exception as recover_exc: # noqa: BLE001
|
|
last_error = recover_exc
|
|
logger.warning(
|
|
"multi-device recover() failed (%d/%d): %s",
|
|
attempt_index + 1,
|
|
self.recovery_attempts,
|
|
recover_exc,
|
|
exc_info=True,
|
|
)
|
|
# If recovery was interrupted by a stop request, do not loop back
|
|
# for another acquisition attempt; let shutdown proceed.
|
|
if stop_event is not None and stop_event.is_set():
|
|
break
|
|
|
|
assert last_error is not None
|
|
raise last_error
|
|
|
|
def _acquire_native_collection_with_transient_retries(
|
|
self,
|
|
collection_id: int,
|
|
*,
|
|
stop_event: threading.Event | None = None,
|
|
) -> SweepCollection:
|
|
"""Acquire one collection, retrying device-healthy failures in place.
|
|
|
|
A ``TransientCollectionError`` (dropped/incomplete datapoint, no-progress
|
|
timeout, or cross-device cycle misalignment) leaves the USB transports
|
|
alive, so it is recovered by simply re-collecting: the controller auto-idled
|
|
on failure, so the next ``configure_continuous_sweep`` re-sends
|
|
``SWEEP_SETTINGS`` (re-arm). This costs ~one sweep period instead of the
|
|
multi-second ``close()``/reopen that genuine transport death requires.
|
|
Non-transient errors propagate immediately to the reopen-based recovery.
|
|
"""
|
|
transient_error: TransientCollectionError | None = None
|
|
for transient_attempt in range(_MAX_TRANSIENT_COLLECTION_RETRIES + 1):
|
|
try:
|
|
return self._acquire_native_collection(collection_id)
|
|
except TransientCollectionError as exc:
|
|
transient_error = exc
|
|
if transient_attempt >= _MAX_TRANSIENT_COLLECTION_RETRIES:
|
|
raise
|
|
if stop_event is not None and stop_event.is_set():
|
|
raise
|
|
logger.debug(
|
|
"transient multi-device collection error (%d/%d); re-arming and re-collecting "
|
|
"without USB reopen: %s",
|
|
transient_attempt + 1,
|
|
_MAX_TRANSIENT_COLLECTION_RETRIES,
|
|
exc,
|
|
)
|
|
assert transient_error is not None # loop either returns or raises
|
|
raise transient_error
|
|
|
|
def _acquire_native_collection(self, collection_id: int) -> SweepCollection:
|
|
if self._controller is None:
|
|
raise RuntimeError("Multi-device controller is not open")
|
|
assert self._sweep_configuration is not None
|
|
|
|
self._controller.configure_continuous_sweep(self._sweep_configuration)
|
|
result = self._controller.collect_running_sweep_cycle(
|
|
datapoint_timeout_seconds=LIBREVNA_NATIVE_SWEEP_TIMEOUT_SECONDS,
|
|
)
|
|
frequencies = np.asarray(result.frequencies_hz, dtype=np.float32)
|
|
traces = [
|
|
TraceData(
|
|
combo=ComboKey(input_pos, output_pos),
|
|
frequency_hz=frequencies,
|
|
s21=np.asarray(result.s_parameters[input_pos, output_pos], dtype=np.complex64),
|
|
capture_start_ns=result.capture_start_ns,
|
|
capture_end_ns=result.capture_end_ns,
|
|
)
|
|
for output_pos in range(self.output_position_count)
|
|
for input_pos in range(2 * len(self.slave_serials))
|
|
]
|
|
|
|
return SweepCollection(
|
|
collection_id=int(collection_id),
|
|
monotonic_ns=time.monotonic_ns(),
|
|
traces=traces,
|
|
capture_start_ns=result.capture_start_ns,
|
|
capture_end_ns=result.capture_end_ns,
|
|
)
|