some fixes
This commit is contained in:
@@ -320,14 +320,18 @@ class KamilAdcService:
|
||||
executable_path = str(Path(adc.executable_path).expanduser())
|
||||
return [executable_path, *adc.args, f"tty:{adc.tty_path}"]
|
||||
|
||||
def open(self) -> None:
|
||||
"""Launch the collector and start the TTY reader thread."""
|
||||
def open(self, *, stop_event: threading.Event | None = None) -> None:
|
||||
"""Launch the collector and start the TTY reader thread.
|
||||
|
||||
An optional `stop_event` lets a caller abort the TTY-wait loop promptly
|
||||
(e.g. on shutdown) instead of blocking for the full startup timeout.
|
||||
"""
|
||||
if self._reader is not None:
|
||||
return
|
||||
previous_tty_identity = _prepare_tty_path_for_collector(self.config.radar.kamil_adc.tty_path)
|
||||
try:
|
||||
self._start_process()
|
||||
self._wait_for_tty(previous_tty_identity)
|
||||
self._wait_for_tty(previous_tty_identity, stop_event=stop_event)
|
||||
reader = KamilAdcTtyReader(self.config.radar.kamil_adc.tty_path)
|
||||
reader.open()
|
||||
self._reader = reader
|
||||
@@ -416,15 +420,28 @@ class KamilAdcService:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait(timeout=1.0)
|
||||
|
||||
def _wait_for_tty(self, previous_identity: tuple[object, ...] | None) -> None:
|
||||
def _wait_for_tty(
|
||||
self,
|
||||
previous_identity: tuple[object, ...] | None,
|
||||
*,
|
||||
stop_event: threading.Event | None = None,
|
||||
) -> None:
|
||||
adc = self.config.radar.kamil_adc
|
||||
deadline = time.monotonic() + adc.startup_timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
# Abort promptly if a stop was requested mid-wait.
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
raise RuntimeError("Kamil ADC startup aborted before TTY became available")
|
||||
KamilAdcTtyReader._raise_if_process_exited(self._process)
|
||||
identity = _tty_identity(adc.tty_path)
|
||||
if identity is not None and identity != previous_identity:
|
||||
return
|
||||
time.sleep(0.05)
|
||||
# Use the stop event's wait() so a set() breaks the poll immediately.
|
||||
if stop_event is not None:
|
||||
if stop_event.wait(0.05):
|
||||
raise RuntimeError("Kamil ADC startup aborted before TTY became available")
|
||||
else:
|
||||
time.sleep(0.05)
|
||||
raise TimeoutError(
|
||||
f"Timed out waiting for Kamil ADC TTY `{adc.tty_path}` to be created by the collector"
|
||||
)
|
||||
|
||||
@@ -157,8 +157,19 @@ class USBTransport:
|
||||
logger.debug("USB disconnect requested")
|
||||
self._stop_event.set()
|
||||
|
||||
if self._rx_thread is not None and self._rx_thread.is_alive():
|
||||
self._rx_thread.join(timeout=1.0)
|
||||
rx_thread = self._rx_thread
|
||||
if rx_thread is not None and rx_thread.is_alive():
|
||||
rx_thread.join(timeout=1.0)
|
||||
if rx_thread.is_alive():
|
||||
# The RX thread is wedged inside libusb; closing the handle or
|
||||
# context now would risk a use-after-free in the still-running
|
||||
# bulkRead. Deliberately leak both rather than crash the process.
|
||||
logger.error(
|
||||
"USB RX thread did not stop within timeout; leaking USB handle/context "
|
||||
"to avoid use-after-free (serial=%s)",
|
||||
self.connected_serial,
|
||||
)
|
||||
return
|
||||
self._rx_thread = None
|
||||
|
||||
if self._handle is not None:
|
||||
|
||||
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass, field
|
||||
import logging
|
||||
import math
|
||||
import threading
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -26,6 +27,12 @@ logger = logging.getLogger(__name__)
|
||||
# of all entries (1.75 s today) plus the cost of close()/open() themselves.
|
||||
_REOPEN_BACKOFF_SECONDS: tuple[float, ...] = (0.25, 0.5, 1.0)
|
||||
|
||||
# Hard ceiling on the wall-clock time a single acquire_collection() may spend in
|
||||
# the recovery path (backoff sleeps + close()/open() cost across every retry).
|
||||
# Bounding this keeps a shutdown that arrives mid-recovery comfortably under the
|
||||
# systemd TimeoutStopSec so the unit is never SIGKILLed for hanging on exit.
|
||||
_MAX_RECOVERY_WALL_SECONDS: float = 10.0
|
||||
|
||||
_INPUT_S_PARAMETERS_BY_OUTPUT: dict[int, tuple[str, ...]] = {
|
||||
0: ("s31", "s41", "s51", "s61"),
|
||||
1: ("s32", "s42", "s52", "s62"),
|
||||
@@ -101,13 +108,22 @@ class MultiDeviceLibreVnaService:
|
||||
except Exception as exc: # noqa: BLE001 — recovery path, never propagate
|
||||
logger.warning("Multi-device close() ignored transport error: %s", exc)
|
||||
|
||||
def recover(self) -> None:
|
||||
def recover(
|
||||
self,
|
||||
*,
|
||||
stop_event: threading.Event | None = None,
|
||||
deadline_monotonic: float | None = None,
|
||||
) -> None:
|
||||
"""Reopen native device transports after a failed acquisition.
|
||||
|
||||
Tries several short backoffs so a transient USB stall does not kill the
|
||||
producer on the very first retry. Raises the last error only after
|
||||
every attempt failed — the outer acquisition loop is expected to count
|
||||
these as recovery_attempts.
|
||||
|
||||
When `stop_event` is supplied the backoff waits on it instead of
|
||||
sleeping, so a shutdown request aborts the loop immediately; an optional
|
||||
`deadline_monotonic` caps the total wall-time spent here.
|
||||
"""
|
||||
if self._using_mock_backend:
|
||||
return
|
||||
@@ -115,7 +131,21 @@ class MultiDeviceLibreVnaService:
|
||||
|
||||
last_error: Exception | None = None
|
||||
for attempt_index, delay_s in enumerate(_REOPEN_BACKOFF_SECONDS, start=1):
|
||||
time.sleep(delay_s)
|
||||
# Bail out the instant a stop is requested or the recovery budget is
|
||||
# spent, rather than committing to another (re)open attempt.
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
logger.info("Multi-device recover() aborted: stop requested")
|
||||
return
|
||||
if deadline_monotonic is not None and time.monotonic() >= deadline_monotonic:
|
||||
logger.warning("Multi-device recover() aborted: recovery time budget exhausted")
|
||||
break
|
||||
# Interruptible backoff: wait() returns early the moment stop is set.
|
||||
if stop_event is not None:
|
||||
if stop_event.wait(delay_s):
|
||||
logger.info("Multi-device recover() aborted: stop requested")
|
||||
return
|
||||
else:
|
||||
time.sleep(delay_s)
|
||||
try:
|
||||
self.open()
|
||||
if self._controller is not None:
|
||||
@@ -151,15 +181,27 @@ class MultiDeviceLibreVnaService:
|
||||
power_dbm=float(sweep.power_dbm),
|
||||
)
|
||||
|
||||
def acquire_collection(self, collection_id: int = 1) -> SweepCollection:
|
||||
"""Acquire one complete virtual 2x4 matrix in canonical combo order."""
|
||||
def acquire_collection(
|
||||
self,
|
||||
collection_id: int = 1,
|
||||
*,
|
||||
stop_event: threading.Event | None = None,
|
||||
) -> SweepCollection:
|
||||
"""Acquire one complete virtual 2x4 matrix in canonical combo order.
|
||||
|
||||
When `stop_event` is supplied it is threaded into the recovery path so a
|
||||
shutdown request aborts the backoff/retry loops promptly (default `None`
|
||||
preserves the original blocking behavior).
|
||||
"""
|
||||
if self._sweep_configuration is None:
|
||||
raise RuntimeError("Multi-device service is not configured")
|
||||
capture_start_ns = time.monotonic_ns()
|
||||
if self._using_mock_backend:
|
||||
collection = self._acquire_mock_collection(collection_id, capture_start_ns)
|
||||
else:
|
||||
collection = self._acquire_native_collection_with_recovery(collection_id, capture_start_ns)
|
||||
collection = self._acquire_native_collection_with_recovery(
|
||||
collection_id, capture_start_ns, stop_event=stop_event
|
||||
)
|
||||
collection.capture_end_ns = time.monotonic_ns()
|
||||
return collection
|
||||
|
||||
@@ -167,8 +209,13 @@ class MultiDeviceLibreVnaService:
|
||||
self,
|
||||
collection_id: int,
|
||||
capture_start_ns: int,
|
||||
*,
|
||||
stop_event: threading.Event | None = None,
|
||||
) -> SweepCollection:
|
||||
last_error: Exception | None = None
|
||||
# Cap total recovery wall-time across all retries so a shutdown that
|
||||
# lands mid-recovery stays well under the systemd TimeoutStopSec.
|
||||
recovery_deadline = time.monotonic() + _MAX_RECOVERY_WALL_SECONDS
|
||||
for attempt_index in range(self.recovery_attempts + 1):
|
||||
try:
|
||||
return self._acquire_native_collection(collection_id, capture_start_ns)
|
||||
@@ -176,6 +223,13 @@ class MultiDeviceLibreVnaService:
|
||||
last_error = exc
|
||||
if attempt_index >= self.recovery_attempts:
|
||||
break
|
||||
# Stop the moment shutdown is requested or the recovery budget is
|
||||
# spent — do not start another reconnect we cannot finish in time.
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
break
|
||||
if time.monotonic() >= recovery_deadline:
|
||||
logger.warning("multi-device recovery time budget exhausted; giving up")
|
||||
break
|
||||
logger.warning(
|
||||
"multi-device acquisition failed, reconnecting devices (%d/%d): %s",
|
||||
attempt_index + 1,
|
||||
@@ -188,7 +242,7 @@ class MultiDeviceLibreVnaService:
|
||||
# attempt and try again on the next loop iteration, so a
|
||||
# transient USB hiccup cannot kill the whole producer.
|
||||
try:
|
||||
self.recover()
|
||||
self.recover(stop_event=stop_event, deadline_monotonic=recovery_deadline)
|
||||
except Exception as recover_exc: # noqa: BLE001
|
||||
last_error = recover_exc
|
||||
logger.warning(
|
||||
@@ -198,6 +252,10 @@ class MultiDeviceLibreVnaService:
|
||||
recover_exc,
|
||||
exc_info=True,
|
||||
)
|
||||
# If recovery was interrupted by a stop request, do not loop back
|
||||
# for another acquisition attempt; let shutdown proceed.
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
break
|
||||
|
||||
assert last_error is not None
|
||||
raise last_error
|
||||
|
||||
@@ -180,9 +180,12 @@ class GpioOutputLines:
|
||||
self._line_fd = int(request.fd)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close line request and chip file descriptors."""
|
||||
self._close_line_fd()
|
||||
self._close_chip_fd()
|
||||
"""Close line request and chip file descriptors (best-effort, idempotent)."""
|
||||
try:
|
||||
self._close_line_fd()
|
||||
finally:
|
||||
# Ensure the chip fd is always closed even if closing the line fd raised.
|
||||
self._close_chip_fd()
|
||||
|
||||
def set_values(self, values: Sequence[int]) -> None:
|
||||
"""Apply output values for all requested lines."""
|
||||
@@ -212,16 +215,22 @@ class GpioOutputLines:
|
||||
raise RuntimeError(f"Failed to set GPIO output values: {exc}") from exc
|
||||
|
||||
def _close_line_fd(self) -> None:
|
||||
"""Close line file descriptor if currently open."""
|
||||
"""Close line file descriptor if currently open (best-effort, idempotent)."""
|
||||
if self._line_fd >= 0:
|
||||
os.close(self._line_fd)
|
||||
self._line_fd = -1
|
||||
try:
|
||||
os.close(self._line_fd)
|
||||
finally:
|
||||
# Always clear the fd so close() stays idempotent even if os.close raised.
|
||||
self._line_fd = -1
|
||||
|
||||
def _close_chip_fd(self) -> None:
|
||||
"""Close chip file descriptor if currently open."""
|
||||
"""Close chip file descriptor if currently open (best-effort, idempotent)."""
|
||||
if self._chip_fd >= 0:
|
||||
os.close(self._chip_fd)
|
||||
self._chip_fd = -1
|
||||
try:
|
||||
os.close(self._chip_fd)
|
||||
finally:
|
||||
# Always clear the fd so close() stays idempotent even if os.close raised.
|
||||
self._chip_fd = -1
|
||||
|
||||
|
||||
class GpioLineEventWatcher:
|
||||
@@ -316,18 +325,27 @@ class GpioLineEventWatcher:
|
||||
return int(event.id)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close line request and chip file descriptors."""
|
||||
self._close_line_fd()
|
||||
self._close_chip_fd()
|
||||
"""Close line request and chip file descriptors (best-effort, idempotent)."""
|
||||
try:
|
||||
self._close_line_fd()
|
||||
finally:
|
||||
# Ensure the chip fd is always closed even if closing the line fd raised.
|
||||
self._close_chip_fd()
|
||||
|
||||
def _close_line_fd(self) -> None:
|
||||
"""Close line file descriptor if currently open."""
|
||||
"""Close line file descriptor if currently open (best-effort, idempotent)."""
|
||||
if self._line_fd >= 0:
|
||||
os.close(self._line_fd)
|
||||
self._line_fd = -1
|
||||
try:
|
||||
os.close(self._line_fd)
|
||||
finally:
|
||||
# Always clear the fd so close() stays idempotent even if os.close raised.
|
||||
self._line_fd = -1
|
||||
|
||||
def _close_chip_fd(self) -> None:
|
||||
"""Close chip file descriptor if currently open."""
|
||||
"""Close chip file descriptor if currently open (best-effort, idempotent)."""
|
||||
if self._chip_fd >= 0:
|
||||
os.close(self._chip_fd)
|
||||
self._chip_fd = -1
|
||||
try:
|
||||
os.close(self._chip_fd)
|
||||
finally:
|
||||
# Always clear the fd so close() stays idempotent even if os.close raised.
|
||||
self._chip_fd = -1
|
||||
|
||||
Reference in New Issue
Block a user