some fixes

This commit is contained in:
Ayzen
2026-06-05 14:40:10 +03:00
parent 22942d9dc9
commit bbea744459
35 changed files with 1797 additions and 297 deletions
+22 -5
View File
@@ -320,14 +320,18 @@ class KamilAdcService:
executable_path = str(Path(adc.executable_path).expanduser())
return [executable_path, *adc.args, f"tty:{adc.tty_path}"]
def open(self) -> None:
"""Launch the collector and start the TTY reader thread."""
def open(self, *, stop_event: threading.Event | None = None) -> None:
"""Launch the collector and start the TTY reader thread.
An optional `stop_event` lets a caller abort the TTY-wait loop promptly
(e.g. on shutdown) instead of blocking for the full startup timeout.
"""
if self._reader is not None:
return
previous_tty_identity = _prepare_tty_path_for_collector(self.config.radar.kamil_adc.tty_path)
try:
self._start_process()
self._wait_for_tty(previous_tty_identity)
self._wait_for_tty(previous_tty_identity, stop_event=stop_event)
reader = KamilAdcTtyReader(self.config.radar.kamil_adc.tty_path)
reader.open()
self._reader = reader
@@ -416,15 +420,28 @@ class KamilAdcService:
os.killpg(process.pid, signal.SIGKILL)
process.wait(timeout=1.0)
def _wait_for_tty(self, previous_identity: tuple[object, ...] | None) -> None:
def _wait_for_tty(
self,
previous_identity: tuple[object, ...] | None,
*,
stop_event: threading.Event | None = None,
) -> None:
adc = self.config.radar.kamil_adc
deadline = time.monotonic() + adc.startup_timeout_s
while time.monotonic() < deadline:
# Abort promptly if a stop was requested mid-wait.
if stop_event is not None and stop_event.is_set():
raise RuntimeError("Kamil ADC startup aborted before TTY became available")
KamilAdcTtyReader._raise_if_process_exited(self._process)
identity = _tty_identity(adc.tty_path)
if identity is not None and identity != previous_identity:
return
time.sleep(0.05)
# Use the stop event's wait() so a set() breaks the poll immediately.
if stop_event is not None:
if stop_event.wait(0.05):
raise RuntimeError("Kamil ADC startup aborted before TTY became available")
else:
time.sleep(0.05)
raise TimeoutError(
f"Timed out waiting for Kamil ADC TTY `{adc.tty_path}` to be created by the collector"
)
@@ -157,8 +157,19 @@ class USBTransport:
logger.debug("USB disconnect requested")
self._stop_event.set()
if self._rx_thread is not None and self._rx_thread.is_alive():
self._rx_thread.join(timeout=1.0)
rx_thread = self._rx_thread
if rx_thread is not None and rx_thread.is_alive():
rx_thread.join(timeout=1.0)
if rx_thread.is_alive():
# The RX thread is wedged inside libusb; closing the handle or
# context now would risk a use-after-free in the still-running
# bulkRead. Deliberately leak both rather than crash the process.
logger.error(
"USB RX thread did not stop within timeout; leaking USB handle/context "
"to avoid use-after-free (serial=%s)",
self.connected_serial,
)
return
self._rx_thread = None
if self._handle is not None:
@@ -5,6 +5,7 @@ from __future__ import annotations
from dataclasses import dataclass, field
import logging
import math
import threading
import time
from typing import TYPE_CHECKING
@@ -26,6 +27,12 @@ logger = logging.getLogger(__name__)
# of all entries (1.75 s today) plus the cost of close()/open() themselves.
_REOPEN_BACKOFF_SECONDS: tuple[float, ...] = (0.25, 0.5, 1.0)
# Hard ceiling on the wall-clock time a single acquire_collection() may spend in
# the recovery path (backoff sleeps + close()/open() cost across every retry).
# Bounding this keeps a shutdown that arrives mid-recovery comfortably under the
# systemd TimeoutStopSec so the unit is never SIGKILLed for hanging on exit.
_MAX_RECOVERY_WALL_SECONDS: float = 10.0
_INPUT_S_PARAMETERS_BY_OUTPUT: dict[int, tuple[str, ...]] = {
0: ("s31", "s41", "s51", "s61"),
1: ("s32", "s42", "s52", "s62"),
@@ -101,13 +108,22 @@ class MultiDeviceLibreVnaService:
except Exception as exc: # noqa: BLE001 — recovery path, never propagate
logger.warning("Multi-device close() ignored transport error: %s", exc)
def recover(self) -> None:
def recover(
self,
*,
stop_event: threading.Event | None = None,
deadline_monotonic: float | None = None,
) -> None:
"""Reopen native device transports after a failed acquisition.
Tries several short backoffs so a transient USB stall does not kill the
producer on the very first retry. Raises the last error only after
every attempt failed — the outer acquisition loop is expected to count
these as recovery_attempts.
When `stop_event` is supplied the backoff waits on it instead of
sleeping, so a shutdown request aborts the loop immediately; an optional
`deadline_monotonic` caps the total wall-time spent here.
"""
if self._using_mock_backend:
return
@@ -115,7 +131,21 @@ class MultiDeviceLibreVnaService:
last_error: Exception | None = None
for attempt_index, delay_s in enumerate(_REOPEN_BACKOFF_SECONDS, start=1):
time.sleep(delay_s)
# Bail out the instant a stop is requested or the recovery budget is
# spent, rather than committing to another (re)open attempt.
if stop_event is not None and stop_event.is_set():
logger.info("Multi-device recover() aborted: stop requested")
return
if deadline_monotonic is not None and time.monotonic() >= deadline_monotonic:
logger.warning("Multi-device recover() aborted: recovery time budget exhausted")
break
# Interruptible backoff: wait() returns early the moment stop is set.
if stop_event is not None:
if stop_event.wait(delay_s):
logger.info("Multi-device recover() aborted: stop requested")
return
else:
time.sleep(delay_s)
try:
self.open()
if self._controller is not None:
@@ -151,15 +181,27 @@ class MultiDeviceLibreVnaService:
power_dbm=float(sweep.power_dbm),
)
def acquire_collection(self, collection_id: int = 1) -> SweepCollection:
"""Acquire one complete virtual 2x4 matrix in canonical combo order."""
def acquire_collection(
self,
collection_id: int = 1,
*,
stop_event: threading.Event | None = None,
) -> SweepCollection:
"""Acquire one complete virtual 2x4 matrix in canonical combo order.
When `stop_event` is supplied it is threaded into the recovery path so a
shutdown request aborts the backoff/retry loops promptly (default `None`
preserves the original blocking behavior).
"""
if self._sweep_configuration is None:
raise RuntimeError("Multi-device service is not configured")
capture_start_ns = time.monotonic_ns()
if self._using_mock_backend:
collection = self._acquire_mock_collection(collection_id, capture_start_ns)
else:
collection = self._acquire_native_collection_with_recovery(collection_id, capture_start_ns)
collection = self._acquire_native_collection_with_recovery(
collection_id, capture_start_ns, stop_event=stop_event
)
collection.capture_end_ns = time.monotonic_ns()
return collection
@@ -167,8 +209,13 @@ class MultiDeviceLibreVnaService:
self,
collection_id: int,
capture_start_ns: int,
*,
stop_event: threading.Event | None = None,
) -> SweepCollection:
last_error: Exception | None = None
# Cap total recovery wall-time across all retries so a shutdown that
# lands mid-recovery stays well under the systemd TimeoutStopSec.
recovery_deadline = time.monotonic() + _MAX_RECOVERY_WALL_SECONDS
for attempt_index in range(self.recovery_attempts + 1):
try:
return self._acquire_native_collection(collection_id, capture_start_ns)
@@ -176,6 +223,13 @@ class MultiDeviceLibreVnaService:
last_error = exc
if attempt_index >= self.recovery_attempts:
break
# Stop the moment shutdown is requested or the recovery budget is
# spent — do not start another reconnect we cannot finish in time.
if stop_event is not None and stop_event.is_set():
break
if time.monotonic() >= recovery_deadline:
logger.warning("multi-device recovery time budget exhausted; giving up")
break
logger.warning(
"multi-device acquisition failed, reconnecting devices (%d/%d): %s",
attempt_index + 1,
@@ -188,7 +242,7 @@ class MultiDeviceLibreVnaService:
# attempt and try again on the next loop iteration, so a
# transient USB hiccup cannot kill the whole producer.
try:
self.recover()
self.recover(stop_event=stop_event, deadline_monotonic=recovery_deadline)
except Exception as recover_exc: # noqa: BLE001
last_error = recover_exc
logger.warning(
@@ -198,6 +252,10 @@ class MultiDeviceLibreVnaService:
recover_exc,
exc_info=True,
)
# If recovery was interrupted by a stop request, do not loop back
# for another acquisition attempt; let shutdown proceed.
if stop_event is not None and stop_event.is_set():
break
assert last_error is not None
raise last_error
@@ -180,9 +180,12 @@ class GpioOutputLines:
self._line_fd = int(request.fd)
def close(self) -> None:
"""Close line request and chip file descriptors."""
self._close_line_fd()
self._close_chip_fd()
"""Close line request and chip file descriptors (best-effort, idempotent)."""
try:
self._close_line_fd()
finally:
# Ensure the chip fd is always closed even if closing the line fd raised.
self._close_chip_fd()
def set_values(self, values: Sequence[int]) -> None:
"""Apply output values for all requested lines."""
@@ -212,16 +215,22 @@ class GpioOutputLines:
raise RuntimeError(f"Failed to set GPIO output values: {exc}") from exc
def _close_line_fd(self) -> None:
"""Close line file descriptor if currently open."""
"""Close line file descriptor if currently open (best-effort, idempotent)."""
if self._line_fd >= 0:
os.close(self._line_fd)
self._line_fd = -1
try:
os.close(self._line_fd)
finally:
# Always clear the fd so close() stays idempotent even if os.close raised.
self._line_fd = -1
def _close_chip_fd(self) -> None:
"""Close chip file descriptor if currently open."""
"""Close chip file descriptor if currently open (best-effort, idempotent)."""
if self._chip_fd >= 0:
os.close(self._chip_fd)
self._chip_fd = -1
try:
os.close(self._chip_fd)
finally:
# Always clear the fd so close() stays idempotent even if os.close raised.
self._chip_fd = -1
class GpioLineEventWatcher:
@@ -316,18 +325,27 @@ class GpioLineEventWatcher:
return int(event.id)
def close(self) -> None:
"""Close line request and chip file descriptors."""
self._close_line_fd()
self._close_chip_fd()
"""Close line request and chip file descriptors (best-effort, idempotent)."""
try:
self._close_line_fd()
finally:
# Ensure the chip fd is always closed even if closing the line fd raised.
self._close_chip_fd()
def _close_line_fd(self) -> None:
"""Close line file descriptor if currently open."""
"""Close line file descriptor if currently open (best-effort, idempotent)."""
if self._line_fd >= 0:
os.close(self._line_fd)
self._line_fd = -1
try:
os.close(self._line_fd)
finally:
# Always clear the fd so close() stays idempotent even if os.close raised.
self._line_fd = -1
def _close_chip_fd(self) -> None:
"""Close chip file descriptor if currently open."""
"""Close chip file descriptor if currently open (best-effort, idempotent)."""
if self._chip_fd >= 0:
os.close(self._chip_fd)
self._chip_fd = -1
try:
os.close(self._chip_fd)
finally:
# Always clear the fd so close() stays idempotent even if os.close raised.
self._chip_fd = -1