some fixes and improvements
This commit is contained in:
Ayzen
@@ -1,4 +1,22 @@
|
||||
"""Service for acquiring sweeps from the external Kamil ADC collector."""
|
||||
"""Service for acquiring sweeps from the external Kamil ADC collector.
|
||||
|
||||
The external `kamil_adc` binary publishes its samples on a PTY/TTY device as a
|
||||
stream of 8-byte frames:
|
||||
|
||||
* **Start marker**: `0x000A 0xFFFF 0xFFFF 0xFFFF` — delimits sweep boundaries.
|
||||
* **Point frame**: `0x000A step real_i16 imag_i16` — one complex sample per
|
||||
frame, with `step` running 1, 2, …, N for an N-point sweep.
|
||||
|
||||
The hardware emits sweeps continuously, faster than callers tend to invoke
|
||||
:meth:`KamilAdcService.acquire`. To avoid TTY-buffer overruns and stale data,
|
||||
a daemon thread drains the device end of the TTY non-stop, parses complete
|
||||
sweeps as they arrive, and publishes the **latest** one to a one-slot mailbox.
|
||||
:meth:`acquire` simply waits for the next sweep to appear in that mailbox.
|
||||
|
||||
Sweep length is determined by the first sweep observed at runtime and stays
|
||||
constant for the life of the service; any later mismatch is treated as a
|
||||
protocol violation rather than something to silently discard.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -13,6 +31,7 @@ import signal
|
||||
import stat
|
||||
import struct
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
@@ -22,233 +41,255 @@ from python_app.models.run_config_model import RadarSweepModel, RunConfigModel
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Wire-format constants for the Kamil ADC TTY protocol.
|
||||
KAMIL_ADC_MARKER = 0x000A
|
||||
KAMIL_ADC_START_STEP = 0xFFFF
|
||||
KAMIL_ADC_FRAME_BYTES = 8
|
||||
KAMIL_ADC_MAX_STEP = 0xFFFE
|
||||
|
||||
_RAW_FRAME_STRUCT = struct.Struct("<HHHH")
|
||||
_POINT_FRAME_STRUCT = struct.Struct("<HHhh")
|
||||
_START_FRAME = _RAW_FRAME_STRUCT.pack(
|
||||
KAMIL_ADC_MARKER,
|
||||
KAMIL_ADC_START_STEP,
|
||||
KAMIL_ADC_START_STEP,
|
||||
KAMIL_ADC_START_STEP,
|
||||
_START_FRAME: bytes = struct.pack(
|
||||
"<HHHH", KAMIL_ADC_MARKER, KAMIL_ADC_START_STEP, KAMIL_ADC_START_STEP, KAMIL_ADC_START_STEP
|
||||
)
|
||||
# Point frames carry signed 16-bit real/imag components; start markers reuse
|
||||
# the same 8-byte slot but with all four words unsigned. Comparing the raw
|
||||
# bytes against :data:`_START_FRAME` is therefore the correct boundary check.
|
||||
_POINT_STRUCT = struct.Struct("<HHhh")
|
||||
|
||||
# Larger TTY reads keep up with bursty USB CDC-ACM writers without raising the
|
||||
# syscall rate. 64 KiB matches the typical Linux PTY buffer size.
|
||||
_READ_CHUNK_BYTES = 65536
|
||||
# select() poll interval inside the reader thread — short enough to react to
|
||||
# `close()` requests, long enough that idle CPU stays near zero.
|
||||
_READ_POLL_INTERVAL_S = 0.1
|
||||
|
||||
|
||||
class KamilAdcFrameParser:
|
||||
"""Strict parser for Kamil ADC 4-word TTY frames."""
|
||||
|
||||
@staticmethod
|
||||
def is_packet_start(frame: bytes) -> bool:
|
||||
"""Return whether `frame` is the packet-start marker."""
|
||||
return frame == _START_FRAME
|
||||
|
||||
@staticmethod
|
||||
def parse_point(frame: bytes, expected_step: int) -> complex:
|
||||
"""Parse one `0x000A step real imag` frame and validate ordering."""
|
||||
if len(frame) != KAMIL_ADC_FRAME_BYTES:
|
||||
raise ValueError(
|
||||
f"Kamil ADC frame must be {KAMIL_ADC_FRAME_BYTES} bytes, got {len(frame)}"
|
||||
)
|
||||
marker, step, real, imag = _POINT_FRAME_STRUCT.unpack(frame)
|
||||
if marker != KAMIL_ADC_MARKER:
|
||||
raise ValueError(f"Kamil ADC marker mismatch: got 0x{marker:04x}, expected 0x000a")
|
||||
if step != expected_step:
|
||||
raise ValueError(f"Kamil ADC step mismatch: got {step}, expected {expected_step}")
|
||||
return complex(real, imag)
|
||||
def _parse_point_frame(frame: bytes, expected_step: int) -> complex:
|
||||
"""Parse one 8-byte point frame; validate marker and step ordering."""
|
||||
marker, step, real, imag = _POINT_STRUCT.unpack(frame)
|
||||
if marker != KAMIL_ADC_MARKER:
|
||||
raise ValueError(f"Kamil ADC marker mismatch: got 0x{marker:04x}, expected 0x000a")
|
||||
if step != expected_step:
|
||||
raise ValueError(f"Kamil ADC step mismatch: got {step}, expected {expected_step}")
|
||||
return complex(real, imag)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class KamilAdcTtyReader:
|
||||
"""Read full Kamil ADC sweep packets from a nonblocking TTY stream."""
|
||||
"""Background-thread TTY reader publishing the latest completed sweep.
|
||||
|
||||
The reader spawns a daemon thread on :meth:`open` which continuously
|
||||
drains the TTY, parses frames into complete sweeps, and stores the most
|
||||
recent one in a single-slot mailbox. Consumers call :meth:`read_sweep` to
|
||||
take that sweep; if a newer one arrives before the consumer reads, it
|
||||
overwrites the previous unread value — by design, since consumers always
|
||||
want the freshest data.
|
||||
"""
|
||||
|
||||
tty_path: str
|
||||
_fd: int | None = field(init=False, default=None, repr=False)
|
||||
_buffer: bytearray = field(init=False, default_factory=bytearray, repr=False)
|
||||
_packet_start_pending: bool = field(init=False, default=False, repr=False)
|
||||
_thread: threading.Thread | None = field(init=False, default=None, repr=False)
|
||||
_stop_event: threading.Event = field(init=False, default_factory=threading.Event, repr=False)
|
||||
_mailbox_cv: threading.Condition = field(init=False, default_factory=threading.Condition, repr=False)
|
||||
_latest_sweep: np.ndarray | None = field(init=False, default=None, repr=False)
|
||||
_reader_error: Exception | None = field(init=False, default=None, repr=False)
|
||||
_locked_points: int | None = field(init=False, default=None, repr=False)
|
||||
_published_count: int = field(init=False, default=0, repr=False)
|
||||
|
||||
def open(self) -> None:
|
||||
"""Open the configured TTY path for binary reads."""
|
||||
"""Open the TTY and start the background reader thread."""
|
||||
if self._fd is not None:
|
||||
return
|
||||
self._fd = os.open(self.tty_path, os.O_RDONLY | os.O_NOCTTY | os.O_NONBLOCK)
|
||||
self._stop_event.clear()
|
||||
self._latest_sweep = None
|
||||
self._reader_error = None
|
||||
self._locked_points = None
|
||||
self._published_count = 0
|
||||
self._thread = threading.Thread(
|
||||
target=self._reader_loop,
|
||||
name=f"kamil-adc-tty-reader[{self.tty_path}]",
|
||||
daemon=True,
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the TTY file descriptor."""
|
||||
if self._fd is None:
|
||||
return
|
||||
try:
|
||||
os.close(self._fd)
|
||||
finally:
|
||||
self._fd = None
|
||||
self._buffer.clear()
|
||||
self._packet_start_pending = False
|
||||
"""Stop the reader thread and close the TTY descriptor."""
|
||||
self._stop_event.set()
|
||||
with self._mailbox_cv:
|
||||
self._mailbox_cv.notify_all()
|
||||
if self._thread is not None:
|
||||
self._thread.join(timeout=1.0)
|
||||
self._thread = None
|
||||
if self._fd is not None:
|
||||
try:
|
||||
os.close(self._fd)
|
||||
finally:
|
||||
self._fd = None
|
||||
self._latest_sweep = None
|
||||
self._reader_error = None
|
||||
self._locked_points = None
|
||||
|
||||
@property
|
||||
def locked_points(self) -> int | None:
|
||||
"""Return the sweep point count established by the first sweep, or `None`."""
|
||||
return self._locked_points
|
||||
|
||||
@property
|
||||
def published_count(self) -> int:
|
||||
"""Return the total number of sweeps the reader thread has produced."""
|
||||
with self._mailbox_cv:
|
||||
return self._published_count
|
||||
|
||||
def read_sweep(
|
||||
self,
|
||||
*,
|
||||
timeout_s: float,
|
||||
process: subprocess.Popen[bytes] | None = None,
|
||||
expected_points: int | None = None,
|
||||
) -> np.ndarray:
|
||||
"""Read one full packet, optionally discarding packets with an unexpected point count."""
|
||||
if self._fd is None:
|
||||
"""Wait for and return the next published sweep.
|
||||
|
||||
Raises :class:`TimeoutError` if no sweep arrives within `timeout_s`,
|
||||
:class:`RuntimeError` if the external collector process exited, and
|
||||
propagates any exception caught by the reader thread.
|
||||
"""
|
||||
if self._thread is None:
|
||||
raise RuntimeError("Kamil ADC TTY reader is not open")
|
||||
if expected_points is not None:
|
||||
if expected_points <= 0:
|
||||
raise ValueError("Kamil ADC expected points must be > 0")
|
||||
if expected_points > KAMIL_ADC_MAX_STEP:
|
||||
raise ValueError(f"Kamil ADC expected points must be <= {KAMIL_ADC_MAX_STEP}")
|
||||
|
||||
deadline = time.monotonic() + float(timeout_s)
|
||||
while True:
|
||||
values = self._read_one_sweep(deadline, process)
|
||||
if expected_points is None or int(values.size) == int(expected_points):
|
||||
return values
|
||||
logger.warning(
|
||||
"Discarding Kamil ADC sweep with %d points; expected %d",
|
||||
int(values.size),
|
||||
int(expected_points),
|
||||
)
|
||||
with self._mailbox_cv:
|
||||
while True:
|
||||
# Always deliver a pending sweep first: if the reader thread
|
||||
# both published a sweep and then died, the consumer should
|
||||
# still see the good data and only meet the error on the next
|
||||
# call.
|
||||
if self._latest_sweep is not None:
|
||||
sweep = self._latest_sweep
|
||||
self._latest_sweep = None
|
||||
return sweep
|
||||
if self._reader_error is not None:
|
||||
raise self._reader_error
|
||||
self._raise_if_process_exited(process)
|
||||
remaining_s = deadline - time.monotonic()
|
||||
if remaining_s <= 0.0:
|
||||
raise TimeoutError(
|
||||
f"Timed out waiting for Kamil ADC sweep after {float(timeout_s):.3f}s"
|
||||
)
|
||||
# Wake periodically so we can re-check process liveness.
|
||||
self._mailbox_cv.wait(timeout=min(_READ_POLL_INTERVAL_S, remaining_s))
|
||||
|
||||
def _read_one_sweep(
|
||||
self,
|
||||
deadline: float,
|
||||
process: subprocess.Popen[bytes] | None,
|
||||
) -> np.ndarray:
|
||||
"""Read one packet from start marker to the next start marker."""
|
||||
if self._packet_start_pending:
|
||||
self._packet_start_pending = False
|
||||
else:
|
||||
self._read_until_packet_start(deadline, process)
|
||||
# ------------------------------------------------------------------
|
||||
# Reader-thread internals
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _reader_loop(self) -> None:
|
||||
"""Drain TTY → parse frames → publish completed sweeps until stop."""
|
||||
buffer = bytearray()
|
||||
try:
|
||||
if not self._skip_to_first_start_marker(buffer):
|
||||
return
|
||||
while not self._stop_event.is_set():
|
||||
sweep = self._read_one_sweep(buffer)
|
||||
if sweep is None:
|
||||
return
|
||||
self._publish_sweep(sweep)
|
||||
except Exception as exc: # noqa: BLE001 — surfaced to the consumer via read_sweep
|
||||
self._publish_error(exc)
|
||||
|
||||
def _skip_to_first_start_marker(self, buffer: bytearray) -> bool:
|
||||
"""Discard pre-roll bytes until a start marker is consumed from `buffer`."""
|
||||
while not self._stop_event.is_set():
|
||||
start_index = buffer.find(_START_FRAME)
|
||||
if start_index >= 0:
|
||||
del buffer[: start_index + KAMIL_ADC_FRAME_BYTES]
|
||||
return True
|
||||
# Keep just enough trailing bytes that a marker split across read
|
||||
# boundaries can still be reassembled on the next chunk.
|
||||
if len(buffer) >= KAMIL_ADC_FRAME_BYTES:
|
||||
del buffer[: -(KAMIL_ADC_FRAME_BYTES - 1)]
|
||||
if not self._read_more(buffer):
|
||||
return False
|
||||
return False
|
||||
|
||||
def _read_one_sweep(self, buffer: bytearray) -> np.ndarray | None:
|
||||
"""Parse frames from `buffer` until the next start marker; return the sweep."""
|
||||
values: list[complex] = []
|
||||
expected_step = 1
|
||||
while True:
|
||||
frame = self._read_frame(deadline, process, received_points=len(values))
|
||||
if KamilAdcFrameParser.is_packet_start(frame):
|
||||
while not self._stop_event.is_set():
|
||||
while len(buffer) < KAMIL_ADC_FRAME_BYTES:
|
||||
if not self._read_more(buffer):
|
||||
return None
|
||||
frame = bytes(buffer[:KAMIL_ADC_FRAME_BYTES])
|
||||
del buffer[:KAMIL_ADC_FRAME_BYTES]
|
||||
|
||||
if frame == _START_FRAME:
|
||||
if not values:
|
||||
# Two consecutive markers — ignore the empty sweep and keep parsing.
|
||||
continue
|
||||
self._packet_start_pending = True
|
||||
self._validate_and_lock_point_count(len(values))
|
||||
return np.asarray(values, dtype=np.complex64)
|
||||
|
||||
if expected_step > KAMIL_ADC_MAX_STEP:
|
||||
raise RuntimeError(f"Kamil ADC sweep exceeded {KAMIL_ADC_MAX_STEP} points without packet end")
|
||||
values.append(KamilAdcFrameParser.parse_point(frame, expected_step))
|
||||
if self._locked_points is not None and expected_step > self._locked_points:
|
||||
raise RuntimeError(
|
||||
f"Kamil ADC sweep exceeded locked point count {self._locked_points} "
|
||||
"without a start marker"
|
||||
)
|
||||
values.append(_parse_point_frame(frame, expected_step))
|
||||
expected_step += 1
|
||||
return None
|
||||
|
||||
def discard_pending(self, process: subprocess.Popen[bytes] | None = None) -> None:
|
||||
"""Discard stale bytes while keeping the newest packet-start boundary."""
|
||||
if self._fd is None:
|
||||
raise RuntimeError("Kamil ADC TTY reader is not open")
|
||||
self._buffer.clear()
|
||||
self._packet_start_pending = False
|
||||
fd = self._require_fd()
|
||||
while True:
|
||||
self._raise_if_process_exited(process)
|
||||
def _validate_and_lock_point_count(self, points: int) -> None:
|
||||
"""Lock the point count on the first sweep; reject mismatches thereafter."""
|
||||
if self._locked_points is None:
|
||||
self._locked_points = points
|
||||
logger.info("Kamil ADC sweep point count locked to %d", points)
|
||||
return
|
||||
if points != self._locked_points:
|
||||
raise RuntimeError(
|
||||
f"Kamil ADC sweep length changed: locked={self._locked_points}, got={points}"
|
||||
)
|
||||
|
||||
def _read_more(self, buffer: bytearray) -> bool:
|
||||
"""Block on `select` until bytes arrive, then append them to `buffer`.
|
||||
|
||||
Returns `False` if the reader was asked to stop, `True` if at least one
|
||||
byte was appended. Raises on stream-level errors.
|
||||
"""
|
||||
fd = self._fd
|
||||
if fd is None:
|
||||
return False
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
readable, _, _ = select.select([fd], [], [], 0.0)
|
||||
readable, _, _ = select.select([fd], [], [], _READ_POLL_INTERVAL_S)
|
||||
except InterruptedError:
|
||||
continue
|
||||
if not readable:
|
||||
return
|
||||
continue
|
||||
try:
|
||||
chunk = os.read(fd, 4096)
|
||||
chunk = os.read(fd, _READ_CHUNK_BYTES)
|
||||
except BlockingIOError:
|
||||
return
|
||||
continue
|
||||
except OSError as exc:
|
||||
if exc.errno in {errno.EAGAIN, errno.EWOULDBLOCK}:
|
||||
return
|
||||
raise RuntimeError(f"Failed to drain Kamil ADC TTY `{self.tty_path}`: {exc}") from exc
|
||||
continue
|
||||
raise RuntimeError(
|
||||
f"Failed to read Kamil ADC TTY `{self.tty_path}`: {exc}"
|
||||
) from exc
|
||||
if not chunk:
|
||||
raise RuntimeError(f"Kamil ADC TTY `{self.tty_path}` closed while draining")
|
||||
self._buffer.extend(chunk)
|
||||
self._keep_latest_packet_start_tail()
|
||||
raise RuntimeError(f"Kamil ADC TTY `{self.tty_path}` closed while reading")
|
||||
buffer.extend(chunk)
|
||||
return True
|
||||
return False
|
||||
|
||||
def _keep_latest_packet_start_tail(self) -> None:
|
||||
"""Keep only bytes from the latest complete packet-start marker onward."""
|
||||
start_index = self._buffer.rfind(_START_FRAME)
|
||||
if start_index >= 0:
|
||||
del self._buffer[:start_index]
|
||||
return
|
||||
if len(self._buffer) >= KAMIL_ADC_FRAME_BYTES:
|
||||
del self._buffer[:-KAMIL_ADC_FRAME_BYTES + 1]
|
||||
def _publish_sweep(self, sweep: np.ndarray) -> None:
|
||||
"""Store `sweep` as the latest mailbox value, overwriting any prior unread one."""
|
||||
with self._mailbox_cv:
|
||||
self._latest_sweep = sweep
|
||||
self._published_count += 1
|
||||
self._mailbox_cv.notify()
|
||||
|
||||
def _read_until_packet_start(
|
||||
self,
|
||||
deadline: float,
|
||||
process: subprocess.Popen[bytes] | None,
|
||||
) -> None:
|
||||
while True:
|
||||
start_index = self._buffer.find(_START_FRAME)
|
||||
if start_index >= 0:
|
||||
del self._buffer[: start_index + KAMIL_ADC_FRAME_BYTES]
|
||||
return
|
||||
if len(self._buffer) >= KAMIL_ADC_FRAME_BYTES:
|
||||
del self._buffer[:-KAMIL_ADC_FRAME_BYTES + 1]
|
||||
self._read_available(deadline, process)
|
||||
|
||||
def _read_frame(
|
||||
self,
|
||||
deadline: float,
|
||||
process: subprocess.Popen[bytes] | None,
|
||||
*,
|
||||
received_points: int,
|
||||
expected_points: int | None = None,
|
||||
) -> bytes:
|
||||
while len(self._buffer) < KAMIL_ADC_FRAME_BYTES:
|
||||
self._read_available(deadline, process, received_points, expected_points)
|
||||
frame = bytes(self._buffer[:KAMIL_ADC_FRAME_BYTES])
|
||||
del self._buffer[:KAMIL_ADC_FRAME_BYTES]
|
||||
return frame
|
||||
|
||||
def _read_available(
|
||||
self,
|
||||
deadline: float,
|
||||
process: subprocess.Popen[bytes] | None,
|
||||
received_points: int | None = None,
|
||||
expected_points: int | None = None,
|
||||
) -> None:
|
||||
self._raise_if_process_exited(process)
|
||||
remaining_s = deadline - time.monotonic()
|
||||
if remaining_s <= 0.0:
|
||||
if received_points is None or expected_points is None:
|
||||
if received_points is not None:
|
||||
raise TimeoutError(
|
||||
f"Timed out waiting for Kamil ADC sweep end: received {received_points} points"
|
||||
)
|
||||
raise TimeoutError("Timed out waiting for Kamil ADC packet-start marker")
|
||||
raise TimeoutError(
|
||||
f"Timed out waiting for Kamil ADC sweep: received {received_points}/{expected_points} points"
|
||||
)
|
||||
|
||||
fd = self._require_fd()
|
||||
wait_s = min(0.05, remaining_s)
|
||||
try:
|
||||
readable, _, _ = select.select([fd], [], [], wait_s)
|
||||
except InterruptedError:
|
||||
return
|
||||
if not readable:
|
||||
return
|
||||
|
||||
try:
|
||||
chunk = os.read(fd, 4096)
|
||||
except BlockingIOError:
|
||||
return
|
||||
except OSError as exc:
|
||||
if exc.errno in {errno.EAGAIN, errno.EWOULDBLOCK}:
|
||||
return
|
||||
raise RuntimeError(f"Failed to read Kamil ADC TTY `{self.tty_path}`: {exc}") from exc
|
||||
if not chunk:
|
||||
raise RuntimeError(f"Kamil ADC TTY `{self.tty_path}` closed while reading")
|
||||
self._buffer.extend(chunk)
|
||||
|
||||
def _require_fd(self) -> int:
|
||||
if self._fd is None:
|
||||
raise RuntimeError("Kamil ADC TTY reader is not open")
|
||||
return self._fd
|
||||
def _publish_error(self, exc: Exception) -> None:
|
||||
"""Record `exc` as the reader fault and wake any waiter."""
|
||||
with self._mailbox_cv:
|
||||
self._reader_error = exc
|
||||
self._mailbox_cv.notify_all()
|
||||
|
||||
@staticmethod
|
||||
def _raise_if_process_exited(process: subprocess.Popen[bytes] | None) -> None:
|
||||
@@ -261,14 +302,13 @@ class KamilAdcTtyReader:
|
||||
|
||||
@dataclass(slots=True)
|
||||
class KamilAdcService:
|
||||
"""Launch `kamil_adc` and acquire TTY sweeps."""
|
||||
"""Launch the external `kamil_adc` collector and serve its sweeps."""
|
||||
|
||||
config: RunConfigModel
|
||||
_process: subprocess.Popen[bytes] | None = field(init=False, default=None, repr=False)
|
||||
_reader: KamilAdcTtyReader | None = field(init=False, default=None, repr=False)
|
||||
_settings: RadarSweepModel | None = field(init=False, default=None, repr=False)
|
||||
_frequency_hz: np.ndarray | None = field(init=False, default=None, repr=False)
|
||||
_expected_points: int | None = field(init=False, default=None, repr=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self._validate_config()
|
||||
@@ -281,10 +321,9 @@ class KamilAdcService:
|
||||
return [executable_path, *adc.args, f"tty:{adc.tty_path}"]
|
||||
|
||||
def open(self) -> None:
|
||||
"""Launch the collector and open its TTY stream."""
|
||||
"""Launch the collector and start the TTY reader thread."""
|
||||
if self._reader is not None:
|
||||
return
|
||||
|
||||
previous_tty_identity = _prepare_tty_path_for_collector(self.config.radar.kamil_adc.tty_path)
|
||||
try:
|
||||
self._start_process()
|
||||
@@ -297,50 +336,40 @@ class KamilAdcService:
|
||||
raise
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close TTY and stop the external collector."""
|
||||
"""Stop the TTY reader and the external collector process."""
|
||||
if self._reader is not None:
|
||||
with suppress(Exception):
|
||||
self._reader.close()
|
||||
self._reader = None
|
||||
|
||||
self._stop_process()
|
||||
|
||||
def configure(self, sweep: RadarSweepModel) -> None:
|
||||
"""Store sweep settings used to construct the synthetic frequency axis."""
|
||||
"""Store sweep settings used to build the synthetic frequency axis."""
|
||||
self._validate_sweep(sweep)
|
||||
self._settings = sweep
|
||||
self._frequency_hz = None
|
||||
self._expected_points = None
|
||||
|
||||
def read_device_limits(self) -> dict[str, float | int]:
|
||||
"""Kamil ADC has no runtime-readable sweep limit API."""
|
||||
raise RuntimeError("Kamil ADC device limits are not available")
|
||||
|
||||
def acquire(self) -> SweepResult:
|
||||
"""Acquire one Kamil ADC sweep as S21; fill S11 with explicit zeros."""
|
||||
"""Return the most recent completed sweep as S21 (S11 filled with zeros)."""
|
||||
if self._settings is None:
|
||||
raise RuntimeError("Kamil ADC service is not configured")
|
||||
if self._reader is None:
|
||||
raise RuntimeError("Kamil ADC service is not open")
|
||||
process = self._process
|
||||
if process is None or process.poll() is not None:
|
||||
code = None if process is None else process.poll()
|
||||
raise RuntimeError(f"Kamil ADC process is not running (code={code})")
|
||||
return_code = None if process is None else process.poll()
|
||||
raise RuntimeError(f"Kamil ADC process is not running (code={return_code})")
|
||||
|
||||
self._reader.discard_pending(process)
|
||||
s21 = self._reader.read_sweep(
|
||||
timeout_s=self.config.radar.kamil_adc.sweep_timeout_s,
|
||||
process=process,
|
||||
expected_points=self._expected_points,
|
||||
)
|
||||
points = int(s21.size)
|
||||
if points <= 0:
|
||||
raise RuntimeError("Kamil ADC sweep contained no points")
|
||||
if self._expected_points is None:
|
||||
self._expected_points = points
|
||||
self._frequency_hz = self._build_frequency_axis(points)
|
||||
logger.info("Kamil ADC sweep point count locked to %d", points)
|
||||
if self._frequency_hz is None:
|
||||
if self._frequency_hz is None or self._frequency_hz.size != points:
|
||||
self._frequency_hz = self._build_frequency_axis(points)
|
||||
return SweepResult(
|
||||
x=self._frequency_hz.copy(),
|
||||
@@ -350,10 +379,13 @@ class KamilAdcService:
|
||||
},
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Process / TTY lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _start_process(self) -> None:
|
||||
if self._process is not None and self._process.poll() is None:
|
||||
return
|
||||
|
||||
adc = self.config.radar.kamil_adc
|
||||
env = os.environ.copy()
|
||||
env.update(adc.env)
|
||||
@@ -371,11 +403,8 @@ class KamilAdcService:
|
||||
def _stop_process(self) -> None:
|
||||
process = self._process
|
||||
self._process = None
|
||||
if process is None:
|
||||
if process is None or process.poll() is not None:
|
||||
return
|
||||
if process.poll() is not None:
|
||||
return
|
||||
|
||||
with suppress(ProcessLookupError):
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
try:
|
||||
@@ -383,7 +412,6 @@ class KamilAdcService:
|
||||
return
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
|
||||
with suppress(ProcessLookupError):
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait(timeout=1.0)
|
||||
@@ -401,6 +429,10 @@ class KamilAdcService:
|
||||
f"Timed out waiting for Kamil ADC TTY `{adc.tty_path}` to be created by the collector"
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Validation helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _validate_config(self) -> None:
|
||||
if not self.config.is_kamil_adc:
|
||||
raise RuntimeError("KamilAdcService requires radar.model='kamil_adc'")
|
||||
|
||||
@@ -216,11 +216,13 @@ class MultiDeviceVnaController:
|
||||
|
||||
def _configure_reference_clocks(self) -> None:
|
||||
for device_connection in self._all_devices:
|
||||
# 1 s ACK timeout plus one retry caps worst-case at ~2 s per device
|
||||
# so a stuck reference apply cannot stall recovery for minutes.
|
||||
self._send_command_and_wait_for_acknowledgement(
|
||||
device_connection,
|
||||
PacketType.REFERENCE_SETTINGS,
|
||||
build_reference_settings_payload(0, self._force_external_reference),
|
||||
timeout_seconds=3.0,
|
||||
timeout_seconds=1.0,
|
||||
retry_count=1,
|
||||
)
|
||||
|
||||
@@ -241,6 +243,8 @@ class MultiDeviceVnaController:
|
||||
(self._master_device, True),
|
||||
]
|
||||
for device_connection, is_synchronization_master in sweep_configuration_commands:
|
||||
# 1 s ACK timeout plus one retry caps worst-case at ~2 s per device
|
||||
# so a stuck sweep apply cannot stall recovery for minutes.
|
||||
self._send_command_and_wait_for_acknowledgement(
|
||||
device_connection,
|
||||
PacketType.SWEEP_SETTINGS,
|
||||
@@ -250,7 +254,7 @@ class MultiDeviceVnaController:
|
||||
synchronization_enabled=self._synchronization_enabled,
|
||||
master_stimulus_ports=master_stimulus_ports,
|
||||
),
|
||||
timeout_seconds=3.0,
|
||||
timeout_seconds=1.0,
|
||||
retry_count=1,
|
||||
)
|
||||
self._last_applied_sweep_configuration = replace(sweep_configuration)
|
||||
|
||||
@@ -23,6 +23,19 @@ from python_app.hardware_full.librevna_multi_device_driver.transport import Libr
|
||||
|
||||
LIBREVNA_NATIVE_SWEEP_TIMEOUT_SECONDS = 1.5
|
||||
|
||||
# Hard upper bound on how long one full sweep cycle is allowed to take from
|
||||
# the moment the collection thread enters its loop. Even a device that keeps
|
||||
# streaming valid-looking datapoints will be abandoned once this deadline
|
||||
# elapses, so the caller's recovery loop can re-open it instead of waiting
|
||||
# forever. Kept comfortably above the worst real-world cycle (≈ points / IFBW).
|
||||
_MAX_FULL_CYCLE_SECONDS = 8.0
|
||||
|
||||
# Maximum time we let collection threads linger after `stop_collection_requested`
|
||||
# has been set. They are all daemon threads and self-poll the flag every
|
||||
# ~0.2 s, so a 2 s grace period is generous. Past this point we stop joining
|
||||
# and let the orphan thread die when the producer process exits.
|
||||
_THREAD_JOIN_TIMEOUT_SECONDS = 2.0
|
||||
|
||||
|
||||
def collect_complete_running_sweep_cycles(
|
||||
*,
|
||||
@@ -82,8 +95,13 @@ def collect_complete_running_sweep_cycles(
|
||||
) -> None:
|
||||
datapoints_received = 0
|
||||
expected_datapoint_count = cycle_count * point_count
|
||||
last_datapoint_timestamp = time.monotonic()
|
||||
collection_loop_start = last_datapoint_timestamp
|
||||
loop_start_timestamp = time.monotonic()
|
||||
# Tracks the last time we actually accepted a datapoint into the cycle.
|
||||
# Crucially, *not* updated on rejected datapoints — a device that keeps
|
||||
# streaming valid-looking frames the handler ignores (e.g. waiting on
|
||||
# point_index=0, or after cycle_count has been reached) must still hit
|
||||
# the per-device timeout and trigger recovery instead of looping forever.
|
||||
last_consumed_timestamp = loop_start_timestamp
|
||||
has_consumed_any_datapoint = False
|
||||
|
||||
while datapoints_received < expected_datapoint_count:
|
||||
@@ -91,19 +109,19 @@ def collect_complete_running_sweep_cycles(
|
||||
return
|
||||
|
||||
now = time.monotonic()
|
||||
remaining_timeout_seconds = (last_datapoint_timestamp + datapoint_timeout_seconds) - now
|
||||
remaining_timeout_seconds = (last_consumed_timestamp + datapoint_timeout_seconds) - now
|
||||
if remaining_timeout_seconds <= 0:
|
||||
collection_errors.append(
|
||||
TimeoutError(
|
||||
f"No datapoints from {device_connection.serial_number} for "
|
||||
f"No usable datapoints from {device_connection.serial_number} for "
|
||||
f"{datapoint_timeout_seconds:.1f} s "
|
||||
f"(received {datapoints_received}/{point_count})"
|
||||
f"(received {datapoints_received}/{expected_datapoint_count})"
|
||||
)
|
||||
)
|
||||
stop_collection_requested.set()
|
||||
return
|
||||
|
||||
if not has_consumed_any_datapoint and (now - collection_loop_start) > cycle_start_guard_seconds:
|
||||
if not has_consumed_any_datapoint and (now - loop_start_timestamp) > cycle_start_guard_seconds:
|
||||
collection_errors.append(
|
||||
TimeoutError(
|
||||
f"Device {device_connection.serial_number} streamed datapoints but never "
|
||||
@@ -114,6 +132,22 @@ def collect_complete_running_sweep_cycles(
|
||||
stop_collection_requested.set()
|
||||
return
|
||||
|
||||
# Hard wallclock deadline for the entire cycle. Even if every
|
||||
# datapoint refreshes `last_consumed_timestamp` and the per-packet
|
||||
# timeout never trips, we still bail out once the cycle has dragged
|
||||
# on for too long — this is the safety net the per-device timeout
|
||||
# cannot provide by itself.
|
||||
if (now - loop_start_timestamp) > _MAX_FULL_CYCLE_SECONDS:
|
||||
collection_errors.append(
|
||||
TimeoutError(
|
||||
f"Device {device_connection.serial_number} did not finish a sweep cycle "
|
||||
f"within {_MAX_FULL_CYCLE_SECONDS:.1f} s "
|
||||
f"(received {datapoints_received}/{expected_datapoint_count})"
|
||||
)
|
||||
)
|
||||
stop_collection_requested.set()
|
||||
return
|
||||
|
||||
try:
|
||||
packet_type, payload = device_connection.receive_packet(
|
||||
timeout_seconds=min(1.0, remaining_timeout_seconds)
|
||||
@@ -136,9 +170,11 @@ def collect_complete_running_sweep_cycles(
|
||||
|
||||
parsed_datapoint = parse_vna_datapoint_payload(payload)
|
||||
if parsed_datapoint and 0 <= parsed_datapoint.point_index < point_count:
|
||||
last_datapoint_timestamp = time.monotonic()
|
||||
datapoint_was_consumed = handle_datapoint(parsed_datapoint)
|
||||
if datapoint_was_consumed:
|
||||
# Only refreshed on accepted datapoints so the no-progress
|
||||
# timeout above stays honest about real cycle progress.
|
||||
last_consumed_timestamp = time.monotonic()
|
||||
has_consumed_any_datapoint = True
|
||||
datapoint_counts_by_device_serial[device_connection.serial_number] += 1
|
||||
datapoints_received += 1
|
||||
@@ -269,8 +305,35 @@ def collect_complete_running_sweep_cycles(
|
||||
|
||||
for collection_thread in collection_threads:
|
||||
collection_thread.start()
|
||||
|
||||
# Bounded join. Threads self-poll `stop_collection_requested` at most every
|
||||
# ~0.2 s (the queue.get timeout inside `receive_packet`), so a 2 s grace
|
||||
# period is more than enough for a cooperative shutdown. Anything still
|
||||
# alive after that is treated as an orphan: we set the flag a second time,
|
||||
# record an error so callers go through recovery, and stop waiting. The
|
||||
# thread is a daemon and will die with the producer process.
|
||||
deadline = time.monotonic() + _THREAD_JOIN_TIMEOUT_SECONDS
|
||||
for collection_thread in collection_threads:
|
||||
collection_thread.join()
|
||||
remaining_seconds = deadline - time.monotonic()
|
||||
collection_thread.join(timeout=max(0.0, remaining_seconds))
|
||||
|
||||
stalled_threads = [
|
||||
collection_thread for collection_thread in collection_threads if collection_thread.is_alive()
|
||||
]
|
||||
if stalled_threads:
|
||||
stop_collection_requested.set()
|
||||
# Give them one more short window in case they were just slow to react.
|
||||
secondary_deadline = time.monotonic() + 0.5
|
||||
for stalled_thread in stalled_threads:
|
||||
stalled_thread.join(timeout=max(0.0, secondary_deadline - time.monotonic()))
|
||||
still_stalled = [stalled_thread for stalled_thread in stalled_threads if stalled_thread.is_alive()]
|
||||
if still_stalled:
|
||||
collection_errors.append(
|
||||
RuntimeError(
|
||||
"Sweep collector thread(s) failed to stop within the join deadline: "
|
||||
+ ", ".join(stalled_thread.name for stalled_thread in still_stalled)
|
||||
)
|
||||
)
|
||||
|
||||
if collection_errors:
|
||||
raise RuntimeError(f"Sweep collection failed: {collection_errors[0]}") from collection_errors[0]
|
||||
|
||||
@@ -20,6 +20,12 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Delays applied between successive USB reopen attempts inside recover(). Picked
|
||||
# to give libusb time to re-enumerate a stuck device while staying short enough
|
||||
# that a healthy reconnect feels instant. The total worst-case wait is the sum
|
||||
# of all entries (1.75 s today) plus the cost of close()/open() themselves.
|
||||
_REOPEN_BACKOFF_SECONDS: tuple[float, ...] = (0.25, 0.5, 1.0)
|
||||
|
||||
_INPUT_S_PARAMETERS_BY_OUTPUT: dict[int, tuple[str, ...]] = {
|
||||
0: ("s31", "s41", "s51", "s61"),
|
||||
1: ("s32", "s42", "s52", "s62"),
|
||||
@@ -77,18 +83,60 @@ class MultiDeviceLibreVnaService:
|
||||
self._controller = None
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close native device transports."""
|
||||
if self._controller is not None:
|
||||
self._controller.close()
|
||||
self._controller = None
|
||||
"""Close native device transports; never raises.
|
||||
|
||||
Recovery loops rely on `close()` being safe to call on a half-open or
|
||||
already-broken controller. We swallow any transport-level exception here
|
||||
and just drop the reference so the next `open()` starts fresh.
|
||||
"""
|
||||
controller = self._controller
|
||||
self._controller = None
|
||||
if controller is None:
|
||||
return
|
||||
try:
|
||||
controller.close()
|
||||
except Exception as exc: # noqa: BLE001 — recovery path, never propagate
|
||||
logger.warning("Multi-device close() ignored transport error: %s", exc)
|
||||
|
||||
def recover(self) -> None:
|
||||
"""Reopen native device transports after a failed acquisition."""
|
||||
"""Reopen native device transports after a failed acquisition.
|
||||
|
||||
Tries several short backoffs so a transient USB stall does not kill the
|
||||
producer on the very first retry. Raises the last error only after
|
||||
every attempt failed — the outer acquisition loop is expected to count
|
||||
these as recovery_attempts.
|
||||
"""
|
||||
if self._using_mock_backend:
|
||||
return
|
||||
self.close()
|
||||
time.sleep(0.25)
|
||||
self.open()
|
||||
|
||||
last_error: Exception | None = None
|
||||
for attempt_index, delay_s in enumerate(_REOPEN_BACKOFF_SECONDS, start=1):
|
||||
time.sleep(delay_s)
|
||||
try:
|
||||
self.open()
|
||||
if self._controller is not None:
|
||||
logger.info(
|
||||
"Multi-device reopen succeeded on attempt %d/%d (after %.2fs)",
|
||||
attempt_index,
|
||||
len(_REOPEN_BACKOFF_SECONDS),
|
||||
delay_s,
|
||||
)
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001 — propagate only the last failure
|
||||
last_error = exc
|
||||
logger.warning(
|
||||
"Multi-device reopen attempt %d/%d failed after %.2fs: %s",
|
||||
attempt_index,
|
||||
len(_REOPEN_BACKOFF_SECONDS),
|
||||
delay_s,
|
||||
exc,
|
||||
)
|
||||
self.close() # tidy partially-opened state before next try
|
||||
|
||||
if last_error is not None:
|
||||
raise last_error
|
||||
raise RuntimeError("Multi-device recover() exhausted all reopen attempts")
|
||||
|
||||
def configure(self, sweep: RadarSweepModel) -> None:
|
||||
"""Store sweep settings for subsequent full-matrix acquisitions."""
|
||||
@@ -132,7 +180,21 @@ class MultiDeviceLibreVnaService:
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
self.recover()
|
||||
# recover() may itself fail when libusb cannot re-enumerate the
|
||||
# device fast enough; treat that as the same kind of recovery
|
||||
# attempt and try again on the next loop iteration, so a
|
||||
# transient USB hiccup cannot kill the whole producer.
|
||||
try:
|
||||
self.recover()
|
||||
except Exception as recover_exc: # noqa: BLE001
|
||||
last_error = recover_exc
|
||||
logger.warning(
|
||||
"multi-device recover() failed (%d/%d): %s",
|
||||
attempt_index + 1,
|
||||
self.recovery_attempts,
|
||||
recover_exc,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
assert last_error is not None
|
||||
raise last_error
|
||||
|
||||
@@ -10,12 +10,15 @@ synchronized SCPI round trip.
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pyvisa
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData
|
||||
from python_app.models.run_config_model import RadarSweepModel
|
||||
|
||||
@@ -324,7 +327,12 @@ class Sn9000Service:
|
||||
try:
|
||||
instrument.read_bytes(1, break_on_termchar=True)
|
||||
return
|
||||
except Exception:
|
||||
except pyvisa.errors.VisaIOError as exc:
|
||||
# Timeouts on a trailing newline are routine; anything else
|
||||
# likely means HiSLIP framing is out of sync and the next
|
||||
# request will hang — surface it in the logs.
|
||||
if exc.error_code != pyvisa.constants.StatusCode.error_timeout:
|
||||
logger.warning("SN9000 terminator drain failed: %s", exc)
|
||||
return
|
||||
|
||||
def _read_response_bytes(self, count: int) -> bytes:
|
||||
|
||||
Reference in New Issue
Block a user