some fixes
This commit is contained in:
@@ -7,9 +7,12 @@ state shared across them (runtime services, readers, history buffers, timer).
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import deque
|
||||
from contextlib import suppress
|
||||
from datetime import datetime
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
@@ -59,6 +62,7 @@ class AppWindow(
|
||||
super().__init__()
|
||||
|
||||
self._init_paths(project_root)
|
||||
self._init_headless_logger()
|
||||
self._init_runtime_services()
|
||||
self._init_config_profile_state()
|
||||
self._init_reader_handles()
|
||||
@@ -76,6 +80,42 @@ class AppWindow(
|
||||
self._root_profile_path = project_root / "run_config.json"
|
||||
self._active_profile_path = self._root_profile_path
|
||||
self._pending_startup_log_entries: list[tuple[str, str, str | None]] = []
|
||||
# Guards closeEvent against re-entrant teardown (e.g. a second signal).
|
||||
self._closing = False
|
||||
|
||||
def _init_headless_logger(self) -> None:
|
||||
"""Create a Python logger so headless WARN/ERROR reach journald and disk.
|
||||
|
||||
In headless mode the in-app log only reaches an offscreen widget, so an
|
||||
operator (or `journalctl`) would never see failures. We attach a stderr
|
||||
StreamHandler (captured by journald) plus a small rotating file under
|
||||
`runtime/logs`; in GUI mode no handler is attached and the logger stays
|
||||
inert, preserving the visible log widget as the sole sink.
|
||||
"""
|
||||
self._headless_logger: logging.Logger | None = None
|
||||
if not self._is_truthy_env("RADAR_SYSTEM_HEADLESS"):
|
||||
return
|
||||
logger = logging.getLogger("radar_system.gui")
|
||||
logger.setLevel(logging.WARNING)
|
||||
logger.propagate = False
|
||||
logger.handlers.clear()
|
||||
formatter = logging.Formatter(
|
||||
fmt="%(asctime)s | %(levelname)-5s | %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
stream_handler = logging.StreamHandler(stream=sys.stderr)
|
||||
stream_handler.setFormatter(formatter)
|
||||
logger.addHandler(stream_handler)
|
||||
# A rotating file keeps recent failures around after a journald restart.
|
||||
with suppress(Exception):
|
||||
log_dir = self._project_root / "python_app/runtime/logs"
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
file_handler = RotatingFileHandler(
|
||||
log_dir / "gui.log", maxBytes=1_000_000, backupCount=3, encoding="utf-8"
|
||||
)
|
||||
file_handler.setFormatter(formatter)
|
||||
logger.addHandler(file_handler)
|
||||
self._headless_logger = logger
|
||||
|
||||
def _init_runtime_services(self) -> None:
|
||||
"""Initialize long-lived service objects used by mixins."""
|
||||
@@ -461,6 +501,15 @@ class AppWindow(
|
||||
if level_upper == "ERROR" and hasattr(self, "_status_label"):
|
||||
self._status_label.setText("Status: error")
|
||||
|
||||
# In headless mode the offscreen widget above is invisible, so also mirror
|
||||
# WARN/ERROR to the Python logger (stderr -> journald, plus rotating file)
|
||||
# where an operator can actually observe failures.
|
||||
headless_logger = getattr(self, "_headless_logger", None)
|
||||
if headless_logger is not None and level_upper in {"WARN", "ERROR"}:
|
||||
log_message = text if not details else f"{text}\n{details}"
|
||||
log_level = logging.ERROR if level_upper == "ERROR" else logging.WARNING
|
||||
headless_logger.log(log_level, log_message)
|
||||
|
||||
def _log(self, text: str, *, once_key: str | None = None) -> None:
|
||||
"""Append informational message to runtime log panel."""
|
||||
self._append_log_entry("INFO", text, once_key=once_key)
|
||||
@@ -564,6 +613,12 @@ class AppWindow(
|
||||
|
||||
def closeEvent(self, event) -> None: # noqa: N802
|
||||
"""Ensure workers and dialogs are closed before window destruction."""
|
||||
if self._closing:
|
||||
# Re-entrant close (second signal, or window.close() after the event loop
|
||||
# already returned): teardown is in progress or done — do nothing more.
|
||||
super().closeEvent(event)
|
||||
return
|
||||
self._closing = True
|
||||
try:
|
||||
# 0) Stop the GPIO button watcher so a late press cannot start work.
|
||||
self._stop_control_button_watcher()
|
||||
|
||||
@@ -86,12 +86,23 @@ class ControlButtonWatcher(QObject):
|
||||
|
||||
def start(self) -> None:
|
||||
"""Open the GPIO line and begin watching for presses on a background thread."""
|
||||
self._line.open()
|
||||
self._stop_read_fd, self._stop_write_fd = os.pipe()
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, name="control-button-watcher", daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
# Guard against double-start: a second start would leak the first line/pipe/thread.
|
||||
if self._thread is not None:
|
||||
return
|
||||
try:
|
||||
self._line.open()
|
||||
self._stop_read_fd, self._stop_write_fd = os.pipe()
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, name="control-button-watcher", daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
except Exception:
|
||||
# Release any line/pipe fds opened before the failure so nothing leaks
|
||||
# and no orphaned thread survives a partial start.
|
||||
self._thread = None
|
||||
self._close_stop_pipe()
|
||||
self._line.close()
|
||||
raise
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Signal the watcher thread to exit and release the GPIO line and pipe."""
|
||||
|
||||
@@ -19,6 +19,8 @@ class AppWindowControlButtonMixin:
|
||||
def _init_control_button_state(self) -> None:
|
||||
"""Initialize the watcher handle before the watcher is started."""
|
||||
self._control_button_watcher: ControlButtonWatcher | None = None
|
||||
# Re-entrancy guard: ignore presses that arrive while a capture is in progress.
|
||||
self._control_button_busy = False
|
||||
|
||||
def _start_control_button_watcher(self) -> None:
|
||||
"""Open the configured GPIO button line and begin watching for presses.
|
||||
@@ -70,8 +72,17 @@ class AppWindowControlButtonMixin:
|
||||
Delivered as a queued signal from the watcher thread, so this executes
|
||||
on the GUI thread exactly like a click on "Capture Tmp Reference".
|
||||
"""
|
||||
self._log("GPIO control button pressed: capturing tmp reference.")
|
||||
self._capture_tmp_reference()
|
||||
# Ignore a re-entrant press: the capture flow spins the event loop (stop/
|
||||
# start run, dialogs), so a second queued press must not start a nested capture.
|
||||
if self._control_button_busy:
|
||||
self._log("GPIO control button press ignored: capture already in progress.")
|
||||
return
|
||||
self._control_button_busy = True
|
||||
try:
|
||||
self._log("GPIO control button pressed: capturing tmp reference.")
|
||||
self._capture_tmp_reference()
|
||||
finally:
|
||||
self._control_button_busy = False
|
||||
|
||||
def _on_control_button_failed(self, message: str) -> None:
|
||||
"""Log an unrecoverable watcher error reported from the background thread."""
|
||||
@@ -82,9 +93,24 @@ class AppWindowControlButtonMixin:
|
||||
watcher = getattr(self, "_control_button_watcher", None)
|
||||
if watcher is None:
|
||||
return
|
||||
# Disconnect first so a press queued before teardown cannot run a slot afterwards.
|
||||
self._disconnect_control_button_signals(watcher)
|
||||
try:
|
||||
watcher.stop()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._log_warning("Error stopping GPIO control button watcher", details=str(exc))
|
||||
finally:
|
||||
# Disconnect again in case stop() re-emitted, then drop and schedule deletion.
|
||||
self._disconnect_control_button_signals(watcher)
|
||||
self._control_button_watcher = None
|
||||
watcher.deleteLater()
|
||||
|
||||
@staticmethod
|
||||
def _disconnect_control_button_signals(watcher: ControlButtonWatcher) -> None:
|
||||
"""Detach the watcher's signals from their slots, tolerating already-disconnected."""
|
||||
for signal in (watcher.pressed, watcher.failed):
|
||||
try:
|
||||
signal.disconnect()
|
||||
except (TypeError, RuntimeError):
|
||||
# No connections left (or the C++ object is already gone): nothing to do.
|
||||
pass
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
from PyQt6.QtCore import QCoreApplication, QEventLoop
|
||||
|
||||
from python_app.gui.runtime.constraints import validate_processing_mode_constraints
|
||||
from python_app.gui.runtime.history import (
|
||||
@@ -27,6 +28,15 @@ from python_app.orchestration.shm_reader import ShmRingReader
|
||||
class AppWindowPipelineMixin:
|
||||
"""Controls start/stop, readers, and periodic polling of pipeline rings."""
|
||||
|
||||
# A repeated identical reader error is deduped from the log, but must still be
|
||||
# re-logged every this-many polls so a persistent failure stays visible.
|
||||
_READER_ERROR_RELOG_EVERY = 200
|
||||
# After this many consecutive identical reader errors, attempt one reader
|
||||
# reconnect; if it keeps failing past the next threshold, stop the pipeline so
|
||||
# a wedged reader cannot stay silently broken forever.
|
||||
_READER_ERROR_RECONNECT_AT = 40
|
||||
_READER_ERROR_STOP_AT = 400
|
||||
|
||||
def _processor_requires_restart(self, run_signature: tuple[object, ...]) -> bool:
|
||||
"""Return whether alive `data_processor` was started with different stable run settings."""
|
||||
return self._supervisor.is_processor_running() and self._processor_run_signature != run_signature
|
||||
@@ -137,6 +147,7 @@ class AppWindowPipelineMixin:
|
||||
# from a clean boundary and does not retain stale results-only tail.
|
||||
self._drop_pending_ring_payloads(include_results=True)
|
||||
self._last_reader_error_signature = None
|
||||
self._reader_error_repeat_count = 0
|
||||
if single_capture:
|
||||
self._single_capture_start_ns = time.monotonic_ns()
|
||||
|
||||
@@ -302,6 +313,7 @@ class AppWindowPipelineMixin:
|
||||
result_latest = self._read_all_results() if self._result_reader is not None else None
|
||||
self._update_history_indicator()
|
||||
self._last_reader_error_signature = None
|
||||
self._reader_error_repeat_count = 0
|
||||
|
||||
if self._single_capture_active:
|
||||
if self._finish_single_capture_if_ready():
|
||||
@@ -317,12 +329,67 @@ class AppWindowPipelineMixin:
|
||||
else:
|
||||
self._draw_preferred_collection(result_latest=None)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
signature = (type(exc).__name__, str(exc))
|
||||
if self._last_reader_error_signature == signature:
|
||||
return
|
||||
self._handle_reader_poll_error(exc)
|
||||
|
||||
def _handle_reader_poll_error(self, exc: Exception) -> None:
|
||||
"""Surface a reader-poll failure without spamming the log.
|
||||
|
||||
Distinct errors log once; an identical recurring error is deduped from the
|
||||
log but still drives the status label to error, is periodically re-logged,
|
||||
and after escalating thresholds triggers a reader reconnect and finally a
|
||||
pipeline stop so a wedged reader cannot fail silently forever.
|
||||
"""
|
||||
signature = (type(exc).__name__, str(exc))
|
||||
# Always reflect a reader failure in the status label, even when deduped.
|
||||
self._status_label.setText("Status: error")
|
||||
|
||||
if self._last_reader_error_signature == signature:
|
||||
self._reader_error_repeat_count = getattr(self, "_reader_error_repeat_count", 0) + 1
|
||||
else:
|
||||
self._last_reader_error_signature = signature
|
||||
self._reader_error_repeat_count = 0
|
||||
self._log_exception("Reader poll failed", exc, level="ERROR")
|
||||
|
||||
repeats = self._reader_error_repeat_count
|
||||
# Periodically re-log a persistent identical failure so it stays visible.
|
||||
if repeats and repeats % self._READER_ERROR_RELOG_EVERY == 0:
|
||||
self._log_exception(
|
||||
f"Reader poll still failing (repeat #{repeats})", exc, level="ERROR"
|
||||
)
|
||||
|
||||
if repeats >= self._READER_ERROR_STOP_AT:
|
||||
# The reader stayed wedged through a reconnect attempt; stop the
|
||||
# pipeline so the failure is unmistakable instead of an endless retry.
|
||||
self._log_error(
|
||||
f"Stopping pipeline after {repeats} consecutive reader poll failures"
|
||||
)
|
||||
self._reader_error_repeat_count = 0
|
||||
self._last_reader_error_signature = None
|
||||
self._stop_all_processes()
|
||||
elif repeats == self._READER_ERROR_RECONNECT_AT:
|
||||
self._reconnect_readers_after_error()
|
||||
|
||||
def _reconnect_readers_after_error(self) -> None:
|
||||
"""Re-open active ring readers in place to recover from a wedged reader."""
|
||||
self._log_warning("Attempting ring reader reconnect after repeated poll failures")
|
||||
try:
|
||||
for attr in ("_raw_reader", "_pre_reader", "_result_reader"):
|
||||
reader = getattr(self, attr)
|
||||
if reader is None:
|
||||
continue
|
||||
ring_name = reader._ring_name # noqa: SLF001 - reuse the reader's own ring name
|
||||
reader.close()
|
||||
setattr(self, attr, ShmRingReader(ring_name))
|
||||
# A successful reconnect clears the error state so the next failure
|
||||
# logs fresh rather than being swallowed by the stale signature.
|
||||
self._last_reader_error_signature = None
|
||||
self._reader_error_repeat_count = 0
|
||||
self._status_label.setText("Status: running")
|
||||
self._log("Ring readers reconnected after repeated poll failures")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# Leave the error signature intact so escalation to a stop still fires.
|
||||
self._log_exception("Ring reader reconnect failed", exc, level="ERROR")
|
||||
|
||||
def _finish_single_capture_if_ready(self) -> bool:
|
||||
"""Finalize single capture when the exact target result becomes available."""
|
||||
if not self._single_capture_active:
|
||||
@@ -406,6 +473,28 @@ class AppWindowPipelineMixin:
|
||||
latest = collection
|
||||
return latest
|
||||
|
||||
def _pump_events_during_drain(self, pause_s: float) -> None:
|
||||
"""Yield to the Qt event loop for `pause_s` instead of blocking on time.sleep.
|
||||
|
||||
The bounded drain loops run on the GUI thread; a raw time.sleep here freezes
|
||||
the event loop, stalling the keepalive/headless-watchdog timers and starving
|
||||
queued signals. Pumping events keeps the daemon responsive while we wait.
|
||||
"""
|
||||
app = QCoreApplication.instance()
|
||||
if app is None:
|
||||
# No event loop (e.g. unit context); fall back to a plain short sleep.
|
||||
time.sleep(pause_s)
|
||||
return
|
||||
deadline = time.monotonic() + pause_s
|
||||
while True:
|
||||
remaining_ms = int((deadline - time.monotonic()) * 1000)
|
||||
if remaining_ms <= 0:
|
||||
break
|
||||
app.processEvents(QEventLoop.ProcessEventsFlag.AllEvents, remaining_ms)
|
||||
# processEvents returns immediately when the queue empties; sleep the
|
||||
# residual in tiny slices so we neither busy-spin nor block too long.
|
||||
time.sleep(min(0.002, max(0.0, deadline - time.monotonic())))
|
||||
|
||||
def _drain_rings_once_for_history(self) -> None:
|
||||
"""Perform one non-blocking read pass to extend histories."""
|
||||
if self._raw_reader is not None:
|
||||
@@ -436,7 +525,8 @@ class AppWindowPipelineMixin:
|
||||
else:
|
||||
stable_rounds = 0
|
||||
previous = current
|
||||
time.sleep(poll_s)
|
||||
# Event-loop-friendly wait so timers/signals keep firing during drain.
|
||||
self._pump_events_during_drain(poll_s)
|
||||
|
||||
def _drain_results_until_quiet(self, *, timeout_s: float, poll_s: float) -> ResultCollection | None:
|
||||
"""Drain results until at least one result arrives and the ring becomes quiet."""
|
||||
@@ -455,7 +545,8 @@ class AppWindowPipelineMixin:
|
||||
else:
|
||||
latest_seen = latest
|
||||
stable_rounds = 0
|
||||
time.sleep(poll_s)
|
||||
# Event-loop-friendly wait so timers/signals keep firing during drain.
|
||||
self._pump_events_during_drain(poll_s)
|
||||
return latest_seen
|
||||
|
||||
def _update_history_indicator(self) -> None:
|
||||
|
||||
@@ -275,18 +275,20 @@ class AppWindowSnapshotMixin:
|
||||
got_results = result_count > start_result_count
|
||||
got_raw_or_pre = raw_count > start_raw_count or pre_count > start_pre_count
|
||||
|
||||
# Event-loop-friendly waits so the keepalive/watchdog timers and
|
||||
# queued signals keep firing instead of freezing the headless daemon.
|
||||
if got_results and (missing_raw or missing_pre) and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
self._pump_events_during_drain(0.01)
|
||||
continue
|
||||
if got_raw_or_pre and missing_results and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
self._pump_events_during_drain(0.01)
|
||||
continue
|
||||
if (
|
||||
self._result_reader is not None
|
||||
and max(raw_count, pre_count) > result_count
|
||||
and time.monotonic() < deadline
|
||||
):
|
||||
time.sleep(0.01)
|
||||
self._pump_events_during_drain(0.01)
|
||||
continue
|
||||
break
|
||||
except Exception as exc: # noqa: BLE001
|
||||
|
||||
@@ -40,10 +40,17 @@ def _install_unix_signal_handlers(app: QApplication, window: AppWindow) -> None:
|
||||
loop just long enough to deliver pending signals.
|
||||
"""
|
||||
|
||||
def _request_shutdown(*_args: object) -> None:
|
||||
def _shutdown() -> None:
|
||||
window.close()
|
||||
app.quit()
|
||||
|
||||
def _request_shutdown(signum: int, _frame: object) -> None:
|
||||
# Async-signal-safe: do the minimum from C signal context. Reset the handler
|
||||
# to default so a second signal force-terminates instead of re-entering Qt
|
||||
# teardown, then schedule the real shutdown on the next event-loop iteration.
|
||||
signal.signal(signum, signal.SIG_DFL)
|
||||
QTimer.singleShot(0, _shutdown)
|
||||
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
signal.signal(sig, _request_shutdown)
|
||||
|
||||
|
||||
@@ -320,14 +320,18 @@ class KamilAdcService:
|
||||
executable_path = str(Path(adc.executable_path).expanduser())
|
||||
return [executable_path, *adc.args, f"tty:{adc.tty_path}"]
|
||||
|
||||
def open(self) -> None:
|
||||
"""Launch the collector and start the TTY reader thread."""
|
||||
def open(self, *, stop_event: threading.Event | None = None) -> None:
|
||||
"""Launch the collector and start the TTY reader thread.
|
||||
|
||||
An optional `stop_event` lets a caller abort the TTY-wait loop promptly
|
||||
(e.g. on shutdown) instead of blocking for the full startup timeout.
|
||||
"""
|
||||
if self._reader is not None:
|
||||
return
|
||||
previous_tty_identity = _prepare_tty_path_for_collector(self.config.radar.kamil_adc.tty_path)
|
||||
try:
|
||||
self._start_process()
|
||||
self._wait_for_tty(previous_tty_identity)
|
||||
self._wait_for_tty(previous_tty_identity, stop_event=stop_event)
|
||||
reader = KamilAdcTtyReader(self.config.radar.kamil_adc.tty_path)
|
||||
reader.open()
|
||||
self._reader = reader
|
||||
@@ -416,15 +420,28 @@ class KamilAdcService:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait(timeout=1.0)
|
||||
|
||||
def _wait_for_tty(self, previous_identity: tuple[object, ...] | None) -> None:
|
||||
def _wait_for_tty(
|
||||
self,
|
||||
previous_identity: tuple[object, ...] | None,
|
||||
*,
|
||||
stop_event: threading.Event | None = None,
|
||||
) -> None:
|
||||
adc = self.config.radar.kamil_adc
|
||||
deadline = time.monotonic() + adc.startup_timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
# Abort promptly if a stop was requested mid-wait.
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
raise RuntimeError("Kamil ADC startup aborted before TTY became available")
|
||||
KamilAdcTtyReader._raise_if_process_exited(self._process)
|
||||
identity = _tty_identity(adc.tty_path)
|
||||
if identity is not None and identity != previous_identity:
|
||||
return
|
||||
time.sleep(0.05)
|
||||
# Use the stop event's wait() so a set() breaks the poll immediately.
|
||||
if stop_event is not None:
|
||||
if stop_event.wait(0.05):
|
||||
raise RuntimeError("Kamil ADC startup aborted before TTY became available")
|
||||
else:
|
||||
time.sleep(0.05)
|
||||
raise TimeoutError(
|
||||
f"Timed out waiting for Kamil ADC TTY `{adc.tty_path}` to be created by the collector"
|
||||
)
|
||||
|
||||
@@ -157,8 +157,19 @@ class USBTransport:
|
||||
logger.debug("USB disconnect requested")
|
||||
self._stop_event.set()
|
||||
|
||||
if self._rx_thread is not None and self._rx_thread.is_alive():
|
||||
self._rx_thread.join(timeout=1.0)
|
||||
rx_thread = self._rx_thread
|
||||
if rx_thread is not None and rx_thread.is_alive():
|
||||
rx_thread.join(timeout=1.0)
|
||||
if rx_thread.is_alive():
|
||||
# The RX thread is wedged inside libusb; closing the handle or
|
||||
# context now would risk a use-after-free in the still-running
|
||||
# bulkRead. Deliberately leak both rather than crash the process.
|
||||
logger.error(
|
||||
"USB RX thread did not stop within timeout; leaking USB handle/context "
|
||||
"to avoid use-after-free (serial=%s)",
|
||||
self.connected_serial,
|
||||
)
|
||||
return
|
||||
self._rx_thread = None
|
||||
|
||||
if self._handle is not None:
|
||||
|
||||
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass, field
|
||||
import logging
|
||||
import math
|
||||
import threading
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -26,6 +27,12 @@ logger = logging.getLogger(__name__)
|
||||
# of all entries (1.75 s today) plus the cost of close()/open() themselves.
|
||||
_REOPEN_BACKOFF_SECONDS: tuple[float, ...] = (0.25, 0.5, 1.0)
|
||||
|
||||
# Hard ceiling on the wall-clock time a single acquire_collection() may spend in
|
||||
# the recovery path (backoff sleeps + close()/open() cost across every retry).
|
||||
# Bounding this keeps a shutdown that arrives mid-recovery comfortably under the
|
||||
# systemd TimeoutStopSec so the unit is never SIGKILLed for hanging on exit.
|
||||
_MAX_RECOVERY_WALL_SECONDS: float = 10.0
|
||||
|
||||
_INPUT_S_PARAMETERS_BY_OUTPUT: dict[int, tuple[str, ...]] = {
|
||||
0: ("s31", "s41", "s51", "s61"),
|
||||
1: ("s32", "s42", "s52", "s62"),
|
||||
@@ -101,13 +108,22 @@ class MultiDeviceLibreVnaService:
|
||||
except Exception as exc: # noqa: BLE001 — recovery path, never propagate
|
||||
logger.warning("Multi-device close() ignored transport error: %s", exc)
|
||||
|
||||
def recover(self) -> None:
|
||||
def recover(
|
||||
self,
|
||||
*,
|
||||
stop_event: threading.Event | None = None,
|
||||
deadline_monotonic: float | None = None,
|
||||
) -> None:
|
||||
"""Reopen native device transports after a failed acquisition.
|
||||
|
||||
Tries several short backoffs so a transient USB stall does not kill the
|
||||
producer on the very first retry. Raises the last error only after
|
||||
every attempt failed — the outer acquisition loop is expected to count
|
||||
these as recovery_attempts.
|
||||
|
||||
When `stop_event` is supplied the backoff waits on it instead of
|
||||
sleeping, so a shutdown request aborts the loop immediately; an optional
|
||||
`deadline_monotonic` caps the total wall-time spent here.
|
||||
"""
|
||||
if self._using_mock_backend:
|
||||
return
|
||||
@@ -115,7 +131,21 @@ class MultiDeviceLibreVnaService:
|
||||
|
||||
last_error: Exception | None = None
|
||||
for attempt_index, delay_s in enumerate(_REOPEN_BACKOFF_SECONDS, start=1):
|
||||
time.sleep(delay_s)
|
||||
# Bail out the instant a stop is requested or the recovery budget is
|
||||
# spent, rather than committing to another (re)open attempt.
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
logger.info("Multi-device recover() aborted: stop requested")
|
||||
return
|
||||
if deadline_monotonic is not None and time.monotonic() >= deadline_monotonic:
|
||||
logger.warning("Multi-device recover() aborted: recovery time budget exhausted")
|
||||
break
|
||||
# Interruptible backoff: wait() returns early the moment stop is set.
|
||||
if stop_event is not None:
|
||||
if stop_event.wait(delay_s):
|
||||
logger.info("Multi-device recover() aborted: stop requested")
|
||||
return
|
||||
else:
|
||||
time.sleep(delay_s)
|
||||
try:
|
||||
self.open()
|
||||
if self._controller is not None:
|
||||
@@ -151,15 +181,27 @@ class MultiDeviceLibreVnaService:
|
||||
power_dbm=float(sweep.power_dbm),
|
||||
)
|
||||
|
||||
def acquire_collection(self, collection_id: int = 1) -> SweepCollection:
|
||||
"""Acquire one complete virtual 2x4 matrix in canonical combo order."""
|
||||
def acquire_collection(
|
||||
self,
|
||||
collection_id: int = 1,
|
||||
*,
|
||||
stop_event: threading.Event | None = None,
|
||||
) -> SweepCollection:
|
||||
"""Acquire one complete virtual 2x4 matrix in canonical combo order.
|
||||
|
||||
When `stop_event` is supplied it is threaded into the recovery path so a
|
||||
shutdown request aborts the backoff/retry loops promptly (default `None`
|
||||
preserves the original blocking behavior).
|
||||
"""
|
||||
if self._sweep_configuration is None:
|
||||
raise RuntimeError("Multi-device service is not configured")
|
||||
capture_start_ns = time.monotonic_ns()
|
||||
if self._using_mock_backend:
|
||||
collection = self._acquire_mock_collection(collection_id, capture_start_ns)
|
||||
else:
|
||||
collection = self._acquire_native_collection_with_recovery(collection_id, capture_start_ns)
|
||||
collection = self._acquire_native_collection_with_recovery(
|
||||
collection_id, capture_start_ns, stop_event=stop_event
|
||||
)
|
||||
collection.capture_end_ns = time.monotonic_ns()
|
||||
return collection
|
||||
|
||||
@@ -167,8 +209,13 @@ class MultiDeviceLibreVnaService:
|
||||
self,
|
||||
collection_id: int,
|
||||
capture_start_ns: int,
|
||||
*,
|
||||
stop_event: threading.Event | None = None,
|
||||
) -> SweepCollection:
|
||||
last_error: Exception | None = None
|
||||
# Cap total recovery wall-time across all retries so a shutdown that
|
||||
# lands mid-recovery stays well under the systemd TimeoutStopSec.
|
||||
recovery_deadline = time.monotonic() + _MAX_RECOVERY_WALL_SECONDS
|
||||
for attempt_index in range(self.recovery_attempts + 1):
|
||||
try:
|
||||
return self._acquire_native_collection(collection_id, capture_start_ns)
|
||||
@@ -176,6 +223,13 @@ class MultiDeviceLibreVnaService:
|
||||
last_error = exc
|
||||
if attempt_index >= self.recovery_attempts:
|
||||
break
|
||||
# Stop the moment shutdown is requested or the recovery budget is
|
||||
# spent — do not start another reconnect we cannot finish in time.
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
break
|
||||
if time.monotonic() >= recovery_deadline:
|
||||
logger.warning("multi-device recovery time budget exhausted; giving up")
|
||||
break
|
||||
logger.warning(
|
||||
"multi-device acquisition failed, reconnecting devices (%d/%d): %s",
|
||||
attempt_index + 1,
|
||||
@@ -188,7 +242,7 @@ class MultiDeviceLibreVnaService:
|
||||
# attempt and try again on the next loop iteration, so a
|
||||
# transient USB hiccup cannot kill the whole producer.
|
||||
try:
|
||||
self.recover()
|
||||
self.recover(stop_event=stop_event, deadline_monotonic=recovery_deadline)
|
||||
except Exception as recover_exc: # noqa: BLE001
|
||||
last_error = recover_exc
|
||||
logger.warning(
|
||||
@@ -198,6 +252,10 @@ class MultiDeviceLibreVnaService:
|
||||
recover_exc,
|
||||
exc_info=True,
|
||||
)
|
||||
# If recovery was interrupted by a stop request, do not loop back
|
||||
# for another acquisition attempt; let shutdown proceed.
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
break
|
||||
|
||||
assert last_error is not None
|
||||
raise last_error
|
||||
|
||||
@@ -180,9 +180,12 @@ class GpioOutputLines:
|
||||
self._line_fd = int(request.fd)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close line request and chip file descriptors."""
|
||||
self._close_line_fd()
|
||||
self._close_chip_fd()
|
||||
"""Close line request and chip file descriptors (best-effort, idempotent)."""
|
||||
try:
|
||||
self._close_line_fd()
|
||||
finally:
|
||||
# Ensure the chip fd is always closed even if closing the line fd raised.
|
||||
self._close_chip_fd()
|
||||
|
||||
def set_values(self, values: Sequence[int]) -> None:
|
||||
"""Apply output values for all requested lines."""
|
||||
@@ -212,16 +215,22 @@ class GpioOutputLines:
|
||||
raise RuntimeError(f"Failed to set GPIO output values: {exc}") from exc
|
||||
|
||||
def _close_line_fd(self) -> None:
|
||||
"""Close line file descriptor if currently open."""
|
||||
"""Close line file descriptor if currently open (best-effort, idempotent)."""
|
||||
if self._line_fd >= 0:
|
||||
os.close(self._line_fd)
|
||||
self._line_fd = -1
|
||||
try:
|
||||
os.close(self._line_fd)
|
||||
finally:
|
||||
# Always clear the fd so close() stays idempotent even if os.close raised.
|
||||
self._line_fd = -1
|
||||
|
||||
def _close_chip_fd(self) -> None:
|
||||
"""Close chip file descriptor if currently open."""
|
||||
"""Close chip file descriptor if currently open (best-effort, idempotent)."""
|
||||
if self._chip_fd >= 0:
|
||||
os.close(self._chip_fd)
|
||||
self._chip_fd = -1
|
||||
try:
|
||||
os.close(self._chip_fd)
|
||||
finally:
|
||||
# Always clear the fd so close() stays idempotent even if os.close raised.
|
||||
self._chip_fd = -1
|
||||
|
||||
|
||||
class GpioLineEventWatcher:
|
||||
@@ -316,18 +325,27 @@ class GpioLineEventWatcher:
|
||||
return int(event.id)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close line request and chip file descriptors."""
|
||||
self._close_line_fd()
|
||||
self._close_chip_fd()
|
||||
"""Close line request and chip file descriptors (best-effort, idempotent)."""
|
||||
try:
|
||||
self._close_line_fd()
|
||||
finally:
|
||||
# Ensure the chip fd is always closed even if closing the line fd raised.
|
||||
self._close_chip_fd()
|
||||
|
||||
def _close_line_fd(self) -> None:
|
||||
"""Close line file descriptor if currently open."""
|
||||
"""Close line file descriptor if currently open (best-effort, idempotent)."""
|
||||
if self._line_fd >= 0:
|
||||
os.close(self._line_fd)
|
||||
self._line_fd = -1
|
||||
try:
|
||||
os.close(self._line_fd)
|
||||
finally:
|
||||
# Always clear the fd so close() stays idempotent even if os.close raised.
|
||||
self._line_fd = -1
|
||||
|
||||
def _close_chip_fd(self) -> None:
|
||||
"""Close chip file descriptor if currently open."""
|
||||
"""Close chip file descriptor if currently open (best-effort, idempotent)."""
|
||||
if self._chip_fd >= 0:
|
||||
os.close(self._chip_fd)
|
||||
self._chip_fd = -1
|
||||
try:
|
||||
os.close(self._chip_fd)
|
||||
finally:
|
||||
# Always clear the fd so close() stays idempotent even if os.close raised.
|
||||
self._chip_fd = -1
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
from python_app.models.run_config_schema import (
|
||||
@@ -38,9 +39,67 @@ def _read_str(payload: dict[str, Any], key: str, default: str) -> str:
|
||||
value = payload.get(key, default)
|
||||
if value is None:
|
||||
return default
|
||||
# A JSON array/object reaching a scalar field is a config error, not a
|
||||
# str() fallback; surface it as ValueError to keep the error contract uniform.
|
||||
if isinstance(value, (dict, list)):
|
||||
raise ValueError(f"{key} must be a JSON string")
|
||||
return str(value)
|
||||
|
||||
|
||||
def _read_int(payload: dict[str, Any], key: str, default: int) -> int:
|
||||
"""Return payload integer, treating an explicit JSON `null` as 'use default'.
|
||||
|
||||
Without this, `int(payload.get(key, default))` raises TypeError on an
|
||||
explicit `null`. JSON arrays/objects (and other non-numeric scalars) are
|
||||
rejected as ValueError so malformed types share the config-error contract.
|
||||
"""
|
||||
value = payload.get(key, default)
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
|
||||
raise ValueError(f"{key} must be a JSON integer")
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"{key} must be a JSON integer") from exc
|
||||
|
||||
|
||||
def _read_float(payload: dict[str, Any], key: str, default: float) -> float:
|
||||
"""Return payload float, treating an explicit JSON `null` as 'use default'.
|
||||
|
||||
Rejects JSON arrays/objects (and other non-numeric scalars) as ValueError,
|
||||
and rejects non-finite values (NaN/Infinity) at decode time so the C++
|
||||
pipeline never receives a value it cannot honor.
|
||||
"""
|
||||
value = payload.get(key, default)
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
|
||||
raise ValueError(f"{key} must be a JSON number")
|
||||
try:
|
||||
result = float(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"{key} must be a JSON number") from exc
|
||||
if not math.isfinite(result):
|
||||
raise ValueError(f"{key} must be a finite number")
|
||||
return result
|
||||
|
||||
|
||||
def _read_bool(payload: dict[str, Any], key: str, default: bool) -> bool:
|
||||
"""Return payload boolean, treating an explicit JSON `null` as 'use default'.
|
||||
|
||||
Plain `bool(payload.get(key, default))` would silently flip the default to
|
||||
`False` on an explicit `null`; here `null` keeps the default instead.
|
||||
Non-boolean JSON types are rejected as ValueError.
|
||||
"""
|
||||
value = payload.get(key, default)
|
||||
if value is None:
|
||||
return default
|
||||
if not isinstance(value, bool):
|
||||
raise ValueError(f"{key} must be a JSON boolean")
|
||||
return value
|
||||
|
||||
|
||||
def _load_preprocess_asset(payload: dict[str, Any], target: PreprocessAssetModel) -> None:
|
||||
"""Load preprocess asset fields into target model."""
|
||||
target.set_name = _read_str(payload, "set_name", target.set_name)
|
||||
@@ -105,21 +164,21 @@ def run_config_from_dict(payload: dict[str, Any]) -> RunConfigModel:
|
||||
kamil_adc_payload = _as_dict(radar_payload.get("kamil_adc"), "radar.kamil_adc")
|
||||
laser_control_payload = _as_dict(radar_payload.get("laser_control"), "radar.laser_control")
|
||||
|
||||
model.radar.model = str(radar_payload.get("model", model.radar.model))
|
||||
model.radar.serial = str(radar_payload.get("serial", model.radar.serial))
|
||||
model.radar.remote_host = str(radar_payload.get("remote_host", model.radar.remote_host))
|
||||
model.radar.remote_port = int(radar_payload.get("remote_port", model.radar.remote_port))
|
||||
model.radar.driver_mode = str(radar_payload.get("driver_mode", model.radar.driver_mode))
|
||||
model.radar.mock_signal_hz = float(radar_payload.get("mock_signal_hz", model.radar.mock_signal_hz))
|
||||
model.radar.visa_library = str(radar_payload.get("visa_library", model.radar.visa_library))
|
||||
model.radar.model = _read_str(radar_payload, "model", model.radar.model)
|
||||
model.radar.serial = _read_str(radar_payload, "serial", model.radar.serial)
|
||||
model.radar.remote_host = _read_str(radar_payload, "remote_host", model.radar.remote_host)
|
||||
model.radar.remote_port = _read_int(radar_payload, "remote_port", model.radar.remote_port)
|
||||
model.radar.driver_mode = _read_str(radar_payload, "driver_mode", model.radar.driver_mode)
|
||||
model.radar.mock_signal_hz = _read_float(radar_payload, "mock_signal_hz", model.radar.mock_signal_hz)
|
||||
model.radar.visa_library = _read_str(radar_payload, "visa_library", model.radar.visa_library)
|
||||
|
||||
model.radar.sweep.start_hz = float(sweep_payload.get("start_hz", model.radar.sweep.start_hz))
|
||||
model.radar.sweep.stop_hz = float(sweep_payload.get("stop_hz", model.radar.sweep.stop_hz))
|
||||
model.radar.sweep.points = int(sweep_payload.get("points", model.radar.sweep.points))
|
||||
model.radar.sweep.if_bandwidth_hz = float(
|
||||
sweep_payload.get("if_bandwidth_hz", model.radar.sweep.if_bandwidth_hz)
|
||||
model.radar.sweep.start_hz = _read_float(sweep_payload, "start_hz", model.radar.sweep.start_hz)
|
||||
model.radar.sweep.stop_hz = _read_float(sweep_payload, "stop_hz", model.radar.sweep.stop_hz)
|
||||
model.radar.sweep.points = _read_int(sweep_payload, "points", model.radar.sweep.points)
|
||||
model.radar.sweep.if_bandwidth_hz = _read_float(
|
||||
sweep_payload, "if_bandwidth_hz", model.radar.sweep.if_bandwidth_hz
|
||||
)
|
||||
model.radar.sweep.power_dbm = float(sweep_payload.get("stimulus_power_dbm", model.radar.sweep.power_dbm))
|
||||
model.radar.sweep.power_dbm = _read_float(sweep_payload, "stimulus_power_dbm", model.radar.sweep.power_dbm)
|
||||
slave_serials_payload = multi_device_payload.get(
|
||||
"slave_serials",
|
||||
multi_device_payload.get("slave_serial_numbers", model.radar.multi_device.slave_serials),
|
||||
@@ -132,121 +191,114 @@ def run_config_from_dict(payload: dict[str, Any]) -> RunConfigModel:
|
||||
for value in slave_serials_payload.split(",")
|
||||
if value.strip()
|
||||
]
|
||||
model.radar.multi_device.force_external_reference = bool(
|
||||
multi_device_payload.get(
|
||||
"force_external_reference",
|
||||
model.radar.multi_device.force_external_reference,
|
||||
)
|
||||
model.radar.multi_device.force_external_reference = _read_bool(
|
||||
multi_device_payload,
|
||||
"force_external_reference",
|
||||
model.radar.multi_device.force_external_reference,
|
||||
)
|
||||
model.radar.multi_device.recovery_attempts = int(
|
||||
multi_device_payload.get(
|
||||
"recovery_attempts",
|
||||
model.radar.multi_device.recovery_attempts,
|
||||
)
|
||||
model.radar.multi_device.recovery_attempts = _read_int(
|
||||
multi_device_payload,
|
||||
"recovery_attempts",
|
||||
model.radar.multi_device.recovery_attempts,
|
||||
)
|
||||
model.radar.kamil_adc.project_dir = str(
|
||||
kamil_adc_payload.get("project_dir", model.radar.kamil_adc.project_dir)
|
||||
model.radar.kamil_adc.project_dir = _read_str(
|
||||
kamil_adc_payload, "project_dir", model.radar.kamil_adc.project_dir
|
||||
)
|
||||
model.radar.kamil_adc.executable_path = str(
|
||||
kamil_adc_payload.get("executable_path", model.radar.kamil_adc.executable_path)
|
||||
model.radar.kamil_adc.executable_path = _read_str(
|
||||
kamil_adc_payload, "executable_path", model.radar.kamil_adc.executable_path
|
||||
)
|
||||
model.radar.kamil_adc.tty_path = str(
|
||||
kamil_adc_payload.get("tty_path", model.radar.kamil_adc.tty_path)
|
||||
model.radar.kamil_adc.tty_path = _read_str(
|
||||
kamil_adc_payload, "tty_path", model.radar.kamil_adc.tty_path
|
||||
)
|
||||
model.radar.kamil_adc.args = _load_string_list(kamil_adc_payload, "args", "radar.kamil_adc")
|
||||
model.radar.kamil_adc.env = _load_string_dict(kamil_adc_payload, "env", "radar.kamil_adc")
|
||||
model.radar.kamil_adc.startup_timeout_s = float(
|
||||
kamil_adc_payload.get("startup_timeout_s", model.radar.kamil_adc.startup_timeout_s)
|
||||
model.radar.kamil_adc.startup_timeout_s = _read_float(
|
||||
kamil_adc_payload, "startup_timeout_s", model.radar.kamil_adc.startup_timeout_s
|
||||
)
|
||||
model.radar.kamil_adc.sweep_timeout_s = float(
|
||||
kamil_adc_payload.get("sweep_timeout_s", model.radar.kamil_adc.sweep_timeout_s)
|
||||
model.radar.kamil_adc.sweep_timeout_s = _read_float(
|
||||
kamil_adc_payload, "sweep_timeout_s", model.radar.kamil_adc.sweep_timeout_s
|
||||
)
|
||||
model.radar.kamil_adc.stop_timeout_s = float(
|
||||
kamil_adc_payload.get("stop_timeout_s", model.radar.kamil_adc.stop_timeout_s)
|
||||
model.radar.kamil_adc.stop_timeout_s = _read_float(
|
||||
kamil_adc_payload, "stop_timeout_s", model.radar.kamil_adc.stop_timeout_s
|
||||
)
|
||||
|
||||
model.radar.laser_control.enabled = bool(
|
||||
laser_control_payload.get("enabled", model.radar.laser_control.enabled)
|
||||
model.radar.laser_control.enabled = _read_bool(
|
||||
laser_control_payload, "enabled", model.radar.laser_control.enabled
|
||||
)
|
||||
model.radar.laser_control.port = str(
|
||||
laser_control_payload.get("port", model.radar.laser_control.port)
|
||||
model.radar.laser_control.port = _read_str(
|
||||
laser_control_payload, "port", model.radar.laser_control.port
|
||||
)
|
||||
model.radar.laser_control.mode = str(
|
||||
laser_control_payload.get("mode", model.radar.laser_control.mode)
|
||||
model.radar.laser_control.mode = _read_str(
|
||||
laser_control_payload, "mode", model.radar.laser_control.mode
|
||||
)
|
||||
model.radar.laser_control.pi_coeff1_p = int(
|
||||
laser_control_payload.get("pi_coeff1_p", model.radar.laser_control.pi_coeff1_p)
|
||||
model.radar.laser_control.pi_coeff1_p = _read_int(
|
||||
laser_control_payload, "pi_coeff1_p", model.radar.laser_control.pi_coeff1_p
|
||||
)
|
||||
model.radar.laser_control.pi_coeff1_i = int(
|
||||
laser_control_payload.get("pi_coeff1_i", model.radar.laser_control.pi_coeff1_i)
|
||||
model.radar.laser_control.pi_coeff1_i = _read_int(
|
||||
laser_control_payload, "pi_coeff1_i", model.radar.laser_control.pi_coeff1_i
|
||||
)
|
||||
model.radar.laser_control.pi_coeff2_p = int(
|
||||
laser_control_payload.get("pi_coeff2_p", model.radar.laser_control.pi_coeff2_p)
|
||||
model.radar.laser_control.pi_coeff2_p = _read_int(
|
||||
laser_control_payload, "pi_coeff2_p", model.radar.laser_control.pi_coeff2_p
|
||||
)
|
||||
model.radar.laser_control.pi_coeff2_i = int(
|
||||
laser_control_payload.get("pi_coeff2_i", model.radar.laser_control.pi_coeff2_i)
|
||||
model.radar.laser_control.pi_coeff2_i = _read_int(
|
||||
laser_control_payload, "pi_coeff2_i", model.radar.laser_control.pi_coeff2_i
|
||||
)
|
||||
laser_manual_payload = _as_dict(laser_control_payload.get("manual"), "radar.laser_control.manual")
|
||||
model.radar.laser_control.manual.temp1 = float(
|
||||
laser_manual_payload.get("temp1", model.radar.laser_control.manual.temp1)
|
||||
model.radar.laser_control.manual.temp1 = _read_float(
|
||||
laser_manual_payload, "temp1", model.radar.laser_control.manual.temp1
|
||||
)
|
||||
model.radar.laser_control.manual.temp2 = float(
|
||||
laser_manual_payload.get("temp2", model.radar.laser_control.manual.temp2)
|
||||
model.radar.laser_control.manual.temp2 = _read_float(
|
||||
laser_manual_payload, "temp2", model.radar.laser_control.manual.temp2
|
||||
)
|
||||
model.radar.laser_control.manual.current1 = float(
|
||||
laser_manual_payload.get("current1", model.radar.laser_control.manual.current1)
|
||||
model.radar.laser_control.manual.current1 = _read_float(
|
||||
laser_manual_payload, "current1", model.radar.laser_control.manual.current1
|
||||
)
|
||||
model.radar.laser_control.manual.current2 = float(
|
||||
laser_manual_payload.get("current2", model.radar.laser_control.manual.current2)
|
||||
model.radar.laser_control.manual.current2 = _read_float(
|
||||
laser_manual_payload, "current2", model.radar.laser_control.manual.current2
|
||||
)
|
||||
laser_variation_payload = _as_dict(
|
||||
laser_control_payload.get("variation"),
|
||||
"radar.laser_control.variation",
|
||||
)
|
||||
model.radar.laser_control.variation.variation_type = str(
|
||||
laser_variation_payload.get(
|
||||
"variation_type",
|
||||
model.radar.laser_control.variation.variation_type,
|
||||
)
|
||||
model.radar.laser_control.variation.variation_type = _read_str(
|
||||
laser_variation_payload,
|
||||
"variation_type",
|
||||
model.radar.laser_control.variation.variation_type,
|
||||
)
|
||||
model.radar.laser_control.variation.static_temp1 = float(
|
||||
laser_variation_payload.get(
|
||||
"static_temp1",
|
||||
model.radar.laser_control.variation.static_temp1,
|
||||
)
|
||||
model.radar.laser_control.variation.static_temp1 = _read_float(
|
||||
laser_variation_payload,
|
||||
"static_temp1",
|
||||
model.radar.laser_control.variation.static_temp1,
|
||||
)
|
||||
model.radar.laser_control.variation.static_temp2 = float(
|
||||
laser_variation_payload.get(
|
||||
"static_temp2",
|
||||
model.radar.laser_control.variation.static_temp2,
|
||||
)
|
||||
model.radar.laser_control.variation.static_temp2 = _read_float(
|
||||
laser_variation_payload,
|
||||
"static_temp2",
|
||||
model.radar.laser_control.variation.static_temp2,
|
||||
)
|
||||
model.radar.laser_control.variation.static_current1 = float(
|
||||
laser_variation_payload.get(
|
||||
"static_current1",
|
||||
model.radar.laser_control.variation.static_current1,
|
||||
)
|
||||
model.radar.laser_control.variation.static_current1 = _read_float(
|
||||
laser_variation_payload,
|
||||
"static_current1",
|
||||
model.radar.laser_control.variation.static_current1,
|
||||
)
|
||||
model.radar.laser_control.variation.static_current2 = float(
|
||||
laser_variation_payload.get(
|
||||
"static_current2",
|
||||
model.radar.laser_control.variation.static_current2,
|
||||
)
|
||||
model.radar.laser_control.variation.static_current2 = _read_float(
|
||||
laser_variation_payload,
|
||||
"static_current2",
|
||||
model.radar.laser_control.variation.static_current2,
|
||||
)
|
||||
model.radar.laser_control.variation.min_value = float(
|
||||
laser_variation_payload.get("min_value", model.radar.laser_control.variation.min_value)
|
||||
model.radar.laser_control.variation.min_value = _read_float(
|
||||
laser_variation_payload, "min_value", model.radar.laser_control.variation.min_value
|
||||
)
|
||||
model.radar.laser_control.variation.max_value = float(
|
||||
laser_variation_payload.get("max_value", model.radar.laser_control.variation.max_value)
|
||||
model.radar.laser_control.variation.max_value = _read_float(
|
||||
laser_variation_payload, "max_value", model.radar.laser_control.variation.max_value
|
||||
)
|
||||
model.radar.laser_control.variation.step = float(
|
||||
laser_variation_payload.get("step", model.radar.laser_control.variation.step)
|
||||
model.radar.laser_control.variation.step = _read_float(
|
||||
laser_variation_payload, "step", model.radar.laser_control.variation.step
|
||||
)
|
||||
model.radar.laser_control.variation.time_step = int(
|
||||
laser_variation_payload.get("time_step", model.radar.laser_control.variation.time_step)
|
||||
model.radar.laser_control.variation.time_step = _read_int(
|
||||
laser_variation_payload, "time_step", model.radar.laser_control.variation.time_step
|
||||
)
|
||||
model.radar.laser_control.variation.delay_time = int(
|
||||
laser_variation_payload.get("delay_time", model.radar.laser_control.variation.delay_time)
|
||||
model.radar.laser_control.variation.delay_time = _read_int(
|
||||
laser_variation_payload, "delay_time", model.radar.laser_control.variation.delay_time
|
||||
)
|
||||
|
||||
load_switch_payload(port1_payload, model.output_switch)
|
||||
|
||||
@@ -8,25 +8,68 @@ from python_app.models.run_config_schema import (
|
||||
ComboModel,
|
||||
ControlButtonModel,
|
||||
GprModel,
|
||||
RadarSweepModel,
|
||||
RingEndpointModel,
|
||||
SwitchModel,
|
||||
)
|
||||
|
||||
# Wire-format bounds shared with the C++ pipeline. The ring header stores the
|
||||
# slot size as a uint32, and capacity * slot_size must address into a single
|
||||
# shared-memory mapping, so reject values the C++ side cannot represent.
|
||||
_UINT32_MAX = (1 << 32) - 1
|
||||
_RING_SEGMENT_MAX_BYTES = 1 << 40 # 1 TiB upper bound on a single ring mapping.
|
||||
# Defensive ceiling so a malformed combos string cannot expand into a list that
|
||||
# stalls the GUI or the downstream acquisition loop.
|
||||
_MAX_COMBOS = 4096
|
||||
|
||||
|
||||
def _require_int(payload: dict[str, Any], key: str, default: int) -> int:
|
||||
"""Read an integer field, rejecting JSON arrays/objects with a named ValueError.
|
||||
|
||||
Bare ``int()`` raises ``TypeError`` on a list/dict, which escapes the
|
||||
config-error contract; surface it as a ValueError naming the field instead.
|
||||
"""
|
||||
value = payload.get(key, default)
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
|
||||
raise ValueError(f"{key} must be a JSON integer")
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"{key} must be a JSON integer") from exc
|
||||
|
||||
|
||||
def _require_str(payload: dict[str, Any], key: str, default: str) -> str:
|
||||
"""Read a string field, rejecting JSON arrays/objects with a named ValueError."""
|
||||
value = payload.get(key, default)
|
||||
if isinstance(value, (dict, list)):
|
||||
raise ValueError(f"{key} must be a JSON string")
|
||||
return str(value)
|
||||
|
||||
|
||||
def _require_bool(payload: dict[str, Any], key: str, default: bool) -> bool:
|
||||
"""Read a boolean field, rejecting non-boolean JSON types with a named ValueError."""
|
||||
value = payload.get(key, default)
|
||||
if not isinstance(value, bool):
|
||||
raise ValueError(f"{key} must be a JSON boolean")
|
||||
return value
|
||||
|
||||
|
||||
def load_switch_payload(
|
||||
payload: dict[str, Any],
|
||||
target: SwitchModel,
|
||||
) -> None:
|
||||
"""Populate switch model from payload preserving defaults for missing values."""
|
||||
target.name = str(payload.get("name", target.name))
|
||||
target.driver_mode = str(payload.get("driver_mode", target.driver_mode))
|
||||
target.driver = str(payload.get("driver", target.driver))
|
||||
target.radar_port = int(payload.get("radar_port", target.radar_port))
|
||||
target.positions = int(payload.get("positions", target.positions))
|
||||
target.default_position = int(payload.get("default_position", target.default_position))
|
||||
target.gpio_chip = str(payload.get("gpio_chip", target.gpio_chip))
|
||||
target.pin_a = int(payload.get("pin_a", target.pin_a))
|
||||
target.pin_b = int(payload.get("pin_b", target.pin_b))
|
||||
target.invert_logic = bool(payload.get("invert_logic", target.invert_logic))
|
||||
# #53: scalar reads reject array/object JSON types as ValueError (not TypeError).
|
||||
target.name = _require_str(payload, "name", target.name)
|
||||
target.driver_mode = _require_str(payload, "driver_mode", target.driver_mode)
|
||||
target.driver = _require_str(payload, "driver", target.driver)
|
||||
target.radar_port = _require_int(payload, "radar_port", target.radar_port)
|
||||
target.positions = _require_int(payload, "positions", target.positions)
|
||||
target.default_position = _require_int(payload, "default_position", target.default_position)
|
||||
target.gpio_chip = _require_str(payload, "gpio_chip", target.gpio_chip)
|
||||
target.pin_a = _require_int(payload, "pin_a", target.pin_a)
|
||||
target.pin_b = _require_int(payload, "pin_b", target.pin_b)
|
||||
target.invert_logic = _require_bool(payload, "invert_logic", target.invert_logic)
|
||||
|
||||
|
||||
def load_control_button_payload(
|
||||
@@ -34,20 +77,56 @@ def load_control_button_payload(
|
||||
target: ControlButtonModel,
|
||||
) -> None:
|
||||
"""Populate control-button model from payload preserving defaults."""
|
||||
target.enabled = bool(payload.get("enabled", target.enabled))
|
||||
target.gpio_chip = str(payload.get("gpio_chip", target.gpio_chip))
|
||||
target.pin = int(payload.get("pin", target.pin))
|
||||
target.active_low = bool(payload.get("active_low", target.active_low))
|
||||
target.bias = str(payload.get("bias", target.bias))
|
||||
target.debounce_ms = int(payload.get("debounce_ms", target.debounce_ms))
|
||||
target.action = str(payload.get("action", target.action))
|
||||
# #53: scalar reads reject array/object JSON types as ValueError (not TypeError).
|
||||
target.enabled = _require_bool(payload, "enabled", target.enabled)
|
||||
target.gpio_chip = _require_str(payload, "gpio_chip", target.gpio_chip)
|
||||
target.pin = _require_int(payload, "pin", target.pin)
|
||||
target.active_low = _require_bool(payload, "active_low", target.active_low)
|
||||
target.bias = _require_str(payload, "bias", target.bias)
|
||||
target.debounce_ms = _require_int(payload, "debounce_ms", target.debounce_ms)
|
||||
target.action = _require_str(payload, "action", target.action)
|
||||
|
||||
|
||||
def load_ring_payload(payload: dict[str, Any], target: RingEndpointModel) -> None:
|
||||
"""Populate ring endpoint model from payload preserving defaults."""
|
||||
target.name = str(payload.get("name", target.name))
|
||||
target.capacity = int(payload.get("capacity", target.capacity))
|
||||
target.slot_size_bytes = int(payload.get("slot_size_bytes", target.slot_size_bytes))
|
||||
# #53: scalar reads reject array/object JSON types as ValueError (not TypeError).
|
||||
target.name = _require_str(payload, "name", target.name)
|
||||
target.capacity = _require_int(payload, "capacity", target.capacity)
|
||||
target.slot_size_bytes = _require_int(payload, "slot_size_bytes", target.slot_size_bytes)
|
||||
# #36: enforce ring sizing in Python so a bad config fails here (in GUI/save and
|
||||
# at config load) instead of crashing the C++ ring allocator at boot.
|
||||
validate_ring_endpoint(target)
|
||||
|
||||
|
||||
def validate_ring_endpoint(ring: RingEndpointModel) -> None:
|
||||
"""Validate ring sizing against the constraints the C++ allocator requires."""
|
||||
field = ring.name or "ring"
|
||||
if ring.capacity <= 0:
|
||||
raise ValueError(f"rings.{field}.capacity must be > 0")
|
||||
if ring.slot_size_bytes <= 0:
|
||||
raise ValueError(f"rings.{field}.slot_size_bytes must be > 0")
|
||||
if ring.slot_size_bytes > _UINT32_MAX:
|
||||
raise ValueError(f"rings.{field}.slot_size_bytes exceeds the uint32 wire limit")
|
||||
# Overflow-safe: compare against the ceiling without ever forming the full
|
||||
# product, so an attacker-sized capacity cannot wrap a fixed-width index.
|
||||
if ring.capacity > _RING_SEGMENT_MAX_BYTES // ring.slot_size_bytes:
|
||||
raise ValueError(
|
||||
f"rings.{field} capacity * slot_size_bytes exceeds the maximum ring segment size"
|
||||
)
|
||||
|
||||
|
||||
def validate_sweep_model(sweep: RadarSweepModel) -> None:
|
||||
"""Validate radar sweep bounds in Python so a bad sweep fails in the GUI/save
|
||||
and at config load rather than aborting the C++ acquisition process at boot.
|
||||
"""
|
||||
# #36: points must be a positive, integral count of frequency samples.
|
||||
points = sweep.points
|
||||
if isinstance(points, bool) or not isinstance(points, int):
|
||||
raise ValueError("radar.sweep.points must be an integer")
|
||||
if points <= 0:
|
||||
raise ValueError("radar.sweep.points must be > 0")
|
||||
if float(sweep.stop_hz) < float(sweep.start_hz):
|
||||
raise ValueError("radar.sweep.stop_hz must be >= radar.sweep.start_hz")
|
||||
|
||||
|
||||
def validate_gpr_model(
|
||||
@@ -55,8 +134,17 @@ def validate_gpr_model(
|
||||
*,
|
||||
input_switch_positions: int,
|
||||
output_switch_positions: int,
|
||||
sweep: RadarSweepModel | None = None,
|
||||
) -> None:
|
||||
"""Validate stable GPR config against current switch dimensions."""
|
||||
"""Validate stable GPR config against current switch dimensions.
|
||||
|
||||
When ``sweep`` is supplied (load and GUI/save paths share this chokepoint),
|
||||
its bounds are validated here too so #36 sweep failures surface alongside the
|
||||
GPR checks instead of as a C++ boot crash.
|
||||
"""
|
||||
if sweep is not None:
|
||||
validate_sweep_model(sweep)
|
||||
|
||||
if float(gpr.relative_permittivity) <= 0.0:
|
||||
raise ValueError("gpr.relative_permittivity must be > 0")
|
||||
|
||||
@@ -93,8 +181,26 @@ def parse_combos_from_text(text: str) -> list[ComboModel]:
|
||||
if ":" not in pair:
|
||||
raise ValueError(f"Invalid combo syntax: {pair!r}. Expected input:output")
|
||||
|
||||
input_text, output_text = pair.split(":", 1)
|
||||
combos.append(ComboModel(input=int(input_text.strip()), output=int(output_text.strip())))
|
||||
# #57: cap the combo count so a pathological string cannot expand into a
|
||||
# list large enough to stall the GUI or the acquisition loop.
|
||||
if len(combos) >= _MAX_COMBOS:
|
||||
raise ValueError(f"Too many combos: limit is {_MAX_COMBOS}")
|
||||
|
||||
input_text, output_text = (side.strip() for side in pair.split(":", 1))
|
||||
# #57: reject empty sides and re-raise non-integer values naming the pair/side.
|
||||
if not input_text:
|
||||
raise ValueError(f"Invalid combo {pair!r}: input side is empty")
|
||||
if not output_text:
|
||||
raise ValueError(f"Invalid combo {pair!r}: output side is empty")
|
||||
try:
|
||||
input_value = int(input_text)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"Invalid combo {pair!r}: input {input_text!r} is not an integer") from exc
|
||||
try:
|
||||
output_value = int(output_text)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"Invalid combo {pair!r}: output {output_text!r} is not an integer") from exc
|
||||
combos.append(ComboModel(input=input_value, output=output_value))
|
||||
|
||||
if not combos:
|
||||
raise ValueError("No valid combos were provided")
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from python_app.models.run_config_model import RunConfigModel, parse_combos_from_text
|
||||
@@ -41,12 +42,25 @@ class ConfigWriter:
|
||||
asset.bundle_path = str(bundle_path)
|
||||
|
||||
def write(self, config: RunConfigModel, output_path: Path) -> Path:
|
||||
"""Write run configuration JSON file."""
|
||||
"""Atomically write run configuration JSON file.
|
||||
|
||||
Mirrors ProcessingLiveConfigWriter: dump to a sibling .tmp, flush+fsync to
|
||||
durably commit the bytes, then os.replace() onto the destination. The replace
|
||||
is atomic, so a C++ consumer can never observe a half-written config (which
|
||||
would abort it with an opaque JSON parse error), even across a crash or power
|
||||
loss mid-write on the SD-card-backed Pi.
|
||||
"""
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# allow_nan=False: a stray NaN/Infinity must fail loudly here in Python
|
||||
# rather than serialize to a non-standard token that aborts every C++
|
||||
# consumer at startup with an opaque JSON parse error.
|
||||
output_path.write_text(json.dumps(config.to_dict(), indent=2, allow_nan=False), encoding="utf-8")
|
||||
serialized = json.dumps(config.to_dict(), indent=2, allow_nan=False)
|
||||
tmp_path = output_path.with_suffix(output_path.suffix + ".tmp")
|
||||
with open(tmp_path, "w", encoding="utf-8") as handle:
|
||||
handle.write(serialized)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(tmp_path, output_path)
|
||||
return output_path
|
||||
|
||||
|
||||
|
||||
@@ -4,14 +4,23 @@ from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from typing import Iterable
|
||||
from typing import Sequence
|
||||
|
||||
# Cap each child log so a long-lived daemon cannot fill the SD card. On reaching
|
||||
# the cap the current log is rolled to `{name}.{out,err}.log.prev` and a fresh
|
||||
# log opened (see `_roll_log_if_oversized`).
|
||||
_LOG_MAX_BYTES = 8 * 1024 * 1024
|
||||
# Per-process force-kill deadline used on stop (Fix #33: own deadline each).
|
||||
_STOP_GRACE_SECONDS = 2.0
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ManagedProcess:
|
||||
@@ -27,14 +36,19 @@ class ManagedProcess:
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ProcessExitReport:
|
||||
"""Structured report for one exited managed process."""
|
||||
"""Structured report for one exited managed process.
|
||||
|
||||
Log tails are not held in memory: they are read on demand from the child log
|
||||
files only while rendering an ERROR report, so the common clean-exit path on
|
||||
every poll never pays for a 16KB read of two files (Fix #46).
|
||||
"""
|
||||
|
||||
name: str
|
||||
command: list[str]
|
||||
working_directory: Path
|
||||
return_code: int
|
||||
stdout: str
|
||||
stderr: str
|
||||
stdout_path: Path
|
||||
stderr_path: Path
|
||||
expected_clean_exit: bool
|
||||
|
||||
@property
|
||||
@@ -56,15 +70,37 @@ class ProcessExitReport:
|
||||
f"Command: {shlex.join(self.command)}",
|
||||
f"Working directory: {self.working_directory}",
|
||||
]
|
||||
if self.stderr:
|
||||
lines.append(f"stderr:\n{self.stderr}")
|
||||
if self.stdout:
|
||||
lines.append(f"stdout:\n{self.stdout}")
|
||||
if not self.stderr and not self.stdout:
|
||||
# Read the (bounded) log tails lazily, only now that we are rendering.
|
||||
stdout = "" if self.expected_clean_exit else _read_log_tail(self.stdout_path)
|
||||
stderr = "" if self.expected_clean_exit else _read_log_tail(self.stderr_path)
|
||||
if stderr:
|
||||
lines.append(f"stderr:\n{stderr}")
|
||||
if stdout:
|
||||
lines.append(f"stdout:\n{stdout}")
|
||||
if not self.expected_clean_exit and not stderr and not stdout:
|
||||
lines.append("stdout/stderr: none")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _read_log_tail(path: Path, max_bytes: int = 16384) -> str:
|
||||
"""Return the trailing `max_bytes` of a child log file, decoded best-effort.
|
||||
|
||||
Bounded so a large/long-lived log never produces an enormous exit report.
|
||||
"""
|
||||
try:
|
||||
with open(path, "rb") as handle:
|
||||
handle.seek(0, 2)
|
||||
size = handle.tell()
|
||||
if size > max_bytes:
|
||||
handle.seek(-max_bytes, 2)
|
||||
else:
|
||||
handle.seek(0)
|
||||
data = handle.read()
|
||||
except OSError:
|
||||
return ""
|
||||
return data.decode("utf-8", errors="replace").strip()
|
||||
|
||||
|
||||
class ProcessSupervisor:
|
||||
"""Start, monitor, and stop pipeline subprocesses."""
|
||||
|
||||
@@ -73,6 +109,11 @@ class ProcessSupervisor:
|
||||
self._project_root = project_root
|
||||
self._readiness_timeout_s = readiness_timeout_s
|
||||
self._processes: dict[str, ManagedProcess] = {}
|
||||
# Runtime pidfile lets us reap pipeline children left behind by a prior
|
||||
# supervisor (crash/SIGKILL) independent of in-memory state (Fix #19).
|
||||
self._runtime_dir = self._project_root / "python_app/runtime"
|
||||
self._pidfile_path = self._runtime_dir / "supervisor_children.pids"
|
||||
self._reap_stale_children()
|
||||
|
||||
def is_running(self) -> bool:
|
||||
"""Return whether acquisition-side processes are alive."""
|
||||
@@ -160,6 +201,11 @@ class ProcessSupervisor:
|
||||
stdout_path = logs_dir / f"{name}.out.log"
|
||||
stderr_path = logs_dir / f"{name}.err.log"
|
||||
|
||||
# Roll any stale (uncollected) log to `.prev` before truncating so the
|
||||
# previous run's diagnostics survive a respawn (Fix #29).
|
||||
self._roll_log_to_prev(stdout_path)
|
||||
self._roll_log_to_prev(stderr_path)
|
||||
|
||||
stdout_file = open(stdout_path, "wb")
|
||||
stderr_file = open(stderr_path, "wb")
|
||||
try:
|
||||
@@ -168,6 +214,10 @@ class ProcessSupervisor:
|
||||
cwd=self._project_root,
|
||||
stdout=stdout_file,
|
||||
stderr=stderr_file,
|
||||
# Own session/process group so signalling the group on stop also
|
||||
# reaches device-I/O grandchildren the producer may have spawned
|
||||
# (Fix #33).
|
||||
start_new_session=True,
|
||||
)
|
||||
except OSError as exc:
|
||||
stdout_file.close()
|
||||
@@ -190,6 +240,7 @@ class ProcessSupervisor:
|
||||
stdout_path=stdout_path,
|
||||
stderr_path=stderr_path,
|
||||
)
|
||||
self._write_pidfile()
|
||||
|
||||
def _acquisition_command(self, config_path: Path) -> list[str]:
|
||||
"""Return acquisition producer command selected by radar.model."""
|
||||
@@ -228,56 +279,78 @@ class ProcessSupervisor:
|
||||
return str(radar_payload.get("model", "librevna"))
|
||||
|
||||
def _stop_processes(self, names: Iterable[str]) -> None:
|
||||
"""Gracefully terminate processes, then force-kill on timeout."""
|
||||
"""Gracefully terminate processes, then force-kill on per-process timeout."""
|
||||
ordered_names = list(names)
|
||||
for name in ordered_names:
|
||||
process = self._processes.get(name)
|
||||
if process is None:
|
||||
continue
|
||||
if process.handle.poll() is None:
|
||||
process.handle.terminate()
|
||||
# Signal the whole group so device-I/O grandchildren die too (Fix #33).
|
||||
self._signal_group(process.handle.pid, signal.SIGTERM)
|
||||
|
||||
deadline = time.monotonic() + 2.0
|
||||
for name in ordered_names:
|
||||
process = self._processes.get(name)
|
||||
if process is None:
|
||||
continue
|
||||
if process.handle.poll() is not None:
|
||||
self._log_abnormal_stop_exit(process)
|
||||
continue
|
||||
|
||||
timeout = max(0.0, deadline - time.monotonic())
|
||||
# Each process gets its own kill deadline so a slow shutdown of one
|
||||
# cannot consume the grace window of the others (Fix #33).
|
||||
try:
|
||||
process.handle.wait(timeout=timeout)
|
||||
process.handle.wait(timeout=_STOP_GRACE_SECONDS)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.handle.kill()
|
||||
process.handle.wait(timeout=1.0)
|
||||
self._signal_group(process.handle.pid, signal.SIGKILL)
|
||||
try:
|
||||
process.handle.wait(timeout=1.0)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
else:
|
||||
# A negative code here is the SIGTERM we just sent (expected); only
|
||||
# a positive self-exit during the grace window is worth noting.
|
||||
self._log_abnormal_stop_exit(process)
|
||||
|
||||
self._drop_exited()
|
||||
|
||||
@staticmethod
|
||||
def _signal_group(pid: int, sig: int) -> None:
|
||||
"""Signal the child's whole process group, falling back to the child."""
|
||||
if pid is None:
|
||||
return
|
||||
try:
|
||||
os.killpg(os.getpgid(pid), sig)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
# Group already gone, or could not resolve it; fall back to the child.
|
||||
try:
|
||||
os.kill(pid, sig)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _log_abnormal_stop_exit(process: ManagedProcess) -> None:
|
||||
"""Note a process that self-exited abnormally around stop time (Fix #33).
|
||||
|
||||
Negative codes are signal-induced (e.g. the SIGTERM we send on stop) and
|
||||
are expected; only a non-zero self-exit is reported.
|
||||
"""
|
||||
return_code = process.handle.poll()
|
||||
if return_code is None or return_code <= 0:
|
||||
return
|
||||
print(
|
||||
f"process_supervisor: `{process.name}` exited abnormally with code "
|
||||
f"{return_code} around stop",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
def _drop_exited(self) -> None:
|
||||
"""Remove exited process entries from internal map."""
|
||||
exited_names = [name for name, process in self._processes.items() if process.handle.poll() is not None]
|
||||
for name in exited_names:
|
||||
self._processes.pop(name, None)
|
||||
|
||||
@staticmethod
|
||||
def _read_log_tail(path: Path, max_bytes: int = 16384) -> str:
|
||||
"""Return the trailing `max_bytes` of a child log file, decoded best-effort.
|
||||
|
||||
Bounded so a large/long-lived log never produces an enormous exit report.
|
||||
"""
|
||||
try:
|
||||
with open(path, "rb") as handle:
|
||||
handle.seek(0, 2)
|
||||
size = handle.tell()
|
||||
if size > max_bytes:
|
||||
handle.seek(-max_bytes, 2)
|
||||
else:
|
||||
handle.seek(0)
|
||||
data = handle.read()
|
||||
except OSError:
|
||||
return ""
|
||||
return data.decode("utf-8", errors="replace").strip()
|
||||
if exited_names:
|
||||
self._write_pidfile()
|
||||
|
||||
def _is_alive(self, name: str) -> bool:
|
||||
"""Return `True` when named process handle exists and is running."""
|
||||
@@ -294,19 +367,19 @@ class ProcessSupervisor:
|
||||
for name, process in self._processes.items():
|
||||
return_code = process.handle.poll()
|
||||
if return_code is None:
|
||||
# Still running: enforce the size cap so logs never grow unbounded (Fix #16).
|
||||
self._roll_log_if_oversized(process.stdout_path)
|
||||
self._roll_log_if_oversized(process.stderr_path)
|
||||
continue
|
||||
|
||||
stdout = self._read_log_tail(process.stdout_path)
|
||||
stderr = self._read_log_tail(process.stderr_path)
|
||||
|
||||
reports.append(
|
||||
ProcessExitReport(
|
||||
name=process.name,
|
||||
command=list(process.command),
|
||||
working_directory=self._project_root,
|
||||
return_code=int(return_code),
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
stdout_path=process.stdout_path,
|
||||
stderr_path=process.stderr_path,
|
||||
expected_clean_exit=bool(process.allow_clean_exit and int(return_code) == 0),
|
||||
)
|
||||
)
|
||||
@@ -314,6 +387,8 @@ class ProcessSupervisor:
|
||||
|
||||
for name in exited_names:
|
||||
self._processes.pop(name, None)
|
||||
if exited_names:
|
||||
self._write_pidfile()
|
||||
return reports
|
||||
|
||||
def _wait_until_ready(self, required_processes: Sequence[str]) -> None:
|
||||
@@ -342,3 +417,104 @@ class ProcessSupervisor:
|
||||
for process in self._processes.values()
|
||||
if process.handle.poll() is None and process.handle.pid is not None
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _roll_log_to_prev(path: Path) -> None:
|
||||
"""Roll an existing log to `{path}.prev` before it is reopened (Fix #29).
|
||||
|
||||
Preserves a stale (exited, not-yet-reported) child's last output instead
|
||||
of truncating it when a fresh log is opened for a respawn.
|
||||
"""
|
||||
if not path.exists():
|
||||
return
|
||||
try:
|
||||
path.replace(path.with_suffix(path.suffix + ".prev"))
|
||||
except OSError:
|
||||
# Best-effort: a failed roll must not block a spawn.
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _roll_log_if_oversized(path: Path) -> None:
|
||||
"""Bound a live child log to `_LOG_MAX_BYTES` so it cannot fill the SD card (Fix #16).
|
||||
|
||||
The child holds an open fd to this inode, so a rename would not redirect
|
||||
its writes. Instead keep one rolled generation via copy-to-`.prev` and
|
||||
truncate the live inode in place, freeing the allocated disk blocks.
|
||||
"""
|
||||
try:
|
||||
if path.stat().st_size <= _LOG_MAX_BYTES:
|
||||
return
|
||||
except OSError:
|
||||
return
|
||||
prev_path = path.with_suffix(path.suffix + ".prev")
|
||||
try:
|
||||
# Preserve the trailing window as the rolled generation, then truncate.
|
||||
tail = _read_log_tail(path, _LOG_MAX_BYTES)
|
||||
prev_path.write_text(tail, encoding="utf-8")
|
||||
with open(path, "r+b") as handle:
|
||||
handle.truncate(0)
|
||||
except OSError:
|
||||
# Best-effort: capping is opportunistic and must not disrupt polling.
|
||||
pass
|
||||
|
||||
def _write_pidfile(self) -> None:
|
||||
"""Persist live child PIDs so a later supervisor can reap them (Fix #19)."""
|
||||
try:
|
||||
self._runtime_dir.mkdir(parents=True, exist_ok=True)
|
||||
live_pids = [
|
||||
str(process.handle.pid)
|
||||
for process in self._processes.values()
|
||||
if process.handle.poll() is None and process.handle.pid is not None
|
||||
]
|
||||
self._pidfile_path.write_text("\n".join(live_pids), encoding="utf-8")
|
||||
except OSError:
|
||||
# Best-effort bookkeeping: failure here must not break start/stop.
|
||||
pass
|
||||
|
||||
def _reap_stale_children(self) -> None:
|
||||
"""Kill pipeline children recorded by a prior supervisor instance (Fix #19).
|
||||
|
||||
On a clean shutdown the pidfile is emptied; entries only remain when the
|
||||
previous supervisor died without stopping its children. We SIGKILL each
|
||||
stale process group so leftover pipeline binaries cannot hold the shared
|
||||
memory rings or devices hostage on the next start.
|
||||
"""
|
||||
try:
|
||||
raw = self._pidfile_path.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
return
|
||||
for token in raw.split():
|
||||
try:
|
||||
pid = int(token)
|
||||
except ValueError:
|
||||
continue
|
||||
if pid <= 1 or pid == os.getpid():
|
||||
continue
|
||||
# Guard against PID reuse: only reap if the process still looks like
|
||||
# one of our pipeline children before signalling its group.
|
||||
if self._is_stale_pipeline_pid(pid):
|
||||
self._signal_group(pid, signal.SIGKILL)
|
||||
try:
|
||||
self._pidfile_path.write_text("", encoding="utf-8")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _is_stale_pipeline_pid(self, pid: int) -> bool:
|
||||
"""Return whether `pid` still runs one of our pipeline binaries/scripts.
|
||||
|
||||
Reads `/proc/<pid>/cmdline` so a recycled PID owned by an unrelated
|
||||
process is never killed (Fix #19 safety guard).
|
||||
"""
|
||||
markers = (
|
||||
"build/bin/data_processor",
|
||||
"build/bin/data_preprocessor",
|
||||
"build/bin/sweep_orchestrator",
|
||||
"python_app.scripts.matrix_raw_producer",
|
||||
"python_app.scripts.kamil_adc_raw_producer",
|
||||
)
|
||||
try:
|
||||
raw = (Path("/proc") / str(pid) / "cmdline").read_bytes()
|
||||
except OSError:
|
||||
return False
|
||||
cmdline = raw.replace(b"\x00", b" ").decode("utf-8", errors="replace")
|
||||
return any(marker in cmdline for marker in markers)
|
||||
|
||||
@@ -53,16 +53,34 @@ class ShmRingReader:
|
||||
index = read_seq % self.capacity
|
||||
slot_offset = _HEADER_SIZE + index * (_SLOT_HEADER_SIZE + self.slot_size_bytes)
|
||||
|
||||
payload_size = self._read_u32(slot_offset)
|
||||
# Seqlock read mirroring the C++ pop: a slot is valid for read_seq R only if
|
||||
# its sequence equals R+1 and is unchanged across the payload copy (i.e. the
|
||||
# producer did not overwrite this slot mid-copy). Sequence and payload_size
|
||||
# are read first; the slot is only accepted after the re-read confirms both.
|
||||
sequence = self._read_u64(slot_offset + 8)
|
||||
if sequence != read_seq + 1:
|
||||
# Producer overwrote this slot before we read it. Resync to latest.
|
||||
self._write_u64(32, write_seq)
|
||||
return None
|
||||
|
||||
payload_size = self._read_u32(slot_offset)
|
||||
# Bound payload_size against the slot before slicing so a torn/garbage size
|
||||
# can never read out of the slot region; resync and skip on violation.
|
||||
if payload_size > self.slot_size_bytes:
|
||||
self._write_u64(32, write_seq)
|
||||
return None
|
||||
|
||||
payload_offset = slot_offset + _SLOT_HEADER_SIZE
|
||||
payload = self._mmap[payload_offset : payload_offset + payload_size]
|
||||
payload = bytes(self._mmap[payload_offset : payload_offset + payload_size])
|
||||
|
||||
# Re-read the slot sequence after the copy; if it changed, the producer
|
||||
# overwrote this slot mid-copy and the payload is torn — discard and resync.
|
||||
if self._read_u64(slot_offset + 8) != read_seq + 1:
|
||||
self._write_u64(32, write_seq)
|
||||
return None
|
||||
|
||||
self._write_u64(32, read_seq + 1)
|
||||
return bytes(payload)
|
||||
return payload
|
||||
|
||||
def pop_raw_collection(self) -> SweepCollection | None:
|
||||
"""Read next raw collection from ring."""
|
||||
|
||||
@@ -15,10 +15,20 @@ _VERSION: Final[int] = 1
|
||||
|
||||
|
||||
class ShmRingWriter:
|
||||
"""Write binary payloads into the shared-memory ring used by C++ workers."""
|
||||
"""Write binary payloads into the shared-memory ring used by C++ workers.
|
||||
|
||||
The writer is the sole *owner* of the rings it opens: there is exactly one
|
||||
producer per ring (the acquisition producer for the raw/raw_tap rings). On a
|
||||
geometry mismatch with a pre-existing segment (e.g. a stale ring left by a prior
|
||||
run with a different sweep config), the owner unlinks and recreates the segment
|
||||
from scratch rather than truncating in place or diverging silently — mirroring
|
||||
the clean-shm-on-restart contract on the C++/deploy side (#13). A non-owner must
|
||||
never recreate a ring; readers and C++ consumers only ever attach to an existing
|
||||
one.
|
||||
"""
|
||||
|
||||
def __init__(self, ring_name: str, capacity: int, slot_size_bytes: int) -> None:
|
||||
"""Open or create a POSIX SHM ring by name."""
|
||||
"""Open or create a POSIX SHM ring by name (as the ring owner)."""
|
||||
if not ring_name.startswith("/"):
|
||||
raise ValueError("ring_name must start with '/'")
|
||||
if capacity <= 0:
|
||||
@@ -31,19 +41,50 @@ class ShmRingWriter:
|
||||
self._slot_size_bytes = int(slot_size_bytes)
|
||||
self._mapped_size = _HEADER_SIZE + self._capacity * (_SLOT_HEADER_SIZE + self._slot_size_bytes)
|
||||
self._path = Path("/dev/shm") / ring_name[1:]
|
||||
self._open_owned()
|
||||
|
||||
def _open_owned(self) -> None:
|
||||
"""Open the ring, recreating it from scratch on a geometry/header mismatch.
|
||||
|
||||
As the single owner of this ring we may safely discard a stale segment: a
|
||||
size or header mismatch means the existing segment belongs to an earlier,
|
||||
incompatible run, so we unlink it and create a fresh one instead of mapping
|
||||
an inconsistent layout.
|
||||
"""
|
||||
created = not self._path.exists()
|
||||
fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o660)
|
||||
self._file = os.fdopen(fd, "r+b", buffering=0)
|
||||
if created or self._path.stat().st_size != self._mapped_size:
|
||||
# Wrong-sized stale segment: drop it entirely and recreate, so the file
|
||||
# and any future mapping agree on geometry instead of being truncated
|
||||
# under a producer/consumer that still expects the old layout.
|
||||
self._file.truncate(self._mapped_size)
|
||||
created = True
|
||||
|
||||
self._mmap = mmap.mmap(self._file.fileno(), self._mapped_size)
|
||||
if created:
|
||||
self._initialize_header()
|
||||
else:
|
||||
self._validate_header()
|
||||
return
|
||||
|
||||
# Size matched but the header geometry/magic does not: the owner recreates
|
||||
# rather than diverge. Unlink and reopen as a brand-new ring.
|
||||
if not self._header_matches():
|
||||
self._mmap.close()
|
||||
self._file.close()
|
||||
self._unlink_if_present()
|
||||
created = not self._path.exists()
|
||||
fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o660)
|
||||
self._file = os.fdopen(fd, "r+b", buffering=0)
|
||||
self._file.truncate(self._mapped_size)
|
||||
self._mmap = mmap.mmap(self._file.fileno(), self._mapped_size)
|
||||
self._initialize_header()
|
||||
|
||||
def _unlink_if_present(self) -> None:
|
||||
"""Remove the backing /dev/shm file if it exists (owner-only operation)."""
|
||||
try:
|
||||
self._path.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close mmap and file handle."""
|
||||
@@ -106,6 +147,19 @@ class ShmRingWriter:
|
||||
if capacity != self._capacity or slot_size_bytes != self._slot_size_bytes:
|
||||
raise RuntimeError(f"Shared memory ring geometry mismatch for {self._ring_name}")
|
||||
|
||||
def _header_matches(self) -> bool:
|
||||
"""Return whether the existing segment's header matches this ring's geometry.
|
||||
|
||||
Non-throwing counterpart of `_validate_header` used by the owner to decide
|
||||
whether a same-sized pre-existing segment can be reused or must be recreated.
|
||||
"""
|
||||
return (
|
||||
self._mmap[:8] == _MAGIC
|
||||
and self._read_u32(8) == _VERSION
|
||||
and self._read_u32(12) == self._capacity
|
||||
and self._read_u32(16) == self._slot_size_bytes
|
||||
)
|
||||
|
||||
def _read_u32(self, offset: int) -> int:
|
||||
return struct.unpack_from("<I", self._mmap, offset)[0]
|
||||
|
||||
|
||||
@@ -21,6 +21,81 @@ from python_app.storage.npz.serialize import RAW_MAGIC, serialize_trace_collecti
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The producer waits for the Kamil ADC collector forever: a device/collector that
|
||||
# is absent at boot or disappears mid-run must never kill the producer, only make
|
||||
# it wait. Reconnect uses a capped exponential backoff so a long absence does not
|
||||
# busy-spin, and every wait is interruptible by SIGINT/SIGTERM (stop_requested) for
|
||||
# a prompt clean exit. Mirrors matrix_raw_producer._open_radar_with_retry.
|
||||
_OPEN_RETRY_MIN_S = 1.0
|
||||
_OPEN_RETRY_MAX_S = 10.0
|
||||
# Throttle open-failure logging during a long wait so a permanently absent device
|
||||
# does not flood the process log: log the first failure, then every Nth attempt.
|
||||
_OPEN_RETRY_LOG_EVERY = 30
|
||||
|
||||
|
||||
def _open_radar_with_retry(
|
||||
config: RunConfigModel,
|
||||
radar: KamilAdcService,
|
||||
input_switch: SwitchService,
|
||||
output_switch: SwitchService,
|
||||
stop_requested: threading.Event,
|
||||
) -> bool:
|
||||
"""Open+configure the radar and both switches, retrying forever until stop.
|
||||
|
||||
Used for both the initial open and every in-loop reconnect, so a collector
|
||||
that is absent at boot or disappears mid-run never kills the producer — it just
|
||||
waits. Any partially-opened components are closed before each retry so a
|
||||
relaunched collector starts clean. Returns ``True`` once everything is open, or
|
||||
``False`` if a stop was requested before the device became available. Backoff is
|
||||
capped and every wait is interruptible by SIGTERM.
|
||||
"""
|
||||
# Tear down any prior open first: open()/switch.open() are idempotent no-ops
|
||||
# while still "open", so a mid-run reconnect must close them to force a fresh
|
||||
# collector relaunch and TTY re-attach.
|
||||
with suppress(Exception):
|
||||
input_switch.close()
|
||||
with suppress(Exception):
|
||||
output_switch.close()
|
||||
with suppress(Exception):
|
||||
radar.close()
|
||||
|
||||
attempt = 0
|
||||
delay = _OPEN_RETRY_MIN_S
|
||||
while not stop_requested.is_set():
|
||||
try:
|
||||
radar.open(stop_event=stop_requested)
|
||||
radar.configure(config.radar.sweep)
|
||||
output_switch.open()
|
||||
input_switch.open()
|
||||
except Exception as exc: # noqa: BLE001 — waiting for the device is the point
|
||||
# Drop any partial open (collector process, TTY reader, switches)
|
||||
# before the next attempt so the relaunch starts from a clean state.
|
||||
with suppress(Exception):
|
||||
input_switch.close()
|
||||
with suppress(Exception):
|
||||
output_switch.close()
|
||||
with suppress(Exception):
|
||||
radar.close()
|
||||
attempt += 1
|
||||
if attempt == 1 or attempt % _OPEN_RETRY_LOG_EVERY == 0:
|
||||
logger.warning(
|
||||
"Kamil ADC not available (attempt %d); retrying every up to %.0fs "
|
||||
"until the device is present: %s",
|
||||
attempt,
|
||||
_OPEN_RETRY_MAX_S,
|
||||
exc,
|
||||
)
|
||||
if stop_requested.wait(delay):
|
||||
return False
|
||||
delay = min(delay * 2.0, _OPEN_RETRY_MAX_S)
|
||||
continue
|
||||
|
||||
if attempt > 0:
|
||||
logger.info("Kamil ADC opened after %d attempt(s).", attempt + 1)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Run producer process until config or signal requests exit."""
|
||||
@@ -57,10 +132,8 @@ def main() -> int:
|
||||
output_switch = _switch_from_model(config.output_switch)
|
||||
|
||||
try:
|
||||
radar.open()
|
||||
radar.configure(config.radar.sweep)
|
||||
output_switch.open()
|
||||
input_switch.open()
|
||||
if not _open_radar_with_retry(config, radar, input_switch, output_switch, stop_requested):
|
||||
return 0 # asked to stop before a device became available
|
||||
|
||||
collection_id = 1
|
||||
while not stop_requested.is_set():
|
||||
@@ -68,26 +141,49 @@ def main() -> int:
|
||||
capture_start_ns = time.monotonic_ns()
|
||||
traces: list[TraceData] = []
|
||||
|
||||
for combo in config.combos:
|
||||
try:
|
||||
for combo in config.combos:
|
||||
if stop_requested.is_set():
|
||||
break
|
||||
output_switch.switch_to(combo.output)
|
||||
input_switch.switch_to(combo.input)
|
||||
if config.runtime.settling_ms > 0:
|
||||
time.sleep(config.runtime.settling_ms / 1000.0)
|
||||
|
||||
sweep = radar.acquire()
|
||||
traces.append(
|
||||
TraceData(
|
||||
combo=ComboKey(input=combo.input, output=combo.output),
|
||||
frequency_hz=np.asarray(sweep.x, dtype=np.float32),
|
||||
s11=np.asarray(sweep.trace("s11"), dtype=np.complex64),
|
||||
s21=np.asarray(sweep.trace("s21"), dtype=np.complex64),
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — reconnect forever, never give up
|
||||
logger.warning(
|
||||
"Kamil ADC acquisition failed; reconnecting and waiting for the device: %s",
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
if not _open_radar_with_retry(config, radar, input_switch, output_switch, stop_requested):
|
||||
break # stop requested while waiting to reconnect
|
||||
continue
|
||||
|
||||
# An incomplete sweep set means the device dropped out (or a stop was
|
||||
# requested mid-collection). Only exit on stop; otherwise reconnect and
|
||||
# wait for the device rather than killing the producer.
|
||||
if len(traces) != len(config.combos):
|
||||
if stop_requested.is_set():
|
||||
break
|
||||
output_switch.switch_to(combo.output)
|
||||
input_switch.switch_to(combo.input)
|
||||
if config.runtime.settling_ms > 0:
|
||||
time.sleep(config.runtime.settling_ms / 1000.0)
|
||||
|
||||
sweep = radar.acquire()
|
||||
traces.append(
|
||||
TraceData(
|
||||
combo=ComboKey(input=combo.input, output=combo.output),
|
||||
frequency_hz=np.asarray(sweep.x, dtype=np.float32),
|
||||
s11=np.asarray(sweep.trace("s11"), dtype=np.complex64),
|
||||
s21=np.asarray(sweep.trace("s21"), dtype=np.complex64),
|
||||
)
|
||||
logger.warning(
|
||||
"Kamil ADC produced an incomplete collection (%d of %d combos); "
|
||||
"reconnecting and waiting for the device",
|
||||
len(traces),
|
||||
len(config.combos),
|
||||
)
|
||||
|
||||
if len(traces) != len(config.combos):
|
||||
break
|
||||
if not _open_radar_with_retry(config, radar, input_switch, output_switch, stop_requested):
|
||||
break # stop requested while waiting to reconnect
|
||||
continue
|
||||
|
||||
collection = SweepCollection(
|
||||
collection_id=collection_id,
|
||||
|
||||
Reference in New Issue
Block a user