some fixes
This commit is contained in:
@@ -7,9 +7,12 @@ state shared across them (runtime services, readers, history buffers, timer).
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import deque
|
||||
from contextlib import suppress
|
||||
from datetime import datetime
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
@@ -59,6 +62,7 @@ class AppWindow(
|
||||
super().__init__()
|
||||
|
||||
self._init_paths(project_root)
|
||||
self._init_headless_logger()
|
||||
self._init_runtime_services()
|
||||
self._init_config_profile_state()
|
||||
self._init_reader_handles()
|
||||
@@ -76,6 +80,42 @@ class AppWindow(
|
||||
self._root_profile_path = project_root / "run_config.json"
|
||||
self._active_profile_path = self._root_profile_path
|
||||
self._pending_startup_log_entries: list[tuple[str, str, str | None]] = []
|
||||
# Guards closeEvent against re-entrant teardown (e.g. a second signal).
|
||||
self._closing = False
|
||||
|
||||
def _init_headless_logger(self) -> None:
|
||||
"""Create a Python logger so headless WARN/ERROR reach journald and disk.
|
||||
|
||||
In headless mode the in-app log only reaches an offscreen widget, so an
|
||||
operator (or `journalctl`) would never see failures. We attach a stderr
|
||||
StreamHandler (captured by journald) plus a small rotating file under
|
||||
`runtime/logs`; in GUI mode no handler is attached and the logger stays
|
||||
inert, preserving the visible log widget as the sole sink.
|
||||
"""
|
||||
self._headless_logger: logging.Logger | None = None
|
||||
if not self._is_truthy_env("RADAR_SYSTEM_HEADLESS"):
|
||||
return
|
||||
logger = logging.getLogger("radar_system.gui")
|
||||
logger.setLevel(logging.WARNING)
|
||||
logger.propagate = False
|
||||
logger.handlers.clear()
|
||||
formatter = logging.Formatter(
|
||||
fmt="%(asctime)s | %(levelname)-5s | %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
stream_handler = logging.StreamHandler(stream=sys.stderr)
|
||||
stream_handler.setFormatter(formatter)
|
||||
logger.addHandler(stream_handler)
|
||||
# A rotating file keeps recent failures around after a journald restart.
|
||||
with suppress(Exception):
|
||||
log_dir = self._project_root / "python_app/runtime/logs"
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
file_handler = RotatingFileHandler(
|
||||
log_dir / "gui.log", maxBytes=1_000_000, backupCount=3, encoding="utf-8"
|
||||
)
|
||||
file_handler.setFormatter(formatter)
|
||||
logger.addHandler(file_handler)
|
||||
self._headless_logger = logger
|
||||
|
||||
def _init_runtime_services(self) -> None:
|
||||
"""Initialize long-lived service objects used by mixins."""
|
||||
@@ -461,6 +501,15 @@ class AppWindow(
|
||||
if level_upper == "ERROR" and hasattr(self, "_status_label"):
|
||||
self._status_label.setText("Status: error")
|
||||
|
||||
# In headless mode the offscreen widget above is invisible, so also mirror
|
||||
# WARN/ERROR to the Python logger (stderr -> journald, plus rotating file)
|
||||
# where an operator can actually observe failures.
|
||||
headless_logger = getattr(self, "_headless_logger", None)
|
||||
if headless_logger is not None and level_upper in {"WARN", "ERROR"}:
|
||||
log_message = text if not details else f"{text}\n{details}"
|
||||
log_level = logging.ERROR if level_upper == "ERROR" else logging.WARNING
|
||||
headless_logger.log(log_level, log_message)
|
||||
|
||||
def _log(self, text: str, *, once_key: str | None = None) -> None:
|
||||
"""Append informational message to runtime log panel."""
|
||||
self._append_log_entry("INFO", text, once_key=once_key)
|
||||
@@ -564,6 +613,12 @@ class AppWindow(
|
||||
|
||||
def closeEvent(self, event) -> None: # noqa: N802
|
||||
"""Ensure workers and dialogs are closed before window destruction."""
|
||||
if self._closing:
|
||||
# Re-entrant close (second signal, or window.close() after the event loop
|
||||
# already returned): teardown is in progress or done — do nothing more.
|
||||
super().closeEvent(event)
|
||||
return
|
||||
self._closing = True
|
||||
try:
|
||||
# 0) Stop the GPIO button watcher so a late press cannot start work.
|
||||
self._stop_control_button_watcher()
|
||||
|
||||
@@ -86,12 +86,23 @@ class ControlButtonWatcher(QObject):
|
||||
|
||||
def start(self) -> None:
|
||||
"""Open the GPIO line and begin watching for presses on a background thread."""
|
||||
self._line.open()
|
||||
self._stop_read_fd, self._stop_write_fd = os.pipe()
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, name="control-button-watcher", daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
# Guard against double-start: a second start would leak the first line/pipe/thread.
|
||||
if self._thread is not None:
|
||||
return
|
||||
try:
|
||||
self._line.open()
|
||||
self._stop_read_fd, self._stop_write_fd = os.pipe()
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, name="control-button-watcher", daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
except Exception:
|
||||
# Release any line/pipe fds opened before the failure so nothing leaks
|
||||
# and no orphaned thread survives a partial start.
|
||||
self._thread = None
|
||||
self._close_stop_pipe()
|
||||
self._line.close()
|
||||
raise
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Signal the watcher thread to exit and release the GPIO line and pipe."""
|
||||
|
||||
@@ -19,6 +19,8 @@ class AppWindowControlButtonMixin:
|
||||
def _init_control_button_state(self) -> None:
|
||||
"""Initialize the watcher handle before the watcher is started."""
|
||||
self._control_button_watcher: ControlButtonWatcher | None = None
|
||||
# Re-entrancy guard: ignore presses that arrive while a capture is in progress.
|
||||
self._control_button_busy = False
|
||||
|
||||
def _start_control_button_watcher(self) -> None:
|
||||
"""Open the configured GPIO button line and begin watching for presses.
|
||||
@@ -70,8 +72,17 @@ class AppWindowControlButtonMixin:
|
||||
Delivered as a queued signal from the watcher thread, so this executes
|
||||
on the GUI thread exactly like a click on "Capture Tmp Reference".
|
||||
"""
|
||||
self._log("GPIO control button pressed: capturing tmp reference.")
|
||||
self._capture_tmp_reference()
|
||||
# Ignore a re-entrant press: the capture flow spins the event loop (stop/
|
||||
# start run, dialogs), so a second queued press must not start a nested capture.
|
||||
if self._control_button_busy:
|
||||
self._log("GPIO control button press ignored: capture already in progress.")
|
||||
return
|
||||
self._control_button_busy = True
|
||||
try:
|
||||
self._log("GPIO control button pressed: capturing tmp reference.")
|
||||
self._capture_tmp_reference()
|
||||
finally:
|
||||
self._control_button_busy = False
|
||||
|
||||
def _on_control_button_failed(self, message: str) -> None:
|
||||
"""Log an unrecoverable watcher error reported from the background thread."""
|
||||
@@ -82,9 +93,24 @@ class AppWindowControlButtonMixin:
|
||||
watcher = getattr(self, "_control_button_watcher", None)
|
||||
if watcher is None:
|
||||
return
|
||||
# Disconnect first so a press queued before teardown cannot run a slot afterwards.
|
||||
self._disconnect_control_button_signals(watcher)
|
||||
try:
|
||||
watcher.stop()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._log_warning("Error stopping GPIO control button watcher", details=str(exc))
|
||||
finally:
|
||||
# Disconnect again in case stop() re-emitted, then drop and schedule deletion.
|
||||
self._disconnect_control_button_signals(watcher)
|
||||
self._control_button_watcher = None
|
||||
watcher.deleteLater()
|
||||
|
||||
@staticmethod
|
||||
def _disconnect_control_button_signals(watcher: ControlButtonWatcher) -> None:
|
||||
"""Detach the watcher's signals from their slots, tolerating already-disconnected."""
|
||||
for signal in (watcher.pressed, watcher.failed):
|
||||
try:
|
||||
signal.disconnect()
|
||||
except (TypeError, RuntimeError):
|
||||
# No connections left (or the C++ object is already gone): nothing to do.
|
||||
pass
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
from PyQt6.QtCore import QCoreApplication, QEventLoop
|
||||
|
||||
from python_app.gui.runtime.constraints import validate_processing_mode_constraints
|
||||
from python_app.gui.runtime.history import (
|
||||
@@ -27,6 +28,15 @@ from python_app.orchestration.shm_reader import ShmRingReader
|
||||
class AppWindowPipelineMixin:
|
||||
"""Controls start/stop, readers, and periodic polling of pipeline rings."""
|
||||
|
||||
# A repeated identical reader error is deduped from the log, but must still be
|
||||
# re-logged every this-many polls so a persistent failure stays visible.
|
||||
_READER_ERROR_RELOG_EVERY = 200
|
||||
# After this many consecutive identical reader errors, attempt one reader
|
||||
# reconnect; if it keeps failing past the next threshold, stop the pipeline so
|
||||
# a wedged reader cannot stay silently broken forever.
|
||||
_READER_ERROR_RECONNECT_AT = 40
|
||||
_READER_ERROR_STOP_AT = 400
|
||||
|
||||
def _processor_requires_restart(self, run_signature: tuple[object, ...]) -> bool:
|
||||
"""Return whether alive `data_processor` was started with different stable run settings."""
|
||||
return self._supervisor.is_processor_running() and self._processor_run_signature != run_signature
|
||||
@@ -137,6 +147,7 @@ class AppWindowPipelineMixin:
|
||||
# from a clean boundary and does not retain stale results-only tail.
|
||||
self._drop_pending_ring_payloads(include_results=True)
|
||||
self._last_reader_error_signature = None
|
||||
self._reader_error_repeat_count = 0
|
||||
if single_capture:
|
||||
self._single_capture_start_ns = time.monotonic_ns()
|
||||
|
||||
@@ -302,6 +313,7 @@ class AppWindowPipelineMixin:
|
||||
result_latest = self._read_all_results() if self._result_reader is not None else None
|
||||
self._update_history_indicator()
|
||||
self._last_reader_error_signature = None
|
||||
self._reader_error_repeat_count = 0
|
||||
|
||||
if self._single_capture_active:
|
||||
if self._finish_single_capture_if_ready():
|
||||
@@ -317,12 +329,67 @@ class AppWindowPipelineMixin:
|
||||
else:
|
||||
self._draw_preferred_collection(result_latest=None)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
signature = (type(exc).__name__, str(exc))
|
||||
if self._last_reader_error_signature == signature:
|
||||
return
|
||||
self._handle_reader_poll_error(exc)
|
||||
|
||||
def _handle_reader_poll_error(self, exc: Exception) -> None:
|
||||
"""Surface a reader-poll failure without spamming the log.
|
||||
|
||||
Distinct errors log once; an identical recurring error is deduped from the
|
||||
log but still drives the status label to error, is periodically re-logged,
|
||||
and after escalating thresholds triggers a reader reconnect and finally a
|
||||
pipeline stop so a wedged reader cannot fail silently forever.
|
||||
"""
|
||||
signature = (type(exc).__name__, str(exc))
|
||||
# Always reflect a reader failure in the status label, even when deduped.
|
||||
self._status_label.setText("Status: error")
|
||||
|
||||
if self._last_reader_error_signature == signature:
|
||||
self._reader_error_repeat_count = getattr(self, "_reader_error_repeat_count", 0) + 1
|
||||
else:
|
||||
self._last_reader_error_signature = signature
|
||||
self._reader_error_repeat_count = 0
|
||||
self._log_exception("Reader poll failed", exc, level="ERROR")
|
||||
|
||||
repeats = self._reader_error_repeat_count
|
||||
# Periodically re-log a persistent identical failure so it stays visible.
|
||||
if repeats and repeats % self._READER_ERROR_RELOG_EVERY == 0:
|
||||
self._log_exception(
|
||||
f"Reader poll still failing (repeat #{repeats})", exc, level="ERROR"
|
||||
)
|
||||
|
||||
if repeats >= self._READER_ERROR_STOP_AT:
|
||||
# The reader stayed wedged through a reconnect attempt; stop the
|
||||
# pipeline so the failure is unmistakable instead of an endless retry.
|
||||
self._log_error(
|
||||
f"Stopping pipeline after {repeats} consecutive reader poll failures"
|
||||
)
|
||||
self._reader_error_repeat_count = 0
|
||||
self._last_reader_error_signature = None
|
||||
self._stop_all_processes()
|
||||
elif repeats == self._READER_ERROR_RECONNECT_AT:
|
||||
self._reconnect_readers_after_error()
|
||||
|
||||
def _reconnect_readers_after_error(self) -> None:
|
||||
"""Re-open active ring readers in place to recover from a wedged reader."""
|
||||
self._log_warning("Attempting ring reader reconnect after repeated poll failures")
|
||||
try:
|
||||
for attr in ("_raw_reader", "_pre_reader", "_result_reader"):
|
||||
reader = getattr(self, attr)
|
||||
if reader is None:
|
||||
continue
|
||||
ring_name = reader._ring_name # noqa: SLF001 - reuse the reader's own ring name
|
||||
reader.close()
|
||||
setattr(self, attr, ShmRingReader(ring_name))
|
||||
# A successful reconnect clears the error state so the next failure
|
||||
# logs fresh rather than being swallowed by the stale signature.
|
||||
self._last_reader_error_signature = None
|
||||
self._reader_error_repeat_count = 0
|
||||
self._status_label.setText("Status: running")
|
||||
self._log("Ring readers reconnected after repeated poll failures")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# Leave the error signature intact so escalation to a stop still fires.
|
||||
self._log_exception("Ring reader reconnect failed", exc, level="ERROR")
|
||||
|
||||
def _finish_single_capture_if_ready(self) -> bool:
|
||||
"""Finalize single capture when the exact target result becomes available."""
|
||||
if not self._single_capture_active:
|
||||
@@ -406,6 +473,28 @@ class AppWindowPipelineMixin:
|
||||
latest = collection
|
||||
return latest
|
||||
|
||||
def _pump_events_during_drain(self, pause_s: float) -> None:
|
||||
"""Yield to the Qt event loop for `pause_s` instead of blocking on time.sleep.
|
||||
|
||||
The bounded drain loops run on the GUI thread; a raw time.sleep here freezes
|
||||
the event loop, stalling the keepalive/headless-watchdog timers and starving
|
||||
queued signals. Pumping events keeps the daemon responsive while we wait.
|
||||
"""
|
||||
app = QCoreApplication.instance()
|
||||
if app is None:
|
||||
# No event loop (e.g. unit context); fall back to a plain short sleep.
|
||||
time.sleep(pause_s)
|
||||
return
|
||||
deadline = time.monotonic() + pause_s
|
||||
while True:
|
||||
remaining_ms = int((deadline - time.monotonic()) * 1000)
|
||||
if remaining_ms <= 0:
|
||||
break
|
||||
app.processEvents(QEventLoop.ProcessEventsFlag.AllEvents, remaining_ms)
|
||||
# processEvents returns immediately when the queue empties; sleep the
|
||||
# residual in tiny slices so we neither busy-spin nor block too long.
|
||||
time.sleep(min(0.002, max(0.0, deadline - time.monotonic())))
|
||||
|
||||
def _drain_rings_once_for_history(self) -> None:
|
||||
"""Perform one non-blocking read pass to extend histories."""
|
||||
if self._raw_reader is not None:
|
||||
@@ -436,7 +525,8 @@ class AppWindowPipelineMixin:
|
||||
else:
|
||||
stable_rounds = 0
|
||||
previous = current
|
||||
time.sleep(poll_s)
|
||||
# Event-loop-friendly wait so timers/signals keep firing during drain.
|
||||
self._pump_events_during_drain(poll_s)
|
||||
|
||||
def _drain_results_until_quiet(self, *, timeout_s: float, poll_s: float) -> ResultCollection | None:
|
||||
"""Drain results until at least one result arrives and the ring becomes quiet."""
|
||||
@@ -455,7 +545,8 @@ class AppWindowPipelineMixin:
|
||||
else:
|
||||
latest_seen = latest
|
||||
stable_rounds = 0
|
||||
time.sleep(poll_s)
|
||||
# Event-loop-friendly wait so timers/signals keep firing during drain.
|
||||
self._pump_events_during_drain(poll_s)
|
||||
return latest_seen
|
||||
|
||||
def _update_history_indicator(self) -> None:
|
||||
|
||||
@@ -275,18 +275,20 @@ class AppWindowSnapshotMixin:
|
||||
got_results = result_count > start_result_count
|
||||
got_raw_or_pre = raw_count > start_raw_count or pre_count > start_pre_count
|
||||
|
||||
# Event-loop-friendly waits so the keepalive/watchdog timers and
|
||||
# queued signals keep firing instead of freezing the headless daemon.
|
||||
if got_results and (missing_raw or missing_pre) and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
self._pump_events_during_drain(0.01)
|
||||
continue
|
||||
if got_raw_or_pre and missing_results and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
self._pump_events_during_drain(0.01)
|
||||
continue
|
||||
if (
|
||||
self._result_reader is not None
|
||||
and max(raw_count, pre_count) > result_count
|
||||
and time.monotonic() < deadline
|
||||
):
|
||||
time.sleep(0.01)
|
||||
self._pump_events_during_drain(0.01)
|
||||
continue
|
||||
break
|
||||
except Exception as exc: # noqa: BLE001
|
||||
|
||||
@@ -40,10 +40,17 @@ def _install_unix_signal_handlers(app: QApplication, window: AppWindow) -> None:
|
||||
loop just long enough to deliver pending signals.
|
||||
"""
|
||||
|
||||
def _request_shutdown(*_args: object) -> None:
|
||||
def _shutdown() -> None:
|
||||
window.close()
|
||||
app.quit()
|
||||
|
||||
def _request_shutdown(signum: int, _frame: object) -> None:
|
||||
# Async-signal-safe: do the minimum from C signal context. Reset the handler
|
||||
# to default so a second signal force-terminates instead of re-entering Qt
|
||||
# teardown, then schedule the real shutdown on the next event-loop iteration.
|
||||
signal.signal(signum, signal.SIG_DFL)
|
||||
QTimer.singleShot(0, _shutdown)
|
||||
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
signal.signal(sig, _request_shutdown)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user