some fixes

This commit is contained in:
Ayzen
2026-06-05 14:40:10 +03:00
parent 22942d9dc9
commit bbea744459
35 changed files with 1797 additions and 297 deletions
+55
View File
@@ -7,9 +7,12 @@ state shared across them (runtime services, readers, history buffers, timer).
from __future__ import annotations
from collections import deque
from contextlib import suppress
from datetime import datetime
import html
import json
import logging
from logging.handlers import RotatingFileHandler
import os
from pathlib import Path
import sys
@@ -59,6 +62,7 @@ class AppWindow(
super().__init__()
self._init_paths(project_root)
self._init_headless_logger()
self._init_runtime_services()
self._init_config_profile_state()
self._init_reader_handles()
@@ -76,6 +80,42 @@ class AppWindow(
self._root_profile_path = project_root / "run_config.json"
self._active_profile_path = self._root_profile_path
self._pending_startup_log_entries: list[tuple[str, str, str | None]] = []
# Guards closeEvent against re-entrant teardown (e.g. a second signal).
self._closing = False
def _init_headless_logger(self) -> None:
"""Create a Python logger so headless WARN/ERROR reach journald and disk.
In headless mode the in-app log only reaches an offscreen widget, so an
operator (or `journalctl`) would never see failures. We attach a stderr
StreamHandler (captured by journald) plus a small rotating file under
`runtime/logs`; in GUI mode no handler is attached and the logger stays
inert, preserving the visible log widget as the sole sink.
"""
self._headless_logger: logging.Logger | None = None
if not self._is_truthy_env("RADAR_SYSTEM_HEADLESS"):
return
logger = logging.getLogger("radar_system.gui")
logger.setLevel(logging.WARNING)
logger.propagate = False
logger.handlers.clear()
formatter = logging.Formatter(
fmt="%(asctime)s | %(levelname)-5s | %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
stream_handler = logging.StreamHandler(stream=sys.stderr)
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
# A rotating file keeps recent failures around after a journald restart.
with suppress(Exception):
log_dir = self._project_root / "python_app/runtime/logs"
log_dir.mkdir(parents=True, exist_ok=True)
file_handler = RotatingFileHandler(
log_dir / "gui.log", maxBytes=1_000_000, backupCount=3, encoding="utf-8"
)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
self._headless_logger = logger
def _init_runtime_services(self) -> None:
"""Initialize long-lived service objects used by mixins."""
@@ -461,6 +501,15 @@ class AppWindow(
if level_upper == "ERROR" and hasattr(self, "_status_label"):
self._status_label.setText("Status: error")
# In headless mode the offscreen widget above is invisible, so also mirror
# WARN/ERROR to the Python logger (stderr -> journald, plus rotating file)
# where an operator can actually observe failures.
headless_logger = getattr(self, "_headless_logger", None)
if headless_logger is not None and level_upper in {"WARN", "ERROR"}:
log_message = text if not details else f"{text}\n{details}"
log_level = logging.ERROR if level_upper == "ERROR" else logging.WARNING
headless_logger.log(log_level, log_message)
def _log(self, text: str, *, once_key: str | None = None) -> None:
"""Append informational message to runtime log panel."""
self._append_log_entry("INFO", text, once_key=once_key)
@@ -564,6 +613,12 @@ class AppWindow(
def closeEvent(self, event) -> None: # noqa: N802
"""Ensure workers and dialogs are closed before window destruction."""
if self._closing:
# Re-entrant close (second signal, or window.close() after the event loop
# already returned): teardown is in progress or done — do nothing more.
super().closeEvent(event)
return
self._closing = True
try:
# 0) Stop the GPIO button watcher so a late press cannot start work.
self._stop_control_button_watcher()
+17 -6
View File
@@ -86,12 +86,23 @@ class ControlButtonWatcher(QObject):
def start(self) -> None:
"""Open the GPIO line and begin watching for presses on a background thread."""
self._line.open()
self._stop_read_fd, self._stop_write_fd = os.pipe()
self._thread = threading.Thread(
target=self._run, name="control-button-watcher", daemon=True
)
self._thread.start()
# Guard against double-start: a second start would leak the first line/pipe/thread.
if self._thread is not None:
return
try:
self._line.open()
self._stop_read_fd, self._stop_write_fd = os.pipe()
self._thread = threading.Thread(
target=self._run, name="control-button-watcher", daemon=True
)
self._thread.start()
except Exception:
# Release any line/pipe fds opened before the failure so nothing leaks
# and no orphaned thread survives a partial start.
self._thread = None
self._close_stop_pipe()
self._line.close()
raise
def stop(self) -> None:
"""Signal the watcher thread to exit and release the GPIO line and pipe."""
@@ -19,6 +19,8 @@ class AppWindowControlButtonMixin:
def _init_control_button_state(self) -> None:
"""Initialize the watcher handle before the watcher is started."""
self._control_button_watcher: ControlButtonWatcher | None = None
# Re-entrancy guard: ignore presses that arrive while a capture is in progress.
self._control_button_busy = False
def _start_control_button_watcher(self) -> None:
"""Open the configured GPIO button line and begin watching for presses.
@@ -70,8 +72,17 @@ class AppWindowControlButtonMixin:
Delivered as a queued signal from the watcher thread, so this executes
on the GUI thread exactly like a click on "Capture Tmp Reference".
"""
self._log("GPIO control button pressed: capturing tmp reference.")
self._capture_tmp_reference()
# Ignore a re-entrant press: the capture flow spins the event loop (stop/
# start run, dialogs), so a second queued press must not start a nested capture.
if self._control_button_busy:
self._log("GPIO control button press ignored: capture already in progress.")
return
self._control_button_busy = True
try:
self._log("GPIO control button pressed: capturing tmp reference.")
self._capture_tmp_reference()
finally:
self._control_button_busy = False
def _on_control_button_failed(self, message: str) -> None:
"""Log an unrecoverable watcher error reported from the background thread."""
@@ -82,9 +93,24 @@ class AppWindowControlButtonMixin:
watcher = getattr(self, "_control_button_watcher", None)
if watcher is None:
return
# Disconnect first so a press queued before teardown cannot run a slot afterwards.
self._disconnect_control_button_signals(watcher)
try:
watcher.stop()
except Exception as exc: # noqa: BLE001
self._log_warning("Error stopping GPIO control button watcher", details=str(exc))
finally:
# Disconnect again in case stop() re-emitted, then drop and schedule deletion.
self._disconnect_control_button_signals(watcher)
self._control_button_watcher = None
watcher.deleteLater()
@staticmethod
def _disconnect_control_button_signals(watcher: ControlButtonWatcher) -> None:
"""Detach the watcher's signals from their slots, tolerating already-disconnected."""
for signal in (watcher.pressed, watcher.failed):
try:
signal.disconnect()
except (TypeError, RuntimeError):
# No connections left (or the C++ object is already gone): nothing to do.
pass
@@ -4,6 +4,7 @@ from __future__ import annotations
import time
from PyQt6.QtCore import QCoreApplication, QEventLoop
from python_app.gui.runtime.constraints import validate_processing_mode_constraints
from python_app.gui.runtime.history import (
@@ -27,6 +28,15 @@ from python_app.orchestration.shm_reader import ShmRingReader
class AppWindowPipelineMixin:
"""Controls start/stop, readers, and periodic polling of pipeline rings."""
# A repeated identical reader error is deduped from the log, but must still be
# re-logged every this-many polls so a persistent failure stays visible.
_READER_ERROR_RELOG_EVERY = 200
# After this many consecutive identical reader errors, attempt one reader
# reconnect; if it keeps failing past the next threshold, stop the pipeline so
# a wedged reader cannot stay silently broken forever.
_READER_ERROR_RECONNECT_AT = 40
_READER_ERROR_STOP_AT = 400
def _processor_requires_restart(self, run_signature: tuple[object, ...]) -> bool:
"""Return whether alive `data_processor` was started with different stable run settings."""
return self._supervisor.is_processor_running() and self._processor_run_signature != run_signature
@@ -137,6 +147,7 @@ class AppWindowPipelineMixin:
# from a clean boundary and does not retain stale results-only tail.
self._drop_pending_ring_payloads(include_results=True)
self._last_reader_error_signature = None
self._reader_error_repeat_count = 0
if single_capture:
self._single_capture_start_ns = time.monotonic_ns()
@@ -302,6 +313,7 @@ class AppWindowPipelineMixin:
result_latest = self._read_all_results() if self._result_reader is not None else None
self._update_history_indicator()
self._last_reader_error_signature = None
self._reader_error_repeat_count = 0
if self._single_capture_active:
if self._finish_single_capture_if_ready():
@@ -317,12 +329,67 @@ class AppWindowPipelineMixin:
else:
self._draw_preferred_collection(result_latest=None)
except Exception as exc: # noqa: BLE001
signature = (type(exc).__name__, str(exc))
if self._last_reader_error_signature == signature:
return
self._handle_reader_poll_error(exc)
def _handle_reader_poll_error(self, exc: Exception) -> None:
"""Surface a reader-poll failure without spamming the log.
Distinct errors log once; an identical recurring error is deduped from the
log but still drives the status label to error, is periodically re-logged,
and after escalating thresholds triggers a reader reconnect and finally a
pipeline stop so a wedged reader cannot fail silently forever.
"""
signature = (type(exc).__name__, str(exc))
# Always reflect a reader failure in the status label, even when deduped.
self._status_label.setText("Status: error")
if self._last_reader_error_signature == signature:
self._reader_error_repeat_count = getattr(self, "_reader_error_repeat_count", 0) + 1
else:
self._last_reader_error_signature = signature
self._reader_error_repeat_count = 0
self._log_exception("Reader poll failed", exc, level="ERROR")
repeats = self._reader_error_repeat_count
# Periodically re-log a persistent identical failure so it stays visible.
if repeats and repeats % self._READER_ERROR_RELOG_EVERY == 0:
self._log_exception(
f"Reader poll still failing (repeat #{repeats})", exc, level="ERROR"
)
if repeats >= self._READER_ERROR_STOP_AT:
# The reader stayed wedged through a reconnect attempt; stop the
# pipeline so the failure is unmistakable instead of an endless retry.
self._log_error(
f"Stopping pipeline after {repeats} consecutive reader poll failures"
)
self._reader_error_repeat_count = 0
self._last_reader_error_signature = None
self._stop_all_processes()
elif repeats == self._READER_ERROR_RECONNECT_AT:
self._reconnect_readers_after_error()
def _reconnect_readers_after_error(self) -> None:
"""Re-open active ring readers in place to recover from a wedged reader."""
self._log_warning("Attempting ring reader reconnect after repeated poll failures")
try:
for attr in ("_raw_reader", "_pre_reader", "_result_reader"):
reader = getattr(self, attr)
if reader is None:
continue
ring_name = reader._ring_name # noqa: SLF001 - reuse the reader's own ring name
reader.close()
setattr(self, attr, ShmRingReader(ring_name))
# A successful reconnect clears the error state so the next failure
# logs fresh rather than being swallowed by the stale signature.
self._last_reader_error_signature = None
self._reader_error_repeat_count = 0
self._status_label.setText("Status: running")
self._log("Ring readers reconnected after repeated poll failures")
except Exception as exc: # noqa: BLE001
# Leave the error signature intact so escalation to a stop still fires.
self._log_exception("Ring reader reconnect failed", exc, level="ERROR")
def _finish_single_capture_if_ready(self) -> bool:
"""Finalize single capture when the exact target result becomes available."""
if not self._single_capture_active:
@@ -406,6 +473,28 @@ class AppWindowPipelineMixin:
latest = collection
return latest
def _pump_events_during_drain(self, pause_s: float) -> None:
"""Yield to the Qt event loop for `pause_s` instead of blocking on time.sleep.
The bounded drain loops run on the GUI thread; a raw time.sleep here freezes
the event loop, stalling the keepalive/headless-watchdog timers and starving
queued signals. Pumping events keeps the daemon responsive while we wait.
"""
app = QCoreApplication.instance()
if app is None:
# No event loop (e.g. unit context); fall back to a plain short sleep.
time.sleep(pause_s)
return
deadline = time.monotonic() + pause_s
while True:
remaining_ms = int((deadline - time.monotonic()) * 1000)
if remaining_ms <= 0:
break
app.processEvents(QEventLoop.ProcessEventsFlag.AllEvents, remaining_ms)
# processEvents returns immediately when the queue empties; sleep the
# residual in tiny slices so we neither busy-spin nor block too long.
time.sleep(min(0.002, max(0.0, deadline - time.monotonic())))
def _drain_rings_once_for_history(self) -> None:
"""Perform one non-blocking read pass to extend histories."""
if self._raw_reader is not None:
@@ -436,7 +525,8 @@ class AppWindowPipelineMixin:
else:
stable_rounds = 0
previous = current
time.sleep(poll_s)
# Event-loop-friendly wait so timers/signals keep firing during drain.
self._pump_events_during_drain(poll_s)
def _drain_results_until_quiet(self, *, timeout_s: float, poll_s: float) -> ResultCollection | None:
"""Drain results until at least one result arrives and the ring becomes quiet."""
@@ -455,7 +545,8 @@ class AppWindowPipelineMixin:
else:
latest_seen = latest
stable_rounds = 0
time.sleep(poll_s)
# Event-loop-friendly wait so timers/signals keep firing during drain.
self._pump_events_during_drain(poll_s)
return latest_seen
def _update_history_indicator(self) -> None:
@@ -275,18 +275,20 @@ class AppWindowSnapshotMixin:
got_results = result_count > start_result_count
got_raw_or_pre = raw_count > start_raw_count or pre_count > start_pre_count
# Event-loop-friendly waits so the keepalive/watchdog timers and
# queued signals keep firing instead of freezing the headless daemon.
if got_results and (missing_raw or missing_pre) and time.monotonic() < deadline:
time.sleep(0.01)
self._pump_events_during_drain(0.01)
continue
if got_raw_or_pre and missing_results and time.monotonic() < deadline:
time.sleep(0.01)
self._pump_events_during_drain(0.01)
continue
if (
self._result_reader is not None
and max(raw_count, pre_count) > result_count
and time.monotonic() < deadline
):
time.sleep(0.01)
self._pump_events_during_drain(0.01)
continue
break
except Exception as exc: # noqa: BLE001
+8 -1
View File
@@ -40,10 +40,17 @@ def _install_unix_signal_handlers(app: QApplication, window: AppWindow) -> None:
loop just long enough to deliver pending signals.
"""
def _request_shutdown(*_args: object) -> None:
def _shutdown() -> None:
window.close()
app.quit()
def _request_shutdown(signum: int, _frame: object) -> None:
# Async-signal-safe: do the minimum from C signal context. Reset the handler
# to default so a second signal force-terminates instead of re-entering Qt
# teardown, then schedule the real shutdown on the next event-loop iteration.
signal.signal(signum, signal.SIG_DFL)
QTimer.singleShot(0, _shutdown)
for sig in (signal.SIGINT, signal.SIGTERM):
signal.signal(sig, _request_shutdown)