some fixes
This commit is contained in:
@@ -19,6 +19,8 @@ class AppWindowControlButtonMixin:
|
||||
def _init_control_button_state(self) -> None:
|
||||
"""Initialize the watcher handle before the watcher is started."""
|
||||
self._control_button_watcher: ControlButtonWatcher | None = None
|
||||
# Re-entrancy guard: ignore presses that arrive while a capture is in progress.
|
||||
self._control_button_busy = False
|
||||
|
||||
def _start_control_button_watcher(self) -> None:
|
||||
"""Open the configured GPIO button line and begin watching for presses.
|
||||
@@ -70,8 +72,17 @@ class AppWindowControlButtonMixin:
|
||||
Delivered as a queued signal from the watcher thread, so this executes
|
||||
on the GUI thread exactly like a click on "Capture Tmp Reference".
|
||||
"""
|
||||
self._log("GPIO control button pressed: capturing tmp reference.")
|
||||
self._capture_tmp_reference()
|
||||
# Ignore a re-entrant press: the capture flow spins the event loop (stop/
|
||||
# start run, dialogs), so a second queued press must not start a nested capture.
|
||||
if self._control_button_busy:
|
||||
self._log("GPIO control button press ignored: capture already in progress.")
|
||||
return
|
||||
self._control_button_busy = True
|
||||
try:
|
||||
self._log("GPIO control button pressed: capturing tmp reference.")
|
||||
self._capture_tmp_reference()
|
||||
finally:
|
||||
self._control_button_busy = False
|
||||
|
||||
def _on_control_button_failed(self, message: str) -> None:
|
||||
"""Log an unrecoverable watcher error reported from the background thread."""
|
||||
@@ -82,9 +93,24 @@ class AppWindowControlButtonMixin:
|
||||
watcher = getattr(self, "_control_button_watcher", None)
|
||||
if watcher is None:
|
||||
return
|
||||
# Disconnect first so a press queued before teardown cannot run a slot afterwards.
|
||||
self._disconnect_control_button_signals(watcher)
|
||||
try:
|
||||
watcher.stop()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._log_warning("Error stopping GPIO control button watcher", details=str(exc))
|
||||
finally:
|
||||
# Disconnect again in case stop() re-emitted, then drop and schedule deletion.
|
||||
self._disconnect_control_button_signals(watcher)
|
||||
self._control_button_watcher = None
|
||||
watcher.deleteLater()
|
||||
|
||||
@staticmethod
|
||||
def _disconnect_control_button_signals(watcher: ControlButtonWatcher) -> None:
|
||||
"""Detach the watcher's signals from their slots, tolerating already-disconnected."""
|
||||
for signal in (watcher.pressed, watcher.failed):
|
||||
try:
|
||||
signal.disconnect()
|
||||
except (TypeError, RuntimeError):
|
||||
# No connections left (or the C++ object is already gone): nothing to do.
|
||||
pass
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
from PyQt6.QtCore import QCoreApplication, QEventLoop
|
||||
|
||||
from python_app.gui.runtime.constraints import validate_processing_mode_constraints
|
||||
from python_app.gui.runtime.history import (
|
||||
@@ -27,6 +28,15 @@ from python_app.orchestration.shm_reader import ShmRingReader
|
||||
class AppWindowPipelineMixin:
|
||||
"""Controls start/stop, readers, and periodic polling of pipeline rings."""
|
||||
|
||||
# A repeated identical reader error is deduped from the log, but must still be
|
||||
# re-logged every this-many polls so a persistent failure stays visible.
|
||||
_READER_ERROR_RELOG_EVERY = 200
|
||||
# After this many consecutive identical reader errors, attempt one reader
|
||||
# reconnect; if it keeps failing past the next threshold, stop the pipeline so
|
||||
# a wedged reader cannot stay silently broken forever.
|
||||
_READER_ERROR_RECONNECT_AT = 40
|
||||
_READER_ERROR_STOP_AT = 400
|
||||
|
||||
def _processor_requires_restart(self, run_signature: tuple[object, ...]) -> bool:
|
||||
"""Return whether alive `data_processor` was started with different stable run settings."""
|
||||
return self._supervisor.is_processor_running() and self._processor_run_signature != run_signature
|
||||
@@ -137,6 +147,7 @@ class AppWindowPipelineMixin:
|
||||
# from a clean boundary and does not retain stale results-only tail.
|
||||
self._drop_pending_ring_payloads(include_results=True)
|
||||
self._last_reader_error_signature = None
|
||||
self._reader_error_repeat_count = 0
|
||||
if single_capture:
|
||||
self._single_capture_start_ns = time.monotonic_ns()
|
||||
|
||||
@@ -302,6 +313,7 @@ class AppWindowPipelineMixin:
|
||||
result_latest = self._read_all_results() if self._result_reader is not None else None
|
||||
self._update_history_indicator()
|
||||
self._last_reader_error_signature = None
|
||||
self._reader_error_repeat_count = 0
|
||||
|
||||
if self._single_capture_active:
|
||||
if self._finish_single_capture_if_ready():
|
||||
@@ -317,12 +329,67 @@ class AppWindowPipelineMixin:
|
||||
else:
|
||||
self._draw_preferred_collection(result_latest=None)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
signature = (type(exc).__name__, str(exc))
|
||||
if self._last_reader_error_signature == signature:
|
||||
return
|
||||
self._handle_reader_poll_error(exc)
|
||||
|
||||
def _handle_reader_poll_error(self, exc: Exception) -> None:
|
||||
"""Surface a reader-poll failure without spamming the log.
|
||||
|
||||
Distinct errors log once; an identical recurring error is deduped from the
|
||||
log but still drives the status label to error, is periodically re-logged,
|
||||
and after escalating thresholds triggers a reader reconnect and finally a
|
||||
pipeline stop so a wedged reader cannot fail silently forever.
|
||||
"""
|
||||
signature = (type(exc).__name__, str(exc))
|
||||
# Always reflect a reader failure in the status label, even when deduped.
|
||||
self._status_label.setText("Status: error")
|
||||
|
||||
if self._last_reader_error_signature == signature:
|
||||
self._reader_error_repeat_count = getattr(self, "_reader_error_repeat_count", 0) + 1
|
||||
else:
|
||||
self._last_reader_error_signature = signature
|
||||
self._reader_error_repeat_count = 0
|
||||
self._log_exception("Reader poll failed", exc, level="ERROR")
|
||||
|
||||
repeats = self._reader_error_repeat_count
|
||||
# Periodically re-log a persistent identical failure so it stays visible.
|
||||
if repeats and repeats % self._READER_ERROR_RELOG_EVERY == 0:
|
||||
self._log_exception(
|
||||
f"Reader poll still failing (repeat #{repeats})", exc, level="ERROR"
|
||||
)
|
||||
|
||||
if repeats >= self._READER_ERROR_STOP_AT:
|
||||
# The reader stayed wedged through a reconnect attempt; stop the
|
||||
# pipeline so the failure is unmistakable instead of an endless retry.
|
||||
self._log_error(
|
||||
f"Stopping pipeline after {repeats} consecutive reader poll failures"
|
||||
)
|
||||
self._reader_error_repeat_count = 0
|
||||
self._last_reader_error_signature = None
|
||||
self._stop_all_processes()
|
||||
elif repeats == self._READER_ERROR_RECONNECT_AT:
|
||||
self._reconnect_readers_after_error()
|
||||
|
||||
def _reconnect_readers_after_error(self) -> None:
|
||||
"""Re-open active ring readers in place to recover from a wedged reader."""
|
||||
self._log_warning("Attempting ring reader reconnect after repeated poll failures")
|
||||
try:
|
||||
for attr in ("_raw_reader", "_pre_reader", "_result_reader"):
|
||||
reader = getattr(self, attr)
|
||||
if reader is None:
|
||||
continue
|
||||
ring_name = reader._ring_name # noqa: SLF001 - reuse the reader's own ring name
|
||||
reader.close()
|
||||
setattr(self, attr, ShmRingReader(ring_name))
|
||||
# A successful reconnect clears the error state so the next failure
|
||||
# logs fresh rather than being swallowed by the stale signature.
|
||||
self._last_reader_error_signature = None
|
||||
self._reader_error_repeat_count = 0
|
||||
self._status_label.setText("Status: running")
|
||||
self._log("Ring readers reconnected after repeated poll failures")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# Leave the error signature intact so escalation to a stop still fires.
|
||||
self._log_exception("Ring reader reconnect failed", exc, level="ERROR")
|
||||
|
||||
def _finish_single_capture_if_ready(self) -> bool:
|
||||
"""Finalize single capture when the exact target result becomes available."""
|
||||
if not self._single_capture_active:
|
||||
@@ -406,6 +473,28 @@ class AppWindowPipelineMixin:
|
||||
latest = collection
|
||||
return latest
|
||||
|
||||
def _pump_events_during_drain(self, pause_s: float) -> None:
|
||||
"""Yield to the Qt event loop for `pause_s` instead of blocking on time.sleep.
|
||||
|
||||
The bounded drain loops run on the GUI thread; a raw time.sleep here freezes
|
||||
the event loop, stalling the keepalive/headless-watchdog timers and starving
|
||||
queued signals. Pumping events keeps the daemon responsive while we wait.
|
||||
"""
|
||||
app = QCoreApplication.instance()
|
||||
if app is None:
|
||||
# No event loop (e.g. unit context); fall back to a plain short sleep.
|
||||
time.sleep(pause_s)
|
||||
return
|
||||
deadline = time.monotonic() + pause_s
|
||||
while True:
|
||||
remaining_ms = int((deadline - time.monotonic()) * 1000)
|
||||
if remaining_ms <= 0:
|
||||
break
|
||||
app.processEvents(QEventLoop.ProcessEventsFlag.AllEvents, remaining_ms)
|
||||
# processEvents returns immediately when the queue empties; sleep the
|
||||
# residual in tiny slices so we neither busy-spin nor block too long.
|
||||
time.sleep(min(0.002, max(0.0, deadline - time.monotonic())))
|
||||
|
||||
def _drain_rings_once_for_history(self) -> None:
|
||||
"""Perform one non-blocking read pass to extend histories."""
|
||||
if self._raw_reader is not None:
|
||||
@@ -436,7 +525,8 @@ class AppWindowPipelineMixin:
|
||||
else:
|
||||
stable_rounds = 0
|
||||
previous = current
|
||||
time.sleep(poll_s)
|
||||
# Event-loop-friendly wait so timers/signals keep firing during drain.
|
||||
self._pump_events_during_drain(poll_s)
|
||||
|
||||
def _drain_results_until_quiet(self, *, timeout_s: float, poll_s: float) -> ResultCollection | None:
|
||||
"""Drain results until at least one result arrives and the ring becomes quiet."""
|
||||
@@ -455,7 +545,8 @@ class AppWindowPipelineMixin:
|
||||
else:
|
||||
latest_seen = latest
|
||||
stable_rounds = 0
|
||||
time.sleep(poll_s)
|
||||
# Event-loop-friendly wait so timers/signals keep firing during drain.
|
||||
self._pump_events_during_drain(poll_s)
|
||||
return latest_seen
|
||||
|
||||
def _update_history_indicator(self) -> None:
|
||||
|
||||
@@ -275,18 +275,20 @@ class AppWindowSnapshotMixin:
|
||||
got_results = result_count > start_result_count
|
||||
got_raw_or_pre = raw_count > start_raw_count or pre_count > start_pre_count
|
||||
|
||||
# Event-loop-friendly waits so the keepalive/watchdog timers and
|
||||
# queued signals keep firing instead of freezing the headless daemon.
|
||||
if got_results and (missing_raw or missing_pre) and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
self._pump_events_during_drain(0.01)
|
||||
continue
|
||||
if got_raw_or_pre and missing_results and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
self._pump_events_during_drain(0.01)
|
||||
continue
|
||||
if (
|
||||
self._result_reader is not None
|
||||
and max(raw_count, pre_count) > result_count
|
||||
and time.monotonic() < deadline
|
||||
):
|
||||
time.sleep(0.01)
|
||||
self._pump_events_during_drain(0.01)
|
||||
continue
|
||||
break
|
||||
except Exception as exc: # noqa: BLE001
|
||||
|
||||
Reference in New Issue
Block a user