some fixes

python_app/gui/controllers/app_window_pipeline_mixin.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import time
 
+from PyQt6.QtCore import QCoreApplication, QEventLoop
 
 from python_app.gui.runtime.constraints import validate_processing_mode_constraints
 from python_app.gui.runtime.history import (
@@ -27,6 +28,15 @@ from python_app.orchestration.shm_reader import ShmRingReader
 class AppWindowPipelineMixin:
     """Controls start/stop, readers, and periodic polling of pipeline rings."""
 
+    # A repeated identical reader error is deduped from the log, but must still be
+    # re-logged every this-many polls so a persistent failure stays visible.
+    _READER_ERROR_RELOG_EVERY = 200
+    # After this many consecutive identical reader errors, attempt one reader
+    # reconnect; if it keeps failing past the next threshold, stop the pipeline so
+    # a wedged reader cannot stay silently broken forever.
+    _READER_ERROR_RECONNECT_AT = 40
+    _READER_ERROR_STOP_AT = 400
+
     def _processor_requires_restart(self, run_signature: tuple[object, ...]) -> bool:
         """Return whether alive `data_processor` was started with different stable run settings."""
         return self._supervisor.is_processor_running() and self._processor_run_signature != run_signature
@@ -137,6 +147,7 @@ class AppWindowPipelineMixin:
             # from a clean boundary and does not retain stale results-only tail.
             self._drop_pending_ring_payloads(include_results=True)
             self._last_reader_error_signature = None
+            self._reader_error_repeat_count = 0
             if single_capture:
                 self._single_capture_start_ns = time.monotonic_ns()
 
@@ -302,6 +313,7 @@ class AppWindowPipelineMixin:
             result_latest = self._read_all_results() if self._result_reader is not None else None
             self._update_history_indicator()
             self._last_reader_error_signature = None
+            self._reader_error_repeat_count = 0
 
             if self._single_capture_active:
                 if self._finish_single_capture_if_ready():
@@ -317,12 +329,67 @@ class AppWindowPipelineMixin:
             else:
                 self._draw_preferred_collection(result_latest=None)
         except Exception as exc:  # noqa: BLE001
-            signature = (type(exc).__name__, str(exc))
-            if self._last_reader_error_signature == signature:
-                return
+            self._handle_reader_poll_error(exc)
+
+    def _handle_reader_poll_error(self, exc: Exception) -> None:
+        """Surface a reader-poll failure without spamming the log.
+
+        Distinct errors log once; an identical recurring error is deduped from the
+        log but still drives the status label to error, is periodically re-logged,
+        and after escalating thresholds triggers a reader reconnect and finally a
+        pipeline stop so a wedged reader cannot fail silently forever.
+        """
+        signature = (type(exc).__name__, str(exc))
+        # Always reflect a reader failure in the status label, even when deduped.
+        self._status_label.setText("Status: error")
+
+        if self._last_reader_error_signature == signature:
+            self._reader_error_repeat_count = getattr(self, "_reader_error_repeat_count", 0) + 1
+        else:
             self._last_reader_error_signature = signature
+            self._reader_error_repeat_count = 0
             self._log_exception("Reader poll failed", exc, level="ERROR")
 
+        repeats = self._reader_error_repeat_count
+        # Periodically re-log a persistent identical failure so it stays visible.
+        if repeats and repeats % self._READER_ERROR_RELOG_EVERY == 0:
+            self._log_exception(
+                f"Reader poll still failing (repeat #{repeats})", exc, level="ERROR"
+            )
+
+        if repeats >= self._READER_ERROR_STOP_AT:
+            # The reader stayed wedged through a reconnect attempt; stop the
+            # pipeline so the failure is unmistakable instead of an endless retry.
+            self._log_error(
+                f"Stopping pipeline after {repeats} consecutive reader poll failures"
+            )
+            self._reader_error_repeat_count = 0
+            self._last_reader_error_signature = None
+            self._stop_all_processes()
+        elif repeats == self._READER_ERROR_RECONNECT_AT:
+            self._reconnect_readers_after_error()
+
+    def _reconnect_readers_after_error(self) -> None:
+        """Re-open active ring readers in place to recover from a wedged reader."""
+        self._log_warning("Attempting ring reader reconnect after repeated poll failures")
+        try:
+            for attr in ("_raw_reader", "_pre_reader", "_result_reader"):
+                reader = getattr(self, attr)
+                if reader is None:
+                    continue
+                ring_name = reader._ring_name  # noqa: SLF001 - reuse the reader's own ring name
+                reader.close()
+                setattr(self, attr, ShmRingReader(ring_name))
+            # A successful reconnect clears the error state so the next failure
+            # logs fresh rather than being swallowed by the stale signature.
+            self._last_reader_error_signature = None
+            self._reader_error_repeat_count = 0
+            self._status_label.setText("Status: running")
+            self._log("Ring readers reconnected after repeated poll failures")
+        except Exception as exc:  # noqa: BLE001
+            # Leave the error signature intact so escalation to a stop still fires.
+            self._log_exception("Ring reader reconnect failed", exc, level="ERROR")
+
     def _finish_single_capture_if_ready(self) -> bool:
         """Finalize single capture when the exact target result becomes available."""
         if not self._single_capture_active:
@@ -406,6 +473,28 @@ class AppWindowPipelineMixin:
                 latest = collection
         return latest
 
+    def _pump_events_during_drain(self, pause_s: float) -> None:
+        """Yield to the Qt event loop for `pause_s` instead of blocking on time.sleep.
+
+        The bounded drain loops run on the GUI thread; a raw time.sleep here freezes
+        the event loop, stalling the keepalive/headless-watchdog timers and starving
+        queued signals. Pumping events keeps the daemon responsive while we wait.
+        """
+        app = QCoreApplication.instance()
+        if app is None:
+            # No event loop (e.g. unit context); fall back to a plain short sleep.
+            time.sleep(pause_s)
+            return
+        deadline = time.monotonic() + pause_s
+        while True:
+            remaining_ms = int((deadline - time.monotonic()) * 1000)
+            if remaining_ms <= 0:
+                break
+            app.processEvents(QEventLoop.ProcessEventsFlag.AllEvents, remaining_ms)
+            # processEvents returns immediately when the queue empties; sleep the
+            # residual in tiny slices so we neither busy-spin nor block too long.
+            time.sleep(min(0.002, max(0.0, deadline - time.monotonic())))
+
     def _drain_rings_once_for_history(self) -> None:
         """Perform one non-blocking read pass to extend histories."""
         if self._raw_reader is not None:
@@ -436,7 +525,8 @@ class AppWindowPipelineMixin:
             else:
                 stable_rounds = 0
                 previous = current
-            time.sleep(poll_s)
+            # Event-loop-friendly wait so timers/signals keep firing during drain.
+            self._pump_events_during_drain(poll_s)
 
     def _drain_results_until_quiet(self, *, timeout_s: float, poll_s: float) -> ResultCollection | None:
         """Drain results until at least one result arrives and the ring becomes quiet."""
@@ -455,7 +545,8 @@ class AppWindowPipelineMixin:
             else:
                 latest_seen = latest
                 stable_rounds = 0
-            time.sleep(poll_s)
+            # Event-loop-friendly wait so timers/signals keep firing during drain.
+            self._pump_events_during_drain(poll_s)
         return latest_seen
 
     def _update_history_indicator(self) -> None: