web UI added and refactoring done

2026-06-06 00:06:30 +03:00
parent 3c30a12d4a
commit af6005d68f
65 changed files with 3630 additions and 4720 deletions
@@ -22,6 +22,7 @@ from python_app.orchestration.preprocess_assets import (
    preprocess_asset_model,
    runtime_preprocess_asset_keys,
 )
+from python_app.orchestration.restart_policy import RestartPolicy
 from python_app.orchestration.shm_reader import ShmRingReader


@@ -36,6 +37,15 @@ class AppWindowPipelineMixin:
    # a wedged reader cannot stay silently broken forever.
    _READER_ERROR_RECONNECT_AT = 40
    _READER_ERROR_STOP_AT = 400
+    # If a pipeline child exits unexpectedly while the run should be live, relaunch
+    # the whole pipeline from the last-written runtime config. It retries FOREVER with
+    # capped back-off (this is an unattended appliance — it must keep trying to come
+    # back, never permanently stop); the failure streak resets once a healthy poll sees
+    # data. The "if it breaks it comes back up" contract holds in BOTH GUI and headless.
+    _RESTART_POLICY = RestartPolicy(min_interval_s=3.0, max_interval_s=60.0)
+    # Safety backstop so a wedged/dropping processor cannot leave a single capture
+    # polling forever with no completion and no error. Generous, not a tight deadline.
+    _SINGLE_CAPTURE_TIMEOUT_S = 300.0

    def _processor_requires_restart(self, run_signature: tuple[object, ...]) -> bool:
        """Return whether alive `data_processor` was started with different stable run settings."""
@@ -148,6 +158,12 @@ class AppWindowPipelineMixin:
            self._drop_pending_ring_payloads(include_results=True)
            self._last_reader_error_signature = None
            self._reader_error_repeat_count = 0
+            # Record what to relaunch if a child later dies unexpectedly. Only a
+            # continuous run auto-restarts; a single capture is bounded by its deadline.
+            self._active_run_config = config
+            self._active_run_config_path = config_path
+            self._pipeline_should_run = not single_capture
+            self._pipeline_restart_count = 0
            if single_capture:
                self._single_capture_start_ns = time.monotonic_ns()

@@ -246,6 +262,7 @@ class AppWindowPipelineMixin:

    def _stop_run(self) -> None:
        """Stop acquisition-side processes and close readers as needed."""
+        self._pipeline_should_run = False  # an explicit stop disables crash auto-restart
        was_running = self._supervisor.is_running()
        if was_running:
            self._supervisor.stop_orchestrator()
@@ -271,6 +288,7 @@ class AppWindowPipelineMixin:

    def _stop_all_processes(self) -> None:
        """Stop all managed pipeline processes and close all readers."""
+        self._pipeline_should_run = False  # an explicit stop disables crash auto-restart
        was_running = self._supervisor.is_running() or self._supervisor.is_processor_running()
        self._supervisor.stop_all()
        self._drain_rings_until_quiet(timeout_s=0.25, poll_s=0.02)
@@ -299,12 +317,8 @@ class AppWindowPipelineMixin:

    def _poll_rings(self) -> None:
        """Poll readers, ingest history, and trigger rendering."""
-        for report in self._supervisor.collect_exit_reports():
-            if report.level == "INFO":
-                self._log(report.format())
-                continue
-            self._status_label.setText("Status: error")
-            self._log_error(report.format())
+        self._web_update_snapshot()  # guarded internally; never raises
+        self._handle_process_exit_reports()

        try:
            if self._raw_reader is not None:
@@ -318,9 +332,13 @@ class AppWindowPipelineMixin:
            if self._single_capture_active:
                if self._finish_single_capture_if_ready():
                    return
+                self._check_single_capture_deadline()
                return

            if result_latest is not None:
+                # Genuine data flowed: the pipeline is healthy, so reset the
+                # crash-storm budget (it only caps consecutive crash-restarts).
+                self._pipeline_restart_count = 0
                render_started_ns = time.monotonic_ns()
                self._draw_preferred_collection(result_latest=result_latest)
                self._pipeline_metrics.record(
@@ -331,6 +349,88 @@ class AppWindowPipelineMixin:
        except Exception as exc:  # noqa: BLE001
            self._handle_reader_poll_error(exc)

+    def _handle_process_exit_reports(self) -> None:
+        """Log child exits and auto-restart the pipeline on an unexpected death.
+
+        Runs first in the poll tick and is fully guarded: it must never raise, or
+        it would abort the Qt slot. An unexpected (non-clean) exit while the run is
+        meant to be live triggers a bounded relaunch — in both GUI and headless.
+        """
+        unexpected = False
+        try:
+            for report in self._supervisor.collect_exit_reports():
+                if report.level == "INFO":
+                    self._log(report.format())
+                    continue
+                self._status_label.setText("Status: error")
+                self._log_error(report.format())
+                unexpected = True
+        except Exception as exc:  # noqa: BLE001 - the poll tick must survive this
+            self._log_exception("Failed to collect process exit reports", exc, level="ERROR")
+            return
+
+        if unexpected and getattr(self, "_pipeline_should_run", False):
+            self._recover_pipeline_after_crash()
+
+    def _recover_pipeline_after_crash(self) -> None:
+        """Relaunch the pipeline after an unexpected child exit (GUI and headless).
+
+        Re-spawns from the already-written runtime config — no widgets, no dialogs,
+        no re-validation — so it is safe to call from the poll tick. Retries forever
+        with capped back-off (never gives up); a healthy poll resets the failure streak.
+        """
+        now = time.monotonic()
+        consecutive_failures = getattr(self, "_pipeline_restart_count", 0)
+        if not self._RESTART_POLICY.should_restart_now(
+            now_s=now,
+            last_restart_s=getattr(self, "_last_pipeline_restart_s", 0.0),
+            consecutive_failures=consecutive_failures,
+        ):
+            return  # still inside the current back-off window; let it settle
+        self._last_pipeline_restart_s = now
+        self._pipeline_restart_count = consecutive_failures + 1
+
+        config = getattr(self, "_active_run_config", None)
+        config_path = getattr(self, "_active_run_config_path", None)
+        if config is None or config_path is None:
+            self._pipeline_should_run = False
+            self._log_error("Cannot auto-restart pipeline: no active run configuration recorded.")
+            return
+
+        self._log_error(
+            "Pipeline process exited unexpectedly; restarting "
+            f"(attempt {self._pipeline_restart_count}, next back-off "
+            f"{self._RESTART_POLICY.backoff_for(self._pipeline_restart_count):.0f}s)."
+        )
+        try:
+            # Note: do NOT drain rings here — draining pumps the Qt event loop, which
+            # would re-enter _poll_rings mid-restart (and reset the crash-storm count).
+            self._supervisor.stop_all()
+            self._close_readers(keep_results=False)
+            self._supervisor.start(config_path, allow_clean_orchestrator_exit=False)
+            self._raw_reader = ShmRingReader(config.rings.raw_tap.name)
+            self._pre_reader = ShmRingReader(config.rings.preprocessed_tap.name)
+            self._result_reader = ShmRingReader(config.rings.results.name)
+            self._processor_run_signature = self._build_processor_run_signature(config)
+            self._drop_pending_ring_payloads(include_results=True)
+            self._status_label.setText("Status: running")
+            self._log("Pipeline auto-restarted after crash.")
+        except Exception as exc:  # noqa: BLE001 - retry on the next crash signal
+            self._log_exception("Pipeline auto-restart failed; will retry", exc, level="ERROR")
+
+    def _check_single_capture_deadline(self) -> None:
+        """Fail a single capture that never completes so it cannot hang forever."""
+        start = self._single_capture_start_ns
+        if start is None:
+            return
+        if time.monotonic_ns() - start <= int(self._SINGLE_CAPTURE_TIMEOUT_S * 1e9):
+            return
+        self._log_error(
+            f"Single capture timed out after {self._SINGLE_CAPTURE_TIMEOUT_S:.0f}s "
+            "with no result; stopping."
+        )
+        self._stop_run()
+
    def _handle_reader_poll_error(self, exc: Exception) -> None:
        """Surface a reader-poll failure without spamming the log.