UI updates

python_app/orchestration/process_supervisor.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 from dataclasses import dataclass
 from pathlib import Path
+import shlex
 import subprocess
 import time
 from typing import Iterable
@@ -16,9 +17,50 @@ class ManagedProcess:
 
     name: str
     command: list[str]
+    allow_clean_exit: bool
     handle: subprocess.Popen[str]
 
 
+@dataclass(slots=True)
+class ProcessExitReport:
+    """Structured report for one exited managed process."""
+
+    name: str
+    command: list[str]
+    working_directory: Path
+    return_code: int
+    stdout: str
+    stderr: str
+    expected_clean_exit: bool
+
+    @property
+    def level(self) -> str:
+        """Return log level appropriate for this exit report."""
+        return "INFO" if self.expected_clean_exit else "ERROR"
+
+    def format(self) -> str:
+        """Render human-readable multiline exit report."""
+        if self.expected_clean_exit:
+            headline = f"Process `{self.name}` completed normally with code {self.return_code}."
+        elif self.return_code == 0:
+            headline = f"Process `{self.name}` exited unexpectedly with code 0."
+        else:
+            headline = f"Process `{self.name}` exited with code {self.return_code}."
+
+        lines = [
+            headline,
+            f"Command: {shlex.join(self.command)}",
+            f"Working directory: {self.working_directory}",
+        ]
+        if self.stderr:
+            lines.append(f"stderr:\n{self.stderr}")
+        if self.stdout:
+            lines.append(f"stdout:\n{self.stdout}")
+        if not self.stderr and not self.stdout:
+            lines.append("stdout/stderr: none")
+        return "\n".join(lines)
+
+
 class ProcessSupervisor:
     """Start, monitor, and stop pipeline subprocesses."""
 
@@ -36,7 +78,7 @@ class ProcessSupervisor:
         """Return whether data processor process is alive."""
         return self._is_alive("data_processor")
 
-    def start(self, config_path: Path) -> None:
+    def start(self, config_path: Path, *, allow_clean_orchestrator_exit: bool = False) -> None:
         """Start required pipeline binaries and wait until they are ready."""
         if self.is_running():
             raise RuntimeError("Acquisition processes are already running")
@@ -59,16 +101,22 @@ class ProcessSupervisor:
             ],
         }
         processor_was_running = self.is_processor_running()
-        required_processes: list[str] = ["data_preprocessor", "sweep_orchestrator"]
+        required_processes: list[str] = ["data_preprocessor"]
+        if not allow_clean_orchestrator_exit:
+            required_processes.append("sweep_orchestrator")
         if not processor_was_running:
             required_processes.insert(0, "data_processor")
 
         try:
             if not processor_was_running:
-                self._spawn("data_processor", command_specs["data_processor"])
+                self._spawn("data_processor", command_specs["data_processor"], allow_clean_exit=False)
 
-            self._spawn("data_preprocessor", command_specs["data_preprocessor"])
-            self._spawn("sweep_orchestrator", command_specs["sweep_orchestrator"])
+            self._spawn("data_preprocessor", command_specs["data_preprocessor"], allow_clean_exit=False)
+            self._spawn(
+                "sweep_orchestrator",
+                command_specs["sweep_orchestrator"],
+                allow_clean_exit=allow_clean_orchestrator_exit,
+            )
             self._wait_until_ready(required_processes)
         except Exception:
             if processor_was_running:
@@ -93,20 +141,32 @@ class ProcessSupervisor:
         """Stop all managed processes."""
         self._stop_processes(["sweep_orchestrator", "data_preprocessor", "data_processor"])
 
-    def _spawn(self, name: str, command: list[str]) -> None:
+    def _spawn(self, name: str, command: list[str], *, allow_clean_exit: bool) -> None:
         """Spawn one process unless same process is already alive."""
         existing = self._processes.get(name)
         if existing is not None and existing.handle.poll() is None:
             return
 
-        handle = subprocess.Popen(
-            command,
-            cwd=self._project_root,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            text=True,
+        try:
+            handle = subprocess.Popen(
+                command,
+                cwd=self._project_root,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+            )
+        except OSError as exc:
+            command_text = shlex.join(command)
+            raise RuntimeError(
+                f"Failed to spawn {name} with command `{command_text}` from `{self._project_root}`: "
+                f"{type(exc).__name__}: {exc}"
+            ) from exc
+        self._processes[name] = ManagedProcess(
+            name=name,
+            command=command,
+            allow_clean_exit=allow_clean_exit,
+            handle=handle,
         )
-        self._processes[name] = ManagedProcess(name=name, command=command, handle=handle)
 
     def _stop_processes(self, names: Iterable[str]) -> None:
         """Gracefully terminate processes, then force-kill on timeout."""
@@ -148,9 +208,9 @@ class ProcessSupervisor:
             return False
         return process.handle.poll() is None
 
-    def collect_crash_reports(self) -> list[str]:
-        """Collect stderr/stdout reports from processes that have exited."""
-        reports: list[str] = []
+    def collect_exit_reports(self) -> list[ProcessExitReport]:
+        """Collect reports for managed processes that have exited."""
+        reports: list[ProcessExitReport] = []
         exited_names: list[str] = []
 
         for name, process in self._processes.items():
@@ -165,13 +225,17 @@ class ProcessSupervisor:
             if process.handle.stderr is not None:
                 stderr = process.handle.stderr.read().strip()
 
-            details = stderr
-            if stdout and stderr:
-                details = f"{stderr}\nstdout:\n{stdout}"
-            elif stdout:
-                details = f"stdout:\n{stdout}"
-
-            reports.append(f"{process.name} exited with code {return_code}: {details}")
+            reports.append(
+                ProcessExitReport(
+                    name=process.name,
+                    command=list(process.command),
+                    working_directory=self._project_root,
+                    return_code=int(return_code),
+                    stdout=stdout,
+                    stderr=stderr,
+                    expected_clean_exit=bool(process.allow_clean_exit and int(return_code) == 0),
+                )
+            )
             exited_names.append(name)
 
         for name in exited_names:
@@ -183,15 +247,19 @@ class ProcessSupervisor:
         deadline = time.monotonic() + self._readiness_timeout_s
 
         while time.monotonic() < deadline:
-            crashed = self.collect_crash_reports()
-            if crashed:
-                raise RuntimeError("; ".join(crashed))
+            exit_reports = self.collect_exit_reports()
+            unexpected_reports = [report for report in exit_reports if not report.expected_clean_exit]
+            if unexpected_reports:
+                raise RuntimeError("; ".join(report.format() for report in unexpected_reports))
             if all(self._is_alive(process_name) for process_name in required_processes):
                 return
             time.sleep(0.05)
 
         names = ", ".join(required_processes)
-        raise RuntimeError(f"Timed out waiting for processes to start: {names}")
+        raise RuntimeError(
+            f"Timed out waiting for processes to start: {names}. "
+            f"Alive processes: {self.pids() or 'none'}"
+        )
 
     def pids(self) -> dict[str, int]:
         """Return PID mapping for currently alive managed processes."""