"""Process supervisor for lifecycle management of C++ pipeline binaries.""" from __future__ import annotations from dataclasses import dataclass import json import logging import os from pathlib import Path import shlex import signal import subprocess import sys import time from typing import Iterable from typing import Sequence logger = logging.getLogger(__name__) # Cap each child log so a long-lived daemon cannot fill the SD card. On reaching # the cap the current log is rolled to `{name}.{out,err}.log.prev` and a fresh # log opened (see `_roll_log_if_oversized`). _LOG_MAX_BYTES = 8 * 1024 * 1024 # Per-process force-kill deadline used on stop; each child gets its own window. _STOP_GRACE_SECONDS = 2.0 # Cmdline marker for the Kamil ADC collector binary. The collector runs in its own # session, so it can outlive a force-killed acquisition producer while still holding # the L-Card E-502; we hunt it by name as a guaranteed device-release backstop. _KAMIL_COLLECTOR_MARKER = b"kamil_adc_collector" @dataclass(slots=True) class ManagedProcess: """Metadata and subprocess handle for one managed child process.""" name: str command: list[str] allow_clean_exit: bool handle: subprocess.Popen[bytes] stdout_path: Path stderr_path: Path @dataclass(slots=True) class ProcessExitReport: """Structured report for one exited managed process. Log tails are not held in memory: they are read on demand from the child log files only while rendering an ERROR report, so the common clean-exit path on every poll never pays for a 16KB read of two files. """ name: str command: list[str] working_directory: Path return_code: int stdout_path: Path stderr_path: Path expected_clean_exit: bool @property def level(self) -> str: """Return log level appropriate for this exit report.""" return "INFO" if self.expected_clean_exit else "ERROR" def format(self) -> str: """Render human-readable multiline exit report.""" if self.expected_clean_exit: headline = f"Process `{self.name}` completed normally with code {self.return_code}." elif self.return_code == 0: headline = f"Process `{self.name}` exited unexpectedly with code 0." else: headline = f"Process `{self.name}` exited with code {self.return_code}." lines = [ headline, f"Command: {shlex.join(self.command)}", f"Working directory: {self.working_directory}", ] # Read the (bounded) log tails lazily, only now that we are rendering. stdout = "" if self.expected_clean_exit else _read_log_tail(self.stdout_path) stderr = "" if self.expected_clean_exit else _read_log_tail(self.stderr_path) if stderr: lines.append(f"stderr:\n{stderr}") if stdout: lines.append(f"stdout:\n{stdout}") if not self.expected_clean_exit and not stderr and not stdout: lines.append("stdout/stderr: none") return "\n".join(lines) def _read_log_tail(path: Path, max_bytes: int = 16384) -> str: """Return the trailing `max_bytes` of a child log file, decoded best-effort. Bounded so a large/long-lived log never produces an enormous exit report. """ try: with open(path, "rb") as handle: handle.seek(0, 2) size = handle.tell() if size > max_bytes: handle.seek(-max_bytes, 2) else: handle.seek(0) data = handle.read() except OSError: return "" return data.decode("utf-8", errors="replace").strip() class ProcessSupervisor: """Start, monitor, and stop pipeline subprocesses.""" def __init__(self, project_root: Path, readiness_timeout_s: float = 3.0) -> None: """Create supervisor bound to repository root directory.""" self._project_root = project_root self._readiness_timeout_s = readiness_timeout_s self._processes: dict[str, ManagedProcess] = {} # Runtime pidfile lets us reap pipeline children left behind by a prior # supervisor (crash/SIGKILL) independent of in-memory state. self._runtime_dir = self._project_root / "python_app/runtime" self._pidfile_path = self._runtime_dir / "supervisor_children.pids" logger.debug( "ProcessSupervisor init: root=%s readiness_timeout=%.1fs", self._project_root, self._readiness_timeout_s, ) self._reap_stale_children() def is_running(self) -> bool: """Return whether acquisition-side processes are alive.""" return self._is_alive("data_preprocessor") or self._is_alive("sweep_orchestrator") def is_processor_running(self) -> bool: """Return whether data processor process is alive.""" return self._is_alive("data_processor") def start(self, config_path: Path, *, allow_clean_orchestrator_exit: bool = False) -> None: """Start required pipeline binaries and wait until they are ready.""" if self.is_running(): raise RuntimeError("Acquisition processes are already running") # Clean slate: an ADC collector orphaned by a prior crash/force-kill can # still hold the E-502 and make a fresh acquisition fail to open. self._kill_kamil_collectors() logger.info( "Starting pipeline from config %s (allow_clean_orchestrator_exit=%s)", config_path, allow_clean_orchestrator_exit, ) acquisition_command = self._acquisition_command(config_path) command_specs = { "data_processor": [ str(self._project_root / "build/bin/data_processor"), "--config", str(config_path), ], "data_preprocessor": [ str(self._project_root / "build/bin/data_preprocessor"), "--config", str(config_path), ], "sweep_orchestrator": acquisition_command, } processor_was_running = self.is_processor_running() required_processes: list[str] = ["data_preprocessor"] if not allow_clean_orchestrator_exit: required_processes.append("sweep_orchestrator") if not processor_was_running: required_processes.insert(0, "data_processor") try: if not processor_was_running: self._spawn("data_processor", command_specs["data_processor"], allow_clean_exit=False) self._spawn("data_preprocessor", command_specs["data_preprocessor"], allow_clean_exit=False) self._spawn( "sweep_orchestrator", command_specs["sweep_orchestrator"], allow_clean_exit=allow_clean_orchestrator_exit, ) self._wait_until_ready(required_processes) except Exception: logger.exception("Pipeline start failed; tearing down spawned processes") if processor_was_running: self.stop() else: self.stop_all() raise logger.info("Pipeline started; live pids=%s", self.pids() or "none") def stop(self) -> None: """Stop acquisition-side processes, keep processor process intact.""" self._stop_processes(["sweep_orchestrator", "data_preprocessor"]) self._kill_kamil_collectors() def stop_orchestrator(self) -> None: """Stop orchestrator process only.""" self._stop_processes(["sweep_orchestrator"]) def stop_preprocessor(self) -> None: """Stop preprocessor process only.""" self._stop_processes(["data_preprocessor"]) def stop_all(self) -> None: """Stop all managed processes.""" self._stop_processes(["sweep_orchestrator", "data_preprocessor", "data_processor"]) self._kill_kamil_collectors() def _spawn(self, name: str, command: list[str], *, allow_clean_exit: bool) -> None: """Spawn one process unless same process is already alive. Child stdout/stderr are redirected to per-process log files rather than captured pipes: a long-running child (e.g. an acquisition producer waiting for its device) would otherwise fill the OS pipe buffer once nobody drains it and block on write. Files never back-pressure the child, and they keep a persistent log we can read for exit reports and tail for diagnostics. """ existing = self._processes.get(name) if existing is not None and existing.handle.poll() is None: logger.debug("Spawn skipped: `%s` already running (pid=%s)", name, existing.handle.pid) return logs_dir = self._project_root / "python_app/runtime/logs" logs_dir.mkdir(parents=True, exist_ok=True) stdout_path = logs_dir / f"{name}.out.log" stderr_path = logs_dir / f"{name}.err.log" # Roll any stale (uncollected) log to `.prev` before truncating so the # previous run's diagnostics survive a respawn. self._roll_log_to_prev(stdout_path) self._roll_log_to_prev(stderr_path) stdout_file = open(stdout_path, "wb") stderr_file = open(stderr_path, "wb") try: handle = subprocess.Popen( command, cwd=self._project_root, stdout=stdout_file, stderr=stderr_file, # Own session/process group so signalling the group on stop also # reaches device-I/O grandchildren the producer may have spawned. start_new_session=True, ) except OSError as exc: stdout_file.close() stderr_file.close() command_text = shlex.join(command) logger.error("Failed to spawn `%s`: %s: %s", name, type(exc).__name__, exc) raise RuntimeError( f"Failed to spawn {name} with command `{command_text}` from `{self._project_root}`: " f"{type(exc).__name__}: {exc}" ) from exc finally: # The child holds its own dup'd fds; the parent's copies are not needed. stdout_file.close() stderr_file.close() self._processes[name] = ManagedProcess( name=name, command=command, allow_clean_exit=allow_clean_exit, handle=handle, stdout_path=stdout_path, stderr_path=stderr_path, ) logger.info("Spawned `%s` (pid=%d)", name, handle.pid) self._write_pidfile() def _acquisition_command(self, config_path: Path) -> list[str]: """Return acquisition producer command selected by radar.model.""" radar_model = self._read_radar_model(config_path) logger.debug("Selecting acquisition producer for radar.model=%s", radar_model) if radar_model in {"librevna_multi", "sn9000"}: return [ sys.executable, "-m", "python_app.scripts.matrix_raw_producer", "--config", str(config_path), ] if radar_model == "kamil_adc": return [ sys.executable, "-m", "python_app.scripts.kamil_adc_raw_producer", "--config", str(config_path), ] return [ str(self._project_root / "build/bin/sweep_orchestrator"), "--config", str(config_path), ] @staticmethod def _read_radar_model(config_path: Path) -> str: """Read `radar.model` cheaply without constructing the full config model.""" payload = json.loads(config_path.read_text(encoding="utf-8")) if not isinstance(payload, dict): return "librevna" radar_payload = payload.get("radar") if not isinstance(radar_payload, dict): return "librevna" return str(radar_payload.get("model", "librevna")) def _stop_processes(self, names: Iterable[str]) -> None: """Gracefully terminate processes, then force-kill on per-process timeout.""" ordered_names = list(names) for name in ordered_names: process = self._processes.get(name) if process is None: continue if process.handle.poll() is None: logger.info("Stopping `%s` (pid=%d): sending SIGTERM to group", name, process.handle.pid) # Signal the whole group so device-I/O grandchildren die too. self._signal_group(process.handle.pid, signal.SIGTERM) for name in ordered_names: process = self._processes.get(name) if process is None: continue if process.handle.poll() is not None: self._log_abnormal_stop_exit(process) continue # Each process gets its own kill deadline so a slow shutdown of one # cannot consume the grace window of the others. try: process.handle.wait(timeout=_STOP_GRACE_SECONDS) except subprocess.TimeoutExpired: logger.warning( "`%s` did not exit within %.1fs of SIGTERM; sending SIGKILL", name, _STOP_GRACE_SECONDS, ) self._signal_group(process.handle.pid, signal.SIGKILL) try: process.handle.wait(timeout=1.0) except subprocess.TimeoutExpired: logger.error("`%s` still alive after SIGKILL", name) else: # A negative code here is the SIGTERM we just sent (expected); only # a positive self-exit during the grace window is worth noting. self._log_abnormal_stop_exit(process) self._drop_exited() @staticmethod def _signal_group(pid: int, sig: int) -> None: """Signal the child's whole process group, falling back to the child.""" if pid is None: return try: os.killpg(os.getpgid(pid), sig) except (ProcessLookupError, PermissionError): # Group already gone, or could not resolve it; fall back to the child. try: os.kill(pid, sig) except (ProcessLookupError, PermissionError): pass def _kill_kamil_collectors(self) -> None: """SIGKILL any Kamil ADC collector found by name — a guaranteed device backstop. The collector survives a force-killed producer (own session) and can hang the E-502, so it is hunted by cmdline and its group killed hard, regardless of who its parent is. Best-effort and never raises. """ own_pid = os.getpid() try: pids = [int(entry.name) for entry in Path("/proc").iterdir() if entry.name.isdigit()] except OSError: return killed = 0 for pid in pids: if pid <= 1 or pid == own_pid: continue try: cmdline = (Path("/proc") / str(pid) / "cmdline").read_bytes() except OSError: continue if _KAMIL_COLLECTOR_MARKER in cmdline: self._signal_group(pid, signal.SIGKILL) killed += 1 if killed: logger.warning("Force-killed %d orphaned Kamil ADC collector process(es)", killed) @staticmethod def _log_abnormal_stop_exit(process: ManagedProcess) -> None: """Log a process that self-exited abnormally around stop time. Negative codes are signal-induced (e.g. the SIGTERM we send on stop) and are expected; only a non-zero self-exit is reported. """ return_code = process.handle.poll() if return_code is None or return_code <= 0: return logger.warning("`%s` exited abnormally with code %d around stop", process.name, return_code) def _drop_exited(self) -> None: """Remove exited process entries from internal map.""" exited_names = [name for name, process in self._processes.items() if process.handle.poll() is not None] for name in exited_names: self._processes.pop(name, None) if exited_names: self._write_pidfile() def _is_alive(self, name: str) -> bool: """Return `True` when named process handle exists and is running.""" process = self._processes.get(name) if process is None: return False return process.handle.poll() is None def collect_exit_reports(self) -> list[ProcessExitReport]: """Collect reports for managed processes that have exited.""" reports: list[ProcessExitReport] = [] exited_names: list[str] = [] for name, process in self._processes.items(): return_code = process.handle.poll() if return_code is None: # Still running: enforce the size cap so logs never grow unbounded. self._roll_log_if_oversized(process.stdout_path) self._roll_log_if_oversized(process.stderr_path) continue logger.debug("Reaped `%s` with exit code %d", process.name, int(return_code)) reports.append( ProcessExitReport( name=process.name, command=list(process.command), working_directory=self._project_root, return_code=int(return_code), stdout_path=process.stdout_path, stderr_path=process.stderr_path, expected_clean_exit=bool(process.allow_clean_exit and int(return_code) == 0), ) ) exited_names.append(name) for name in exited_names: self._processes.pop(name, None) if exited_names: self._write_pidfile() return reports def _wait_until_ready(self, required_processes: Sequence[str]) -> None: """Wait until all required processes are alive or timeout/crash occurs.""" deadline = time.monotonic() + self._readiness_timeout_s logger.debug("Waiting for processes to become ready: %s", ", ".join(required_processes)) while time.monotonic() < deadline: exit_reports = self.collect_exit_reports() unexpected_reports = [report for report in exit_reports if not report.expected_clean_exit] if unexpected_reports: raise RuntimeError("; ".join(report.format() for report in unexpected_reports)) if all(self._is_alive(process_name) for process_name in required_processes): logger.debug("All required processes ready") return time.sleep(0.05) names = ", ".join(required_processes) raise RuntimeError( f"Timed out waiting for processes to start: {names}. " f"Alive processes: {self.pids() or 'none'}" ) def pids(self) -> dict[str, int]: """Return PID mapping for currently alive managed processes.""" return { process.name: process.handle.pid for process in self._processes.values() if process.handle.poll() is None and process.handle.pid is not None } @staticmethod def _roll_log_to_prev(path: Path) -> None: """Roll an existing log to `{path}.prev` before it is reopened. Preserves a stale (exited, not-yet-reported) child's last output instead of truncating it when a fresh log is opened for a respawn. """ if not path.exists(): return try: path.replace(path.with_suffix(path.suffix + ".prev")) except OSError: # Best-effort: a failed roll must not block a spawn. pass @staticmethod def _roll_log_if_oversized(path: Path) -> None: """Bound a live child log to `_LOG_MAX_BYTES` so it cannot fill the SD card. The child holds an open fd to this inode, so a rename would not redirect its writes. Instead keep one rolled generation via copy-to-`.prev` and truncate the live inode in place, freeing the allocated disk blocks. """ try: if path.stat().st_size <= _LOG_MAX_BYTES: return except OSError: return prev_path = path.with_suffix(path.suffix + ".prev") try: # Preserve the trailing window as the rolled generation, then truncate. tail = _read_log_tail(path, _LOG_MAX_BYTES) prev_path.write_text(tail, encoding="utf-8") with open(path, "r+b") as handle: handle.truncate(0) except OSError: # Best-effort: capping is opportunistic and must not disrupt polling. pass def _write_pidfile(self) -> None: """Persist live child PIDs so a later supervisor can reap them.""" try: self._runtime_dir.mkdir(parents=True, exist_ok=True) live_pids = [ str(process.handle.pid) for process in self._processes.values() if process.handle.poll() is None and process.handle.pid is not None ] self._pidfile_path.write_text("\n".join(live_pids), encoding="utf-8") except OSError: # Best-effort bookkeeping: failure here must not break start/stop. pass def _reap_stale_children(self) -> None: """Kill pipeline children recorded by a prior supervisor instance. On a clean shutdown the pidfile is emptied; entries only remain when the previous supervisor died without stopping its children. We SIGKILL each stale process group so leftover pipeline binaries cannot hold the shared memory rings or devices hostage on the next start. """ try: raw = self._pidfile_path.read_text(encoding="utf-8") except OSError: return for token in raw.split(): try: pid = int(token) except ValueError: continue if pid <= 1 or pid == os.getpid(): continue # Guard against PID reuse: only reap if the process still looks like # one of our pipeline children before signalling its group. if self._is_stale_pipeline_pid(pid): logger.warning("Reaping stale pipeline child from prior run (pid=%d)", pid) self._signal_group(pid, signal.SIGKILL) try: self._pidfile_path.write_text("", encoding="utf-8") except OSError: pass # The ADC collector is a grandchild (spawned by the producer), not in the # pidfile, so reap it separately by name. self._kill_kamil_collectors() def _is_stale_pipeline_pid(self, pid: int) -> bool: """Return whether `pid` still runs one of our pipeline binaries/scripts. Reads `/proc//cmdline` so a recycled PID owned by an unrelated process is never killed. """ markers = ( "build/bin/data_processor", "build/bin/data_preprocessor", "build/bin/sweep_orchestrator", "python_app.scripts.matrix_raw_producer", "python_app.scripts.kamil_adc_raw_producer", ) try: raw = (Path("/proc") / str(pid) / "cmdline").read_bytes() except OSError: return False cmdline = raw.replace(b"\x00", b" ").decode("utf-8", errors="replace") return any(marker in cmdline for marker in markers)