"""Process supervisor for lifecycle management of pipeline subprocesses.""" from __future__ import annotations from dataclasses import dataclass import logging import os from pathlib import Path import shlex import signal import subprocess import time from typing import Iterable from typing import Sequence from python_app.file_io import atomic_write_text from python_app.orchestration.producer_selection import acquisition_command logger = logging.getLogger(__name__) # Cap each child log so a long-lived daemon cannot fill the SD card. On reaching # the cap the current log is rolled to `{name}.{out,err}.log.prev` and a fresh # log opened (see `_roll_log_if_oversized`). _LOG_MAX_BYTES = 8 * 1024 * 1024 # Per-process force-kill deadline used on stop; each child gets its own window. _STOP_GRACE_SECONDS = 2.0 @dataclass(slots=True) class ManagedProcess: """Metadata and subprocess handle for one managed child process.""" name: str command: list[str] allow_clean_exit: bool handle: subprocess.Popen[bytes] stdout_path: Path stderr_path: Path @dataclass(slots=True) class ProcessExitReport: """Structured report for one exited managed process. Log tails are not held in memory: they are read on demand from the child log files only while rendering an ERROR report, so the common clean-exit path on every poll never pays for a 16KB read of two files. """ name: str command: list[str] working_directory: Path return_code: int stdout_path: Path stderr_path: Path expected_clean_exit: bool @property def level(self) -> str: """Return log level appropriate for this exit report.""" return "INFO" if self.expected_clean_exit else "ERROR" def format(self) -> str: """Render human-readable multiline exit report.""" if self.expected_clean_exit: headline = f"Process `{self.name}` completed normally with code {self.return_code}." elif self.return_code == 0: headline = f"Process `{self.name}` exited unexpectedly with code 0." else: headline = f"Process `{self.name}` exited with code {self.return_code}." lines = [ headline, f"Command: {shlex.join(self.command)}", f"Working directory: {self.working_directory}", ] # Read the (bounded) log tails lazily, only now that we are rendering. stdout = "" if self.expected_clean_exit else _read_log_tail(self.stdout_path) stderr = "" if self.expected_clean_exit else _read_log_tail(self.stderr_path) if stderr: lines.append(f"stderr:\n{stderr}") if stdout: lines.append(f"stdout:\n{stdout}") if not self.expected_clean_exit and not stderr and not stdout: lines.append("stdout/stderr: none") return "\n".join(lines) def _read_log_tail(path: Path, max_bytes: int = 16384) -> str: """Return the trailing `max_bytes` of a child log file, decoded best-effort. Bounded so a large/long-lived log never produces an enormous exit report. """ try: with open(path, "rb") as handle: handle.seek(0, 2) size = handle.tell() if size > max_bytes: handle.seek(-max_bytes, 2) else: handle.seek(0) data = handle.read() except OSError: return "" # Drop NULs: logs written by an older supervisor can carry a sparse hole from # the pre-O_APPEND truncate bug, and a tail landing in it would otherwise turn # an exit report (or a rolled `.prev`) into megabytes of NUL padding. return data.replace(b"\0", b"").decode("utf-8", errors="replace").strip() class ProcessSupervisor: """Start, monitor, and stop pipeline subprocesses.""" def __init__(self, project_root: Path, readiness_timeout_s: float = 3.0) -> None: """Create supervisor bound to repository root directory.""" self._project_root = project_root self._readiness_timeout_s = readiness_timeout_s self._processes: dict[str, ManagedProcess] = {} # Runtime pidfile lets us reap pipeline children left behind by a prior # supervisor (crash/SIGKILL) independent of in-memory state. self._runtime_dir = self._project_root / "python_app/runtime" self._pidfile_path = self._runtime_dir / "supervisor_children.pids" logger.debug( "ProcessSupervisor init: root=%s readiness_timeout=%.1fs", self._project_root, self._readiness_timeout_s, ) self._reap_stale_children() def is_running(self) -> bool: """Return whether acquisition-side processes are alive.""" return self._is_alive("data_preprocessor") or self._is_alive("acquisition") def is_processor_running(self) -> bool: """Return whether data processor process is alive.""" return self._is_alive("data_processor") def start(self, config_path: Path, *, allow_clean_acquisition_exit: bool = False) -> None: """Start required pipeline binaries and wait until they are ready.""" if self.is_running(): raise RuntimeError("Acquisition processes are already running") logger.info( "Starting pipeline from config %s (allow_clean_acquisition_exit=%s)", config_path, allow_clean_acquisition_exit, ) acquisition_command = self._acquisition_command(config_path) command_specs = { "data_processor": [ str(self._project_root / "build/bin/data_processor"), "--config", str(config_path), ], "data_preprocessor": [ str(self._project_root / "build/bin/data_preprocessor"), "--config", str(config_path), ], "acquisition": acquisition_command, } processor_was_running = self.is_processor_running() required_processes: list[str] = ["data_preprocessor"] if not allow_clean_acquisition_exit: required_processes.append("acquisition") if not processor_was_running: required_processes.insert(0, "data_processor") try: if not processor_was_running: self._spawn("data_processor", command_specs["data_processor"], allow_clean_exit=False) self._spawn("data_preprocessor", command_specs["data_preprocessor"], allow_clean_exit=False) self._spawn( "acquisition", command_specs["acquisition"], allow_clean_exit=allow_clean_acquisition_exit, ) self._wait_until_ready(required_processes) except Exception: logger.exception("Pipeline start failed; tearing down spawned processes") if processor_was_running: self.stop() else: self.stop_all() raise logger.info("Pipeline started; live pids=%s", self.pids() or "none") def stop(self) -> None: """Stop acquisition-side processes, keep processor process intact.""" self._stop_processes(["acquisition", "data_preprocessor"]) def stop_acquisition(self) -> None: """Stop acquisition process only.""" self._stop_processes(["acquisition"]) def stop_preprocessor(self) -> None: """Stop preprocessor process only.""" self._stop_processes(["data_preprocessor"]) def stop_all(self) -> None: """Stop all managed processes.""" self._stop_processes(["acquisition", "data_preprocessor", "data_processor"]) def _spawn(self, name: str, command: list[str], *, allow_clean_exit: bool) -> None: """Spawn one process unless same process is already alive. Child stdout/stderr are redirected to per-process log files rather than captured pipes: a long-running child (e.g. an acquisition producer waiting for its device) would otherwise fill the OS pipe buffer once nobody drains it and block on write. Files never back-pressure the child, and they keep a persistent log we can read for exit reports and tail for diagnostics. """ existing = self._processes.get(name) if existing is not None and existing.handle.poll() is None: logger.debug("Spawn skipped: `%s` already running (pid=%s)", name, existing.handle.pid) return logs_dir = self._project_root / "python_app/runtime/logs" logs_dir.mkdir(parents=True, exist_ok=True) stdout_path = logs_dir / f"{name}.out.log" stderr_path = logs_dir / f"{name}.err.log" # Roll any stale (uncollected) log to `.prev` before truncating so the # previous run's diagnostics survive a respawn. self._roll_log_to_prev(stdout_path) self._roll_log_to_prev(stderr_path) # O_APPEND ("ab"), not "wb": the child inherits these fds and keeps its own # file offset. Without O_APPEND, the in-place truncate in # `_roll_log_if_oversized` leaves that offset far past the new end of file, # so the next write lands there and the kernel fills everything before it # with a hole of NUL bytes — the log becomes unreadable and the size cap # stops working entirely. O_APPEND makes the kernel seek to EOF atomically # on every write, so a truncate genuinely restarts the file at offset 0. # `_roll_log_to_prev` above already renamed any previous log away, so not # truncating on open costs nothing. try: with stdout_path.open("ab") as stdout_file, stderr_path.open("ab") as stderr_file: handle = subprocess.Popen( command, cwd=self._project_root, stdout=stdout_file, stderr=stderr_file, # Device-I/O grandchildren share this group and stop with it. start_new_session=True, ) except OSError as exc: command_text = shlex.join(command) logger.error("Failed to spawn `%s`: %s: %s", name, type(exc).__name__, exc) raise RuntimeError( f"Failed to spawn {name} with command `{command_text}` from `{self._project_root}`: " f"{type(exc).__name__}: {exc}" ) from exc self._processes[name] = ManagedProcess( name=name, command=command, allow_clean_exit=allow_clean_exit, handle=handle, stdout_path=stdout_path, stderr_path=stderr_path, ) logger.info("Spawned `%s` (pid=%d)", name, handle.pid) self._write_pidfile() def _acquisition_command(self, config_path: Path) -> list[str]: """Select the same acquisition producer as the shell launcher.""" return acquisition_command(config_path) def _stop_processes(self, names: Iterable[str]) -> None: """Gracefully terminate processes, then force-kill on per-process timeout.""" ordered_names = list(names) for name in ordered_names: process = self._processes.get(name) if process is None: continue if process.handle.poll() is None: logger.info("Stopping `%s` (pid=%d): sending SIGTERM to group", name, process.handle.pid) # Signal the whole group so device-I/O grandchildren die too. self._signal_group(process.handle.pid, signal.SIGTERM) for name in ordered_names: process = self._processes.get(name) if process is None: continue if process.handle.poll() is not None: self._log_abnormal_stop_exit(process) continue # Each process gets its own kill deadline so a slow shutdown of one # cannot consume the grace window of the others. try: process.handle.wait(timeout=_STOP_GRACE_SECONDS) except subprocess.TimeoutExpired: logger.warning( "`%s` did not exit within %.1fs of SIGTERM; sending SIGKILL", name, _STOP_GRACE_SECONDS, ) self._signal_group(process.handle.pid, signal.SIGKILL) try: process.handle.wait(timeout=1.0) except subprocess.TimeoutExpired: logger.error("`%s` still alive after SIGKILL", name) else: # A negative code here is the SIGTERM we just sent (expected); only # a positive self-exit during the grace window is worth noting. self._log_abnormal_stop_exit(process) self._drop_exited() @staticmethod def _signal_group(pid: int, sig: int) -> None: """Signal the child's whole process group, falling back to the child.""" if pid is None: return try: # Managed children lead their own sessions. Never signal an unrelated # shell's group if a stale PID or an external collector does not. if os.getpgid(pid) == pid: os.killpg(pid, sig) else: os.kill(pid, sig) except (ProcessLookupError, PermissionError): # Group already gone, or could not resolve it; fall back to the child. try: os.kill(pid, sig) except (ProcessLookupError, PermissionError): pass @staticmethod def _log_abnormal_stop_exit(process: ManagedProcess) -> None: """Log a process that self-exited abnormally around stop time. Negative codes are signal-induced (e.g. the SIGTERM we send on stop) and are expected; only a non-zero self-exit is reported. """ return_code = process.handle.poll() if return_code is None or return_code <= 0: return logger.warning("`%s` exited abnormally with code %d around stop", process.name, return_code) def _drop_exited(self) -> None: """Remove exited process entries from internal map.""" exited_names = [name for name, process in self._processes.items() if process.handle.poll() is not None] for name in exited_names: self._processes.pop(name, None) if exited_names: self._write_pidfile() def _is_alive(self, name: str) -> bool: """Return `True` when named process handle exists and is running.""" process = self._processes.get(name) if process is None: return False return process.handle.poll() is None def collect_exit_reports(self) -> list[ProcessExitReport]: """Collect reports for managed processes that have exited.""" reports: list[ProcessExitReport] = [] exited_names: list[str] = [] for name, process in self._processes.items(): return_code = process.handle.poll() if return_code is None: # Still running: enforce the size cap so logs never grow unbounded. self._roll_log_if_oversized(process.stdout_path) self._roll_log_if_oversized(process.stderr_path) continue logger.debug("Reaped `%s` with exit code %d", process.name, int(return_code)) reports.append( ProcessExitReport( name=process.name, command=list(process.command), working_directory=self._project_root, return_code=int(return_code), stdout_path=process.stdout_path, stderr_path=process.stderr_path, expected_clean_exit=bool(process.allow_clean_exit and int(return_code) == 0), ) ) exited_names.append(name) for name in exited_names: self._processes.pop(name, None) if exited_names: self._write_pidfile() return reports def _wait_until_ready(self, required_processes: Sequence[str]) -> None: """Wait until all required processes are alive or timeout/crash occurs.""" deadline = time.monotonic() + self._readiness_timeout_s logger.debug("Waiting for processes to become ready: %s", ", ".join(required_processes)) while time.monotonic() < deadline: exit_reports = self.collect_exit_reports() unexpected_reports = [report for report in exit_reports if not report.expected_clean_exit] if unexpected_reports: raise RuntimeError("; ".join(report.format() for report in unexpected_reports)) if all(self._is_alive(process_name) for process_name in required_processes): logger.debug("All required processes ready") return time.sleep(0.05) names = ", ".join(required_processes) raise RuntimeError( f"Timed out waiting for processes to start: {names}. " f"Alive processes: {self.pids() or 'none'}" ) def pids(self) -> dict[str, int]: """Return PID mapping for currently alive managed processes.""" return { process.name: process.handle.pid for process in self._processes.values() if process.handle.poll() is None and process.handle.pid is not None } @staticmethod def _roll_log_to_prev(path: Path) -> None: """Roll an existing log to `{path}.prev` before it is reopened. Preserves a stale (exited, not-yet-reported) child's last output instead of truncating it when a fresh log is opened for a respawn. """ if not path.exists(): return try: path.replace(path.with_suffix(path.suffix + ".prev")) except OSError: # Best-effort: a failed roll must not block a spawn. pass @staticmethod def _roll_log_if_oversized(path: Path) -> None: """Bound a live child log to `_LOG_MAX_BYTES` so it cannot fill the SD card. The child holds an open fd to this inode, so a rename would not redirect its writes. Instead keep one rolled generation via copy-to-`.prev` and truncate the live inode in place, freeing the allocated disk blocks. This relies on the child's fd being opened with O_APPEND (see `_spawn`): only then does the child resume writing at offset 0 after the truncate. With a plain write fd it would keep writing at its stale offset, punching a multi-hundred-megabyte NUL hole and defeating the cap. """ try: if path.stat().st_size <= _LOG_MAX_BYTES: return except OSError: return prev_path = path.with_suffix(path.suffix + ".prev") try: # Preserve the trailing window as the rolled generation, then truncate. tail = _read_log_tail(path, _LOG_MAX_BYTES) prev_path.write_text(tail, encoding="utf-8") with open(path, "r+b") as handle: handle.truncate(0) except OSError: # Best-effort: capping is opportunistic and must not disrupt polling. pass def _write_pidfile(self) -> None: """Persist live child PIDs so a later supervisor can reap them.""" try: self._runtime_dir.mkdir(parents=True, exist_ok=True) live_pids = [ str(process.handle.pid) for process in self._processes.values() if process.handle.poll() is None and process.handle.pid is not None ] atomic_write_text(self._pidfile_path, "\n".join(live_pids)) except OSError: # Best-effort bookkeeping: failure here must not break start/stop. pass def _reap_stale_children(self) -> None: """Kill pipeline children recorded by a prior supervisor instance. On a clean shutdown the pidfile is emptied; entries only remain when the previous supervisor died without stopping its children. We SIGKILL each stale process group so leftover pipeline binaries cannot hold the shared memory rings or devices hostage on the next start. """ try: raw = self._pidfile_path.read_text(encoding="utf-8") except OSError: return for token in raw.split(): try: pid = int(token) except ValueError: continue if pid <= 1 or pid == os.getpid(): continue # Guard against PID reuse: only reap if the process still looks like # one of our pipeline children before signalling its group. if self._is_stale_pipeline_pid(pid): logger.warning("Reaping stale pipeline child from prior run (pid=%d)", pid) self._signal_group(pid, signal.SIGKILL) try: atomic_write_text(self._pidfile_path, "") except OSError: pass def _is_stale_pipeline_pid(self, pid: int) -> bool: """Return whether `pid` still runs one of our pipeline binaries/scripts. Reads `/proc//cmdline` so a recycled PID owned by an unrelated process is never killed. """ native_names = {"data_processor", "data_preprocessor"} producer_modules = { "python_app.scripts.matrix_raw_producer", "python_app.scripts.rph_radar_raw_producer", } try: raw = (Path("/proc") / str(pid) / "cmdline").read_bytes() except OSError: return False argv = raw.decode("utf-8", errors="replace").rstrip("\x00").split("\x00") executable = Path(argv[0]) if executable.name in native_names: expected = self._project_root / "build/bin" / executable.name return executable == expected or executable == Path("build/bin") / executable.name is_python = executable.name in {"python", "python3"} or executable.name.startswith("python3.") return is_python and len(argv) >= 3 and argv[1] == "-m" and argv[2] in producer_modules