521 lines
20 KiB
Python
521 lines
20 KiB
Python
"""Process supervisor for lifecycle management of C++ pipeline binaries."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shlex
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from typing import Iterable
|
|
from typing import Sequence
|
|
|
|
# Cap each child log so a long-lived daemon cannot fill the SD card. On reaching
|
|
# the cap the current log is rolled to `{name}.{out,err}.log.prev` and a fresh
|
|
# log opened (see `_roll_log_if_oversized`).
|
|
_LOG_MAX_BYTES = 8 * 1024 * 1024
|
|
# Per-process force-kill deadline used on stop (Fix #33: own deadline each).
|
|
_STOP_GRACE_SECONDS = 2.0
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ManagedProcess:
|
|
"""Metadata and subprocess handle for one managed child process."""
|
|
|
|
name: str
|
|
command: list[str]
|
|
allow_clean_exit: bool
|
|
handle: subprocess.Popen[bytes]
|
|
stdout_path: Path
|
|
stderr_path: Path
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ProcessExitReport:
|
|
"""Structured report for one exited managed process.
|
|
|
|
Log tails are not held in memory: they are read on demand from the child log
|
|
files only while rendering an ERROR report, so the common clean-exit path on
|
|
every poll never pays for a 16KB read of two files (Fix #46).
|
|
"""
|
|
|
|
name: str
|
|
command: list[str]
|
|
working_directory: Path
|
|
return_code: int
|
|
stdout_path: Path
|
|
stderr_path: Path
|
|
expected_clean_exit: bool
|
|
|
|
@property
|
|
def level(self) -> str:
|
|
"""Return log level appropriate for this exit report."""
|
|
return "INFO" if self.expected_clean_exit else "ERROR"
|
|
|
|
def format(self) -> str:
|
|
"""Render human-readable multiline exit report."""
|
|
if self.expected_clean_exit:
|
|
headline = f"Process `{self.name}` completed normally with code {self.return_code}."
|
|
elif self.return_code == 0:
|
|
headline = f"Process `{self.name}` exited unexpectedly with code 0."
|
|
else:
|
|
headline = f"Process `{self.name}` exited with code {self.return_code}."
|
|
|
|
lines = [
|
|
headline,
|
|
f"Command: {shlex.join(self.command)}",
|
|
f"Working directory: {self.working_directory}",
|
|
]
|
|
# Read the (bounded) log tails lazily, only now that we are rendering.
|
|
stdout = "" if self.expected_clean_exit else _read_log_tail(self.stdout_path)
|
|
stderr = "" if self.expected_clean_exit else _read_log_tail(self.stderr_path)
|
|
if stderr:
|
|
lines.append(f"stderr:\n{stderr}")
|
|
if stdout:
|
|
lines.append(f"stdout:\n{stdout}")
|
|
if not self.expected_clean_exit and not stderr and not stdout:
|
|
lines.append("stdout/stderr: none")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _read_log_tail(path: Path, max_bytes: int = 16384) -> str:
|
|
"""Return the trailing `max_bytes` of a child log file, decoded best-effort.
|
|
|
|
Bounded so a large/long-lived log never produces an enormous exit report.
|
|
"""
|
|
try:
|
|
with open(path, "rb") as handle:
|
|
handle.seek(0, 2)
|
|
size = handle.tell()
|
|
if size > max_bytes:
|
|
handle.seek(-max_bytes, 2)
|
|
else:
|
|
handle.seek(0)
|
|
data = handle.read()
|
|
except OSError:
|
|
return ""
|
|
return data.decode("utf-8", errors="replace").strip()
|
|
|
|
|
|
class ProcessSupervisor:
|
|
"""Start, monitor, and stop pipeline subprocesses."""
|
|
|
|
def __init__(self, project_root: Path, readiness_timeout_s: float = 3.0) -> None:
|
|
"""Create supervisor bound to repository root directory."""
|
|
self._project_root = project_root
|
|
self._readiness_timeout_s = readiness_timeout_s
|
|
self._processes: dict[str, ManagedProcess] = {}
|
|
# Runtime pidfile lets us reap pipeline children left behind by a prior
|
|
# supervisor (crash/SIGKILL) independent of in-memory state (Fix #19).
|
|
self._runtime_dir = self._project_root / "python_app/runtime"
|
|
self._pidfile_path = self._runtime_dir / "supervisor_children.pids"
|
|
self._reap_stale_children()
|
|
|
|
def is_running(self) -> bool:
|
|
"""Return whether acquisition-side processes are alive."""
|
|
return self._is_alive("data_preprocessor") or self._is_alive("sweep_orchestrator")
|
|
|
|
def is_processor_running(self) -> bool:
|
|
"""Return whether data processor process is alive."""
|
|
return self._is_alive("data_processor")
|
|
|
|
def start(self, config_path: Path, *, allow_clean_orchestrator_exit: bool = False) -> None:
|
|
"""Start required pipeline binaries and wait until they are ready."""
|
|
if self.is_running():
|
|
raise RuntimeError("Acquisition processes are already running")
|
|
|
|
acquisition_command = self._acquisition_command(config_path)
|
|
command_specs = {
|
|
"data_processor": [
|
|
str(self._project_root / "build/bin/data_processor"),
|
|
"--config",
|
|
str(config_path),
|
|
],
|
|
"data_preprocessor": [
|
|
str(self._project_root / "build/bin/data_preprocessor"),
|
|
"--config",
|
|
str(config_path),
|
|
],
|
|
"sweep_orchestrator": acquisition_command,
|
|
}
|
|
processor_was_running = self.is_processor_running()
|
|
required_processes: list[str] = ["data_preprocessor"]
|
|
if not allow_clean_orchestrator_exit:
|
|
required_processes.append("sweep_orchestrator")
|
|
if not processor_was_running:
|
|
required_processes.insert(0, "data_processor")
|
|
|
|
try:
|
|
if not processor_was_running:
|
|
self._spawn("data_processor", command_specs["data_processor"], allow_clean_exit=False)
|
|
|
|
self._spawn("data_preprocessor", command_specs["data_preprocessor"], allow_clean_exit=False)
|
|
self._spawn(
|
|
"sweep_orchestrator",
|
|
command_specs["sweep_orchestrator"],
|
|
allow_clean_exit=allow_clean_orchestrator_exit,
|
|
)
|
|
self._wait_until_ready(required_processes)
|
|
except Exception:
|
|
if processor_was_running:
|
|
self.stop()
|
|
else:
|
|
self.stop_all()
|
|
raise
|
|
|
|
def stop(self) -> None:
|
|
"""Stop acquisition-side processes, keep processor process intact."""
|
|
self._stop_processes(["sweep_orchestrator", "data_preprocessor"])
|
|
|
|
def stop_orchestrator(self) -> None:
|
|
"""Stop orchestrator process only."""
|
|
self._stop_processes(["sweep_orchestrator"])
|
|
|
|
def stop_preprocessor(self) -> None:
|
|
"""Stop preprocessor process only."""
|
|
self._stop_processes(["data_preprocessor"])
|
|
|
|
def stop_all(self) -> None:
|
|
"""Stop all managed processes."""
|
|
self._stop_processes(["sweep_orchestrator", "data_preprocessor", "data_processor"])
|
|
|
|
def _spawn(self, name: str, command: list[str], *, allow_clean_exit: bool) -> None:
|
|
"""Spawn one process unless same process is already alive.
|
|
|
|
Child stdout/stderr are redirected to per-process log files rather than
|
|
captured pipes: a long-running child (e.g. an acquisition producer waiting
|
|
for its device) would otherwise fill the OS pipe buffer once nobody drains
|
|
it and block on write. Files never back-pressure the child, and they keep
|
|
a persistent log we can read for exit reports and tail for diagnostics.
|
|
"""
|
|
existing = self._processes.get(name)
|
|
if existing is not None and existing.handle.poll() is None:
|
|
return
|
|
|
|
logs_dir = self._project_root / "python_app/runtime/logs"
|
|
logs_dir.mkdir(parents=True, exist_ok=True)
|
|
stdout_path = logs_dir / f"{name}.out.log"
|
|
stderr_path = logs_dir / f"{name}.err.log"
|
|
|
|
# Roll any stale (uncollected) log to `.prev` before truncating so the
|
|
# previous run's diagnostics survive a respawn (Fix #29).
|
|
self._roll_log_to_prev(stdout_path)
|
|
self._roll_log_to_prev(stderr_path)
|
|
|
|
stdout_file = open(stdout_path, "wb")
|
|
stderr_file = open(stderr_path, "wb")
|
|
try:
|
|
handle = subprocess.Popen(
|
|
command,
|
|
cwd=self._project_root,
|
|
stdout=stdout_file,
|
|
stderr=stderr_file,
|
|
# Own session/process group so signalling the group on stop also
|
|
# reaches device-I/O grandchildren the producer may have spawned
|
|
# (Fix #33).
|
|
start_new_session=True,
|
|
)
|
|
except OSError as exc:
|
|
stdout_file.close()
|
|
stderr_file.close()
|
|
command_text = shlex.join(command)
|
|
raise RuntimeError(
|
|
f"Failed to spawn {name} with command `{command_text}` from `{self._project_root}`: "
|
|
f"{type(exc).__name__}: {exc}"
|
|
) from exc
|
|
finally:
|
|
# The child holds its own dup'd fds; the parent's copies are not needed.
|
|
stdout_file.close()
|
|
stderr_file.close()
|
|
|
|
self._processes[name] = ManagedProcess(
|
|
name=name,
|
|
command=command,
|
|
allow_clean_exit=allow_clean_exit,
|
|
handle=handle,
|
|
stdout_path=stdout_path,
|
|
stderr_path=stderr_path,
|
|
)
|
|
self._write_pidfile()
|
|
|
|
def _acquisition_command(self, config_path: Path) -> list[str]:
|
|
"""Return acquisition producer command selected by radar.model."""
|
|
radar_model = self._read_radar_model(config_path)
|
|
if radar_model in {"librevna_multi", "sn9000"}:
|
|
return [
|
|
sys.executable,
|
|
"-m",
|
|
"python_app.scripts.matrix_raw_producer",
|
|
"--config",
|
|
str(config_path),
|
|
]
|
|
if radar_model == "kamil_adc":
|
|
return [
|
|
sys.executable,
|
|
"-m",
|
|
"python_app.scripts.kamil_adc_raw_producer",
|
|
"--config",
|
|
str(config_path),
|
|
]
|
|
return [
|
|
str(self._project_root / "build/bin/sweep_orchestrator"),
|
|
"--config",
|
|
str(config_path),
|
|
]
|
|
|
|
@staticmethod
|
|
def _read_radar_model(config_path: Path) -> str:
|
|
"""Read `radar.model` cheaply without constructing the full config model."""
|
|
payload = json.loads(config_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
return "librevna"
|
|
radar_payload = payload.get("radar")
|
|
if not isinstance(radar_payload, dict):
|
|
return "librevna"
|
|
return str(radar_payload.get("model", "librevna"))
|
|
|
|
def _stop_processes(self, names: Iterable[str]) -> None:
|
|
"""Gracefully terminate processes, then force-kill on per-process timeout."""
|
|
ordered_names = list(names)
|
|
for name in ordered_names:
|
|
process = self._processes.get(name)
|
|
if process is None:
|
|
continue
|
|
if process.handle.poll() is None:
|
|
# Signal the whole group so device-I/O grandchildren die too (Fix #33).
|
|
self._signal_group(process.handle.pid, signal.SIGTERM)
|
|
|
|
for name in ordered_names:
|
|
process = self._processes.get(name)
|
|
if process is None:
|
|
continue
|
|
if process.handle.poll() is not None:
|
|
self._log_abnormal_stop_exit(process)
|
|
continue
|
|
|
|
# Each process gets its own kill deadline so a slow shutdown of one
|
|
# cannot consume the grace window of the others (Fix #33).
|
|
try:
|
|
process.handle.wait(timeout=_STOP_GRACE_SECONDS)
|
|
except subprocess.TimeoutExpired:
|
|
self._signal_group(process.handle.pid, signal.SIGKILL)
|
|
try:
|
|
process.handle.wait(timeout=1.0)
|
|
except subprocess.TimeoutExpired:
|
|
pass
|
|
else:
|
|
# A negative code here is the SIGTERM we just sent (expected); only
|
|
# a positive self-exit during the grace window is worth noting.
|
|
self._log_abnormal_stop_exit(process)
|
|
|
|
self._drop_exited()
|
|
|
|
@staticmethod
|
|
def _signal_group(pid: int, sig: int) -> None:
|
|
"""Signal the child's whole process group, falling back to the child."""
|
|
if pid is None:
|
|
return
|
|
try:
|
|
os.killpg(os.getpgid(pid), sig)
|
|
except (ProcessLookupError, PermissionError):
|
|
# Group already gone, or could not resolve it; fall back to the child.
|
|
try:
|
|
os.kill(pid, sig)
|
|
except (ProcessLookupError, PermissionError):
|
|
pass
|
|
|
|
@staticmethod
|
|
def _log_abnormal_stop_exit(process: ManagedProcess) -> None:
|
|
"""Note a process that self-exited abnormally around stop time (Fix #33).
|
|
|
|
Negative codes are signal-induced (e.g. the SIGTERM we send on stop) and
|
|
are expected; only a non-zero self-exit is reported.
|
|
"""
|
|
return_code = process.handle.poll()
|
|
if return_code is None or return_code <= 0:
|
|
return
|
|
print(
|
|
f"process_supervisor: `{process.name}` exited abnormally with code "
|
|
f"{return_code} around stop",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
def _drop_exited(self) -> None:
|
|
"""Remove exited process entries from internal map."""
|
|
exited_names = [name for name, process in self._processes.items() if process.handle.poll() is not None]
|
|
for name in exited_names:
|
|
self._processes.pop(name, None)
|
|
if exited_names:
|
|
self._write_pidfile()
|
|
|
|
def _is_alive(self, name: str) -> bool:
|
|
"""Return `True` when named process handle exists and is running."""
|
|
process = self._processes.get(name)
|
|
if process is None:
|
|
return False
|
|
return process.handle.poll() is None
|
|
|
|
def collect_exit_reports(self) -> list[ProcessExitReport]:
|
|
"""Collect reports for managed processes that have exited."""
|
|
reports: list[ProcessExitReport] = []
|
|
exited_names: list[str] = []
|
|
|
|
for name, process in self._processes.items():
|
|
return_code = process.handle.poll()
|
|
if return_code is None:
|
|
# Still running: enforce the size cap so logs never grow unbounded (Fix #16).
|
|
self._roll_log_if_oversized(process.stdout_path)
|
|
self._roll_log_if_oversized(process.stderr_path)
|
|
continue
|
|
|
|
reports.append(
|
|
ProcessExitReport(
|
|
name=process.name,
|
|
command=list(process.command),
|
|
working_directory=self._project_root,
|
|
return_code=int(return_code),
|
|
stdout_path=process.stdout_path,
|
|
stderr_path=process.stderr_path,
|
|
expected_clean_exit=bool(process.allow_clean_exit and int(return_code) == 0),
|
|
)
|
|
)
|
|
exited_names.append(name)
|
|
|
|
for name in exited_names:
|
|
self._processes.pop(name, None)
|
|
if exited_names:
|
|
self._write_pidfile()
|
|
return reports
|
|
|
|
def _wait_until_ready(self, required_processes: Sequence[str]) -> None:
|
|
"""Wait until all required processes are alive or timeout/crash occurs."""
|
|
deadline = time.monotonic() + self._readiness_timeout_s
|
|
|
|
while time.monotonic() < deadline:
|
|
exit_reports = self.collect_exit_reports()
|
|
unexpected_reports = [report for report in exit_reports if not report.expected_clean_exit]
|
|
if unexpected_reports:
|
|
raise RuntimeError("; ".join(report.format() for report in unexpected_reports))
|
|
if all(self._is_alive(process_name) for process_name in required_processes):
|
|
return
|
|
time.sleep(0.05)
|
|
|
|
names = ", ".join(required_processes)
|
|
raise RuntimeError(
|
|
f"Timed out waiting for processes to start: {names}. "
|
|
f"Alive processes: {self.pids() or 'none'}"
|
|
)
|
|
|
|
def pids(self) -> dict[str, int]:
|
|
"""Return PID mapping for currently alive managed processes."""
|
|
return {
|
|
process.name: process.handle.pid
|
|
for process in self._processes.values()
|
|
if process.handle.poll() is None and process.handle.pid is not None
|
|
}
|
|
|
|
@staticmethod
|
|
def _roll_log_to_prev(path: Path) -> None:
|
|
"""Roll an existing log to `{path}.prev` before it is reopened (Fix #29).
|
|
|
|
Preserves a stale (exited, not-yet-reported) child's last output instead
|
|
of truncating it when a fresh log is opened for a respawn.
|
|
"""
|
|
if not path.exists():
|
|
return
|
|
try:
|
|
path.replace(path.with_suffix(path.suffix + ".prev"))
|
|
except OSError:
|
|
# Best-effort: a failed roll must not block a spawn.
|
|
pass
|
|
|
|
@staticmethod
|
|
def _roll_log_if_oversized(path: Path) -> None:
|
|
"""Bound a live child log to `_LOG_MAX_BYTES` so it cannot fill the SD card (Fix #16).
|
|
|
|
The child holds an open fd to this inode, so a rename would not redirect
|
|
its writes. Instead keep one rolled generation via copy-to-`.prev` and
|
|
truncate the live inode in place, freeing the allocated disk blocks.
|
|
"""
|
|
try:
|
|
if path.stat().st_size <= _LOG_MAX_BYTES:
|
|
return
|
|
except OSError:
|
|
return
|
|
prev_path = path.with_suffix(path.suffix + ".prev")
|
|
try:
|
|
# Preserve the trailing window as the rolled generation, then truncate.
|
|
tail = _read_log_tail(path, _LOG_MAX_BYTES)
|
|
prev_path.write_text(tail, encoding="utf-8")
|
|
with open(path, "r+b") as handle:
|
|
handle.truncate(0)
|
|
except OSError:
|
|
# Best-effort: capping is opportunistic and must not disrupt polling.
|
|
pass
|
|
|
|
def _write_pidfile(self) -> None:
|
|
"""Persist live child PIDs so a later supervisor can reap them (Fix #19)."""
|
|
try:
|
|
self._runtime_dir.mkdir(parents=True, exist_ok=True)
|
|
live_pids = [
|
|
str(process.handle.pid)
|
|
for process in self._processes.values()
|
|
if process.handle.poll() is None and process.handle.pid is not None
|
|
]
|
|
self._pidfile_path.write_text("\n".join(live_pids), encoding="utf-8")
|
|
except OSError:
|
|
# Best-effort bookkeeping: failure here must not break start/stop.
|
|
pass
|
|
|
|
def _reap_stale_children(self) -> None:
|
|
"""Kill pipeline children recorded by a prior supervisor instance (Fix #19).
|
|
|
|
On a clean shutdown the pidfile is emptied; entries only remain when the
|
|
previous supervisor died without stopping its children. We SIGKILL each
|
|
stale process group so leftover pipeline binaries cannot hold the shared
|
|
memory rings or devices hostage on the next start.
|
|
"""
|
|
try:
|
|
raw = self._pidfile_path.read_text(encoding="utf-8")
|
|
except OSError:
|
|
return
|
|
for token in raw.split():
|
|
try:
|
|
pid = int(token)
|
|
except ValueError:
|
|
continue
|
|
if pid <= 1 or pid == os.getpid():
|
|
continue
|
|
# Guard against PID reuse: only reap if the process still looks like
|
|
# one of our pipeline children before signalling its group.
|
|
if self._is_stale_pipeline_pid(pid):
|
|
self._signal_group(pid, signal.SIGKILL)
|
|
try:
|
|
self._pidfile_path.write_text("", encoding="utf-8")
|
|
except OSError:
|
|
pass
|
|
|
|
def _is_stale_pipeline_pid(self, pid: int) -> bool:
|
|
"""Return whether `pid` still runs one of our pipeline binaries/scripts.
|
|
|
|
Reads `/proc/<pid>/cmdline` so a recycled PID owned by an unrelated
|
|
process is never killed (Fix #19 safety guard).
|
|
"""
|
|
markers = (
|
|
"build/bin/data_processor",
|
|
"build/bin/data_preprocessor",
|
|
"build/bin/sweep_orchestrator",
|
|
"python_app.scripts.matrix_raw_producer",
|
|
"python_app.scripts.kamil_adc_raw_producer",
|
|
)
|
|
try:
|
|
raw = (Path("/proc") / str(pid) / "cmdline").read_bytes()
|
|
except OSError:
|
|
return False
|
|
cmdline = raw.replace(b"\x00", b" ").decode("utf-8", errors="replace")
|
|
return any(marker in cmdline for marker in markers)
|