awe/radar_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improved logging

Browse Source
This commit is contained in:
Ayzen
2026-06-06 00:52:52 +03:00
parent af6005d68f
commit aea49f6128
65 changed files with 1206 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python_app/orchestration/process_supervisor.py
+46 -21
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
from dataclasses import dataclass
import json
import logging
import os
from pathlib import Path
import shlex
@@ -14,11 +15,13 @@ import time
from typing import Iterable
from typing import Sequence
logger = logging.getLogger(__name__)
# Cap each child log so a long-lived daemon cannot fill the SD card. On reaching
# the cap the current log is rolled to `{name}.{out,err}.log.prev` and a fresh
# log opened (see `_roll_log_if_oversized`).
_LOG_MAX_BYTES = 8 * 1024 * 1024
# Per-process force-kill deadline used on stop (Fix #33: own deadline each).
# Per-process force-kill deadline used on stop; each child gets its own window.
_STOP_GRACE_SECONDS = 2.0
@@ -40,7 +43,7 @@ class ProcessExitReport:
Log tails are not held in memory: they are read on demand from the child log
files only while rendering an ERROR report, so the common clean-exit path on
every poll never pays for a 16KB read of two files (Fix #46).
every poll never pays for a 16KB read of two files.
"""
name: str
@@ -110,9 +113,14 @@ class ProcessSupervisor:
self._readiness_timeout_s = readiness_timeout_s
self._processes: dict[str, ManagedProcess] = {}
# Runtime pidfile lets us reap pipeline children left behind by a prior
# supervisor (crash/SIGKILL) independent of in-memory state (Fix #19).
# supervisor (crash/SIGKILL) independent of in-memory state.
self._runtime_dir = self._project_root / "python_app/runtime"
self._pidfile_path = self._runtime_dir / "supervisor_children.pids"
logger.debug(
"ProcessSupervisor init: root=%s readiness_timeout=%.1fs",
self._project_root,
self._readiness_timeout_s,
)
self._reap_stale_children()
def is_running(self) -> bool:
@@ -128,6 +136,11 @@ class ProcessSupervisor:
if self.is_running():
raise RuntimeError("Acquisition processes are already running")
logger.info(
"Starting pipeline from config %s (allow_clean_orchestrator_exit=%s)",
config_path,
allow_clean_orchestrator_exit,
)
acquisition_command = self._acquisition_command(config_path)
command_specs = {
"data_processor": [
@@ -161,12 +174,15 @@ class ProcessSupervisor:
)
self._wait_until_ready(required_processes)
except Exception:
logger.exception("Pipeline start failed; tearing down spawned processes")
if processor_was_running:
self.stop()
else:
self.stop_all()
raise
logger.info("Pipeline started; live pids=%s", self.pids() or "none")
def stop(self) -> None:
"""Stop acquisition-side processes, keep processor process intact."""
self._stop_processes(["sweep_orchestrator", "data_preprocessor"])
@@ -194,6 +210,7 @@ class ProcessSupervisor:
"""
existing = self._processes.get(name)
if existing is not None and existing.handle.poll() is None:
logger.debug("Spawn skipped: `%s` already running (pid=%s)", name, existing.handle.pid)
return
logs_dir = self._project_root / "python_app/runtime/logs"
@@ -202,7 +219,7 @@ class ProcessSupervisor:
stderr_path = logs_dir / f"{name}.err.log"
# Roll any stale (uncollected) log to `.prev` before truncating so the
# previous run's diagnostics survive a respawn (Fix #29).
# previous run's diagnostics survive a respawn.
self._roll_log_to_prev(stdout_path)
self._roll_log_to_prev(stderr_path)
@@ -215,14 +232,14 @@ class ProcessSupervisor:
stdout=stdout_file,
stderr=stderr_file,
# Own session/process group so signalling the group on stop also
# reaches device-I/O grandchildren the producer may have spawned
# (Fix #33).
# reaches device-I/O grandchildren the producer may have spawned.
start_new_session=True,
)
except OSError as exc:
stdout_file.close()
stderr_file.close()
command_text = shlex.join(command)
logger.error("Failed to spawn `%s`: %s: %s", name, type(exc).__name__, exc)
raise RuntimeError(
f"Failed to spawn {name} with command `{command_text}` from `{self._project_root}`: "
f"{type(exc).__name__}: {exc}"
@@ -240,11 +257,13 @@ class ProcessSupervisor:
stdout_path=stdout_path,
stderr_path=stderr_path,
)
logger.info("Spawned `%s` (pid=%d)", name, handle.pid)
self._write_pidfile()
def _acquisition_command(self, config_path: Path) -> list[str]:
"""Return acquisition producer command selected by radar.model."""
radar_model = self._read_radar_model(config_path)
logger.debug("Selecting acquisition producer for radar.model=%s", radar_model)
if radar_model in {"librevna_multi", "sn9000"}:
return [
sys.executable,
@@ -286,7 +305,8 @@ class ProcessSupervisor:
if process is None:
continue
if process.handle.poll() is None:
# Signal the whole group so device-I/O grandchildren die too (Fix #33).
logger.info("Stopping `%s` (pid=%d): sending SIGTERM to group", name, process.handle.pid)
# Signal the whole group so device-I/O grandchildren die too.
self._signal_group(process.handle.pid, signal.SIGTERM)
for name in ordered_names:
@@ -298,15 +318,20 @@ class ProcessSupervisor:
continue
# Each process gets its own kill deadline so a slow shutdown of one
# cannot consume the grace window of the others (Fix #33).
# cannot consume the grace window of the others.
try:
process.handle.wait(timeout=_STOP_GRACE_SECONDS)
except subprocess.TimeoutExpired:
logger.warning(
"`%s` did not exit within %.1fs of SIGTERM; sending SIGKILL",
name,
_STOP_GRACE_SECONDS,
)
self._signal_group(process.handle.pid, signal.SIGKILL)
try:
process.handle.wait(timeout=1.0)
except subprocess.TimeoutExpired:
pass
logger.error("`%s` still alive after SIGKILL", name)
else:
# A negative code here is the SIGTERM we just sent (expected); only
# a positive self-exit during the grace window is worth noting.
@@ -330,7 +355,7 @@ class ProcessSupervisor:
@staticmethod
def _log_abnormal_stop_exit(process: ManagedProcess) -> None:
"""Note a process that self-exited abnormally around stop time (Fix #33).
"""Log a process that self-exited abnormally around stop time.
Negative codes are signal-induced (e.g. the SIGTERM we send on stop) and
are expected; only a non-zero self-exit is reported.
@@ -338,11 +363,7 @@ class ProcessSupervisor:
return_code = process.handle.poll()
if return_code is None or return_code <= 0:
return
print(
f"process_supervisor: `{process.name}` exited abnormally with code "
f"{return_code} around stop",
file=sys.stderr,
)
logger.warning("`%s` exited abnormally with code %d around stop", process.name, return_code)
def _drop_exited(self) -> None:
"""Remove exited process entries from internal map."""
@@ -367,11 +388,12 @@ class ProcessSupervisor:
for name, process in self._processes.items():
return_code = process.handle.poll()
if return_code is None:
# Still running: enforce the size cap so logs never grow unbounded (Fix #16).
# Still running: enforce the size cap so logs never grow unbounded.
self._roll_log_if_oversized(process.stdout_path)
self._roll_log_if_oversized(process.stderr_path)
continue
logger.debug("Reaped `%s` with exit code %d", process.name, int(return_code))
reports.append(
ProcessExitReport(
name=process.name,
@@ -394,6 +416,7 @@ class ProcessSupervisor:
def _wait_until_ready(self, required_processes: Sequence[str]) -> None:
"""Wait until all required processes are alive or timeout/crash occurs."""
deadline = time.monotonic() + self._readiness_timeout_s
logger.debug("Waiting for processes to become ready: %s", ", ".join(required_processes))
while time.monotonic() < deadline:
exit_reports = self.collect_exit_reports()
@@ -401,6 +424,7 @@ class ProcessSupervisor:
if unexpected_reports:
raise RuntimeError("; ".join(report.format() for report in unexpected_reports))
if all(self._is_alive(process_name) for process_name in required_processes):
logger.debug("All required processes ready")
return
time.sleep(0.05)
@@ -420,7 +444,7 @@ class ProcessSupervisor:
@staticmethod
def _roll_log_to_prev(path: Path) -> None:
"""Roll an existing log to `{path}.prev` before it is reopened (Fix #29).
"""Roll an existing log to `{path}.prev` before it is reopened.
Preserves a stale (exited, not-yet-reported) child's last output instead
of truncating it when a fresh log is opened for a respawn.
@@ -435,7 +459,7 @@ class ProcessSupervisor:
@staticmethod
def _roll_log_if_oversized(path: Path) -> None:
"""Bound a live child log to `_LOG_MAX_BYTES` so it cannot fill the SD card (Fix #16).
"""Bound a live child log to `_LOG_MAX_BYTES` so it cannot fill the SD card.
The child holds an open fd to this inode, so a rename would not redirect
its writes. Instead keep one rolled generation via copy-to-`.prev` and
@@ -458,7 +482,7 @@ class ProcessSupervisor:
pass
def _write_pidfile(self) -> None:
"""Persist live child PIDs so a later supervisor can reap them (Fix #19)."""
"""Persist live child PIDs so a later supervisor can reap them."""
try:
self._runtime_dir.mkdir(parents=True, exist_ok=True)
live_pids = [
@@ -472,7 +496,7 @@ class ProcessSupervisor:
pass
def _reap_stale_children(self) -> None:
"""Kill pipeline children recorded by a prior supervisor instance (Fix #19).
"""Kill pipeline children recorded by a prior supervisor instance.
On a clean shutdown the pidfile is emptied; entries only remain when the
previous supervisor died without stopping its children. We SIGKILL each
@@ -493,6 +517,7 @@ class ProcessSupervisor:
# Guard against PID reuse: only reap if the process still looks like
# one of our pipeline children before signalling its group.
if self._is_stale_pipeline_pid(pid):
logger.warning("Reaping stale pipeline child from prior run (pid=%d)", pid)
self._signal_group(pid, signal.SIGKILL)
try:
self._pidfile_path.write_text("", encoding="utf-8")
@@ -503,7 +528,7 @@ class ProcessSupervisor:
"""Return whether `pid` still runs one of our pipeline binaries/scripts.
Reads `/proc/<pid>/cmdline` so a recycled PID owned by an unrelated
process is never killed (Fix #19 safety guard).
process is never killed.
"""
markers = (
"build/bin/data_processor",