init commit

python_app/orchestration/__init__.py

@@ -0,0 +1 @@
+"""Runtime orchestration utilities for process control and IPC."""

python_app/orchestration/config_writer.py

@@ -0,0 +1,42 @@
+"""Helpers for writing runtime configs and preprocessing bundles."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from python_app.models.run_config_model import RunConfigModel, parse_combos_from_text
+from python_app.storage.npz_store import NpzStore
+
+
+class ConfigWriter:
+    """Write runtime artifacts consumed by C++ processes."""
+
+    def __init__(self, runtime_dir: Path) -> None:
+        """Create writer rooted at runtime directory."""
+        self._runtime_dir = runtime_dir
+        self._runtime_dir.mkdir(parents=True, exist_ok=True)
+
+    def prepare_bundles(
+        self,
+        store: NpzStore,
+        radar_key: str,
+        calibration_set: str,
+        reference_set: str,
+    ) -> tuple[Path, Path]:
+        """Export calibration/reference sets into binary bundles for preprocessor."""
+        calibration_bundle = self._runtime_dir / "calibration_bundle.bin"
+        reference_bundle = self._runtime_dir / "reference_bundle.bin"
+
+        store.export_set_bundle("calibration", radar_key, calibration_set, calibration_bundle)
+        store.export_set_bundle("reference", radar_key, reference_set, reference_bundle)
+        return calibration_bundle, reference_bundle
+
+    def write(self, config: RunConfigModel, output_path: Path) -> Path:
+        """Write run configuration JSON file."""
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        output_path.write_text(json.dumps(config.to_dict(), indent=2), encoding="utf-8")
+        return output_path
+
+
+__all__ = ["ConfigWriter", "parse_combos_from_text"]

python_app/orchestration/live_processing_config.py

@@ -0,0 +1,61 @@
+"""Live processing settings model and atomic JSON writer."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import json
+from pathlib import Path
+
+
+@dataclass(slots=True)
+class ProcessingLiveConfig:
+    """Runtime-adjustable processing parameters shared with data processor."""
+
+    processor_mode: str = "pass_through"
+    gain_db: float = 0.0
+    phase_deg: float = 0.0
+    bscan_axis: str = "abs"
+    bscan_cut_m: float = 0.824
+    bscan_max_depth_m: float = 1.0
+    bscan_gain: float = 1.0
+    bscan_start_freq_mhz: float = 100.0
+    bscan_stop_freq_mhz: float = 8800.0
+    history_command_seq: int = 0
+    history_command: str = "none"
+
+    def to_dict(self) -> dict[str, float | str | int]:
+        """Convert live config to JSON-serializable dictionary."""
+        return {
+            "processor_mode": str(self.processor_mode),
+            "gain_db": float(self.gain_db),
+            "phase_deg": float(self.phase_deg),
+            "bscan_axis": str(self.bscan_axis),
+            "bscan_cut_m": float(self.bscan_cut_m),
+            "bscan_max_depth_m": float(self.bscan_max_depth_m),
+            "bscan_gain": float(self.bscan_gain),
+            "bscan_start_freq_mhz": float(self.bscan_start_freq_mhz),
+            "bscan_stop_freq_mhz": float(self.bscan_stop_freq_mhz),
+            "history_command_seq": int(self.history_command_seq),
+            "history_command": str(self.history_command),
+        }
+
+
+class ProcessingLiveConfigWriter:
+    """Atomic writer for processing live-config file."""
+
+    def __init__(self, config_path: Path) -> None:
+        """Create writer targeting `config_path`."""
+        self._config_path = config_path
+        self._config_path.parent.mkdir(parents=True, exist_ok=True)
+
+    @property
+    def path(self) -> Path:
+        """Return destination config path."""
+        return self._config_path
+
+    def write(self, config: ProcessingLiveConfig) -> Path:
+        """Atomically write config by temp-file replace."""
+        temp_path = self._config_path.with_suffix(self._config_path.suffix + ".tmp")
+        temp_path.write_text(json.dumps(config.to_dict(), indent=2), encoding="utf-8")
+        temp_path.replace(self._config_path)
+        return self._config_path

python_app/orchestration/process_supervisor.py

@@ -0,0 +1,202 @@
+"""Process supervisor for lifecycle management of C++ pipeline binaries."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+import subprocess
+import time
+from typing import Iterable
+from typing import Sequence
+
+
+@dataclass(slots=True)
+class ManagedProcess:
+    """Metadata and subprocess handle for one managed child process."""
+
+    name: str
+    command: list[str]
+    handle: subprocess.Popen[str]
+
+
+class ProcessSupervisor:
+    """Start, monitor, and stop pipeline subprocesses."""
+
+    def __init__(self, project_root: Path, readiness_timeout_s: float = 3.0) -> None:
+        """Create supervisor bound to repository root directory."""
+        self._project_root = project_root
+        self._readiness_timeout_s = readiness_timeout_s
+        self._processes: dict[str, ManagedProcess] = {}
+
+    def is_running(self) -> bool:
+        """Return whether acquisition-side processes are alive."""
+        return self._is_alive("data_preprocessor") or self._is_alive("sweep_orchestrator")
+
+    def is_processor_running(self) -> bool:
+        """Return whether data processor process is alive."""
+        return self._is_alive("data_processor")
+
+    def start(self, config_path: Path) -> None:
+        """Start required pipeline binaries and wait until they are ready."""
+        if self.is_running():
+            raise RuntimeError("Acquisition processes are already running")
+
+        command_specs = {
+            "data_processor": [
+                str(self._project_root / "build/bin/data_processor"),
+                "--config",
+                str(config_path),
+            ],
+            "data_preprocessor": [
+                str(self._project_root / "build/bin/data_preprocessor"),
+                "--config",
+                str(config_path),
+            ],
+            "sweep_orchestrator": [
+                str(self._project_root / "build/bin/sweep_orchestrator"),
+                "--config",
+                str(config_path),
+            ],
+        }
+        processor_was_running = self.is_processor_running()
+        required_processes: list[str] = ["data_preprocessor", "sweep_orchestrator"]
+        if not processor_was_running:
+            required_processes.insert(0, "data_processor")
+
+        try:
+            if not processor_was_running:
+                self._spawn("data_processor", command_specs["data_processor"])
+
+            self._spawn("data_preprocessor", command_specs["data_preprocessor"])
+            self._spawn("sweep_orchestrator", command_specs["sweep_orchestrator"])
+            self._wait_until_ready(required_processes)
+        except Exception:
+            if processor_was_running:
+                self.stop()
+            else:
+                self.stop_all()
+            raise
+
+    def stop(self) -> None:
+        """Stop acquisition-side processes, keep processor process intact."""
+        self._stop_processes(["sweep_orchestrator", "data_preprocessor"])
+
+    def stop_orchestrator(self) -> None:
+        """Stop orchestrator process only."""
+        self._stop_processes(["sweep_orchestrator"])
+
+    def stop_preprocessor(self) -> None:
+        """Stop preprocessor process only."""
+        self._stop_processes(["data_preprocessor"])
+
+    def stop_all(self) -> None:
+        """Stop all managed processes."""
+        self._stop_processes(["sweep_orchestrator", "data_preprocessor", "data_processor"])
+
+    def _spawn(self, name: str, command: list[str]) -> None:
+        """Spawn one process unless same process is already alive."""
+        existing = self._processes.get(name)
+        if existing is not None and existing.handle.poll() is None:
+            return
+
+        handle = subprocess.Popen(
+            command,
+            cwd=self._project_root,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+        self._processes[name] = ManagedProcess(name=name, command=command, handle=handle)
+
+    def _stop_processes(self, names: Iterable[str]) -> None:
+        """Gracefully terminate processes, then force-kill on timeout."""
+        ordered_names = list(names)
+        for name in ordered_names:
+            process = self._processes.get(name)
+            if process is None:
+                continue
+            if process.handle.poll() is None:
+                process.handle.terminate()
+
+        deadline = time.monotonic() + 2.0
+        for name in ordered_names:
+            process = self._processes.get(name)
+            if process is None:
+                continue
+            if process.handle.poll() is not None:
+                continue
+
+            timeout = max(0.0, deadline - time.monotonic())
+            try:
+                process.handle.wait(timeout=timeout)
+            except subprocess.TimeoutExpired:
+                process.handle.kill()
+                process.handle.wait(timeout=1.0)
+
+        self._drop_exited()
+
+    def _drop_exited(self) -> None:
+        """Remove exited process entries from internal map."""
+        exited_names = [name for name, process in self._processes.items() if process.handle.poll() is not None]
+        for name in exited_names:
+            self._processes.pop(name, None)
+
+    def _is_alive(self, name: str) -> bool:
+        """Return `True` when named process handle exists and is running."""
+        process = self._processes.get(name)
+        if process is None:
+            return False
+        return process.handle.poll() is None
+
+    def collect_crash_reports(self) -> list[str]:
+        """Collect stderr/stdout reports from processes that have exited."""
+        reports: list[str] = []
+        exited_names: list[str] = []
+
+        for name, process in self._processes.items():
+            return_code = process.handle.poll()
+            if return_code is None:
+                continue
+
+            stderr = ""
+            stdout = ""
+            if process.handle.stdout is not None:
+                stdout = process.handle.stdout.read().strip()
+            if process.handle.stderr is not None:
+                stderr = process.handle.stderr.read().strip()
+
+            details = stderr
+            if stdout and stderr:
+                details = f"{stderr}\nstdout:\n{stdout}"
+            elif stdout:
+                details = f"stdout:\n{stdout}"
+
+            reports.append(f"{process.name} exited with code {return_code}: {details}")
+            exited_names.append(name)
+
+        for name in exited_names:
+            self._processes.pop(name, None)
+        return reports
+
+    def _wait_until_ready(self, required_processes: Sequence[str]) -> None:
+        """Wait until all required processes are alive or timeout/crash occurs."""
+        deadline = time.monotonic() + self._readiness_timeout_s
+
+        while time.monotonic() < deadline:
+            crashed = self.collect_crash_reports()
+            if crashed:
+                raise RuntimeError("; ".join(crashed))
+            if all(self._is_alive(process_name) for process_name in required_processes):
+                return
+            time.sleep(0.05)
+
+        names = ", ".join(required_processes)
+        raise RuntimeError(f"Timed out waiting for processes to start: {names}")
+
+    def pids(self) -> dict[str, int]:
+        """Return PID mapping for currently alive managed processes."""
+        return {
+            process.name: process.handle.pid
+            for process in self._processes.values()
+            if process.handle.poll() is None and process.handle.pid is not None
+        }

python_app/orchestration/shm/__init__.py

@@ -0,0 +1,21 @@
+"""Shared-memory ring readers and payload decoders."""
+
+from python_app.orchestration.shm.binary_cursor import ByteCursor
+from python_app.orchestration.shm.decoder import (
+    PREPROC_MAGIC,
+    RAW_MAGIC,
+    RESULT_MAGIC,
+    decode_result_collection,
+    decode_trace_collection,
+)
+from python_app.orchestration.shm.ring_reader import ShmRingReader
+
+__all__ = [
+    "ByteCursor",
+    "PREPROC_MAGIC",
+    "RAW_MAGIC",
+    "RESULT_MAGIC",
+    "ShmRingReader",
+    "decode_result_collection",
+    "decode_trace_collection",
+]

python_app/orchestration/shm/binary_cursor.py

@@ -0,0 +1,50 @@
+"""Byte-wise cursor utilities for decoding binary ring payloads."""
+
+from __future__ import annotations
+
+import struct
+
+
+class ByteCursor:
+    """Read primitive values from bytes while tracking offset."""
+
+    def __init__(self, payload: bytes) -> None:
+        """Create cursor at start of payload."""
+        self.payload = payload
+        self.offset = 0
+
+    def read_u8(self) -> int:
+        """Read unsigned 8-bit integer."""
+        value = struct.unpack_from("<B", self.payload, self.offset)[0]
+        self.offset += 1
+        return value
+
+    def read_u16(self) -> int:
+        """Read unsigned 16-bit integer."""
+        value = struct.unpack_from("<H", self.payload, self.offset)[0]
+        self.offset += 2
+        return value
+
+    def read_u32(self) -> int:
+        """Read unsigned 32-bit integer."""
+        value = struct.unpack_from("<I", self.payload, self.offset)[0]
+        self.offset += 4
+        return value
+
+    def read_u64(self) -> int:
+        """Read unsigned 64-bit integer."""
+        value = struct.unpack_from("<Q", self.payload, self.offset)[0]
+        self.offset += 8
+        return value
+
+    def read_f32(self) -> float:
+        """Read 32-bit float."""
+        value = struct.unpack_from("<f", self.payload, self.offset)[0]
+        self.offset += 4
+        return float(value)
+
+    def read_bytes(self, size: int) -> bytes:
+        """Read raw byte slice of fixed size."""
+        data = self.payload[self.offset : self.offset + size]
+        self.offset += size
+        return data

python_app/orchestration/shm/decoder.py

@@ -0,0 +1,114 @@
+"""Binary decoders for raw/preprocessed/result payload collections."""
+
+from __future__ import annotations
+
+import numpy as np
+
+from python_app.models.dataset_model import (
+    ComboKey,
+    ResultBlock,
+    ResultCollection,
+    ResultPayload,
+    SweepCollection,
+    TraceData,
+)
+from python_app.orchestration.shm.binary_cursor import ByteCursor
+
+RAW_MAGIC = 0x31574152
+PREPROC_MAGIC = 0x31525050
+RESULT_MAGIC = 0x314C5352
+
+
+def decode_trace_collection(payload: bytes, expected_magic: int) -> SweepCollection:
+    """Decode one raw/preprocessed collection from binary payload."""
+    cursor = ByteCursor(payload)
+    magic = cursor.read_u32()
+    if magic != expected_magic:
+        raise ValueError("Unexpected trace collection magic")
+
+    collection_id = cursor.read_u64()
+    monotonic_ns = cursor.read_u64()
+    trace_count = cursor.read_u32()
+
+    traces: list[TraceData] = []
+    for _ in range(trace_count):
+        input_pos = cursor.read_u32()
+        output_pos = cursor.read_u32()
+        point_count = cursor.read_u32()
+
+        freq_bytes = point_count * 4
+        freq = np.frombuffer(cursor.read_bytes(freq_bytes), dtype="<f4").astype(np.float32, copy=False)
+
+        interleaved_bytes = point_count * 8
+        interleaved = np.frombuffer(cursor.read_bytes(interleaved_bytes), dtype="<f4")
+        s21 = (interleaved[0::2] + 1j * interleaved[1::2]).astype(np.complex64, copy=False)
+
+        traces.append(
+            TraceData(
+                combo=ComboKey(input_pos=input_pos, output_pos=output_pos),
+                frequency_hz=freq,
+                s21=s21,
+            )
+        )
+
+    return SweepCollection(collection_id=collection_id, monotonic_ns=monotonic_ns, traces=traces)
+
+
+def decode_result_collection(payload: bytes) -> ResultCollection:
+    """Decode one processed result collection from binary payload."""
+    cursor = ByteCursor(payload)
+    magic = cursor.read_u32()
+    if magic != RESULT_MAGIC:
+        raise ValueError("Unexpected result collection magic")
+
+    collection_id = cursor.read_u64()
+    monotonic_ns = cursor.read_u64()
+    block_count = cursor.read_u32()
+
+    blocks: list[ResultBlock] = []
+    for _ in range(block_count):
+        input_pos = cursor.read_u32()
+        output_pos = cursor.read_u32()
+        payload_count = cursor.read_u32()
+
+        payloads: list[ResultPayload] = []
+        for _ in range(payload_count):
+            kind = cursor.read_u8()
+            name_size = cursor.read_u16()
+            name = cursor.read_bytes(name_size).decode("utf-8")
+
+            if kind == 1:
+                point_count = cursor.read_u32()
+                freq = np.frombuffer(cursor.read_bytes(point_count * 4), dtype="<f4").astype(np.float32, copy=False)
+                interleaved = np.frombuffer(cursor.read_bytes(point_count * 8), dtype="<f4")
+                trace = (interleaved[0::2] + 1j * interleaved[1::2]).astype(np.complex64, copy=False)
+                payloads.append(
+                    ResultPayload(
+                        processing_name=name,
+                        kind=kind,
+                        frequency_hz=freq,
+                        trace=trace,
+                    )
+                )
+            elif kind == 2:
+                scalar_value = cursor.read_f32()
+                payloads.append(
+                    ResultPayload(
+                        processing_name=name,
+                        kind=kind,
+                        frequency_hz=np.array([], dtype=np.float32),
+                        trace=np.array([], dtype=np.complex64),
+                        scalar_value=scalar_value,
+                    )
+                )
+            else:
+                raise ValueError(f"Unsupported result payload kind: {kind}")
+
+        blocks.append(
+            ResultBlock(
+                combo=ComboKey(input_pos=input_pos, output_pos=output_pos),
+                payloads=payloads,
+            )
+        )
+
+    return ResultCollection(collection_id=collection_id, monotonic_ns=monotonic_ns, blocks=blocks)

python_app/orchestration/shm/ring_reader.py

@@ -0,0 +1,150 @@
+"""POSIX shared-memory ring reader implementation."""
+
+from __future__ import annotations
+
+import mmap
+from pathlib import Path
+import struct
+import time
+from typing import Final
+
+from python_app.models.dataset_model import ResultCollection, SweepCollection
+from python_app.orchestration.shm.decoder import (
+    PREPROC_MAGIC,
+    RAW_MAGIC,
+    decode_result_collection,
+    decode_trace_collection,
+)
+
+_HEADER_SIZE: Final[int] = 64
+_SLOT_HEADER_SIZE: Final[int] = 16
+_MAGIC: Final[bytes] = b"RDRRING2"
+_VERSION: Final[int] = 1
+
+
+class ShmRingReader:
+    """Read binary payloads from a lock-free ring in `/dev/shm`."""
+
+    def __init__(self, ring_name: str, open_timeout_s: float = 2.0, open_poll_s: float = 0.01) -> None:
+        """Open and validate ring by name, for example `/radar_results`."""
+        if not ring_name.startswith("/"):
+            raise ValueError("ring_name must start with '/'")
+
+        self._ring_name = ring_name
+        self._path = Path("/dev/shm") / ring_name[1:]
+        self._wait_for_ring_file(timeout_s=open_timeout_s, poll_s=open_poll_s)
+
+        self._file = self._path.open("r+b", buffering=0)
+        self._mmap = mmap.mmap(self._file.fileno(), 0)
+        self._validate_header_with_wait(timeout_s=1.0, poll_s=0.002)
+
+    def close(self) -> None:
+        """Close mmap and file handle."""
+        self._mmap.close()
+        self._file.close()
+
+    def pop_payload(self) -> bytes | None:
+        """Read next payload from ring, or `None` if no unread payload exists."""
+        write_seq = self._read_u64(24)
+        read_seq = self._read_u64(32)
+        if read_seq >= write_seq:
+            return None
+
+        index = read_seq % self.capacity
+        slot_offset = _HEADER_SIZE + index * (_SLOT_HEADER_SIZE + self.slot_size_bytes)
+
+        payload_size = self._read_u32(slot_offset)
+        sequence = self._read_u64(slot_offset + 8)
+        if sequence != read_seq + 1:
+            self._write_u64(32, write_seq)
+            return None
+
+        payload_offset = slot_offset + _SLOT_HEADER_SIZE
+        payload = self._mmap[payload_offset : payload_offset + payload_size]
+        self._write_u64(32, read_seq + 1)
+        return bytes(payload)
+
+    def pop_raw_collection(self) -> SweepCollection | None:
+        """Read next raw collection from ring."""
+        payload = self.pop_payload()
+        if payload is None:
+            return None
+        return decode_trace_collection(payload, RAW_MAGIC)
+
+    def pop_preprocessed_collection(self) -> SweepCollection | None:
+        """Read next preprocessed collection from ring."""
+        payload = self.pop_payload()
+        if payload is None:
+            return None
+        return decode_trace_collection(payload, PREPROC_MAGIC)
+
+    def pop_result_collection(self) -> ResultCollection | None:
+        """Read next processed result collection from ring."""
+        payload = self.pop_payload()
+        if payload is None:
+            return None
+        return decode_result_collection(payload)
+
+    def drop_all(self) -> int:
+        """Mark all unread slots as consumed and return number of dropped payloads."""
+        write_seq = self._read_u64(24)
+        read_seq = self._read_u64(32)
+        if read_seq >= write_seq:
+            return 0
+        dropped = int(write_seq - read_seq)
+        self._write_u64(32, write_seq)
+        return dropped
+
+    @property
+    def capacity(self) -> int:
+        """Number of slots in ring."""
+        return self._read_u32(12)
+
+    @property
+    def slot_size_bytes(self) -> int:
+        """Maximum payload size per slot."""
+        return self._read_u32(16)
+
+    def _validate_header_with_wait(self, timeout_s: float, poll_s: float) -> None:
+        """Wait for ring header to contain expected magic and version."""
+        deadline = time.monotonic() + timeout_s
+        while True:
+            magic = self._mmap[:8]
+            version = self._read_u32(8)
+            if magic == _MAGIC and version == _VERSION:
+                return
+
+            if time.monotonic() >= deadline:
+                if magic != _MAGIC:
+                    got_magic = bytes(magic).hex()
+                    expected_magic = _MAGIC.hex()
+                    raise RuntimeError(
+                        f"Shared memory ring magic mismatch for {self._ring_name}: "
+                        f"got=0x{got_magic}, expected=0x{expected_magic}"
+                    )
+                raise RuntimeError(
+                    f"Shared memory ring version mismatch for {self._ring_name}: "
+                    f"got={version}, expected={_VERSION}"
+                )
+
+            time.sleep(poll_s)
+
+    def _wait_for_ring_file(self, timeout_s: float, poll_s: float) -> None:
+        """Wait until ring file appears in `/dev/shm`."""
+        deadline = time.monotonic() + timeout_s
+        while not self._path.exists():
+            if time.monotonic() >= deadline:
+                raise FileNotFoundError(f"Shared memory ring does not exist: {self._ring_name}")
+            time.sleep(poll_s)
+
+    def _read_u32(self, offset: int) -> int:
+        """Read little-endian u32 at mmap offset."""
+        return struct.unpack_from("<I", self._mmap, offset)[0]
+
+    def _read_u64(self, offset: int) -> int:
+        """Read little-endian u64 at mmap offset."""
+        return struct.unpack_from("<Q", self._mmap, offset)[0]
+
+    def _write_u64(self, offset: int, value: int) -> None:
+        """Write little-endian u64 at mmap offset."""
+        struct.pack_into("<Q", self._mmap, offset, value)

python_app/orchestration/shm_reader.py

@@ -0,0 +1,21 @@
+"""Facade exports for shared-memory ring reader and decoders."""
+
+from python_app.orchestration.shm.binary_cursor import ByteCursor
+from python_app.orchestration.shm.decoder import (
+    PREPROC_MAGIC,
+    RAW_MAGIC,
+    RESULT_MAGIC,
+    decode_result_collection,
+    decode_trace_collection,
+)
+from python_app.orchestration.shm.ring_reader import ShmRingReader
+
+__all__ = [
+    "ByteCursor",
+    "PREPROC_MAGIC",
+    "RAW_MAGIC",
+    "RESULT_MAGIC",
+    "ShmRingReader",
+    "decode_result_collection",
+    "decode_trace_collection",
+]