Rework radar system
This commit is contained in:
Ayzen
@@ -6,14 +6,13 @@ import logging
|
||||
from pathlib import Path
|
||||
|
||||
from python_app.file_io import atomic_write_json
|
||||
from python_app.models.run_config_model import RunConfigModel, parse_combos_from_text
|
||||
from python_app.models.run_config_schema import RunConfigModel
|
||||
from python_app.orchestration.preprocess_assets import (
|
||||
PREPROCESS_ASSET_KEYS,
|
||||
PREPROCESS_ASSET_SPECS,
|
||||
PREPROCESS_ASSET_NAMES,
|
||||
preprocess_asset_model,
|
||||
runtime_preprocess_asset_keys,
|
||||
)
|
||||
from python_app.storage.npz_store import NpzStore
|
||||
from python_app.storage.npz.store import NpzStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -33,18 +32,14 @@ class ConfigWriter:
|
||||
config: RunConfigModel,
|
||||
) -> None:
|
||||
"""Export selected preprocess sets into runtime bundles and update config paths."""
|
||||
for key in PREPROCESS_ASSET_KEYS:
|
||||
for key in PREPROCESS_ASSET_NAMES:
|
||||
preprocess_asset_model(config, key).bundle_path = ""
|
||||
|
||||
for key in runtime_preprocess_asset_keys(config):
|
||||
spec = PREPROCESS_ASSET_SPECS[key]
|
||||
asset = preprocess_asset_model(config, key)
|
||||
bundle_path = self._runtime_dir / spec.runtime_filename
|
||||
bundle_path = self._runtime_dir / f"{key}_bundle.bin"
|
||||
logger.debug("Exporting preprocess bundle %s (set=%s) -> %s", key, asset.set_name, bundle_path)
|
||||
if key == "s21_object_calibration":
|
||||
store.export_object_calibration_bundle(radar_key, asset.set_name, bundle_path, config)
|
||||
else:
|
||||
store.export_set_bundle(spec.set_kind, radar_key, asset.set_name, bundle_path)
|
||||
store.export_set_bundle(key, radar_key, asset.set_name, bundle_path)
|
||||
asset.bundle_path = str(bundle_path)
|
||||
|
||||
def write(self, config: RunConfigModel, output_path: Path) -> Path:
|
||||
@@ -52,6 +47,3 @@ class ConfigWriter:
|
||||
atomic_write_json(output_path, config.to_dict(), durable=True)
|
||||
logger.debug("Wrote run config to %s", output_path)
|
||||
return output_path
|
||||
|
||||
|
||||
__all__ = ["ConfigWriter", "parse_combos_from_text"]
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
"""Helpers for extracting GPR objects from result collections.
|
||||
|
||||
Locator TCP delivery now lives in the C++ data_processor. This module retains
|
||||
only the inspection helpers that the GUI uses for plotting.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
|
||||
from python_app.models.dataset_model import ResultCollection, ResultPayload
|
||||
|
||||
|
||||
def collection_payload_by_name(
|
||||
collection: ResultCollection,
|
||||
name: str,
|
||||
kind: int | None = None,
|
||||
) -> ResultPayload | None:
|
||||
"""Return the first collection payload matching name and optional kind."""
|
||||
for payload in collection.collection_payloads:
|
||||
if payload.processing_name != name:
|
||||
continue
|
||||
if kind is not None and int(payload.kind) != int(kind):
|
||||
continue
|
||||
return payload
|
||||
return None
|
||||
|
||||
|
||||
def collection_payloads_by_prefix(
|
||||
collection: ResultCollection,
|
||||
prefix: str,
|
||||
kind: int | None = None,
|
||||
) -> list[ResultPayload]:
|
||||
"""Return collection payloads matching a processing-name prefix."""
|
||||
payloads: list[ResultPayload] = []
|
||||
for payload in collection.collection_payloads:
|
||||
if not str(payload.processing_name).startswith(prefix):
|
||||
continue
|
||||
if kind is not None and int(payload.kind) != int(kind):
|
||||
continue
|
||||
payloads.append(payload)
|
||||
return payloads
|
||||
|
||||
|
||||
def collection_has_gpr_payloads(collection: ResultCollection) -> bool:
|
||||
"""Return whether collection carries GPR-specific collection payloads."""
|
||||
return any(
|
||||
str(payload.processing_name).startswith("gpr_")
|
||||
for payload in collection.collection_payloads
|
||||
)
|
||||
|
||||
|
||||
def gpr_object_rows(collection: ResultCollection) -> np.ndarray:
|
||||
"""Return object rows as `[x_m, z_m, score]` from a GPR collection."""
|
||||
points_payload = collection_payload_by_name(collection, "gpr_points", kind=4)
|
||||
if points_payload is not None:
|
||||
points = np.asarray(points_payload.table, dtype=np.float32)
|
||||
if points.ndim == 2 and points.shape[1] >= 3:
|
||||
return points[:, :3]
|
||||
|
||||
centers_payload = collection_payload_by_name(collection, "gpr_region_centers", kind=4)
|
||||
if centers_payload is not None:
|
||||
centers = np.asarray(centers_payload.table, dtype=np.float32)
|
||||
if centers.ndim == 2 and centers.shape[1] >= 3:
|
||||
return centers[:, :3]
|
||||
|
||||
return np.zeros((0, 3), dtype=np.float32)
|
||||
|
||||
|
||||
def apply_object_draw_limits(
|
||||
rows: np.ndarray,
|
||||
limits: tuple[int, int] | None,
|
||||
) -> np.ndarray:
|
||||
"""Apply the object count/top-M drawing rules to already-filtered `[x, z, score]` rows.
|
||||
|
||||
`limits` is `(max_detected_objects, draw_top_objects)`, or `None` to disable
|
||||
(legacy GPR). When more than ``max_detected_objects`` survive, ALL are hidden
|
||||
(the scene is too cluttered to be meaningful); otherwise the top ``draw_top_objects``
|
||||
rows are kept (rows arrive already sorted by score descending).
|
||||
"""
|
||||
if limits is None or rows.size == 0:
|
||||
return rows
|
||||
max_detected_objects, draw_top_objects = limits
|
||||
if rows.shape[0] > int(max_detected_objects):
|
||||
return np.zeros((0, rows.shape[1]), dtype=rows.dtype)
|
||||
return rows[: max(0, int(draw_top_objects))]
|
||||
|
||||
|
||||
def filter_object_rows(
|
||||
rows: np.ndarray,
|
||||
*,
|
||||
min_score: float,
|
||||
x_bounds: tuple[float, float],
|
||||
z_bounds: tuple[float, float],
|
||||
draw_limits: tuple[int, int] | None,
|
||||
) -> np.ndarray:
|
||||
"""Filter `[x_m, z_m, score]` object rows for display/broadcast.
|
||||
|
||||
Drops non-finite rows, rows below ``min_score`` (the caller passes the mode's own
|
||||
threshold — a normalized float for coherent GPR, a pair count for legacy GPR; the
|
||||
comparison is identical either way), and rows outside the visible X/Z window, then
|
||||
applies ``draw_limits``. Mode-agnostic: all semantics enter through the parameters.
|
||||
"""
|
||||
if rows.size == 0:
|
||||
return rows
|
||||
x_min, x_max = x_bounds
|
||||
z_min, z_max = z_bounds
|
||||
visible_mask = (
|
||||
np.all(np.isfinite(rows[:, :3]), axis=1)
|
||||
& (rows[:, 2] >= min_score)
|
||||
& (rows[:, 0] >= x_min)
|
||||
& (rows[:, 0] <= x_max)
|
||||
& (rows[:, 1] >= z_min)
|
||||
& (rows[:, 1] <= z_max)
|
||||
)
|
||||
return apply_object_draw_limits(rows[visible_mask], draw_limits)
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
from dataclasses import asdict, dataclass, field
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
@@ -15,14 +15,9 @@ class ProcessingLiveConfig:
|
||||
"""Runtime-adjustable processing parameters shared with data processor."""
|
||||
|
||||
processor_mode: str = "pass_through"
|
||||
pass_through_channel: str = "s21"
|
||||
pass_through_fixed_y_enabled: bool = False
|
||||
pass_through_y_min_db: float = -100.0
|
||||
pass_through_y_max_db: float = 0.0
|
||||
bscan_axis: str = "abs"
|
||||
bscan_channel: str = "s21"
|
||||
bscan_cut_m: float = 0.0
|
||||
bscan_max_depth_m: float = 8.0
|
||||
bscan_cut_m: float = 0.824
|
||||
bscan_max_depth_m: float = 1.0
|
||||
bscan_gain: float = 1.0
|
||||
bscan_start_freq_mhz: float = 100.0
|
||||
bscan_stop_freq_mhz: float = 8800.0
|
||||
@@ -39,8 +34,7 @@ class ProcessingLiveConfig:
|
||||
gpr_object_min_frac: float = 0.7
|
||||
gpr_score_mode: str = "combined"
|
||||
# Backprojection intra-sweep speed-correction mode: "int_minus" (full
|
||||
# correction) or "int_focus" (focusing residual only). Mirrors Python
|
||||
# Horns_motion_3libre.py MOTION_CORRECTION_MODE.
|
||||
# correction) or "int_focus" (focusing residual only).
|
||||
gpr_motion_mode: str = "int_minus"
|
||||
gpr_max_detected_objects_to_draw: int = 5
|
||||
gpr_draw_top_m_objects: int = 2
|
||||
@@ -89,8 +83,8 @@ class ProcessingLiveConfig:
|
||||
# and keeps using `gpr_speed_m_s` from this file.
|
||||
ignore_socket_speed: bool = False
|
||||
reprocess_current_result: bool = True
|
||||
history_command_seq: int = 0
|
||||
history_command: str = "none"
|
||||
history_clear_before_ns: int = 0
|
||||
history_removed: list[dict[str, int]] = field(default_factory=list)
|
||||
settings_revision: int = 0
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
|
||||
@@ -69,10 +69,6 @@ class PipelineMetrics:
|
||||
self._buffers: dict[str, deque[int]] = {}
|
||||
logger.debug("PipelineMetrics init: report_every=%d", self._report_every)
|
||||
|
||||
def set_log_sink(self, log_sink: Callable[[str], None] | None) -> None:
|
||||
"""Reassign the log sink (used when the GUI log appears after init)."""
|
||||
self._log_sink = log_sink
|
||||
|
||||
def record(self, name: str, duration_ns: int) -> MetricReport | None:
|
||||
"""Append one sample. Return a flushed report if the buffer is full."""
|
||||
if duration_ns <= 0:
|
||||
|
||||
@@ -2,38 +2,14 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from python_app.models.run_config_schema import PreprocessAssetModel, RunConfigModel
|
||||
|
||||
from python_app.models.run_config_model import PreprocessAssetModel, RunConfigModel
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PreprocessAssetSpec:
|
||||
"""Stable description of one preprocess asset."""
|
||||
|
||||
display_name: str
|
||||
set_kind: str
|
||||
runtime_filename: str
|
||||
channel: str
|
||||
|
||||
|
||||
PREPROCESS_ASSET_SPECS = {
|
||||
"s21_calibration": PreprocessAssetSpec("S21 Calibration", "s21_calibration", "s21_calibration_bundle.bin", "s21"),
|
||||
"s21_reference": PreprocessAssetSpec("S21 Reference", "s21_reference", "s21_reference_bundle.bin", "s21"),
|
||||
"s21_object_calibration": PreprocessAssetSpec("Object Calibration", "s21_object_calibration", "s21_object_calibration_bundle.bin", "s21"),
|
||||
"s11_open": PreprocessAssetSpec("S11 Open", "s11_open", "s11_open_calibration_bundle.bin", "s11"),
|
||||
"s11_short": PreprocessAssetSpec("S11 Short", "s11_short", "s11_short_calibration_bundle.bin", "s11"),
|
||||
"s11_load": PreprocessAssetSpec("S11 Load", "s11_load", "s11_load_calibration_bundle.bin", "s11"),
|
||||
"s11_reference": PreprocessAssetSpec("S11 Reference", "s11_reference", "s11_reference_bundle.bin", "s11"),
|
||||
PREPROCESS_ASSET_NAMES = {
|
||||
"s21_calibration": "S21 Calibration",
|
||||
"s21_reference": "S21 Reference",
|
||||
"s21_object_calibration": "Object Calibration",
|
||||
}
|
||||
|
||||
PREPROCESS_ASSET_KEYS = tuple(PREPROCESS_ASSET_SPECS.keys())
|
||||
S21_PREPROCESS_ASSET_KEYS = ("s21_calibration", "s21_reference")
|
||||
S11_PREPROCESS_ASSET_KEYS = ("s11_open", "s11_short", "s11_load", "s11_reference")
|
||||
S11_PREPROCESS_CALIBRATION_KEYS = ("s11_open", "s11_short", "s11_load")
|
||||
VISIBLE_PREPROCESS_ASSET_KEYS = (*S21_PREPROCESS_ASSET_KEYS, "s21_object_calibration")
|
||||
CAPTURABLE_PREPROCESS_ASSET_KEYS = S21_PREPROCESS_ASSET_KEYS
|
||||
REQUIRED_PREPROCESS_ASSET_KEYS = S21_PREPROCESS_ASSET_KEYS
|
||||
REQUIRED_PREPROCESS_ASSET_KEYS = ("s21_calibration", "s21_reference")
|
||||
|
||||
|
||||
def preprocess_asset_model(config: RunConfigModel, key: str) -> PreprocessAssetModel:
|
||||
@@ -44,46 +20,16 @@ def preprocess_asset_model(config: RunConfigModel, key: str) -> PreprocessAssetM
|
||||
return config.preprocess.s21.reference
|
||||
if key == "s21_object_calibration":
|
||||
return config.preprocess.s21.object_calibration
|
||||
if key == "s11_open":
|
||||
return config.preprocess.s11.calibration.open
|
||||
if key == "s11_short":
|
||||
return config.preprocess.s11.calibration.short
|
||||
if key == "s11_load":
|
||||
return config.preprocess.s11.calibration.load
|
||||
if key == "s11_reference":
|
||||
return config.preprocess.s11.reference
|
||||
raise KeyError(f"Unknown preprocess asset key: {key}")
|
||||
|
||||
|
||||
def preprocess_asset_display_name(key: str) -> str:
|
||||
"""Return human-readable label for preprocess asset."""
|
||||
return PREPROCESS_ASSET_SPECS[key].display_name
|
||||
|
||||
|
||||
def preprocess_asset_channel(key: str) -> str:
|
||||
"""Return associated trace channel for preprocess asset."""
|
||||
return PREPROCESS_ASSET_SPECS[key].channel
|
||||
|
||||
|
||||
def preprocess_asset_set_name(config: RunConfigModel, key: str) -> str:
|
||||
"""Return normalized selected set name for one preprocess asset."""
|
||||
return str(preprocess_asset_model(config, key).set_name).strip()
|
||||
return PREPROCESS_ASSET_NAMES[key]
|
||||
|
||||
|
||||
def runtime_preprocess_asset_keys(config: RunConfigModel) -> tuple[str, ...]:
|
||||
"""Return preprocess assets that should be exported and validated for runtime."""
|
||||
enabled_keys: list[str] = list(REQUIRED_PREPROCESS_ASSET_KEYS)
|
||||
if preprocess_asset_set_name(config, "s21_object_calibration"):
|
||||
enabled_keys.append("s21_object_calibration")
|
||||
|
||||
has_full_s11_calibration = all(
|
||||
preprocess_asset_set_name(config, key)
|
||||
for key in S11_PREPROCESS_CALIBRATION_KEYS
|
||||
)
|
||||
if not has_full_s11_calibration:
|
||||
return tuple(enabled_keys)
|
||||
|
||||
enabled_keys.extend(S11_PREPROCESS_CALIBRATION_KEYS)
|
||||
if preprocess_asset_set_name(config, "s11_reference"):
|
||||
enabled_keys.append("s11_reference")
|
||||
return tuple(enabled_keys)
|
||||
"""The through/reference pair plus the selected optional object correction."""
|
||||
return (*REQUIRED_PREPROCESS_ASSET_KEYS, "s21_object_calibration") if (
|
||||
config.preprocess.s21.object_calibration.set_name.strip()
|
||||
) else REQUIRED_PREPROCESS_ASSET_KEYS
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Process supervisor for lifecycle management of C++ pipeline binaries."""
|
||||
"""Process supervisor for lifecycle management of pipeline subprocesses."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -133,13 +133,13 @@ class ProcessSupervisor:
|
||||
|
||||
def is_running(self) -> bool:
|
||||
"""Return whether acquisition-side processes are alive."""
|
||||
return self._is_alive("data_preprocessor") or self._is_alive("sweep_orchestrator")
|
||||
return self._is_alive("data_preprocessor") or self._is_alive("acquisition")
|
||||
|
||||
def is_processor_running(self) -> bool:
|
||||
"""Return whether data processor process is alive."""
|
||||
return self._is_alive("data_processor")
|
||||
|
||||
def start(self, config_path: Path, *, allow_clean_orchestrator_exit: bool = False) -> None:
|
||||
def start(self, config_path: Path, *, allow_clean_acquisition_exit: bool = False) -> None:
|
||||
"""Start required pipeline binaries and wait until they are ready."""
|
||||
if self.is_running():
|
||||
raise RuntimeError("Acquisition processes are already running")
|
||||
@@ -148,9 +148,9 @@ class ProcessSupervisor:
|
||||
self._kill_kamil_collectors()
|
||||
|
||||
logger.info(
|
||||
"Starting pipeline from config %s (allow_clean_orchestrator_exit=%s)",
|
||||
"Starting pipeline from config %s (allow_clean_acquisition_exit=%s)",
|
||||
config_path,
|
||||
allow_clean_orchestrator_exit,
|
||||
allow_clean_acquisition_exit,
|
||||
)
|
||||
acquisition_command = self._acquisition_command(config_path)
|
||||
command_specs = {
|
||||
@@ -164,12 +164,12 @@ class ProcessSupervisor:
|
||||
"--config",
|
||||
str(config_path),
|
||||
],
|
||||
"sweep_orchestrator": acquisition_command,
|
||||
"acquisition": acquisition_command,
|
||||
}
|
||||
processor_was_running = self.is_processor_running()
|
||||
required_processes: list[str] = ["data_preprocessor"]
|
||||
if not allow_clean_orchestrator_exit:
|
||||
required_processes.append("sweep_orchestrator")
|
||||
if not allow_clean_acquisition_exit:
|
||||
required_processes.append("acquisition")
|
||||
if not processor_was_running:
|
||||
required_processes.insert(0, "data_processor")
|
||||
|
||||
@@ -179,9 +179,9 @@ class ProcessSupervisor:
|
||||
|
||||
self._spawn("data_preprocessor", command_specs["data_preprocessor"], allow_clean_exit=False)
|
||||
self._spawn(
|
||||
"sweep_orchestrator",
|
||||
command_specs["sweep_orchestrator"],
|
||||
allow_clean_exit=allow_clean_orchestrator_exit,
|
||||
"acquisition",
|
||||
command_specs["acquisition"],
|
||||
allow_clean_exit=allow_clean_acquisition_exit,
|
||||
)
|
||||
self._wait_until_ready(required_processes)
|
||||
except Exception:
|
||||
@@ -196,12 +196,12 @@ class ProcessSupervisor:
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Stop acquisition-side processes, keep processor process intact."""
|
||||
self._stop_processes(["sweep_orchestrator", "data_preprocessor"])
|
||||
self._stop_processes(["acquisition", "data_preprocessor"])
|
||||
self._kill_kamil_collectors()
|
||||
|
||||
def stop_orchestrator(self) -> None:
|
||||
"""Stop orchestrator process only."""
|
||||
self._stop_processes(["sweep_orchestrator"])
|
||||
def stop_acquisition(self) -> None:
|
||||
"""Stop acquisition process only."""
|
||||
self._stop_processes(["acquisition"])
|
||||
|
||||
def stop_preprocessor(self) -> None:
|
||||
"""Stop preprocessor process only."""
|
||||
@@ -209,7 +209,7 @@ class ProcessSupervisor:
|
||||
|
||||
def stop_all(self) -> None:
|
||||
"""Stop all managed processes."""
|
||||
self._stop_processes(["sweep_orchestrator", "data_preprocessor", "data_processor"])
|
||||
self._stop_processes(["acquisition", "data_preprocessor", "data_processor"])
|
||||
self._kill_kamil_collectors()
|
||||
|
||||
def _spawn(self, name: str, command: list[str], *, allow_clean_exit: bool) -> None:
|
||||
@@ -276,7 +276,7 @@ class ProcessSupervisor:
|
||||
|
||||
def _acquisition_command(self, config_path: Path) -> list[str]:
|
||||
"""Select the same acquisition producer as the shell launcher."""
|
||||
return acquisition_command(self._project_root, config_path)
|
||||
return acquisition_command(config_path)
|
||||
|
||||
def _stop_processes(self, names: Iterable[str]) -> None:
|
||||
"""Gracefully terminate processes, then force-kill on per-process timeout."""
|
||||
@@ -552,7 +552,7 @@ class ProcessSupervisor:
|
||||
Reads `/proc/<pid>/cmdline` so a recycled PID owned by an unrelated
|
||||
process is never killed.
|
||||
"""
|
||||
native_names = {"data_processor", "data_preprocessor", "sweep_orchestrator"}
|
||||
native_names = {"data_processor", "data_preprocessor"}
|
||||
producer_modules = {
|
||||
"python_app.scripts.matrix_raw_producer",
|
||||
"python_app.scripts.kamil_adc_raw_producer",
|
||||
|
||||
@@ -7,29 +7,25 @@ from pathlib import Path
|
||||
import sys
|
||||
|
||||
from python_app.models.run_config_schema import RadarModel
|
||||
from python_app.models.json_fields import json_object, read_str
|
||||
|
||||
|
||||
def acquisition_command(project_root: Path, config_path: Path) -> list[str]:
|
||||
def acquisition_command(config_path: Path) -> list[str]:
|
||||
"""Select replay first, then the configured model's acquisition producer."""
|
||||
payload = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
radar = payload.get("radar", {}) if isinstance(payload, dict) else {}
|
||||
radar = radar if isinstance(radar, dict) else {}
|
||||
replay = radar.get("replay", {})
|
||||
dataset = replay.get("dataset_dir") if isinstance(replay, dict) else None
|
||||
defaults = RadarModel()
|
||||
model = radar.get("model")
|
||||
if model is None and isinstance(payload, dict):
|
||||
model = defaults.model
|
||||
driver_mode = radar.get("driver_mode")
|
||||
if driver_mode is None:
|
||||
driver_mode = defaults.driver_mode
|
||||
if model in ("librevna", "librevna_multi") and driver_mode == "mock":
|
||||
model = "librevna_multi"
|
||||
module = "replay_raw_producer" if isinstance(dataset, str) and dataset.strip() else {
|
||||
payload = json_object(json.loads(config_path.read_text(encoding="utf-8")), "run config")
|
||||
radar = json_object(payload.get("radar"), "radar")
|
||||
replay = json_object(radar.get("replay"), "radar.replay")
|
||||
dataset = read_str(replay, "dataset_dir", "")
|
||||
model = read_str(radar, "model", RadarModel().model)
|
||||
producers = {
|
||||
"librevna": "matrix_raw_producer",
|
||||
"librevna_multi": "matrix_raw_producer",
|
||||
"kamil_adc": "kamil_adc_raw_producer",
|
||||
}.get(model if isinstance(model, str) else "")
|
||||
command = [sys.executable, "-m", f"python_app.scripts.{module}"] if module else [
|
||||
str(project_root / "build/bin/sweep_orchestrator"),
|
||||
]
|
||||
return [*command, "--config", str(config_path)]
|
||||
}
|
||||
if dataset.strip():
|
||||
module = "replay_raw_producer"
|
||||
else:
|
||||
if model not in producers:
|
||||
raise ValueError(f"Unsupported radar model: {model}")
|
||||
module = producers[model]
|
||||
return [sys.executable, "-m", f"python_app.scripts.{module}", "--config", str(config_path)]
|
||||
|
||||
@@ -1,23 +1 @@
|
||||
"""Shared-memory ring readers and payload decoders."""
|
||||
|
||||
from python_app.orchestration.shm.binary_cursor import ByteCursor
|
||||
from python_app.orchestration.shm.decoder import (
|
||||
PREPROC_MAGIC,
|
||||
RAW_MAGIC,
|
||||
RESULT_MAGIC,
|
||||
decode_result_collection,
|
||||
decode_trace_collection,
|
||||
)
|
||||
from python_app.orchestration.shm.ring_reader import ShmRingReader
|
||||
from python_app.orchestration.shm.ring_writer import ShmRingWriter
|
||||
|
||||
__all__ = [
|
||||
"ByteCursor",
|
||||
"PREPROC_MAGIC",
|
||||
"RAW_MAGIC",
|
||||
"RESULT_MAGIC",
|
||||
"ShmRingReader",
|
||||
"ShmRingWriter",
|
||||
"decode_result_collection",
|
||||
"decode_trace_collection",
|
||||
]
|
||||
"""Shared-memory rings and payload decoding."""
|
||||
|
||||
@@ -14,9 +14,9 @@ from python_app.models.dataset_model import (
|
||||
)
|
||||
from python_app.orchestration.shm.binary_cursor import ByteCursor
|
||||
|
||||
RAW_MAGIC = 0x32574152
|
||||
PREPROC_MAGIC = 0x32525050
|
||||
RESULT_MAGIC = 0x324C5352 # RSL2: adds processing_duration_ns after monotonic_ns
|
||||
RAW_MAGIC = 0x33574152 # RAW3
|
||||
PREPROC_MAGIC = 0x33505250 # PRP3
|
||||
RESULT_MAGIC = 0x334C5352 # RSL3: typed acquisition and processing metadata
|
||||
|
||||
|
||||
def decode_trace_collection(payload: bytes, expected_magic: int) -> SweepCollection:
|
||||
@@ -39,40 +39,23 @@ def decode_trace_collection(payload: bytes, expected_magic: int) -> SweepCollect
|
||||
freq_bytes = point_count * 4
|
||||
freq = np.frombuffer(cursor.read_bytes(freq_bytes), dtype="<f4").astype(np.float32, copy=False)
|
||||
|
||||
interleaved_bytes = point_count * 8
|
||||
s11_interleaved = np.frombuffer(cursor.read_bytes(interleaved_bytes), dtype="<f4")
|
||||
s21_interleaved = np.frombuffer(cursor.read_bytes(interleaved_bytes), dtype="<f4")
|
||||
s11 = (s11_interleaved[0::2] + 1j * s11_interleaved[1::2]).astype(np.complex64, copy=False)
|
||||
s21 = (s21_interleaved[0::2] + 1j * s21_interleaved[1::2]).astype(np.complex64, copy=False)
|
||||
s21 = np.frombuffer(cursor.read_bytes(point_count * 8), dtype="<c8")
|
||||
|
||||
traces.append(
|
||||
TraceData(
|
||||
combo=ComboKey(input=input_pos, output=output_pos),
|
||||
frequency_hz=freq,
|
||||
s11=s11,
|
||||
s21=s21,
|
||||
)
|
||||
)
|
||||
|
||||
# Optional trailer, written after the trace blocks by newer producers: the
|
||||
# collection capture window, then a per-trace window table. Both stages are
|
||||
# optional so payloads from an older producer still decode (the timestamps
|
||||
# simply stay zero).
|
||||
capture_start_ns = 0
|
||||
capture_end_ns = 0
|
||||
if cursor.remaining_bytes() != 0:
|
||||
if cursor.remaining_bytes() < 16:
|
||||
raise ValueError("Truncated capture window in trace collection")
|
||||
capture_start_ns = cursor.read_u64()
|
||||
capture_end_ns = cursor.read_u64()
|
||||
|
||||
if cursor.remaining_bytes() != 0:
|
||||
trace_time_count = cursor.read_u32()
|
||||
if trace_time_count != len(traces):
|
||||
raise ValueError("Per-trace capture window count does not match trace count")
|
||||
for trace in traces:
|
||||
trace.capture_start_ns = cursor.read_u64()
|
||||
trace.capture_end_ns = cursor.read_u64()
|
||||
capture_start_ns = cursor.read_u64()
|
||||
capture_end_ns = cursor.read_u64()
|
||||
if cursor.read_u32() != len(traces):
|
||||
raise ValueError("Per-trace capture window count does not match trace count")
|
||||
for trace in traces:
|
||||
trace.capture_start_ns = cursor.read_u64()
|
||||
trace.capture_end_ns = cursor.read_u64()
|
||||
|
||||
if cursor.remaining_bytes() != 0:
|
||||
raise ValueError("Unexpected trailing bytes in trace collection")
|
||||
@@ -98,8 +81,7 @@ def decode_result_collection(payload: bytes) -> ResultCollection:
|
||||
if kind == 1:
|
||||
point_count = cursor.read_u32()
|
||||
freq = np.frombuffer(cursor.read_bytes(point_count * 4), dtype="<f4").astype(np.float32, copy=False)
|
||||
interleaved = np.frombuffer(cursor.read_bytes(point_count * 8), dtype="<f4")
|
||||
trace = (interleaved[0::2] + 1j * interleaved[1::2]).astype(np.complex64, copy=False)
|
||||
trace = np.frombuffer(cursor.read_bytes(point_count * 8), dtype="<c8")
|
||||
return ResultPayload(
|
||||
processing_name=name,
|
||||
kind=kind,
|
||||
@@ -150,6 +132,10 @@ def decode_result_collection(payload: bytes) -> ResultCollection:
|
||||
collection_id = cursor.read_u64()
|
||||
monotonic_ns = cursor.read_u64()
|
||||
processing_duration_ns = cursor.read_u64()
|
||||
settings_revision = cursor.read_u64()
|
||||
capture_start_ns = cursor.read_u64()
|
||||
capture_end_ns = cursor.read_u64()
|
||||
speed_m_s = cursor.read_f32()
|
||||
collection_payload_count = cursor.read_u32()
|
||||
block_count = cursor.read_u32()
|
||||
|
||||
@@ -174,10 +160,17 @@ def decode_result_collection(payload: bytes) -> ResultCollection:
|
||||
)
|
||||
)
|
||||
|
||||
if cursor.remaining_bytes() != 0:
|
||||
raise ValueError("Unexpected trailing bytes in result collection")
|
||||
|
||||
return ResultCollection(
|
||||
collection_id=collection_id,
|
||||
monotonic_ns=monotonic_ns,
|
||||
processing_duration_ns=processing_duration_ns,
|
||||
settings_revision=settings_revision,
|
||||
capture_start_ns=capture_start_ns,
|
||||
capture_end_ns=capture_end_ns,
|
||||
speed_m_s=speed_m_s,
|
||||
collection_payloads=collection_payloads,
|
||||
blocks=blocks,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Shared ring layout; keep in sync with common_cpp/ipc/src/shm_ring.cpp."""
|
||||
|
||||
from pathlib import Path
|
||||
import struct
|
||||
|
||||
MAGIC = b"RDRRING2"
|
||||
VERSION = 1
|
||||
HEADER = struct.Struct("<8sIIIIQQQ16x")
|
||||
SLOT_HEADER_SIZE = 16
|
||||
|
||||
|
||||
def ring_path(name: str) -> Path:
|
||||
if not name.startswith("/") or len(name) < 2 or "/" in name[1:] or "\x00" in name:
|
||||
raise ValueError("ring_name must be '/' followed by a single nonempty filename")
|
||||
return Path("/dev/shm") / name[1:]
|
||||
@@ -4,9 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import mmap
|
||||
from pathlib import Path
|
||||
import time
|
||||
from typing import Final
|
||||
|
||||
from python_app.models.dataset_model import ResultCollection, SweepCollection
|
||||
from python_app.orchestration.shm import atomics
|
||||
@@ -16,31 +14,23 @@ from python_app.orchestration.shm.decoder import (
|
||||
decode_result_collection,
|
||||
decode_trace_collection,
|
||||
)
|
||||
from python_app.orchestration.shm.ring_layout import HEADER, MAGIC, SLOT_HEADER_SIZE, VERSION, ring_path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_HEADER_SIZE: Final[int] = 64
|
||||
_SLOT_HEADER_SIZE: Final[int] = 16
|
||||
_MAGIC: Final[bytes] = b"RDRRING2"
|
||||
_VERSION: Final[int] = 1
|
||||
|
||||
|
||||
class ShmRingReader:
|
||||
"""Read binary payloads from a lock-free ring in `/dev/shm`."""
|
||||
|
||||
def __init__(self, ring_name: str, open_timeout_s: float = 2.0, open_poll_s: float = 0.01) -> None:
|
||||
"""Open and validate ring by name, for example `/radar_results`."""
|
||||
if not ring_name.startswith("/") or len(ring_name) < 2 or "/" in ring_name[1:] or "\x00" in ring_name:
|
||||
raise ValueError("ring_name must be '/' followed by a single nonempty filename")
|
||||
|
||||
self._ring_name = ring_name
|
||||
self._path = Path("/dev/shm") / ring_name[1:]
|
||||
self.name = ring_name
|
||||
self._path = ring_path(ring_name)
|
||||
self._wait_for_ring_file(timeout_s=open_timeout_s, poll_s=open_poll_s)
|
||||
|
||||
self._file = self._path.open("r+b", buffering=0)
|
||||
self._mmap: mmap.mmap | None = None
|
||||
with self._path.open("r+b", buffering=0) as file:
|
||||
self._mmap = mmap.mmap(file.fileno(), 0)
|
||||
try:
|
||||
self._mmap = mmap.mmap(self._file.fileno(), 0)
|
||||
self._validate_header_with_wait(timeout_s=1.0, poll_s=0.002)
|
||||
except BaseException:
|
||||
# A fail-fast open (absent/incompatible ring) must not leak the fd/mapping.
|
||||
@@ -48,18 +38,15 @@ class ShmRingReader:
|
||||
raise
|
||||
logger.debug(
|
||||
"Opened SHM ring reader %s (capacity=%d, slot_size=%d bytes)",
|
||||
self._ring_name,
|
||||
self.name,
|
||||
self.capacity,
|
||||
self.slot_size_bytes,
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close mmap and file handle."""
|
||||
if self._mmap is not None:
|
||||
self._mmap.close()
|
||||
self._mmap = None
|
||||
self._file.close()
|
||||
logger.debug("Closed SHM ring reader %s", self._ring_name)
|
||||
"""Close the mapping and its owned file descriptor; safe to repeat."""
|
||||
self._mmap.close()
|
||||
logger.debug("Closed SHM ring reader %s", self.name)
|
||||
|
||||
def pop_payload(self) -> bytes | None:
|
||||
"""Read next payload from ring, or `None` if no unread payload exists."""
|
||||
@@ -69,7 +56,7 @@ class ShmRingReader:
|
||||
return None
|
||||
|
||||
index = read_seq % self.capacity
|
||||
slot_offset = _HEADER_SIZE + index * (_SLOT_HEADER_SIZE + self.slot_size_bytes)
|
||||
slot_offset = HEADER.size + index * (SLOT_HEADER_SIZE + self.slot_size_bytes)
|
||||
|
||||
# Seqlock read mirroring the C++ pop: a slot is valid for read_seq R only if
|
||||
# its sequence equals R+1 and is unchanged across the payload copy (i.e. the
|
||||
@@ -78,7 +65,7 @@ class ShmRingReader:
|
||||
sequence = self._read_u64(slot_offset + 8)
|
||||
if sequence != read_seq + 1:
|
||||
# Producer overwrote this slot before we read it. Resync to latest.
|
||||
logger.debug("Ring %s: slot lapped before read, resyncing to write_seq=%d", self._ring_name, write_seq)
|
||||
logger.debug("Ring %s: slot lapped before read, resyncing to write_seq=%d", self.name, write_seq)
|
||||
atomics.advance(self._mmap, 32, write_seq)
|
||||
return None
|
||||
|
||||
@@ -88,21 +75,21 @@ class ShmRingReader:
|
||||
if payload_size > self.slot_size_bytes:
|
||||
logger.debug(
|
||||
"Ring %s: payload_size %d exceeds slot %d, resyncing",
|
||||
self._ring_name,
|
||||
self.name,
|
||||
payload_size,
|
||||
self.slot_size_bytes,
|
||||
)
|
||||
atomics.advance(self._mmap, 32, write_seq)
|
||||
return None
|
||||
|
||||
payload_offset = slot_offset + _SLOT_HEADER_SIZE
|
||||
payload_offset = slot_offset + SLOT_HEADER_SIZE
|
||||
payload = bytes(self._mmap[payload_offset : payload_offset + payload_size])
|
||||
atomics.fence()
|
||||
|
||||
# Re-read the slot sequence after the copy; if it changed, the producer
|
||||
# overwrote this slot mid-copy and the payload is torn — discard and resync.
|
||||
if self._read_u64(slot_offset + 8) != read_seq + 1:
|
||||
logger.debug("Ring %s: slot overwritten mid-copy, discarding torn payload", self._ring_name)
|
||||
logger.debug("Ring %s: slot overwritten mid-copy, discarding torn payload", self.name)
|
||||
atomics.advance(self._mmap, 32, write_seq)
|
||||
return None
|
||||
|
||||
@@ -130,39 +117,6 @@ class ShmRingReader:
|
||||
return None
|
||||
return decode_result_collection(payload)
|
||||
|
||||
def peek_latest_payload(self) -> bytes | None:
|
||||
"""Return the most recently published payload WITHOUT consuming it.
|
||||
|
||||
Reads the newest slot through the seqlock and never advances `read_seq`, so a
|
||||
viewer can peek the freshest frame while the ring's real consumer keeps its own
|
||||
cursor — the two coexist without stealing each other's payloads. Inherently
|
||||
latest-wins: always the freshest published frame, or `None` when nothing has
|
||||
been published yet or the slot is being overwritten at this instant.
|
||||
"""
|
||||
write_seq = self._read_u64(24)
|
||||
if write_seq == 0:
|
||||
return None
|
||||
|
||||
latest_seq = write_seq - 1
|
||||
index = latest_seq % self.capacity
|
||||
slot_offset = _HEADER_SIZE + index * (_SLOT_HEADER_SIZE + self.slot_size_bytes)
|
||||
|
||||
# Seqlock read with NO cursor advance: accept the slot only if its sequence
|
||||
# equals the published value both before and after the copy (i.e. the producer
|
||||
# did not lap this slot mid-read). Never touch read_seq, so the real consumer
|
||||
# is undisturbed.
|
||||
if self._read_u64(slot_offset + 8) != latest_seq + 1:
|
||||
return None
|
||||
payload_size = self._read_u32(slot_offset)
|
||||
if payload_size > self.slot_size_bytes:
|
||||
return None
|
||||
payload_offset = slot_offset + _SLOT_HEADER_SIZE
|
||||
payload = bytes(self._mmap[payload_offset : payload_offset + payload_size])
|
||||
atomics.fence()
|
||||
if self._read_u64(slot_offset + 8) != latest_seq + 1:
|
||||
return None
|
||||
return payload
|
||||
|
||||
def drop_all(self) -> int:
|
||||
"""Mark all unread slots as consumed and return number of dropped payloads."""
|
||||
write_seq = self._read_u64(24)
|
||||
@@ -171,7 +125,7 @@ class ShmRingReader:
|
||||
return 0
|
||||
dropped = int(write_seq - read_seq)
|
||||
atomics.advance(self._mmap, 32, write_seq)
|
||||
logger.debug("Ring %s: dropped %d unread payload(s)", self._ring_name, dropped)
|
||||
logger.debug("Ring %s: dropped %d unread payload(s)", self.name, dropped)
|
||||
return dropped
|
||||
|
||||
@property
|
||||
@@ -190,27 +144,27 @@ class ShmRingReader:
|
||||
while True:
|
||||
magic = self._mmap[:8]
|
||||
version = self._read_u32(8)
|
||||
if magic == _MAGIC and version == _VERSION:
|
||||
if magic == MAGIC and version == VERSION:
|
||||
capacity = self.capacity
|
||||
slot_size = self.slot_size_bytes
|
||||
expected_size = _HEADER_SIZE + capacity * (_SLOT_HEADER_SIZE + slot_size)
|
||||
expected_size = HEADER.size + capacity * (SLOT_HEADER_SIZE + slot_size)
|
||||
if capacity > 0 and slot_size > 0 and slot_size % 8 == 0 and expected_size == len(self._mmap):
|
||||
return
|
||||
|
||||
if time.monotonic() >= deadline:
|
||||
if magic != _MAGIC:
|
||||
if magic != MAGIC:
|
||||
got_magic = bytes(magic).hex()
|
||||
expected_magic = _MAGIC.hex()
|
||||
expected_magic = MAGIC.hex()
|
||||
raise RuntimeError(
|
||||
f"Shared memory ring magic mismatch for {self._ring_name}: "
|
||||
f"Shared memory ring magic mismatch for {self.name}: "
|
||||
f"got=0x{got_magic}, expected=0x{expected_magic}"
|
||||
)
|
||||
if version != _VERSION:
|
||||
if version != VERSION:
|
||||
raise RuntimeError(
|
||||
f"Shared memory ring version mismatch for {self._ring_name}: "
|
||||
f"got={version}, expected={_VERSION}"
|
||||
f"Shared memory ring version mismatch for {self.name}: "
|
||||
f"got={version}, expected={VERSION}"
|
||||
)
|
||||
raise RuntimeError(f"Shared memory ring geometry mismatch for {self._ring_name}")
|
||||
raise RuntimeError(f"Shared memory ring geometry mismatch for {self.name}")
|
||||
|
||||
time.sleep(poll_s)
|
||||
|
||||
@@ -219,12 +173,12 @@ class ShmRingReader:
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while True:
|
||||
try:
|
||||
if self._path.stat().st_size >= _HEADER_SIZE:
|
||||
if self._path.stat().st_size >= HEADER.size:
|
||||
return
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
if time.monotonic() >= deadline:
|
||||
raise FileNotFoundError(f"Shared memory ring does not exist: {self._ring_name}")
|
||||
raise FileNotFoundError(f"Shared memory ring does not exist: {self.name}")
|
||||
time.sleep(poll_s)
|
||||
|
||||
def _read_u32(self, offset: int) -> int:
|
||||
|
||||
@@ -5,117 +5,74 @@ from __future__ import annotations
|
||||
import logging
|
||||
import mmap
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from python_app.models.run_config_schema import RingEndpointModel
|
||||
from python_app.models.run_config_validation import validate_ring_endpoint
|
||||
from python_app.orchestration.shm import atomics
|
||||
logger = logging.getLogger(__name__)
|
||||
from python_app.orchestration.shm.ring_layout import HEADER, MAGIC, SLOT_HEADER_SIZE, VERSION, ring_path
|
||||
|
||||
_HEADER_SIZE: Final[int] = 64
|
||||
_SLOT_HEADER_SIZE: Final[int] = 16
|
||||
_MAGIC: Final[bytes] = b"RDRRING2"
|
||||
_VERSION: Final[int] = 1
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ShmRingWriter:
|
||||
"""Write binary payloads into the shared-memory ring used by C++ workers.
|
||||
"""The sole producer owns the ring and replaces incompatible stale segments.
|
||||
|
||||
The writer is the sole *owner* of the rings it opens: there is exactly one
|
||||
producer per ring (the acquisition producer for the raw/raw_tap rings). On a
|
||||
geometry mismatch with a pre-existing segment (e.g. a stale ring left by a prior
|
||||
run with a different sweep config), the owner unlinks and recreates the segment
|
||||
from scratch rather than truncating in place or diverging silently — mirroring
|
||||
the clean-shm-on-restart contract on the C++/deploy side (#13). A non-owner must
|
||||
never recreate a ring; readers and C++ consumers only ever attach to an existing
|
||||
one.
|
||||
Replacement uses unlink plus a new inode, preserving existing readers' mappings.
|
||||
Compatible segments retain their unread payloads and sequence counters.
|
||||
"""
|
||||
|
||||
def __init__(self, ring_name: str, capacity: int, slot_size_bytes: int) -> None:
|
||||
"""Open or create a POSIX SHM ring by name (as the ring owner)."""
|
||||
if not ring_name.startswith("/") or len(ring_name) < 2 or "/" in ring_name[1:] or "\x00" in ring_name:
|
||||
raise ValueError("ring_name must be '/' followed by a single nonempty filename")
|
||||
self._path = ring_path(ring_name)
|
||||
validate_ring_endpoint(RingEndpointModel(ring_name, capacity, slot_size_bytes))
|
||||
|
||||
self._ring_name = ring_name
|
||||
self._capacity = int(capacity)
|
||||
self._slot_size_bytes = int(slot_size_bytes)
|
||||
self._mapped_size = _HEADER_SIZE + self._capacity * (_SLOT_HEADER_SIZE + self._slot_size_bytes)
|
||||
self._path = Path("/dev/shm") / ring_name[1:]
|
||||
self._file = self._mmap = None
|
||||
self._capacity = capacity
|
||||
self._slot_size_bytes = slot_size_bytes
|
||||
self._mapped_size = HEADER.size + capacity * (SLOT_HEADER_SIZE + slot_size_bytes)
|
||||
self._mmap = self._open_owned()
|
||||
|
||||
def _open_owned(self) -> mmap.mmap:
|
||||
"""Reuse a compatible segment, otherwise replace it without truncating it."""
|
||||
try:
|
||||
self._open_owned()
|
||||
except BaseException:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def _open_owned(self) -> None:
|
||||
"""Open the ring, recreating it from scratch on a geometry/header mismatch.
|
||||
|
||||
As the single owner of this ring we may safely discard a stale segment: a
|
||||
size or header mismatch means the existing segment belongs to an earlier,
|
||||
incompatible run, so we unlink it and create a fresh one instead of mapping
|
||||
an inconsistent layout.
|
||||
"""
|
||||
created = not self._path.exists()
|
||||
fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o660)
|
||||
self._file = os.fdopen(fd, "r+b", buffering=0)
|
||||
if created or self._path.stat().st_size != self._mapped_size:
|
||||
# Wrong-sized stale segment: drop it entirely and recreate, so the file
|
||||
# and any future mapping agree on geometry instead of being truncated
|
||||
# under a producer/consumer that still expects the old layout.
|
||||
if not created:
|
||||
logger.info("Ring %s: stale segment with wrong size, recreating", self._ring_name)
|
||||
self._file.close()
|
||||
self._unlink_if_present()
|
||||
fd = os.open(self._path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o660)
|
||||
self._file = os.fdopen(fd, "r+b", buffering=0)
|
||||
self._file.truncate(self._mapped_size)
|
||||
created = True
|
||||
|
||||
self._mmap = mmap.mmap(self._file.fileno(), self._mapped_size)
|
||||
if created:
|
||||
self._initialize_header()
|
||||
logger.debug(
|
||||
"Created SHM ring writer %s (capacity=%d, slot_size=%d bytes)",
|
||||
self._ring_name,
|
||||
self._capacity,
|
||||
self._slot_size_bytes,
|
||||
)
|
||||
return
|
||||
|
||||
# Size matched but the header geometry/magic does not: the owner recreates
|
||||
# rather than diverge. Unlink and reopen as a brand-new ring.
|
||||
if not self._header_matches():
|
||||
logger.warning("Ring %s: header/geometry mismatch on existing segment, recreating", self._ring_name)
|
||||
self._mmap.close()
|
||||
self._file.close()
|
||||
self._unlink_if_present()
|
||||
fd = os.open(self._path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o660)
|
||||
self._file = os.fdopen(fd, "r+b", buffering=0)
|
||||
self._file.truncate(self._mapped_size)
|
||||
self._mmap = mmap.mmap(self._file.fileno(), self._mapped_size)
|
||||
self._initialize_header()
|
||||
else:
|
||||
logger.debug("Reusing existing SHM ring writer %s", self._ring_name)
|
||||
|
||||
def _unlink_if_present(self) -> None:
|
||||
"""Remove the backing /dev/shm file if it exists (owner-only operation)."""
|
||||
try:
|
||||
self._path.unlink()
|
||||
with self._path.open("r+b", buffering=0) as file:
|
||||
if os.fstat(file.fileno()).st_size == self._mapped_size:
|
||||
mapping = mmap.mmap(file.fileno(), self._mapped_size)
|
||||
try:
|
||||
if self._header_matches(mapping):
|
||||
return mapping
|
||||
except BaseException:
|
||||
mapping.close()
|
||||
raise
|
||||
mapping.close()
|
||||
logger.warning("Ring %s: stale segment, recreating", self._ring_name)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
self._path.unlink(missing_ok=True)
|
||||
fd = os.open(self._path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o660)
|
||||
with os.fdopen(fd, "r+b", buffering=0) as file:
|
||||
file.truncate(self._mapped_size)
|
||||
mapping = mmap.mmap(file.fileno(), self._mapped_size)
|
||||
try:
|
||||
# New slots are already zero-filled. Publish the version last so readers
|
||||
# cannot accept a partially initialized header.
|
||||
mapping[:HEADER.size] = HEADER.pack(MAGIC, 0, self._capacity, self._slot_size_bytes, 0, 0, 0, 0)
|
||||
atomics.store(mapping, 8, VERSION, 4)
|
||||
except BaseException:
|
||||
mapping.close()
|
||||
raise
|
||||
return mapping
|
||||
|
||||
def _header_matches(self, mapping: mmap.mmap) -> bool:
|
||||
return (
|
||||
atomics.load(mapping, 8, 4) == VERSION
|
||||
and mapping[:8] == MAGIC
|
||||
and atomics.load(mapping, 12, 4) == self._capacity
|
||||
and atomics.load(mapping, 16, 4) == self._slot_size_bytes
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close mmap and file handle."""
|
||||
if self._mmap is not None:
|
||||
self._mmap.close()
|
||||
self._mmap = None
|
||||
if self._file is not None:
|
||||
self._file.close()
|
||||
self._file = None
|
||||
logger.debug("Closed SHM ring writer %s", self._ring_name)
|
||||
"""Close the mapping and its owned file descriptor; safe to repeat."""
|
||||
self._mmap.close()
|
||||
|
||||
def push(self, payload: bytes) -> bool:
|
||||
"""Push one payload with overwrite-oldest semantics on overflow."""
|
||||
@@ -137,11 +94,11 @@ class ShmRingWriter:
|
||||
self._write_u64(40, self._read_u64(40) + 1)
|
||||
|
||||
index = write_seq % self._capacity
|
||||
slot_offset = _HEADER_SIZE + index * (_SLOT_HEADER_SIZE + self._slot_size_bytes)
|
||||
payload_offset = slot_offset + _SLOT_HEADER_SIZE
|
||||
slot_offset = HEADER.size + index * (SLOT_HEADER_SIZE + self._slot_size_bytes)
|
||||
payload_offset = slot_offset + SLOT_HEADER_SIZE
|
||||
|
||||
# Invalidate the previous publication before changing its bytes. A peek or
|
||||
# a lapped consumer must not accept the old sequence during this overwrite.
|
||||
# Invalidate the previous publication before changing its bytes. A lapped
|
||||
# consumer must not accept the old sequence during this overwrite.
|
||||
self._write_u64(slot_offset + 8, 0)
|
||||
atomics.fence()
|
||||
self._write_u32(slot_offset, len(payload))
|
||||
@@ -151,50 +108,16 @@ class ShmRingWriter:
|
||||
self._write_u64(24, write_seq + 1)
|
||||
return True
|
||||
|
||||
@property
|
||||
def ring_name(self) -> str:
|
||||
"""Return the POSIX SHM ring name."""
|
||||
return self._ring_name
|
||||
|
||||
@property
|
||||
def slot_size_bytes(self) -> int:
|
||||
"""Return maximum payload size per slot."""
|
||||
return self._slot_size_bytes
|
||||
|
||||
def _initialize_header(self) -> None:
|
||||
# A new ftruncate-extended segment is already zero-filled by the kernel;
|
||||
# avoid allocating and copying another complete ring just to initialize it.
|
||||
self._mmap[:_HEADER_SIZE] = b"\x00" * _HEADER_SIZE
|
||||
self._mmap[:8] = _MAGIC
|
||||
self._write_u32(8, _VERSION)
|
||||
self._write_u32(12, self._capacity)
|
||||
self._write_u32(16, self._slot_size_bytes)
|
||||
self._write_u32(20, 0)
|
||||
self._write_u64(24, 0)
|
||||
self._write_u64(32, 0)
|
||||
self._write_u64(40, 0)
|
||||
|
||||
def _header_matches(self) -> bool:
|
||||
"""Return whether the existing segment's header matches this ring's geometry.
|
||||
|
||||
Used by the owner to decide whether a same-sized pre-existing segment can be
|
||||
reused as-is or must be unlinked and recreated.
|
||||
"""
|
||||
return (
|
||||
self._mmap[:8] == _MAGIC
|
||||
and self._read_u32(8) == _VERSION
|
||||
and self._read_u32(12) == self._capacity
|
||||
and self._read_u32(16) == self._slot_size_bytes
|
||||
)
|
||||
|
||||
def _read_u32(self, offset: int) -> int:
|
||||
return atomics.load(self._mmap, offset, 4)
|
||||
|
||||
def _read_u64(self, offset: int) -> int:
|
||||
return atomics.load(self._mmap, offset, 8)
|
||||
|
||||
def _write_u32(self, offset: int, value: int) -> None:
|
||||
atomics.store(self._mmap, offset, int(value), 4)
|
||||
atomics.store(self._mmap, offset, value, 4)
|
||||
|
||||
def _write_u64(self, offset: int, value: int) -> None:
|
||||
atomics.store(self._mmap, offset, int(value), 8)
|
||||
atomics.store(self._mmap, offset, value, 8)
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
"""Facade exports for shared-memory ring reader and decoders."""
|
||||
|
||||
from python_app.orchestration.shm.binary_cursor import ByteCursor
|
||||
from python_app.orchestration.shm.decoder import (
|
||||
PREPROC_MAGIC,
|
||||
RAW_MAGIC,
|
||||
RESULT_MAGIC,
|
||||
decode_result_collection,
|
||||
decode_trace_collection,
|
||||
)
|
||||
from python_app.orchestration.shm.ring_reader import ShmRingReader
|
||||
from python_app.orchestration.shm.ring_writer import ShmRingWriter
|
||||
|
||||
__all__ = [
|
||||
"ByteCursor",
|
||||
"PREPROC_MAGIC",
|
||||
"RAW_MAGIC",
|
||||
"RESULT_MAGIC",
|
||||
"ShmRingReader",
|
||||
"ShmRingWriter",
|
||||
"decode_result_collection",
|
||||
"decode_trace_collection",
|
||||
]
|
||||
Reference in New Issue
Block a user