awe/radar_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

projector version

Browse Source
This commit is contained in:
awe
2026-09-15 19:06:05 +03:00
parent 62c0266a6b
commit b3339d101c
33 changed files with 2605 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python_app/scripts/create_neutral_preprocess_sets.py
+90
View File
@@ -0,0 +1,90 @@
"""CLI to create neutral S21 preprocess sets for a profile (headless).
Mirrors the GUI's "Create neutral sets" button: builds calibration = 1+0j and
reference = 0 collections for every combo of the profile and saves them under
the profile's radar key, so the data_preprocessor becomes a bit-exact
pass-through. Needed for replay profiles whose recorded data is already
preprocessed (see run_config_examples/run_config_replay_vna.example.json).
Usage:
python -m python_app.scripts.create_neutral_preprocess_sets \
--config run_config_examples/run_config_replay_vna.example.json \
[--set-name neutral_replay] [--force]
"""
from __future__ import annotations
import argparse
import logging
from pathlib import Path
from python_app.models.run_config_model import RunConfigModel
from python_app.storage.npz_store import NpzStore, radar_key_from_config
from python_app.workflows.kamil_adc_neutral_preprocess import (
build_neutral_s21_sets,
supports_neutral_preprocess_sets,
)
logger = logging.getLogger(__name__)
PROJECT_ROOT = Path(__file__).resolve().parents[2]
def main() -> int:
"""Create and save the neutral S21 calibration/reference pair."""
parser = argparse.ArgumentParser(description="Create neutral S21 preprocess sets")
parser.add_argument("--config", required=True, type=Path, help="Path to a run_config profile")
parser.add_argument("--set-name", default="neutral_replay", help="Name for both saved sets")
parser.add_argument(
"--force", action="store_true", help="Overwrite the sets if they already exist"
)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
config = RunConfigModel.load_from_path(args.config)
config.ensure_combos()
if not supports_neutral_preprocess_sets(config):
raise RuntimeError(
"Neutral S21 sets are available only for kamil_adc and librevna_multi"
)
radar_key = radar_key_from_config(
model_name=config.radar.model,
serial=config.radar.serial,
sweep_start_hz=config.radar.sweep.start_hz,
sweep_stop_hz=config.radar.sweep.stop_hz,
sweep_points=config.radar.sweep.points,
ifbw_hz=config.radar.sweep.if_bandwidth_hz,
power_dbm=config.radar.sweep.power_dbm,
extra_serials=config.radar_key_extra_parts() or None,
)
store = NpzStore(PROJECT_ROOT / "python_app/data")
existing = [
kind
for kind in ("s21_calibration", "s21_reference")
if args.set_name in store.list_sets(kind, radar_key)
]
if existing and not args.force:
logger.info(
"Sets '%s' already exist for %s (radar_key=%s); nothing to do (use --force to overwrite).",
args.set_name, ", ".join(existing), radar_key,
)
return 0
calibration, reference = build_neutral_s21_sets(config)
store.save_set("s21_calibration", radar_key, args.set_name, calibration)
store.save_set("s21_reference", radar_key, args.set_name, reference)
logger.info(
"Neutral S21 sets saved: set=%s, radar_key=%s, combos=%d, points=%d",
args.set_name,
radar_key,
len(calibration.traces),
int(calibration.traces[0].frequency_hz.size),
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
python_app/scripts/replay_raw_producer.py
+245
View File
@@ -0,0 +1,245 @@
"""Raw producer that replays a recorded snapshot dataset (no hardware).
Selected by the supervisor instead of the model's normal acquisition producer
when ``radar.replay.dataset_dir`` is non-empty. The dataset is a directory of
snapshot folders named ``NNNN_idK_ns<timestamp>`` (the VNA_AlGo
preprocessed-capture layout), each holding ``i{input}_o{output}_s21.npy`` and
``i{input}_o{output}_freq.npy`` files for every run combo.
The recorded data is already background-subtracted, so the profile must pair
this producer with NEUTRAL preprocess sets (calibration = 1, reference = 0,
no notch) — then the C++ preprocessor passes the traces through bit-exactly
and the processors (e.g. sar3d) see the recorded scene.
Timeline: snapshot timestamps are re-anchored to this process's monotonic
clock, preserving the recorded inter-frame gaps. On loop, the seam advances by
the median inter-frame gap, so motion-model consumers (sar3d aperture offsets)
never see time jump backwards. ``collection_id`` increases forever.
"""
from __future__ import annotations
import argparse
import logging
from pathlib import Path
import re
import signal
import statistics
import threading
import time
import numpy as np
from python_app.logging_setup import coerce_level
from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData
from python_app.models.run_config_model import RunConfigModel
from python_app.orchestration.shm import ShmRingWriter
from python_app.storage.npz.serialize import RAW_MAGIC, serialize_trace_collection
logger = logging.getLogger(__name__)
_SNAPSHOT_NAME = re.compile(r"^\d+_id\d+_ns(\d+)$")
class ReplayDataset:
"""Recorded snapshots preloaded into memory, with their recorded timeline."""
def __init__(self, snapshots: list[list[TraceData]], timestamps_ns: list[int]) -> None:
self.snapshots = snapshots
self.timestamps_ns = timestamps_ns
gaps = [b - a for a, b in zip(timestamps_ns, timestamps_ns[1:])]
# Median gap doubles as the loop-seam step and the capture-window span;
# a single-snapshot dataset falls back to an arbitrary sane cadence.
self.median_gap_ns = int(statistics.median(gaps)) if gaps else 160_000_000
def __len__(self) -> int:
return len(self.snapshots)
def _load_dataset(dataset_dir: Path, combos: list[ComboKey], expected_points: int) -> ReplayDataset:
"""Preload every snapshot's traces; fail loudly on a malformed dataset."""
if not dataset_dir.is_dir():
raise RuntimeError(f"radar.replay.dataset_dir is not a directory: {dataset_dir}")
named: list[tuple[str, int]] = []
for entry in sorted(dataset_dir.iterdir()):
match = _SNAPSHOT_NAME.match(entry.name)
if entry.is_dir() and match:
named.append((entry.name, int(match.group(1))))
if not named:
raise RuntimeError(
f"No snapshot folders (NNNN_idK_ns<ts>) found in {dataset_dir}"
)
snapshots: list[list[TraceData]] = []
timestamps_ns: list[int] = []
for name, timestamp_ns in named:
folder = dataset_dir / name
traces: list[TraceData] = []
for combo in combos:
s21_path = folder / f"i{combo.input}_o{combo.output}_s21.npy"
freq_path = folder / f"i{combo.input}_o{combo.output}_freq.npy"
if not s21_path.is_file() or not freq_path.is_file():
raise RuntimeError(
f"Snapshot {name} is missing files for combo "
f"input={combo.input} output={combo.output}: expected "
f"{s21_path.name} and {freq_path.name}. Check run.combos vs the dataset."
)
s21 = np.load(s21_path).astype(np.complex64, copy=False)
freq = np.load(freq_path).astype(np.float32, copy=False)
if s21.size != freq.size:
raise RuntimeError(f"{s21_path}: s21 size {s21.size} != freq size {freq.size}")
if expected_points and s21.size != expected_points:
raise RuntimeError(
f"{s21_path}: {s21.size} points, but radar.sweep.points = "
f"{expected_points}. Align the profile's sweep with the dataset."
)
traces.append(
TraceData(
combo=combo,
frequency_hz=freq,
s11=np.zeros_like(s21),
s21=s21,
)
)
snapshots.append(traces)
timestamps_ns.append(timestamp_ns)
return ReplayDataset(snapshots, timestamps_ns)
def _build_collection(
dataset: ReplayDataset,
index: int,
collection_id: int,
frame_ns: int,
) -> SweepCollection:
"""Wrap one preloaded snapshot in a SweepCollection at the given frame time."""
span_ns = dataset.median_gap_ns
start_ns = frame_ns - span_ns // 2
end_ns = frame_ns + span_ns // 2
traces = []
for source in dataset.snapshots[index]:
traces.append(
TraceData(
combo=source.combo,
frequency_hz=source.frequency_hz,
s11=source.s11,
s21=source.s21,
capture_start_ns=start_ns,
capture_end_ns=end_ns,
)
)
return SweepCollection(
collection_id=collection_id,
monotonic_ns=frame_ns,
traces=traces,
capture_start_ns=start_ns,
capture_end_ns=end_ns,
)
def main() -> int:
"""Replay the configured dataset into the raw SHM rings until stopped."""
parser = argparse.ArgumentParser(description="Replay a recorded dataset to raw SHM rings")
parser.add_argument("--config", required=True, type=Path, help="Path to run_config.json")
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
stop_requested = threading.Event()
def request_stop(_signum: int, _frame: object) -> None:
stop_requested.set()
signal.signal(signal.SIGINT, request_stop)
signal.signal(signal.SIGTERM, request_stop)
config = RunConfigModel.load_from_path(args.config)
config.apply_device_model_constraints()
logging.getLogger("python_app").setLevel(coerce_level(config.logging.level))
replay = config.radar.replay
if not replay.dataset_dir:
raise RuntimeError("replay_raw_producer requires a non-empty radar.replay.dataset_dir")
rate_scale = float(replay.rate_scale) if replay.rate_scale > 0 else 1.0
combos = [ComboKey(input=combo.input, output=combo.output) for combo in config.combos]
dataset = _load_dataset(Path(replay.dataset_dir), combos, int(config.radar.sweep.points))
logger.info(
"Replay producer starting: dataset=%s, snapshots=%d, combos=%d, "
"median_gap=%.3f s, loop=%s, rate_scale=%.2f",
replay.dataset_dir, len(dataset), len(combos),
dataset.median_gap_ns / 1e9, replay.loop, rate_scale,
)
raw_writer = ShmRingWriter(
config.rings.raw.name,
config.rings.raw.capacity,
config.rings.raw.slot_size_bytes,
)
raw_tap_writer = ShmRingWriter(
config.rings.raw_tap.name,
config.rings.raw_tap.capacity,
config.rings.raw_tap.slot_size_bytes,
)
try:
base_ns = time.monotonic_ns()
wall_start_s = time.monotonic()
first_ts = dataset.timestamps_ns[0]
cycle_span_ns = dataset.timestamps_ns[-1] - first_ts + dataset.median_gap_ns
collection_id = 1
publish_failures = 0
cycle = 0
while not stop_requested.is_set():
for index in range(len(dataset)):
if stop_requested.is_set():
break
# Monotonic replay timeline: recorded offsets re-anchored at
# base_ns, each loop shifted by one full cycle span.
frame_ns = (
base_ns
+ (dataset.timestamps_ns[index] - first_ts)
+ cycle * cycle_span_ns
)
# Publish on the recorded cadence (scaled); waiting via the stop
# event keeps shutdown prompt.
deadline_s = wall_start_s + (frame_ns - base_ns) / 1e9 / rate_scale
delay_s = deadline_s - time.monotonic()
if delay_s > 0 and stop_requested.wait(delay_s):
break
collection = _build_collection(dataset, index, collection_id, frame_ns)
payload = serialize_trace_collection(collection, RAW_MAGIC)
if raw_writer.push(payload):
raw_tap_writer.push(payload) # best-effort GUI tap; never fatal
else:
publish_failures += 1
if publish_failures == 1 or publish_failures % 100 == 0:
logger.error(
"Replay payload %d B exceeds ring slot %d B; dropping collection %d (drops=%d)",
len(payload), raw_writer.slot_size_bytes, collection_id, publish_failures,
)
if collection_id == 1 or collection_id % 50 == 0:
logger.info(
"replay collection %d published (snapshot %d/%d, cycle %d)",
collection_id, index + 1, len(dataset), cycle,
)
collection_id += 1
if not config.runtime.continuous:
stop_requested.set()
break
if not replay.loop:
break
cycle += 1
finally:
raw_tap_writer.close()
raw_writer.close()
logger.info("replay raw producer stopped")
return 0
if __name__ == "__main__":
raise SystemExit(main())
python_app/scripts/replay_smoke_run.py
+246
View File
@@ -0,0 +1,246 @@
"""Headless smoke: replay the recorded VNA_AlGo dataset through the full pipeline.
Loads the replay profile, creates neutral preprocess sets, launches the whole
pipeline (replay_raw_producer -> data_preprocessor -> data_processor) via the
ProcessSupervisor with unique ring names, forces the sar3d processor through
processing_live.json, and asserts that:
- sar3d projection payloads arrive with a positive peak;
- collection ids increase monotonically (including across a dataset loop);
- the preprocessor is transparent: a preprocessed_tap trace equals the
recorded snapshot's s21 within float32 rounding.
Usage:
python -m python_app.scripts.replay_smoke_run \
[--config run_config_examples/run_config_replay_vna.example.json] \
[--duration 20.0]
"""
from __future__ import annotations
import argparse
import json
import logging
import os
from pathlib import Path
import time
import numpy as np
from python_app.models.run_config_model import RunConfigModel
from python_app.orchestration.config_writer import ConfigWriter
from python_app.orchestration.live_processing_config import (
ProcessingLiveConfig,
ProcessingLiveConfigWriter,
)
from python_app.orchestration.process_supervisor import ProcessSupervisor
from python_app.orchestration.shm import ShmRingReader
from python_app.orchestration.shm.decoder import PREPROC_MAGIC, decode_trace_collection
from python_app.storage.npz_store import NpzStore, radar_key_from_config
from python_app.workflows.kamil_adc_neutral_preprocess import build_neutral_s21_sets
logger = logging.getLogger(__name__)
PROJECT_ROOT = Path(__file__).resolve().parents[2]
def _make_unique_ring_name(prefix: str) -> str:
"""Build unique POSIX SHM ring name."""
return f"/{prefix}_{os.getpid()}_{time.monotonic_ns()}"
def _sar3d_live_config(profile_path: Path) -> ProcessingLiveConfig:
"""Build a sar3d live config from the profile's gui.processing.sar3d section.
RunConfigModel does not carry the GUI section (it belongs to the GUI profile
layer), so the sar3d defaults are read straight from the profile JSON.
"""
payload = json.loads(profile_path.read_text(encoding="utf-8"))
sar3d = (
payload.get("gui", {}).get("processing", {}).get("sar3d", {})
if isinstance(payload, dict)
else {}
)
def _positions(text: str) -> list[int]:
return [int(value) for value in str(text).split(",") if value.strip()]
live = ProcessingLiveConfig(processor_mode="sar3d", ignore_socket_speed=True)
if not isinstance(sar3d, dict) or not sar3d:
return live
live.gpr_input_positions = _positions(sar3d.get("input_positions", ""))
live.gpr_output_positions = _positions(sar3d.get("output_positions", ""))
live.gpr_min_depth_m = float(sar3d.get("min_depth_m", live.gpr_min_depth_m))
live.gpr_max_depth_m = float(sar3d.get("max_depth_m", live.gpr_max_depth_m))
live.gpr_start_freq_mhz = float(sar3d.get("start_freq_mhz", live.gpr_start_freq_mhz))
live.gpr_stop_freq_mhz = float(sar3d.get("stop_freq_mhz", live.gpr_stop_freq_mhz))
live.gpr_speed_m_s = float(sar3d.get("speed_m_s", live.gpr_speed_m_s))
live.gpr_look_angle_deg = float(sar3d.get("look_angle_deg", live.gpr_look_angle_deg))
live.gpr_direction_sign = float(sar3d.get("direction_sign", live.gpr_direction_sign))
live.gpr_apply_freq_phase_correction = bool(
sar3d.get("apply_freq_phase_correction", live.gpr_apply_freq_phase_correction)
)
live.sar3d_frame_radius = int(sar3d.get("frame_radius", live.sar3d_frame_radius))
live.sar3d_edge_frame_weight = float(
sar3d.get("edge_frame_weight", live.sar3d_edge_frame_weight)
)
live.sar3d_pitch_deg = float(sar3d.get("pitch_deg", live.sar3d_pitch_deg))
live.sar3d_center_depth_m = float(sar3d.get("center_depth_m", live.sar3d_center_depth_m))
live.sar3d_half_width_x_m = float(sar3d.get("half_width_x_m", live.sar3d_half_width_x_m))
live.sar3d_half_width_y_m = float(sar3d.get("half_width_y_m", live.sar3d_half_width_y_m))
live.sar3d_half_width_z_m = float(sar3d.get("half_width_z_m", live.sar3d_half_width_z_m))
live.sar3d_grid_nx = int(sar3d.get("grid_nx", live.sar3d_grid_nx))
live.sar3d_grid_ny = int(sar3d.get("grid_ny", live.sar3d_grid_ny))
live.sar3d_grid_nz = int(sar3d.get("grid_nz", live.sar3d_grid_nz))
live.sar3d_use_compensation = bool(sar3d.get("use_compensation", live.sar3d_use_compensation))
live.sar3d_normalize_pairs = bool(sar3d.get("normalize_pairs", live.sar3d_normalize_pairs))
return live
def _check_preprocessor_transparency(
tap_reader: ShmRingReader, dataset_dir: Path
) -> bool:
"""Compare one preprocessed trace against the recorded snapshot's s21."""
payload = tap_reader.peek_latest_payload()
if payload is None:
logger.warning("No preprocessed_tap payload available for transparency check")
return False
preprocessed = decode_trace_collection(payload, PREPROC_MAGIC)
# Any snapshot works: neutral preprocessing is time-invariant, so match the
# trace against every recorded snapshot and accept the best one.
trace = preprocessed.traces[0]
combo_file = f"i{trace.combo.input}_o{trace.combo.output}_s21.npy"
best_delta = np.inf
for folder in sorted(p for p in dataset_dir.iterdir() if p.name[:1].isdigit()):
recorded = np.load(folder / combo_file).astype(np.complex64)
if recorded.size != np.asarray(trace.s21).size:
continue
delta = float(np.max(np.abs(np.asarray(trace.s21) - recorded)))
best_delta = min(best_delta, delta)
if best_delta == 0.0:
break
logger.info("Preprocessor transparency: max|delta| vs best snapshot = %.3e", best_delta)
return best_delta < 1e-6
def main() -> int:
"""Run the replay pipeline smoke and report pass/fail."""
parser = argparse.ArgumentParser(description="Headless replay pipeline smoke")
parser.add_argument(
"--config",
type=Path,
default=PROJECT_ROOT / "run_config_examples/run_config_replay_vna.example.json",
)
parser.add_argument("--duration", type=float, default=20.0)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
config = RunConfigModel.load_from_path(args.config)
config.ensure_combos()
dataset_dir = Path(config.radar.replay.dataset_dir)
if not dataset_dir.is_dir():
raise RuntimeError(f"Replay dataset not found: {dataset_dir}")
config.rings.raw.name = _make_unique_ring_name("radar_raw_replay_smoke")
config.rings.raw_tap.name = _make_unique_ring_name("radar_raw_tap_replay_smoke")
config.rings.preprocessed.name = _make_unique_ring_name("radar_preprocessed_replay_smoke")
config.rings.preprocessed_tap.name = _make_unique_ring_name("radar_preprocessed_tap_replay_smoke")
config.rings.results.name = _make_unique_ring_name("radar_results_replay_smoke")
# Private live-config path so the smoke never clobbers the GUI's file.
live_path = PROJECT_ROOT / "python_app/runtime/processing_live_replay_smoke.json"
config.runtime.processing_live_config_path = str(
live_path.relative_to(PROJECT_ROOT)
)
radar_key = radar_key_from_config(
model_name=config.radar.model,
serial=config.radar.serial,
sweep_start_hz=config.radar.sweep.start_hz,
sweep_stop_hz=config.radar.sweep.stop_hz,
sweep_points=config.radar.sweep.points,
ifbw_hz=config.radar.sweep.if_bandwidth_hz,
power_dbm=config.radar.sweep.power_dbm,
extra_serials=config.radar_key_extra_parts() or None,
)
store = NpzStore(PROJECT_ROOT / "python_app/data")
set_name = config.preprocess.s21.calibration.set_name or "neutral_replay"
if set_name not in store.list_sets("s21_calibration", radar_key):
calibration, reference = build_neutral_s21_sets(config)
store.save_set("s21_calibration", radar_key, set_name, calibration)
store.save_set("s21_reference", radar_key, set_name, reference)
logger.info("Created neutral sets '%s' (radar_key=%s)", set_name, radar_key)
config.preprocess.s21.calibration.set_name = set_name
config.preprocess.s21.reference.set_name = set_name
ProcessingLiveConfigWriter(live_path).write(_sar3d_live_config(args.config))
config_writer = ConfigWriter(PROJECT_ROOT / "python_app/runtime")
config_writer.prepare_preprocess_bundles(store, radar_key, config)
config_path = config_writer.write(
config, PROJECT_ROOT / "python_app/runtime/run_config_replay_smoke.json"
)
supervisor = ProcessSupervisor(PROJECT_ROOT)
result_reader: ShmRingReader | None = None
tap_reader: ShmRingReader | None = None
failures: list[str] = []
try:
supervisor.start(config_path)
result_reader = ShmRingReader(config.rings.results.name, open_timeout_s=15.0)
tap_reader = ShmRingReader(config.rings.preprocessed_tap.name, open_timeout_s=15.0)
deadline = time.monotonic() + args.duration
received = 0
sar3d_results = 0
last_peak = 0.0
last_id = 0
ids_monotonic = True
while time.monotonic() < deadline:
result = result_reader.pop_result_collection()
if result is None:
time.sleep(0.02)
continue
received += 1
if last_id and result.collection_id <= last_id:
ids_monotonic = False
last_id = result.collection_id
names = {p.processing_name: p for p in result.collection_payloads}
if "sar3d_proj_xz" in names and "sar3d_proj_xy" in names:
sar3d_results += 1
peak_payload = names.get("sar3d_peak")
if peak_payload is not None:
last_peak = float(peak_payload.scalar_value)
logger.info(
"Results: total=%d, with sar3d projections=%d, last collection_id=%d, last peak=%.4e",
received, sar3d_results, last_id, last_peak,
)
if received == 0:
failures.append("no result collections received")
if sar3d_results == 0:
failures.append("no sar3d projection payloads received")
if not (last_peak > 0.0):
failures.append(f"sar3d peak is not positive: {last_peak}")
if not ids_monotonic:
failures.append("collection ids were not monotonically increasing")
if not _check_preprocessor_transparency(tap_reader, dataset_dir):
failures.append("preprocessor is not transparent vs recorded snapshots")
finally:
supervisor.stop_all()
if result_reader is not None:
result_reader.close()
if tap_reader is not None:
tap_reader.close()
if failures:
for failure in failures:
logger.error("FAIL: %s", failure)
return 1
logger.info("REPLAY SMOKE OK")
return 0
if __name__ == "__main__":
raise SystemExit(main())