Rework radar system

This commit is contained in:
Ayzen
2026-10-06 12:32:26 +03:00
parent 552cdc688a
commit 5bb86b7165
299 changed files with 13537 additions and 27763 deletions
@@ -0,0 +1,580 @@
"""Select preprocessing assets and own paused hardware capture sessions."""
from __future__ import annotations
from collections.abc import Callable
from contextlib import nullcontext
from copy import deepcopy
from dataclasses import asdict
import logging
from PyQt6.QtCore import QTimer
from PyQt6.QtWidgets import QWidget
from python_app.gui.controllers.object_calibration_capture import ObjectSweepCapture
from python_app.gui.controllers.object_calibration_controller import ObjectCalibrationController
from python_app.gui.controllers.pipeline_controller import PipelineController
from python_app.gui.controllers.plot_controller import PlotController
from python_app.gui.controllers.sections.views import DataView, PrimaryView
from python_app.gui.object_calibration_dialog import ObjectCalibrationDialog
from python_app.gui.preprocess_dialog import PreprocessDialog
from python_app.gui.runtime.pipeline_pause import PipelinePause
from python_app.gui.trace_png_export import export_trace_png
from python_app.models.gui_profile_schema import GuiPreprocessDialogStateModel
from python_app.models.run_config_schema import ComboModel, GprModel, RunConfigModel
from python_app.orchestration.preprocess_assets import (
PREPROCESS_ASSET_NAMES,
preprocess_asset_display_name, preprocess_asset_model,
)
from python_app.storage.npz.store import NpzStore
from python_app.workflows.neutral_preprocess import build_neutral_s21_sets, supports_neutral_preprocess_sets
from python_app.workflows.object_calibration import capture_provenance, geometry_signature
from python_app.workflows.radar_config_variants import RadarConfigVariant, scan_radar_config_variants
from python_app.workflows.reference_workflow import capture_reference_set
from python_app.workflows.sequential_capture_workflow import CaptureBatch, SavedCaptureSet, SequentialCaptureSession
logger = logging.getLogger(__name__)
TMP_REFERENCE_SET_NAME = "tmp_reference"
class PreprocessController:
def __init__(
self, parent: QWidget, config: RunConfigModel, state: GuiPreprocessDialogStateModel,
store: NpzStore, pipeline: PipelineController, plots: PlotController, *,
build_config: Callable[[], RunConfigModel], current_radar_key: Callable[[], str],
geometry: Callable[[], GprModel], summary: Callable[[dict[str, str]], None],
error: Callable[[str, Exception], None],
) -> None:
self._parent, self._store, self._pipeline, self._plots = parent, store, pipeline, plots
self._build_config, self._current_radar_key, self._geometry = build_config, current_radar_key, geometry
self._summary, self._error = summary, error
self._defaults, self._settings = config, deepcopy(state)
self.selected_sets = {key: preprocess_asset_model(config, key).set_name for key in PREPROCESS_ASSET_NAMES}
self.radar_key = config.radar_key()
self.dialog: PreprocessDialog | None = None
self.capture_session: SequentialCaptureSession | None = None
self.capture_busy = False
self._tmp_reference_busy = False
self._capture_pause: PipelinePause | None = None
self._radar_variants: list[RadarConfigVariant] = []
self.object_calibration = ObjectCalibrationController(
parent, store, build_config, self._capture_object, self._select_object, error,
)
def bind_actions(self, primary: PrimaryView, data: DataView) -> None:
primary.preprocess.clicked.connect(self.open_panel)
data.capture_tmp.clicked.connect(self.capture_tmp_reference)
@property
def object_capture(self) -> ObjectSweepCapture | None:
return self.object_calibration.capture
def settings(self) -> GuiPreprocessDialogStateModel:
if self.dialog is not None:
self._settings = GuiPreprocessDialogStateModel(
set_name=self.dialog.set_name(), radar_config_dir=self.dialog.radar_config_dir(),
use_all_radar_configs=self.dialog.use_all_radar_configs(),
median_sweep_count=self.dialog.median_sweep_count(),
)
return deepcopy(self._settings)
def apply_profile(self, config: RunConfigModel, state: GuiPreprocessDialogStateModel) -> None:
self._defaults, self._settings = config, deepcopy(state)
self.selected_sets.update({key: preprocess_asset_model(config, key).set_name for key in PREPROCESS_ASSET_NAMES})
self.radar_key = config.radar_key()
self._summary(self.selected_sets)
if self.dialog is not None:
self._apply_dialog_settings()
def _apply_dialog_settings(self) -> None:
assert self.dialog is not None
self.dialog.apply_settings(self._settings, self.selected_sets,
neutral_visible=supports_neutral_preprocess_sets(self._defaults))
def ensure_dialog(self) -> PreprocessDialog:
if self.dialog is None:
self.dialog = PreprocessDialog(self._parent)
self._apply_dialog_settings()
self.dialog.refresh_requested.connect(self._refresh_requested)
self.dialog.selection_changed.connect(self._selection_changed)
self.dialog.radar_config_dir_changed.connect(self._refresh_requested)
self.dialog.multi_radar_option_changed.connect(self._refresh_requested)
self.dialog.start_sequence_requested.connect(self.start_sequence)
self.dialog.capture_next_requested.connect(self.capture_next)
self.dialog.capture_all_requested.connect(self.capture_all)
self.dialog.undo_last_requested.connect(self.undo)
self.dialog.finalize_sequence_requested.connect(self.finalize)
self.dialog.abort_sequence_requested.connect(self.abort)
self.dialog.create_neutral_sets_requested.connect(self.create_neutral_sets)
self.dialog.create_object_calibration_requested.connect(
lambda: self.object_calibration.open_creation(self.settings().set_name))
self._update_capture_state()
return self.dialog
def open_panel(self) -> None:
try:
dialog = self.ensure_dialog()
self.refresh_sets()
self._update_capture_state()
dialog.showMaximized()
dialog.raise_()
dialog.activateWindow()
except Exception as exc:
self._error("Failed to open preprocessing panel", exc)
def busy_reason(self) -> str | None:
if self._pipeline.stopping:
return "Wait for pipeline shutdown to finish"
if self.object_capture is not None:
return "Wait for object-calibration capture to finish"
if self.capture_session is not None:
return "Finish the active capture sequence first"
if self.capture_busy or self._tmp_reference_busy:
return "Wait for preprocessing capture to finish"
return None
def data_action_allowed(self, *, deletion: bool = False) -> bool:
if self._pipeline.closing:
return False
if self._pipeline.stopping:
self._error("Cannot perform data action", RuntimeError(self.busy_reason()))
return False
if self.object_capture is not None:
self._error("Wait for object-calibration capture to finish", RuntimeError("Calibration capture is active"))
return False
if deletion and self.capture_session is not None:
self._error("Cannot modify runtime history during active capture sequence", RuntimeError(self.capture_state_details()))
return False
return True
def _can_capture(self) -> bool:
if self._pipeline.closing:
return False
reason = self.busy_reason()
if reason:
self._error(reason, RuntimeError(self.capture_state_details()))
return False
return True
def capture_state_details(self) -> str:
if self.capture_session is None:
return "Capture session: none"
state = self.capture_session.state()
lines = ["Capture session:", f"- kind={state.kind}", f"- progress={state.captured_count}/{state.total_count}"]
if state.current_combo is not None:
lines.append(f"- current_combo=input={state.current_combo.input}, output={state.current_combo.output}")
return "\n".join(lines)
def reset_selection(self) -> None:
try:
radar_key = self._current_radar_key()
except Exception:
logger.debug("Skipping preprocess-selection reset; radar key unavailable", exc_info=True)
return
if radar_key == self.radar_key:
return
previous_key, self.radar_key = self.radar_key, radar_key
cleared = {key: value for key, value in self.selected_sets.items() if value}
self.selected_sets.update(dict.fromkeys(PREPROCESS_ASSET_NAMES, ""))
self._summary(self.selected_sets)
if self.dialog is not None:
self.dialog.set_selected_sets(self.selected_sets, emit_signal=False)
self._refresh_requested()
if cleared:
logger.info("Preprocess selection reset: %s -> %s; cleared %s", previous_key, radar_key, cleared)
def _selection_changed(self) -> None:
previous = dict(self.selected_sets)
self.selected_sets.update(self.ensure_dialog().selection_snapshot())
self._summary(self.selected_sets)
changes = [f"{preprocess_asset_display_name(key)}: {previous.get(key) or '<not selected>'} -> "
f"{self.selected_sets.get(key) or '<not selected>'}" for key in PREPROCESS_ASSET_NAMES
if previous.get(key) != self.selected_sets.get(key)]
if changes:
logger.info("Preprocess selection changed: %s", "; ".join(changes))
def refresh_sets(self) -> None:
self.radar_key = self._current_radar_key()
dialog = self.ensure_dialog()
self.settings()
available = {key: self._store.list_sets(key, self.radar_key)
for key in PREPROCESS_ASSET_NAMES}
dialog.set_available_sets(available)
self._refresh_variants()
for key, names in available.items():
selected = self.selected_sets.get(key, "")
if selected and selected not in names:
logger.warning("%s set %s is unavailable for radar key %s", key, selected, self.radar_key)
dialog.set_selected_sets(self.selected_sets)
dialog.set_neutral_sets_visible(supports_neutral_preprocess_sets(self._defaults))
self._summary(self.selected_sets)
logger.info("Preprocess set lists refreshed: radar_key=%s, counts=%s", self.radar_key,
{key: len(names) for key, names in available.items()})
def _refresh_requested(self) -> None:
try:
self.refresh_sets()
except Exception as exc:
self._error("Failed to refresh preprocessing sets", exc)
def _refresh_variants(self) -> None:
self._radar_variants, summary = scan_radar_config_variants(
self.settings().radar_config_dir, base_config=self._build_config())
if self.dialog is not None:
values = asdict(summary)
values.pop("issues")
self.dialog.set_radar_config_summary(**values)
if summary.issues:
logger.warning("Some radar config JSON files were skipped:\n%s", "\n".join(summary.issues))
def capture_tmp_reference(self) -> None:
if self._tmp_reference_busy:
logger.info("Tmp reference capture already in progress; ignoring duplicate request")
return
if not self._can_capture():
return
self._tmp_reference_busy = True
try:
with self._pipeline.pause():
try:
config = self._build_config()
radar_key, collection = capture_reference_set(
config, TMP_REFERENCE_SET_NAME, self._store,
median_sweep_count=self.settings().median_sweep_count,
)
self.selected_sets["s21_reference"] = TMP_REFERENCE_SET_NAME
self.radar_key = radar_key
# Reference reload must keep the processor and its TCP locator alive.
self._pipeline.preprocessing_changed()
self._summary(self.selected_sets)
if collection.traces:
self._plots.draw_trace(collection.traces[-1], "Tmp S21 Reference last trace")
if self.dialog is not None:
self.refresh_sets()
self.dialog.set_status(f"S21 Reference set saved: {TMP_REFERENCE_SET_NAME} ({len(collection.traces)} traces)")
logger.info("Tmp S21 Reference captured: set=%s, key=%s, traces=%d; runtime history reset",
TMP_REFERENCE_SET_NAME, radar_key, len(collection.traces))
finally:
self._tmp_reference_busy = False
except InterruptedError:
self._pipeline.operation_cancelled()
except Exception as exc:
self._error("Failed to capture tmp reference", exc)
finally:
self._tmp_reference_busy = False
def create_neutral_sets(self) -> None:
if not self._can_capture():
return
try:
dialog = self.ensure_dialog()
name, config = dialog.set_name(), self._build_config()
if not name:
raise ValueError("Set name is required")
if not supports_neutral_preprocess_sets(config):
raise ValueError("Neutral S21 sets are not supported for this radar configuration")
radar_key = config.radar_key()
for kind in ("s21_calibration", "s21_reference"):
if name in self._store.list_sets(kind, radar_key):
raise FileExistsError(f"Set '{name}' already exists for {preprocess_asset_display_name(kind)}")
with self._pipeline.pause():
calibration, reference = build_neutral_s21_sets(config)
self._store.save_sets(radar_key, name, {"s21_calibration": calibration, "s21_reference": reference})
self.selected_sets.update(s21_calibration=name, s21_reference=name)
self.radar_key = radar_key
self._pipeline.preprocessing_changed(restart_processor=True)
self.refresh_sets()
dialog.set_status(f"Neutral S21 sets saved: {name} ({len(calibration.traces)} combos, "
f"{calibration.traces[0].frequency_hz.size} points)")
logger.info("Neutral S21 sets saved: set=%s, radar_key=%s", name, radar_key)
except InterruptedError:
self._pipeline.operation_cancelled()
except Exception as exc:
self._error("Failed to create neutral S21 sets", exc)
def start_sequence(self, kind: str) -> None:
if not self._can_capture():
return
pause = None
try:
dialog = self.ensure_dialog()
name = dialog.set_name()
if not name:
raise ValueError("Set name is required")
pause = self._pipeline.pause()
self._capture_pause = pause
config, settings = self._build_config(), self.settings()
if settings.use_all_radar_configs:
self._refresh_variants()
if not self._radar_variants:
raise ValueError("No valid radar config variants were found in the selected directory")
keys = [variant.radar_key for variant in self._radar_variants]
else:
keys = [config.radar_key()]
duplicates = [key for key in keys if name in self._store.list_sets(kind, key)]
if duplicates:
raise FileExistsError(f"Set '{name}' already exists for {kind}: {', '.join(duplicates)}")
session = SequentialCaptureSession(
config, kind, name, median_sweep_count=settings.median_sweep_count,
radar_variants=self._radar_variants if settings.use_all_radar_configs else None,
)
session.open()
self.capture_session = session
dialog.clear_capture_log()
dialog.reset_preview()
dialog.set_status(f"{preprocess_asset_display_name(kind)} sequence started")
self._plots.trace.clear()
self._update_capture_state()
logger.info("%s sequence started: set=%s, combos=%d, radar_variants=%d, median_sweeps=%d",
kind, name, session.state().total_count, len(keys), settings.median_sweep_count)
except InterruptedError:
self._pipeline.operation_cancelled()
self._release_pause()
except Exception as exc:
if pause is not None:
self._cleanup_session()
if pause is not None:
self._release_pause()
self._error(f"Failed to start {kind} sequence", exc)
def _begin_capture(self) -> bool:
if self._pipeline.closing or self.capture_busy:
logger.info("Preprocess combo capture already in progress; ignoring duplicate request")
return False
self.capture_busy = True
self._update_capture_state()
return True
def capture_next(self) -> None:
self._capture_remaining(all_remaining=False)
def capture_all(self) -> None:
self._capture_remaining(all_remaining=True)
def _capture_remaining(self, *, all_remaining: bool) -> None:
session = self.capture_session
if session is None or session.is_complete():
self._error("Cannot capture preprocess data", RuntimeError("No incomplete capture sequence"))
return
if not self._begin_capture():
return
sequence = self._pipeline.stop_generation
if all_remaining:
self.ensure_dialog().set_status(f"{preprocess_asset_display_name(session.kind)} batch capture started")
try:
while self.capture_session is session and self._pipeline.can_resume(sequence):
try:
captured = session.capture_current_combo()
except Exception as exc:
state = session.state()
self._error("Failed to capture combo; previous captures kept, retry when ready", exc)
self.ensure_dialog().set_status(
f"{preprocess_asset_display_name(session.kind)} capture failed: "
f"{state.captured_count}/{state.total_count} kept, ready to retry")
return
self._record_capture(session, captured)
if not all_remaining or session.is_complete():
break
self._pipeline.pump_events(0.01)
if all_remaining and self.capture_session is session and not session.is_complete():
self.ensure_dialog().set_status(
f"{preprocess_asset_display_name(session.kind)} batch capture paused; previous captures kept")
finally:
self.capture_busy = False
# Queued clicks must reach disabled buttons before they are re-enabled.
QTimer.singleShot(0, self._update_capture_state)
def _record_capture(self, session: SequentialCaptureSession, captured: CaptureBatch) -> None:
dialog, state = self.ensure_dialog(), session.state()
display = preprocess_asset_display_name(session.kind)
trace = captured.display_trace
details = f" | radar_configs={len(captured.traces)}" if state.variant_count > 1 else ""
dialog.append_capture_log_entry(kind=display, captured_count=state.captured_count, total_count=state.total_count,
input_pos=trace.combo.input, output_pos=trace.combo.output, extra_details=details)
dialog.draw_last_trace(trace, title=f"{display} captured")
self._plots.draw_trace(trace, f"{display} last trace")
logger.info("%s capture: %d/%d | input=%d output=%d%s", display, state.captured_count,
state.total_count, trace.combo.input, trace.combo.output, details)
self._update_capture_state()
if session.is_complete():
dialog.set_status(f"{display} sequence complete. Review captures or save the set.")
def undo(self) -> None:
session = self.capture_session
if session is None:
self._error("No active capture sequence", RuntimeError(self.capture_state_details()))
return
try:
session.undo_last_capture()
dialog, state = self.ensure_dialog(), session.state()
display = preprocess_asset_display_name(session.kind)
captures = session.captured_batches()
extra = f" | radar_configs={state.variant_count}" if state.variant_count > 1 else ""
dialog.set_capture_log_entries([
f"{display}: {index}/{state.total_count} | input={item.combo.input} output={item.combo.output}{extra}"
for index, item in enumerate(captures, start=1)])
trace = session.last_captured_trace()
if trace is None:
dialog.reset_preview()
self._plots.trace.clear()
else:
dialog.draw_last_trace(trace, title=f"{display} last trace")
self._plots.draw_trace(trace, f"{display} last trace")
dialog.set_status(f"{display} last capture removed. Ready to recapture.")
self._update_capture_state()
logger.info("%s undo last capture; remaining=%d/%d", display, state.captured_count, state.total_count)
except Exception as exc:
self._error("Failed to undo last preprocess capture", exc)
def finalize(self) -> None:
session = self.capture_session
if session is None or not session.is_complete():
self._error("Capture sequence is not complete", RuntimeError(self.capture_state_details()))
return
try:
try:
dialog = self.ensure_dialog()
kind, name = session.kind, session.set_name
saved = session.finalize(self._store)
status = (f"{len(saved)} radar variants" if len(saved) > 1
else f"{len(saved[0].collection.traces)} traces")
if self.settings().use_all_radar_configs:
self._export_previews(session, saved)
self._cleanup_session()
self.selected_sets[kind] = name
self._summary(self.selected_sets)
dialog.set_status(f"{preprocess_asset_display_name(kind)} set saved: {name} ({status})")
logger.info("%s sequence saved: set=%s (%s)", kind, name, status)
self.refresh_sets()
finally:
if self.capture_session is None:
self._release_pause()
except Exception as exc:
self._error("Failed to save preprocess set", exc)
def _export_previews(self, session: SequentialCaptureSession, saved: list[SavedCaptureSet]) -> None:
for item in saved:
for trace in item.collection.traces:
try:
path = self._store.preview_png_dir(session.kind, session.set_name, item.radar_key) / f"i{trace.combo.input}_o{trace.combo.output}.png"
export_trace_png(trace, path, title=f"{preprocess_asset_display_name(session.kind)} | {item.display_name} | "
f"input={trace.combo.input} output={trace.combo.output}")
except Exception:
# Data are already saved; rendering failure must not abort completion.
logger.warning("Failed to save preview PNG for %s", item.display_name, exc_info=True)
def abort(self, *, resume_pipeline: bool = True) -> None:
if self.capture_session is None:
return
try:
try:
kind = self.capture_session.kind
self._cleanup_session()
self.ensure_dialog().set_status(f"{preprocess_asset_display_name(kind)} sequence aborted")
logger.info("%s sequence aborted", kind)
finally:
self._release_pause(resume=resume_pipeline)
except Exception as exc:
self._error("Failed to abort capture sequence", exc)
def _cleanup_session(self) -> None:
session, self.capture_session = self.capture_session, None
try:
if session is not None:
session.close()
except Exception:
self._release_pause(resume=False)
raise
finally:
self._update_capture_state()
def _release_pause(self, *, resume: bool = True) -> None:
pause, self._capture_pause = self._capture_pause, None
if pause is not None:
pause.finish(resume=resume)
def _update_capture_state(self) -> None:
# A queued post-capture callback can outlive the window and its Qt children.
if self._pipeline.closing or self.dialog is None:
return
state = self.capture_session.state() if self.capture_session is not None else None
combo = state.current_combo if state else None
self.dialog.set_capture_state(
kind=state.kind if state else None, captured_count=state.captured_count if state else 0,
total_count=state.total_count if state else 0, next_input=combo.input if combo else None,
next_output=combo.output if combo else None, can_undo=state.can_undo if state else False,
can_finalize=state.is_complete if state else False,
can_capture_all=bool(state and state.supports_batch_capture and not state.is_complete and combo),
variant_count=state.variant_count if state else 1, actions_enabled=not self.capture_busy,
)
def _capture_object(self, dialog: ObjectCalibrationDialog) -> ObjectSweepCapture:
reason = self.busy_reason()
if reason or self._pipeline.closing or self._pipeline.stopping:
raise RuntimeError(reason or "Pipeline is stopping")
if self._pipeline.single_capture_active or self._pipeline.recording.active:
raise RuntimeError("Finish the single capture or disk recording before taking calibration data")
config = self._build_config()
if not config.rings.preprocessed_tap.name:
raise ValueError("Object calibration requires rings.preprocessed_tap; enable it in the run profile")
if config.radar.replay.dataset_dir.strip():
raise ValueError("Object-calibration capture requires live acquisition; disable archive replay")
config.preprocess.s21.object_calibration.set_name = ""
config.preprocess.s21.object_calibration.bundle_path = ""
tx, rx = sorted({entry.output_pos for entry in config.gpr.tx_geometry}), sorted({entry.input_pos for entry in config.gpr.rx_geometry})
if not tx or not rx:
raise ValueError("Configure Tx/Rx antenna positions before taking calibration data")
config.combos = [ComboModel(input=i, output=o) for o in tx for i in rx]
radar_key = config.radar_key()
through = self._store.load_set("s21_calibration", radar_key, config.preprocess.s21.calibration.set_name)
provenance = capture_provenance(config.to_dict(), radar_key, through)
return ObjectSweepCapture(self._pipeline, dialog, config, self.settings().median_sweep_count, provenance)
def _select_object(self, name: str) -> None:
self.selected_sets["s21_object_calibration"] = name
self.ensure_dialog().set_set_name(name)
self._summary(self.selected_sets)
self._refresh_requested()
logger.info("Selected object calibration: %s", name)
def invalidate_object_calibration_after_geometry_change(self) -> None:
selected = self.selected_sets.get("s21_object_calibration", "")
active_config = self._pipeline.active_config if self._pipeline.running else None
active = active_config.preprocess.s21.object_calibration.set_name if active_config else ""
if not selected and not active:
return
geometry = geometry_signature({"gpr": asdict(self._geometry())})
selections = [(selected, self.radar_key)]
if active_config is not None:
selections.append((active, active_config.radar_key()))
invalid: set[str] = set()
for name, radar_key in dict.fromkeys(selections):
if not name:
continue
try:
_, metadata = self._store.load_set_with_metadata("s21_object_calibration", radar_key, name)
if metadata["provenance"]["geometry"] != geometry:
raise ValueError("Antenna geometry differs from the captured calibration")
except Exception as exc:
invalid.add(name)
logger.warning("Object calibration %s disabled for changed antenna geometry: %s", name, exc)
if not invalid:
return
try:
with self._pipeline.pause() if active in invalid else nullcontext():
if selected in invalid:
self.selected_sets["s21_object_calibration"] = ""
self._summary(self.selected_sets)
if self.dialog is not None:
self.dialog.set_selected_sets(self.selected_sets, emit_signal=False)
except InterruptedError:
self._pipeline.operation_cancelled()
except Exception as exc:
self._error("Failed to update object calibration after geometry change", exc)
def shutdown(self) -> None:
self.object_calibration.shutdown()
self.abort(resume_pipeline=False)
if self.dialog is not None:
self.dialog.close()