diff --git a/.gitignore b/.gitignore index deb8359..1a911db 100644 --- a/.gitignore +++ b/.gitignore @@ -245,9 +245,10 @@ __marimo__/ python_app/runtime SHARE_INTERNET_TO_PI.md -# Локальная обвязка Claude Code: инструкции, хуки, скилы, конфиги инструментов. -# В репозиторий не идёт. +# Локальная обвязка Claude Code и Codex: инструкции, хуки, скилы, конфиги инструментов. +# В репозиторий не идёт. AGENTS.md — симлинк на CLAUDE.md для Codex. CLAUDE.md +AGENTS.md .claude/ .graphifyignore pyrightconfig.json diff --git a/Horns_motion_3libre.py b/Horns_motion_3libre.py deleted file mode 100644 index 23233b6..0000000 --- a/Horns_motion_3libre.py +++ /dev/null @@ -1,1326 +0,0 @@ -""" -MIMO GPR — coherent time-domain BackProjection -================================================= - -Эта ячейка полностью независима от верхнего эллипсного алгоритма: - 1. загружает те же S21-данные; - 2. строит oversampled A-сканы через тот же частотный сдвиг перед IFFT; - 3. для каждой точки (x,z) вычисляет tau_ij = (Rtx + Rrx) / v; - 4. интерполирует комплексный A-скан в этой задержке; - 5. когерентно суммирует комплексные вклады всех Tx/Rx-пар с компенсацией geo·pattern. - -Это coherent BP: суммируются комплексные h_ij(tau), затем строится |sum h_ij|. -""" - -import builtins -import json -import numpy as np -import matplotlib.pyplot as plt -from scipy.ndimage import gaussian_filter, label -from pathlib import Path -from dataclasses import dataclass, field -from typing import Dict, List, Tuple - -_print_raw = builtins.print -PRINT_DIAGNOSTICS = False - - -def print(*args, **kwargs): - if PRINT_DIAGNOSTICS: - _print_raw(*args, **kwargs) - - -# ══════════════════════════════════════════════════════ -# 0.1 ИЗМЕНЯЕМЫЕ ПАРАМЕТРЫ -# ══════════════════════════════════════════════════════ -INPUT_IDX = [0, 1, 2, 3] -OUTPUT_IDX = [0, 1] - -# Частотный диапазон и глубинный gate. -F_START = 38 * 1e8 -F_STOP = 6.0 * 1e9 -MIN_DEPTH = 3.0 -MAX_DEPTH = 12.0 - -# Движение радара во время кадра. -# Подтвержденная схема измерения: Tx0(f0), Tx1(f0), Tx0(f1), Tx1(f1), ... -# 'int_minus' — полная intra-frequency correction к центру interleaved sweep; -# 'int_focus' — focus-like correction без чистого Z-сдвига. -SPEED_M_S = 1.75 # скорость в м/с -LOOK_ANGLE_DEG = 7.5 # угол наклона радара, град -TX_SWEEP_TIME_S = 0.0714 # Половинное время свипа, с -MOTION_CORRECTION_MODE = 'int_minus' # 'int_minus' или 'int_focus' - -# Данные и вычитание среднего фона. -BG_SUBTRACT = True -DATA_PATH = Path('/Users/ivan_root/Downloads/Telegram_dwnld/20260608_moving/20260608_3.5-6_751_10k_-3p_2nd/preprocessed/0015_id16_ns8216344309213') -BG_PATH = DATA_PATH.parent - -# Убрать паразитные боковые лепестки с 2D карты. -BP_REMOVE_SIDELOBE_OBJECTS = True # True: убрать SL-кандидаты из финальной таблицы и разметки -BP_MAX_DETECTED_OBJECTS_TO_DRAW = 8 # N: если найдено больше объектов, цели на карте не рисуются -BP_DRAW_TOP_M_OBJECTS = 4 # M: если найдено <= N, рисуются только первые M по текущему score -BP_OBJECT_MIN_FRAC = 0.7 # остановка: пик ниже этой доли от глобального максимума - -# Компенсация затухания разделена на 2 части: -# range: геометрическое расхождение 1/(Rtx*Rrx) -# angle: диаграмма направленности cos_tx^2 * cos_rx^2 -COMP_RANGE_POWER = 0.1 - -# Метрика для ранжирования целей. -# 'peak' — порядок по максимумам coherent BP; -# 'combined' — coherent peak + coherence factor + prominence + contrast. -BP_SCORE_MODE = 'combined' - -# ══════════════════════════════════════════════════════ -# 0.2 КОНФИГИ АНТЕНН И ДВИЖЕНИЯ -# ══════════════════════════════════════════════════════ - -# Физические координаты антенн по их реальным индексам, [м]. -# Формат: (x, y, z), где X - поперечная ось, Z - дальность вдоль оси радара, -# Y - нормаль к плоскости XOZ. BP-карта строится в плоскости y=BP_PLANE_Y. -# Геометрию лучше переносить из config_profile.json текущего эксперимента. -BP_PLANE_Y = 0.0 - -TX_POSITIONS = { - 0: (-75.0 * 0.01, 0.0, 0.0), - 1: ( 75.0 * 0.01, 0.0, 0.0), -} - -RX_POSITIONS = { - 0: ( 19.0 * 0.01, 0.0, 0.0), - 1: ( 45.0 * 0.01, 0.0, 0.0), - 2: (-45.0 * 0.01, 0.0, 0.0), - 3: (-19.0 * 0.01, 0.0, 0.0), -} - - -@dataclass -class MotionConfig: - """ - Конфигурация speed-correction для interleaved Tx-by-frequency режима. - - Реальная схема измерения: - Tx0(f0), Tx1(f0), Tx0(f1), Tx1(f1), ... - - Поэтому меж-Tx сдвиг A-сканов не используется. Коррекция движения делается - только в частотной области до IFFT: для каждой частоты учитывается время, - в которое эта частота была измерена данной Tx-группой. - - speed_m_s: - Линейная скорость радара во время кадра. - - look_angle_deg: - Угол между направлением движения и осью дальности Z. - - tx_sweep_time_s: - Базовое время одной VNA-линейки частот. Для одной Tx-группы соседние - частоты измеряются через N_tx таких временных шагов, поэтому effective - frequency sweep этой Tx-группы равен N_tx * tx_sweep_time_s. - - motion_mode: - 'int_minus' — полная intra-frequency correction к центру interleaved sweep; - 'int_focus' — удалить аффинную по частоте часть phase(f), оставив - focus-like остаток без чистого сдвига Z. - - pair_order_phys: - Порядок физических Tx/Rx-пар в данных. Пары с одинаковым tx_phys имеют - один interleave-slot и отличаются только Rx. - - direction_sign: - Знак движения вдоль оси дальности. +1 означает, что при росте времени - радар смещается в сторону увеличения Z. - """ - speed_m_s: float = 0.0 - look_angle_deg: float = 0.0 - tx_sweep_time_s: float = 0.15 - motion_mode: str = 'int_minus' - pair_order_phys: List[Tuple[int, int]] = field(default_factory=lambda: [ - (tx_phys, rx_phys) - for tx_phys in sorted(OUTPUT_IDX) - for rx_phys in sorted(INPUT_IDX) - ]) - direction_sign: float = +1.0 - - -MOTION_CONFIG = MotionConfig( - speed_m_s=SPEED_M_S, - look_angle_deg=LOOK_ANGLE_DEG, - tx_sweep_time_s=TX_SWEEP_TIME_S, - motion_mode=MOTION_CORRECTION_MODE, - pair_order_phys=[ - (tx_phys, rx_phys) - for tx_phys in sorted(OUTPUT_IDX) - for rx_phys in sorted(INPUT_IDX) - ], - direction_sign=+1.0, -) - -# ══════════════════════════════════════════════════════ -# 0.3 НЕИЗМЕНЯЕМЫЕ ПАРАМЕТРЫ (ЛУЧШЕ НЕ ТРОГАТЬ) -# ══════════════════════════════════════════════════════ - -eps_r = 1.0 -v = 3e8 / np.sqrt(eps_r) - - -def positions_to_xyz(position_dict): - coords = [] - for idx in sorted(position_dict): - pos = np.asarray(position_dict[idx], dtype=float) - if pos.ndim == 0: - pos = np.array([float(pos), 0.0, 0.0], dtype=float) - elif pos.shape == (2,): - pos = np.array([pos[0], pos[1], 0.0], dtype=float) - elif pos.shape != (3,): - raise ValueError(f'Позиция антенны {idx} должна быть x, (x,y) или (x,y,z), получено {pos}') - coords.append(pos) - return np.vstack(coords) - - -tx_xyz = positions_to_xyz(TX_POSITIONS) -rx_xyz = positions_to_xyz(RX_POSITIONS) -x_tx, y_tx, z_tx = tx_xyz.T -x_rx, y_rx, z_rx = rx_xyz.T - -# Сетка BP-карты. -x_ant = np.concatenate([x_tx, x_rx]) -x_min, x_max = x_ant.min() - 2.0, x_ant.max() + 2.0 -z_min, z_max = 0.1, MAX_DEPTH -NX_BP = 300 -NZ_BP = 300 - -# Oversampling A-сканов: повышает плотность точек по t, но не физическое разрешение. -BP_OVERSAMPLE = 8 -BP_WINDOW = True - -# Нормировка каналов Tx/Rx перед когерентным сложением. -# Масштаб считается по |complex A-scan| в выбранном диапазоне глубин. -PAIR_NORMALIZE = True -PAIR_NORM_PERCENTILE = 50.0 -PAIR_NORM_DEPTH_MIN = MIN_DEPTH -PAIR_NORM_DEPTH_MAX = MAX_DEPTH -PAIR_NORM_EPS = 1e-15 - -# Компенсация нормируется на точку под виртуальным центром пары на глубине COMP_REF_DEPTH. -COMP_ANGLE_POWER = 0.0 -COMP_RANGE_WEIGHT_MAX = 5.0 -COMP_ANGLE_WEIGHT_MAX = 2.0 -COMP_WEIGHT_MAX = 8.0 -COMP_REF_DEPTH = 5.0 -BP_VALIDATE_COMPENSATION = False # clean mode: не считаем отдельную карту без compensation - -# Сглаживание только для удобства поиска/визуализации максимума. -BP_SMOOTH_SIGMA = 1.5 - -# Параметры поиска объектов на BP-карте. -MAX_OBJECTS = 10 -BP_REGION_THRESH_FRAC = 0.75 # область объекта: связная область выше этой доли от локального пика -BP_SUPPRESS_THRESH_FRAC = 0.2 # подавление: более широкая связная область вокруг найденного пика -BP_SUPPRESS_USE_WINDOW = True # False: подавлять всю связанную область; True: ограничить окно вокруг пика -BP_SUPPRESS_RX_CM = 80.0 # используется только если BP_SUPPRESS_USE_WINDOW=True -BP_SUPPRESS_RZ_CM = 40.0 # используется только если BP_SUPPRESS_USE_WINDOW=True -BP_MIN_REGION_AREA_CM2 = 10.0 # отсечение совсем мелких шумовых пятен - -# Компактный центроид вокруг локального максимума, устойчивее центроида всей вытянутой области. -BP_CENTER_USE_COMPACT = True -BP_CENTER_RX_CM = 60.0 -BP_CENTER_RZ_CM = 25.0 -BP_CENTER_THRESH_FRAC = 0.88 -BP_CENTER_WEIGHT_POWER = 2.0 - -# Диагностика боковых лепестков coherent BP. -BP_SIDELOBE_DETECT = True -BP_SIDELOBE_RANGE_RMS_TOL_CM = 20.0 # RMS-разница бистатических глубин по всем парам -BP_SIDELOBE_MIN_DX_CM = 35.0 # боковой лепесток должен быть заметно смещен по X -BP_SIDELOBE_MAX_DZ_CM = 70.0 # но находиться примерно на той же глубине -BP_SIDELOBE_MAX_REL_PEAK = 0.85 # кандидат должен быть слабее родительского максимума - -# Фазовая метрика внутри compact-области объекта. -BP_PHASE_WEIGHT_POWER = 1.0 - -# Локальная выраженность объекта над окружающим фоном. -BP_LOCAL_BG_RX_CM = 120.0 -BP_LOCAL_BG_RZ_CM = 80.0 -BP_LOCAL_BG_PERCENTILE = 50.0 -BP_LOCAL_CONTRAST_EPS = 1e-12 - -# Экспериментальный score для ранжирования найденных объектов. -BP_SCORE_COMPUTE_INCOHERENT = True -BP_SCORE_COH_PEAK_WEIGHT = 0.45 -BP_SCORE_COHERENCE_FACTOR_WEIGHT = 0.25 -BP_SCORE_PROMINENCE_WEIGHT = 0.20 -BP_SCORE_CONTRAST_WEIGHT = 0.10 -BP_SCORE_CONTRAST_CAP = 6.0 -BP_SCORE_CF_EPS = 1e-12 - - -def _build_phys_to_logical_maps(output_idx, input_idx): - tx_map = {phys: k for k, phys in enumerate(sorted(output_idx))} - rx_map = {phys: k for k, phys in enumerate(sorted(input_idx))} - return tx_map, rx_map - - -def _tx_event_order_from_pairs(pair_order_phys): - tx_events = [] - for tx_phys, _ in pair_order_phys: - if tx_phys not in tx_events: - tx_events.append(tx_phys) - return tx_events - - -def compute_pair_timestamps(config: MotionConfig, - output_idx=OUTPUT_IDX, - input_idx=INPUT_IDX) -> Tuple[Dict[Tuple[int, int], Dict], List[Dict]]: - """Возвращает interleave-slot для каждой логической Tx/Rx-пары.""" - tx_phys_to_log, rx_phys_to_log = _build_phys_to_logical_maps(output_idx, input_idx) - expected_pairs = {(tx, rx) for tx in output_idx for rx in input_idx} - observed_pairs = set(config.pair_order_phys) - missing_pairs = expected_pairs - observed_pairs - extra_pairs = observed_pairs - expected_pairs - if missing_pairs: - raise ValueError(f'pair_order_phys не содержит пары: {sorted(missing_pairs)}') - if extra_pairs: - raise ValueError(f'pair_order_phys содержит лишние пары: {sorted(extra_pairs)}') - if len(config.pair_order_phys) != len(observed_pairs): - raise ValueError('pair_order_phys содержит повторяющиеся пары') - - tx_event_order = _tx_event_order_from_pairs(config.pair_order_phys) - if not tx_event_order: - return {}, [] - - n_tx_events = len(tx_event_order) - pair_timestamps: Dict[Tuple[int, int], Dict] = {} - rows: List[Dict] = [] - - for order_idx, (tx_phys, rx_phys) in enumerate(config.pair_order_phys): - if tx_phys not in tx_phys_to_log: - raise ValueError(f'Tx {tx_phys} отсутствует в OUTPUT_IDX={output_idx}') - if rx_phys not in rx_phys_to_log: - raise ValueError(f'Rx {rx_phys} отсутствует в INPUT_IDX={input_idx}') - - event_idx = tx_event_order.index(tx_phys) - row = { - 'order_idx': order_idx, - 'tx_event_idx': event_idx, - 'tx_interleave_slot': event_idx, - 'tx_interleave_slots': n_tx_events, - 'tx_phys': tx_phys, - 'rx_phys': rx_phys, - 'i_tx': tx_phys_to_log[tx_phys], - 'i_rx': rx_phys_to_log[rx_phys], - 'base_sweep_time_s': config.tx_sweep_time_s, - } - rows.append(row) - pair_timestamps[(row['i_tx'], row['i_rx'])] = row.copy() - - return pair_timestamps, rows - - -def print_motion_summary(rows: List[Dict], config: MotionConfig): - if config.motion_mode not in ('int_minus', 'int_focus'): - raise ValueError("motion_mode must be 'int_minus' or 'int_focus'") - - n_slots = max((r['tx_interleave_slots'] for r in rows), default=len(OUTPUT_IDX)) - print(f'Motion correction: {config.motion_mode}; speed={config.speed_m_s:.3f} м/с, ' - f'angle={config.look_angle_deg:.2f}°, tx_sweep={config.tx_sweep_time_s*1e3:.2f} мс') - print(f'Frequency timing: interleaved Tx-by-frequency; Tx slots={n_slots}; ' - f'effective Tx frequency span≈{n_slots*config.tx_sweep_time_s*1e3:.2f} мс') - print(f" {'pair':<8} {'slot':>4} {'tx_phys':>7} {'rx_phys':>7}") - for r in rows: - pair = f"Tx{r['i_tx']}-Rx{r['i_rx']}" - print(f" {pair:<8} {r['tx_interleave_slot']:>4d} {r['tx_phys']:>7d} {r['rx_phys']:>7d}") - - -def compute_frequency_sample_times(freq_full: np.ndarray, - f_start: float, - f_stop: float, - pair_info: Dict, - sweep_time_s: float) -> Tuple[np.ndarray, np.ndarray, float]: - """Возвращает индексы частот, абсолютное время каждой точки и центр Tx-sweep.""" - mask = (freq_full >= f_start) & (freq_full <= f_stop) - idx = np.flatnonzero(mask) - if idx.size == 0: - raise ValueError('После частотной обрезки не осталось точек для phase correction') - - n_full = len(freq_full) - if n_full < 2: - return idx, np.zeros(idx.size, dtype=float), 0.0 - - n_slots = int(pair_info.get('tx_interleave_slots', len(OUTPUT_IDX))) - slot = int(pair_info.get('tx_interleave_slot', pair_info.get('tx_event_idx', 0))) - dt_base = sweep_time_s / (n_full - 1) - - full_indices = np.arange(n_full, dtype=float) - t_all = (n_slots * full_indices + slot) * dt_base - t_abs = t_all[idx] - t_center = 0.5 * (float(t_all[0]) + float(t_all[-1])) - return idx, t_abs, t_center - - -def apply_intra_sweep_phase_correction(s21: np.ndarray, - freq_full: np.ndarray, - pair_info: Dict, - config: MotionConfig, - wave_speed: float, - f_start: float, - f_stop: float): - """Frequency-domain speed correction до IFFT для interleaved Tx-by-frequency.""" - if config.motion_mode not in ('int_minus', 'int_focus'): - raise ValueError("motion_mode must be 'int_minus' or 'int_focus'") - - s21_corr = np.array(s21, dtype=np.complex128, copy=True) - idx, t_abs, t_center_s = compute_frequency_sample_times( - freq_full=freq_full, - f_start=f_start, - f_stop=f_stop, - pair_info=pair_info, - sweep_time_s=config.tx_sweep_time_s, - ) - - dt_intra = t_abs - t_center_s - theta = np.radians(config.look_angle_deg) - delta_range_intra = config.direction_sign * config.speed_m_s * np.cos(theta) * dt_intra - delta_path_intra = 2.0 * delta_range_intra - dtau_intra = delta_path_intra / wave_speed - - phi = 2.0 * np.pi * freq_full[idx] * dtau_intra - - # int_focus убирает постоянную + линейную по частоте часть phase(f). - # Оставшийся нелинейный остаток улучшает фокус, почти не сдвигая Z. - if config.motion_mode == 'int_focus' and np.any(dtau_intra != 0.0): - f_rel = freq_full[idx].astype(float) - float(np.mean(freq_full[idx])) - x_fit = np.column_stack([np.ones_like(f_rel), f_rel]) - beta_fit, *_ = np.linalg.lstsq(x_fit, phi, rcond=None) - phi = phi - x_fit @ beta_fit - - if np.any(dtau_intra != 0.0): - s21_corr[idx] *= np.exp(-1j * phi) - - meta = { - 'enabled': bool(np.any(dtau_intra != 0.0)), - 'motion_mode': config.motion_mode, - 'freq_idx': idx, - 't_abs_s': t_abs, - 'dt_intra_s': dt_intra, - 't_center_s': t_center_s, - 'intra_dtau_s': dtau_intra, - 'total_dtau_s': dtau_intra, - 'delta_range_m': delta_range_intra, - 'delta_path_m': delta_path_intra, - 'phi_rad': phi, - } - return s21_corr, meta - - -pair_timestamps, pair_timing_rows = compute_pair_timestamps(MOTION_CONFIG) -print_motion_summary(pair_timing_rows, MOTION_CONFIG) - -# ══════════════════════════════════════════════════════ -# 1. ЗАГРУЗКА ДАННЫХ -# ══════════════════════════════════════════════════════ - -def load_mimo_data(data_path, input_idx, output_idx): - data_path = Path(data_path) - s21_data = {} - freq_data = {} - - for f in data_path.glob('i*_o*_s21.npy'): - name = f.stem - parts = name.split('_') - i_tx_phys = int(parts[1][1:]) - i_rx_phys = int(parts[0][1:]) - - if i_tx_phys not in output_idx or i_rx_phys not in input_idx: - continue - - i_tx = sorted(output_idx).index(i_tx_phys) - i_rx = sorted(input_idx).index(i_rx_phys) - - s21_data[(i_tx, i_rx)] = np.load(f) - freq_file = data_path / f'i{i_rx_phys}_o{i_tx_phys}_freq.npy' - freq_data[(i_tx, i_rx)] = np.load(freq_file) - - return s21_data, freq_data - - -def compute_background(bg_path, input_idx, output_idx): - bg_path = Path(bg_path) - snapshots = [s for s in sorted(bg_path.glob('*/')) if s.is_dir()] - - if len(snapshots) == 0: - print('Фоновые снимки не найдены, BG_SUBTRACT отключен.') - return None - - print(f'Вычисление фона по {len(snapshots)} снимкам...', end=' ', flush=True) - bg_sum = {} - bg_count = {} - - for snap_dir in snapshots: - for f in snap_dir.glob('i*_o*_s21.npy'): - name = f.stem - parts = name.split('_') - i_tx_phys = int(parts[1][1:]) - i_rx_phys = int(parts[0][1:]) - - if i_tx_phys not in output_idx or i_rx_phys not in input_idx: - continue - - i_tx = sorted(output_idx).index(i_tx_phys) - i_rx = sorted(input_idx).index(i_rx_phys) - key = (i_tx, i_rx) - - s21 = np.load(f) - if key not in bg_sum: - bg_sum[key] = np.zeros_like(s21, dtype=np.complex128) - bg_count[key] = 0 - bg_sum[key] += s21 - bg_count[key] += 1 - - bg = {key: bg_sum[key] / bg_count[key] for key in bg_sum} - print(f'готово. Пар: {len(bg)}') - return bg - - -s21_data, freq_data = load_mimo_data(DATA_PATH, INPUT_IDX, OUTPUT_IDX) -N_tx = len(OUTPUT_IDX) -N_rx = len(INPUT_IDX) -N_pairs = len(s21_data) - -assert len(x_tx) == N_tx, f'x_tx должен содержать {N_tx} элементов' -assert len(x_rx) == N_rx, f'x_rx должен содержать {N_rx} элементов' -assert N_pairs > 0, 'Не найдено ни одной Tx/Rx-пары. Проверьте DATA_PATH.' - -print('Геометрия антенн, [м]:') -for local_i, phys_i in enumerate(sorted(TX_POSITIONS)): - print(f' Tx{local_i} / o{phys_i}: x={x_tx[local_i]:+.3f}, y={y_tx[local_i]:+.3f}, z={z_tx[local_i]:+.3f}') -for local_j, phys_j in enumerate(sorted(RX_POSITIONS)): - print(f' Rx{local_j} / i{phys_j}: x={x_rx[local_j]:+.3f}, y={y_rx[local_j]:+.3f}, z={z_rx[local_j]:+.3f}') - -if BG_SUBTRACT: - background = compute_background(BG_PATH, INPUT_IDX, OUTPUT_IDX) - if background is None: - BG_SUBTRACT = False -else: - background = None - print('BG_SUBTRACT = False, вычитание фона отключено.') - -first_key = list(freq_data.keys())[0] -freqs = freq_data[first_key] -freq_mask = (freqs >= F_START) & (freqs <= F_STOP) -freqs_bp = freqs[freq_mask] - -if len(freqs_bp) < 2: - raise ValueError('В выбранном частотном диапазоне меньше двух точек.') - -f_min = float(freqs_bp[0]) -f_max = float(freqs_bp[-1]) -BW = f_max - f_min -df_values = np.diff(freqs_bp) -df_median = float(np.median(df_values)) -df_min = float(np.min(df_values)) -df_max = float(np.max(df_values)) -df_rel_spread = (df_max - df_min) / (df_median + 1e-30) -range_resolution = v / (2 * BW) -unambiguous_depth = v / (2 * df_median) -unambiguous_total_path = v / df_median - -print(f'Загружено пар Tx/Rx: {N_pairs}') -print(f'Частотный диапазон BP: {f_min/1e9:.3f} - {f_max/1e9:.3f} ГГц') -print(f'Точек частоты в BP-диапазоне: {len(freqs_bp)}') -print(f'Шаг частоты df: median={df_median/1e6:.3f} МГц, ' - f'min={df_min/1e6:.3f} МГц, max={df_max/1e6:.3f} МГц') -print(f'Неравномерность df: {(df_rel_spread*100):.3f}% от median') -print(f'Полоса B: {BW/1e9:.3f} ГГц') -print(f'Теоретический предел разрешения по глубине deltaZ = {range_resolution*100:.2f} см') -print(f'Unambiguous range по глубине = {unambiguous_depth:.2f} м ' - f'(max total path = {unambiguous_total_path:.2f} м)') -print(f'BP_OVERSAMPLE = {BP_OVERSAMPLE}') -print(f'MOTION_CORRECTION_MODE = {MOTION_CORRECTION_MODE}') -print('FREQUENCY_TIMING = interleaved_tx_by_frequency') - - -# ══════════════════════════════════════════════════════ -# 2. OVERSAMPLED A-СКАНЫ -# ══════════════════════════════════════════════════════ - -def compute_ascan_bp(s21, freq, f_start, f_stop, window=True, oversample=8): - """ - S21(f) -> A-скан с правильным частотным сдвигом и oversampling. - - Частотный шаг df остается тем же, а n_fft увеличивается в oversample раз. - Поэтому временная сетка становится плотнее: dt = 1 / (n_fft * df). - """ - mask = (freq >= f_start) & (freq <= f_stop) - freq_cut = freq[mask] - s21_cut = s21[mask] - - if len(freq_cut) < 2: - raise ValueError('После обрезки по частоте осталось меньше двух точек.') - - df = float(np.median(np.diff(freq_cut))) - n = len(freq_cut) - k0 = int(round(freq_cut[0] / df)) - - min_len = 2 * (k0 + n - 1) - n_fft_base = 1 << int(np.ceil(np.log2(min_len))) - n_fft = int(n_fft_base * oversample) - - dt = 1.0 / (n_fft * df) - t_sec = np.arange(n_fft, dtype=float) * dt - - s = s21_cut * np.hanning(n) if window else s21_cut.copy() - - H = np.zeros(n_fft, dtype=np.complex128) - H[k0:k0 + n] = s - - h_complex = np.fft.ifft(H) - a_abs = np.abs(h_complex) - - return t_sec, a_abs, h_complex, n_fft_base, n_fft - - - -print('Вычисление oversampled A-сканов...', end=' ', flush=True) -A_bp = {} -H_bp = {} -T_bp = {} -Z_bp = {} -n_fft_info = {} -pair_norm_info = {} -phase_meta = {} - -for (i, j), s21 in s21_data.items(): - s21_proc = np.array(s21, dtype=np.complex128, copy=True) - if BG_SUBTRACT and background is not None and (i, j) in background: - s21_proc = s21_proc - background[(i, j)] - - if (i, j) not in pair_timestamps: - raise KeyError(f'Нет временной информации для пары {(i, j)}') - - s21_corr, meta = apply_intra_sweep_phase_correction( - s21=s21_proc, - freq_full=freq_data[(i, j)], - pair_info=pair_timestamps[(i, j)], - config=MOTION_CONFIG, - wave_speed=v, - f_start=F_START, - f_stop=F_STOP, - ) - phase_meta[(i, j)] = meta - - t_pair, a_pair, h_pair, n_fft_base, n_fft = compute_ascan_bp( - s21_corr, - freq_data[(i, j)], - f_start=F_START, - f_stop=F_STOP, - window=BP_WINDOW, - oversample=BP_OVERSAMPLE, - ) - T_bp[(i, j)] = t_pair - Z_bp[(i, j)] = t_pair * v / 2 - A_bp[(i, j)] = a_pair - H_bp[(i, j)] = h_pair - n_fft_info[(i, j)] = (n_fft_base, n_fft) - -# Robust per-pair amplitude normalization. It equalizes channel scale, not phase. -for key in sorted(H_bp.keys()): - z_axis = Z_bp[key] - h_abs = np.abs(H_bp[key]) - norm_mask = (z_axis >= PAIR_NORM_DEPTH_MIN) & (z_axis <= PAIR_NORM_DEPTH_MAX) - if not np.any(norm_mask): - norm_mask = np.ones_like(z_axis, dtype=bool) - - median_val = float(np.median(h_abs[norm_mask])) - p75_val = float(np.percentile(h_abs[norm_mask], 75)) - p95_val = float(np.percentile(h_abs[norm_mask], 95)) - scale = float(np.percentile(h_abs[norm_mask], PAIR_NORM_PERCENTILE)) - if not np.isfinite(scale) or scale <= PAIR_NORM_EPS: - scale = 1.0 - - pair_norm_info[key] = { - 'median': median_val, - 'p75': p75_val, - 'p95': p95_val, - 'scale': scale, - } - - if PAIR_NORMALIZE: - H_bp[key] = H_bp[key] / (scale + PAIR_NORM_EPS) - A_bp[key] = np.abs(H_bp[key]) - - -z_h_bp = Z_bp[first_key] -t_h_bp = T_bp[first_key] -print('готово.') -print(f'dt = {(t_h_bp[1] - t_h_bp[0])*1e12:.2f} пс') -print(f'dz_sample = {(z_h_bp[1] - z_h_bp[0])*100:.3f} см') -if phase_meta: - delta_ranges = np.concatenate([m['delta_range_m'] for m in phase_meta.values()]) - total_dtau_vals = np.concatenate([m['total_dtau_s'] for m in phase_meta.values()]) - phis = np.concatenate([m['phi_rad'] for m in phase_meta.values()]) - print(f'Motion mode до IFFT: {MOTION_CONFIG.motion_mode}') - print('Frequency timing: interleaved_tx_by_frequency') - print(f'Total dtau(f) range = {total_dtau_vals.min()*1e9:+.3f}..{total_dtau_vals.max()*1e9:+.3f} нс') - print(f'Intra-sweep dz range = {delta_ranges.min()*100:+.3f}..{delta_ranges.max()*100:+.3f} см') - print(f'Motion phi range = {phis.min():+.3e}..{phis.max():+.3e} рад') - -print('\n' + '=' * 86) -print(f' НОРМИРОВКА КАНАЛОВ Tx/Rx: enabled={PAIR_NORMALIZE}, ' - f'percentile={PAIR_NORM_PERCENTILE:.1f}, ' - f'z=[{PAIR_NORM_DEPTH_MIN:.2f}, {PAIR_NORM_DEPTH_MAX:.2f}] м') -print('=' * 86) -print(f" {'Pair':<8} {'median':>12} {'p75':>12} {'p95':>12} {'scale':>12} {'rel_scale':>12}") -print('-' * 86) -scales = np.array([v['scale'] for v in pair_norm_info.values()], dtype=float) -scale_ref = float(np.median(scales)) if len(scales) else 1.0 -for key in sorted(pair_norm_info.keys()): - info = pair_norm_info[key] - rel_scale = info['scale'] / (scale_ref + PAIR_NORM_EPS) - print(f" Tx{key[0]}-Rx{key[1]:<3} {info['median']:>12.4e} {info['p75']:>12.4e} " - f"{info['p95']:>12.4e} {info['scale']:>12.4e} {rel_scale:>12.3f}") -print('=' * 86) - - -# ══════════════════════════════════════════════════════ -# 3. TIME-DOMAIN COHERENT BACKPROJECTION -# ══════════════════════════════════════════════════════ - -x_grid_bp = np.linspace(x_min, x_max, NX_BP) -z_grid_bp = np.linspace(z_min, z_max, NZ_BP) -XX_bp, ZZ_bp = np.meshgrid(x_grid_bp, z_grid_bp) -depth_gate = (ZZ_bp >= MIN_DEPTH) & (ZZ_bp <= MAX_DEPTH) - - -def bistatic_ranges(i_tx, i_rx, XX, ZZ, yy=BP_PLANE_Y): - Rtx = np.sqrt((XX - x_tx[i_tx])**2 + (yy - y_tx[i_tx])**2 + (ZZ - z_tx[i_tx])**2) - Rrx = np.sqrt((XX - x_rx[i_rx])**2 + (yy - y_rx[i_rx])**2 + (ZZ - z_rx[i_rx])**2) - return Rtx, Rrx - - -def antenna_boresight_cos_z(R, z_ant, ZZ): - # В локальных координатах радара все антенны смотрят вдоль +Z. - return (ZZ - z_ant) / (R + 1e-12) - - -def attenuation_components_map(i_tx, i_rx, XX, ZZ): - Rtx, Rrx = bistatic_ranges(i_tx, i_rx, XX, ZZ) - geo = 1.0 / (Rtx * Rrx + 1e-12) - cos_tx = antenna_boresight_cos_z(Rtx, z_tx[i_tx], ZZ) - cos_rx = antenna_boresight_cos_z(Rrx, z_rx[i_rx], ZZ) - angle = cos_tx**2 * cos_rx**2 - return geo + 1e-30, angle + 1e-30 - - -def attenuation_components_at_ref_depth(i_tx, i_rx, z_ref): - xc = (x_tx[i_tx] + x_rx[i_rx]) / 2.0 - Rtx, Rrx = bistatic_ranges(i_tx, i_rx, xc, z_ref) - geo = 1.0 / (Rtx * Rrx + 1e-12) - cos_tx = antenna_boresight_cos_z(Rtx, z_tx[i_tx], z_ref) - cos_rx = antenna_boresight_cos_z(Rrx, z_rx[i_rx], z_ref) - angle = cos_tx**2 * cos_rx**2 - return geo + 1e-30, angle + 1e-30 - - -def bp_compensation_weight(i_tx, i_rx, XX, ZZ): - geo, angle = attenuation_components_map(i_tx, i_rx, XX, ZZ) - geo_ref, angle_ref = attenuation_components_at_ref_depth(i_tx, i_rx, COMP_REF_DEPTH) - - geo_norm = geo / geo_ref - angle_norm = angle / angle_ref - - range_weight = 1.0 / (geo_norm ** COMP_RANGE_POWER + 1e-12) - angle_weight = 1.0 / (angle_norm ** COMP_ANGLE_POWER + 1e-12) - - range_weight = np.clip(range_weight, 0.0, COMP_RANGE_WEIGHT_MAX) - angle_weight = np.clip(angle_weight, 0.0, COMP_ANGLE_WEIGHT_MAX) - - weight = range_weight * angle_weight - return np.clip(weight, 0.0, COMP_WEIGHT_MAX) - - -def interpolate_ascan_amplitude(tau, t_axis, a_axis): - return np.interp(tau.ravel(), t_axis, a_axis, left=0.0, right=0.0).reshape(tau.shape) - - -def interpolate_ascan_complex(tau, t_axis, h_axis): - h_real = np.interp(tau.ravel(), t_axis, h_axis.real, left=0.0, right=0.0) - h_imag = np.interp(tau.ravel(), t_axis, h_axis.imag, left=0.0, right=0.0) - return (h_real + 1j * h_imag).reshape(tau.shape) - - -def backproject_coherent(H, T, compensate=True): - bp_complex = np.zeros_like(XX_bp, dtype=np.complex128) - contribution_count = np.zeros_like(XX_bp, dtype=float) - - for i in range(N_tx): - for j in range(N_rx): - key = (i, j) - if key not in H: - continue - - Rtx, Rrx = bistatic_ranges(i, j, XX_bp, ZZ_bp) - tau_ref = (Rtx + Rrx) / v - tau = tau_ref - valid = depth_gate & (tau >= T[key][0]) & (tau <= T[key][-1]) - - h_tau = interpolate_ascan_complex(tau, T[key], H[key]) - h_tau = np.where(valid, h_tau, 0.0 + 0.0j) - - if compensate: - w = bp_compensation_weight(i, j, XX_bp, ZZ_bp) - w = np.where(valid, w, 0.0) - else: - w = np.where(valid, 1.0, 0.0) - - bp_complex += h_tau * w - contribution_count += valid.astype(float) - - bp_complex = bp_complex / (contribution_count + 1e-12) - bp_complex = np.where(depth_gate, bp_complex, 0.0 + 0.0j) - bp_abs = np.abs(bp_complex) - return bp_abs, bp_complex - - -def backproject_coherent_and_incoherent(H, T, compensate=True): - """Одним проходом строит coherent |sum h| и incoherent sum |h| BP-карты.""" - bp_complex = np.zeros_like(XX_bp, dtype=np.complex128) - bp_incoherent = np.zeros_like(XX_bp, dtype=float) - contribution_count = np.zeros_like(XX_bp, dtype=float) - - for i in range(N_tx): - for j in range(N_rx): - key = (i, j) - if key not in H: - continue - - Rtx, Rrx = bistatic_ranges(i, j, XX_bp, ZZ_bp) - tau_ref = (Rtx + Rrx) / v - tau = tau_ref - valid = depth_gate & (tau >= T[key][0]) & (tau <= T[key][-1]) - - h_tau = interpolate_ascan_complex(tau, T[key], H[key]) - h_tau = np.where(valid, h_tau, 0.0 + 0.0j) - - if compensate: - w = bp_compensation_weight(i, j, XX_bp, ZZ_bp) - w = np.where(valid, w, 0.0) - else: - w = np.where(valid, 1.0, 0.0) - - bp_complex += h_tau * w - bp_incoherent += np.abs(h_tau) * w - contribution_count += valid.astype(float) - - bp_complex = bp_complex / (contribution_count + 1e-12) - bp_complex = np.where(depth_gate, bp_complex, 0.0 + 0.0j) - bp_abs = np.abs(bp_complex) - - bp_incoherent = bp_incoherent / (contribution_count + 1e-12) - bp_incoherent = np.where(depth_gate, bp_incoherent, 0.0) - - bp_cf = bp_abs / (bp_incoherent + BP_SCORE_CF_EPS) - bp_cf = np.where(depth_gate, bp_cf, 0.0) - bp_cf = np.clip(bp_cf, 0.0, 1.0) - return bp_abs, bp_complex, bp_incoherent, bp_cf - - -def normalize_bp_map(bp_raw, smooth_sigma=BP_SMOOTH_SIGMA): - bp_norm = bp_raw / (bp_raw.max() + 1e-12) - bp_norm = gaussian_filter(bp_norm, sigma=smooth_sigma) - bp_norm = np.where(depth_gate, bp_norm, 0.0) - return bp_norm / (bp_norm.max() + 1e-12) - -def component_containing_peak(image, iz, ix, threshold, window_mask=None): - mask = image >= threshold - if window_mask is not None: - mask &= window_mask - - labels, n_labels = label(mask, structure=np.ones((3, 3), dtype=int)) - if n_labels == 0 or labels[iz, ix] == 0: - fallback = np.zeros_like(image, dtype=bool) - fallback[iz, ix] = True - return fallback - - return labels == labels[iz, ix] - - -def weighted_centroid(image, region_mask, threshold=0.0): - values = image[region_mask] - weights = np.clip(values - threshold, 0.0, None) - if weights.sum() <= 1e-15: - weights = values.copy() - if weights.sum() <= 1e-15: - iz, ix = np.argwhere(region_mask)[0] - return x_grid_bp[ix], z_grid_bp[iz] - - x_vals = XX_bp[region_mask] - z_vals = ZZ_bp[region_mask] - return (x_vals * weights).sum() / weights.sum(), (z_vals * weights).sum() / weights.sum() - - - -def compact_peak_centroid(image, iz, ix, peak): - x0 = x_grid_bp[ix] - z0 = z_grid_bp[iz] - window_mask = ( - (np.abs(XX_bp - x0) <= BP_CENTER_RX_CM / 100.0) & - (np.abs(ZZ_bp - z0) <= BP_CENTER_RZ_CM / 100.0) - ) - threshold = BP_CENTER_THRESH_FRAC * peak - center_mask = window_mask & (image >= threshold) - - if center_mask.sum() == 0: - center_mask = window_mask.copy() - center_mask[iz, ix] = True - - values = image[center_mask] - weights = np.clip(values - threshold, 0.0, None) ** BP_CENTER_WEIGHT_POWER - if weights.sum() <= 1e-15: - weights = values.copy() - if weights.sum() <= 1e-15: - return x0, z0, center_mask - - x_vals = XX_bp[center_mask] - z_vals = ZZ_bp[center_mask] - x_c = (x_vals * weights).sum() / weights.sum() - z_c = (z_vals * weights).sum() / weights.sum() - return x_c, z_c, center_mask - -def find_bp_objects(bp_image): - """ - CLEAN-подобный поиск объектов на BP-карте. - - Для каждого шага берется максимум рабочей карты, вокруг него выделяется - связная область выше BP_REGION_THRESH_FRAC от локального пика, затем - считается взвешенный центроид этой области. После этого более широкая - область вокруг той же цели подавляется на рабочей карте. - """ - work = bp_image.copy() - objects = [] - global_peak = float(work.max()) - stop_level = BP_OBJECT_MIN_FRAC * global_peak - dx_cm = abs(x_grid_bp[1] - x_grid_bp[0]) * 100 - dz_cm = abs(z_grid_bp[1] - z_grid_bp[0]) * 100 - pixel_area_cm2 = dx_cm * dz_cm - - for step in range(MAX_OBJECTS): - peak = float(work.max()) - if peak <= stop_level or peak <= 0: - break - - iz, ix = np.unravel_index(np.argmax(work), work.shape) - x_peak_local = x_grid_bp[ix] - z_peak_local = z_grid_bp[iz] - - region_threshold = BP_REGION_THRESH_FRAC * peak - region_mask = component_containing_peak(work, iz, ix, region_threshold) - region_area_cm2 = float(region_mask.sum() * pixel_area_cm2) - - if region_area_cm2 < BP_MIN_REGION_AREA_CM2: - work[iz, ix] = 0.0 - continue - - x_region_centroid, z_region_centroid = weighted_centroid(work, region_mask, threshold=region_threshold) - if BP_CENTER_USE_COMPACT: - x_centroid, z_centroid, center_mask = compact_peak_centroid(work, iz, ix, peak) - else: - x_centroid, z_centroid = x_region_centroid, z_region_centroid - center_mask = region_mask.copy() - - center_area_cm2 = float(center_mask.sum() * pixel_area_cm2) - region_values = work[region_mask] - objects.append({ - 'index': len(objects) + 1, - 'ix_peak': int(ix), - 'iz_peak': int(iz), - 'x_peak': float(x_peak_local), - 'z_peak': float(z_peak_local), - 'x': float(x_centroid), - 'z': float(z_centroid), - 'x_region': float(x_region_centroid), - 'z_region': float(z_region_centroid), - 'peak': peak, - 'area_cm2': region_area_cm2, - 'center_area_cm2': center_area_cm2, - 'region_mask': region_mask.copy(), - 'center_mask': center_mask.copy(), - 'mean_value': float(region_values.mean()), - 'sum_value': float(region_values.sum()), - }) - - if BP_SUPPRESS_USE_WINDOW: - suppress_window = ( - (np.abs(XX_bp - x_peak_local) <= BP_SUPPRESS_RX_CM / 100.0) & - (np.abs(ZZ_bp - z_peak_local) <= BP_SUPPRESS_RZ_CM / 100.0) - ) - else: - suppress_window = None - - suppress_threshold = BP_SUPPRESS_THRESH_FRAC * peak - suppress_mask = component_containing_peak( - work, iz, ix, suppress_threshold, window_mask=suppress_window - ) - - # Если широкий порог дал слишком маленькую область, подавляем хотя бы область детекции. - if suppress_mask.sum() < region_mask.sum(): - suppress_mask = region_mask - - work[suppress_mask] = 0.0 - - return objects, work - - -def circular_phase_stats(bp_complex_map, mask): - if mask.sum() == 0: - return np.nan, np.nan, np.nan - - h = bp_complex_map[mask] - amp = np.abs(h) - valid = amp > 0 - if not np.any(valid): - return np.nan, np.nan, np.nan - - phase = np.angle(h[valid]) - weights = amp[valid] ** BP_PHASE_WEIGHT_POWER - if weights.sum() <= 1e-15: - weights = np.ones_like(phase) - - vec = np.sum(weights * np.exp(1j * phase)) / (np.sum(weights) + 1e-15) - phase_mean = float(np.angle(vec)) - phase_coherence = float(np.abs(vec)) - phase_circular_variance = float(1.0 - phase_coherence) - return phase_mean, phase_coherence, phase_circular_variance - - -def add_phase_metrics(objects, bp_complex_map): - for obj in objects: - mask = obj.get('center_mask', obj['region_mask']) - phase_mean, phase_coh, phase_var = circular_phase_stats(bp_complex_map, mask) - obj['phase_mean_rad'] = phase_mean - obj['phase_coherence'] = phase_coh - obj['phase_circular_variance'] = phase_var - return objects - - -def add_local_prominence_metrics(objects, bp_image): - for obj in objects: - x0 = obj['x_peak'] - z0 = obj['z_peak'] - outer_mask = ( - (np.abs(XX_bp - x0) <= BP_LOCAL_BG_RX_CM / 100.0) & - (np.abs(ZZ_bp - z0) <= BP_LOCAL_BG_RZ_CM / 100.0) & - depth_gate - ) - bg_mask = outer_mask & (~obj['region_mask']) - if bg_mask.sum() < 10: - bg_mask = depth_gate & (~obj['region_mask']) - - if bg_mask.sum() == 0: - bg_level = 0.0 - bg_p75 = 0.0 - else: - bg_values = bp_image[bg_mask] - bg_level = float(np.percentile(bg_values, BP_LOCAL_BG_PERCENTILE)) - bg_p75 = float(np.percentile(bg_values, 75)) - - peak = float(obj['peak']) - obj['local_bg'] = bg_level - obj['local_bg_p75'] = bg_p75 - obj['prominence'] = peak - bg_level - obj['contrast'] = peak / (bg_level + BP_LOCAL_CONTRAST_EPS) - return objects - - -def add_incoherent_support_metrics(objects, bp_incoherent_image, bp_cf_image=None): - for obj in objects: - if bp_incoherent_image is None: - obj['incoh_peak'] = 0.0 - obj['incoh_mean'] = 0.0 - obj['incoh_center_mean'] = 0.0 - obj['coherence_factor_peak_raw'] = 0.0 - obj['coherence_factor_center_raw'] = 0.0 - obj['coherence_factor_peak'] = 0.0 - obj['coherence_factor_center'] = 0.0 - continue - - region_mask = obj['region_mask'] - center_mask = obj.get('center_mask', region_mask) - if region_mask.sum() == 0: - obj['incoh_peak'] = 0.0 - obj['incoh_mean'] = 0.0 - else: - region_values = bp_incoherent_image[region_mask] - obj['incoh_peak'] = float(region_values.max()) - obj['incoh_mean'] = float(region_values.mean()) - - if center_mask.sum() == 0: - obj['incoh_center_mean'] = obj['incoh_mean'] - else: - obj['incoh_center_mean'] = float(bp_incoherent_image[center_mask].mean()) - - if bp_cf_image is None: - obj['coherence_factor_peak_raw'] = float( - obj.get('peak', 0.0) / (obj['incoh_peak'] + BP_SCORE_CF_EPS) - ) - obj['coherence_factor_center_raw'] = float( - obj.get('mean_value', 0.0) / (obj['incoh_center_mean'] + BP_SCORE_CF_EPS) - ) - else: - iz_peak = int(obj.get('iz_peak', 0)) - ix_peak = int(obj.get('ix_peak', 0)) - obj['coherence_factor_peak_raw'] = float(bp_cf_image[iz_peak, ix_peak]) - if center_mask.sum() == 0: - obj['coherence_factor_center_raw'] = obj['coherence_factor_peak_raw'] - else: - obj['coherence_factor_center_raw'] = float(bp_cf_image[center_mask].mean()) - - obj['coherence_factor_peak'] = float(np.clip(obj['coherence_factor_peak_raw'], 0.0, 1.0)) - obj['coherence_factor_center'] = float(np.clip(obj['coherence_factor_center_raw'], 0.0, 1.0)) - return objects - - -def _contrast_score_unit(contrast): - if not np.isfinite(contrast): - return 0.0 - if BP_SCORE_CONTRAST_CAP <= 1.0: - return 0.0 - return float(np.clip((contrast - 1.0) / (BP_SCORE_CONTRAST_CAP - 1.0), 0.0, 1.0)) - - -def add_bp_score_metrics(objects): - total_weight = ( - BP_SCORE_COH_PEAK_WEIGHT + - BP_SCORE_COHERENCE_FACTOR_WEIGHT + - BP_SCORE_PROMINENCE_WEIGHT + - BP_SCORE_CONTRAST_WEIGHT - ) - if total_weight <= 0: - total_weight = 1.0 - - for obj in objects: - coh_peak_score = float(np.clip(obj.get('peak', 0.0), 0.0, 1.0)) - coherence_factor_score = float(np.clip(obj.get('coherence_factor_peak', 0.0), 0.0, 1.0)) - prominence_score = float(np.clip(obj.get('prominence', 0.0), 0.0, 1.0)) - contrast_score = _contrast_score_unit(obj.get('contrast', np.nan)) - - score_combined = ( - BP_SCORE_COH_PEAK_WEIGHT * coh_peak_score + - BP_SCORE_COHERENCE_FACTOR_WEIGHT * coherence_factor_score + - BP_SCORE_PROMINENCE_WEIGHT * prominence_score + - BP_SCORE_CONTRAST_WEIGHT * contrast_score - ) / total_weight - - obj['score_old'] = coh_peak_score - obj['score_new'] = float(score_combined) - obj['score_coh_peak_part'] = coh_peak_score - obj['score_cf_part'] = coherence_factor_score - obj['score_prominence_part'] = prominence_score - obj['score_contrast_part'] = contrast_score - obj['score_selected'] = obj['score_new'] if BP_SCORE_MODE == 'combined' else obj['score_old'] - return objects - - -def prepare_bp_objects_for_display(objects, remove_sidelobes=True): - if remove_sidelobes: - visible = [obj for obj in objects if not obj.get('sidelobe_candidate', False)] - else: - visible = list(objects) - - if BP_SCORE_MODE == 'combined': - visible = sorted(visible, key=lambda obj: obj.get('score_new', 0.0), reverse=True) - else: - visible = sorted(visible, key=lambda obj: obj.get('score_old', obj.get('peak', 0.0)), reverse=True) - - for rank, obj in enumerate(visible, start=1): - obj['display_index'] = rank - return visible - - -def bistatic_depth_signature(x_obj, z_obj): - signature = [] - for i in range(N_tx): - for j in range(N_rx): - Rtx, Rrx = bistatic_ranges(i, j, x_obj, z_obj) - signature.append(0.5 * (Rtx + Rrx)) - return np.array(signature, dtype=float) - - -def mark_sidelobe_candidates(objects): - for obj in objects: - obj['sidelobe_candidate'] = False - obj['sidelobe_parent'] = None - obj['sidelobe_range_rms_cm'] = np.nan - obj['sidelobe_dx_cm'] = np.nan - obj['sidelobe_dz_cm'] = np.nan - - if not BP_SIDELOBE_DETECT: - return objects - - signatures = [bistatic_depth_signature(obj['x'], obj['z']) for obj in objects] - - for k, obj in enumerate(objects): - best_parent = None - best_rms_cm = np.inf - best_dx_cm = np.nan - best_dz_cm = np.nan - - for p in range(k): - parent = objects[p] - rel_peak = obj['peak'] / (parent['peak'] + 1e-12) - dx_cm = abs(obj['x'] - parent['x']) * 100.0 - dz_cm = abs(obj['z'] - parent['z']) * 100.0 - rms_cm = float(np.sqrt(np.mean((signatures[k] - signatures[p])**2)) * 100.0) - - is_candidate = ( - rel_peak <= BP_SIDELOBE_MAX_REL_PEAK and - dx_cm >= BP_SIDELOBE_MIN_DX_CM and - dz_cm <= BP_SIDELOBE_MAX_DZ_CM and - rms_cm <= BP_SIDELOBE_RANGE_RMS_TOL_CM - ) - - if is_candidate and rms_cm < best_rms_cm: - best_parent = parent - best_rms_cm = rms_cm - best_dx_cm = dx_cm - best_dz_cm = dz_cm - - if best_parent is not None: - obj['sidelobe_candidate'] = True - obj['sidelobe_parent'] = best_parent['index'] - obj['sidelobe_range_rms_cm'] = best_rms_cm - obj['sidelobe_dx_cm'] = best_dx_cm - obj['sidelobe_dz_cm'] = best_dz_cm - - return objects - - -if BP_SCORE_COMPUTE_INCOHERENT: - print('Расчет coherent + incoherent time-domain BP...', end=' ', flush=True) - bp_raw, bp_complex, bp_incoh_raw, bp_cf_map = backproject_coherent_and_incoherent(H_bp, T_bp, compensate=True) - bp_incoh_map_s = normalize_bp_map(bp_incoh_raw) -else: - print('Расчет coherent time-domain BP...', end=' ', flush=True) - bp_raw, bp_complex = backproject_coherent(H_bp, T_bp, compensate=True) - bp_incoh_raw = None - bp_incoh_map_s = None - bp_cf_map = None -bp_map_s = normalize_bp_map(bp_raw) -print('готово.') -print(f'BP score mode: {BP_SCORE_MODE} (peak=старый, combined=экспериментальный)') - -if BP_VALIDATE_COMPENSATION: - print('Расчет coherent BP без compensation для валидации...', end=' ', flush=True) - bp_raw_nocomp, bp_complex_nocomp = backproject_coherent(H_bp, T_bp, compensate=False) - bp_map_nocomp_s = normalize_bp_map(bp_raw_nocomp) - print('готово.') -else: - bp_raw_nocomp = None - bp_complex_nocomp = None - bp_map_nocomp_s = None - -bp_objects_all, bp_residual = find_bp_objects(bp_map_s) -bp_objects_all = add_phase_metrics(bp_objects_all, bp_complex) -bp_objects_all = add_local_prominence_metrics(bp_objects_all, bp_map_s) -bp_objects_all = add_incoherent_support_metrics(bp_objects_all, bp_incoh_map_s, bp_cf_map) -bp_objects_all = mark_sidelobe_candidates(bp_objects_all) -bp_objects_all = add_bp_score_metrics(bp_objects_all) - -bp_map_display = bp_map_s.copy() -if BP_REMOVE_SIDELOBE_OBJECTS: - for obj in bp_objects_all: - if obj['sidelobe_candidate']: - bp_map_display[obj['region_mask']] = 0.0 -bp_objects = prepare_bp_objects_for_display(bp_objects_all, remove_sidelobes=BP_REMOVE_SIDELOBE_OBJECTS) -bp_detected_object_count = len(bp_objects) -if bp_detected_object_count > BP_MAX_DETECTED_OBJECTS_TO_DRAW: - bp_objects_to_plot = [] -else: - bp_objects_to_plot = bp_objects[:BP_DRAW_TOP_M_OBJECTS] - -iz_max, ix_max = np.unravel_index(np.argmax(bp_map_s), bp_map_s.shape) -x_peak = x_grid_bp[ix_max] -z_peak = z_grid_bp[iz_max] -peak_value = bp_map_s[iz_max, ix_max] -main_obj = bp_objects[0] if bp_objects else None - -target_results = [ - { - 'rank': int(rank), - 'x_cm': round(float(obj['x'] * 100.0), 1), - 'z_cm': round(float(obj['z'] * 100.0), 1), - 'combined_score': round(float(obj.get('score_new', np.nan)), 4), - } - for rank, obj in enumerate(bp_objects_to_plot, start=1) -] -_print_raw(json.dumps({'targets': target_results}, ensure_ascii=False, indent=2)) - - -# ══════════════════════════════════════════════════════ -# 4. ТОЛЬКО BP-КАРТА -# ══════════════════════════════════════════════════════ - -fig, ax = plt.subplots(figsize=(12, 7)) -im = ax.imshow( - bp_map_display, - extent=[x_grid_bp[0]*100, x_grid_bp[-1]*100, z_grid_bp[-1]*100, z_grid_bp[0]*100], - aspect='auto', - cmap='jet', - vmin=0.25, - vmax=0.95, -) -plt.colorbar(im, ax=ax, label='Нормированная |coherent BP|') -ax.plot(x_tx * 100, z_tx * 100, 'r^', ms=12, label='Tx', zorder=5) -ax.plot(x_rx * 100, z_rx * 100, 'bv', ms=12, label='Rx', zorder=5) - -for obj in bp_objects_to_plot: - ax.contour( - x_grid_bp * 100, - z_grid_bp * 100, - obj['region_mask'].astype(float), - levels=[0.5], - colors='white', - linewidths=0.9, - alpha=0.75, - ) - ax.contour( - x_grid_bp * 100, - z_grid_bp * 100, - obj['center_mask'].astype(float), - levels=[0.5], - colors='cyan', - linewidths=0.8, - alpha=0.85, - ) - if obj['sidelobe_candidate']: - ax.plot(obj['x'] * 100, obj['z'] * 100, 'x', color='yellow', ms=9, - mew=2.0, zorder=7) - label_text = f"{obj['index']} SL" - text_color = 'yellow' - else: - ax.plot(obj['x'] * 100, obj['z'] * 100, 'wo', ms=7, - markeredgecolor='k', mew=0.8, zorder=6) - label_text = str(obj.get('display_index', obj['index'])) - text_color = 'white' - ax.text(obj['x'] * 100 + 3, obj['z'] * 100, label_text, - color=text_color, fontsize=9, weight='bold', zorder=7) - -if len(bp_objects_to_plot) > 0: - ax.plot([], [], 'wo', ms=7, markeredgecolor='k', mew=0.8, label='Центроид области') - if not BP_REMOVE_SIDELOBE_OBJECTS: - ax.plot([], [], 'x', color='yellow', ms=9, mew=2.0, label='Sidelobe candidate') - -ax.axhline(MIN_DEPTH * 100, color='white', lw=1.0, ls='--', alpha=0.75) -ax.set_xlabel('X [см]') -ax.set_ylabel('Глубина Z [см]') -map_title = f'Time-domain coherent BackProjection | motion={MOTION_CORRECTION_MODE}, score={BP_SCORE_MODE}, shown={len(bp_objects_to_plot)}/{bp_detected_object_count}' -if BP_REMOVE_SIDELOBE_OBJECTS: - map_title += ' (SL области скрыты на карте)' -ax.set_title(map_title) -ax.set_xlim(x_grid_bp[0] * 100, x_grid_bp[-1] * 100) -ax.set_ylim(z_grid_bp[-1] * 100, 0) -ax.legend(loc='lower right', fontsize=9) -ax.grid(alpha=0.22) -ax.invert_yaxis() -plt.tight_layout() -plt.show() diff --git a/Makefile b/Makefile index 9bd1806..e97f7c2 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,6 @@ CXX := g++ PYTHON ?= .venv/bin/python CXXFLAGS := -std=c++20 -O2 -Wall -Wextra -Wpedantic -pthread -MMD -MP LDFLAGS := -pthread -lrt -ORCH_LDFLAGS := $(LDFLAGS) -lusb-1.0 BUILD_DIR := build BIN_DIR := $(BUILD_DIR)/bin @@ -10,7 +9,7 @@ BIN_DIR := $(BUILD_DIR)/bin # --- CUDA for the sar3d processor (optional). Its kernels are compiled at run # time by NVRTC and launched through the CUDA driver API, so no nvcc is needed: # only nvrtc.h, cuda.h, libnvrtc and the NVIDIA driver's libcuda. Headers and -# libnvrtc come from the pip nvidia-cuda-* packages in .venv (requirements-cuda.txt, +# libnvrtc come from the pip nvidia-cuda-* packages in .venv (requirements.txt, # x86_64) or from a system toolkit in /usr/local/cuda (Jetson). Without them # data_processor still builds; selecting sar3d shows a CUDA warning. CUDA_ROOTS := $(wildcard $(CURDIR)/.venv/lib/python3*/site-packages/nvidia/*) /usr/local/cuda @@ -33,12 +32,7 @@ INCLUDES := \ -Idata_acq_and_processing/common_cpp/config/include \ -Idata_acq_and_processing/third_party \ -Idata_acq_and_processing/third_party/eigen \ - -Idata_acq_and_processing/sweep_orchestrator/include \ - -Idata_acq_and_processing/sweep_orchestrator/device_drivers/interfaces \ - -Idata_acq_and_processing/sweep_orchestrator/device_drivers/radar \ - -Idata_acq_and_processing/sweep_orchestrator/device_drivers/switches \ - -Idata_acq_and_processing/preprocessing/calibration_master/include \ - -Idata_acq_and_processing/preprocessing/reference_master/include \ + -Idata_acq_and_processing/preprocessing/bundles/include \ -Idata_acq_and_processing/preprocessing/data_preprocessor/include \ -Idata_acq_and_processing/processing/processors/include \ -Idata_acq_and_processing/processing/data_processor/include \ @@ -49,20 +43,8 @@ COMMON_SOURCES := \ data_acq_and_processing/common_cpp/ipc/src/shared_types.cpp \ data_acq_and_processing/common_cpp/config/src/run_config.cpp -ORCH_SOURCES := \ - data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_lifecycle.cpp \ - data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_transport.cpp \ - data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_protocol.cpp \ - data_acq_and_processing/sweep_orchestrator/device_drivers/switches/h7992_minimal_driver.cpp \ - data_acq_and_processing/sweep_orchestrator/device_drivers/switches/hmc349a_minimal_driver.cpp \ - data_acq_and_processing/sweep_orchestrator/src/sweep_orchestrator.cpp \ - data_acq_and_processing/sweep_orchestrator/src/main.cpp - PREPROC_SOURCES := \ - data_acq_and_processing/preprocessing/calibration_master/src/calibration_master.cpp \ - data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp \ - data_acq_and_processing/preprocessing/calibration_master/src/through_calibrator.cpp \ - data_acq_and_processing/preprocessing/reference_master/src/reference_master.cpp \ + data_acq_and_processing/preprocessing/bundles/src/channel_bundle_support.cpp \ data_acq_and_processing/preprocessing/data_preprocessor/src/data_preprocessor.cpp \ data_acq_and_processing/preprocessing/data_preprocessor/src/main.cpp @@ -87,42 +69,38 @@ PROCESSOR_SOURCES := \ # time. Built on demand (not part of `all`) for radar.model == kamil_adc. KAMIL_COLLECTOR_DIR := data_acq_and_processing/kamil_adc_collector # The collector now also drives the RF switches itself (switch-aware mode), so it -# reuses the orchestrator's switch drivers and the shared run_config loader, which +# uses the common H7992 driver and run_config loader, which # pull in the common config/ipc/locator headers and the vendored nlohmann/json. KAMIL_COLLECTOR_INCLUDES := \ -I$(KAMIL_COLLECTOR_DIR)/include \ -I$(KAMIL_COLLECTOR_DIR)/vendor/lcard \ -Idata_acq_and_processing/common_cpp/ipc/include \ -Idata_acq_and_processing/common_cpp/config/include \ - -Idata_acq_and_processing/sweep_orchestrator/device_drivers/interfaces \ - -Idata_acq_and_processing/sweep_orchestrator/device_drivers/switches \ + -Idata_acq_and_processing/common_cpp/hardware/include \ -Idata_acq_and_processing/processing/locator/include \ -Idata_acq_and_processing/third_party KAMIL_COLLECTOR_LDFLAGS := -pthread -ldl -lutil # Collector sources, including the shared drivers/config loader it reuses. These # are compiled into a private build/kamil/ tree (see rule below) so they never -# collide with the orchestrator's objects, which use different compile flags. +# collide with pipeline objects, which use different include paths. KAMIL_COLLECTOR_SOURCES := \ $(KAMIL_COLLECTOR_DIR)/src/main.cpp \ $(KAMIL_COLLECTOR_DIR)/src/tty_protocol_writer.cpp \ $(KAMIL_COLLECTOR_DIR)/src/capture_file_writer.cpp \ data_acq_and_processing/common_cpp/config/src/run_config.cpp \ data_acq_and_processing/common_cpp/ipc/src/shared_types.cpp \ - data_acq_and_processing/sweep_orchestrator/device_drivers/switches/h7992_minimal_driver.cpp \ - data_acq_and_processing/sweep_orchestrator/device_drivers/switches/hmc349a_minimal_driver.cpp + data_acq_and_processing/common_cpp/hardware/src/h7992_minimal_driver.cpp -SWEEP_ORCH_OBJS := $(addprefix $(BUILD_DIR)/,$(COMMON_SOURCES:.cpp=.o) $(ORCH_SOURCES:.cpp=.o)) PREPROCESSOR_OBJS := $(addprefix $(BUILD_DIR)/,$(COMMON_SOURCES:.cpp=.o) $(PREPROC_SOURCES:.cpp=.o)) DATA_PROCESSOR_OBJS := $(addprefix $(BUILD_DIR)/,$(COMMON_SOURCES:.cpp=.o) $(PROCESSOR_SOURCES:.cpp=.o)) KAMIL_BUILD_DIR := $(BUILD_DIR)/kamil -KAMIL_COLLECTOR_OBJS := $(addprefix $(KAMIL_BUILD_DIR)/,$(notdir $(KAMIL_COLLECTOR_SOURCES:.cpp=.o))) -DEPFILES := $(sort $(SWEEP_ORCH_OBJS:.o=.d) $(PREPROCESSOR_OBJS:.o=.d) $(DATA_PROCESSOR_OBJS:.o=.d) $(KAMIL_COLLECTOR_OBJS:.o=.d)) +KAMIL_COLLECTOR_OBJS := $(addprefix $(KAMIL_BUILD_DIR)/,$(KAMIL_COLLECTOR_SOURCES:.cpp=.o)) +DEPFILES := $(sort $(PREPROCESSOR_OBJS:.o=.d) $(DATA_PROCESSOR_OBJS:.o=.d) $(KAMIL_COLLECTOR_OBJS:.o=.d)) NATIVE_TEST_SOURCES := $(COMMON_SOURCES) $(LOCATOR_SOURCES) \ data_acq_and_processing/tests/native_regressions.cpp \ data_acq_and_processing/processing/data_processor/src/processing_live_config.cpp \ $(filter-out %/main.cpp,$(PREPROC_SOURCES)) \ - data_acq_and_processing/sweep_orchestrator/src/sweep_orchestrator.cpp \ $(KAMIL_COLLECTOR_DIR)/src/capture_file_writer.cpp \ $(KAMIL_COLLECTOR_DIR)/src/tty_protocol_writer.cpp NATIVE_BUILD_DIR := $(BUILD_DIR)/native_tests @@ -133,7 +111,6 @@ $(shell mkdir -p $(NATIVE_BUILD_DIR) && echo '$(NATIVE_CXXFLAGS)' | cmp -s - $(N DEPFILES += $(NATIVE_TEST_OBJS:.o=.d) TARGETS := \ - $(BIN_DIR)/sweep_orchestrator \ $(BIN_DIR)/data_preprocessor \ $(BIN_DIR)/data_processor @@ -147,33 +124,20 @@ check-native: $(NATIVE_BUILD_DIR)/native_regressions $(NATIVE_BUILD_DIR)/native_regressions run_config_examples/run_config_simulator.example.json SAR3D_TEST_SOURCES := data_acq_and_processing/tests/sar3d_fixture.cpp \ + data_acq_and_processing/common_cpp/ipc/src/shared_types.cpp \ data_acq_and_processing/processing/processors/src/sar3d_processor.cpp \ data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp SAR3D_TEST_OBJS := $(addprefix $(NATIVE_BUILD_DIR)/,$(SAR3D_TEST_SOURCES:.cpp=.o)) -SAR3D_EMULATION_OBJ := $(NATIVE_BUILD_DIR)/data_acq_and_processing/tests/sar3d_kernel_emulation.o -SAR3D_KERNEL_TEST_OBJ := $(NATIVE_BUILD_DIR)/data_acq_and_processing/tests/sar3d_kernel_regressions.o -SAR3D_EMULATION_HEADERS := $(NATIVE_BUILD_DIR)/sar3d_kernels.inc $(NATIVE_BUILD_DIR)/sar3d_helpers.inc -DEPFILES += $(SAR3D_TEST_OBJS:.o=.d) $(SAR3D_EMULATION_OBJ:.o=.d) $(SAR3D_KERNEL_TEST_OBJ:.o=.d) +DEPFILES += $(SAR3D_TEST_OBJS:.o=.d) -check-sar3d: $(NATIVE_BUILD_DIR)/sar3d_fixture $(NATIVE_BUILD_DIR)/sar3d_math_fixture $(NATIVE_BUILD_DIR)/sar3d_kernel_regressions - $(NATIVE_BUILD_DIR)/sar3d_kernel_regressions - $(PYTHON) data_acq_and_processing/tests/test_sar3d_oracle.py +# The sar3d processor on the real CUDA backend against the notebook reference; +# without a usable GPU the test checks the warning and skips the rest. +check-sar3d: $(NATIVE_BUILD_DIR)/sar3d_fixture + SAR3D_FIXTURE="$(abspath $<)" $(PYTHON) data_acq_and_processing/tests/test_sar3d.py $(NATIVE_BUILD_DIR)/sar3d_fixture: $(SAR3D_TEST_OBJS) $(CXX) $(NATIVE_CXXFLAGS) $^ -o $@ $(LDFLAGS) $(SAR3D_LDFLAGS) -$(NATIVE_BUILD_DIR)/sar3d_math_fixture: $(filter-out $(NATIVE_BUILD_DIR)/data_acq_and_processing/processing/processors/src/sar3d_gpu.o,$(SAR3D_TEST_OBJS)) $(SAR3D_EMULATION_OBJ) - $(CXX) $(NATIVE_CXXFLAGS) $^ -o $@ $(LDFLAGS) - -$(NATIVE_BUILD_DIR)/sar3d_kernel_regressions: $(SAR3D_KERNEL_TEST_OBJ) $(SAR3D_EMULATION_OBJ) - $(CXX) $(NATIVE_CXXFLAGS) $^ -o $@ $(LDFLAGS) - -$(SAR3D_EMULATION_HEADERS) &: data_acq_and_processing/tests/generate_sar3d_emulation.py data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp data_acq_and_processing/processing/processors/src/sar3d_gpu.hpp - $(PYTHON) data_acq_and_processing/tests/generate_sar3d_emulation.py $(NATIVE_BUILD_DIR) - -$(SAR3D_EMULATION_OBJ): NATIVE_CXXFLAGS += -I$(NATIVE_BUILD_DIR) -$(SAR3D_EMULATION_OBJ): $(SAR3D_EMULATION_HEADERS) - $(NATIVE_BUILD_DIR)/data_acq_and_processing/processing/processors/src/sar3d_gpu.o: NATIVE_CXXFLAGS += $(SAR3D_CXXFLAGS) $(NATIVE_BUILD_DIR)/data_acq_and_processing/processing/processors/src/sar3d_gpu.o: $(SAR3D_CUDA_STAMP) @@ -184,10 +148,6 @@ $(NATIVE_BUILD_DIR)/%.o: %.cpp $(NATIVE_FLAGS_STAMP) @mkdir -p $(dir $@) $(CXX) $(NATIVE_CXXFLAGS) $(INCLUDES) -I$(KAMIL_COLLECTOR_DIR)/include -c $< -o $@ -$(BIN_DIR)/sweep_orchestrator: $(SWEEP_ORCH_OBJS) - @mkdir -p $(BIN_DIR) - $(CXX) $(SWEEP_ORCH_OBJS) -o $@ $(ORCH_LDFLAGS) - $(BIN_DIR)/data_preprocessor: $(PREPROCESSOR_OBJS) @mkdir -p $(BIN_DIR) $(CXX) $(PREPROCESSOR_OBJS) -o $@ $(LDFLAGS) @@ -200,12 +160,7 @@ $(BIN_DIR)/data_processor: $(DATA_PROCESSOR_OBJS) $(SAR3D_CUDA_STAMP) $(BUILD_DIR)/data_acq_and_processing/processing/processors/src/sar3d_gpu.o: CXXFLAGS += $(SAR3D_CXXFLAGS) $(BUILD_DIR)/data_acq_and_processing/processing/processors/src/sar3d_gpu.o: $(SAR3D_CUDA_STAMP) -# The collector compiles into its own build/kamil/ tree with collector-only -# includes, so the shared driver/config sources it reuses never collide -# with the orchestrator's objects of the same name. Basenames are unique, so a -# vpath lets one pattern rule find every source. -vpath %.cpp $(sort $(dir $(KAMIL_COLLECTOR_SOURCES))) - +# Keep source-relative paths so moving a source also replaces its dependencies. $(KAMIL_BUILD_DIR)/%.o: %.cpp @mkdir -p $(dir $@) $(CXX) $(CXXFLAGS) $(KAMIL_COLLECTOR_INCLUDES) -c $< -o $@ diff --git a/README.md b/README.md index 791bd54..0ca219a 100644 --- a/README.md +++ b/README.md @@ -2,25 +2,25 @@ A real-time radar acquisition and imaging system for vector-network-analyzer (VNA) and ADC front-ends. It drives the radar through a frequency sweep, calibrates and -preprocesses the measured S-parameters, runs a selectable imaging/detection +preprocesses the measured S21 traces, runs a selectable imaging/detection processor, and presents the live result in a desktop GUI and an in-process web interface. The same application runs interactively on a workstation or unattended as a headless Raspberry Pi appliance. ## Overview -The system is split into a **C++ real-time pipeline** and a **Python application** -that supervises it and handles visualization and control. +The system uses **Python acquisition producers**, **C++ preprocessing and processing**, +and a **Python application** for supervision, visualization and control. The pipeline runs as three independent processes connected by lock-free POSIX shared-memory ring buffers. Each stage advances on its own, and a slow consumer never blocks a producer — the rings are latest-wins, so a reader that falls behind -simply skips to the freshest frame: +skips overwritten frames and continues from the retained tail: -1. **Acquisition** — `sweep_orchestrator` (or a hardware-specific producer) - configures the radar, performs the frequency sweep, and publishes raw S11/S21 - traces. -2. **Preprocessing** — `data_preprocessor` applies through/OSL calibration, +1. **Acquisition** — Python producers configure the radar, acquire frequency sweeps, + and publish raw S21 traces. Both single and multi-device LibreVNA use + `matrix_raw_producer`; Kamil ADC uses `kamil_adc_raw_producer` with its C++ collector. +2. **Preprocessing** — `data_preprocessor` applies through calibration, reference subtraction, optional frequency-domain notch filtering, and an optional object-based MIMO correction of S21. 3. **Processing** — `data_processor` runs the selected processing mode and emits @@ -28,9 +28,9 @@ simply skips to the freshest frame: or 3D SAR views). Detected objects are additionally broadcast to subscribers over a TCP locator server. -The **Python application** (`python_app`) starts and monitors the C++ binaries, -reads the result rings, renders the live view with pyqtgraph, and exposes the same -controls plus a pixel-identical live plot through an embedded web UI. +The **Python application** (`python_app`) starts and monitors the producer and C++ processes, +reads the result rings, renders the live view with pyqtgraph, and exposes acquisition +controls, processing settings and a copy of the live plot through an embedded web UI. ``` radar hardware @@ -45,12 +45,13 @@ controls plus a pixel-identical live plot through an embedded web UI. - **librevna** — single LibreVNA. - **librevna_multi** — synchronized multi-device LibreVNA. -- **kamil_adc** — external Kamil ADC acquisition process, with optional laser control. +- **kamil_adc** — [Kamil ADC collector](data_acq_and_processing/kamil_adc_collector/README.md), + with optional laser control and [temperature monitoring](python_app/hardware_full/laser_control/monitoring/README.md). - **mock / simulator** — driver-less mode for development without hardware. ## Processing modes -- **pass_through** — magnitude and phase of the selected channel for each combo. +- **pass_through** — magnitude and phase of S21 for each combo. - **bscan** — depth-versus-sweep B-scan heatmap. - **gpr** — coherent back-projection GPR imaging with object detection. - **legacy_gpr** — ellipse-intersection MIMO localization (point and extended/region modes). @@ -58,30 +59,29 @@ controls plus a pixel-identical live plot through an embedded web UI. on an NVIDIA GPU through CUDA: front (X-Y), side (Z-Y), top (X-Z) and oblique maximum projections of the imaged volume. Its settings carry the notebook's parameter names; the scene window is - fixed (X −2…2 m, height −0.5…2 m above the ground), its range extent follows + set by the selected antennas with a 2 m X margin and height −0.5…2 m above the ground; its range extent follows `MIN_DEPTH`/`MAX_DEPTH`, and the voxels are cubes with edge `SAR_VOXEL_M` (default 2 cm). The radar motion is corrected from each trace's capture timestamps, so the sweep timing needs no parameters. -For matrix-radar B-scans, **Next combination** advances output first, then input, -and wraps to the first pair. The desktop fields and browser channel selector -redraw the selected cached B-scan immediately without restarting acquisition. -**Save Last 1000** and **Save S21 JSON** use a fixed limit of 1000 recent -measurements from each available stage; older GUI profiles may still contain -`save_count`, which is ignored. **Start + Record** retains its separate count for -recording future measurements to disk. +For B-scans on every radar model, **B-scan pair** selects a pair from the displayed +history independently of the run combinations. **Next combination** advances output +first, then input, and wraps to the first recorded pair; it is enabled with at least +two pairs. Desktop and browser selectors redraw the cached B-scan immediately. Tx/Rx antenna positions are edited in tables with switch position and X/Y/Z in metres, on both the desktop and web interface. Coordinate and legacy-GPR medium changes apply to the running processor and recompute its retained result. Profiles -preserve all three coordinates. GPR/SAR views wait for a result marked with the -current settings revision, so an older in-flight result cannot overwrite new geometry. +preserve all three coordinates. A new table row becomes active after X is entered; +desktop, web and profile geometry use one Python validator; C++ validates its own +input. GPR/SAR views wait for a result marked with the current settings revision, +so an older in-flight result cannot overwrite new geometry. ## Repository layout | Path | Contents | | --- | --- | -| `data_acq_and_processing/` | C++ pipeline: `sweep_orchestrator/`, `preprocessing/`, `processing/` (including `locator/`), shared `common_cpp/` (config, IPC rings), and vendored `third_party/` (Eigen, nlohmann/json). | +| `data_acq_and_processing/` | C++ pipeline: `preprocessing/`, `processing/` (including `locator/`), `kamil_adc_collector/`, shared `common_cpp/` (config, IPC rings, H7992 driver), and vendored `third_party/` (Eigen, nlohmann/json). | | `python_app/` | Python application: `gui/`, `orchestration/` (process supervisor, shared-memory readers, config), `hardware_full/` (device drivers and services), `webui/`, `storage/`, `models/`, `workflows/`, `scripts/` (producers and tools), and `tests/`. | | `run_config_examples/` | Ready-made configuration profiles, one per hardware setup. | | `deploy/` | systemd daemon installer, and the USB update kit for the Pi and Jetson (`pack-usb.sh`, `update.sh`). | @@ -94,7 +94,7 @@ current settings revision, so an older in-flight result cannot overwrite new geo **Raspberry Pi 5**; hardware PWM gating for the generator sweep is Pi 5 specific (GPIO12/13/18/19, `dtoverlay=pwm-2chan`). - A C++20 compiler and `make`; Eigen and nlohmann/json are vendored. -- `libusb-1.0` development headers and `libatomic.so.1` (`libatomic1` on Debian/Ubuntu). +- `libusb-1.0` runtime and `libatomic.so.1` (`libatomic1` on Debian/Ubuntu). Python ring readers/writers use lock-free atomic loads, stores and cursor CAS through libatomic, including memory barriers for ARM. Slot sizes must be multiples of 8 bytes, and the complete ring mapping must not exceed 1 TiB. @@ -102,7 +102,7 @@ current settings revision, so an older in-flight result cannot overwrite new geo - For `sar3d`: an NVIDIA GPU with its driver (`libcuda`). The kernels are compiled at run time by NVRTC, so there is no nvcc step; `make` needs only `nvrtc.h`, `cuda.h` and `libnvrtc`, taken from the pip - `nvidia-cuda-*` packages in `.venv` (`requirements-cuda.txt`, x86_64) or from + `nvidia-cuda-*` packages in `.venv` (`requirements.txt`, x86_64) or from a system toolkit in `/usr/local/cuda` (Jetson). Without these dependencies or a usable GPU, SAR3D shows a warning above the plots and does not calculate an image. Other processing modes remain available. @@ -115,57 +115,50 @@ From the repository root: python3 -m venv .venv .venv/bin/python -m pip install --upgrade pip .venv/bin/python -m pip install -r requirements.txt -make # build the C++ binaries into build/bin/ +make # data_preprocessor and data_processor in build/bin/ +make kamil_adc_collector # also required for Kamil ADC acquisition ``` `make` builds the sar3d GPU code when it finds CUDA (see Requirements) and prints a note when it does not. After installing the CUDA requirements into an existing venv, run `make` again: it notices the change and rebuilds that part. -For SAR3D on x86_64, install the optional packages with -`.venv/bin/python -m pip install -r requirements-cuda.txt`, then run `make`. +CUDA 13 is installed from `requirements.txt` on x86_64. For GPUs older than +sm_75, install NVRTC 12 outside `.venv` and build with +`make BUILD_DIR=/tmp/build-nvrtc12 CUDA_ROOTS="/nvidia/cuda_nvrtc /nvidia/cuda_runtime"`. +A separate build directory keeps the two CUDA builds independent. -Run the desktop GUI: +Run the desktop GUI through the launcher for build and single-instance handling: ```bash -.venv/bin/python -m python_app.gui.main -# or, with the build step and single-instance handling: -./start.sh +./start.sh --profile run_config.json # explicit local mock profile +# or choose a hardware profile from run_config_examples/ ``` -`start.sh` supports `--headless` (offscreen, auto-start — for unattended Pi use), -`--profile PATH`, `--producer-only`, `--skip-build`, and `--clean-shm`; see -`./start.sh --help`. The acquisition device (LibreVNA, Kamil ADC, …) is detected -automatically from the active config's `radar.model` — there is no per-device flag. +`start.sh` also supports `--headless` (offscreen, auto-start), `--auto-start`, +`--producer-only`, `--skip-build`, `--build-only`, and `--clean-shm`; see +`./start.sh --help`. Pass `--profile PATH` so launcher provisioning and the GUI +use the same profile. The producer is selected from `radar.model`, with dataset +replay taking precedence. There is no per-device flag. Provision dependencies +before unattended use: headless startup does not install missing packages. + +For development, `.venv/bin/python -m python_app.gui.main` starts the GUI directly, +without the launcher's build step or single-instance lock. ## Jetson (aarch64) bring-up -Ubuntu 22.04 on Jetson has glibc 2.35, and PyQt6 6.8+ ships only -`manylinux_2_39` aarch64 wheels — `requirements.txt` therefore pins PyQt6 to -`<6.8` on aarch64 (6.7.1 is the last release with a compatible wheel). With -that pin the stock `./start.sh` provisioning works; the extra steps are only -for the CUDA-backed `sar3d` processor: +`requirements.txt` pins PyQt6 below 6.8 on aarch64 for the older glibc used by +Jetson images. CUDA comes from JetPack in `/usr/local/cuda`; pip CUDA packages +are installed only on x86_64. Use the same build commands and launcher as above. +For dataset replay, first copy the dataset locally and update +`radar.replay.dataset_dir` in the profile; example paths are machine-specific: ```bash -# 1. copy the replay dataset (for dataset-replay profiles) and point -# radar.replay.dataset_dir in the profile at the local copy — the path in -# run_config_examples/*.json is machine-specific. - -# 2. normal launch; make takes the JetPack CUDA from /usr/local/cuda -# (requirements.txt installs no pip nvidia-* packages on aarch64) and the -# driver's /usr/lib/aarch64-linux-gnu/nvidia/libcuda.so.1: ./start.sh --profile run_config_examples/run_config_replay_vna.example.json ``` -Ubuntu 22.04 ships Python 3.10, and the host parses the radar stream in Python: -the same parsing code ran 1.6 times slower on 3.10 than on 3.13 (the Pi's version) -in a benchmark. Use Python 3.13 for the venv, installed with uv so the system -`python3` stays untouched (apt depends on it): - -```bash -curl -LsSf https://astral.sh/uv/install.sh | sh # then open a new terminal -uv python install 3.13 -bash radar_update/update.sh --python 3.13 # see Deployment -``` +The USB updater can create a fresh venv with an already installed interpreter, +for example `bash radar_update/update.sh --python 3.13`. The system Python does +not need to be replaced. ## Running without hardware (emulation) @@ -180,31 +173,38 @@ along the antenna array and between depths 3.25–4.25 m over an eight-second cy Each S21 uses the actual transmitter–target–receiver distance from the configured XYZ coordinates; coincident antennas produce identical signals. All combinations share the same target position and measured capture window, with collections paced -at 50 ms intervals. B-scan's default range is 0–8 m; stored profiles retain their -chosen range. The simulator models a stationary radar, so radar speed should be zero. +at 50 ms intervals. The root mock profile explicitly selects a 0–8 m B-scan +range. Both shipped mock profiles use a 0.445 m vertical antenna baseline +(Rx1/Rx3 and, for the four-output matrix, Tx2/Tx3), avoiding coincident pairs. +The general defaults are 0.824–1.0 m; stored profiles retain their range. The +simulator models a stationary radar, so radar speed should be zero. The scene uses the antenna positions from its startup profile; live geometry edits change the reconstruction of captured sweeps, whose samples remain unchanged. The preprocessor still requires S21 calibration/reference sets. Start the GUI: ```bash -./start.sh +./start.sh --profile run_config.json ``` -In **Preprocessing Panel**, use **Create neutral sets**, then **Start**. This makes +Open **Preprocessing**, use **Create Neutral S21 Sets**, then **Start**. This makes identity calibration + zero reference, preserving the simulated S21 samples. **Dataset replay (recorded scene).** When `radar.replay.dataset_dir` is set in the profile, the supervisor launches `python_app.scripts.replay_raw_producer` instead of the normal producer: it -replays recorded snapshots into the raw ring at the recorded cadence, looping -forever with a monotonic timeline. Pair it with NEUTRAL preprocess sets so the -already-preprocessed recording passes through the preprocessor bit-exactly. -A ready profile for the VNA_AlGo tank005 dataset is provided: +replays the collection directories in that folder into the raw ring. Point the +path at a saved stage such as `dataset/preprocessed`, containing +`NNNN_idK_ns/` directories with `meta.json` and `i*_o*_{freq,s21}.npy`. +Replay follows the recorded cadence scaled by `radar.replay.rate_scale`; +`radar.replay.loop` defaults to true. Trace windows preserve the recorded spacing +for SAR, even with accelerated replay. Each collection starts at its actual publication +time for acquisition identity and retains the original duration for legacy GPR. +For an already-preprocessed recording, select neutral through/reference sets +and disable notch and object correction to preserve the recorded S21 values. +A ready profile for the VNA_AlGo tank005 dataset is provided. Its neutral sets are +created once with the **Create Neutral S21 Sets** button of the preprocessing panel: ```bash -# one-time: neutral sets for this profile (or use the GUI button) -.venv/bin/python -m python_app.scripts.create_neutral_preprocess_sets \ - --config run_config_examples/run_config_replay_vna.example.json # interactive GUI (mode sar3d is pre-selected by the profile's gui section): ./start.sh --profile run_config_examples/run_config_replay_vna.example.json ``` @@ -214,77 +214,195 @@ a faster preview matters more than spatial resolution. Processing failures appea above the plots and a failed single capture finishes with its explanation while retaining the acquired data. -The web interface is served in-process at `http://:8080`. It mirrors the live -plot and offers Start / Stop / Capture controls and the active processor settings. +## Web interface -Manual dataset and JSON export keep the last 1000 available collections from each -stage independently, including raw or preprocessed sweeps without a result. -Streaming recording keeps every collection received after arming until the requested -result count or Stop; Stop and closing the window flush accepted data. Acquisition -rings use latest-wins delivery, so disk recording cannot recover sweeps overwritten -before the GUI receives them. NumPy datasets preserve both S-parameters, capture +The embedded server listens at `http://:8080`; set +`RADAR_SYSTEM_WEBUI_PORT` to use another port. It is available in desktop and +headless mode. Browser actions are: + +- **Start**, **Single Capture**, **Stop**, **Tmp Reference** and **Remove Last Measurement**; +- load a profile from `run_configs/*.json` using **Load**; +- **Save Last 1000** and **Start + Record** with a path, optional name and recording count; +- edit processor settings and antenna tables with **Apply**, or clear history with **Reset history**; +- select a **B-scan pair** or advance with **Next combination**. + +Copy profiles from `run_config_examples/` into `run_configs/` to offer them in the +browser. JSON export and calibration dialogs are desktop actions. PNG capture +runs only while a browser is connected and the plot has changed, including zoom +and resize, at most five times per second. Each client retains the latest frame, +status and settings independently; reconnecting refreshes settings, and **Apply** +sends only edited fields. + +## Data and preprocessing + +### Snapshots and recording + +**Save Last 1000** and **Save S21 JSON** keep up to 1000 available collections from +each stage independently, including raw or preprocessed sweeps without a result. +The limit is fixed; legacy profile `save_count` values are ignored. Saving +keeps an active disk recording running. +Without raw/preprocessed taps, result snapshots remain available, but S21 JSON +requires retained traces. +**Start + Record to Disk** (**Start + Record** in the browser) has a separate count +for future results. It keeps collections received after arming until that count, +Stop, or exhaustion of the bounded writer queue. If storage falls behind, +recording stops accepting data, drains the queue and final chunk, and reports the limit. +Stop and closing the window also flush accepted data without blocking the GUI. +Pausing acquisition for a capture or history action ends an active recording: +the log reports the accepted count, requested count and stop reason before draining +the writer. Latest-wins rings cannot recover sweeps overwritten before the GUI +receives them. NumPy datasets preserve S21, capture windows, every supported result payload, and processing time. Failed exports remove their staging files, leaving existing datasets intact. Companion profiles retain the radar, antenna layout and preprocessing used for acquisition, including when -another profile is loaded after Stop. +another profile is loaded after Stop. Dataset name prefixes also describe the acquisition; +a manual save is cancelled if acquisition is restarted during its final history drain. + +Each NumPy dataset includes `processing_settings.json`, mapping result revisions to +the live settings used for processing. Results retain their revision, capture window, +and actual `speed_m_s` (including speed supplied by the locator). Settings +changed while acquisition is stopped are recorded for the reprocessed results as well. +**Remove Last Measurement** uses capture order and removes matching collections +from every stage; the processor suppresses that sweep if it is still pending. +**Clear Runtime History** (**Reset history** in the browser) clears all retained +stages and rejects late results from before the cutoff. These actions leave saved +files intact. A sweep is identified by `(collection_id, capture_start_ns)`; all +stages preserve the producer's capture window. `monotonic_ns` is diagnostic only. + +**Save S21 JSON** can add exports to an existing NumPy dataset. Existing JSON files +are never overwritten; missing raw/preprocessed counterparts produce empty arrays, +and stages pair only by the exact acquisition key. Its `vna_config_profile.json` companion +leaves the NumPy dataset's `config_profile.json` intact. A streaming write failure +keeps all collections already published; only the incomplete collection is discarded. +The offline snapshot converter uses this same exporter. Legacy snapshots without +capture windows retain repeated IDs through one-to-one time/index matching; +processing completion timestamps are not compared with acquisition timestamps. + +### Calibration and reference sets Calibration and reference sets keep their trace arrays and metadata together in one -atomically replaced NPZ archive, with a readable JSON sidecar. Earlier NPZ/JSON pairs +atomically replaced NPZ archive, with a readable JSON sidecar. Both files and their +directory entries are synced before successful completion. Existing file permissions +are retained; new files and directories respect the process umask. Earlier NPZ/JSON pairs remain supported. Invalid trace shapes, duplicate combinations and non-finite calibration/reference values are rejected before a saved set is replaced. Median captures also require identical frequency axes and channel coverage. -In **Preprocessing Panel**, create or select the ordinary S21 calibration and -empty-scene reference first. **Create…** in the **Object Calibration** row opens a separate step: -take fresh sweeps, then enter the measured reflector centre X/Y/Z in metres. -Height above ground, radar pitch and sphere radius follow `mimo_calibration.py`; -several reflector positions may contribute to one calibration. The calculation -shows each pair's phase at the target before/after correction and its coherence. -The selected reference is already subtracted, and the previous object correction -is disabled while these sweeps are acquired. Saved object corrections apply on the -next **Start**, after notch filtering and before any processor; **Off** disables -them. Changing antenna geometry disables an incompatible correction and reloads -acquisition without restarting the application. **Import…** in that row -accepts the example's `cal_*.npy` / `cal_meta.json` format; the example and GUI share -one numerical implementation. +Open **Preprocessing** to create or select the ordinary S21 calibration and +empty-scene reference. The processing order is through calibration → calibrated +reference subtraction → notch → object correction → the selected processor. -Dataset replay restores both recorded S-parameters and orders long recordings by -their numeric index; older S21-only datasets still replay with zero S11. Offline -JSON conversion and the GUI share the same sweep-matching code. +### Object calibration + +**Create…** in the **Object Calibration** row opens a separate dialog. Use +**Capture data** to take fresh preprocessed sweeps, then enter the measured +reflector centre X/Y/Z in metres and use **Calculate and save**. +Coordinates follow the notebook: X across the array, Y above the ground, Z along +the ground. +Height above ground, radar pitch and sphere radius follow `MIMO_calibration.ipynb`; +several reflector positions may contribute to one calibration. The only diagnostic +plot shows the phase of each antenna pair's contribution at the target point, +before and after correction, as in cell 5 of `MIMO_calibration.ipynb`: Tx rows, +Rx columns, phase relative to the complex sum of each map, with cell values and +standard deviation in degrees on a cyclic ±180° scale. +With partial antenna geometry, only the specified Tx × Rx pairs contribute to +the calculation and phase maps. Acquisition still retains every physical stage; +pairs outside that geometry receive unit correction and appear in `unchanged_combos`. +The selected reference is already subtracted, and the previous object correction +is disabled while these sweeps are acquired. A capture owns an exclusive pipeline +pause until it finishes or is cancelled, including during automatic-restart backoff. +An explicit **Stop** cancels automatic resumption. + +Object corrections use the same NPZ collection format as other preprocessing sets: +S21 holds each pair's multiplier on the captured frequency grid. Metadata retain +the capture radar key, antenna geometry, through-set name and content hash, target +positions, estimation settings and phase diagnostics. Saving uses the capture's +configuration even if the current profile changes during calculation. The next +**Start** checks the selected through set's S21 and frequency arrays, geometry and +coverage of the requested combinations before applying the correction after notch +filtering and before any processor. **Off** disables it. Changing antenna geometry +disables an incompatible correction and reloads acquisition without restarting +the application. Changing relative permittivity only updates processing settings. + +Legacy `cal_*.npy` / `cal_meta.json` folders are not application calibration sets. +Capture a set through **Create…** to use object correction; the legacy files on +disk are left intact. Ordinary through/reference NPZ sets need no conversion. + +Older NPZ sets and snapshots remain readable: extra S11 arrays are ignored; +new trace files contain only frequency and S21. + +## Runtime interfaces + +Tap rings are optional: an empty `rings.raw_tap.name` or +`rings.preprocessed_tap.name` disables that copy and its GUI history. Core processing +and single capture still work without taps; object calibration needs the preprocessed +tap. All producers share raw publication and hardware retry handling. + +Raw and preprocessed IPC use **RAW3** and **PRP3** respectively: each trace contains +float32 frequencies followed by complex64 S21 (12 bytes per point). The complete +frame requires collection and per-trace capture windows; its size is `44 + combos * (28 + 12 * points)` bytes. +Older RAW2/PPR2 bundles are incompatible and are rebuilt from NPZ sets at Start; +update Python and C++ together. Results use RSL3; older RSL2 frames are rejected. + +RF switches use H7992 with 1–4 positions and two GPIO control lines. Profiles use +`driver_mode`, GPIO settings and the position count. An old `driver: "h7992"` is accepted; +other explicit driver values or enabled inversion are rejected before GPIO opens. +Two positions use the existing H7992 mapping `(A,B) = (0,0), (0,1)`; physical +wiring and channel numbers follow the profile. + +Single-device LibreVNA acquires the configured `run.combos` in order. Multi-device +acquisition keeps the full matrix: SAR3D derives intra-sweep stage timing from +the complete list of outputs sharing a capture window. ## Configuration -The active profile is `run_config.json`. It selects the radar model and sweep, the -switch matrix and acquisition combos, the preprocessing calibration/reference -assets, the GPR geometry, and the shared-memory ring sizes. Profiles for each -hardware setup are provided in `run_config_examples/` — copy one to -`run_config.json`, or pass it with `--profile`, to use it. +A profile selects the radar model and sweep, switches and acquisition combos, +preprocessing sets, antenna geometry, ring sizes and GUI settings. The GUI loads +`RADAR_SYSTEM_PROFILE` (set by `start.sh --profile`), otherwise the last profile in +`python_app/runtime/gui_session_state.json`, otherwise root `run_config.json`. +If a selected external profile cannot be loaded, it reports the error and falls +back to the root profile. **Load Config** and **Save Config** manage desktop profiles. + +Hardware examples are in `run_config_examples/`. The root `run_config.json` is +intentionally mock; select a hardware profile explicitly on a real device. A +change of model, serials or sweep parameters changes the `radar_key` used to find +calibration/reference sets; select or capture matching sets before **Start**. ## Deployment On a Raspberry Pi, `deploy/install-daemon.sh` installs a systemd service that runs -the application headless and restarts it on failure. A single-instance lock ensures -the interactive GUI and the daemon never run at the same time, since they share the -radar, the shared-memory rings, and the locator port. +the application through `start.sh --headless --skip-build` and restarts it on +failure. The launcher holds a single-instance lock for the radar, rings and locator +port; an interactive launch first asks systemd to stop an active daemon. **Updating a device from a USB flash drive.** On the development machine -`deploy/pack-usb.sh` writes the host part of the last commit (no venv, build, data -or local configs) and the update script into `radar_update/` on the flash drive. +`deploy/pack-usb.sh /path/to/drive` writes the host part of the **last commit** +and its update script into `radar_update/` on the flash drive. Uncommitted changes, +venv, build outputs, recorded data and local configs are excluded. On the Pi or Jetson, close the program and run, from the drive's folder: ```bash bash radar_update/update.sh # or: --python 3.13 for a fresh venv ``` -It installs the newest archive into `~/Documents/radar_system` (or `RADAR_DIR`); -the current version is renamed to `radar_system`, the next free number. Kept from -it: `run_config.json`, profiles in `run_config_examples/` changed or added on the -device (the release version of a changed one lands next to it as `.new`), -`python_app/runtime/` without logs, `python_app/data*` and any other top-level -file or folder that is not part of the release, as hard links. The venv is copied -independently so package updates cannot modify the rollback environment (or a new -one is built with `--python`). It builds the C++ pipeline and ADC collector and restarts -`radar.service` if it was running. To go back: -`rm -rf radar_system && mv radar_system radar_system`. +Run the updater as the normal device user. It installs the newest archive into +`~/Documents/radar_system` (or `RADAR_DIR`), renaming the current version to the +next free `radar_system`. It preserves: + +- `run_config.json` and changed/added `run_config_examples/*.json` by independent + copy; a conflicting release profile is placed beside the device profile as `.new`; +- `python_app/runtime/` by copy, excluding logs and `*.pids`; +- `python_app/data*` and extra non-hidden top-level entries absent from the release + by hard links, excluding `build`, `graphify-out`, `device_firmware`, `docs` and `RELEASE`; +- the venv by independent copy, or a fresh one when `--python` is supplied. + +Saved data is written through replacement or new files, preserving the backup's +hard links. `RELEASE` gets a new inode. The updater builds the C++ pipeline and ADC +collector, then restarts `radar.service` only if it was running before the update. +Keep the device's real hardware profile: the release's root profile is mock. + +For rollback, stop `radar.service` and close the GUI, rename the new installation +aside to preserve measurements taken since the update, then rename the chosen +backup to `radar_system`. Restart the service if it is normally used. ## Tests @@ -302,10 +420,93 @@ also exercise mutated frames, ring concurrency, locator input and ADC spool reco Firmware host tests check cart timer rollover and SBUS packing; they do not validate physical timing. Real devices, GPIO and the CUDA backend need their own hardware run. +`check-sar3d` runs the real CUDA processor (`data_acq_and_processing/tests/test_sar3d.py`). +It compares it with `sar3d_reference.npz`: compact float32/complex64 inputs, the three +maximum projections of the notebook's volume, and its joint peak/level for four +compensation/normalization and timing cases. The expected values come from unchanged +functions in cells 0 and 4 of the original notebook. The same test checks the published +status for invalid input, a result too large for the ring slot, an empty and a very weak +signal. Without a usable GPU it checks only the CUDA warning and skips the rest; the +notebook itself is not needed to run it. To regenerate the reference, pass the +original notebook (it is kept outside the repository): + +```bash +.venv/bin/python data_acq_and_processing/tests/generate_sar3d_reference.py \ + --notebook /path/to/Multi_antenna_3D_clean_v1.ipynb +``` + +The archive records cell-source SHA256 hashes and NumPy/SciPy versions. Another build +can be checked with `make check-sar3d BUILD_DIR=/path/to/build` or by running +`test_sar3d.py` with `SAR3D_FIXTURE=/path/to/build/native_tests/sar3d_fixture`. +For older GPUs, pass the same `BUILD_DIR` and NVRTC 12 `CUDA_ROOTS` used to build. + +For development, install `pyright` into the venv, then run +`.venv/bin/pyright python_app` to check all Python modules and tests. + +## Application architecture + Invalid device/switch settings and undersized raw/preprocessed slots fail before acquisition starts. Oversized processed results report the required slot size in the log; switching to a mode that fits resumes processing without a restart. +`AppWindow` assembles controllers and shuts them down. `WindowView` owns layout +and typed sections; controllers receive concrete dependencies, without GUI mixins +or duplicate window state. `ProfileController` owns startup selection, session +state and profile persistence. `RadarSettingsController` applies device limits, +while `LiveSettingsController` publishes revisions and updates plots. + +GUI scalar settings use dataclass metadata in `models/gui_profile_schema.py`: +profile validation, desktop values, live processing fields, processing log summaries +and web forms share those bindings. Hardware fields and geometry have their own +typed mappings and validators. Section builders return typed widget groups; their +controllers connect signals directly, with no string table of action callbacks. +`gui/controllers/config_controller.py` reads and applies scalar metadata without owning +the window or acquisition processes. Web batches roll back together when validation +or live-file publication fails; desktop settings also revert to the last publication. +Profile loading blocks editor signals until all settings are populated. + +`HistoryController` owns retained acquisitions; `PlotController` selects the view +and supplies independent B-scan, GPR, trace and SAR renderers with typed settings. +Renderers own their plots and display caches, without access to the acquisition +processes. Stopped B-scan replay uses the last run's ring, matches exact acquisition +keys and the requested settings revision, and cancels when that run changes. With +no retained preprocessed sweeps (including disabled taps), the processor recomputes +its own retained history. Capture previews preserve the selected processing view. +`PipelineController` owns processes, ring readers, polling/recovery, Single Capture +and replay. Its pause prevents recovery from starting a producer over a hardware +capture; Stop or Close cancels any pending resumption, including during a drain. +Validation rejects Start before stopping existing processes. Pauses resume only +continuous acquisition, retry failed resumption with the usual backoff, and release +ownership before presenting capture errors. GPIO errors are logged without dialogs. +Rendering failures leave acquisition and the locator running and log each cause once; +repeated ring-read failures retain their separate recovery and stop policy. +`PreprocessController` owns calibration/reference sessions and asset selection. +One `SequentialCaptureSession` handles both current settings and a list of sweep +variants; a single variant adds no extra acquisition or reconfiguration. Multi-device +collection always captures one complete synchronized cycle. The native +`DataPreprocessor` owns all three bundles in `preprocessing/bundles`: through, +through-calibrated reference and object correction, applied in that order with notch +filtering before object correction. +Object capture checks both acquisition processes and releases its pause only after +completion or cancellation; losing the producer reports a failure immediately. +`SnapshotController` coordinates manual exports, paused Clear/Remove operations +and Start + Record. +History deletion is retained before publishing it to the processor, so a failed +settings write can be retried without restoring deleted sweeps. `RecordingController` +owns a bounded writer session; its disk thread receives data and storage only. +A processor restart during Start + Record does not finish the newly armed session. +Temporary references and neutral sets clear retained processing history without +restarting the locator while the pipeline is paused. + +`WebBridge` queues browser actions onto Qt and exposes immutable status/frame +snapshots. Clear and Remove use separate action endpoints; live settings only edit +settings. Trace display options (including its Y range) do not publish processing +revisions. A timeout cancels queued actions; an action already running reports that +state without inviting a duplicate request. The GPIO controller invokes the same +reference-capture action as desktop and web, and ignores queued presses after Close. +`LogController` batches worker logs and returns web-action errors to the browser +without opening a desktop modal. Stop/Close also cancels delayed launcher startup. + GUI profiles, run configs, session state and live processing settings use shared atomic writers with unique temporary files. Profiles and run/session settings sync their file and directory; frequent live settings avoid extra disk syncs. A failed diff --git a/data_acq_and_processing/common_cpp/config/include/pipeline_runtime.hpp b/data_acq_and_processing/common_cpp/config/include/pipeline_runtime.hpp new file mode 100644 index 0000000..e2ff966 --- /dev/null +++ b/data_acq_and_processing/common_cpp/config/include/pipeline_runtime.hpp @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "run_config.hpp" + +namespace radar::runtime { + +inline std::atomic stop_requested{false}; + +inline auto initialize(int argc, char** argv) -> config::RunConfig { + const auto stop = [](int) { stop_requested.store(true, std::memory_order_relaxed); }; + std::signal(SIGINT, stop); + std::signal(SIGTERM, stop); + std::signal(SIGPIPE, SIG_IGN); + std::string path = "run_config.json"; + for (int index = 1; index + 1 < argc; ++index) { + if (std::string(argv[index]) == "--config") { path = argv[++index]; } + } + return config::load_run_config(path); +} + +class FatalPipelineError : public std::runtime_error { + public: + using std::runtime_error::runtime_error; +}; + +// Drop isolated malformed collections; make persistent failures conspicuous. +class CollectionErrors { + public: + void progress() { consecutive_ = 0; } + void report(const char* worker, const std::exception& error) { + ++total_; + if (++consecutive_ % 500 == 0) { + std::cerr << worker << ": " << consecutive_ + << " consecutive failures — likely a permanent configuration error: " + << error.what() << '\n'; + } else if (total_ % 100 == 1) { + std::cerr << worker << ": dropped collection after error (count=" + << total_ << "): " << error.what() << '\n'; + } + } + private: + std::uint64_t total_ = 0; + std::uint64_t consecutive_ = 0; +}; + +} // namespace radar::runtime diff --git a/data_acq_and_processing/common_cpp/config/include/run_config.hpp b/data_acq_and_processing/common_cpp/config/include/run_config.hpp index 19ca67c..0d79507 100644 --- a/data_acq_and_processing/common_cpp/config/include/run_config.hpp +++ b/data_acq_and_processing/common_cpp/config/include/run_config.hpp @@ -14,40 +14,15 @@ enum class DriverMode { Native, }; -enum class SwitchDriverKind { - H7992, - HMC349A, -}; - -struct RadarSweepSettings { - // Frequency sweep bounds and acquisition settings. - float start_hz = 1'000'000.0F; - float stop_hz = 6'000'000'000.0F; - std::uint32_t points = 501; - float if_bandwidth_hz = 1'000.0F; - float power_dbm = -10.0F; -}; - -struct RadarConfig { - // Radar device identity and runtime mode. - std::string model = "librevna"; - std::string serial{}; - DriverMode driver_mode = DriverMode::Mock; - RadarSweepSettings sweep{}; -}; - struct SwitchConfig { // One RF switch attached to a specific radar port. std::string name{}; DriverMode driver_mode = DriverMode::Mock; - SwitchDriverKind driver_kind = SwitchDriverKind::H7992; - std::uint32_t radar_port = 0; std::uint32_t positions = 4; std::uint32_t default_position = 0; std::string gpio_chip = "/dev/gpiochip0"; std::int32_t pin_a = 17; std::int32_t pin_b = 27; - bool invert_logic = false; }; struct RingEndpointConfig { @@ -59,22 +34,18 @@ struct RingEndpointConfig { struct RingsConfig { RingEndpointConfig raw{.name = "/radar_raw"}; - RingEndpointConfig raw_tap{.name = "/radar_raw_tap"}; RingEndpointConfig preprocessed{.name = "/radar_preprocessed"}; RingEndpointConfig preprocessed_tap{.name = "/radar_preprocessed_tap"}; RingEndpointConfig results{.name = "/radar_results"}; }; struct RuntimeConfig { - // Runtime pacing and behavior flags for C++ workers. - std::uint32_t settling_ms = 0; + // Runtime pacing and live processing configuration. std::uint32_t idle_sleep_ms = 2; - bool continuous = true; std::string processing_live_config_path = "python_app/runtime/processing_live.json"; }; struct PreprocessAssetConfig { - std::string set_name{}; std::string bundle_path{}; }; @@ -84,17 +55,6 @@ struct S21PreprocessConfig { PreprocessAssetConfig object_calibration{}; }; -struct S11CalibrationPreprocessConfig { - PreprocessAssetConfig open{}; - PreprocessAssetConfig short_standard{}; - PreprocessAssetConfig load{}; -}; - -struct S11PreprocessConfig { - S11CalibrationPreprocessConfig calibration{}; - PreprocessAssetConfig reference{}; -}; - struct PreprocessNotchBandConfig { float low_hz = 0.0F; float high_hz = 0.0F; @@ -110,7 +70,6 @@ struct PreprocessNotchConfig { struct PreprocessConfig { // Channel-specific preprocessing assets selected by Python GUI layer. S21PreprocessConfig s21{}; - S11PreprocessConfig s11{}; PreprocessNotchConfig notch{}; }; @@ -140,7 +99,7 @@ struct GprConfig { struct RunConfig { // Single configuration object shared by all C++ processes. - RadarConfig radar{}; + std::string radar_model = "librevna"; SwitchConfig input_switch{}; SwitchConfig output_switch{}; RingsConfig rings{}; diff --git a/data_acq_and_processing/common_cpp/config/src/run_config.cpp b/data_acq_and_processing/common_cpp/config/src/run_config.cpp index 064919b..674295d 100644 --- a/data_acq_and_processing/common_cpp/config/src/run_config.cpp +++ b/data_acq_and_processing/common_cpp/config/src/run_config.cpp @@ -160,7 +160,6 @@ using Json = nlohmann::json; const auto* asset_obj = as_object(required_field(object, key), asset_context); PreprocessAssetConfig asset{}; - asset.set_name = optional_string(*asset_obj, "set_name", ""); asset.bundle_path = optional_string(*asset_obj, "bundle_path", ""); return asset; } @@ -242,33 +241,7 @@ using Json = nlohmann::json; throw std::runtime_error("Unsupported driver_mode value: " + value); } -[[nodiscard]] auto parse_switch_driver_kind(const std::string& value) -> SwitchDriverKind { - if (value == "h7992") { - return SwitchDriverKind::H7992; - } - if (value == "hmc349a") { - return SwitchDriverKind::HMC349A; - } - throw std::runtime_error("Unsupported switch driver value: " + value); -} - -void validate_radar_sweep(const RadarSweepSettings& sweep) { - if (sweep.start_hz < 0.0F || sweep.stop_hz < 0.0F || !(sweep.if_bandwidth_hz > 0.0F)) { - throw std::runtime_error("radar.sweep frequencies must be non-negative and IF bandwidth must be > 0"); - } - if (sweep.points == 0U) { - throw std::runtime_error("radar.sweep.points must be > 0"); - } - if (sweep.stop_hz < sweep.start_hz) { - throw std::runtime_error("radar.sweep.stop_hz must be >= start_hz"); - } -} - void validate_switch_config(const SwitchConfig& config) { - if (config.radar_port != 1U && config.radar_port != 2U) { - throw std::runtime_error("Switch radar_port must be 1 or 2 for " + config.name); - } - if (config.positions == 0U) { throw std::runtime_error("Switch positions must be > 0 for " + config.name); } @@ -284,21 +257,11 @@ void validate_switch_config(const SwitchConfig& config) { throw std::runtime_error("gpio_chip is required for native switch mode: " + config.name); } - if (config.driver_kind == SwitchDriverKind::H7992) { - if (config.positions > 4U) { - throw std::runtime_error("Native H7992 switch driver supports positions <= 4 for " + config.name); - } - if (config.pin_a < 0 || config.pin_b < 0 || config.pin_a == config.pin_b) { - throw std::runtime_error("pin_a/pin_b are invalid for native H7992 mode: " + config.name); - } - return; + if (config.positions > 4U) { + throw std::runtime_error("Native H7992 switch driver supports positions <= 4 for " + config.name); } - - if (config.positions > 2U) { - throw std::runtime_error("Native HMC349A switch driver supports positions <= 2 for " + config.name); - } - if (config.pin_a < 0) { - throw std::runtime_error("pin_a (control) is required for native HMC349A mode: " + config.name); + if (config.pin_a < 0 || config.pin_b < 0 || config.pin_a == config.pin_b) { + throw std::runtime_error("pin_a/pin_b are invalid for native H7992 mode: " + config.name); } } @@ -350,22 +313,24 @@ void validate_combo_key(const ipc::ComboKey& key, const RunConfig& config) { [[nodiscard]] auto parse_switch_config( const Json& object, - const std::string& default_name, - std::uint32_t default_radar_port + const std::string& default_name ) -> SwitchConfig { const auto* switch_obj = as_object(object, default_name); SwitchConfig config{}; config.name = optional_string(*switch_obj, "name", default_name); config.driver_mode = parse_driver_mode(optional_string(*switch_obj, "driver_mode", "mock")); - config.driver_kind = parse_switch_driver_kind(optional_string(*switch_obj, "driver", "h7992")); - config.radar_port = optional_u32(*switch_obj, "radar_port", default_radar_port); + if (optional_string(*switch_obj, "driver", "h7992") != "h7992") { + throw std::runtime_error("Only H7992 switches are supported; check GPIO wiring before updating " + default_name); + } + if (optional_bool(*switch_obj, "invert_logic", false)) { + throw std::runtime_error("Switch invert_logic is unsupported for H7992: " + default_name); + } config.positions = optional_u32(*switch_obj, "positions", 4); config.default_position = optional_u32(*switch_obj, "default_position", 0); config.gpio_chip = optional_string(*switch_obj, "gpio_chip", "/dev/gpiochip0"); config.pin_a = optional_i32(*switch_obj, "pin_a", 17); config.pin_b = optional_i32(*switch_obj, "pin_b", 27); - config.invert_logic = optional_bool(*switch_obj, "invert_logic", false); validate_switch_config(config); @@ -397,6 +362,9 @@ void validate_combo_key(const ipc::ComboKey& key, const RunConfig& config) { if (endpoint.slot_size_bytes % 8U != 0U) { throw std::runtime_error("Ring slot_size_bytes must be a multiple of 8 for " + key); } + if (endpoint.name.empty() && key == "preprocessed_tap") { + return endpoint; + } if (endpoint.name.empty() || endpoint.name.front() != '/') { throw std::runtime_error("Ring name must start with '/' for " + key); } @@ -462,59 +430,18 @@ auto load_run_config(const std::string& path) -> RunConfig { { const auto* radar_obj = as_object(required_field(*root_obj, "radar"), "radar"); - config.radar.model = optional_string(*radar_obj, "model", "librevna"); - config.radar.serial = optional_string(*radar_obj, "serial", ""); - config.radar.driver_mode = parse_driver_mode(optional_string(*radar_obj, "driver_mode", "mock")); - - const auto* sweep_obj = as_object(required_field(*radar_obj, "sweep"), "radar.sweep"); - config.radar.sweep.start_hz = - number_to_f32(as_number(required_field(*sweep_obj, "start_hz"), "radar.sweep.start_hz"), "radar.sweep.start_hz"); - config.radar.sweep.stop_hz = - number_to_f32(as_number(required_field(*sweep_obj, "stop_hz"), "radar.sweep.stop_hz"), "radar.sweep.stop_hz"); - if (const auto* points_value = optional_field(*sweep_obj, "points"); points_value != nullptr) { - config.radar.sweep.points = - number_to_u32(as_number(*points_value, "radar.sweep.points"), "radar.sweep.points"); - } else if (config.radar.model != "kamil_adc") { - throw std::runtime_error("Missing required config field: points"); - } - config.radar.sweep.if_bandwidth_hz = number_to_f32( - as_number(required_field(*sweep_obj, "if_bandwidth_hz"), "radar.sweep.if_bandwidth_hz"), "radar.sweep.if_bandwidth_hz" - ); - config.radar.sweep.power_dbm = optional_f32(*sweep_obj, "stimulus_power_dbm", -10.0F); - validate_radar_sweep(config.radar.sweep); + config.radar_model = optional_string(*radar_obj, "model", "librevna"); } { const auto* switches_obj = as_object(required_field(*root_obj, "switches"), "switches"); - - if (const auto* port1_value = optional_field(*switches_obj, "port1"); port1_value != nullptr) { - const auto* port1_obj = as_object(*port1_value, "switches.port1"); - const auto* port2_obj = as_object(required_field(*switches_obj, "port2"), "switches.port2"); - config.output_switch = parse_switch_config(*port1_obj, "port1", 1); - config.input_switch = parse_switch_config(*port2_obj, "port2", 2); - } else { - config.input_switch = parse_switch_config( - *as_object(required_field(*switches_obj, "input"), "switches.input"), - "input", - 2 - ); - config.output_switch = parse_switch_config( - *as_object(required_field(*switches_obj, "output"), "switches.output"), - "output", - 1 - ); - } - - if (config.input_switch.radar_port == config.output_switch.radar_port) { - throw std::runtime_error("Switch radar_port mapping must be unique for two switches"); - } + config.output_switch = parse_switch_config(required_field(*switches_obj, "port1"), "port1"); + config.input_switch = parse_switch_config(required_field(*switches_obj, "port2"), "port2"); } { const auto* run_obj = as_object(required_field(*root_obj, "run"), "run"); - config.runtime.settling_ms = optional_u32(*run_obj, "settling_ms", 0); config.runtime.idle_sleep_ms = optional_u32(*run_obj, "idle_sleep_ms", 2); - config.runtime.continuous = optional_bool(*run_obj, "continuous", true); config.runtime.processing_live_config_path = optional_string( *run_obj, "processing_live_config_path", @@ -560,17 +487,6 @@ auto load_run_config(const std::string& path) -> RunConfig { parse_preprocess_asset(*s21_obj, "object_calibration", "preprocess.s21"); } - const auto* s11_obj = as_object(required_field(*preprocess_obj, "s11"), "preprocess.s11"); - const auto* s11_calibration_obj = - as_object(required_field(*s11_obj, "calibration"), "preprocess.s11.calibration"); - config.preprocess.s11.calibration.open = - parse_preprocess_asset(*s11_calibration_obj, "open", "preprocess.s11.calibration"); - config.preprocess.s11.calibration.short_standard = - parse_preprocess_asset(*s11_calibration_obj, "short", "preprocess.s11.calibration"); - config.preprocess.s11.calibration.load = - parse_preprocess_asset(*s11_calibration_obj, "load", "preprocess.s11.calibration"); - config.preprocess.s11.reference = parse_preprocess_asset(*s11_obj, "reference", "preprocess.s11"); - if (const auto* notch_value = optional_field(*preprocess_obj, "notch"); notch_value != nullptr) { config.preprocess.notch = parse_preprocess_notch(*as_object(*notch_value, "preprocess.notch"), "preprocess.notch"); } @@ -585,7 +501,6 @@ auto load_run_config(const std::string& path) -> RunConfig { { const auto* rings_obj = as_object(required_field(*root_obj, "rings"), "rings"); config.rings.raw = parse_ring_endpoint(*rings_obj, "raw", config.rings.raw); - config.rings.raw_tap = parse_ring_endpoint(*rings_obj, "raw_tap", config.rings.raw_tap); config.rings.preprocessed = parse_ring_endpoint(*rings_obj, "preprocessed", config.rings.preprocessed); config.rings.preprocessed_tap = parse_ring_endpoint( *rings_obj, diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/switches/h7992_minimal_driver.hpp b/data_acq_and_processing/common_cpp/hardware/include/h7992_minimal_driver.hpp similarity index 75% rename from data_acq_and_processing/sweep_orchestrator/device_drivers/switches/h7992_minimal_driver.hpp rename to data_acq_and_processing/common_cpp/hardware/include/h7992_minimal_driver.hpp index 3133e20..bc555dd 100644 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/switches/h7992_minimal_driver.hpp +++ b/data_acq_and_processing/common_cpp/hardware/include/h7992_minimal_driver.hpp @@ -4,7 +4,6 @@ #include #include "run_config.hpp" -#include "switch_driver.hpp" namespace radar::drivers { @@ -27,15 +26,14 @@ struct H7992MinimalDriverSettings { * Native mode uses Linux GPIO character-device API (uAPI v2). * Mock mode keeps state in memory without touching hardware. */ -class H7992MinimalDriver final : public SwitchDriver { +class H7992MinimalDriver { public: explicit H7992MinimalDriver(H7992MinimalDriverSettings settings); - ~H7992MinimalDriver() override; // Leaves RF path safe even if close() was skipped. + ~H7992MinimalDriver(); // Leaves RF path safe even if close() was skipped. - void open() override; - void close() override; - [[nodiscard]] auto position_count() const -> std::uint32_t override; - void switch_to(std::uint32_t position) override; + void open(); + void close(); + void switch_to(std::uint32_t position); private: void open_native(); diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/switches/h7992_minimal_driver.cpp b/data_acq_and_processing/common_cpp/hardware/src/h7992_minimal_driver.cpp similarity index 98% rename from data_acq_and_processing/sweep_orchestrator/device_drivers/switches/h7992_minimal_driver.cpp rename to data_acq_and_processing/common_cpp/hardware/src/h7992_minimal_driver.cpp index 3032d5b..0405492 100644 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/switches/h7992_minimal_driver.cpp +++ b/data_acq_and_processing/common_cpp/hardware/src/h7992_minimal_driver.cpp @@ -83,10 +83,6 @@ void H7992MinimalDriver::close() { is_open_ = false; } -auto H7992MinimalDriver::position_count() const -> std::uint32_t { - return settings_.positions; -} - void H7992MinimalDriver::switch_to(std::uint32_t position) { if (!is_open_) { throw std::runtime_error("Switch driver is not open for " + settings_.name); diff --git a/data_acq_and_processing/common_cpp/ipc/include/shared_types.hpp b/data_acq_and_processing/common_cpp/ipc/include/shared_types.hpp index 65721e4..7270479 100644 --- a/data_acq_and_processing/common_cpp/ipc/include/shared_types.hpp +++ b/data_acq_and_processing/common_cpp/ipc/include/shared_types.hpp @@ -34,10 +34,8 @@ struct ComboKeyHash { struct SweepTraceBlock { ComboKey combo{}; - // Frequency axis in Hz. Must have the same size as `s11` and `s21`. + // Frequency axis in Hz. Must have the same size as `s21`. std::vector frequency_hz{}; - // Complex S11 samples for matching frequency points. - std::vector s11{}; // Complex S21 samples for matching frequency points. std::vector s21{}; // Monotonic window in which THIS trace's sweep was measured, excluding the @@ -92,11 +90,12 @@ struct ResultBlock { struct ResultCollection { std::uint64_t collection_id = 0; std::uint64_t monotonic_ns = 0; - // Wall-clock duration of `process_collection()` for this collection, - // measured on the data_processor side with a monotonic clock. Used by the - // Python pipeline to report processing-time metrics. Zero is a valid - // "unmeasured" sentinel for legacy producers. + // Processing duration and acquisition identity, independent of stage completion time. std::uint64_t processing_duration_ns = 0; + std::uint64_t settings_revision = 0; + std::uint64_t capture_start_ns = 0; + std::uint64_t capture_end_ns = 0; + float speed_m_s = 0.0F; std::vector collection_payloads{}; std::vector blocks{}; }; @@ -111,7 +110,6 @@ struct ResultCollection { -> PreprocessedCollection; [[nodiscard]] auto serialize_result_collection(const ResultCollection& collection) -> std::vector; -[[nodiscard]] auto deserialize_result_collection(std::span bytes) -> ResultCollection; // Monotonic timestamp helper used for produced collections. [[nodiscard]] auto current_monotonic_ns() -> std::uint64_t; diff --git a/data_acq_and_processing/common_cpp/ipc/include/shm_ring.hpp b/data_acq_and_processing/common_cpp/ipc/include/shm_ring.hpp index a86a20b..282c57c 100644 --- a/data_acq_and_processing/common_cpp/ipc/include/shm_ring.hpp +++ b/data_acq_and_processing/common_cpp/ipc/include/shm_ring.hpp @@ -25,15 +25,7 @@ class ShmRing { std::uint32_t slot_size_bytes ) -> ShmRing; - // Opens ring by name and validates wire-format header fields. - [[nodiscard]] static auto open_existing(const std::string& name) -> ShmRing; - // Removes shared memory object by name (safe if it does not exist). - static void unlink_ring(const std::string& name); - - [[nodiscard]] auto is_open() const -> bool; - [[nodiscard]] auto capacity() const -> std::uint32_t; [[nodiscard]] auto slot_size_bytes() const -> std::uint32_t; - [[nodiscard]] auto dropped_count() const -> std::uint64_t; // Pushes payload with overwrite-oldest policy on overflow. [[nodiscard]] auto push(std::span payload) -> bool; @@ -57,8 +49,10 @@ class ShmRing { void close(); static void validate_name(const std::string& name); - // Validates that mapped_size covers the geometry declared in the header - // (fix #50: reject undersized/corrupt mappings to prevent OOB slot access). + [[nodiscard]] static auto checked_mapped_size( + std::uint32_t capacity, std::uint32_t slot_size_bytes, const std::string& name + ) -> std::size_t; + // Reject undersized/corrupt mappings before accessing any slot. static void validate_geometry(const Header& header, std::size_t mapped_size, const std::string& name); }; diff --git a/data_acq_and_processing/common_cpp/ipc/src/shared_types.cpp b/data_acq_and_processing/common_cpp/ipc/src/shared_types.cpp index 42590a9..b85ade5 100644 --- a/data_acq_and_processing/common_cpp/ipc/src/shared_types.cpp +++ b/data_acq_and_processing/common_cpp/ipc/src/shared_types.cpp @@ -1,5 +1,6 @@ #include "shared_types.hpp" +#include #include #include #include @@ -12,9 +13,11 @@ namespace radar::ipc { namespace { -constexpr std::uint32_t kRawCollectionMagic = 0x32574152U; // RAW2 -constexpr std::uint32_t kPreprocessedCollectionMagic = 0x32525050U; // PRP2 -constexpr std::uint32_t kResultCollectionMagic = 0x324C5352U; // RSL2 +static_assert(std::endian::native == std::endian::little, "IPC requires a little-endian host"); + +constexpr std::uint32_t kRawCollectionMagic = 0x33574152U; // RAW3 +constexpr std::uint32_t kPreprocessedCollectionMagic = 0x33505250U; // PRP3 +constexpr std::uint32_t kResultCollectionMagic = 0x334C5352U; // RSL3 template concept TriviallySerializable = std::is_trivially_copyable_v; @@ -75,16 +78,6 @@ class BinaryReader { return value; } - [[nodiscard]] auto read_string() -> std::string { - const auto size = read(); - ensure_available(size); - - const auto* begin = reinterpret_cast(bytes_.data() + offset_); - std::string value(begin, begin + size); - offset_ += size; - return value; - } - [[nodiscard]] auto is_consumed() const -> bool { return offset_ == bytes_.size(); } @@ -106,7 +99,6 @@ class BinaryReader { void write_trace_block(BinaryWriter& writer, const SweepTraceBlock& trace) { ensure_equal_sizes(trace.frequency_hz.size(), trace.s21.size(), "Sweep trace"); - ensure_equal_sizes(trace.frequency_hz.size(), trace.s11.size(), "Sweep trace S11"); writer.write(trace.combo.input_pos); writer.write(trace.combo.output_pos); @@ -116,11 +108,6 @@ void write_trace_block(BinaryWriter& writer, const SweepTraceBlock& trace) { writer.write(frequency_hz); } - for (const auto& point : trace.s11) { - writer.write(point.re); - writer.write(point.im); - } - for (const auto& point : trace.s21) { writer.write(point.re); writer.write(point.im); @@ -144,23 +131,15 @@ void require_count_fits(std::uint64_t count, std::size_t min_bytes_each, BinaryR trace.combo.output_pos = reader.read(); const auto point_count = reader.read(); - // Each point encodes frequency (4B) + s11 (8B) + s21 (8B) = 20 bytes. - require_count_fits(point_count, 20U, reader); + // Each point encodes frequency (4B) + s21 (8B) = 12 bytes. + require_count_fits(point_count, 12U, reader); trace.frequency_hz.reserve(point_count); - trace.s11.reserve(point_count); trace.s21.reserve(point_count); for (std::uint32_t index = 0; index < point_count; ++index) { trace.frequency_hz.push_back(reader.read()); } - for (std::uint32_t index = 0; index < point_count; ++index) { - trace.s11.push_back(Complex32{ - .re = reader.read(), - .im = reader.read(), - }); - } - for (std::uint32_t index = 0; index < point_count; ++index) { trace.s21.push_back(Complex32{ .re = reader.read(), @@ -171,10 +150,7 @@ void require_count_fits(std::uint64_t count, std::size_t min_bytes_each, BinaryR return trace; } -// The capture windows live in a TRAILER after the trace blocks rather than inside -// them, so a reader built before they existed still decodes every trace and simply -// stops early. The trailer grows the same way: collection window first, then the -// per-trace window table (one pair per trace, in trace order). +// Capture trailer: collection window, then one per-trace window in trace order. void write_trace_collection(BinaryWriter& writer, std::uint32_t magic, const RawSweepCollection& collection) { writer.write(magic); writer.write(collection.collection_id); @@ -213,22 +189,9 @@ void write_trace_collection(BinaryWriter& writer, std::uint32_t magic, const Raw collection.traces.push_back(read_trace_block(reader)); } - // Each trailer stage is optional: a payload from an older producer stops after - // the trace blocks (or after the collection window) and leaves the rest zeroed. - if (reader.remaining_bytes() == 0U) { - return collection; - } - if (reader.remaining_bytes() < (sizeof(std::uint64_t) * 2U)) { - throw std::runtime_error("Truncated capture window in trace collection"); - } - collection.capture_start_ns = reader.read(); collection.capture_end_ns = reader.read(); - if (reader.remaining_bytes() == 0U) { - return collection; - } - const auto trace_time_count = reader.read(); if (trace_time_count != collection.traces.size()) { throw std::runtime_error("Per-trace capture window count does not match trace count"); @@ -254,24 +217,6 @@ void write_trace_result_payload(BinaryWriter& writer, const ResultPayload& paylo } } -auto read_trace_result_payload(BinaryReader& reader, ResultPayload* payload) -> void { - const auto point_count = reader.read(); - require_count_fits(point_count, 12U, reader); - - payload->frequency_hz.reserve(point_count); - payload->trace.reserve(point_count); - - for (std::uint32_t index = 0; index < point_count; ++index) { - payload->frequency_hz.push_back(reader.read()); - } - for (std::uint32_t index = 0; index < point_count; ++index) { - payload->trace.push_back(Complex32{ - .re = reader.read(), - .im = reader.read(), - }); - } -} - void write_image_result_payload(BinaryWriter& writer, const ResultPayload& payload) { const auto x_count = checked_count_to_u32(payload.image_x_axis.size(), "Result image x axis point count"); const auto y_count = checked_count_to_u32(payload.image_y_axis.size(), "Result image y axis point count"); @@ -293,27 +238,6 @@ void write_image_result_payload(BinaryWriter& writer, const ResultPayload& paylo } } -auto read_image_result_payload(BinaryReader& reader, ResultPayload* payload) -> void { - const auto x_count = reader.read(); - const auto y_count = reader.read(); - const auto value_count = static_cast(x_count) * y_count; - require_count_fits(value_count + x_count + y_count, sizeof(float), reader); - - payload->image_x_axis.reserve(x_count); - payload->image_y_axis.reserve(y_count); - payload->image_values.reserve(static_cast(value_count)); - - for (std::uint32_t index = 0; index < x_count; ++index) { - payload->image_x_axis.push_back(reader.read()); - } - for (std::uint32_t index = 0; index < y_count; ++index) { - payload->image_y_axis.push_back(reader.read()); - } - for (std::size_t index = 0; index < value_count; ++index) { - payload->image_values.push_back(reader.read()); - } -} - void write_table_result_payload(BinaryWriter& writer, const ResultPayload& payload) { const auto column_count = payload.table_columns; if (column_count == 0U && !payload.table_values.empty()) { @@ -333,21 +257,6 @@ void write_table_result_payload(BinaryWriter& writer, const ResultPayload& paylo } } -auto read_table_result_payload(BinaryReader& reader, ResultPayload* payload) -> void { - const auto column_count = reader.read(); - const auto row_count = reader.read(); - if (column_count == 0U && row_count != 0U) { - throw std::runtime_error("Result table cannot have rows without columns"); - } - const auto value_count = static_cast(column_count) * row_count; - require_count_fits(value_count, sizeof(float), reader); - payload->table_columns = column_count; - payload->table_values.reserve(static_cast(value_count)); - for (std::size_t index = 0; index < value_count; ++index) { - payload->table_values.push_back(reader.read()); - } -} - void write_result_payload(BinaryWriter& writer, const ResultPayload& payload) { writer.write(static_cast(payload.kind)); writer.write_string(payload.processing_name); @@ -370,29 +279,6 @@ void write_result_payload(BinaryWriter& writer, const ResultPayload& payload) { } } -[[nodiscard]] auto read_result_payload(BinaryReader& reader) -> ResultPayload { - ResultPayload payload{}; - payload.kind = static_cast(reader.read()); - payload.processing_name = reader.read_string(); - - switch (payload.kind) { - case ResultKind::TraceComplex: - read_trace_result_payload(reader, &payload); - return payload; - case ResultKind::ScalarF32: - payload.scalar_value = reader.read(); - return payload; - case ResultKind::ImageF32: - read_image_result_payload(reader, &payload); - return payload; - case ResultKind::TableF32: - read_table_result_payload(reader, &payload); - return payload; - default: - throw std::runtime_error("Unsupported result payload kind in stream"); - } -} - void write_result_block(BinaryWriter& writer, const ResultBlock& block) { writer.write(block.combo.input_pos); writer.write(block.combo.output_pos); @@ -403,21 +289,6 @@ void write_result_block(BinaryWriter& writer, const ResultBlock& block) { } } -[[nodiscard]] auto read_result_block(BinaryReader& reader) -> ResultBlock { - ResultBlock block{}; - block.combo.input_pos = reader.read(); - block.combo.output_pos = reader.read(); - - const auto payload_count = reader.read(); - require_count_fits(payload_count, 3U, reader); - block.payloads.reserve(payload_count); - for (std::uint32_t index = 0; index < payload_count; ++index) { - block.payloads.push_back(read_result_payload(reader)); - } - - return block; -} - void ensure_reader_consumed(const BinaryReader& reader, const std::string& payload_label) { if (!reader.is_consumed()) { throw std::runtime_error("Unexpected trailing bytes in " + payload_label); @@ -458,6 +329,10 @@ auto serialize_result_collection(const ResultCollection& collection) -> std::vec writer.write(collection.collection_id); writer.write(collection.monotonic_ns); writer.write(collection.processing_duration_ns); + writer.write(collection.settings_revision); + writer.write(collection.capture_start_ns); + writer.write(collection.capture_end_ns); + writer.write(collection.speed_m_s); writer.write(checked_count_to_u32(collection.collection_payloads.size(), "Collection payload count")); writer.write(checked_count_to_u32(collection.blocks.size(), "Result block count")); @@ -471,37 +346,6 @@ auto serialize_result_collection(const ResultCollection& collection) -> std::vec return std::move(writer).finish(); } -auto deserialize_result_collection(std::span bytes) -> ResultCollection { - BinaryReader reader(bytes); - - const auto magic = reader.read(); - if (magic != kResultCollectionMagic) { - throw std::runtime_error("Unexpected result collection magic"); - } - - ResultCollection collection{}; - collection.collection_id = reader.read(); - collection.monotonic_ns = reader.read(); - collection.processing_duration_ns = reader.read(); - - const auto collection_payload_count = reader.read(); - const auto block_count = reader.read(); - require_count_fits(collection_payload_count, 3U, reader); - collection.collection_payloads.reserve(collection_payload_count); - for (std::uint32_t index = 0; index < collection_payload_count; ++index) { - collection.collection_payloads.push_back(read_result_payload(reader)); - } - - require_count_fits(block_count, 12U, reader); - collection.blocks.reserve(block_count); - for (std::uint32_t index = 0; index < block_count; ++index) { - collection.blocks.push_back(read_result_block(reader)); - } - - ensure_reader_consumed(reader, "result collection"); - return collection; -} - auto current_monotonic_ns() -> std::uint64_t { const auto now = std::chrono::steady_clock::now().time_since_epoch(); const auto now_ns = std::chrono::duration_cast(now).count(); diff --git a/data_acq_and_processing/common_cpp/ipc/src/shm_ring.cpp b/data_acq_and_processing/common_cpp/ipc/src/shm_ring.cpp index a37659e..cae935c 100644 --- a/data_acq_and_processing/common_cpp/ipc/src/shm_ring.cpp +++ b/data_acq_and_processing/common_cpp/ipc/src/shm_ring.cpp @@ -44,63 +44,10 @@ constexpr auto kHeaderInitWaitTimeout = std::chrono::milliseconds(1000); constexpr auto kHeaderInitPollInterval = std::chrono::milliseconds(2); constexpr std::uint64_t kMaxMappedSize = 1ULL << 40U; -[[nodiscard]] auto checked_u64_diff(std::uint64_t left, std::uint64_t right) -> std::uint64_t { - return left >= right ? left - right : 0; -} - [[nodiscard]] auto errno_message(const std::string& action, const std::string& name) -> std::runtime_error { return std::runtime_error(action + " " + name + ": " + std::strerror(errno)); } -class ScopedFd { - public: - explicit ScopedFd(int fd) : fd_(fd) {} - - ~ScopedFd() { - if (fd_ >= 0) { - ::close(fd_); - } - } - - ScopedFd(const ScopedFd&) = delete; - auto operator=(const ScopedFd&) -> ScopedFd& = delete; - - ScopedFd(ScopedFd&& other) noexcept : fd_(std::exchange(other.fd_, -1)) {} - - auto release() -> int { - return std::exchange(fd_, -1); - } - - private: - int fd_ = -1; -}; - -class ScopedMmap { - public: - ScopedMmap(void* address, std::size_t size) : address_(address), size_(size) {} - - ~ScopedMmap() { - if (address_ != nullptr && address_ != MAP_FAILED) { - munmap(address_, size_); - } - } - - ScopedMmap(const ScopedMmap&) = delete; - auto operator=(const ScopedMmap&) -> ScopedMmap& = delete; - - ScopedMmap(ScopedMmap&& other) noexcept - : address_(std::exchange(other.address_, nullptr)), - size_(std::exchange(other.size_, 0U)) {} - - auto release() -> std::pair { - return {std::exchange(address_, nullptr), std::exchange(size_, 0U)}; - } - - private: - void* address_ = nullptr; - std::size_t size_ = 0; -}; - } // namespace ShmRing::~ShmRing() { @@ -129,28 +76,7 @@ auto ShmRing::open_or_create( std::uint32_t slot_size_bytes ) -> ShmRing { validate_name(name); - if (capacity == 0U) { - throw std::runtime_error("Ring capacity must be > 0"); - } - if (slot_size_bytes == 0U) { - throw std::runtime_error("Ring slot size must be > 0"); - } - if (slot_size_bytes % alignof(SlotHeader) != 0U) { - throw std::runtime_error("Ring slot size must be a multiple of 8"); - } - - const auto stride = sizeof(SlotHeader) + static_cast(slot_size_bytes); - if (stride > (kMaxMappedSize - sizeof(Header)) / capacity) { - throw std::runtime_error("Shared memory ring exceeds 1 TiB for " + name); - } - const auto mapped_size_u64 = sizeof(Header) + stride * capacity; - if (mapped_size_u64 > kMaxMappedSize - || mapped_size_u64 > std::numeric_limits::max() - || mapped_size_u64 > static_cast(std::numeric_limits::max())) { - throw std::runtime_error("Shared memory ring geometry overflows for " + name); - } - const auto mapped_size = static_cast(mapped_size_u64); - + const auto mapped_size = checked_mapped_size(capacity, slot_size_bytes, name); bool created = false; int fd = shm_open(name.c_str(), O_RDWR | O_CREAT | O_EXCL, 0660); @@ -164,8 +90,8 @@ auto ShmRing::open_or_create( throw errno_message("Failed to open shared memory ring", name); } - ScopedFd scoped_fd(fd); - + ShmRing ring{}; + ring.fd_ = fd; if (created) { if (ftruncate(fd, static_cast(mapped_size)) != 0) { @@ -195,30 +121,29 @@ auto ShmRing::open_or_create( throw errno_message("Failed to mmap shared memory ring", name); } - ScopedMmap scoped_mmap(mapped, mapped_size); - auto* header = static_cast(mapped); + ring.mapped_ = mapped; + ring.mapped_size_ = mapped_size; + auto* header = ring.header_ = static_cast(mapped); if (created) { // ftruncate already supplies zero-filled slots; leave them demand-paged // instead of touching the full ring before any payload is published. std::memset(mapped, 0, sizeof(Header)); std::memcpy(header->magic, kRingMagic.data(), kRingMagic.size()); - header->version = kRingVersion; header->capacity = capacity; header->slot_size_bytes = slot_size_bytes; new (&header->write_seq) std::atomic(0); new (&header->read_seq) std::atomic(0); new (&header->dropped) std::atomic(0); - std::atomic_thread_fence(std::memory_order_release); + // Publish readiness only after geometry and cursors have been initialized. + std::atomic_ref(header->version).store(kRingVersion, std::memory_order_release); } else { const auto deadline = std::chrono::steady_clock::now() + kHeaderInitWaitTimeout; while (true) { - std::atomic_thread_fence(std::memory_order_acquire); - + const bool version_ok = std::atomic_ref(header->version).load(std::memory_order_acquire) == kRingVersion; const bool magic_ok = std::memcmp(header->magic, kRingMagic.data(), kRingMagic.size()) == 0; - const bool version_ok = header->version == kRingVersion; const bool geometry_ok = header->capacity == capacity && header->slot_size_bytes == slot_size_bytes; if (magic_ok && version_ok && geometry_ok) { @@ -247,89 +172,13 @@ auto ShmRing::open_or_create( } } - ShmRing ring{}; - ring.fd_ = scoped_fd.release(); - - const auto [released_mapped, released_size] = scoped_mmap.release(); - ring.mapped_ = released_mapped; - ring.mapped_size_ = released_size; - ring.header_ = static_cast(released_mapped); return ring; } -auto ShmRing::open_existing(const std::string& name) -> ShmRing { - validate_name(name); - - const int fd = shm_open(name.c_str(), O_RDWR, 0660); - if (fd < 0) { - throw errno_message("Failed to open shared memory ring", name); - } - - ScopedFd scoped_fd(fd); - - struct stat info {}; - if (fstat(fd, &info) != 0) { - throw errno_message("Failed to stat shared memory ring", name); - } - - if (info.st_size < static_cast(sizeof(Header))) { - throw std::runtime_error("Shared memory ring size is too small for " + name); - } - - const auto mapped_size = static_cast(info.st_size); - void* mapped = mmap(nullptr, mapped_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (mapped == MAP_FAILED) { - throw errno_message("Failed to mmap shared memory ring", name); - } - - ScopedMmap scoped_mmap(mapped, mapped_size); - auto* header = static_cast(mapped); - - if (std::memcmp(header->magic, kRingMagic.data(), kRingMagic.size()) != 0) { - throw std::runtime_error("Shared memory ring magic mismatch for " + name); - } - if (header->version != kRingVersion) { - throw std::runtime_error("Shared memory ring version mismatch for " + name); - } - // ensure the mapping actually spans every slot the header describes. - validate_geometry(*header, mapped_size, name); - - ShmRing ring{}; - ring.fd_ = scoped_fd.release(); - - const auto [released_mapped, released_size] = scoped_mmap.release(); - ring.mapped_ = released_mapped; - ring.mapped_size_ = released_size; - ring.header_ = static_cast(released_mapped); - return ring; -} - -void ShmRing::unlink_ring(const std::string& name) { - validate_name(name); - if (shm_unlink(name.c_str()) != 0 && errno != ENOENT) { - throw errno_message("Failed to unlink shared memory ring", name); - } -} - -auto ShmRing::is_open() const -> bool { - return header_ != nullptr; -} - -auto ShmRing::capacity() const -> std::uint32_t { - return header_ != nullptr ? header_->capacity : 0; -} - auto ShmRing::slot_size_bytes() const -> std::uint32_t { return header_ != nullptr ? header_->slot_size_bytes : 0; } -auto ShmRing::dropped_count() const -> std::uint64_t { - if (header_ == nullptr) { - return 0; - } - return header_->dropped.load(std::memory_order_acquire); -} - auto ShmRing::push(std::span payload) -> bool { if (header_ == nullptr) { throw std::runtime_error("Ring is not open"); @@ -341,7 +190,7 @@ auto ShmRing::push(std::span payload) -> bool { const auto write_seq = header_->write_seq.load(std::memory_order_relaxed); auto read_seq = header_->read_seq.load(std::memory_order_acquire); - if (checked_u64_diff(write_seq, read_seq) >= header_->capacity) { + if (write_seq >= read_seq && write_seq - read_seq >= header_->capacity) { // Ring full: overwrite the oldest unread slot. Advance read_seq via CAS so a // concurrent consumer's advance is never clobbered (read_seq never moves back). header_->read_seq.compare_exchange_strong( @@ -444,37 +293,41 @@ void ShmRing::close() { } void ShmRing::validate_name(const std::string& name) { - if (name.empty() || name.front() != '/') { - throw std::runtime_error("POSIX shm name must start with '/'"); + if (name.size() < 2U || name.front() != '/' || name.find('/', 1) != std::string::npos + || name.find('\0') != std::string::npos) { + throw std::runtime_error("POSIX shm name must be '/' followed by a single nonempty filename"); } } -void ShmRing::validate_geometry(const Header& header, std::size_t mapped_size, const std::string& name) { - // derive the expected size from the header's own geometry fields and - // require the real mapping to cover it. A bogus capacity/slot_size or a truncated - // mapping would otherwise yield out-of-bounds slot offsets and a SIGSEGV. - const std::uint32_t capacity = header.capacity; - const std::uint32_t slot_size_bytes = header.slot_size_bytes; +auto ShmRing::checked_mapped_size( + std::uint32_t capacity, std::uint32_t slot_size_bytes, const std::string& name +) -> std::size_t { + static_assert(sizeof(Header) == 64 && sizeof(SlotHeader) == 16); if (capacity == 0U) { - throw std::runtime_error("Shared memory ring capacity is zero for " + name); + throw std::runtime_error("Ring capacity must be > 0"); } if (slot_size_bytes == 0U) { - throw std::runtime_error("Shared memory ring slot size is zero for " + name); + throw std::runtime_error("Ring slot size must be > 0"); } if (slot_size_bytes % alignof(SlotHeader) != 0U) { - throw std::runtime_error("Shared memory ring slot size must be a multiple of 8 for " + name); + throw std::runtime_error("Ring slot size must be a multiple of 8"); } const auto stride = sizeof(SlotHeader) + static_cast(slot_size_bytes); if (stride > (kMaxMappedSize - sizeof(Header)) / capacity) { throw std::runtime_error("Shared memory ring exceeds 1 TiB for " + name); } - const auto expected_size = sizeof(Header) + stride * capacity; - if (expected_size > kMaxMappedSize || expected_size > std::numeric_limits::max()) { + const auto mapped_size_u64 = sizeof(Header) + stride * capacity; + if (mapped_size_u64 > kMaxMappedSize + || mapped_size_u64 > std::numeric_limits::max() + || mapped_size_u64 > static_cast(std::numeric_limits::max())) { throw std::runtime_error("Shared memory ring geometry overflows for " + name); } + return static_cast(mapped_size_u64); +} - if (mapped_size < expected_size) { +void ShmRing::validate_geometry(const Header& header, std::size_t mapped_size, const std::string& name) { + if (mapped_size < checked_mapped_size(header.capacity, header.slot_size_bytes, name)) { throw std::runtime_error("Shared memory ring size is too small for " + name); } } diff --git a/data_acq_and_processing/kamil_adc_collector/README.md b/data_acq_and_processing/kamil_adc_collector/README.md index eea20b2..6528a58 100644 --- a/data_acq_and_processing/kamil_adc_collector/README.md +++ b/data_acq_and_processing/kamil_adc_collector/README.md @@ -2,11 +2,11 @@ Standalone acquisition binary for the **L-Card E-502** ADC. It captures the dual-channel main + reference IQ stream and publishes it as 8-byte framed packets -over a PTY, which the Python `kamil_adc` producer -(`python_app/hardware_full/kamil_adc`) reads and processes. +over a PTY. `KamilAdcService` in `python_app/hardware_full/kamil_adc` reads and +processes that stream for `python_app.scripts.kamil_adc_raw_producer`. -This is the data *source* for `radar.model == "kamil_adc"`; it is the Kamil ADC -counterpart to the C++ `sweep_orchestrator` used by LibreVNA devices. +This is the data *source* for `radar.model == "kamil_adc"`, paired with the Python +`kamil_adc_raw_producer`. LibreVNA acquisition uses Python drivers. ## Layout @@ -16,15 +16,13 @@ counterpart to the C++ `sweep_orchestrator` used by LibreVNA devices. | `include/` | Headers for the writers plus `switch_sequencer.h`. | | `vendor/lcard/` | L-Card X502/E502 SDK headers (`x502api.h`, `e502api.h`, `lcard_pstdint.h`), vendored verbatim so the collector builds with no external include path. | -The sources started as a verbatim import of the original standalone tool, but are -**no longer identical to it**: the collector now drives the RF switches itself -(switch-aware mode, `SwitchSequencer`), stepping to the next combination inside -the hardware idle gap between sweeps and flagging a sweep *dirty* when a readout -backlog means the toggle could not have landed in that gap. It therefore reuses -the orchestrator's switch drivers and the shared `run_config` loader, which is why -its build has its own include set and its own `build/kamil/` object tree. - -The wire format towards the Python producer is unchanged. +In switch-aware mode, `SwitchSequencer` drives the RF switches, stepping to the +next combination inside the hardware idle gap between sweeps and flagging a +sweep *dirty* when a readout backlog means the toggle could not have landed in +that gap. The Python reader +drops dirty sweeps; the collector repeats that combination without another +switch transition. The collector uses the common H7992 driver and `run_config` +loader, with its own include set and `build/kamil/` object tree. ## Build @@ -33,16 +31,25 @@ make kamil_adc_collector # -> build/bin/kamil_adc_collector ``` It is **not** part of `make all`; `start.sh` builds it automatically when the -active config selects `radar.model == "kamil_adc"`. The build is self-contained: -it needs only the vendored headers and links `-ldl -lutil -lpthread`. +active config selects `radar.model == "kamil_adc"`, unless `--skip-build` is set. +The Linux build uses the repository's common sources and vendored headers and +links with `-pthread -ldl -lutil`; no installed L-Card SDK is needed to build it. ## Runtime The collector loads the proprietary `libx502api.so` / `libe502api.so` at run time -via `dlopen`, so those libraries must be on `LD_LIBRARY_PATH`. The Kamil ADC -service prepends `~/.local/lib` (their default install location) unless the config -pins `LD_LIBRARY_PATH` in `radar.kamil_adc.env`. +via `dlopen`, so those libraries must be discoverable by the dynamic loader. +The Kamil ADC service prepends `~/.local/lib` (their default install location) +unless the config pins `LD_LIBRARY_PATH` in `radar.kamil_adc.env`. The command-line arguments (capture profile, clocking, the `do8_freq_ref` reference overlay, etc.) are supplied by `radar.kamil_adc.args` in the run config; -the service appends the generated `tty:` argument. +the service appends `tty:` from `radar.kamil_adc.tty_path`. + +When both RF switches use `driver_mode: "native"`, the producer also passes +`config:` to the collector. After each sweep boundary, the collector emits +an 8-byte combo tag `[0x00C0, input_pos, output_pos, dirty]` (four little-endian +16-bit words). The producer accepts only clean sweeps for the requested combo. +Otherwise, the producer selects and settles the switches itself; the collector +runs without `config:` and emits no combo tags. Calibration captures also +use this Python-controlled path. diff --git a/data_acq_and_processing/kamil_adc_collector/include/switch_sequencer.h b/data_acq_and_processing/kamil_adc_collector/include/switch_sequencer.h index f2ec2b9..e0df25b 100644 --- a/data_acq_and_processing/kamil_adc_collector/include/switch_sequencer.h +++ b/data_acq_and_processing/kamil_adc_collector/include/switch_sequencer.h @@ -6,7 +6,7 @@ #include #include "shared_types.hpp" -#include "switch_driver.hpp" +#include "h7992_minimal_driver.hpp" namespace radar::kamil { @@ -47,8 +47,8 @@ class SwitchSequencer { SwitchSequencer( Settings settings, - std::unique_ptr input_switch, - std::unique_ptr output_switch + std::unique_ptr input_switch, + std::unique_ptr output_switch ) : settings_(std::move(settings)), input_switch_(std::move(input_switch)), @@ -130,15 +130,14 @@ class SwitchSequencer { private: void apply(const radar::ipc::ComboKey& combo) { - // Output first, then input — same order the sweep orchestrator uses, so - // both acquisition paths drive an identical hardware sequence. + // Set the transmitter before the receiver. output_switch_->switch_to(combo.output_pos); input_switch_->switch_to(combo.input_pos); } Settings settings_; - std::unique_ptr input_switch_; - std::unique_ptr output_switch_; + std::unique_ptr input_switch_; + std::unique_ptr output_switch_; std::size_t index_ = 0; radar::ipc::ComboKey applied_{}; bool upcoming_dirty_ = false; diff --git a/data_acq_and_processing/kamil_adc_collector/src/main.cpp b/data_acq_and_processing/kamil_adc_collector/src/main.cpp index 258a278..0424b89 100644 --- a/data_acq_and_processing/kamil_adc_collector/src/main.cpp +++ b/data_acq_and_processing/kamil_adc_collector/src/main.cpp @@ -20,7 +20,6 @@ #include "switch_sequencer.h" #include "run_config.hpp" #include "h7992_minimal_driver.hpp" -#include "hmc349a_minimal_driver.hpp" #include #include @@ -1830,21 +1829,8 @@ void print_device_info(const t_x502_info& info) { } // Build one switch driver (native GPIO or mock) from a parsed switch config, -// reusing the exact driver implementations the sweep orchestrator uses so both -// acquisition paths drive identical hardware. -std::unique_ptr make_switch_driver(const radar::config::SwitchConfig& sc) { - if (sc.driver_kind == radar::config::SwitchDriverKind::HMC349A) { - return std::make_unique( - radar::drivers::HMC349AMinimalDriverSettings{ - .name = sc.name, - .mode = sc.driver_mode, - .positions = sc.positions, - .default_position = sc.default_position, - .gpio_chip = sc.gpio_chip, - .pin_a = sc.pin_a, - .invert_logic = sc.invert_logic, - }); - } +// using the shared H7992 GPIO implementation. +std::unique_ptr make_switch_driver(const radar::config::SwitchConfig& sc) { return std::make_unique( radar::drivers::H7992MinimalDriverSettings{ .name = sc.name, diff --git a/data_acq_and_processing/preprocessing/bundles/include/channel_bundle_support.hpp b/data_acq_and_processing/preprocessing/bundles/include/channel_bundle_support.hpp new file mode 100644 index 0000000..5201e8d --- /dev/null +++ b/data_acq_and_processing/preprocessing/bundles/include/channel_bundle_support.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include + +#include "shared_types.hpp" + +namespace radar::preprocessing { + +struct ChannelTrace { + std::vector frequency_hz{}; + std::vector samples{}; +}; + +class S21CalibrationBundle { + public: + void load(const std::string& path); + + void validate_combos(const std::vector& combos) const; + void apply( + const ipc::ComboKey& combo, + std::span frequency_hz, + std::span measured + ) const; + + private: + std::unordered_map traces_by_combo_{}; +}; + +class S21ReferenceBundle { + public: + // Load and through-calibrate the active references once at startup. + void load(const std::string& path, const S21CalibrationBundle& calibration, + const std::vector& combos); + void apply(ipc::SweepTraceBlock& trace) const; + + private: + std::unordered_map traces_by_combo_{}; +}; + +// Object-derived MIMO correction, exported on each pair's runtime frequency grid. +// An empty path disables it; enabled bundles must cover every active pair. +class S21ObjectCalibrationBundle { + public: + void load(const std::string& path); + void validate_combos(const std::vector& combos) const; + void apply(ipc::SweepTraceBlock& trace) const; + + private: + std::unordered_map factors_by_combo_{}; +}; + +} // namespace radar::preprocessing diff --git a/data_acq_and_processing/preprocessing/bundles/src/channel_bundle_support.cpp b/data_acq_and_processing/preprocessing/bundles/src/channel_bundle_support.cpp new file mode 100644 index 0000000..572ea3e --- /dev/null +++ b/data_acq_and_processing/preprocessing/bundles/src/channel_bundle_support.cpp @@ -0,0 +1,255 @@ +#include "channel_bundle_support.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace radar::preprocessing { +namespace { + +constexpr float kFrequencyRelativeTolerance = 1e-5F; +constexpr float kFrequencyAbsoluteTolerance = 1e-3F; +[[nodiscard]] auto combo_to_string(const ipc::ComboKey& combo) -> std::string { + return "input=" + std::to_string(combo.input_pos) + " output=" + std::to_string(combo.output_pos); +} + +[[nodiscard]] auto read_binary_file(const std::string& path, const std::string& bundle_label) + -> std::vector { + std::ifstream stream(path, std::ios::binary); + if (!stream.is_open()) { + throw std::runtime_error("Failed to open " + bundle_label + " bundle: " + path); + } + + return std::vector(std::istreambuf_iterator(stream), std::istreambuf_iterator()); +} + +void validate_channel_trace_layout( + const ChannelTrace& trace, + const std::string& trace_label, + const ipc::ComboKey& combo +) { + if (trace.frequency_hz.size() != trace.samples.size()) { + throw std::runtime_error( + trace_label + " frequency/channel vector size mismatch for combo " + combo_to_string(combo) + ); + } + for (std::size_t i = 0; i < trace.samples.size(); ++i) { + if (!std::isfinite(trace.frequency_hz[i]) || !std::isfinite(trace.samples[i].re) + || !std::isfinite(trace.samples[i].im)) { + throw std::runtime_error(trace_label + " contains non-finite values for combo " + combo_to_string(combo)); + } + } +} + +[[nodiscard]] auto frequency_axes_match(std::span left, std::span right) -> bool { + if (left.size() != right.size()) { + return false; + } + + for (std::size_t index = 0; index < left.size(); ++index) { + if (!std::isfinite(left[index]) || !std::isfinite(right[index])) { + return false; + } + const auto delta = std::fabs(left[index] - right[index]); + const auto scale = std::max(std::fabs(left[index]), std::fabs(right[index])); + const auto tolerance = std::max(kFrequencyAbsoluteTolerance, scale * kFrequencyRelativeTolerance); + if (delta > tolerance) { + return false; + } + } + return true; +} + +void ensure_frequency_axis_match( + std::span expected, + std::span actual, + const std::string& label +) { + if (!frequency_axes_match(expected, actual)) { + throw std::runtime_error(label + " frequency axis mismatch"); + } +} + +[[nodiscard]] auto to_channel_trace( + const ipc::SweepTraceBlock& trace, + const std::string& bundle_label +) -> ChannelTrace { + if (trace.frequency_hz.empty()) { + throw std::runtime_error(bundle_label + " trace contains no frequency samples"); + } + + ChannelTrace channel_trace{}; + channel_trace.frequency_hz = trace.frequency_hz; + channel_trace.samples = trace.s21; + validate_channel_trace_layout(channel_trace, bundle_label + " trace", trace.combo); + return channel_trace; +} + +[[nodiscard]] auto load_channel_traces( + const std::string& path, + const std::string& bundle_label +) -> std::unordered_map { + const auto bytes = read_binary_file(path, bundle_label); + if (bytes.empty()) { + throw std::runtime_error(bundle_label + " bundle is empty: " + path); + } + + const auto collection = ipc::deserialize_raw_collection(bytes); + std::unordered_map traces_by_combo{}; + traces_by_combo.reserve(collection.traces.size()); + for (const auto& trace : collection.traces) { + // #47: reject duplicate combos instead of silently overwriting an earlier trace. + const auto inserted = + traces_by_combo.insert({trace.combo, to_channel_trace(trace, bundle_label)}); + if (!inserted.second) { + throw std::runtime_error( + bundle_label + " bundle contains duplicate combo " + combo_to_string(trace.combo) + ": " + path + ); + } + } + + if (traces_by_combo.empty()) { + throw std::runtime_error(bundle_label + " bundle does not contain traces: " + path); + } + return traces_by_combo; +} + +void validate_bundle_combos( + const std::unordered_map& traces_by_combo, + const std::vector& combos, + const std::string& bundle_label +) { + if (traces_by_combo.empty()) { + return; + } + + for (const auto& combo : combos) { + if (!traces_by_combo.contains(combo)) { + throw std::runtime_error(bundle_label + " is missing combo " + combo_to_string(combo)); + } + } +} + +template +auto resolve_required( + const std::unordered_map& values_by_combo, + const ipc::ComboKey& combo, + const char* value_label +) -> const TValue& { + const auto found = values_by_combo.find(combo); + if (found == values_by_combo.end()) { + throw std::runtime_error(std::string(value_label) + " is missing for combo " + combo_to_string(combo)); + } + return found->second; +} + +} // namespace + +void S21CalibrationBundle::load(const std::string& path) { + if (path.empty()) { + throw std::runtime_error("S21 calibration bundle path must not be empty"); + } + + traces_by_combo_ = load_channel_traces(path, "S21 calibration"); +} + +void S21CalibrationBundle::validate_combos(const std::vector& combos) const { + validate_bundle_combos(traces_by_combo_, combos, "S21 calibration bundle"); +} + +void S21CalibrationBundle::apply( + const ipc::ComboKey& combo, + std::span frequency_hz, + std::span measured +) const { + const auto& calibration_trace = resolve_required(traces_by_combo_, combo, "S21 calibration trace"); + ensure_frequency_axis_match(calibration_trace.frequency_hz, frequency_hz, "S21 calibration"); + if (measured.size() != calibration_trace.samples.size()) { + throw std::runtime_error("S21 calibration vector size mismatch"); + } + constexpr float epsilon = 1e-12F; + for (std::size_t index = 0; index < measured.size(); ++index) { + const auto standard = calibration_trace.samples[index]; + const float magnitude_squared = standard.re * standard.re + standard.im * standard.im; + if (magnitude_squared > epsilon) { + const auto value = measured[index]; + measured[index] = { + (value.re * standard.re + value.im * standard.im) / magnitude_squared, + (value.im * standard.re - value.re * standard.im) / magnitude_squared, + }; + } + } +} + +void S21ReferenceBundle::load( + const std::string& path, + const S21CalibrationBundle& calibration, + const std::vector& combos +) { + const auto raw = load_channel_traces(path, "S21 reference"); + decltype(traces_by_combo_) calibrated; + for (const auto& combo : combos) { + auto trace = resolve_required(raw, combo, "S21 reference"); + calibration.apply(combo, trace.frequency_hz, trace.samples); + validate_channel_trace_layout(trace, "Calibrated S21 reference", combo); + calibrated.emplace(combo, std::move(trace)); + } + traces_by_combo_ = std::move(calibrated); +} + +void S21ReferenceBundle::apply(ipc::SweepTraceBlock& trace) const { + const auto& reference = resolve_required(traces_by_combo_, trace.combo, "Calibrated S21 reference"); + ensure_frequency_axis_match(reference.frequency_hz, trace.frequency_hz, "S21 reference"); + if (trace.s21.size() != reference.samples.size()) { + throw std::runtime_error("S21 reference point count mismatch"); + } + for (std::size_t index = 0; index < trace.s21.size(); ++index) { + trace.s21[index].re -= reference.samples[index].re; + trace.s21[index].im -= reference.samples[index].im; + } +} + +void S21ObjectCalibrationBundle::load(const std::string& path) { + if (path.empty()) { + factors_by_combo_.clear(); + return; + } + factors_by_combo_ = load_channel_traces(path, "S21 object calibration"); +} + +void S21ObjectCalibrationBundle::validate_combos(const std::vector& combos) const { + validate_bundle_combos(factors_by_combo_, combos, "S21 object calibration bundle"); +} + +void S21ObjectCalibrationBundle::apply(ipc::SweepTraceBlock& trace) const { + if (factors_by_combo_.empty()) { return; } + const auto& factors = resolve_required(factors_by_combo_, trace.combo, "S21 object calibration"); + ensure_frequency_axis_match(factors.frequency_hz, trace.frequency_hz, "S21 object calibration"); + if (trace.s21.size() != factors.samples.size()) { + throw std::runtime_error("S21 object calibration vector size mismatch"); + } + for (std::size_t index = 0; index < trace.s21.size(); ++index) { + const auto value = trace.s21[index]; + const auto factor = factors.samples[index]; + const auto corrected = std::complex(value.re, value.im) * std::complex(factor.re, factor.im); + if (!std::isfinite(corrected.real()) || !std::isfinite(corrected.imag()) || + std::abs(corrected.real()) > std::numeric_limits::max() || + std::abs(corrected.imag()) > std::numeric_limits::max()) { + throw std::runtime_error("S21 object calibration result must fit finite float32"); + } + trace.s21[index] = {.re = static_cast(corrected.real()), .im = static_cast(corrected.imag())}; + } +} + +} // namespace radar::preprocessing diff --git a/data_acq_and_processing/preprocessing/calibration_master/include/calibration_master.hpp b/data_acq_and_processing/preprocessing/calibration_master/include/calibration_master.hpp deleted file mode 100644 index 5508d1a..0000000 --- a/data_acq_and_processing/preprocessing/calibration_master/include/calibration_master.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include "calibrator_interface.hpp" -#include "channel_bundle_support.hpp" -#include "shared_types.hpp" - -namespace radar::preprocessing { - -class CalibrationMaster { - public: - // `s21_calibrator` encapsulates the S21 calibration algorithm. - explicit CalibrationMaster(std::unique_ptr s21_calibrator); - - // Loads a serialized raw sweep bundle with one S21 calibration standard per combo. - void load_s21_calibration_bundle(const std::string& path); - // Loads optional one-port OSL calibration data for S11 from raw sweep bundles. - void load_s11_calibration_bundle( - const std::string& open_path, - const std::string& short_path, - const std::string& load_path - ); - // Ensures all runtime combos are present in loaded standards. - void validate_combos(const std::vector& combos) const; - // Applies both channel calibrations to one measured trace. - [[nodiscard]] auto apply_to_trace(const ipc::SweepTraceBlock& measured_trace) const -> ipc::SweepTraceBlock; - [[nodiscard]] auto apply_s21( - const ipc::ComboKey& combo, - std::span frequency_hz, - const std::vector& measured_s21 - ) const -> std::vector; - [[nodiscard]] auto apply_s11( - const ipc::ComboKey& combo, - std::span frequency_hz, - const std::vector& measured_s11 - ) const -> std::vector; - [[nodiscard]] auto has_s11_calibration() const noexcept -> bool; - - private: - std::unique_ptr s21_calibrator_; - S21CalibrationBundle s21_calibration_bundle_{}; - S11CalibrationBundle s11_calibration_bundle_{}; -}; - -[[nodiscard]] auto make_s21_through_calibrator() -> std::unique_ptr; - -} // namespace radar::preprocessing diff --git a/data_acq_and_processing/preprocessing/calibration_master/include/calibrator_interface.hpp b/data_acq_and_processing/preprocessing/calibration_master/include/calibrator_interface.hpp deleted file mode 100644 index 159f9a5..0000000 --- a/data_acq_and_processing/preprocessing/calibration_master/include/calibrator_interface.hpp +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -#include -#include - -#include "shared_types.hpp" - -namespace radar::preprocessing { - -class CalibratorInterface { - public: - virtual ~CalibratorInterface() = default; - - [[nodiscard]] virtual auto name() const -> std::string = 0; - [[nodiscard]] virtual auto apply( - const std::vector& measured, - const std::vector& calibration - ) const -> std::vector = 0; -}; - -} // namespace radar::preprocessing diff --git a/data_acq_and_processing/preprocessing/calibration_master/include/channel_bundle_support.hpp b/data_acq_and_processing/preprocessing/calibration_master/include/channel_bundle_support.hpp deleted file mode 100644 index a4c8a03..0000000 --- a/data_acq_and_processing/preprocessing/calibration_master/include/channel_bundle_support.hpp +++ /dev/null @@ -1,106 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include "shared_types.hpp" - -namespace radar::preprocessing { - -class CalibratorInterface; - -struct ChannelTrace { - std::vector frequency_hz{}; - std::vector samples{}; -}; - -void validate_channel_trace_layout(const ChannelTrace& trace, const std::string& label, const ipc::ComboKey& combo); -[[nodiscard]] auto frequency_axes_match(std::span left, std::span right) -> bool; - -class S21CalibrationBundle { - public: - void load(const std::string& path); - - [[nodiscard]] auto is_enabled() const noexcept -> bool; - void validate_combos(const std::vector& combos) const; - [[nodiscard]] auto apply( - const ipc::ComboKey& combo, - std::span frequency_hz, - const std::vector& measured, - const CalibratorInterface& calibrator - ) const -> std::vector; - - private: - [[nodiscard]] auto resolve(const ipc::ComboKey& combo) const -> const ChannelTrace&; - - std::unordered_map traces_by_combo_{}; -}; - -class S21ReferenceBundle { - public: - void load(const std::string& path); - - [[nodiscard]] auto is_enabled() const noexcept -> bool; - void validate_combos(const std::vector& combos) const; - [[nodiscard]] auto resolve(const ipc::ComboKey& combo) const -> const ChannelTrace&; - - private: - std::unordered_map traces_by_combo_{}; -}; - -// Object-derived MIMO correction, exported on each pair's runtime frequency grid. -// An empty path disables it; enabled bundles must cover every active pair. -class S21ObjectCalibrationBundle { - public: - void load(const std::string& path); - void validate_combos(const std::vector& combos) const; - void apply(ipc::SweepTraceBlock& trace) const; - - private: - std::unordered_map factors_by_combo_{}; -}; - -class S11CalibrationBundle { - public: - struct Coefficients { - std::vector frequency_hz{}; - std::vector directivity{}; - std::vector source_match{}; - std::vector reflection_tracking{}; - }; - - void load( - const std::string& open_path, - const std::string& short_path, - const std::string& load_path - ); - - [[nodiscard]] auto is_enabled() const noexcept -> bool; - void validate_combos(const std::vector& combos) const; - [[nodiscard]] auto apply( - const ipc::ComboKey& combo, - std::span frequency_hz, - const std::vector& measured - ) const -> std::vector; - - private: - [[nodiscard]] auto resolve(const ipc::ComboKey& combo) const -> const Coefficients&; - - std::unordered_map coefficients_by_combo_{}; -}; - -class S11ReferenceBundle { - public: - void load(const std::string& path); - - [[nodiscard]] auto is_enabled() const noexcept -> bool; - void validate_combos(const std::vector& combos) const; - [[nodiscard]] auto resolve(const ipc::ComboKey& combo) const -> const ChannelTrace&; - - private: - std::unordered_map traces_by_combo_{}; -}; - -} // namespace radar::preprocessing diff --git a/data_acq_and_processing/preprocessing/calibration_master/src/calibration_master.cpp b/data_acq_and_processing/preprocessing/calibration_master/src/calibration_master.cpp deleted file mode 100644 index 0e5e126..0000000 --- a/data_acq_and_processing/preprocessing/calibration_master/src/calibration_master.cpp +++ /dev/null @@ -1,63 +0,0 @@ -#include "calibration_master.hpp" - -#include - -namespace radar::preprocessing { - -CalibrationMaster::CalibrationMaster(std::unique_ptr s21_calibrator) - : s21_calibrator_(std::move(s21_calibrator)) { - if (!s21_calibrator_) { - throw std::runtime_error("CalibrationMaster requires a non-null calibrator"); - } -} - -void CalibrationMaster::load_s21_calibration_bundle(const std::string& path) { - s21_calibration_bundle_.load(path); -} - -void CalibrationMaster::load_s11_calibration_bundle( - const std::string& open_path, - const std::string& short_path, - const std::string& load_path -) { - s11_calibration_bundle_.load(open_path, short_path, load_path); -} - -void CalibrationMaster::validate_combos(const std::vector& combos) const { - s21_calibration_bundle_.validate_combos(combos); - s11_calibration_bundle_.validate_combos(combos); -} - -auto CalibrationMaster::apply_to_trace(const ipc::SweepTraceBlock& measured_trace) const -> ipc::SweepTraceBlock { - ipc::SweepTraceBlock output{}; - output.combo = measured_trace.combo; - output.frequency_hz = measured_trace.frequency_hz; - output.s21 = apply_s21(measured_trace.combo, measured_trace.frequency_hz, measured_trace.s21); - output.s11 = apply_s11(measured_trace.combo, measured_trace.frequency_hz, measured_trace.s11); - // Calibration reshapes the samples, not when they were measured. - output.capture_start_ns = measured_trace.capture_start_ns; - output.capture_end_ns = measured_trace.capture_end_ns; - return output; -} - -auto CalibrationMaster::apply_s21( - const ipc::ComboKey& combo, - std::span frequency_hz, - const std::vector& measured_s21 -) const -> std::vector { - return s21_calibration_bundle_.apply(combo, frequency_hz, measured_s21, *s21_calibrator_); -} - -auto CalibrationMaster::apply_s11( - const ipc::ComboKey& combo, - std::span frequency_hz, - const std::vector& measured_s11 -) const -> std::vector { - return s11_calibration_bundle_.apply(combo, frequency_hz, measured_s11); -} - -auto CalibrationMaster::has_s11_calibration() const noexcept -> bool { - return s11_calibration_bundle_.is_enabled(); -} - -} // namespace radar::preprocessing diff --git a/data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp b/data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp deleted file mode 100644 index a130748..0000000 --- a/data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp +++ /dev/null @@ -1,494 +0,0 @@ -#include "channel_bundle_support.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "calibrator_interface.hpp" - -namespace radar::preprocessing { -namespace { - -enum class RawTraceChannel { - S11, - S21, -}; - -constexpr float kFrequencyRelativeTolerance = 1e-5F; -constexpr float kFrequencyAbsoluteTolerance = 1e-3F; -constexpr float kComplexMagnitudeSquaredEpsilon = 1e-12F; -// #30: OSL denominator must be well conditioned relative to the magnitudes of the terms forming it. -constexpr float kOslRelativeConditioning = 1e-6F; -// #30: maximum fraction of degenerate (fallback) OSL points tolerated per combo before load fails. -constexpr float kOslMaxDegenerateFraction = 0.05F; - -[[nodiscard]] auto combo_to_string(const ipc::ComboKey& combo) -> std::string { - return "input=" + std::to_string(combo.input_pos) + " output=" + std::to_string(combo.output_pos); -} - -[[nodiscard]] auto read_binary_file(const std::string& path, const std::string& bundle_label) - -> std::vector { - std::ifstream stream(path, std::ios::binary); - if (!stream.is_open()) { - throw std::runtime_error("Failed to open " + bundle_label + " bundle: " + path); - } - - return std::vector(std::istreambuf_iterator(stream), std::istreambuf_iterator()); -} - -void validate_raw_trace_layout(const ipc::SweepTraceBlock& trace, const std::string& trace_label) { - if (trace.frequency_hz.size() != trace.s11.size()) { - throw std::runtime_error( - trace_label + " frequency/S11 vector size mismatch for combo " + combo_to_string(trace.combo) - ); - } - if (trace.frequency_hz.size() != trace.s21.size()) { - throw std::runtime_error( - trace_label + " frequency/S21 vector size mismatch for combo " + combo_to_string(trace.combo) - ); - } -} - -} // namespace - -void validate_channel_trace_layout( - const ChannelTrace& trace, - const std::string& trace_label, - const ipc::ComboKey& combo -) { - if (trace.frequency_hz.size() != trace.samples.size()) { - throw std::runtime_error( - trace_label + " frequency/channel vector size mismatch for combo " + combo_to_string(combo) - ); - } - for (std::size_t i = 0; i < trace.samples.size(); ++i) { - if (!std::isfinite(trace.frequency_hz[i]) || !std::isfinite(trace.samples[i].re) - || !std::isfinite(trace.samples[i].im)) { - throw std::runtime_error(trace_label + " contains non-finite values for combo " + combo_to_string(combo)); - } - } -} - -[[nodiscard]] auto frequency_axes_match(std::span left, std::span right) -> bool { - if (left.size() != right.size()) { - return false; - } - - for (std::size_t index = 0; index < left.size(); ++index) { - if (!std::isfinite(left[index]) || !std::isfinite(right[index])) { - return false; - } - const auto delta = std::fabs(left[index] - right[index]); - const auto scale = std::max(std::fabs(left[index]), std::fabs(right[index])); - const auto tolerance = std::max(kFrequencyAbsoluteTolerance, scale * kFrequencyRelativeTolerance); - if (delta > tolerance) { - return false; - } - } - return true; -} - -namespace { - -void ensure_frequency_axis_match( - std::span expected, - std::span actual, - const std::string& label -) { - if (!frequency_axes_match(expected, actual)) { - throw std::runtime_error(label + " frequency axis mismatch"); - } -} - -[[nodiscard]] auto to_std_complex(const ipc::Complex32& value) -> std::complex { - return std::complex(value.re, value.im); -} - -[[nodiscard]] auto from_std_complex(const std::complex& value) -> ipc::Complex32 { - return ipc::Complex32{ - .re = value.real(), - .im = value.imag(), - }; -} - -[[nodiscard]] auto pick_channel_samples(const ipc::SweepTraceBlock& trace, RawTraceChannel channel) - -> const std::vector& { - if (channel == RawTraceChannel::S11) { - return trace.s11; - } - return trace.s21; -} - -[[nodiscard]] auto to_channel_trace( - const ipc::SweepTraceBlock& trace, - RawTraceChannel channel, - const std::string& bundle_label -) -> ChannelTrace { - validate_raw_trace_layout(trace, bundle_label + " trace"); - if (trace.frequency_hz.empty()) { - throw std::runtime_error(bundle_label + " trace contains no frequency samples"); - } - - ChannelTrace channel_trace{}; - channel_trace.frequency_hz = trace.frequency_hz; - channel_trace.samples = pick_channel_samples(trace, channel); - validate_channel_trace_layout(channel_trace, bundle_label + " trace", trace.combo); - return channel_trace; -} - -[[nodiscard]] auto load_channel_traces( - const std::string& path, - const std::string& bundle_label, - RawTraceChannel channel -) -> std::unordered_map { - const auto bytes = read_binary_file(path, bundle_label); - if (bytes.empty()) { - throw std::runtime_error(bundle_label + " bundle is empty: " + path); - } - - const auto collection = ipc::deserialize_raw_collection(bytes); - std::unordered_map traces_by_combo{}; - traces_by_combo.reserve(collection.traces.size()); - for (const auto& trace : collection.traces) { - // #47: reject duplicate combos instead of silently overwriting an earlier trace. - const auto inserted = - traces_by_combo.insert({trace.combo, to_channel_trace(trace, channel, bundle_label)}); - if (!inserted.second) { - throw std::runtime_error( - bundle_label + " bundle contains duplicate combo " + combo_to_string(trace.combo) + ": " + path - ); - } - } - - if (traces_by_combo.empty()) { - throw std::runtime_error(bundle_label + " bundle does not contain traces: " + path); - } - return traces_by_combo; -} - -void validate_bundle_combos( - const std::unordered_map& traces_by_combo, - const std::vector& combos, - const std::string& bundle_label -) { - if (traces_by_combo.empty()) { - return; - } - - for (const auto& combo : combos) { - if (!traces_by_combo.contains(combo)) { - throw std::runtime_error(bundle_label + " is missing combo " + combo_to_string(combo)); - } - } -} - -template -auto resolve_required( - const std::unordered_map& values_by_combo, - const ipc::ComboKey& combo, - const char* value_label -) -> const TValue& { - const auto found = values_by_combo.find(combo); - if (found == values_by_combo.end()) { - throw std::runtime_error(std::string(value_label) + " is missing for combo " + combo_to_string(combo)); - } - return found->second; -} - -void ensure_combo_coverage( - const std::unordered_map& expected, - const std::unordered_map& actual, - const std::string& expected_label, - const std::string& actual_label -) { - for (const auto& entry : expected) { - const auto& combo = entry.first; - if (!actual.contains(combo)) { - throw std::runtime_error( - actual_label + " is missing combo " + combo_to_string(combo) + - " required by " + expected_label - ); - } - } -} - -[[nodiscard]] auto solve_osl_coefficients( - const ipc::ComboKey& combo, - const ChannelTrace& open_trace, - const ChannelTrace& short_trace, - const ChannelTrace& load_trace -) -> S11CalibrationBundle::Coefficients { - validate_channel_trace_layout(open_trace, "S11 open calibration", combo); - validate_channel_trace_layout(short_trace, "S11 short calibration", combo); - validate_channel_trace_layout(load_trace, "S11 load calibration", combo); - ensure_frequency_axis_match(open_trace.frequency_hz, short_trace.frequency_hz, "S11 short calibration"); - ensure_frequency_axis_match(open_trace.frequency_hz, load_trace.frequency_hz, "S11 load calibration"); - - S11CalibrationBundle::Coefficients coefficients{}; - coefficients.frequency_hz = open_trace.frequency_hz; - coefficients.directivity.resize(open_trace.frequency_hz.size()); - coefficients.source_match.resize(open_trace.frequency_hz.size()); - coefficients.reflection_tracking.resize(open_trace.frequency_hz.size()); - - // #30: track degenerate points so silently-substituted fallback coefficients cannot pass unnoticed. - std::size_t degenerate_points = 0; - for (std::size_t index = 0; index < open_trace.frequency_hz.size(); ++index) { - const auto load = to_std_complex(load_trace.samples[index]); - const auto open_delta = to_std_complex(open_trace.samples[index]) - load; - const auto short_delta = to_std_complex(short_trace.samples[index]) - load; - const auto denominator = open_delta - short_delta; - - // #30: require the denominator to be well conditioned both absolutely and relative to the - // magnitudes of the open/short deltas it is formed from, so near-cancellations are rejected. - const auto denominator_norm = std::norm(denominator); - const auto relative_scale = std::max(std::norm(open_delta), std::norm(short_delta)); - const auto relative_threshold = relative_scale * (kOslRelativeConditioning * kOslRelativeConditioning); - - std::complex source_match = std::complex(0.0F, 0.0F); - std::complex reflection_tracking = std::complex(1.0F, 0.0F); - if (denominator_norm > kComplexMagnitudeSquaredEpsilon && denominator_norm > relative_threshold) { - source_match = (open_delta + short_delta) / denominator; - reflection_tracking = open_delta * (std::complex(1.0F, 0.0F) - source_match); - } else { - // #30: degenerate point — log the offending combo/frequency and fall back to a pass-through. - ++degenerate_points; - std::cerr << "S11 calibration: degenerate OSL point for combo " << combo_to_string(combo) - << " at " << open_trace.frequency_hz[index] << " Hz (ill-conditioned denominator)\n"; - } - - coefficients.directivity[index] = load_trace.samples[index]; - coefficients.source_match[index] = from_std_complex(source_match); - coefficients.reflection_tracking[index] = from_std_complex(reflection_tracking); - } - - // #30: refuse to load a calibration whose per-combo fallback fraction exceeds the tolerated threshold. - if (!open_trace.frequency_hz.empty()) { - const auto degenerate_fraction = - static_cast(degenerate_points) / static_cast(open_trace.frequency_hz.size()); - if (degenerate_fraction > kOslMaxDegenerateFraction) { - throw std::runtime_error( - "S11 calibration for combo " + combo_to_string(combo) + " has " + - std::to_string(degenerate_points) + " of " + - std::to_string(open_trace.frequency_hz.size()) + - " degenerate OSL points (fraction exceeds tolerance)" - ); - } - } - - return coefficients; -} - -} // namespace - -void S21CalibrationBundle::load(const std::string& path) { - if (path.empty()) { - throw std::runtime_error("S21 calibration bundle path must not be empty"); - } - - traces_by_combo_ = load_channel_traces(path, "S21 calibration", RawTraceChannel::S21); -} - -auto S21CalibrationBundle::is_enabled() const noexcept -> bool { - return !traces_by_combo_.empty(); -} - -void S21CalibrationBundle::validate_combos(const std::vector& combos) const { - validate_bundle_combos(traces_by_combo_, combos, "S21 calibration bundle"); -} - -auto S21CalibrationBundle::apply( - const ipc::ComboKey& combo, - std::span frequency_hz, - const std::vector& measured, - const CalibratorInterface& calibrator -) const -> std::vector { - const auto& calibration_trace = resolve(combo); - ensure_frequency_axis_match(calibration_trace.frequency_hz, frequency_hz, "S21 calibration"); - if (measured.size() != calibration_trace.samples.size()) { - throw std::runtime_error("S21 calibration vector size mismatch"); - } - return calibrator.apply(measured, calibration_trace.samples); -} - -auto S21CalibrationBundle::resolve(const ipc::ComboKey& combo) const -> const ChannelTrace& { - return resolve_required(traces_by_combo_, combo, "S21 calibration trace"); -} - -void S21ReferenceBundle::load(const std::string& path) { - if (path.empty()) { - throw std::runtime_error("S21 reference bundle path must not be empty"); - } - - traces_by_combo_ = load_channel_traces(path, "S21 reference", RawTraceChannel::S21); -} - -auto S21ReferenceBundle::is_enabled() const noexcept -> bool { - return !traces_by_combo_.empty(); -} - -void S21ReferenceBundle::validate_combos(const std::vector& combos) const { - validate_bundle_combos(traces_by_combo_, combos, "S21 reference bundle"); -} - -auto S21ReferenceBundle::resolve(const ipc::ComboKey& combo) const -> const ChannelTrace& { - return resolve_required(traces_by_combo_, combo, "S21 reference trace"); -} - -void S21ObjectCalibrationBundle::load(const std::string& path) { - if (path.empty()) { - factors_by_combo_.clear(); - return; - } - factors_by_combo_ = load_channel_traces(path, "S21 object calibration", RawTraceChannel::S21); -} - -void S21ObjectCalibrationBundle::validate_combos(const std::vector& combos) const { - validate_bundle_combos(factors_by_combo_, combos, "S21 object calibration bundle"); -} - -void S21ObjectCalibrationBundle::apply(ipc::SweepTraceBlock& trace) const { - if (factors_by_combo_.empty()) { return; } - const auto& factors = resolve_required(factors_by_combo_, trace.combo, "S21 object calibration"); - ensure_frequency_axis_match(factors.frequency_hz, trace.frequency_hz, "S21 object calibration"); - if (trace.s21.size() != factors.samples.size()) { - throw std::runtime_error("S21 object calibration vector size mismatch"); - } - for (std::size_t index = 0; index < trace.s21.size(); ++index) { - const auto value = trace.s21[index]; - const auto factor = factors.samples[index]; - const auto corrected = std::complex(value.re, value.im) * std::complex(factor.re, factor.im); - if (!std::isfinite(corrected.real()) || !std::isfinite(corrected.imag()) || - std::abs(corrected.real()) > std::numeric_limits::max() || - std::abs(corrected.imag()) > std::numeric_limits::max()) { - throw std::runtime_error("S21 object calibration result must fit finite float32"); - } - trace.s21[index] = {.re = static_cast(corrected.real()), .im = static_cast(corrected.imag())}; - } -} - -void S11CalibrationBundle::load( - const std::string& open_path, - const std::string& short_path, - const std::string& load_path -) { - if (open_path.empty() && short_path.empty() && load_path.empty()) { - coefficients_by_combo_.clear(); - // #38: S11 calibration disabled because no OSL paths were provided; surface it instead of silently skipping. - std::cerr << "S11 calibration: no open/short/load bundle paths provided; reflection correction disabled\n"; - return; - } - if (open_path.empty() || short_path.empty() || load_path.empty()) { - throw std::runtime_error("S11 calibration requires open, short, and load raw bundle paths"); - } - - const auto open_traces = load_channel_traces(open_path, "S11 open calibration", RawTraceChannel::S11); - const auto short_traces = load_channel_traces(short_path, "S11 short calibration", RawTraceChannel::S11); - const auto load_traces = load_channel_traces(load_path, "S11 load calibration", RawTraceChannel::S11); - - ensure_combo_coverage(open_traces, short_traces, "S11 open calibration", "S11 short calibration"); - ensure_combo_coverage(open_traces, load_traces, "S11 open calibration", "S11 load calibration"); - ensure_combo_coverage(short_traces, open_traces, "S11 short calibration", "S11 open calibration"); - ensure_combo_coverage(load_traces, open_traces, "S11 load calibration", "S11 open calibration"); - - decltype(coefficients_by_combo_) coefficients; - coefficients.reserve(open_traces.size()); - for (const auto& [combo, open_trace] : open_traces) { - coefficients.insert_or_assign( - combo, - solve_osl_coefficients(combo, open_trace, short_traces.at(combo), load_traces.at(combo)) - ); - } - coefficients_by_combo_ = std::move(coefficients); -} - -auto S11CalibrationBundle::is_enabled() const noexcept -> bool { - return !coefficients_by_combo_.empty(); -} - -void S11CalibrationBundle::validate_combos(const std::vector& combos) const { - if (!is_enabled()) { - return; - } - - for (const auto& combo : combos) { - if (!coefficients_by_combo_.contains(combo)) { - throw std::runtime_error("S11 calibration data is missing for combo " + combo_to_string(combo)); - } - } -} - -auto S11CalibrationBundle::apply( - const ipc::ComboKey& combo, - std::span frequency_hz, - const std::vector& measured -) const -> std::vector { - if (!is_enabled()) { - return measured; - } - if (frequency_hz.size() != measured.size()) { - throw std::runtime_error("S11 calibration input frequency/sample size mismatch"); - } - - const auto& coefficients = resolve(combo); - ensure_frequency_axis_match(coefficients.frequency_hz, frequency_hz, "S11 calibration"); - if (measured.size() != coefficients.directivity.size()) { - throw std::runtime_error("S11 calibration vector size mismatch"); - } - - std::vector corrected{}; - corrected.reserve(measured.size()); - for (std::size_t index = 0; index < measured.size(); ++index) { - const auto numerator = to_std_complex(measured[index]) - to_std_complex(coefficients.directivity[index]); - const auto denominator = - to_std_complex(coefficients.reflection_tracking[index]) + - (to_std_complex(coefficients.source_match[index]) * numerator); - - if (std::norm(denominator) > kComplexMagnitudeSquaredEpsilon) { - corrected.push_back(from_std_complex(numerator / denominator)); - } else { - corrected.push_back(from_std_complex(numerator)); - } - } - return corrected; -} - -auto S11CalibrationBundle::resolve(const ipc::ComboKey& combo) const -> const Coefficients& { - return resolve_required(coefficients_by_combo_, combo, "S11 calibration coefficients"); -} - -void S11ReferenceBundle::load(const std::string& path) { - if (path.empty()) { - traces_by_combo_.clear(); - // #38: S11 reference disabled because no path was provided; surface it instead of silently skipping. - std::cerr << "S11 reference: no bundle path provided; reference normalization disabled\n"; - return; - } - - traces_by_combo_ = load_channel_traces(path, "S11 reference", RawTraceChannel::S11); -} - -auto S11ReferenceBundle::is_enabled() const noexcept -> bool { - return !traces_by_combo_.empty(); -} - -void S11ReferenceBundle::validate_combos(const std::vector& combos) const { - validate_bundle_combos(traces_by_combo_, combos, "S11 reference bundle"); -} - -auto S11ReferenceBundle::resolve(const ipc::ComboKey& combo) const -> const ChannelTrace& { - return resolve_required(traces_by_combo_, combo, "S11 reference trace"); -} - -} // namespace radar::preprocessing diff --git a/data_acq_and_processing/preprocessing/calibration_master/src/through_calibrator.cpp b/data_acq_and_processing/preprocessing/calibration_master/src/through_calibrator.cpp deleted file mode 100644 index a146397..0000000 --- a/data_acq_and_processing/preprocessing/calibration_master/src/through_calibrator.cpp +++ /dev/null @@ -1,76 +0,0 @@ -#include "calibration_master.hpp" - -#include -#include -#include -#include - -#include - -namespace radar::preprocessing { -namespace { - -class ThroughCalibrator final : public CalibratorInterface { - public: - [[nodiscard]] auto name() const -> std::string override { - return "through"; - } - - [[nodiscard]] auto apply( - const std::vector& measured, - const std::vector& calibration - ) const -> std::vector override { - if (measured.size() != calibration.size()) { - throw std::runtime_error("Through calibration vector size mismatch"); - } - - static_assert(sizeof(ipc::Complex32) == sizeof(float) * 2U, "Complex32 layout must be two contiguous floats"); - - constexpr float epsilon = 1e-12F; - using InterleavedComplexView = Eigen::Matrix; - - const auto point_count = static_cast(measured.size()); - Eigen::Map measured_view( - reinterpret_cast(measured.data()), - point_count, - 2 - ); - Eigen::Map calibration_view( - reinterpret_cast(calibration.data()), - point_count, - 2 - ); - - const auto measured_re = measured_view.col(0).array(); - const auto measured_im = measured_view.col(1).array(); - const auto calibration_re = calibration_view.col(0).array(); - const auto calibration_im = calibration_view.col(1).array(); - - const Eigen::ArrayXf magnitude_squared = calibration_re.square() + calibration_im.square(); - const auto stable_division_mask = (magnitude_squared > epsilon); - - const Eigen::ArrayXf corrected_re = stable_division_mask.select( - ((measured_re * calibration_re) + (measured_im * calibration_im)) / magnitude_squared, - measured_re - ); - const Eigen::ArrayXf corrected_im = stable_division_mask.select( - ((measured_im * calibration_re) - (measured_re * calibration_im)) / magnitude_squared, - measured_im - ); - - std::vector corrected(measured.size()); - Eigen::Map corrected_view(reinterpret_cast(corrected.data()), point_count, 2); - corrected_view.col(0) = corrected_re.matrix(); - corrected_view.col(1) = corrected_im.matrix(); - - return corrected; - } -}; - -} // namespace - -auto make_s21_through_calibrator() -> std::unique_ptr { - return std::make_unique(); -} - -} // namespace radar::preprocessing diff --git a/data_acq_and_processing/preprocessing/data_preprocessor/include/data_preprocessor.hpp b/data_acq_and_processing/preprocessing/data_preprocessor/include/data_preprocessor.hpp index 7e0de76..98f9d1f 100644 --- a/data_acq_and_processing/preprocessing/data_preprocessor/include/data_preprocessor.hpp +++ b/data_acq_and_processing/preprocessing/data_preprocessor/include/data_preprocessor.hpp @@ -2,21 +2,27 @@ #include #include +#include #include -#include "calibration_master.hpp" -#include "reference_master.hpp" +#include "channel_bundle_support.hpp" #include "run_config.hpp" #include "shm_ring.hpp" namespace radar::preprocessing { +// Per-point notch multipliers: 0 inside every band, 1 outside; with the cosine +// taper a raised-cosine shoulder of taper_width_hz falls from 1 to 0 towards +// each band edge. +[[nodiscard]] auto build_notch_mask( + std::span frequency_hz, + const config::PreprocessNotchConfig& notch +) -> std::vector; + class DataPreprocessor { public: DataPreprocessor( const config::RunConfig& config, - CalibrationMaster& calibration_master, - ReferenceMaster& reference_master, ipc::ShmRing& raw_ring, ipc::ShmRing& preprocessed_ring, ipc::ShmRing* preprocessed_tap_ring = nullptr @@ -26,19 +32,13 @@ class DataPreprocessor { void run(const std::atomic& stop_requested); private: - void validate_startup_prerequisites() const; - [[nodiscard]] auto try_pop_raw_collection( - std::vector& serialized_raw, - ipc::RawSweepCollection* out_collection - ) -> bool; void publish_preprocessed_collection(const ipc::PreprocessedCollection& collection); - [[nodiscard]] auto preprocess_collection(const ipc::RawSweepCollection& raw_collection) const - -> ipc::PreprocessedCollection; + void preprocess_collection(ipc::RawSweepCollection& collection) const; const config::RunConfig& config_; - CalibrationMaster& calibration_master_; - ReferenceMaster& reference_master_; + S21CalibrationBundle calibration_; + S21ReferenceBundle reference_; S21ObjectCalibrationBundle object_calibration_; ipc::ShmRing& raw_ring_; ipc::ShmRing& preprocessed_ring_; diff --git a/data_acq_and_processing/preprocessing/data_preprocessor/src/data_preprocessor.cpp b/data_acq_and_processing/preprocessing/data_preprocessor/src/data_preprocessor.cpp index 537b346..dcc27d7 100644 --- a/data_acq_and_processing/preprocessing/data_preprocessor/src/data_preprocessor.cpp +++ b/data_acq_and_processing/preprocessing/data_preprocessor/src/data_preprocessor.cpp @@ -1,26 +1,25 @@ #include "data_preprocessor.hpp" #include "runtime_wait.hpp" +#include "pipeline_runtime.hpp" #include #include #include -#include +#include #include #include -#include -#include #include namespace radar::preprocessing { namespace { -constexpr double kPi = 3.14159265358979323846; - [[nodiscard]] auto should_apply_notch(const config::PreprocessNotchConfig& notch) -> bool { return notch.enabled && !notch.bands_hz.empty(); } -[[nodiscard]] auto build_notch_mask( +} // namespace + +auto build_notch_mask( std::span frequency_hz, const config::PreprocessNotchConfig& notch ) -> std::vector { @@ -51,15 +50,16 @@ constexpr double kPi = 3.14159265358979323846; continue; } + // Raised-cosine shoulders: 1 at the outer edge, 0 where the band starts. if (frequency >= trans_low_hz && frequency < low_hz) { - const double arg = kPi * (frequency - trans_low_hz) / taper_width_hz; - mask[index] *= static_cast(0.5 * (1.0 - std::cos(arg))); + const double arg = std::numbers::pi * (frequency - trans_low_hz) / taper_width_hz; + mask[index] *= static_cast(0.5 * (1.0 + std::cos(arg))); continue; } if (frequency > high_hz && frequency <= trans_high_hz) { - const double arg = kPi * (frequency - high_hz) / taper_width_hz; - mask[index] *= static_cast(0.5 * (1.0 + std::cos(arg))); + const double arg = std::numbers::pi * (frequency - high_hz) / taper_width_hz; + mask[index] *= static_cast(0.5 * (1.0 - std::cos(arg))); } } } @@ -67,6 +67,8 @@ constexpr double kPi = 3.14159265358979323846; return mask; } +namespace { + void apply_notch_mask(std::vector& samples, std::span mask) { const std::size_t point_count = std::min(samples.size(), mask.size()); for (std::size_t index = 0U; index < point_count; ++index) { @@ -75,132 +77,72 @@ void apply_notch_mask(std::vector& samples, std::span ipc::SweepTraceBlock { - if (!should_apply_notch(notch)) { - return trace; - } - - ipc::SweepTraceBlock filtered = trace; - const auto mask = build_notch_mask(filtered.frequency_hz, notch); - apply_notch_mask(filtered.s21, mask); - apply_notch_mask(filtered.s11, mask); - return filtered; -} - } // namespace DataPreprocessor::DataPreprocessor( const config::RunConfig& config, - CalibrationMaster& calibration_master, - ReferenceMaster& reference_master, ipc::ShmRing& raw_ring, ipc::ShmRing& preprocessed_ring, ipc::ShmRing* preprocessed_tap_ring ) : config_(config), - calibration_master_(calibration_master), - reference_master_(reference_master), raw_ring_(raw_ring), preprocessed_ring_(preprocessed_ring), preprocessed_tap_ring_(preprocessed_tap_ring) { + calibration_.load(config_.preprocess.s21.calibration.bundle_path); + calibration_.validate_combos(config_.run_combos); + reference_.load(config_.preprocess.s21.reference.bundle_path, calibration_, config_.run_combos); object_calibration_.load(config_.preprocess.s21.object_calibration.bundle_path); -} - -void DataPreprocessor::validate_startup_prerequisites() const { - calibration_master_.validate_combos(config_.run_combos); - reference_master_.validate_combos(config_.run_combos); object_calibration_.validate_combos(config_.run_combos); } -auto DataPreprocessor::try_pop_raw_collection( - std::vector& serialized_raw, - ipc::RawSweepCollection* out_collection -) -> bool { - if (!raw_ring_.pop(serialized_raw)) { - return false; - } - - *out_collection = ipc::deserialize_raw_collection(serialized_raw); - return true; -} - void DataPreprocessor::publish_preprocessed_collection(const ipc::PreprocessedCollection& collection) { const auto serialized_preprocessed = ipc::serialize_preprocessed_collection(collection); if (!preprocessed_ring_.push(serialized_preprocessed)) { - throw std::runtime_error("Preprocessed ring slot is too small for serialized collection"); + throw runtime::FatalPipelineError("Preprocessed ring slot is too small for serialized collection"); } if (preprocessed_tap_ring_ != nullptr && !preprocessed_tap_ring_->push(serialized_preprocessed)) { - throw std::runtime_error("Preprocessed tap ring slot is too small for serialized collection"); + throw runtime::FatalPipelineError("Preprocessed tap ring slot is too small for serialized collection"); } } void DataPreprocessor::run(const std::atomic& stop_requested) { - validate_startup_prerequisites(); - std::vector serialized_raw{}; - ipc::RawSweepCollection raw_collection{}; - std::uint64_t error_count = 0; - std::uint64_t consecutive_errors = 0; + runtime::CollectionErrors errors; while (!stop_requested.load(std::memory_order_relaxed)) { try { - if (!try_pop_raw_collection(serialized_raw, &raw_collection)) { + if (!raw_ring_.pop(serialized_raw)) { (void)config::wait_or_stop(config_.runtime.idle_sleep_ms, stop_requested); continue; } - const auto preprocessed_collection = preprocess_collection(raw_collection); - publish_preprocessed_collection(preprocessed_collection); - consecutive_errors = 0; // made progress; reset the escalation counter + auto raw_collection = ipc::deserialize_raw_collection(serialized_raw); + preprocess_collection(raw_collection); + publish_preprocessed_collection(raw_collection); + errors.progress(); + } catch (const runtime::FatalPipelineError&) { + throw; } catch (const std::exception& exc) { - // A ring-slot-too-small failure is a permanent config error (the slot - // cannot hold a serialized collection): swallowing it would silently - // drop every collection forever, so fail fast and let the operator fix it. - if (std::string(exc.what()).find("slot is too small") != std::string::npos) { - throw; - } - // A single malformed or transiently-bad collection must not kill the - // long-running preprocessor: drop it and keep serving the next sweep. - ++error_count; - ++consecutive_errors; - // Escalate a PERSISTENT failure loudly so a deterministic config error - // (which drops every collection forever) is not hidden by the throttle. - if (consecutive_errors % 500 == 0) { - std::cerr << "data_preprocessor: " << consecutive_errors - << " consecutive failures — likely a permanent configuration error: " - << exc.what() << '\n'; - } else if (error_count % 100 == 1) { - std::cerr << "data_preprocessor: dropped collection after error (count=" - << error_count << "): " << exc.what() << '\n'; - } - // Back-off floored at 1ms (the empty-ring sleep is never reached while a - // persistent error keeps the ring non-empty) so the catch cannot busy-spin - // a CPU core even when idle_sleep_ms is configured to 0. + errors.report("data_preprocessor", exc); (void)config::wait_or_stop(std::max(1U, config_.runtime.idle_sleep_ms), stop_requested); } } } -auto DataPreprocessor::preprocess_collection(const ipc::RawSweepCollection& raw_collection) const - -> ipc::PreprocessedCollection { - ipc::PreprocessedCollection preprocessed{}; - preprocessed.collection_id = raw_collection.collection_id; - preprocessed.monotonic_ns = ipc::current_monotonic_ns(); - preprocessed.capture_start_ns = raw_collection.capture_start_ns; - preprocessed.capture_end_ns = raw_collection.capture_end_ns; - preprocessed.traces.reserve(raw_collection.traces.size()); - - for (const auto& raw_trace : raw_collection.traces) { - // Object-derived corrections act on the completed preprocessing output. - const auto calibrated = calibration_master_.apply_to_trace(raw_trace); - const auto referenced = reference_master_.apply_to_trace(calibrated); - auto filtered = apply_notch_filter(referenced, config_.preprocess.notch); - object_calibration_.apply(filtered); - preprocessed.traces.push_back(std::move(filtered)); +void DataPreprocessor::preprocess_collection(ipc::RawSweepCollection& collection) const { + if (!collection.capture_start_ns || collection.capture_end_ns < collection.capture_start_ns) { + throw std::runtime_error("Raw collection requires a valid acquisition window"); } - - return preprocessed; + for (auto& trace : collection.traces) { + calibration_.apply(trace.combo, trace.frequency_hz, trace.s21); + reference_.apply(trace); + if (should_apply_notch(config_.preprocess.notch)) { + const auto mask = build_notch_mask(trace.frequency_hz, config_.preprocess.notch); + apply_notch_mask(trace.s21, mask); + } + // Object-derived corrections act on the completed preprocessing output. + object_calibration_.apply(trace); + } + collection.monotonic_ns = ipc::current_monotonic_ns(); } } // namespace radar::preprocessing diff --git a/data_acq_and_processing/preprocessing/data_preprocessor/src/main.cpp b/data_acq_and_processing/preprocessing/data_preprocessor/src/main.cpp index 9cc8143..a25c067 100644 --- a/data_acq_and_processing/preprocessing/data_preprocessor/src/main.cpp +++ b/data_acq_and_processing/preprocessing/data_preprocessor/src/main.cpp @@ -1,49 +1,14 @@ -#include -#include #include #include -#include +#include -#include "calibration_master.hpp" #include "data_preprocessor.hpp" -#include "reference_master.hpp" -#include "run_config.hpp" +#include "pipeline_runtime.hpp" #include "shm_ring.hpp" -namespace { - -constexpr const char* kDefaultConfigPath = "run_config.json"; - -std::atomic g_stop_requested{false}; - -void signal_handler(int /*signal*/) { - g_stop_requested.store(true, std::memory_order_relaxed); -} - -void install_signal_handlers() { - std::signal(SIGINT, signal_handler); - std::signal(SIGTERM, signal_handler); -} - -[[nodiscard]] auto read_config_path(int argc, char** argv) -> std::string { - std::string config_path = kDefaultConfigPath; - for (int index = 1; index < argc; ++index) { - const std::string arg = argv[index]; - if (arg == "--config" && (index + 1) < argc) { - config_path = argv[++index]; - } - } - return config_path; -} - -} // namespace - int main(int argc, char** argv) { - install_signal_handlers(); - try { - const auto config_path = read_config_path(argc, argv); - const auto config = radar::config::load_run_config(config_path); + const auto config = radar::runtime::initialize(argc, argv); auto raw_ring = radar::ipc::ShmRing::open_or_create( config.rings.raw.name, @@ -55,37 +20,22 @@ int main(int argc, char** argv) { config.rings.preprocessed.capacity, config.rings.preprocessed.slot_size_bytes ); - auto preprocessed_tap_ring = radar::ipc::ShmRing::open_or_create( - config.rings.preprocessed_tap.name, - config.rings.preprocessed_tap.capacity, - config.rings.preprocessed_tap.slot_size_bytes - ); - - // Load preprocessing assets once before entering run loop. - radar::preprocessing::CalibrationMaster calibration_master( - radar::preprocessing::make_s21_through_calibrator() - ); - calibration_master.load_s21_calibration_bundle(config.preprocess.s21.calibration.bundle_path); - calibration_master.load_s11_calibration_bundle( - config.preprocess.s11.calibration.open.bundle_path, - config.preprocess.s11.calibration.short_standard.bundle_path, - config.preprocess.s11.calibration.load.bundle_path - ); - - radar::preprocessing::ReferenceMaster reference_master; - reference_master.load_s21_reference_bundle(config.preprocess.s21.reference.bundle_path); - reference_master.load_s11_reference_bundle(config.preprocess.s11.reference.bundle_path); - reference_master.prepare_calibrated(calibration_master, config.run_combos); + std::optional preprocessed_tap_ring; + if (!config.rings.preprocessed_tap.name.empty()) { + preprocessed_tap_ring = radar::ipc::ShmRing::open_or_create( + config.rings.preprocessed_tap.name, + config.rings.preprocessed_tap.capacity, + config.rings.preprocessed_tap.slot_size_bytes + ); + } radar::preprocessing::DataPreprocessor preprocessor( config, - calibration_master, - reference_master, raw_ring, preprocessed_ring, - &preprocessed_tap_ring + preprocessed_tap_ring ? &*preprocessed_tap_ring : nullptr ); - preprocessor.run(g_stop_requested); + preprocessor.run(radar::runtime::stop_requested); return 0; } catch (const std::exception& exception) { std::cerr << "data_preprocessor error: " << exception.what() << '\n'; diff --git a/data_acq_and_processing/preprocessing/reference_master/include/reference_master.hpp b/data_acq_and_processing/preprocessing/reference_master/include/reference_master.hpp deleted file mode 100644 index 2d3ae70..0000000 --- a/data_acq_and_processing/preprocessing/reference_master/include/reference_master.hpp +++ /dev/null @@ -1,46 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include "channel_bundle_support.hpp" -#include "shared_types.hpp" - -namespace radar::preprocessing { - -class CalibrationMaster; - -class ReferenceMaster { - public: - // Loads a serialized raw sweep bundle with one S21 reference trace per combo. - void load_s21_reference_bundle(const std::string& path); - // Loads optional raw S11 reference data. - void load_s11_reference_bundle(const std::string& path); - // Builds calibrated references in memory using currently loaded raw references. - // Must be called after load_s21_reference_bundle() and after calibration standards are loaded. - void prepare_calibrated(const CalibrationMaster& calibration_master, const std::vector& combos); - // Ensures all runtime combos are present in loaded references. - void validate_combos(const std::vector& combos) const; - // Subtracts both channel references from one calibrated trace. - [[nodiscard]] auto apply_to_trace(const ipc::SweepTraceBlock& calibrated_trace) const -> ipc::SweepTraceBlock; - [[nodiscard]] auto apply_s21( - const ipc::ComboKey& combo, - std::span frequency_hz, - const std::vector& calibrated_s21 - ) const -> std::vector; - [[nodiscard]] auto apply_s11( - const ipc::ComboKey& combo, - std::span frequency_hz, - const std::vector& calibrated_s11 - ) const -> std::vector; - - private: - S21ReferenceBundle raw_s21_reference_bundle_{}; - std::unordered_map calibrated_s21_references_by_combo_{}; - S11ReferenceBundle raw_s11_reference_bundle_{}; - std::unordered_map calibrated_s11_references_by_combo_{}; -}; - -} // namespace radar::preprocessing diff --git a/data_acq_and_processing/preprocessing/reference_master/src/reference_master.cpp b/data_acq_and_processing/preprocessing/reference_master/src/reference_master.cpp deleted file mode 100644 index 7c6bf62..0000000 --- a/data_acq_and_processing/preprocessing/reference_master/src/reference_master.cpp +++ /dev/null @@ -1,180 +0,0 @@ -#include "reference_master.hpp" - -#include -#include -#include -#include -#include - -#include - -#include "calibration_master.hpp" - -namespace radar::preprocessing { -namespace { - -[[nodiscard]] auto combo_to_string(const ipc::ComboKey& combo) -> std::string { - return "input=" + std::to_string(combo.input_pos) + " output=" + std::to_string(combo.output_pos); -} - -} // namespace - -void ReferenceMaster::load_s21_reference_bundle(const std::string& path) { - raw_s21_reference_bundle_.load(path); - calibrated_s21_references_by_combo_.clear(); -} - -void ReferenceMaster::load_s11_reference_bundle(const std::string& path) { - raw_s11_reference_bundle_.load(path); - calibrated_s11_references_by_combo_.clear(); -} - -void ReferenceMaster::prepare_calibrated( - const CalibrationMaster& calibration_master, - const std::vector& combos -) { - if (!raw_s21_reference_bundle_.is_enabled()) { - throw std::runtime_error("Reference bundle must be loaded before prepare_calibrated()"); - } - - calibrated_s21_references_by_combo_.clear(); - calibrated_s21_references_by_combo_.reserve(combos.size()); - for (const auto& combo : combos) { - const auto& raw_reference = raw_s21_reference_bundle_.resolve(combo); - ChannelTrace calibrated{}; - calibrated.frequency_hz = raw_reference.frequency_hz; - calibrated.samples = calibration_master.apply_s21(combo, raw_reference.frequency_hz, raw_reference.samples); - calibrated_s21_references_by_combo_.insert_or_assign(combo, std::move(calibrated)); - } - - calibrated_s11_references_by_combo_.clear(); - if (!raw_s11_reference_bundle_.is_enabled()) { - return; - } - if (!calibration_master.has_s11_calibration()) { - throw std::runtime_error("S11 reference bundle requires S11 calibration bundle"); - } - - calibrated_s11_references_by_combo_.reserve(combos.size()); - for (const auto& combo : combos) { - const auto& raw_reference = raw_s11_reference_bundle_.resolve(combo); - ChannelTrace calibrated{}; - calibrated.frequency_hz = raw_reference.frequency_hz; - calibrated.samples = calibration_master.apply_s11(combo, raw_reference.frequency_hz, raw_reference.samples); - calibrated_s11_references_by_combo_.insert_or_assign(combo, std::move(calibrated)); - } -} - -void ReferenceMaster::validate_combos(const std::vector& combos) const { - if (calibrated_s21_references_by_combo_.empty()) { - throw std::runtime_error( - "Calibrated references are not prepared. Call ReferenceMaster::prepare_calibrated() at startup." - ); - } - - raw_s21_reference_bundle_.validate_combos(combos); - for (const auto& combo : combos) { - if (!calibrated_s21_references_by_combo_.contains(combo)) { - throw std::runtime_error("Calibrated S21 reference data is missing for combo " + combo_to_string(combo)); - } - } - - raw_s11_reference_bundle_.validate_combos(combos); - if (!raw_s11_reference_bundle_.is_enabled()) { - return; - } - if (calibrated_s11_references_by_combo_.empty()) { - throw std::runtime_error("Calibrated S11 references are not prepared"); - } - for (const auto& combo : combos) { - if (!calibrated_s11_references_by_combo_.contains(combo)) { - throw std::runtime_error("Calibrated S11 reference data is missing for combo " + combo_to_string(combo)); - } - } -} - -auto ReferenceMaster::apply_to_trace(const ipc::SweepTraceBlock& calibrated_trace) const -> ipc::SweepTraceBlock { - ipc::SweepTraceBlock output{}; - output.combo = calibrated_trace.combo; - output.frequency_hz = calibrated_trace.frequency_hz; - output.s21 = apply_s21(calibrated_trace.combo, calibrated_trace.frequency_hz, calibrated_trace.s21); - output.s11 = apply_s11(calibrated_trace.combo, calibrated_trace.frequency_hz, calibrated_trace.s11); - // Reference subtraction reshapes the samples, not when they were measured. - output.capture_start_ns = calibrated_trace.capture_start_ns; - output.capture_end_ns = calibrated_trace.capture_end_ns; - return output; -} - -auto ReferenceMaster::apply_s21( - const ipc::ComboKey& combo, - std::span frequency_hz, - const std::vector& calibrated_s21 -) const -> std::vector { - const auto found = calibrated_s21_references_by_combo_.find(combo); - if (found == calibrated_s21_references_by_combo_.end()) { - throw std::runtime_error("Calibrated S21 reference is missing for combo " + combo_to_string(combo)); - } - - const auto& reference = found->second; - validate_channel_trace_layout(reference, "Calibrated S21 reference", combo); - if (calibrated_s21.size() != reference.samples.size()) { - throw std::runtime_error("S21 reference point count mismatch for combo " + combo_to_string(combo)); - } - if (!frequency_axes_match(frequency_hz, reference.frequency_hz)) { - throw std::runtime_error("S21 reference frequency axis mismatch for combo " + combo_to_string(combo)); - } - - std::vector output(calibrated_s21.size()); - - static_assert(sizeof(ipc::Complex32) == sizeof(float) * 2U, "Complex32 layout must be two contiguous floats"); - using InterleavedComplexView = Eigen::Matrix; - const auto point_count = static_cast(calibrated_s21.size()); - - Eigen::Map calibrated_view( - reinterpret_cast(calibrated_s21.data()), - point_count, - 2 - ); - Eigen::Map reference_view( - reinterpret_cast(reference.samples.data()), - point_count, - 2 - ); - Eigen::Map output_view(reinterpret_cast(output.data()), point_count, 2); - - output_view = calibrated_view - reference_view; - return output; -} - -auto ReferenceMaster::apply_s11( - const ipc::ComboKey& combo, - std::span frequency_hz, - const std::vector& calibrated_s11 -) const -> std::vector { - if (!raw_s11_reference_bundle_.is_enabled()) { - return calibrated_s11; - } - - const auto found = calibrated_s11_references_by_combo_.find(combo); - if (found == calibrated_s11_references_by_combo_.end()) { - throw std::runtime_error("Calibrated S11 reference is missing for combo " + combo_to_string(combo)); - } - - const auto& reference = found->second; - validate_channel_trace_layout(reference, "Calibrated S11 reference", combo); - if (calibrated_s11.size() != reference.samples.size()) { - throw std::runtime_error("S11 reference point count mismatch for combo " + combo_to_string(combo)); - } - if (!frequency_axes_match(frequency_hz, reference.frequency_hz)) { - throw std::runtime_error("S11 reference frequency axis mismatch for combo " + combo_to_string(combo)); - } - - std::vector output(calibrated_s11.size()); - for (std::size_t index = 0; index < calibrated_s11.size(); ++index) { - output[index].re = calibrated_s11[index].re - reference.samples[index].re; - output[index].im = calibrated_s11[index].im - reference.samples[index].im; - } - return output; -} - -} // namespace radar::preprocessing diff --git a/data_acq_and_processing/preprocessing/testdata/s11_load_calibration_bundle.bin b/data_acq_and_processing/preprocessing/testdata/s11_load_calibration_bundle.bin deleted file mode 100644 index 6cee6b2..0000000 Binary files a/data_acq_and_processing/preprocessing/testdata/s11_load_calibration_bundle.bin and /dev/null differ diff --git a/data_acq_and_processing/preprocessing/testdata/s11_open_calibration_bundle.bin b/data_acq_and_processing/preprocessing/testdata/s11_open_calibration_bundle.bin deleted file mode 100644 index 283d22a..0000000 Binary files a/data_acq_and_processing/preprocessing/testdata/s11_open_calibration_bundle.bin and /dev/null differ diff --git a/data_acq_and_processing/preprocessing/testdata/s11_reference_bundle.bin b/data_acq_and_processing/preprocessing/testdata/s11_reference_bundle.bin deleted file mode 100644 index 1d64a3b..0000000 Binary files a/data_acq_and_processing/preprocessing/testdata/s11_reference_bundle.bin and /dev/null differ diff --git a/data_acq_and_processing/preprocessing/testdata/s11_short_calibration_bundle.bin b/data_acq_and_processing/preprocessing/testdata/s11_short_calibration_bundle.bin deleted file mode 100644 index 8b7e104..0000000 Binary files a/data_acq_and_processing/preprocessing/testdata/s11_short_calibration_bundle.bin and /dev/null differ diff --git a/data_acq_and_processing/processing/data_processor/include/processing_live_config.hpp b/data_acq_and_processing/processing/data_processor/include/processing_live_config.hpp index 3074803..446baaa 100644 --- a/data_acq_and_processing/processing/data_processor/include/processing_live_config.hpp +++ b/data_acq_and_processing/processing/data_processor/include/processing_live_config.hpp @@ -10,12 +10,6 @@ namespace radar::processing { -enum class HistoryCommand { - None, - RemoveLast, - ClearAll, -}; - enum class LegacyGprMode { Point, Extended, @@ -29,16 +23,16 @@ enum class LegacyGprReferenceMode { FirstTxEvent, }; +struct HistoryTarget { + std::uint64_t collection_id = 0; + std::uint64_t capture_start_ns = 0; +}; + struct ProcessingLiveConfig { std::string processor_mode = "pass_through"; - std::string pass_through_channel = "s21"; - bool pass_through_fixed_y_enabled = false; - float pass_through_y_min_db = -100.0F; - float pass_through_y_max_db = 0.0F; std::string bscan_axis = "abs"; - std::string bscan_channel = "s21"; - float bscan_cut_m = 0.0F; - float bscan_max_depth_m = 8.0F; + float bscan_cut_m = 0.824F; + float bscan_max_depth_m = 1.0F; float bscan_gain = 1.0F; float bscan_start_freq_mhz = 100.0F; float bscan_stop_freq_mhz = 8800.0F; @@ -51,13 +45,12 @@ struct ProcessingLiveConfig { float gpr_max_depth_m = 14.0F; float gpr_range_comp_power = 0.1F; float gpr_comp_power = 0.2F; - // BP object-detection stop level, as a fraction of the global peak (Python - // Horns_motion_3libre.py BP_OBJECT_MIN_FRAC); peaks below it are not objects. + // BP object-detection stop level, as a fraction of the global peak; peaks + // below it are not objects. float gpr_object_min_frac = 0.7F; std::string gpr_score_mode = "combined"; // Backprojection intra-sweep speed-correction mode: "int_minus" (full - // correction) or "int_focus" (focusing residual only). Mirrors the Python - // Horns_motion_3libre.py MOTION_CORRECTION_MODE selector. + // correction) or "int_focus" (focusing residual only). std::string gpr_motion_mode = "int_minus"; float gpr_speed_m_s = 0.0F; float gpr_look_angle_deg = 0.0F; @@ -81,8 +74,8 @@ struct ProcessingLiveConfig { // Default 0 keeps legacy 1D antenna layouts imaging in the antenna plane. float gpr_imaging_plane_y_m = 0.0F; // Coherent BP object visibility (window + the draw limits below) is applied in - // the processor itself, matching Horns_motion_3libre.py — there is NO score - // threshold for it. Only legacy GPR still thresholds, on a pair count. + // the processor itself, with no score threshold. Only legacy GPR thresholds, + // on a pair count. float legacy_gpr_min_visible_pair_count = 0.0F; std::uint32_t gpr_max_detected_objects_to_draw = 0; std::uint32_t gpr_draw_top_m_objects = 0; @@ -111,8 +104,9 @@ struct ProcessingLiveConfig { // "ignore socket speed" checkbox. bool ignore_socket_speed = false; bool reprocess_current_result = true; - std::uint64_t history_command_seq = 0; - HistoryCommand history_command = HistoryCommand::None; + // Persistent desired history: normal settings writes must preserve deletions. + std::uint64_t history_clear_before_ns = 0; + std::vector history_removed{}; std::uint64_t settings_revision = 0; }; diff --git a/data_acq_and_processing/processing/data_processor/src/data_processor.cpp b/data_acq_and_processing/processing/data_processor/src/data_processor.cpp index 3e76a67..14679cc 100644 --- a/data_acq_and_processing/processing/data_processor/src/data_processor.cpp +++ b/data_acq_and_processing/processing/data_processor/src/data_processor.cpp @@ -1,5 +1,6 @@ #include "data_processor.hpp" #include "runtime_wait.hpp" +#include "pipeline_runtime.hpp" #include #include @@ -69,9 +70,7 @@ void DataProcessor::run(const std::atomic& stop_requested) { std::vector preprocessed_history{}; const std::size_t history_limit = replay_history_limit(config_); std::uint64_t last_replayed_revision = live_config_loader_.revision(); - std::uint64_t last_applied_history_command_seq = 0; - std::uint64_t error_count = 0; - std::uint64_t consecutive_errors = 0; + runtime::CollectionErrors errors; std::uint64_t oversized_results = 0; // track the socket-fed speed used for the last reprocess so a change // arriving without a live-config revision bump still triggers a reprocess of @@ -95,17 +94,19 @@ void DataProcessor::run(const std::atomic& stop_requested) { auto& processor = resolve_processor(live_config); const bool socket_speed_dirty = current_socket_speed != last_reprocessed_socket_speed; - if (live_revision != last_replayed_revision || socket_speed_dirty) { - if (live_config.history_command_seq > last_applied_history_command_seq) { - if (live_config.history_command == HistoryCommand::RemoveLast) { - if (!preprocessed_history.empty()) { - preprocessed_history.pop_back(); - } - } else if (live_config.history_command == HistoryCommand::ClearAll) { - preprocessed_history.clear(); - } - last_applied_history_command_seq = live_config.history_command_seq; + const auto was_removed = [&](const ipc::PreprocessedCollection& collection) { + if (live_config.history_clear_before_ns && + collection.capture_start_ns <= live_config.history_clear_before_ns) { + return true; } + return std::any_of(live_config.history_removed.begin(), live_config.history_removed.end(), + [&](const HistoryTarget& target) { + return collection.collection_id == target.collection_id && + collection.capture_start_ns == target.capture_start_ns; + }); + }; + if (live_revision != last_replayed_revision || socket_speed_dirty) { + std::erase_if(preprocessed_history, was_removed); if (!live_config.reprocess_current_result) { // Socket-fed speed updates should affect only future preprocessed collections, @@ -139,6 +140,12 @@ void DataProcessor::run(const std::atomic& stop_requested) { if (preprocessed_ring_.pop(bytes)) { auto preprocessed = ipc::deserialize_preprocessed_collection(bytes); + if (!preprocessed.capture_start_ns || preprocessed.capture_end_ns < preprocessed.capture_start_ns) { + throw std::runtime_error("Preprocessed collection requires a valid acquisition window"); + } + if (was_removed(preprocessed)) { + continue; // A deletion also applies to queued input and later GUI replays. + } // Sweeps the GUI feeds back to recompute its B-scan keep their original // timestamps, so they are no newer than the newest retained sweep. They // are processed like any other but stay out of the history, which must @@ -146,7 +153,7 @@ void DataProcessor::run(const std::atomic& stop_requested) { // background from it), and away from the locator, which reports the // current scene. const bool fed_back = !preprocessed_history.empty() && - preprocessed.monotonic_ns <= preprocessed_history.back().monotonic_ns; + preprocessed.capture_start_ns <= preprocessed_history.back().capture_start_ns; if (!fed_back) { preprocessed_history.push_back(std::move(preprocessed)); while (preprocessed_history.size() > history_limit) { @@ -157,7 +164,7 @@ void DataProcessor::run(const std::atomic& stop_requested) { const auto earlier = std::partition_point( preprocessed_history.begin(), preprocessed_history.end(), - [&](const ipc::PreprocessedCollection& retained) { return retained.monotonic_ns < current.monotonic_ns; } + [&](const ipc::PreprocessedCollection& retained) { return retained.capture_start_ns < current.capture_start_ns; } ); const auto result_collection = process_collection( @@ -170,29 +177,13 @@ void DataProcessor::run(const std::atomic& stop_requested) { if (!fed_back) { publish_locator(result_collection, live_config); } - consecutive_errors = 0; // made progress; reset the escalation counter + errors.progress(); continue; } (void)config::wait_or_stop(config_.runtime.idle_sleep_ms, stop_requested); } catch (const std::exception& exc) { - // A single bad collection (torn ring slot, decode/processing error) must - // not kill the long-running processor: drop it and keep going. - ++error_count; - ++consecutive_errors; - // Escalate a PERSISTENT failure (a deterministic config error drops every - // collection forever): log it loudly and distinctly — not buried in the - // every-100 throttle — so liveness-only supervision still surfaces it. - if (consecutive_errors % 500 == 0) { - std::cerr << "data_processor: " << consecutive_errors - << " consecutive failures — likely a permanent configuration error: " - << exc.what() << '\n'; - } else if (error_count % 100 == 1) { - std::cerr << "data_processor: dropped collection after error (count=" - << error_count << "): " << exc.what() << '\n'; - } - // Back-off (floored at 1ms even if idle_sleep_ms==0) so a persistent error - // cannot busy-spin a core or flood the log. + errors.report("data_processor", exc); (void)config::wait_or_stop(std::max(1U, config_.runtime.idle_sleep_ms), stop_requested); } } @@ -210,15 +201,13 @@ auto DataProcessor::process_collection( result.processing_duration_ns = static_cast( std::chrono::duration_cast(finished_at - started_at).count() ); - // Four 16-bit words retain the uint64 exactly in the existing float32 table format. - ipc::ResultPayload revision{}; - revision.processing_name = "processing_revision"; - revision.kind = ipc::ResultKind::TableF32; - revision.table_columns = 4U; - for (unsigned shift = 0; shift < 64; shift += 16) { - revision.table_values.push_back(static_cast((live_config.settings_revision >> shift) & 0xffffU)); - } - result.collection_payloads.push_back(std::move(revision)); + result.collection_id = preprocessed.collection_id; + result.monotonic_ns = ipc::current_monotonic_ns(); + result.settings_revision = live_config.settings_revision; + result.capture_start_ns = preprocessed.capture_start_ns; + result.capture_end_ns = preprocessed.capture_end_ns; + // Socket speed can change without a GUI revision; retain the actual value per result. + result.speed_m_s = live_config.gpr_speed_m_s; return result; } @@ -268,7 +257,7 @@ auto DataProcessor::build_locator_filter(const ProcessingLiveConfig& live_config filter.draw_limits.reset(); } // Coherent BP already emits the FINAL visible object set from the processor - // (window + N/M, no score threshold — matching Horns_motion_3libre.py), so the + // (window + N/M, no score threshold), so the // socket forwards it verbatim. The default FilterParams passes everything through // (it only drops non-finite rows), keeping the filtering logic in one place. return filter; diff --git a/data_acq_and_processing/processing/data_processor/src/main.cpp b/data_acq_and_processing/processing/data_processor/src/main.cpp index acb1f23..c8c696b 100644 --- a/data_acq_and_processing/processing/data_processor/src/main.cpp +++ b/data_acq_and_processing/processing/data_processor/src/main.cpp @@ -1,43 +1,14 @@ -#include -#include #include #include #include -#include #include "data_processor.hpp" #include "locator/tcp_server.hpp" -#include "run_config.hpp" +#include "pipeline_runtime.hpp" #include "shm_ring.hpp" namespace { -constexpr const char* kDefaultConfigPath = "run_config.json"; - -std::atomic g_stop_requested{false}; - -void signal_handler(int /*signal*/) { - g_stop_requested.store(true, std::memory_order_relaxed); -} - -void install_signal_handlers() { - std::signal(SIGINT, signal_handler); - std::signal(SIGTERM, signal_handler); - // Writing to a peer-closed socket would otherwise terminate the process. - std::signal(SIGPIPE, SIG_IGN); -} - -[[nodiscard]] auto read_config_path(int argc, char** argv) -> std::string { - std::string config_path = kDefaultConfigPath; - for (int index = 1; index < argc; ++index) { - const std::string arg = argv[index]; - if (arg == "--config" && (index + 1) < argc) { - config_path = argv[++index]; - } - } - return config_path; -} - [[nodiscard]] auto start_locator_server(const radar::config::RunConfig& config) -> std::unique_ptr { if (!config.locator_server.enabled) { @@ -56,11 +27,8 @@ void install_signal_handlers() { } // namespace int main(int argc, char** argv) { - install_signal_handlers(); - try { - const auto config_path = read_config_path(argc, argv); - const auto config = radar::config::load_run_config(config_path); + const auto config = radar::runtime::initialize(argc, argv); auto preprocessed_ring = radar::ipc::ShmRing::open_or_create( config.rings.preprocessed.name, @@ -82,7 +50,7 @@ int main(int argc, char** argv) { radar::processing::create_default_processors(), locator_server.get() ); - processor.run(g_stop_requested); + processor.run(radar::runtime::stop_requested); return 0; } catch (const std::exception& exception) { std::cerr << "data_processor error: " << exception.what() << '\n'; diff --git a/data_acq_and_processing/processing/data_processor/src/processing_live_config.cpp b/data_acq_and_processing/processing/data_processor/src/processing_live_config.cpp index da57661..c6465a4 100644 --- a/data_acq_and_processing/processing/data_processor/src/processing_live_config.cpp +++ b/data_acq_and_processing/processing/data_processor/src/processing_live_config.cpp @@ -19,19 +19,6 @@ namespace { using Json = nlohmann::json; -[[nodiscard]] auto parse_history_command(const std::string& value) -> HistoryCommand { - if (value == "none") { - return HistoryCommand::None; - } - if (value == "remove_last") { - return HistoryCommand::RemoveLast; - } - if (value == "clear_all") { - return HistoryCommand::ClearAll; - } - throw std::runtime_error("processing.history_command must be one of: none, remove_last, clear_all"); -} - [[nodiscard]] auto parse_legacy_gpr_mode(const std::string& value, const std::string& field_name) -> LegacyGprMode { if (value == "point") { return LegacyGprMode::Point; @@ -86,13 +73,6 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s } } -[[nodiscard]] auto parse_s_parameter_channel(const std::string& value, const std::string& field_name) -> std::string { - if (value == "s21" || value == "s11") { - return value; - } - throw std::runtime_error(field_name + " must be one of: s21, s11"); -} - [[nodiscard]] auto parse_f32(const Json& value, const std::string& field_name) -> float { if (!value.is_number()) { throw std::runtime_error(field_name + " must be number"); @@ -226,23 +206,7 @@ auto parse_geometry(const Json& value, const char* position_field, std::uint32_t config::validate_gpr_config(*config.gpr, run_config); } assign_optional(root, "processor_mode", config.processor_mode); - if (const auto found = root.find("pass_through_channel"); found != root.end()) { - if (!found->is_string()) { - throw std::runtime_error("processing.pass_through_channel must be string"); - } - config.pass_through_channel = - parse_s_parameter_channel(found->get(), "processing.pass_through_channel"); - } - assign_optional(root, "pass_through_fixed_y_enabled", config.pass_through_fixed_y_enabled); - assign_optional(root, "pass_through_y_min_db", config.pass_through_y_min_db); - assign_optional(root, "pass_through_y_max_db", config.pass_through_y_max_db); assign_optional(root, "bscan_axis", config.bscan_axis); - if (const auto found = root.find("bscan_channel"); found != root.end()) { - if (!found->is_string()) { - throw std::runtime_error("processing.bscan_channel must be string"); - } - config.bscan_channel = parse_s_parameter_channel(found->get(), "processing.bscan_channel"); - } assign_optional(root, "bscan_cut_m", config.bscan_cut_m); assign_optional(root, "bscan_max_depth_m", config.bscan_max_depth_m); assign_optional(root, "bscan_gain", config.bscan_gain); @@ -344,14 +308,23 @@ auto parse_geometry(const Json& value, const char* position_field, std::uint32_t } assign_optional(root, "ignore_socket_speed", config.ignore_socket_speed); assign_optional(root, "reprocess_current_result", config.reprocess_current_result); - if (const auto found = root.find("history_command_seq"); found != root.end()) { - config.history_command_seq = parse_u64_number(*found, "processing.history_command_seq"); + if (const auto found = root.find("history_clear_before_ns"); found != root.end()) { + config.history_clear_before_ns = parse_u64_number(*found, "processing.history_clear_before_ns"); } - if (const auto found = root.find("history_command"); found != root.end()) { - if (!found->is_string()) { - throw std::runtime_error("processing.history_command must be string"); + if (const auto found = root.find("history_removed"); found != root.end()) { + if (!found->is_array()) { + throw std::runtime_error("processing.history_removed must be an array"); + } + for (const auto& value : *found) { + HistoryTarget target{ + parse_u64_number(value.at("collection_id"), "history_removed.collection_id"), + parse_u64_number(value.at("capture_start_ns"), "history_removed.capture_start_ns"), + }; + if (!target.capture_start_ns) { + throw std::runtime_error("history_removed.capture_start_ns must be positive"); + } + config.history_removed.push_back(target); } - config.history_command = parse_history_command(found->get()); } return config; } @@ -393,9 +366,10 @@ auto ProcessingLiveConfigLoader::refresh_if_needed() -> ProcessingLiveConfig { return current_; } - current_ = parse_live_config(json_text, path_, run_config_); + // Remember rejected content too; retry only when the file actually changes. last_json_text_ = json_text; has_last_json_text_ = true; + current_ = parse_live_config(json_text, path_, run_config_); ++revision_; } catch (const std::exception& error) { std::cerr << "data_processor warning: failed to refresh live processing config: " << error.what() << '\n'; diff --git a/data_acq_and_processing/processing/locator/include/locator/tcp_server.hpp b/data_acq_and_processing/processing/locator/include/locator/tcp_server.hpp index d0d261e..1a8f1f0 100644 --- a/data_acq_and_processing/processing/locator/include/locator/tcp_server.hpp +++ b/data_acq_and_processing/processing/locator/include/locator/tcp_server.hpp @@ -36,7 +36,6 @@ class ClientQueue { // Wake any waiter and reject further pushes. Idempotent. void close(); - [[nodiscard]] auto is_closed() const -> bool; private: mutable std::mutex mutex_{}; diff --git a/data_acq_and_processing/processing/locator/src/tcp_server.cpp b/data_acq_and_processing/processing/locator/src/tcp_server.cpp index 4158a51..a65b070 100644 --- a/data_acq_and_processing/processing/locator/src/tcp_server.cpp +++ b/data_acq_and_processing/processing/locator/src/tcp_server.cpp @@ -197,11 +197,6 @@ void ClientQueue::close() { not_empty_.notify_all(); } -auto ClientQueue::is_closed() const -> bool { - std::lock_guard guard(mutex_); - return closed_; -} - // ----- ClientSession -------------------------------------------------------- ClientSession::ClientSession( diff --git a/data_acq_and_processing/processing/processors/include/object_approach_filter.hpp b/data_acq_and_processing/processing/processors/include/object_approach_filter.hpp index 7495def..388f185 100644 --- a/data_acq_and_processing/processing/processors/include/object_approach_filter.hpp +++ b/data_acq_and_processing/processing/processors/include/object_approach_filter.hpp @@ -10,8 +10,7 @@ namespace radar::processing { -// Temporal "approach" filter for coherent-BP detections — a port of the objects-only -// filter in Horns_motion_3libre's demonstrate notebook. +// Temporal "approach" filter for coherent-BP detections. // // It keeps only objects that persist as a *motion-consistent track* across at least // `min_frames` consecutive frames: as the radar moves, a real target reappears at a @@ -30,12 +29,8 @@ namespace radar::processing { // inter-frame gap (`kMaxFrameGapSeconds`), so a stale history from a previous run cannot // spuriously confirm objects. // -// IDEMPOTENT under reprocessing: the data_processor re-runs the last collection (or -// replays the whole window) whenever live settings or the socket speed change, with no -// new sweep. The history is therefore keyed by `frame_id` (the strictly increasing -// collection id): the same id replaces its entry (reprocess of the current frame), a -// smaller id rebuilds from scratch (a replay restart), so repeated reprocessing never -// duplicates frames or falsely confirms a track. +// Reprocessing replaces the same (collection_id, capture_start_ns) entry. Earlier +// capture times rebuild the history when the processor replays its retained window. class ObjectApproachFilter { public: struct Point { @@ -43,20 +38,16 @@ class ObjectApproachFilter { double z_m; }; - // Record `objects` as frame `frame_id` and return, per object, whether it is - // confirmed (ends a >= `min_frames` motion-consistent track). `frame_id` is the - // collection id (identity/order, survives reprocessing); `frame_time_seconds` is the - // frame's wall-clock timestamp (drives the inter-frame interval); `speed_m_s`/ - // `look_angle_deg` are the live motion estimate. `min_frames <= 1` disables filtering. + // Acquisition time drives both ordering and motion intervals. min_frames <= 1 disables filtering. [[nodiscard]] auto confirm( const std::vector& objects, std::uint64_t frame_id, - double frame_time_seconds, + std::uint64_t capture_start_ns, double speed_m_s, double look_angle_deg, std::size_t min_frames ) -> std::vector { - record_frame(Frame{frame_id, frame_time_seconds, objects}); + record_frame(Frame{frame_id, capture_start_ns, objects}); const std::size_t history_depth = std::max(min_frames, 1U); while (history_.size() > history_depth) { history_.pop_front(); @@ -79,17 +70,14 @@ class ObjectApproachFilter { private: struct Frame { std::uint64_t id; - double time_seconds; + std::uint64_t capture_start_ns; std::vector objects; }; - // Append a genuinely new frame, replace the current one on reprocessing (same id), - // or rebuild from scratch when the id steps backward (a replay restart). This keeps - // the history one entry per distinct collection no matter how often settings change. void record_frame(Frame frame) { - if (history_.empty() || frame.id > history_.back().id) { + if (history_.empty() || frame.capture_start_ns > history_.back().capture_start_ns) { history_.push_back(std::move(frame)); - } else if (frame.id == history_.back().id) { + } else if (frame.id == history_.back().id && frame.capture_start_ns == history_.back().capture_start_ns) { history_.back() = std::move(frame); } else { history_.clear(); @@ -97,7 +85,7 @@ class ObjectApproachFilter { } } - // Fixed matching tolerances (Horns_motion notebook 0.2 block). Range decreases as the + // Fixed matching tolerances. Range decreases as the // radar approaches the target, hence the negative Z sign. static constexpr double kXToleranceM = 0.45; static constexpr double kRangeToleranceFraction = 0.85; @@ -122,8 +110,10 @@ class ObjectApproachFilter { for (std::size_t step = 1U; step < min_frames; ++step) { const std::size_t earlier_index = newest - step; const Frame& earlier = history_[earlier_index]; - const double dt = history_[earlier_index + 1U].time_seconds - earlier.time_seconds; - if (!(dt > 0.0) || dt > kMaxFrameGapSeconds) { + const auto later_ns = history_[earlier_index + 1U].capture_start_ns; + if (later_ns <= earlier.capture_start_ns) { return false; } + const double dt = static_cast(later_ns - earlier.capture_start_ns) * 1e-9; + if (dt > kMaxFrameGapSeconds) { return false; // non-monotonic time, or a gap that breaks the track } const double expected_range_shift = range_step_per_second * dt; diff --git a/data_acq_and_processing/processing/processors/src/bscan_processor.cpp b/data_acq_and_processing/processing/processors/src/bscan_processor.cpp index e797977..6206572 100644 --- a/data_acq_and_processing/processing/processors/src/bscan_processor.cpp +++ b/data_acq_and_processing/processing/processors/src/bscan_processor.cpp @@ -93,16 +93,6 @@ void fft_inplace(std::vector>& values, bool inverse) { return std::abs(sample); } -[[nodiscard]] auto selected_trace_samples( - const ipc::SweepTraceBlock& trace, - const ProcessingLiveConfig& live_config -) -> std::span { - if (live_config.bscan_channel == "s11") { - return trace.s11; - } - return trace.s21; -} - [[nodiscard]] auto fallback_profile( const ipc::SweepTraceBlock& trace, std::span selected_samples @@ -125,7 +115,7 @@ void fft_inplace(std::vector>& values, bool inverse) { const ipc::SweepTraceBlock& trace, const ProcessingLiveConfig& live_config ) -> BScanProfile { - const auto selected_samples = selected_trace_samples(trace, live_config); + const auto selected_samples = std::span(trace.s21); const std::size_t point_count = std::min(trace.frequency_hz.size(), selected_samples.size()); if (point_count < 2U) { return fallback_profile(trace, selected_samples); @@ -237,8 +227,6 @@ auto BScanProcessor::process_collection( const ProcessingLiveConfig& live_config ) -> ipc::ResultCollection { ipc::ResultCollection results{}; - results.collection_id = collection.collection_id; - results.monotonic_ns = collection.monotonic_ns; results.blocks.reserve(collection.traces.size()); for (const auto& trace : collection.traces) { diff --git a/data_acq_and_processing/processing/processors/src/gpr_backprojection_processor.ipp b/data_acq_and_processing/processing/processors/src/gpr_backprojection_processor.ipp index 868f63f..034a537 100644 --- a/data_acq_and_processing/processing/processors/src/gpr_backprojection_processor.ipp +++ b/data_acq_and_processing/processing/processors/src/gpr_backprojection_processor.ipp @@ -51,7 +51,7 @@ constexpr double kScoreCfEps = 1e-12; using PairKey = std::uint64_t; -// Frequency-domain speed correction mode (Horns_motion_3libre.py MOTION_CONFIG): +// Frequency-domain speed correction mode: // IntMinus — full intra-frequency correction to the interleaved-sweep centre. // IntFocus — remove the constant + linear-in-frequency part of phi(f), leaving // the focusing residual without a net Z shift. @@ -653,9 +653,7 @@ void validate_collection_trace_order( return traces; } -// Frequency-domain speed correction applied to S21 *before* the IFFT, ported -// from Horns_motion_3libre.py (apply_intra_sweep_phase_correction + -// compute_frequency_sample_times). The measurement is interleaved Tx-by-frequency +// Frequency-domain speed correction applied to S21 *before* the IFFT. The measurement is interleaved Tx-by-frequency // — Tx0(f0), Tx1(f0), Tx0(f1), ... — so each Tx occupies one interleave slot and // its frequency k is sampled at global step (slot_count*k + slot). Each in-band // frequency's phase is rotated back to the centre of the (full) interleaved sweep. @@ -1740,7 +1738,7 @@ void add_bp_score_metrics( } } -// Select the FINAL visible objects exactly as Horns_motion_3libre.py does, so the +// Select the FINAL visible objects here, so the // processor is the single source of truth: the GUI plot and the locator socket both // consume this set verbatim (no second, duplicated filter). Steps, in order: // 1. drop sidelobe candidates (when enabled), then sort by score (peak tie-break); @@ -1844,8 +1842,6 @@ void add_bp_score_metrics( ObjectApproachFilter& approach_filter ) -> ipc::ResultCollection { ipc::ResultCollection results{}; - results.collection_id = collection.collection_id; - results.monotonic_ns = collection.monotonic_ns; const auto selection = build_geometry_selection(run_config, live_config); if (selection.input_positions.empty() || selection.output_positions.empty()) { @@ -1859,9 +1855,8 @@ void add_bp_score_metrics( return results; } - // Coherent BP fixes the medium to vacuum/air (eps_r = 1), matching the Python - // reference Horns_motion_3libre.py (its 0.3 block hardcodes eps_r = 1.0). The - // configurable relative_permittivity stays a legacy-GPR-only knob. + // Coherent BP fixes the medium to vacuum/air (eps_r = 1); the configurable + // relative_permittivity is a legacy-GPR-only knob. const double velocity_mps = kSpeedOfLightMetersPerSec; const double start_hz = static_cast(live_config.gpr_start_freq_mhz) * 1'000'000.0; const double stop_hz = static_cast(live_config.gpr_stop_freq_mhz) * 1'000'000.0; @@ -1948,7 +1943,7 @@ void add_bp_score_metrics( min_depth_m, max_depth_m, std::max(0.0, static_cast(live_config.gpr_range_comp_power)), - 0.0 // angle compensation fixed off (Python Horns_motion_3libre.py COMP_ANGLE_POWER = 0.0) + 0.0 // angle compensation is fixed off for coherent BP ); if (bp.image.empty()) { return results; @@ -1980,7 +1975,7 @@ void add_bp_score_metrics( results.collection_payloads.push_back(build_image_payload("gpr_accumulator", grid.x_grid, grid.z_grid, display_map)); // Final visible set (window + N/M), then the cross-frame approach filter: keep only - // objects confirmed as a >= min_frames motion-consistent track (Horns_motion notebook). + // objects confirmed as a >= min_frames motion-consistent track. const auto visible = select_visible_objects(objects, live_config); std::vector visible_points{}; visible_points.reserve(visible.size()); @@ -1990,7 +1985,7 @@ void add_bp_score_metrics( const auto confirmed = approach_filter.confirm( visible_points, collection.collection_id, - static_cast(collection.monotonic_ns) * 1e-9, + collection.capture_start_ns, static_cast(live_config.gpr_speed_m_s), static_cast(live_config.gpr_look_angle_deg), static_cast(live_config.gpr_object_approach_min_frames) diff --git a/data_acq_and_processing/processing/processors/src/gpr_legacy_processor.ipp b/data_acq_and_processing/processing/processors/src/gpr_legacy_processor.ipp index df595ae..0d398e3 100644 --- a/data_acq_and_processing/processing/processors/src/gpr_legacy_processor.ipp +++ b/data_acq_and_processing/processing/processors/src/gpr_legacy_processor.ipp @@ -853,8 +853,6 @@ void apply_legacy_motion_correction( const ProcessingLiveConfig& live_config ) -> ipc::ResultCollection { ipc::ResultCollection results{}; - results.collection_id = collection.collection_id; - results.monotonic_ns = collection.monotonic_ns; const auto selection = build_geometry_selection(run_config, live_config); if (selection.input_positions.empty() || selection.output_positions.empty()) { @@ -881,7 +879,7 @@ void apply_legacy_motion_correction( // Motion timing is computed once per collection. Matrix radars get one // event per Tx (parallel Rx); sequential radars get one event per pair. - const bool matrix_radar = is_matrix_radar_model(run_config.radar.model); + const bool matrix_radar = is_matrix_radar_model(run_config.radar_model); const auto motion_timing = compute_legacy_motion_timing( selected_traces, matrix_radar, diff --git a/data_acq_and_processing/processing/processors/src/passthrough_processor.cpp b/data_acq_and_processing/processing/processors/src/passthrough_processor.cpp index 068978c..99c7b42 100644 --- a/data_acq_and_processing/processing/processors/src/passthrough_processor.cpp +++ b/data_acq_and_processing/processing/processors/src/passthrough_processor.cpp @@ -10,11 +10,9 @@ auto PassThroughProcessor::process_collection( const config::RunConfig& /*run_config*/, const ipc::PreprocessedCollection& collection, std::span /*previous_collections*/, - const ProcessingLiveConfig& live_config + const ProcessingLiveConfig& /*live_config*/ ) -> ipc::ResultCollection { ipc::ResultCollection results{}; - results.collection_id = collection.collection_id; - results.monotonic_ns = collection.monotonic_ns; results.blocks.reserve(collection.traces.size()); for (const auto& trace : collection.traces) { @@ -22,11 +20,7 @@ auto PassThroughProcessor::process_collection( payload.processing_name = name(); payload.kind = ipc::ResultKind::TraceComplex; payload.frequency_hz = trace.frequency_hz; - if (live_config.pass_through_channel == "s11") { - payload.trace = trace.s11; - } else { - payload.trace = trace.s21; - } + payload.trace = trace.s21; ipc::ResultBlock block{}; block.combo = trace.combo; diff --git a/data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp b/data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp index 57af839..113ea6c 100644 --- a/data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp +++ b/data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp @@ -297,9 +297,8 @@ void check(nvrtcResult result, const char* what) { } } if (target == 0) { - throw std::runtime_error( - "GPU sm_" + std::to_string(device_arch) + " is older than every architecture this NVRTC supports" - ); + throw CudaUnavailable("CUDA GPU sm_" + std::to_string(device_arch) + + " is older than every architecture this NVRTC supports; rebuild with NVRTC 12"); } const bool native = target == device_arch; const std::string arch_option = @@ -387,7 +386,7 @@ void launch(CUfunction kernel, int items, Args... args) { } // namespace -struct CudaBackend { +struct Gpu::Impl { // Declared first, destroyed last: buffers and the module need the context. struct Context { CUdevice device = 0; @@ -395,10 +394,10 @@ struct CudaBackend { Context() { check(cuInit(0U), "init", true); check(cuDeviceGet(&device, 0), "device", true); - check(cuDevicePrimaryCtxRetain(&handle, device), "context"); + check(cuDevicePrimaryCtxRetain(&handle, device), "context", true); if (const CUresult result = cuCtxSetCurrent(handle); result != CUDA_SUCCESS) { cuDevicePrimaryCtxRelease(device); - check(result, "context"); + check(result, "context", true); } } ~Context() { cuDevicePrimaryCtxRelease(device); } @@ -430,8 +429,8 @@ struct CudaBackend { DeviceBuffer images; DeviceBuffer peak_stats; - ~CudaBackend() { cuCtxSetCurrent(context.handle); } - explicit CudaBackend(std::span weights) { + ~Impl() { cuCtxSetCurrent(context.handle); } + explicit Impl(std::span weights) { smooth_radius = static_cast(weights.size() / 2U); taps.upload(weights); } @@ -439,13 +438,13 @@ struct CudaBackend { auto project(const VolumeRequest&) -> Projections; }; -auto CudaBackend::inverse_dft( +auto Gpu::Impl::inverse_dft( std::span> bands, std::size_t points, std::size_t first_bin, std::size_t size ) -> std::vector> { - CudaBackend& gpu = *this; + Impl& gpu = *this; check(cuCtxSetCurrent(gpu.context.handle), "context"); const std::size_t samples = bands.size() / points * size; const int items = checked_int(samples, "A-scan stack"); @@ -456,8 +455,8 @@ auto CudaBackend::inverse_dft( return download>(output, samples); } -auto CudaBackend::project(const VolumeRequest& request) -> Projections { - CudaBackend& gpu = *this; +auto Gpu::Impl::project(const VolumeRequest& request) -> Projections { + Impl& gpu = *this; check(cuCtxSetCurrent(gpu.context.handle), "context"); const int nx = checked_int(request.x.size(), "grid"); @@ -524,17 +523,15 @@ auto CudaBackend::project(const VolumeRequest& request) -> Projections { return result; } +#else + +struct Gpu::Impl {}; + #endif -struct Engine::Impl { +Gpu::Gpu(double smooth_sigma_voxels) { #ifdef SAR3D_WITH_CUDA - std::unique_ptr cuda; -#endif -}; - -Engine::Engine(double smooth_sigma_voxels) : impl_(std::make_unique()) { -#ifdef SAR3D_WITH_CUDA - impl_->cuda = std::make_unique(gaussian_taps(smooth_sigma_voxels)); + impl_ = std::make_unique(gaussian_taps(smooth_sigma_voxels)); #else (void)smooth_sigma_voxels; throw CudaUnavailable("SAR3D requires an NVIDIA GPU with CUDA; this program was built without " @@ -542,23 +539,24 @@ Engine::Engine(double smooth_sigma_voxels) : impl_(std::make_unique()) { #endif } -Engine::~Engine() = default; +Gpu::~Gpu() = default; -auto Engine::inverse_dft(std::span> bands, std::size_t points, - std::size_t first_bin, std::size_t size) -> std::vector> { +// Without CUDA the constructor throws, so the #else branches below only satisfy the compiler. +auto Gpu::inverse_dft(std::span> bands, std::size_t points, + std::size_t first_bin, std::size_t size) -> std::vector> { if (points == 0U || bands.empty() || bands.size() % points != 0U || !std::has_single_bit(size) || size > kMaxIndexedItems || first_bin >= size || points > size - first_bin) { throw std::runtime_error("sar3d invalid inverse FFT dimensions"); } (void)checked_product(bands.size() / points, size, kMaxIndexedItems, "A-scan stack"); #ifdef SAR3D_WITH_CUDA - return impl_->cuda->inverse_dft(bands, points, first_bin, size); + return impl_->inverse_dft(bands, points, first_bin, size); #else throw CudaUnavailable("SAR3D requires an NVIDIA GPU with CUDA"); #endif } -auto Engine::project(const VolumeRequest& request) -> Projections { +auto Gpu::project(const VolumeRequest& request) -> Projections { const auto nx = request.x.size(), ny = request.y.size(), nz = request.z.size(); const auto plane = checked_product(nx, nz, kMaxIndexedItems, "grid plane"); (void)checked_product(plane, ny, kMaxIndexedItems, "grid (32-bit indexing; increase SAR_VOXEL_M)"); @@ -581,11 +579,6 @@ auto Engine::project(const VolumeRequest& request) -> Projections { } } } - for (const auto& sample : request.ascans) { - if (!std::isfinite(sample.real()) || !std::isfinite(sample.imag())) { - throw std::runtime_error("sar3d A-scan contains a non-finite sample"); - } - } for (const auto& term : request.contributions) { if (!std::isfinite(term.weight) || term.weight < 0.0F || !std::isfinite(term.inv_reference_ranges) || term.inv_reference_ranges <= 0.0F) { @@ -600,7 +593,7 @@ auto Engine::project(const VolumeRequest& request) -> Projections { (void)checked_product(static_cast(view.nu), static_cast(view.nv), kMaxIndexedItems, "oblique image"); #ifdef SAR3D_WITH_CUDA - return impl_->cuda->project(request); + return impl_->project(request); #else throw CudaUnavailable("SAR3D requires an NVIDIA GPU with CUDA"); #endif diff --git a/data_acq_and_processing/processing/processors/src/sar3d_gpu.hpp b/data_acq_and_processing/processing/processors/src/sar3d_gpu.hpp index 1ff4cb9..7236a34 100644 --- a/data_acq_and_processing/processing/processors/src/sar3d_gpu.hpp +++ b/data_acq_and_processing/processing/processors/src/sar3d_gpu.hpp @@ -8,7 +8,7 @@ #include #include -// Numerical backend of the sar3d processor: A-scans of the band spectra, and the +// CUDA backend of the sar3d processor: A-scans of the band spectra, and the // backprojection of an aperture into the 3D volume, smoothed and projected. // Only the projections and peak leave the backend. namespace radar::processing::sar3d { @@ -74,12 +74,12 @@ struct Projections { // CUDA context and runtime-compiled kernels. A missing device raises // CudaUnavailable; computation and compilation failures remain ordinary errors. -class Engine { +class Gpu { public: - explicit Engine(double smooth_sigma_voxels); - ~Engine(); - Engine(const Engine&) = delete; - auto operator=(const Engine&) -> Engine& = delete; + explicit Gpu(double smooth_sigma_voxels); + ~Gpu(); + Gpu(const Gpu&) = delete; + auto operator=(const Gpu&) -> Gpu& = delete; // numpy.fft.ifft of each band spectrum placed at bins [first_bin, first_bin + // points) of a `size`-point spectrum; `bands` holds the bands back to back. diff --git a/data_acq_and_processing/processing/processors/src/sar3d_processor.cpp b/data_acq_and_processing/processing/processors/src/sar3d_processor.cpp index af846fc..c26a6c3 100644 --- a/data_acq_and_processing/processing/processors/src/sar3d_processor.cpp +++ b/data_acq_and_processing/processing/processors/src/sar3d_processor.cpp @@ -41,10 +41,10 @@ constexpr float kCompRangeWeightMax = 5.0F; // COMP_RANGE_WEIGHT_MAX constexpr double kEdgeFrameWeight = 0.8; // SAR_EDGE_FRAME_WEIGHT constexpr double kSmoothSigma = 1.5; // BP_SMOOTH_SIGMA, voxels constexpr double kNormEdgeMargin = 0.3; // SAR_NORM_EDGE_MARGIN_M -// Scene window: X across, Y height above the ground (SAR_Y_RANGE_M). Z along the -// ground follows the depth gate. -constexpr double kSceneXMin = -2.0; -constexpr double kSceneXMax = 2.0; +// Scene window: X across spans the selected antennas plus a margin on each side +// (x_ant.min() - 2 ... x_ant.max() + 2), Y is the height above the ground +// (SAR_Y_RANGE_M), Z along the ground follows the depth gate. +constexpr double kSceneXMargin = 2.0; constexpr double kSceneYMin = -0.5; constexpr double kSceneYMax = 2.0; static_assert(kSceneYMax - kSceneYMin > 2.5 * kNormEdgeMargin, "else sar_norm_reference_mask keeps every plane"); @@ -169,8 +169,7 @@ struct TraceClock { }; struct CollectionTiming { - std::uint64_t center_ns = 0U; // integer centre avoids overflow and loss of relative timing - double center_fraction_ns = 0.0; + std::uint64_t center_ns = 0U; // centre of the collection's capture window std::vector clocks; // per collection trace }; @@ -202,7 +201,6 @@ struct CollectionTiming { CollectionTiming timing{}; const auto duration_ns = last - first; timing.center_ns = first + duration_ns / 2U; - timing.center_fraction_ns = 0.5 * static_cast(duration_ns % 2U); for (const auto& trace : collection.traces) { const auto& stages = sweep_outputs.at({trace.capture_start_ns, trace.capture_end_ns}); const auto stage = std::find(stages.begin(), stages.end(), trace.combo.output_pos) - stages.begin(); @@ -315,7 +313,6 @@ void normalize_ascan(std::span> ascan, double dt_s, const Se // A-scans of every selected Tx/Rx pair of one collection, (Tx, Rx) order. struct Frame { std::uint64_t center_ns = 0U; - double center_fraction_ns = 0.0; double dt_s = 0.0; std::size_t samples = 0U; // per A-scan std::vector> ascans{}; // [pair][sample] @@ -324,7 +321,7 @@ struct Frame { [[nodiscard]] auto compute_frame( const ipc::PreprocessedCollection& collection, const Settings& settings, - sar3d::Engine& backend + sar3d::Gpu& backend ) -> Frame { const auto timing = collection_timing(collection); std::vector bands{}; @@ -354,7 +351,6 @@ struct Frame { } Frame frame{}; frame.center_ns = timing.center_ns; - frame.center_fraction_ns = timing.center_fraction_ns; frame.samples = first.size; frame.dt_s = 1.0 / (static_cast(first.size) * first.df_hz); frame.ascans = backend.inverse_dft(spectra, first.spectrum.size(), first.first_bin, first.size); @@ -399,6 +395,15 @@ struct Grid { if (std::abs(cos_p) < 1e-6) { throw std::runtime_error("SAR grid is undefined at a radar look angle of 90 degrees"); } + // The pitch turns the radar about X, so antenna X is already the scene X. + double x_min = std::numeric_limits::infinity(); + double x_max = -std::numeric_limits::infinity(); + for (const auto* antennas : {&settings.tx, &settings.rx}) { + for (const auto& antenna : *antennas) { + x_min = std::min(x_min, antenna[0] - kSceneXMargin); + x_max = std::max(x_max, antenna[0] + kSceneXMargin); + } + } double z_min = std::numeric_limits::infinity(); double z_max = -std::numeric_limits::infinity(); for (const double height : {kSceneYMin, kSceneYMax}) { @@ -414,13 +419,13 @@ struct Grid { const auto axis_size = [&](double span) { return checked_axis_size(std::max(std::nearbyint(span / settings.voxel_m) + 1.0, 2.0)); }; - const auto nx = axis_size(kSceneXMax - kSceneXMin); + const auto nx = axis_size(x_max - x_min); const auto ny = axis_size(kSceneYMax - kSceneYMin); const auto nz = axis_size(z_max - z_min); if (nx > static_cast(std::numeric_limits::max()) / ny / nz) { throw std::runtime_error("SAR grid exceeds 32-bit kernel indexing; increase SAR_VOXEL_M"); } - return {linspace(kSceneXMin, kSceneXMax, nx), linspace(kSceneYMin, kSceneYMax, ny), + return {linspace(x_min, x_max, nx), linspace(kSceneYMin, kSceneYMax, ny), linspace(z_min, z_max, nz)}; } @@ -571,11 +576,11 @@ struct Oblique { } // namespace struct Sar3dProcessor::State { - // A sweep is its id with its timestamp: ids start again from 1 on every run, + // Acquisition identity is (id, capture start): ids restart on every run, // while the processor and this cache outlive runs. using SweepKey = std::pair; [[nodiscard]] static auto sweep_key(const ipc::PreprocessedCollection& collection) -> SweepKey { - return {collection.collection_id, collection.monotonic_ns}; + return {collection.collection_id, collection.capture_start_ns}; } struct CachedFrame { @@ -584,12 +589,12 @@ struct Sar3dProcessor::State { }; std::map frames{}; AscanKey frames_key{}; - std::unique_ptr engine{}; - std::exception_ptr engine_error{}; + std::unique_ptr device{}; + std::exception_ptr device_error{}; // why the GPU could not be opened; not retried std::string last_error{}; std::uint64_t repeated_errors = 0U; - auto frame(const ipc::PreprocessedCollection& collection, const Settings& settings, sar3d::Engine& backend) + auto frame(const ipc::PreprocessedCollection& collection, const Settings& settings, sar3d::Gpu& backend) -> std::shared_ptr { if (const auto found = frames.find(sweep_key(collection)); found != frames.end()) { if (found->second.value == nullptr) { report_error(found->second.error); } @@ -607,13 +612,13 @@ struct Sar3dProcessor::State { return frame; } - auto numerical_engine() -> sar3d::Engine& { - if (engine == nullptr && engine_error == nullptr) { - try { engine = std::make_unique(kSmoothSigma); } - catch (...) { engine_error = std::current_exception(); } + auto gpu() -> sar3d::Gpu& { + if (device == nullptr && device_error == nullptr) { + try { device = std::make_unique(kSmoothSigma); } + catch (...) { device_error = std::current_exception(); } } - if (engine == nullptr) { std::rethrow_exception(engine_error); } - return *engine; + if (device == nullptr) { std::rethrow_exception(device_error); } + return *device; } auto render( @@ -621,14 +626,12 @@ struct Sar3dProcessor::State { std::span previous, const Settings& settings ) -> std::vector { - auto& backend = numerical_engine(); + auto& backend = gpu(); if (settings.ascan_key() != frames_key) { frames.clear(); frames_key = settings.ascan_key(); } - const auto history_count = settings.frame_radius >= previous.size() / 2U + previous.size() % 2U - ? previous.size() : 2U * settings.frame_radius; - const auto recent = previous.last(history_count); + const auto recent = previous.last(std::min(previous.size(), 2U * settings.frame_radius)); std::set retained{sweep_key(collection)}; for (const auto& member : recent) { retained.insert(sweep_key(member)); } // Prune even when the newest frame is invalid, so a bad input stream @@ -657,10 +660,9 @@ struct Sar3dProcessor::State { std::vector offsets_s{}; for (const auto& member : aperture) { - const double delta_ns = member->center_ns >= centre.center_ns - ? static_cast(member->center_ns - centre.center_ns) - : -static_cast(centre.center_ns - member->center_ns); - offsets_s.push_back(1e-9 * (delta_ns + member->center_fraction_ns - centre.center_fraction_ns)); + // Unsigned difference reinterpreted as signed: exact for any two timestamps. + const auto delta_ns = static_cast(member->center_ns - centre.center_ns); + offsets_s.push_back(1e-9 * static_cast(delta_ns)); } const auto weights = frame_weights(offsets_s); const Grid grid = scene_grid(settings); @@ -797,11 +799,18 @@ auto Sar3dProcessor::process_collection( const ProcessingLiveConfig& live_config ) -> ipc::ResultCollection { ipc::ResultCollection results{}; - results.collection_id = collection.collection_id; - results.monotonic_ns = collection.monotonic_ns; try { results.collection_payloads = state_->render(collection, previous_collections, read_settings(run_config, live_config)); + // RSL3 includes the typed metadata in its fixed header. + const auto bytes = ipc::serialize_result_collection(results).size(); + const std::size_t slot = run_config.rings.results.slot_size_bytes; + if (bytes > slot) { + throw std::runtime_error("the views need " + std::to_string(bytes / 1024U) + + " KiB but a results ring slot holds " + std::to_string(slot / 1024U) + + " KiB; increase SAR_VOXEL_M, narrow the depth range or raise " + "rings.results.slot_size_bytes"); + } } catch (const sar3d::CudaUnavailable& error) { state_->report_error(error.what()); results.collection_payloads = {status_payload(3.0F, error.what())}; diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/interfaces/radar_driver.hpp b/data_acq_and_processing/sweep_orchestrator/device_drivers/interfaces/radar_driver.hpp deleted file mode 100644 index ccc7288..0000000 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/interfaces/radar_driver.hpp +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once - -#include - -#include "shared_types.hpp" - -namespace radar::drivers { - -/** - * @brief One forward sweep acquired from the radar. - * - * All vectors must have equal size and aligned indices. - */ -struct SweepTrace { - std::vector frequency_hz{}; - std::vector s11{}; - std::vector s21{}; -}; - -/** - * @brief Minimal radar interface used by the sweep orchestrator. - * - * Implementations are expected to be lightweight: configuration is handled by - * the Python layer, while this interface only opens/closes and acquires one - * forward sweep. - */ -class RadarDriver { - public: - virtual ~RadarDriver() = default; - - /** @brief Open underlying transport and prepare acquisition. */ - virtual void open() = 0; - /** @brief Release all allocated resources. */ - virtual void close() = 0; - /** @brief Acquire one sweep containing the forward traces exposed by the driver. */ - [[nodiscard]] virtual auto acquire_sweep() -> SweepTrace = 0; - -}; - -} // namespace radar::drivers diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/interfaces/switch_driver.hpp b/data_acq_and_processing/sweep_orchestrator/device_drivers/interfaces/switch_driver.hpp deleted file mode 100644 index e99ce8e..0000000 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/interfaces/switch_driver.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include - -namespace radar::drivers { - -/** - * @brief Unified RF switch interface consumed by the orchestrator. - */ -class SwitchDriver { - public: - virtual ~SwitchDriver() = default; - - /** @brief Open underlying transport and set initial switch state. */ - virtual void open() = 0; - /** @brief Release driver resources. */ - virtual void close() = 0; - /** @brief Number of selectable positions exposed by this switch. */ - [[nodiscard]] virtual auto position_count() const -> std::uint32_t = 0; - /** @brief Switch to a 0-based position. */ - virtual void switch_to(std::uint32_t position) = 0; -}; - -} // namespace radar::drivers diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_lifecycle.cpp b/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_lifecycle.cpp deleted file mode 100644 index c9bb66d..0000000 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_lifecycle.cpp +++ /dev/null @@ -1,241 +0,0 @@ -#include "../librevna_minimal_driver.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "librevna_protocol_common.hpp" - -namespace radar::drivers { -namespace detail = radar::drivers::librevna::detail; -namespace { - -constexpr std::uint32_t kNativeAcquireMaxAttempts = 3U; - -// Derive the budget the whole multi-point sweep is allowed to take before the -// first datapoint must arrive (#7): a fixed setup cost plus the expected dwell -// (points / IFBW) with a generous margin, clamped to the overall hard cap. The -// per-gap stall timeout then governs progress once datapoints start flowing. -[[nodiscard]] auto native_initial_sweep_budget(const config::RadarSweepSettings& sweep) - -> std::chrono::milliseconds { - const float points = std::max(1.0F, static_cast(sweep.points)); - const float if_bw = std::max(1.0F, sweep.if_bandwidth_hz); - const double dwell_ms = - (static_cast(points) / static_cast(if_bw)) * 1000.0 * - static_cast(detail::kNativeSweepDwellMargin); - const double total_ms = static_cast(detail::kNativeSweepSetupBudgetMs) + dwell_ms; - const auto budget = std::chrono::milliseconds(static_cast(total_ms)); - return std::clamp( - budget, - std::chrono::milliseconds(detail::kNativeSweepPerGapTimeoutMs), - std::chrono::milliseconds(detail::kNativeSweepHardCapMs) - ); -} - -[[nodiscard]] auto is_retryable_native_acquire_error(std::string_view message) -> bool { - // A stop request must not be retried: drain to the caller immediately (#26). - if (message.find("aborted by stop request") != std::string_view::npos) { - return false; - } - - constexpr std::array kRetryableSubstrings = { - "Timeout waiting for expected LibreVNA packet type", - "Timeout waiting for LibreVNA ACK", - "LibreVNA returned NACK", - "Failed to read LibreVNA USB bulk packet", - "Failed to write LibreVNA USB bulk packet", - // #31: a transient USB error often triggers a brief device re-enumeration, - // so the device may be momentarily absent or its handle invalidated when - // we reconnect. Treat these as retryable; the reconnect path absorbs the - // re-enumeration with a short bounded discovery backoff. - "No compatible LibreVNA USB device found", - "LibreVNA native handle is not open", - "LibreVNA USB handle is not open", - }; - - return std::any_of( - kRetryableSubstrings.begin(), - kRetryableSubstrings.end(), - [message](std::string_view needle) { - return message.find(needle) != std::string_view::npos; - } - ); -} - -} // namespace - -LibreVnaMinimalDriver::LibreVnaMinimalDriver(LibreVnaMinimalDriverSettings settings) : settings_(std::move(settings)) {} - -void LibreVnaMinimalDriver::open() { - if (is_open_) { - return; - } - if (settings_.sweep.points == 0U) { - throw std::runtime_error("Radar sweep points must be > 0"); - } - - open_native(); - - is_open_ = true; -} - -void LibreVnaMinimalDriver::close() { - if (!is_open_) { - return; - } - - close_native(); - - is_open_ = false; -} - -auto LibreVnaMinimalDriver::acquire_sweep() -> SweepTrace { - if (!is_open_) { - throw std::runtime_error("Radar driver is not open"); - } - - std::exception_ptr last_exception{}; - std::string last_message{}; - - for (std::uint32_t attempt = 1; attempt <= kNativeAcquireMaxAttempts; ++attempt) { - try { - return acquire_native(); - } catch (const std::exception& exception) { - last_exception = std::current_exception(); - last_message = exception.what(); - - if (!is_retryable_native_acquire_error(last_message) || attempt == kNativeAcquireMaxAttempts) { - break; - } - if (detail::native_stop_requested()) { - break; // #26: do not waste reconnect attempts while shutting down. - } - - // #31: recover from transient USB/protocol stalls without - // destroying the libusb_context. Release only the interface - // and handle, then reopen against the surviving context so - // we ride out a brief device re-enumeration instead of - // re-initialising libusb from scratch on every retry. - rx_buffer_.clear(); - packet_queue_.clear(); - if (usb_handle_ != nullptr && interface_claimed_) { - libusb_release_interface(usb_handle_, detail::kUsbInterface); - interface_claimed_ = false; - } - if (usb_handle_ != nullptr) { - libusb_close(usb_handle_); - usb_handle_ = nullptr; - } - protocol_version_ = 0; - device_num_ports_ = 0; - open_native(); // Reuses the surviving usb_context_ and retries discovery. - } - } - - if (last_exception != nullptr) { - std::rethrow_exception(last_exception); - } - throw std::runtime_error("Native acquisition failed without exception detail"); -} - -auto LibreVnaMinimalDriver::acquire_native() -> SweepTrace { - if (usb_handle_ == nullptr) { - throw std::runtime_error("LibreVNA native handle is not open"); - } - - // Drop stale datapoints left in queue by previous operations. - while (pop_packet(detail::kPacketVnaDatapoint).has_value()) { - } - - send_packet_no_payload(detail::kPacketInitiateSweep, true); - - SweepTrace trace{}; - trace.frequency_hz.assign(settings_.sweep.points, 0.0F); - trace.s11.assign(settings_.sweep.points, ipc::Complex32{}); - trace.s21.assign(settings_.sweep.points, ipc::Complex32{}); - - std::vector received(settings_.sweep.points, 0U); - std::uint32_t received_count = 0; - - // #7: the per-datapoint deadline is rebased every time a fresh point lands - // (or the first point must arrive within the size-derived budget), so a - // large sweep is not killed by one fixed ~1.5s timeout. An independent hard - // cap bounds the whole sweep against a device that streams forever. - const auto now = std::chrono::steady_clock::now(); - const auto hard_cap_deadline = now + std::chrono::milliseconds(detail::kNativeSweepHardCapMs); - auto gap_deadline = std::min(now + native_initial_sweep_budget(settings_.sweep), hard_cap_deadline); - - while (received_count < settings_.sweep.points) { - // #26: honour a stop request observed during the (chunked) USB wait so - // SIGTERM aborts the sweep promptly instead of after the full budget. - if (detail::native_stop_requested()) { - throw std::runtime_error("Native sweep aborted by stop request"); - } - if (std::chrono::steady_clock::now() >= hard_cap_deadline) { - throw std::runtime_error( - "Native sweep exceeded hard cap (" + std::to_string(received_count) + "/" + - std::to_string(settings_.sweep.points) + " points received)" - ); - } - - const auto wait_deadline = std::min(gap_deadline, hard_cap_deadline); - - NativePacket packet{}; - try { - packet = wait_for_packet(detail::kPacketVnaDatapoint, wait_deadline); - } catch (const std::exception& exception) { - throw std::runtime_error( - "Timeout while collecting VNADatapoints (" + std::to_string(received_count) + "/" + - std::to_string(settings_.sweep.points) + " points received): " + exception.what() - ); - } - - DecodedVnaDatapoint datapoint{}; - if (!decode_vna_datapoint_traces(packet.payload, datapoint)) { - throw std::runtime_error("Failed to decode traces from VNADatapoint packet"); - } - if (datapoint.point_number >= settings_.sweep.points) { - throw std::runtime_error("Received out-of-range VNADatapoint index"); - } - - if (received[datapoint.point_number] == 0U) { - received[datapoint.point_number] = 1U; - ++received_count; - } - - trace.frequency_hz[datapoint.point_number] = datapoint.frequency_hz; - trace.s11[datapoint.point_number] = datapoint.s11; - trace.s21[datapoint.point_number] = datapoint.s21; - - // Extend the per-gap stall timeout now that progress was made. - gap_deadline = std::chrono::steady_clock::now() + - std::chrono::milliseconds(detail::kNativeSweepPerGapTimeoutMs); - } - - return trace; -} - -void LibreVnaMinimalDriver::validate_device_info_payload(std::span payload) const { - if (payload.size() != detail::kDeviceInfoPayloadSize) { - throw std::runtime_error("Unexpected DeviceInfo payload size"); - } - - const auto protocol = detail::read_u16_le(payload, detail::kDeviceInfoProtocolOffset); - if (protocol != detail::kProtocolV14) { - throw std::runtime_error("Unsupported LibreVNA protocol version (expected v14)"); - } - - const auto ports = payload[detail::kDeviceInfoNumPortsOffset]; - if (ports < 2U) { - throw std::runtime_error("LibreVNA reported invalid number of ports"); - } -} - -} // namespace radar::drivers diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_protocol.cpp b/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_protocol.cpp deleted file mode 100644 index a23d62a..0000000 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_protocol.cpp +++ /dev/null @@ -1,148 +0,0 @@ -#include "../librevna_minimal_driver.hpp" - -#include -#include -#include -#include -#include - -#include "librevna_protocol_common.hpp" - -namespace radar::drivers { -namespace detail = radar::drivers::librevna::detail; - -auto LibreVnaMinimalDriver::encode_frame( - std::uint8_t packet_type, - std::span payload -) -> std::vector { - const auto length = payload.size() + detail::kFrameOverheadBytes; - if (length > 0xFFFFU) { - throw std::runtime_error("Protocol frame is too large"); - } - - std::vector frame(length, 0); - frame[0] = detail::kFrameHeader; - detail::write_u16_le(frame, 1, static_cast(length)); - frame[3] = packet_type; - std::copy(payload.begin(), payload.end(), frame.begin() + 4); - - std::uint32_t crc = 0; - if (packet_type != detail::kPacketVnaDatapoint) { - crc = crc32(std::span(frame.data(), frame.size() - 4U)); - } - detail::write_u32_le(frame, frame.size() - 4U, crc); - - return frame; -} - -auto LibreVnaMinimalDriver::decode_vna_datapoint_traces( - std::span payload, - DecodedVnaDatapoint& datapoint_out -) -> bool { - // VNADatapoint payload layout: - // [0..7]=freq_or_time, [8..9]=cdbm, [10..11]=point_number, - // followed by N tuples of {real:f32, imag:f32, flags:u8}. - if (payload.size() < 12U) { - return false; - } - - const auto values_block = payload.size() - 12U; - if ((values_block % 9U) != 0U) { - return false; - } - - const auto num_values = values_block / 9U; - if (num_values == 0U) { - return false; - } - - datapoint_out = DecodedVnaDatapoint{}; - datapoint_out.point_number = detail::read_u16_le(payload, 10); - datapoint_out.frequency_hz = static_cast(detail::read_u64_le(payload, 0)); - - const auto real_offset = 12U; - const auto imag_offset = real_offset + (4U * num_values); - const auto flags_offset = imag_offset + (4U * num_values); - - std::array, 8> ref_by_stage{}; - std::array, 8> s11_measured_by_stage{}; - std::array, 8> s21_measured_by_stage{}; - std::array has_ref{}; - std::array has_s11_measured{}; - std::array has_s21_measured{}; - constexpr float kReferenceMagnitudeSquaredEpsilon = 1e-12F; - - for (std::size_t index = 0; index < num_values; ++index) { - const auto flags = payload[flags_offset + index]; - const auto stage = static_cast(flags >> 5U); - if (stage >= ref_by_stage.size()) { - continue; - } - - const auto value = std::complex( - detail::read_f32_le(payload, real_offset + (4U * index)), - detail::read_f32_le(payload, imag_offset + (4U * index)) - ); - - const bool is_reference = (flags & detail::kReferenceFlagMask) != 0U; - if ((flags & detail::kPort1Mask) != 0U && is_reference) { - ref_by_stage[stage] = value; - has_ref[stage] = true; - } - if ((flags & detail::kPort1Mask) != 0U && !is_reference) { - s11_measured_by_stage[stage] = value; - has_s11_measured[stage] = true; - } - if ((flags & detail::kPort2Mask) != 0U && !is_reference) { - s21_measured_by_stage[stage] = value; - has_s21_measured[stage] = true; - } - } - - // For the forward sweep used in this project, port1 reference is the - // incident signal, port1 measured is reflection (S11 numerator), and - // port2 measured is transmission (S21 numerator). - for (std::size_t stage = 0; stage < ref_by_stage.size(); ++stage) { - if (!has_ref[stage]) { - continue; - } - if (std::norm(ref_by_stage[stage]) <= kReferenceMagnitudeSquaredEpsilon) { - continue; - } - - if (has_s11_measured[stage]) { - const auto s11_ratio = s11_measured_by_stage[stage] / ref_by_stage[stage]; - datapoint_out.s11.re = s11_ratio.real(); - datapoint_out.s11.im = s11_ratio.imag(); - datapoint_out.has_s11 = true; - } - if (has_s21_measured[stage]) { - const auto s21_ratio = s21_measured_by_stage[stage] / ref_by_stage[stage]; - datapoint_out.s21.re = s21_ratio.real(); - datapoint_out.s21.im = s21_ratio.imag(); - datapoint_out.has_s21 = true; - } - if (datapoint_out.has_s21) { - return true; - } - } - - return false; -} - -auto LibreVnaMinimalDriver::crc32(std::span data) -> std::uint32_t { - std::uint32_t crc = 0xFFFFFFFFU; - for (const auto byte : data) { - crc ^= static_cast(byte); - for (int bit = 0; bit < 8; ++bit) { - const auto lsb = crc & 1U; - crc >>= 1U; - if (lsb != 0U) { - crc ^= 0xEDB88320U; - } - } - } - return ~crc; -} - -} // namespace radar::drivers diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_transport.cpp b/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_transport.cpp deleted file mode 100644 index cdb3d53..0000000 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_transport.cpp +++ /dev/null @@ -1,394 +0,0 @@ -#include "../librevna_minimal_driver.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "librevna_protocol_common.hpp" - -namespace radar::drivers { -namespace detail = radar::drivers::librevna::detail; -namespace { - -[[nodiscard]] auto libusb_error_message(const std::string& prefix, int status_code) -> std::string { - return prefix + ": " + libusb_error_name(status_code); -} - -[[nodiscard]] auto is_supported_vid_pid(std::uint16_t vendor_id, std::uint16_t product_id) -> bool { - return std::any_of( - detail::kSupportedUsbIds.begin(), - detail::kSupportedUsbIds.end(), - [vendor_id, product_id](const auto& pair) { - return pair.first == vendor_id && pair.second == product_id; - } - ); -} - -[[nodiscard]] auto parse_serial_string(libusb_device_handle* handle, std::uint8_t serial_index) -> std::string { - if (handle == nullptr || serial_index == 0U) { - return {}; - } - - std::array serial_bytes{}; - const auto length = libusb_get_string_descriptor_ascii( - handle, - serial_index, - serial_bytes.data(), - static_cast(serial_bytes.size()) - ); - if (length <= 0) { - return {}; - } - - return std::string( - reinterpret_cast(serial_bytes.data()), - reinterpret_cast(serial_bytes.data() + length) - ); -} - -[[nodiscard]] auto find_matching_device_handle( - libusb_context* usb_context, - const std::string& expected_serial -) -> libusb_device_handle* { - libusb_device** devices = nullptr; - const auto device_count = libusb_get_device_list(usb_context, &devices); - if (device_count < 0) { - throw std::runtime_error( - libusb_error_message("Failed to enumerate USB devices", static_cast(device_count)) - ); - } - - libusb_device_handle* selected_handle = nullptr; - for (ssize_t index = 0; index < device_count; ++index) { - auto* device = devices[index]; - if (device == nullptr) { - continue; - } - - libusb_device_descriptor descriptor{}; - const auto descriptor_status = libusb_get_device_descriptor(device, &descriptor); - if (descriptor_status != LIBUSB_SUCCESS) { - continue; - } - if (!is_supported_vid_pid(descriptor.idVendor, descriptor.idProduct)) { - continue; - } - - libusb_device_handle* candidate_handle = nullptr; - if (libusb_open(device, &candidate_handle) != LIBUSB_SUCCESS || candidate_handle == nullptr) { - continue; - } - - if (!expected_serial.empty()) { - const auto serial = parse_serial_string(candidate_handle, descriptor.iSerialNumber); - if (serial != expected_serial) { - libusb_close(candidate_handle); - continue; - } - } - - selected_handle = candidate_handle; - break; - } - - libusb_free_device_list(devices, 1); - return selected_handle; -} - -} // namespace - -void LibreVnaMinimalDriver::open_native() { - if (usb_handle_ != nullptr) { - throw std::runtime_error("LibreVNA native state is already initialized"); - } - - // #26: observe SIGINT/SIGTERM (chaining to main()'s handler) so the blocking - // USB loops can bail out promptly during shutdown. Safe to call repeatedly. - detail::install_native_stop_observer(); - - // #31: keep the libusb_context alive across reconnect retries. Only create a - // fresh context the first time; reconnect paths reuse the surviving context - // and merely reopen the handle, avoiding a full libusb teardown/re-init. - if (usb_context_ == nullptr) { - const auto init_status = libusb_init(&usb_context_); - if (init_status != LIBUSB_SUCCESS) { - usb_context_ = nullptr; - throw std::runtime_error(libusb_error_message("Failed to initialize libusb", init_status)); - } - } - - try { - // #31: retry discovery with a short bounded backoff so a brief device - // re-enumeration (common right after a transient USB error) is absorbed - // instead of surfacing as a hard "device not found". - libusb_device_handle* selected_handle = nullptr; - for (int attempt = 0; attempt < detail::kNativeDiscoveryRetryAttempts; ++attempt) { - if (detail::native_stop_requested()) { - break; // #26: abandon discovery promptly during shutdown. - } - selected_handle = find_matching_device_handle(usb_context_, settings_.serial); - if (selected_handle != nullptr) { - break; - } - std::this_thread::sleep_for(std::chrono::milliseconds(detail::kNativeDiscoveryRetryDelayMs)); - } - - if (selected_handle == nullptr) { - const auto serial_hint = settings_.serial.empty() ? std::string() : " for serial '" + settings_.serial + "'"; - throw std::runtime_error("No compatible LibreVNA USB device found" + serial_hint); - } - - usb_handle_ = selected_handle; - - const auto auto_detach_status = libusb_set_auto_detach_kernel_driver(usb_handle_, 1); - if (auto_detach_status != LIBUSB_SUCCESS && auto_detach_status != LIBUSB_ERROR_NOT_SUPPORTED) { - throw std::runtime_error( - libusb_error_message("Failed to configure USB auto-detach kernel driver", auto_detach_status) - ); - } - - const auto claim_status = libusb_claim_interface(usb_handle_, detail::kUsbInterface); - if (claim_status != LIBUSB_SUCCESS) { - throw std::runtime_error(libusb_error_message("Failed to claim LibreVNA USB interface", claim_status)); - } - interface_claimed_ = true; - - rx_buffer_.clear(); - packet_queue_.clear(); - - send_packet_no_payload(detail::kPacketRequestDeviceInfo, true); - const auto info_packet = wait_for_packet( - detail::kPacketDeviceInfo, - std::chrono::steady_clock::now() + std::chrono::seconds(detail::kDeviceInfoTimeoutSeconds) - ); - validate_device_info_payload(info_packet.payload); - protocol_version_ = detail::read_u16_le(info_packet.payload, detail::kDeviceInfoProtocolOffset); - device_num_ports_ = info_packet.payload[detail::kDeviceInfoNumPortsOffset]; - } catch (...) { - close_native(); - throw; - } -} - -void LibreVnaMinimalDriver::close_native() { - rx_buffer_.clear(); - packet_queue_.clear(); - - if (usb_handle_ != nullptr && interface_claimed_) { - libusb_release_interface(usb_handle_, detail::kUsbInterface); - interface_claimed_ = false; - } - - if (usb_handle_ != nullptr) { - libusb_close(usb_handle_); - usb_handle_ = nullptr; - } - - if (usb_context_ != nullptr) { - libusb_exit(usb_context_); - usb_context_ = nullptr; - } - - protocol_version_ = 0; - device_num_ports_ = 0; -} - -void LibreVnaMinimalDriver::send_packet( - std::uint8_t packet_type, - std::span payload, - bool require_ack -) { - if (usb_handle_ == nullptr) { - throw std::runtime_error("Cannot send packet: LibreVNA USB handle is not open"); - } - - auto frame = encode_frame(packet_type, payload); - - int transferred = 0; - const auto transfer_status = libusb_bulk_transfer( - usb_handle_, - detail::kEndpointOut, - reinterpret_cast(frame.data()), - static_cast(frame.size()), - &transferred, - detail::kUsbWriteTimeoutMs - ); - if (transfer_status != LIBUSB_SUCCESS) { - throw std::runtime_error(libusb_error_message("Failed to write LibreVNA USB bulk packet", transfer_status)); - } - if (transferred != static_cast(frame.size())) { - throw std::runtime_error("Incomplete LibreVNA USB bulk write"); - } - - if (require_ack) { - wait_for_ack(std::chrono::steady_clock::now() + std::chrono::milliseconds(detail::kAckTimeoutMs)); - } -} - -void LibreVnaMinimalDriver::send_packet_no_payload(std::uint8_t packet_type, bool require_ack) { - send_packet(packet_type, std::span{}, require_ack); -} - -void LibreVnaMinimalDriver::wait_for_ack(std::chrono::steady_clock::time_point deadline) { - while (std::chrono::steady_clock::now() < deadline) { - // #26: bail out promptly on SIGTERM/SIGINT rather than blocking until the - // ACK deadline expires. - if (detail::native_stop_requested()) { - throw std::runtime_error("LibreVNA ACK wait aborted by stop request"); - } - - std::optional ack_or_nack{}; - for (auto iter = packet_queue_.begin(); iter != packet_queue_.end(); ++iter) { - if (iter->packet_type != detail::kPacketAck && iter->packet_type != detail::kPacketNack) { - continue; - } - ack_or_nack = std::move(*iter); - packet_queue_.erase(iter); - break; - } - - if (ack_or_nack.has_value()) { - if (ack_or_nack->packet_type == detail::kPacketAck) { - return; - } - throw std::runtime_error("LibreVNA returned NACK"); - } - - pump_usb(deadline); - } - - throw std::runtime_error("Timeout waiting for LibreVNA ACK"); -} - -auto LibreVnaMinimalDriver::wait_for_packet( - std::uint8_t expected_type, - std::chrono::steady_clock::time_point deadline -) -> NativePacket { - while (std::chrono::steady_clock::now() < deadline) { - if (auto packet = pop_packet(expected_type); packet.has_value()) { - return *packet; - } - // #26: re-check the stop flag between short USB polls so a shutdown is - // honoured within a few hundred ms even on a long sweep deadline. - if (detail::native_stop_requested()) { - throw std::runtime_error("LibreVNA packet wait aborted by stop request"); - } - pump_usb(deadline); - } - - throw std::runtime_error("Timeout waiting for expected LibreVNA packet type"); -} - -void LibreVnaMinimalDriver::pump_usb(std::chrono::steady_clock::time_point deadline) { - if (usb_handle_ == nullptr) { - throw std::runtime_error("Cannot read packets: LibreVNA USB handle is not open"); - } - - const auto now = std::chrono::steady_clock::now(); - if (now >= deadline) { - return; - } - - const auto remaining_ms = std::chrono::duration_cast(deadline - now).count(); - const auto timeout_ms = static_cast( - std::clamp(remaining_ms, detail::kUsbReadPollMinTimeoutMs, detail::kUsbReadPollMaxTimeoutMs) - ); - - std::array buffer{}; - int transferred = 0; - const auto transfer_status = libusb_bulk_transfer( - usb_handle_, - detail::kEndpointIn, - buffer.data(), - static_cast(buffer.size()), - &transferred, - timeout_ms - ); - - if (transfer_status == LIBUSB_ERROR_TIMEOUT) { - return; - } - if (transfer_status != LIBUSB_SUCCESS) { - throw std::runtime_error(libusb_error_message("Failed to read LibreVNA USB bulk packet", transfer_status)); - } - if (transferred <= 0) { - return; - } - - rx_buffer_.insert(rx_buffer_.end(), buffer.begin(), buffer.begin() + transferred); - decode_frames_from_buffer(); -} - -void LibreVnaMinimalDriver::decode_frames_from_buffer() { - while (true) { - const auto header_iter = std::find(rx_buffer_.begin(), rx_buffer_.end(), detail::kFrameHeader); - if (header_iter == rx_buffer_.end()) { - rx_buffer_.clear(); - return; - } - - if (header_iter != rx_buffer_.begin()) { - rx_buffer_.erase(rx_buffer_.begin(), header_iter); - } - - if (rx_buffer_.size() < 4U) { - return; - } - - const auto declared_length = detail::read_u16_le(rx_buffer_, 1); - if (declared_length < detail::kFrameOverheadBytes || declared_length > detail::kMaxFrameLengthBytes) { - rx_buffer_.erase(rx_buffer_.begin()); - continue; - } - - if (rx_buffer_.size() < declared_length) { - return; - } - - const std::vector frame(rx_buffer_.begin(), rx_buffer_.begin() + declared_length); - rx_buffer_.erase(rx_buffer_.begin(), rx_buffer_.begin() + declared_length); - - const auto packet_type = frame[3]; - const auto received_crc = detail::read_u32_le(frame, frame.size() - 4U); - - if (packet_type == detail::kPacketVnaDatapoint) { - if (received_crc != 0U) { - throw std::runtime_error("Invalid VNADatapoint CRC (expected zero)"); - } - } else { - const auto expected_crc = crc32(std::span(frame.data(), frame.size() - 4U)); - if (received_crc != expected_crc) { - throw std::runtime_error("Invalid LibreVNA packet CRC"); - } - } - - NativePacket packet{}; - packet.packet_type = packet_type; - packet.payload.assign(frame.begin() + 4, frame.end() - 4); - packet_queue_.push_back(std::move(packet)); - } -} - -auto LibreVnaMinimalDriver::pop_packet(std::uint8_t packet_type) -> std::optional { - for (auto iter = packet_queue_.begin(); iter != packet_queue_.end(); ++iter) { - if (iter->packet_type != packet_type) { - continue; - } - - auto packet = std::move(*iter); - packet_queue_.erase(iter); - return packet; - } - - return std::nullopt; -} - -} // namespace radar::drivers diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_protocol_common.hpp b/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_protocol_common.hpp deleted file mode 100644 index 30ce612..0000000 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_protocol_common.hpp +++ /dev/null @@ -1,197 +0,0 @@ -#pragma once - -#include -#include -#include // POSIX sigaction/NSIG for the chaining stop observer (#26). -#include -#include -#include -#include -#include -#include - -namespace radar::drivers::librevna::detail { - -constexpr float kPi = 3.14159265358979323846F; - -// Frame format: -// [0]=header, [1..2]=length_le, [3]=packet_type, [...payload...], [crc32_le] -constexpr std::uint8_t kFrameHeader = 0x5A; -constexpr std::size_t kFrameOverheadBytes = 8; -constexpr std::size_t kMaxFrameLengthBytes = 4096; - -constexpr unsigned char kEndpointOut = 0x01; -constexpr unsigned char kEndpointIn = 0x81; -constexpr int kUsbInterface = 0; - -constexpr std::uint8_t kPacketDeviceInfo = 5; -constexpr std::uint8_t kPacketAck = 7; -constexpr std::uint8_t kPacketNack = 10; -constexpr std::uint8_t kPacketRequestDeviceInfo = 15; -constexpr std::uint8_t kPacketVnaDatapoint = 27; -constexpr std::uint8_t kPacketInitiateSweep = 32; - -constexpr std::size_t kDeviceInfoPayloadSize = 57; -constexpr std::size_t kDeviceInfoProtocolOffset = 0; -constexpr std::size_t kDeviceInfoNumPortsOffset = 54; -constexpr std::uint16_t kProtocolV14 = 14; - -constexpr std::uint8_t kReferenceFlagMask = 0x10; -constexpr std::uint8_t kPort1Mask = 0x01; -constexpr std::uint8_t kPort2Mask = 0x02; - -constexpr std::array, 3> kSupportedUsbIds = { - std::pair{0x0483, 0x564E}, - std::pair{0x0483, 0x4121}, - std::pair{0x1209, 0x4121}, -}; - -constexpr int kUsbWriteTimeoutMs = 500; -constexpr int kAckTimeoutMs = 800; -constexpr int kDeviceInfoTimeoutSeconds = 2; -constexpr std::size_t kUsbReadChunkBytes = 16 * 1024; -constexpr int kUsbReadPollMinTimeoutMs = 1; -constexpr int kUsbReadPollMaxTimeoutMs = 50; - -// Native sweep timing budget (#7). The native deadline is no longer a single -// fixed value for the whole sweep: instead it is derived from the sweep size -// (fixed setup cost + per-point dwell ≈ points/IFBW) and is extended every -// time a fresh datapoint arrives via a per-gap stall timeout, while an overall -// hard cap bounds a wedged device. -constexpr int kNativeSweepSetupBudgetMs = 500; -constexpr float kNativeSweepDwellMargin = 3.0F; -constexpr int kNativeSweepPerGapTimeoutMs = 1500; -constexpr int kNativeSweepHardCapMs = 60'000; - -// Reconnect backoff (#31). After a transient error the device may re-enumerate; -// retry discovery a few times with a short bounded delay before giving up. -constexpr int kNativeDiscoveryRetryAttempts = 10; -constexpr int kNativeDiscoveryRetryDelayMs = 150; - -[[nodiscard]] inline auto read_u16_le(std::span data, std::size_t offset) -> std::uint16_t { - if ((offset + sizeof(std::uint16_t)) > data.size()) { - throw std::runtime_error("Failed to decode uint16 from payload"); - } - return static_cast(data[offset]) | - (static_cast(data[offset + 1]) << 8U); -} - -[[nodiscard]] inline auto read_u32_le(std::span data, std::size_t offset) -> std::uint32_t { - if ((offset + sizeof(std::uint32_t)) > data.size()) { - throw std::runtime_error("Failed to decode uint32 from payload"); - } - return static_cast(data[offset]) | - (static_cast(data[offset + 1]) << 8U) | - (static_cast(data[offset + 2]) << 16U) | - (static_cast(data[offset + 3]) << 24U); -} - -[[nodiscard]] inline auto read_u64_le(std::span data, std::size_t offset) -> std::uint64_t { - if ((offset + sizeof(std::uint64_t)) > data.size()) { - throw std::runtime_error("Failed to decode uint64 from payload"); - } - - std::uint64_t value = 0; - for (std::size_t index = 0; index < sizeof(std::uint64_t); ++index) { - value |= static_cast(data[offset + index]) << (index * 8U); - } - return value; -} - -[[nodiscard]] inline auto read_f32_le(std::span data, std::size_t offset) -> float { - const auto raw = read_u32_le(data, offset); - float out = 0.0F; - std::memcpy(&out, &raw, sizeof(float)); - return out; -} - -inline void write_u16_le(std::span data, std::size_t offset, std::uint16_t value) { - if ((offset + sizeof(std::uint16_t)) > data.size()) { - throw std::runtime_error("Failed to encode uint16 into frame"); - } - - data[offset] = static_cast(value & 0xFFU); - data[offset + 1] = static_cast((value >> 8U) & 0xFFU); -} - -inline void write_u32_le(std::span data, std::size_t offset, std::uint32_t value) { - if ((offset + sizeof(std::uint32_t)) > data.size()) { - throw std::runtime_error("Failed to encode uint32 into frame"); - } - - data[offset] = static_cast(value & 0xFFU); - data[offset + 1] = static_cast((value >> 8U) & 0xFFU); - data[offset + 2] = static_cast((value >> 16U) & 0xFFU); - data[offset + 3] = static_cast((value >> 24U) & 0xFFU); -} - -// Stop responsiveness (#26). The blocking USB loops keep their per-call -// timeouts short and re-poll this flag so a SIGTERM/SIGINT delivered mid-sweep -// is honoured within a few hundred ms instead of only after the whole sweep -// budget has elapsed. -// -// `main()` installs its own SIGINT/SIGTERM handlers before any driver is -// opened, so we chain to (and preserve) whatever handler is already installed -// rather than clobbering it. All state lives in single inline-static slots so -// every translation unit observes the same flag and chain table. -[[nodiscard]] inline auto native_stop_flag() -> volatile std::sig_atomic_t& { - static volatile std::sig_atomic_t flag = 0; - return flag; -} - -[[nodiscard]] inline auto native_chained_handlers() -> std::array& { - static std::array handlers{}; - return handlers; -} - -inline void native_stop_signal_handler(int signal_number) { - native_stop_flag() = 1; - - // Chain to the previously installed disposition (e.g. main()'s handler) so - // process-level stop semantics are unchanged. - if (signal_number < 0 || signal_number >= NSIG) { - return; - } - const auto& prior = native_chained_handlers()[static_cast(signal_number)]; - if ((prior.sa_flags & SA_SIGINFO) == 0 && prior.sa_handler != nullptr && - prior.sa_handler != SIG_DFL && prior.sa_handler != SIG_IGN && - prior.sa_handler != native_stop_signal_handler) { - prior.sa_handler(signal_number); - } -} - -// Lazily install the chaining stop observer for one signal. Must be called -// after main() has installed its handlers (i.e. from open_native()). -inline void install_native_stop_observer_for(int signal_number) { - if (signal_number < 0 || signal_number >= NSIG) { - return; - } - - struct sigaction current{}; - if (sigaction(signal_number, nullptr, ¤t) != 0) { - return; - } - if (current.sa_handler == native_stop_signal_handler) { - return; // Already installed; do not chain to ourselves. - } - - native_chained_handlers()[static_cast(signal_number)] = current; - - struct sigaction action{}; - action.sa_handler = native_stop_signal_handler; - sigemptyset(&action.sa_mask); - action.sa_flags = current.sa_flags & ~SA_SIGINFO; // Keep flags such as SA_RESTART. - sigaction(signal_number, &action, nullptr); -} - -inline void install_native_stop_observer() { - install_native_stop_observer_for(SIGINT); - install_native_stop_observer_for(SIGTERM); -} - -[[nodiscard]] inline auto native_stop_requested() -> bool { - return native_stop_flag() != 0; -} - -} // namespace radar::drivers::librevna::detail - diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna_minimal_driver.hpp b/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna_minimal_driver.hpp deleted file mode 100644 index e6d4bb7..0000000 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna_minimal_driver.hpp +++ /dev/null @@ -1,103 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#if __has_include() -#include -#else -#include -#endif - -#include "radar_driver.hpp" -#include "run_config.hpp" - -namespace radar::drivers { - -/** - * @brief Runtime settings for minimal LibreVNA acquisition. - */ -struct LibreVnaMinimalDriverSettings { - std::string serial{}; - config::RadarSweepSettings sweep{}; -}; - -/** - * @brief Minimal radar driver that acquires forward sweeps from LibreVNA. - * - * This class intentionally keeps scope narrow: open/close transport and - * acquire one sweep. Full device configuration is expected to be done by the - * Python layer before this process starts. - */ -class LibreVnaMinimalDriver final : public RadarDriver { - public: - explicit LibreVnaMinimalDriver(LibreVnaMinimalDriverSettings settings); - - void open() override; - void close() override; - [[nodiscard]] auto acquire_sweep() -> SweepTrace override; - - private: - /** - * @brief Raw protocol packet decoded from framed USB stream. - */ - struct NativePacket { - std::uint8_t packet_type = 0; - std::vector payload{}; - }; - - /** - * @brief One decoded VNADatapoint containing complex values for one point. - */ - struct DecodedVnaDatapoint { - std::uint32_t point_number = 0; - float frequency_hz = 0.0F; - ipc::Complex32 s11{}; - ipc::Complex32 s21{}; - bool has_s11 = false; - bool has_s21 = false; - }; - - [[nodiscard]] auto acquire_native() -> SweepTrace; - - void open_native(); - void close_native(); - - void send_packet(std::uint8_t packet_type, std::span payload, bool require_ack); - void send_packet_no_payload(std::uint8_t packet_type, bool require_ack); - void wait_for_ack(std::chrono::steady_clock::time_point deadline); - [[nodiscard]] auto wait_for_packet(std::uint8_t expected_type, std::chrono::steady_clock::time_point deadline) - -> NativePacket; - void pump_usb(std::chrono::steady_clock::time_point deadline); - void decode_frames_from_buffer(); - - [[nodiscard]] auto pop_packet(std::uint8_t packet_type) -> std::optional; - [[nodiscard]] static auto encode_frame(std::uint8_t packet_type, std::span payload) - -> std::vector; - [[nodiscard]] static auto decode_vna_datapoint_traces( - std::span payload, - DecodedVnaDatapoint& datapoint_out - ) -> bool; - [[nodiscard]] static auto crc32(std::span data) -> std::uint32_t; - void validate_device_info_payload(std::span payload) const; - - LibreVnaMinimalDriverSettings settings_{}; - bool is_open_ = false; - - libusb_context* usb_context_ = nullptr; - libusb_device_handle* usb_handle_ = nullptr; - bool interface_claimed_ = false; - - std::vector rx_buffer_{}; - std::deque packet_queue_{}; - - std::uint16_t protocol_version_ = 0; - std::uint8_t device_num_ports_ = 0; -}; - -} // namespace radar::drivers diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/switches/hmc349a_minimal_driver.cpp b/data_acq_and_processing/sweep_orchestrator/device_drivers/switches/hmc349a_minimal_driver.cpp deleted file mode 100644 index 51067d0..0000000 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/switches/hmc349a_minimal_driver.cpp +++ /dev/null @@ -1,183 +0,0 @@ -#include "hmc349a_minimal_driver.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace radar::drivers { -namespace { - -// A transient ioctl failure (e.g. EINTR/EAGAIN/EBUSY under load) should not be -// immediately fatal; retry a small number of times before giving up. -constexpr int kSwitchIoctlRetries = 3; - -// Compute the GPIO line values for a given logical position, honouring inverted -// control logic. Shared by switch_native() and the teardown default-drive path. -[[nodiscard]] auto make_line_values(const HMC349AMinimalDriverSettings& settings, std::uint32_t position) - -> gpio_v2_line_values { - const std::uint8_t requested_state = static_cast(position & 0x01U); - const std::uint8_t control_state = settings.invert_logic ? (requested_state ^ 0x01U) : requested_state; - - gpio_v2_line_values values{}; - values.mask = (1ULL << 0U); - values.bits = static_cast(control_state); - return values; -} - -void validate_open_settings(const HMC349AMinimalDriverSettings& settings) { - if (settings.positions == 0U || settings.positions > 2U) { - throw std::runtime_error("HMC349A switch positions must be in range [1, 2] for " + settings.name); - } - if (settings.default_position >= settings.positions) { - throw std::runtime_error("Switch default_position out of range for " + settings.name); - } -} - -} // namespace - -HMC349AMinimalDriver::HMC349AMinimalDriver(HMC349AMinimalDriverSettings settings) - : settings_(std::move(settings)) {} - -HMC349AMinimalDriver::~HMC349AMinimalDriver() { - // Safety net: if the owner never called close() (e.g. on crash/unwind), - // still drive the RF path to a known-safe state and release the GPIO lines. - try { - close(); - } catch (...) { - } -} - -void HMC349AMinimalDriver::open() { - if (is_open_) { - return; - } - - validate_open_settings(settings_); - - if (settings_.mode == config::DriverMode::Native) { - open_native(); - } - - is_open_ = true; - switch_to(settings_.default_position); -} - -void HMC349AMinimalDriver::close() { - if (!is_open_) { - return; - } - - if (settings_.mode == config::DriverMode::Native) { - close_native(); - } - - is_open_ = false; -} - -auto HMC349AMinimalDriver::position_count() const -> std::uint32_t { - return settings_.positions; -} - -void HMC349AMinimalDriver::switch_to(std::uint32_t position) { - if (!is_open_) { - throw std::runtime_error("Switch driver is not open for " + settings_.name); - } - if (position >= settings_.positions) { - throw std::runtime_error("Switch position out of range for " + settings_.name); - } - - switch (settings_.mode) { - case config::DriverMode::Mock: - break; - case config::DriverMode::Native: - switch_native(position); - break; - default: - throw std::runtime_error("Unsupported switch driver mode for " + settings_.name); - } - - current_position_ = position; -} - -void HMC349AMinimalDriver::open_native() { - if (settings_.gpio_chip.empty()) { - throw std::runtime_error("gpio_chip is empty for native switch " + settings_.name); - } - if (settings_.pin_a < 0) { - throw std::runtime_error("pin_a (control) is invalid for native switch " + settings_.name); - } - - chip_fd_ = ::open(settings_.gpio_chip.c_str(), O_RDONLY | O_CLOEXEC); - if (chip_fd_ < 0) { - throw std::runtime_error( - "Failed to open gpio chip '" + settings_.gpio_chip + "' for " + settings_.name + ": " + std::strerror(errno) - ); - } - - gpio_v2_line_request request{}; - // Bit 0 controls pin_a. - request.offsets[0] = static_cast(settings_.pin_a); - request.num_lines = 1; - request.config.flags = GPIO_V2_LINE_FLAG_OUTPUT; - std::snprintf(request.consumer, sizeof(request.consumer), "radar_%s", settings_.name.c_str()); - - if (::ioctl(chip_fd_, GPIO_V2_GET_LINE_IOCTL, &request) != 0) { - const auto error = std::strerror(errno); - ::close(chip_fd_); - chip_fd_ = -1; - throw std::runtime_error("Failed to request GPIO lines for " + settings_.name + ": " + error); - } - - line_fd_ = request.fd; - if (line_fd_ < 0) { - ::close(chip_fd_); - chip_fd_ = -1; - throw std::runtime_error("GPIO line request returned invalid fd for " + settings_.name); - } -} - -void HMC349AMinimalDriver::close_native() { - // Leave the RF path in its known-safe default position before releasing the - // line, so shutdown/crash never strands the switch in an arbitrary state. - if (line_fd_ >= 0 && settings_.default_position < settings_.positions) { - auto values = make_line_values(settings_, settings_.default_position); - // Best-effort: do not throw out of the teardown path. - ::ioctl(line_fd_, GPIO_V2_LINE_SET_VALUES_IOCTL, &values); - } - - if (line_fd_ >= 0) { - ::close(line_fd_); - line_fd_ = -1; - } - if (chip_fd_ >= 0) { - ::close(chip_fd_); - chip_fd_ = -1; - } -} - -void HMC349AMinimalDriver::switch_native(std::uint32_t position) { - if (line_fd_ < 0) { - throw std::runtime_error("Native GPIO line fd is not open for " + settings_.name); - } - - auto values = make_line_values(settings_, position); - - // Retry transient ioctl failures so a momentary hiccup is recoverable - // instead of aborting the whole sweep; only the final attempt is fatal. - int last_errno = 0; - for (int attempt = 0; attempt < kSwitchIoctlRetries; ++attempt) { - if (::ioctl(line_fd_, GPIO_V2_LINE_SET_VALUES_IOCTL, &values) == 0) { - return; - } - last_errno = errno; - } - throw std::runtime_error("Failed to switch GPIO state for " + settings_.name + ": " + std::strerror(last_errno)); -} - -} // namespace radar::drivers diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/switches/hmc349a_minimal_driver.hpp b/data_acq_and_processing/sweep_orchestrator/device_drivers/switches/hmc349a_minimal_driver.hpp deleted file mode 100644 index 057da95..0000000 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/switches/hmc349a_minimal_driver.hpp +++ /dev/null @@ -1,53 +0,0 @@ -#pragma once - -#include -#include - -#include "run_config.hpp" -#include "switch_driver.hpp" - -namespace radar::drivers { - -/** - * @brief Runtime settings for the HMC349A GPIO switch driver. - */ -struct HMC349AMinimalDriverSettings { - std::string name{}; - config::DriverMode mode = config::DriverMode::Native; - std::uint32_t positions = 2; - std::uint32_t default_position = 0; - std::string gpio_chip = "/dev/gpiochip0"; - std::int32_t pin_a = 17; ///< Control pin. - bool invert_logic = false; ///< Invert control logic for position mapping. -}; - -/** - * @brief Minimal HMC349A implementation for deterministic GPIO control. - * - * Native mode uses Linux GPIO character-device API (uAPI v2). - * Mock mode keeps state in memory without touching hardware. - */ -class HMC349AMinimalDriver final : public SwitchDriver { - public: - explicit HMC349AMinimalDriver(HMC349AMinimalDriverSettings settings); - ~HMC349AMinimalDriver() override; // Leaves RF path safe even if close() was skipped. - - void open() override; - void close() override; - [[nodiscard]] auto position_count() const -> std::uint32_t override; - void switch_to(std::uint32_t position) override; - - private: - void open_native(); - void close_native(); - void switch_native(std::uint32_t position); - - HMC349AMinimalDriverSettings settings_{}; - bool is_open_ = false; - std::uint32_t current_position_ = 0; - - int chip_fd_ = -1; - int line_fd_ = -1; -}; - -} // namespace radar::drivers diff --git a/data_acq_and_processing/sweep_orchestrator/include/sweep_orchestrator.hpp b/data_acq_and_processing/sweep_orchestrator/include/sweep_orchestrator.hpp deleted file mode 100644 index db05570..0000000 --- a/data_acq_and_processing/sweep_orchestrator/include/sweep_orchestrator.hpp +++ /dev/null @@ -1,60 +0,0 @@ -#pragma once - -#include -#include - -#include "radar_driver.hpp" -#include "run_config.hpp" -#include "shm_ring.hpp" -#include "switch_driver.hpp" - -namespace radar::acq { - -/** - * @brief Produces raw sweep collections for configured switch combinations. - * - * Lifecycle: - * 1. Open radar and switch drivers. - * 2. Iterate configured combinations, acquire one sweep per combination. - * 3. Publish serialized collections into the raw shared-memory ring. - */ -class SweepOrchestrator { - public: - /** - * @param config Runtime config loaded from JSON. - * @param radar_driver Radar device implementation. - * @param input_switch_driver Switch connected to radar input path. - * @param output_switch_driver Switch connected to radar output path. - * @param raw_ring Shared-memory ring for raw collections. - * @param raw_tap_ring Optional tap ring for GUI/debug readers. - */ - SweepOrchestrator( - const config::RunConfig& config, - drivers::RadarDriver& radar_driver, - drivers::SwitchDriver& input_switch_driver, - drivers::SwitchDriver& output_switch_driver, - ipc::ShmRing& raw_ring, - ipc::ShmRing* raw_tap_ring = nullptr - ); - - /** - * @brief Run acquisition loop until stop is requested or single run completes. - */ - void run(const std::atomic& stop_requested); - - private: - /** - * @brief Acquire one collection for all configured combinations. - */ - [[nodiscard]] auto acquire_one_collection(std::uint64_t collection_id, const std::atomic& stop_requested) - -> ipc::RawSweepCollection; - - const config::RunConfig& config_; - drivers::RadarDriver& radar_driver_; - drivers::SwitchDriver& input_switch_driver_; - drivers::SwitchDriver& output_switch_driver_; - ipc::ShmRing& raw_ring_; - ipc::ShmRing* raw_tap_ring_ = nullptr; -}; - -} // namespace radar::acq diff --git a/data_acq_and_processing/sweep_orchestrator/src/main.cpp b/data_acq_and_processing/sweep_orchestrator/src/main.cpp deleted file mode 100644 index 9b1408c..0000000 --- a/data_acq_and_processing/sweep_orchestrator/src/main.cpp +++ /dev/null @@ -1,150 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include "h7992_minimal_driver.hpp" -#include "hmc349a_minimal_driver.hpp" -#include "librevna_minimal_driver.hpp" -#include "radar_driver.hpp" -#include "run_config.hpp" -#include "shm_ring.hpp" -#include "sweep_orchestrator.hpp" - -namespace { - -constexpr const char* kDefaultConfigPath = "run_config.json"; -constexpr const char* kLibreVnaModel = "librevna"; -constexpr const char* kLibreVnaMultiModel = "librevna_multi"; -std::atomic g_stop_requested{false}; - -void signal_handler(int /*signal*/) { - g_stop_requested.store(true, std::memory_order_relaxed); -} - -void install_signal_handlers() { - std::signal(SIGINT, signal_handler); - std::signal(SIGTERM, signal_handler); -} - -[[nodiscard]] auto read_config_path(int argc, char** argv) -> std::string { - std::string config_path = kDefaultConfigPath; - - for (int index = 1; index < argc; ++index) { - const std::string arg = argv[index]; - if (arg == "--config" && (index + 1) < argc) { - config_path = argv[++index]; - } - } - - return config_path; -} - -[[nodiscard]] auto make_radar_driver(const radar::config::RadarConfig& config) - -> std::unique_ptr { - if (config.model == kLibreVnaModel || config.model.empty()) { - if (config.driver_mode != radar::config::DriverMode::Native) { - throw std::runtime_error("LibreVNA simulation uses python_app.scripts.matrix_raw_producer; launch through start.sh"); - } - return std::make_unique( - radar::drivers::LibreVnaMinimalDriverSettings{ - .serial = config.serial, - .sweep = config.sweep, - } - ); - } - - if (config.model == kLibreVnaMultiModel) { - throw std::runtime_error("librevna_multi is handled by python_app.scripts.matrix_raw_producer"); - } - - throw std::runtime_error("Unsupported radar.model for sweep_orchestrator: " + config.model); -} - -[[nodiscard]] auto make_h7992_driver(const radar::config::SwitchConfig& config) - -> std::unique_ptr { - return std::make_unique( - radar::drivers::H7992MinimalDriverSettings{ - .name = config.name, - .mode = config.driver_mode, - .positions = config.positions, - .default_position = config.default_position, - .gpio_chip = config.gpio_chip, - .pin_a = config.pin_a, - .pin_b = config.pin_b, - } - ); -} - -[[nodiscard]] auto make_hmc349a_driver(const radar::config::SwitchConfig& config) - -> std::unique_ptr { - return std::make_unique( - radar::drivers::HMC349AMinimalDriverSettings{ - .name = config.name, - .mode = config.driver_mode, - .positions = config.positions, - .default_position = config.default_position, - .gpio_chip = config.gpio_chip, - .pin_a = config.pin_a, - .invert_logic = config.invert_logic, - } - ); -} - -[[nodiscard]] auto make_switch_driver(const radar::config::SwitchConfig& config) - -> std::unique_ptr { - using radar::config::SwitchDriverKind; - - switch (config.driver_kind) { - case SwitchDriverKind::H7992: - return make_h7992_driver(config); - case SwitchDriverKind::HMC349A: - return make_hmc349a_driver(config); - default: - throw std::runtime_error("Unsupported switch driver kind"); - } -} - -} // namespace - -int main(int argc, char** argv) { - install_signal_handlers(); - - try { - const auto config_path = read_config_path(argc, argv); - const auto config = radar::config::load_run_config(config_path); - - auto raw_ring = radar::ipc::ShmRing::open_or_create( - config.rings.raw.name, - config.rings.raw.capacity, - config.rings.raw.slot_size_bytes - ); - auto raw_tap_ring = radar::ipc::ShmRing::open_or_create( - config.rings.raw_tap.name, - config.rings.raw_tap.capacity, - config.rings.raw_tap.slot_size_bytes - ); - - auto radar_driver = make_radar_driver(config.radar); - - auto input_switch = make_switch_driver(config.input_switch); - auto output_switch = make_switch_driver(config.output_switch); - - radar::acq::SweepOrchestrator orchestrator( - config, - *radar_driver, - *input_switch, - *output_switch, - raw_ring, - &raw_tap_ring - ); - orchestrator.run(g_stop_requested); - return 0; - } catch (const std::exception& exception) { - std::cerr << "sweep_orchestrator error: " << exception.what() << '\n'; - return 1; - } -} diff --git a/data_acq_and_processing/sweep_orchestrator/src/sweep_orchestrator.cpp b/data_acq_and_processing/sweep_orchestrator/src/sweep_orchestrator.cpp deleted file mode 100644 index 1509348..0000000 --- a/data_acq_and_processing/sweep_orchestrator/src/sweep_orchestrator.cpp +++ /dev/null @@ -1,298 +0,0 @@ -#include "sweep_orchestrator.hpp" -#include "runtime_wait.hpp" - -#include -#include -#include -#include -#include -#include - -namespace radar::acq { -namespace { - -// Wait-for-device retry tuning (mirror of python matrix_raw_producer._open_radar_with_retry): -// a device that is absent at boot or disappears mid-run must never kill the orchestrator, -// only make it wait. Backoff is capped so a long absence does not busy-spin, and every wait -// is interruptible by stop_requested for a prompt clean exit. -constexpr std::uint32_t kOpenRetryMinMs = 1'000U; -constexpr std::uint32_t kOpenRetryMaxMs = 10'000U; -// Throttle open-failure logging during a long wait so a permanently absent device does not -// flood the process log: log the first failure, then every Nth attempt. -constexpr std::uint64_t kOpenRetryLogEvery = 30U; - -[[nodiscard]] inline auto should_stop(const std::atomic& stop_requested) -> bool { - return stop_requested.load(std::memory_order_relaxed); -} - -void validate_sweep(const drivers::SweepTrace& sweep) { - if (sweep.frequency_hz.size() != sweep.s11.size() || sweep.frequency_hz.size() != sweep.s21.size()) { - throw std::runtime_error("Radar driver returned inconsistent sweep vectors"); - } -} - -class DriverLifecycleGuard { - public: - DriverLifecycleGuard( - drivers::RadarDriver& radar_driver, - drivers::SwitchDriver& input_switch_driver, - drivers::SwitchDriver& output_switch_driver - ) - : radar_driver_(radar_driver), - input_switch_driver_(input_switch_driver), - output_switch_driver_(output_switch_driver) {} - - ~DriverLifecycleGuard() noexcept { - close_all(); - } - - void open_all() { - active_ = true; // A later open failure still owns partially opened devices. - radar_driver_.open(); - input_switch_driver_.open(); - output_switch_driver_.open(); - } - - // Open all devices, retrying forever with capped exponential backoff until success or - // stop. Used for both the initial open and every in-loop reopen, so a device that is - // absent at boot or disappears mid-run never kills the orchestrator — it just waits. - // Returns false only if a stop was requested before any device became available. - [[nodiscard]] auto open_all_with_retry(const std::atomic& stop_requested) -> bool { - // Drop any partial state from a previous open before retrying. - close_all(); - - std::uint64_t attempt = 0; - std::uint32_t delay_ms = kOpenRetryMinMs; - while (!should_stop(stop_requested)) { - try { - open_all(); - if (attempt > 0) { - std::cerr << "sweep_orchestrator: devices opened after " << (attempt + 1) << " attempt(s)\n"; - } - return true; - } catch (const std::exception& exc) { - // Best-effort: drop any partial open before the next attempt. - close_all(); - ++attempt; - if (attempt == 1 || attempt % kOpenRetryLogEvery == 0) { - std::cerr << "sweep_orchestrator: devices not available (attempt " << attempt - << "); retrying up to every " << (kOpenRetryMaxMs / 1'000U) - << "s until present: " << exc.what() << '\n'; - } - if (config::wait_or_stop(delay_ms, stop_requested)) { - return false; - } - delay_ms = std::min(delay_ms * 2U, kOpenRetryMaxMs); - } - } - return false; - } - - void close_all() noexcept { - if (!active_) { - return; - } - active_ = false; - const auto close = [](auto& driver) { - try { - driver.close(); - } catch (const std::exception& error) { - std::cerr << "sweep_orchestrator: driver cleanup failed: " << error.what() << '\n'; - } catch (...) { - std::cerr << "sweep_orchestrator: driver cleanup failed with an unknown error\n"; - } - }; - close(output_switch_driver_); - close(input_switch_driver_); - close(radar_driver_); - } - - private: - drivers::RadarDriver& radar_driver_; - drivers::SwitchDriver& input_switch_driver_; - drivers::SwitchDriver& output_switch_driver_; - bool active_ = false; -}; - -// Worst-case serialized size of a collection given the configured combo count and sweep -// point count, using the trace wire format (see ipc::write_trace_collection/write_trace_block): -// collection header: magic(4) + collection_id(8) + monotonic_ns(8) + trace_count(4) -// + capture_start_ns(8) + capture_end_ns(8) -// + trace capture window count(4) = 44 bytes -// per trace block: input_pos(4) + output_pos(4) + point_count(4) = 12 bytes -// + per point: frequency(4) + s11(8) + s21(8) = 20 bytes -// + trailer: capture_start_ns(8) + capture_end_ns(8) = 16 bytes -[[nodiscard]] auto worst_case_serialized_bytes(std::size_t combo_count, std::uint32_t sweep_points) -> std::size_t { - constexpr std::size_t kCollectionHeaderBytes = 44U; - constexpr std::size_t kTraceHeaderBytes = 12U; - constexpr std::size_t kBytesPerPoint = 20U; - constexpr std::size_t kTraceTrailerBytes = 16U; - const std::size_t per_trace = - kTraceHeaderBytes + (static_cast(sweep_points) * kBytesPerPoint) + kTraceTrailerBytes; - return kCollectionHeaderBytes + (combo_count * per_trace); -} - -// Publish a collection on the primary raw ring; tap pushes are strictly best-effort. -// Never throws: an oversize payload is logged-and-dropped (the slot-size budget is also -// validated up front at startup, so this guards only against unexpected runtime growth). -// Returns true if the primary push succeeded. -[[nodiscard]] auto publish_collection( - ipc::ShmRing& raw_ring, - ipc::ShmRing* raw_tap_ring, - const ipc::RawSweepCollection& collection, - std::uint64_t& oversize_drop_count -) -> bool { - const auto serialized_collection = ipc::serialize_raw_collection(collection); - if (!raw_ring.push(serialized_collection)) { - // Log-and-drop an oversize payload rather than aborting the long-running loop. - // Throttle so a persistently oversize payload cannot flood the log. - if (oversize_drop_count % 100 == 0) { - std::cerr << "sweep_orchestrator: dropped oversize raw payload (" << serialized_collection.size() - << " bytes > slot " << raw_ring.slot_size_bytes() << "; drop count=" - << (oversize_drop_count + 1) << ")\n"; - } - ++oversize_drop_count; - return false; - } - // Tap ring is for GUI/debug observers only: a tap failure must never abort the primary path. - if (raw_tap_ring != nullptr) { - (void)raw_tap_ring->push(serialized_collection); - } - return true; -} - -} // namespace - -SweepOrchestrator::SweepOrchestrator( - const config::RunConfig& config, - drivers::RadarDriver& radar_driver, - drivers::SwitchDriver& input_switch_driver, - drivers::SwitchDriver& output_switch_driver, - ipc::ShmRing& raw_ring, - ipc::ShmRing* raw_tap_ring -) - : config_(config), - radar_driver_(radar_driver), - input_switch_driver_(input_switch_driver), - output_switch_driver_(output_switch_driver), - raw_ring_(raw_ring), - raw_tap_ring_(raw_tap_ring) {} - -void SweepOrchestrator::run(const std::atomic& stop_requested) { - // Fail fast on a config error: a slot that is too small for the worst-case payload can - // never carry a full collection, so report it clearly at startup instead of dropping - // every collection at runtime. - const auto worst_case_bytes = worst_case_serialized_bytes(config_.run_combos.size(), config_.radar.sweep.points); - if (worst_case_bytes > raw_ring_.slot_size_bytes()) { - throw std::runtime_error( - "Raw ring slot_size_bytes (" + std::to_string(raw_ring_.slot_size_bytes()) - + ") is too small for the worst-case serialized collection (" + std::to_string(worst_case_bytes) - + " bytes for " + std::to_string(config_.run_combos.size()) + " combos x " - + std::to_string(config_.radar.sweep.points) + " points)" - ); - } - - DriverLifecycleGuard lifecycle_guard(radar_driver_, input_switch_driver_, output_switch_driver_); - // Wait for the devices to become available before starting: an absent device - // makes the orchestrator wait, not exit. - if (!lifecycle_guard.open_all_with_retry(stop_requested)) { - return; // stop requested before any device became available - } - - std::uint64_t collection_id = 0; - std::uint64_t oversize_drop_count = 0; - while (!should_stop(stop_requested)) { - try { - auto raw_collection = acquire_one_collection(++collection_id, stop_requested); - if (raw_collection.traces.empty()) { - if (!config_.runtime.continuous) { - break; - } - (void)config::wait_or_stop(config_.runtime.idle_sleep_ms, stop_requested); - continue; - } - - (void)publish_collection(raw_ring_, raw_tap_ring_, raw_collection, oversize_drop_count); - - if (!config_.runtime.continuous) { - break; - } - } catch (const std::exception& exc) { - // Treat any in-loop device error (timeout/NACK/transient USB or socket glitch/ - // device-not-found-after-glitch) as recoverable: close, wait for the device to - // come back, and continue. This mirrors the python producer's reconnect-forever - // policy so a long-running headless appliance survives transient hardware hiccups - // instead of exiting. Genuinely fatal config errors are caught at startup above - // (and in main()), reserving non-zero exit for those. - std::cerr << "sweep_orchestrator: acquisition failed; reopening and waiting for the device: " - << exc.what() << '\n'; - if (!config_.runtime.continuous) { - throw; // single-run mode has no recovery path; surface the failure - } - if (!lifecycle_guard.open_all_with_retry(stop_requested)) { - break; // stop requested while waiting to reopen - } - } - } - - lifecycle_guard.close_all(); -} - -auto SweepOrchestrator::acquire_one_collection( - std::uint64_t collection_id, - const std::atomic& stop_requested -) -> ipc::RawSweepCollection { - ipc::RawSweepCollection collection{}; - collection.collection_id = collection_id; - collection.monotonic_ns = ipc::current_monotonic_ns(); - collection.traces.reserve(config_.run_combos.size()); - collection.capture_start_ns = ipc::current_monotonic_ns(); - bool interrupted = false; - - for (const auto& combo : config_.run_combos) { - if (should_stop(stop_requested)) { - interrupted = true; - break; - } - - // Hardware flow for each combo: - // 1) switch output path, 2) switch input path, 3) optional settle. - output_switch_driver_.switch_to(combo.output_pos); - input_switch_driver_.switch_to(combo.input_pos); - if (config::wait_or_stop(config_.runtime.settling_ms, stop_requested)) { - interrupted = true; - break; - } - - // Stamp around the sweep only: the switch drive and settling above belong to - // neither the previous combo nor this one, so excluding them keeps the window - // an honest "when was this combo actually measured". - const auto sweep_start_ns = ipc::current_monotonic_ns(); - auto sweep = radar_driver_.acquire_sweep(); - const auto sweep_end_ns = ipc::current_monotonic_ns(); - validate_sweep(sweep); - - ipc::SweepTraceBlock trace{}; - trace.combo = combo; - trace.frequency_hz = std::move(sweep.frequency_hz); - trace.s11 = std::move(sweep.s11); - trace.s21 = std::move(sweep.s21); - trace.capture_start_ns = sweep_start_ns; - trace.capture_end_ns = sweep_end_ns; - collection.traces.push_back(std::move(trace)); - } - - if (interrupted) { - // Do not emit partial collections when stop was requested mid-cycle. - collection.traces.clear(); - collection.capture_start_ns = 0; - collection.capture_end_ns = 0; - return collection; - } - - collection.capture_end_ns = collection.traces.empty() ? 0U : ipc::current_monotonic_ns(); - - return collection; -} - -} // namespace radar::acq diff --git a/data_acq_and_processing/tests/generate_sar3d_emulation.py b/data_acq_and_processing/tests/generate_sar3d_emulation.py deleted file mode 100644 index 4adabee..0000000 --- a/data_acq_and_processing/tests/generate_sar3d_emulation.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Extract production CUDA kernels and host helpers for the test-only adapter.""" - -from pathlib import Path -import re -import sys - - -source = (Path(__file__).resolve().parents[1] / "processing/processors/src/sar3d_gpu.cpp").read_text() -kernel = re.search(r'kKernelSource\s*=\s*R"CUDA\((.*?)\)CUDA";', source, re.DOTALL) -if kernel is None: - raise RuntimeError("SAR CUDA kernel literal was not found") -first = source.index("constexpr std::size_t kMaxIndexedItems") -last = source.index("} // namespace", first) -destination = Path(sys.argv[1]) -destination.mkdir(parents=True, exist_ok=True) -(destination / "sar3d_kernels.inc").write_text(kernel.group(1)) -helpers = re.sub(r"^#(?:ifdef SAR3D_WITH_CUDA|endif)$", "", source[first:last], flags=re.MULTILINE) -(destination / "sar3d_helpers.inc").write_text(helpers) diff --git a/data_acq_and_processing/tests/generate_sar3d_reference.py b/data_acq_and_processing/tests/generate_sar3d_reference.py new file mode 100644 index 0000000..76ee957 --- /dev/null +++ b/data_acq_and_processing/tests/generate_sar3d_reference.py @@ -0,0 +1,204 @@ +"""Generate small SAR3D references by executing notebook cells 0 and 4. + +Requires the original, untracked Multi_antenna_3D_clean_v1.ipynb; a missing notebook +is an error. NumPy/SciPy execute its function/class ASTs unchanged. Only parameter +values are replaced; plotting imports and the interactive top-level runs are +excluded. No production SAR implementation or hand-transcribed oracle is used. + +Run from the repository root: + .venv/bin/python data_acq_and_processing/tests/generate_sar3d_reference.py +""" +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +from pathlib import Path +import tempfile + +import numpy as np +import scipy + +ROOT = Path(__file__).resolve().parents[2] +REFERENCE = Path(__file__).with_name("sar3d_reference.npz") + + +def notebook_functions(notebook: Path, parameters: dict) -> tuple[dict, dict]: + cells = json.loads(notebook.read_text(encoding="utf-8"))["cells"] + namespace = {"__name__": __name__} + hashes = {} + for index in (0, 4): + source = "".join(cells[index]["source"]) + hashes[str(index)] = hashlib.sha256(source.encode()).hexdigest() + tree = ast.parse(source, filename=f"{notebook.name}:cell{index}") + definitions = [] + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.ImportFrom)): + definitions.append(node) + elif isinstance(node, ast.Import): + if not any(alias.name.startswith(("matplotlib", "plotly")) for alias in node.names): + definitions.append(node) + elif isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): + try: + namespace[node.targets[0].id] = ast.literal_eval(node.value) + except (ValueError, TypeError): + pass # Dataset loading and the interactive run are supplied below. + namespace.update(parameters) + exec(compile(ast.Module(definitions, type_ignores=[]), f"{notebook.name}:cell{index}", + "exec", dont_inherit=True), namespace) + return namespace, hashes + + +def scene() -> dict: + """Deterministic two-echo input with motion, vertical baselines and four pairs.""" + request = dict( + tx=[[0, -.4, 0, 0], [1, .45, .2, .05]], + rx=[[0, -.2, .1, 0], [1, .3, .35, .02]], f_start=1e9, f_stop=4e9, + min_depth=1., max_depth=3., speed=.4, pitch=13., ground=.3, + voxel=.4, radius=1, power=1.5, compensation=True, normalize=True, + ) + # Match the float32 settings/geometry accepted by the native fixture. + for key in ("tx", "rx"): + request[key] = np.asarray(request[key], dtype=np.float32).astype(float).tolist() + frequency = np.linspace(1e9, 4e9, 201, dtype=np.float32) + members = [] + for frame, offset in enumerate((0, 180_000_000, 510_000_000)): + traces = [] + start = 10_000_000_000 + offset + for tx in range(2): + for rx in range(2): + samples = ((1 + .1 * rx) * np.exp(-2j * np.pi * frequency.astype(float) + * (4.8 + .08 * tx - .03 * frame) / 3e8) + + .35 * np.exp(-2j * np.pi * frequency.astype(float) * (5.6 - .05 * rx) / 3e8)).astype(np.complex64) + traces.append(dict(input=rx, output=tx, start=start, end=start + 165_000_000, + frequency=frequency.tolist(), s21=np.column_stack((samples.real, samples.imag)).tolist())) + members.append(dict(id=frame + 1, timestamp=start + 165_000_000, traces=traces)) + request.update(previous=members[:-1], latest=members[-1]) + return request + + +def write_dataset(directory: Path, request: dict) -> list[Path]: + """Supply the notebook's original config_profile/meta/NPY input contract.""" + profile = {"gpr": {}, "run": {"combos": []}} + for kind, position in (("tx", "output_pos"), ("rx", "input_pos")): + profile["gpr"][f"{kind}_geometry"] = [ + {position: int(index), "x_m": x, "y_m": y, "z_m": z} + for index, x, y, z in request[kind] + ] + profile["run"]["combos"] = [ + {"input": trace["input"], "output": trace["output"]} for trace in request["latest"]["traces"] + ] + (directory / "config_profile.json").write_text(json.dumps(profile), encoding="utf-8") + snapshots = [] + for index, member in enumerate([*request["previous"], request["latest"]]): + snapshot = directory / f'{index:04d}_id{member["id"]}_ns{member["timestamp"]}' + snapshot.mkdir() + metadata = {"traces": []} + for trace in member["traces"]: + stem = f'i{trace["input"]}_o{trace["output"]}' + values = np.asarray(trace["s21"], dtype=np.float32) + np.save(snapshot / f"{stem}_s21.npy", values[:, 0] + 1j * values[:, 1]) + np.save(snapshot / f"{stem}_freq.npy", np.asarray(trace["frequency"], dtype=np.float32)) + metadata["traces"].append(dict(input=trace["input"], output=trace["output"], + capture_start_ns=trace["start"], capture_end_ns=trace["end"])) + (snapshot / "meta.json").write_text(json.dumps(metadata), encoding="utf-8") + snapshots.append(snapshot) + return snapshots + + +def reference(notebook: Path, request: dict, directory: Path) -> tuple[dict, dict]: + snapshots = write_dataset(directory, request) + center = snapshots[max(len(snapshots) - 1 - request["radius"], 0)] + parameters = dict( + INPUT_IDX=[int(row[0]) for row in request["rx"]], + OUTPUT_IDX=[int(row[0]) for row in request["tx"]], + F_START=request["f_start"], F_STOP=request["f_stop"], + MIN_DEPTH=request["min_depth"], MAX_DEPTH=request["max_depth"], + PAIR_NORM_DEPTH_MIN=request["min_depth"], PAIR_NORM_DEPTH_MAX=request["max_depth"], + SPEED_M_S=request["speed"], LOOK_ANGLE_DEG=request["pitch"], + BP_GROUND_DEPTH_M=request["ground"], COMP_RANGE_POWER=request["power"], + SAR_VOXEL_M=(request["voxel"],) * 3, SAR_FRAME_RADIUS=request["radius"], + SAR_USE_COMPENSATION=request["compensation"], SAR_NORMALIZE_EACH_FRAME=request["normalize"], + SAR_SCAN_ROOT=directory, DATA_PATH=center, BG_PATH=directory, v=3e8, + ) + ns, hashes = notebook_functions(notebook, parameters) + tx, rx, order, tx_order, pair_count = ns["load_antenna_config_profile"]( + directory / "config_profile.json", ns["INPUT_IDX"], ns["OUTPUT_IDX"]) + ns["N_tx"], ns["N_rx"] = len(tx), len(rx) + for kind, positions in (("tx", tx), ("rx", rx)): + for axis, values in zip("xyz", ns["positions_to_xyz"](positions).T): + ns[f"{axis}_{kind}"] = values + antenna_x = np.concatenate((ns["x_tx"], ns["x_rx"])) + ns["x_grid_bp"] = np.linspace(antenna_x.min() - 2, antenna_x.max() + 2, ns["NX_BP"]) + sweep_time, _, _ = ns["estimate_sweep_time_from_meta"](directory, n_pairs=pair_count) + config = ns["MotionConfig"]( + speed_m_s=ns["SPEED_M_S"], look_angle_deg=ns["LOOK_ANGLE_DEG"], + tx_sweep_time_s=sweep_time, frame_acq_time_s=sweep_time * pair_count, + pair_order_phys=order, tx_order_full=tuple(tx_order), + n_freq_hint=len(request["latest"]["traces"][0]["frequency"]), + apply_pair_motion_correction=ns["APPLY_PAIR_MOTION_CORRECTION"], + apply_freq_phase_correction=ns["APPLY_FREQ_PHASE_CORRECTION"], + pair_motion_correction_mode=ns["PAIR_MOTION_CORRECTION_MODE"], + motion_phase_sign=ns["MOTION_PHASE_SIGN"], + ) + timestamps, _ = ns["compute_pair_timestamps_for_snapshot"](center, config) + result = ns["sar_compute_volume"](center, None, config, timestamps, verbose=False) + volume, grid = result["volume"], result["grid"] + iy, iz, ix = ns["sar_global_max_index"](volume, grid) + expected = {axis: grid[axis] for axis in "xyz"} + expected.update(front=volume.max(axis=1), side=volume.max(axis=2), top=volume.max(axis=0), + peak=np.array([result["level"], grid["x"][ix], grid["y"][iy], grid["z"][iz]])) + return expected, hashes + + +def generate(notebook: Path, output: Path) -> None: + # Read up front so a missing notebook cannot produce an incomplete artifact. + notebook.read_text(encoding="utf-8") + cases = [ + dict(compensation=True, normalize=True), + dict(compensation=True, normalize=False, speed=-.2, pitch=-7., ground=.5), + dict(compensation=False, normalize=True, f_start=1.3e9, f_stop=3.7e9), + dict(compensation=False, normalize=False, speed=0., pitch=0., radius=0), + ] + arrays, configurations = {}, [] + for index, settings in enumerate(cases): + request = scene() + request.update(settings) + for key in ("min_depth", "max_depth", "speed", "pitch", "ground", "voxel", "power"): + request[key] = float(np.float32(request[key])) + if index == 1: + # A separate, unequal capture window per output instead of interleaving. + for member in [*request["previous"], request["latest"]]: + for trace in member["traces"]: + if trace["output"] == 0: + trace["end"] = trace["start"] + 80_000_000 + else: + trace["start"] += 95_000_000 + with tempfile.TemporaryDirectory(prefix="sar3d_notebook_") as directory: + expected, hashes = reference(notebook, request, Path(directory)) + if index == 0: + arrays["request"] = np.array(json.dumps(request, separators=(",", ":"))) + configurations.append(dict( + settings={key: request[key] for key in settings}, + windows=[[[trace["start"], trace["end"]] for trace in member["traces"]] + for member in [*request["previous"], request["latest"]]], + )) + arrays.update({f"case_{index}_{key}": value for key, value in expected.items()}) + arrays["cases"] = np.array(json.dumps(configurations, separators=(",", ":"))) + arrays["provenance"] = np.array(json.dumps(dict( + notebook=notebook.name, cell_source_sha256=hashes, cells=[0, 4], cases=len(cases), + numpy=np.__version__, scipy=scipy.__version__, + method="Unchanged notebook AST functions; max projections of sar_compute_volume output", + ), sort_keys=True)) + output.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed(output, **arrays) + print(f"Wrote {output}: {output.stat().st_size} bytes, {len(cases)} notebook references") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--notebook", type=Path, default=ROOT / "Multi_antenna_3D_clean_v1.ipynb") + parser.add_argument("--output", type=Path, default=REFERENCE) + args = parser.parse_args() + generate(args.notebook, args.output) diff --git a/data_acq_and_processing/tests/native_regressions.cpp b/data_acq_and_processing/tests/native_regressions.cpp index ab5c2ec..924c2a8 100644 --- a/data_acq_and_processing/tests/native_regressions.cpp +++ b/data_acq_and_processing/tests/native_regressions.cpp @@ -2,10 +2,10 @@ #include "channel_bundle_support.hpp" #include "data_preprocessor.hpp" #include "locator/tcp_server.hpp" +#include "object_approach_filter.hpp" #include "processing_live_config.hpp" #include "runtime_wait.hpp" #include "shm_ring.hpp" -#include "sweep_orchestrator.hpp" #include "tty_protocol_writer.h" #include @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -65,9 +66,9 @@ struct RingName { std::string name; explicit RingName(const std::string& suffix = {}) : name("/radar_native_test_" + std::to_string(::getpid()) + suffix) { - ipc::ShmRing::unlink_ring(name); + ::shm_unlink(name.c_str()); } - ~RingName() { ipc::ShmRing::unlink_ring(name); } + ~RingName() { ::shm_unlink(name.c_str()); } }; auto save_bundle(const Workspace& workspace, const char* name, const ipc::RawSweepCollection& collection) @@ -84,13 +85,15 @@ void write_u32(std::vector& bytes, std::size_t offset, std::uint32 std::memcpy(bytes.data() + offset, &value, sizeof(value)); } -void test_wire_decoders() { +void test_wire_format() { ipc::RawSweepCollection raw{.collection_id = 7, .monotonic_ns = 42, .capture_start_ns = 21, .capture_end_ns = 40, .traces = {{.combo = {1, 2}, .frequency_hz = {1, 2}, - .s11 = {{1, 2}, {3, 4}}, .s21 = {{5, 6}, {7, 8}}, + .s21 = {{5, 6}, {7, 8}}, .capture_start_ns = 25, .capture_end_ns = 39}}}; - ipc::ResultCollection results{.collection_id = 7, .processing_duration_ns = 17, + ipc::ResultCollection results{.collection_id = 7, .monotonic_ns = 0x0102030405060708ULL, + .processing_duration_ns = 17, .settings_revision = 0xfedcba9876543210ULL, + .capture_start_ns = 21, .capture_end_ns = 40, .speed_m_s = -1.5F, .collection_payloads = { {.kind = ipc::ResultKind::TraceComplex, .frequency_hz = {1}, .trace = {{2, 3}}}, {.kind = ipc::ResultKind::ScalarF32, .scalar_value = 4}, @@ -103,32 +106,39 @@ void test_wire_decoders() { const auto raw_bytes = ipc::serialize_raw_collection(raw); const auto pre_bytes = ipc::serialize_preprocessed_collection(raw); const auto result_bytes = ipc::serialize_result_collection(results); + const std::array result_header{ + 'R', 'S', 'L', '3', 7, 0, 0, 0, 0, 0, 0, 0, + 8, 7, 6, 5, 4, 3, 2, 1, 17, 0, 0, 0, 0, 0, 0, 0, + 0x10, 0x32, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe, + 21, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0xc0, 0xbf, 6, 0, 0, 0, 1, 0, 0, 0, + }; + require(std::equal(result_header.begin(), result_header.end(), result_bytes.begin()), + "RSL3 must preserve exact acquisition identity and settings in its 64-byte header"); + require(std::memcmp(raw_bytes.data(), "RAW3", 4) == 0 && + std::memcmp(pre_bytes.data(), "PRP3", 4) == 0, + "trace collections must use the S21-only version 3 format"); + // Collection header + one trace header + two 12-byte points + capture trailer. + require(raw_bytes.size() == 24 + 12 + 2 * 12 + 16 + 4 + 16 && pre_bytes.size() == raw_bytes.size(), + "trace wire format must contain only frequency and S21 samples"); + auto old_raw = raw_bytes; + auto old_pre = pre_bytes; + write_u32(old_raw, 0, 0x32574152U); + write_u32(old_pre, 0, 0x32525050U); + expect_error([&] { (void)ipc::deserialize_raw_collection(old_raw); }); + expect_error([&] { (void)ipc::deserialize_preprocessed_collection(old_pre); }); require(ipc::serialize_raw_collection(ipc::deserialize_raw_collection(raw_bytes)) == raw_bytes, "raw roundtrip changed data"); require(ipc::serialize_preprocessed_collection(ipc::deserialize_preprocessed_collection(pre_bytes)) == pre_bytes, "preprocessed roundtrip changed data"); - require(ipc::serialize_result_collection(ipc::deserialize_result_collection(result_bytes)) == result_bytes, - "result roundtrip changed data or empty shape"); - - for (std::size_t size = 0; size < result_bytes.size(); ++size) { - expect_error([&] { (void)ipc::deserialize_result_collection(std::span(result_bytes).first(size)); }); - } - for (std::size_t count_offset : {28U, 32U, 39U}) { - auto corrupt = result_bytes; - write_u32(corrupt, count_offset, std::numeric_limits::max()); - expect_error([&] { (void)ipc::deserialize_result_collection(corrupt); }); - } - for (auto kind : {ipc::ResultKind::ImageF32, ipc::ResultKind::TableF32}) { - ipc::ResultCollection empty{.collection_payloads = {{.kind = kind}}}; - auto corrupt = ipc::serialize_result_collection(empty); - write_u32(corrupt, 39, std::numeric_limits::max()); - write_u32(corrupt, 43, std::numeric_limits::max()); - expect_error([&] { (void)ipc::deserialize_result_collection(corrupt); }); + for (const std::size_t tail : {36U, 20U}) { + expect_error([&] { (void)ipc::deserialize_raw_collection(std::span(raw_bytes).first(raw_bytes.size() - tail)); }); + expect_error([&] { (void)ipc::deserialize_preprocessed_collection(std::span(pre_bytes).first(pre_bytes.size() - tail)); }); } // Deterministic random mutations include wire-compatible valid frames as seeds. std::mt19937 random(0x52414452U); for (unsigned iteration = 0; iteration < 10'000U; ++iteration) { - auto bytes = iteration % 3U == 0U ? raw_bytes : iteration % 3U == 1U ? pre_bytes : result_bytes; + auto bytes = iteration % 2U == 0U ? raw_bytes : pre_bytes; for (unsigned edit = 0; edit < 1U + random() % 5U; ++edit) { bytes[random() % bytes.size()] = static_cast(random()); } @@ -136,9 +146,8 @@ void test_wire_decoders() { bytes.resize(random() % bytes.size()); } try { - if (iteration % 3U == 0U) { (void)ipc::deserialize_raw_collection(bytes); } - else if (iteration % 3U == 1U) { (void)ipc::deserialize_preprocessed_collection(bytes); } - else { (void)ipc::deserialize_result_collection(bytes); } + if (iteration % 2U == 0U) { (void)ipc::deserialize_raw_collection(bytes); } + else { (void)ipc::deserialize_preprocessed_collection(bytes); } } catch (const std::bad_alloc&) { throw; } catch (const std::runtime_error&) {} } @@ -151,12 +160,35 @@ void test_configuration(const std::string& seed_path, const Workspace& workspace const auto save = [&](const Json& config) { std::ofstream(config_path) << config; }; save(seed); const auto config = config::load_run_config(config_path.string()); + auto no_taps = seed; + no_taps["rings"]["preprocessed_tap"]["name"] = ""; + save(no_taps); + const auto disabled = config::load_run_config(config_path.string()).rings; + require(disabled.preprocessed_tap.name.empty(), + "explicit empty tap names must disable the optional rings"); + for (const auto* key : {"raw", "preprocessed", "results"}) { + auto invalid = no_taps; + invalid["rings"][key]["name"] = ""; + save(invalid); + expect_error([&] { (void)config::load_run_config(config_path.string()); }); + } + for (const auto* port : {"port1", "port2"}) { + auto with_driver = seed; + with_driver["switches"][port]["driver"] = "h7992"; + save(with_driver); + (void)config::load_run_config(config_path.string()); + for (const auto& driver : {Json("hmc349a"), Json("unknown"), Json(7)}) { + with_driver["switches"][port]["driver"] = driver; + save(with_driver); + expect_error([&] { (void)config::load_run_config(config_path.string()); }); + } + } auto with_object_calibration = seed; with_object_calibration["preprocess"]["s21"]["object_calibration"] = Json{{"set_name", "reflector"}, {"bundle_path", "corrections.bin"}}; save(with_object_calibration); const auto selected = config::load_run_config(config_path.string()).preprocess.s21.object_calibration; - require(selected.set_name == "reflector" && selected.bundle_path == "corrections.bin", + require(selected.bundle_path == "corrections.bin", "object calibration asset did not load"); with_object_calibration["preprocess"]["s21"].erase("object_calibration"); save(with_object_calibration); @@ -167,10 +199,8 @@ void test_configuration(const std::string& seed_path, const Workspace& workspace save(with_object_calibration); expect_error([&] { (void)config::load_run_config(config_path.string()); }); } - for (const auto& [pointer, value] : std::array, 8>{{ - {"/radar/sweep/start_hz", -1}, {"/radar/sweep/stop_hz", 1e100}, - {"/radar/sweep/if_bandwidth_hz", 0}, {"/radar/sweep/points", 0}, - {"/radar/sweep/points", 4294967296.0}, {"/run/combos/0/input", 0.0000001}, + for (const auto& [pointer, value] : std::array, 3>{{ + {"/run/combos/0/input", 0.0000001}, {"/rings/raw/slot_size_bytes", 65}, {"/gpr/rx_geometry/0/x_m", 1e100}}}) { auto invalid = seed; invalid[Json::json_pointer(pointer)] = value; @@ -190,11 +220,6 @@ void test_configuration(const std::string& seed_path, const Workspace& workspace require(current.bscan_gain == 2 && loader.revision() == revision, "invalid live config replaced the last valid settings"); } - const std::array nan_axis{std::numeric_limits::quiet_NaN()}; - require(!preprocessing::frequency_axes_match(nan_axis, nan_axis), "NaN frequency axes matched"); - preprocessing::ChannelTrace invalid_trace{.frequency_hz = {1}, - .samples = {{std::numeric_limits::infinity(), 0}}}; - expect_error([&] { preprocessing::validate_channel_trace_layout(invalid_trace, "test", {}); }); } void test_ring() { @@ -245,38 +270,16 @@ void test_ring() { } } -void test_partial_driver_open() { - struct Radar final : drivers::RadarDriver { - unsigned closes = 0; - void open() override {} - void close() override { ++closes; } - auto acquire_sweep() -> drivers::SweepTrace override { return {}; } - } radar; - struct Switch final : drivers::SwitchDriver { - std::atomic& stop; - bool fail_open = false; - bool fail_close = false; - unsigned closes = 0; - explicit Switch(std::atomic& stop_flag) : stop(stop_flag) {} - void open() override { - if (fail_open) { stop.store(true); throw std::runtime_error("open failure"); } - } - void close() override { ++closes; if (fail_close) { throw std::runtime_error("close failure"); } } - auto position_count() const -> std::uint32_t override { return 1; } - void switch_to(std::uint32_t) override {} - }; - std::atomic stop{false}; - Switch input(stop), output(stop); - input.fail_open = true; - output.fail_close = true; - config::RunConfig config; - config.radar.sweep.points = 1; - config.run_combos = {{0, 0}}; - RingName name; - auto ring = ipc::ShmRing::open_or_create(name.name, 2, 256); - acq::SweepOrchestrator(config, radar, input, output, ring).run(stop); - require(radar.closes == 1 && input.closes == 1 && output.closes == 1, - "partial driver startup did not close each driver exactly once"); +void test_acquisition_identity() { + processing::ObjectApproachFilter filter; + const std::vector points{{0, 2}}; + require(!filter.confirm(points, 1, 1'000'000'000ULL, 0, 0, 2).front(), "first frame confirmed a track"); + require(filter.confirm(points, 1, 2'000'000'000ULL, 0, 0, 2).front(), + "reused collection id replaced a distinct acquisition"); + require(filter.confirm(points, 1, 2'000'000'000ULL, 0, 0, 2).front(), + "reprocessing duplicated a frame or lost its predecessor"); + require(!filter.confirm(points, 7, 1'000'000'000ULL, 0, 0, 2).front(), + "an earlier capture time failed to reset replay history"); } void test_interruptible_wait() { @@ -291,6 +294,19 @@ void test_interruptible_wait() { "large wait delayed shutdown"); } +void test_cosine_notch_taper() { + // Band 2.0-2.1 GHz with 0.1 GHz shoulders: 1 at the outer edges, 0.5 halfway, + // 0 from the band edges inwards. + const std::vector frequency_hz{1.85e9F, 1.9e9F, 1.95e9F, 2.0e9F, 2.05e9F, + 2.1e9F, 2.15e9F, 2.2e9F, 2.25e9F}; + const std::vector expected{1.0F, 1.0F, 0.5F, 0.0F, 0.0F, 0.0F, 0.5F, 1.0F, 1.0F}; + const auto mask = preprocessing::build_notch_mask(frequency_hz, + {.enabled = true, .bands_hz = {{2.0e9F, 2.1e9F}}, .taper_width_hz = 1e8F, .taper_type = "cosine"}); + for (std::size_t index = 0U; index < expected.size(); ++index) { + require(std::abs(mask[index] - expected[index]) < 1e-3F, "cosine notch shoulders are not 1 -> 0 towards the band"); + } +} + void test_csv_failures(const Workspace& workspace) { const auto csv = workspace.path / "capture.csv"; const auto spool = workspace.path / "capture.capture_spool.bin"; @@ -328,37 +344,32 @@ void test_csv_failures(const Workspace& workspace) { void test_asset_reload_preserves_data(const Workspace& workspace) { const auto save = [&](const char* name, float sample) { return save_bundle(workspace, name, {.traces = { - {.frequency_hz = {1}, .s11 = {{sample, 0}}, .s21 = {{sample, 0}}}}}); + {.frequency_hz = {1}, .s21 = {{sample, 0}}}}}); }; const auto open = save("open.bin", 1); - const auto short_standard = save("short.bin", -1); - const auto load = save("load.bin", 0); const auto nan = save("nan.bin", std::numeric_limits::quiet_NaN()); - preprocessing::S21ReferenceBundle s21_reference; - s21_reference.load(open); - expect_error([&] { s21_reference.load(nan); }); - require(s21_reference.resolve({}).samples.front().re == 1, "failed S21 reload lost previous reference"); preprocessing::S21CalibrationBundle s21_calibration; - s21_calibration.load(open); + s21_calibration.load(save("through.bin", 2)); expect_error([&] { s21_calibration.load(nan); }); - require(s21_calibration.is_enabled(), "failed S21 reload disabled previous calibration"); - preprocessing::S11ReferenceBundle s11_reference; - s11_reference.load(open); - expect_error([&] { s11_reference.load(nan); }); - require(s11_reference.resolve({}).samples.front().re == 1, "failed S11 reload lost previous reference"); - preprocessing::S11CalibrationBundle s11_calibration; - s11_calibration.load(open, short_standard, load); - expect_error([&] { s11_calibration.load(open, open, load); }); - const std::array axis{1}; - require(s11_calibration.apply({}, axis, {{0.5, 0}}).front().re == 0.5F, - "failed OSL reload lost previous calibration"); + const std::array frequency{1}; + std::array samples{{{6, 8}}}; + s21_calibration.apply({}, frequency, samples); + require(samples[0].re == 3 && samples[0].im == 4, "failed S21 reload changed previous calibration"); + const std::array nan_axis{std::numeric_limits::quiet_NaN()}; + expect_error([&] { s21_calibration.apply({}, nan_axis, samples); }); + preprocessing::S21ReferenceBundle reference; + reference.load(open, s21_calibration, {{}}); + expect_error([&] { reference.load(nan, s21_calibration, {{}}); }); + ipc::SweepTraceBlock trace{.frequency_hz = {1}, .s21 = {{2, 0}}}; + reference.apply(trace); + require(trace.s21[0].re == 1.5F, "failed reference reload changed the calibrated reference"); } void test_object_calibration(const Workspace& workspace) { const ipc::SweepTraceBlock first{.combo = {7, 3}, .frequency_hz = {1e9F, 2e9F}, - .s11 = {{1, 2}, {3, 4}}, .s21 = {{2, 3}, {4, 5}}, .capture_start_ns = 5, .capture_end_ns = 15}; + .s21 = {{2, 3}, {4, 5}}, .capture_start_ns = 5, .capture_end_ns = 15}; const ipc::SweepTraceBlock second{.combo = {1, 9}, .frequency_hz = {3e9F, 4e9F, 5e9F}, - .s11 = {{}, {}, {}}, .s21 = {{6, 7}, {8, 9}, {10, 11}}}; + .s21 = {{6, 7}, {8, 9}, {10, 11}}}; auto factors_first = first; factors_first.s21 = {{0, 2}, {2, 0}}; auto factors_second = second; @@ -390,7 +401,7 @@ void test_object_calibration(const Workspace& workspace) { trace = first; correction.apply(trace); require(ipc::serialize_raw_collection({.traces = {trace}}) == ipc::serialize_raw_collection({.traces = {expected}}), - "object calibration changed S11, timing, pair identity, or multiplication after a failed reload"); + "object calibration changed timing, pair identity, or multiplication after a failed reload"); trace = second; correction.apply(trace); require(trace.s21[0].re == 13 && trace.s21[0].im == 1 && trace.s21[1].re == -8 && trace.s21[1].im == -9 && @@ -406,19 +417,18 @@ void test_object_calibration(const Workspace& workspace) { void test_object_calibration_pipeline(const Workspace& workspace) { const ipc::SweepTraceBlock trace{.combo = {7, 3}, .frequency_hz = {1e9F, 2e9F}, - .s11 = {{7, 3}, {8, 4}}, .s21 = {{2, 4}, {6, 8}}, .capture_start_ns = 5, .capture_end_ns = 15}; + .s21 = {{2, 4}, {6, 8}}, .capture_start_ns = 5, .capture_end_ns = 15}; auto standard = trace; standard.s21 = {{2, 0}, {2, 0}}; auto reference = trace; reference.s21 = {{1, 2}, {2, 2}}; auto factors = trace; factors.s21 = {{0, 2}, {5, 1}}; - preprocessing::CalibrationMaster calibration(preprocessing::make_s21_through_calibrator()); - calibration.load_s21_calibration_bundle(save_bundle(workspace, "pipeline_through.bin", {.traces = {standard}})); - preprocessing::ReferenceMaster references; - references.load_s21_reference_bundle(save_bundle(workspace, "pipeline_reference.bin", {.traces = {reference}})); - references.prepare_calibrated(calibration, {trace.combo}); config::RunConfig config; + config.preprocess.s21.calibration.bundle_path = + save_bundle(workspace, "pipeline_through.bin", {.traces = {standard}}); + config.preprocess.s21.reference.bundle_path = + save_bundle(workspace, "pipeline_reference.bin", {.traces = {reference}}); config.run_combos = {trace.combo}; config.preprocess.s21.object_calibration.bundle_path = save_bundle(workspace, "pipeline_object.bin", {.traces = {factors}}); @@ -426,7 +436,7 @@ void test_object_calibration_pipeline(const Workspace& workspace) { RingName raw_name("_object_raw"), pre_name("_object_pre"); auto raw_ring = ipc::ShmRing::open_or_create(raw_name.name, 2, 4096); auto pre_ring = ipc::ShmRing::open_or_create(pre_name.name, 2, 4096); - preprocessing::DataPreprocessor preprocessor(config, calibration, references, raw_ring, pre_ring); + preprocessing::DataPreprocessor preprocessor(config, raw_ring, pre_ring); require(raw_ring.push(ipc::serialize_raw_collection({.collection_id = 41, .capture_start_ns = 5, .capture_end_ns = 15, .traces = {trace}})), "test raw ring could not hold a sweep"); std::atomic stop{false}; @@ -449,8 +459,7 @@ void test_object_calibration_pipeline(const Workspace& workspace) { "object calibration lost capture identity or timing"); require(output.s21[0].re == -2 && output.s21[0].im == 1 && output.s21[1].re == 0 && output.s21[1].im == 0, "object calibration ran before existing preprocessing or was applied more than once"); - require(output.s11[0].re == 7 && output.s11[0].im == 3 && output.s11[1].re == 0 && output.s11[1].im == 0, - "object calibration changed S11 or bypassed the notch filter"); + } void test_locator_packet_errors() { @@ -506,14 +515,15 @@ int main(int argc, char** argv) { try { require(argc == 2, "Pass the simulator config fixture path"); Workspace workspace; - test_wire_decoders(); + test_wire_format(); test_configuration(argv[1], workspace); test_asset_reload_preserves_data(workspace); test_object_calibration(workspace); test_object_calibration_pipeline(workspace); test_ring(); - test_partial_driver_open(); + test_acquisition_identity(); test_interruptible_wait(); + test_cosine_notch_taper(); test_csv_failures(workspace); test_locator_packet_errors(); test_unread_tty_shutdown(workspace); diff --git a/data_acq_and_processing/tests/sar3d_fixture.cpp b/data_acq_and_processing/tests/sar3d_fixture.cpp index 6fdf010..36a3be4 100644 --- a/data_acq_and_processing/tests/sar3d_fixture.cpp +++ b/data_acq_and_processing/tests/sar3d_fixture.cpp @@ -1,7 +1,9 @@ #include "sar3d_processor.hpp" -#include "../processing/processors/src/sar3d_gpu.hpp" +#include +#include #include +#include #include using Json = nlohmann::json; @@ -29,60 +31,57 @@ auto collection(const Json& source) -> ipc::PreprocessedCollection { for (const auto& sample : complex_samples(entry.at("s21"))) { trace.s21.push_back({sample.real(), sample.imag()}); } + if (result.traces.empty() || trace.capture_start_ns < result.capture_start_ns) { + result.capture_start_ns = trace.capture_start_ns; + } + result.capture_end_ns = std::max(result.capture_end_ns, trace.capture_end_ns); result.traces.push_back(std::move(trace)); } return result; } } +// Runs the sar3d processor once: the JSON request on stdin holds the antenna +// geometry, the live settings and the latest and previous collections; the +// published payloads go to stdout as JSON. int main() { try { const Json input = Json::parse(std::cin); - Json output; - if (input.at("operation") == "processor") { - config::RunConfig config; - for (const auto& row : input.at("tx")) { - config.gpr.tx_geometry.push_back({row.at(0), row.at(1), row.at(2), row.at(3)}); - } - for (const auto& row : input.at("rx")) { - config.gpr.rx_geometry.push_back({row.at(0), row.at(1), row.at(2), row.at(3)}); - } - processing::ProcessingLiveConfig live; - live.gpr_start_freq_mhz = input.at("f_start").get() / 1e6F; - live.gpr_stop_freq_mhz = input.at("f_stop").get() / 1e6F; - live.gpr_min_depth_m = input.at("min_depth"); - live.gpr_max_depth_m = input.at("max_depth"); - live.gpr_speed_m_s = input.at("speed"); - live.gpr_look_angle_deg = input.at("pitch"); - live.sar3d_voxel_m = input.at("voxel"); - live.sar3d_bp_ground_depth_m = input.at("ground"); - live.sar3d_frame_radius = input.at("radius"); - live.sar3d_comp_range_power = input.at("power"); - live.sar3d_use_compensation = input.at("compensation"); - live.sar3d_normalize_each_frame = input.at("normalize"); - std::vector previous; - for (const auto& member : input.at("previous")) { - previous.push_back(collection(member)); - } - processing::Sar3dProcessor processor; - const auto result = processor.process_collection(config, collection(input.at("latest")), previous, live); - output["payloads"] = Json::array(); - for (const auto& payload : result.collection_payloads) { - output["payloads"].push_back({{"name", payload.processing_name}, - {"x", payload.image_x_axis}, {"y", payload.image_y_axis}, - {"image", payload.image_values}, {"table", payload.table_values}, - {"columns", payload.table_columns}}); - } - } else { - processing::sar3d::Engine engine(1.5); - const auto samples = engine.inverse_dft(complex_samples(input.at("bands")), - input.at("points"), input.at("first_bin"), input.at("size")); - output["ascans"] = Json::array(); - for (const auto& sample : samples) { - output["ascans"].push_back({sample.real(), sample.imag()}); - } + config::RunConfig config; + config.rings.results.slot_size_bytes = input.value("slot", config.rings.results.slot_size_bytes); + for (const auto& row : input.at("tx")) { + config.gpr.tx_geometry.push_back({row.at(0), row.at(1), row.at(2), row.at(3)}); } - std::cout << output << '\n'; + for (const auto& row : input.at("rx")) { + config.gpr.rx_geometry.push_back({row.at(0), row.at(1), row.at(2), row.at(3)}); + } + processing::ProcessingLiveConfig live; + live.gpr_start_freq_mhz = input.at("f_start").get() / 1e6F; + live.gpr_stop_freq_mhz = input.at("f_stop").get() / 1e6F; + live.gpr_min_depth_m = input.at("min_depth"); + live.gpr_max_depth_m = input.at("max_depth"); + live.gpr_speed_m_s = input.at("speed"); + live.gpr_look_angle_deg = input.at("pitch"); + live.sar3d_voxel_m = input.at("voxel"); + live.sar3d_bp_ground_depth_m = input.at("ground"); + live.sar3d_frame_radius = input.at("radius"); + live.sar3d_comp_range_power = input.at("power"); + live.sar3d_use_compensation = input.at("compensation"); + live.sar3d_normalize_each_frame = input.at("normalize"); + std::vector previous; + for (const auto& member : input.at("previous")) { + previous.push_back(collection(member)); + } + processing::Sar3dProcessor processor; + const auto result = processor.process_collection(config, collection(input.at("latest")), previous, live); + Json payloads = Json::array(); + for (const auto& payload : result.collection_payloads) { + payloads.push_back({{"name", payload.processing_name}, + {"x", payload.image_x_axis}, {"y", payload.image_y_axis}, + {"image", payload.image_values}, {"table", payload.table_values}, + {"columns", payload.table_columns}}); + } + std::cout << Json{{"payloads", payloads}} << '\n'; } catch (const std::exception& error) { std::cout << Json{{"error", error.what()}} << '\n'; return 1; diff --git a/data_acq_and_processing/tests/sar3d_kernel_emulation.cpp b/data_acq_and_processing/tests/sar3d_kernel_emulation.cpp deleted file mode 100644 index c8469a8..0000000 --- a/data_acq_and_processing/tests/sar3d_kernel_emulation.cpp +++ /dev/null @@ -1,136 +0,0 @@ -// Test-only execution of the exact CUDA kernel source, one logical thread at a time. -// Linked solely into sar3d_math_fixture; never used as a production backend. -#include "../processing/processors/src/sar3d_gpu.hpp" - -#include -#include -#include -#include -#include -#include -#include - -#define __global__ -using std::isfinite; -using std::min; -struct Index { unsigned x; }; -Index blockIdx{0}, blockDim{1}, threadIdx{0}; -struct float2 { float x, y; }; -auto make_float2(float x, float y) -> float2 { return {x, y}; } -void sincospif(float x, float* sine, float* cosine) { - *sine = std::sin(x * 3.14159265358979323846F); - *cosine = std::cos(x * 3.14159265358979323846F); -} -auto __powf(float x, float y) -> float { return std::pow(x, y); } -auto __float_as_uint(float x) -> unsigned int { return std::bit_cast(x); } -auto __double2int_rn(double x) -> int { return static_cast(std::nearbyint(x)); } -void atomicMax(unsigned int* destination, unsigned int value) { *destination = std::max(*destination, value); } -#include "sar3d_kernels.inc" - -auto probe_ascan_upper_bound() -> float { - blockIdx.x = 0; - const double x[]{1073741824.0}, zero[]{0.0}; - const ::Contribution contribution{{}, {}, 1.0F, 1.0F}; - float volume = -1.0F; - ::backproject(nullptr, INT_MAX, &contribution, 1, x, zero, zero, 1, 1, 1, - 1.0, 0.0, 0.0, 0.0, 1.0, 1.0F, 0, 0.0F, 0.0F, &volume); - return volume; -} - -namespace { -template -void each(int count, Operation operation) { - for (blockIdx.x = 0; blockIdx.x < static_cast(count); ++blockIdx.x) { operation(); } -} - -auto float_pairs(std::span> values) -> std::vector { - std::vector pairs; - pairs.reserve(values.size()); - for (const auto value : values) { pairs.push_back({value.real(), value.imag()}); } - return pairs; -} -} - -namespace radar::processing::sar3d { -namespace { -#include "sar3d_helpers.inc" -} - -struct Engine::Impl { std::vector taps; }; -Engine::Engine(double sigma) : impl_(std::make_unique()) { impl_->taps = gaussian_taps(sigma); } -Engine::~Engine() = default; - -auto Engine::inverse_dft(std::span> bands, std::size_t points, - std::size_t first_bin, std::size_t size) -> std::vector> { - if (points == 0U || bands.empty() || bands.size() % points != 0U || !std::has_single_bit(size) || - size > kMaxIndexedItems || first_bin >= size || points > size - first_bin) { - throw std::runtime_error("sar3d invalid inverse FFT dimensions"); - } - const auto count = checked_product(bands.size() / points, size, kMaxIndexedItems, "A-scan stack"); - const auto input = float_pairs(bands); - std::vector output(count); - each(static_cast(count), [&] { - ::inverse_dft(input.data(), points, first_bin, size, count, output.data()); - }); - std::vector> result; - result.reserve(count); - for (const auto value : output) { result.emplace_back(value.x, value.y); } - return result; -} - -auto Engine::project(const VolumeRequest& request) -> Projections { - const int nx = request.x.size(), ny = request.y.size(), nz = request.z.size(); - const int plane = checked_product(nx, nz, kMaxIndexedItems, "grid plane"); - const int voxels = checked_product(plane, ny, kMaxIndexedItems, "grid"); - std::vector<::Contribution> contributions; - for (const auto& term : request.contributions) { - contributions.push_back(std::bit_cast<::Contribution>(term)); - } - const auto ascans = float_pairs(request.ascans); - std::vector volume(voxels), scratch(voxels), levels(ny); - std::vector indices(ny); - each(voxels, [&] { - ::backproject(ascans.data(), request.samples_per_ascan, contributions.data(), contributions.size(), - request.x.data(), request.y.data(), request.z.data(), nx, nz, voxels, - std::cos(request.pitch_rad), std::sin(request.pitch_rad), request.ground_depth_m, - request.min_depth_m, request.max_depth_m, request.samples_per_metre, - request.use_compensation, request.comp_range_power, request.comp_weight_max, volume.data()); - }); - each(ny, [&] { ::peak_planes(volume.data(), plane, ny, levels.data(), indices.data()); }); - Projections raw; - choose_peak(raw, request, levels); - const int radius = impl_->taps.size() / 2; - each(voxels, [&] { - ::smooth(volume.data(), scratch.data(), impl_->taps.data(), radius, nx, nz, voxels, 1); - }); - each(voxels, [&] { - ::smooth(scratch.data(), volume.data(), impl_->taps.data(), radius, nx, nz, voxels, 0); - }); - each(ny, [&] { ::peak_planes(volume.data(), plane, ny, levels.data(), indices.data()); }); - Projections result; - result.raw_level = raw.level; - choose_peak(result, request, levels, indices); - result.front.resize(ny * nx); - result.side.resize(ny * nz); - result.top.resize(plane); - const auto& view = request.oblique; - std::vector oblique(checked_product(view.nu, view.nv, kMaxIndexedItems, "oblique image")); - each(result.front.size(), [&] { - ::max_project(volume.data(), result.front.data(), result.front.size(), nx, plane, 1, nz, nx); - }); - each(result.side.size(), [&] { - ::max_project(volume.data(), result.side.data(), result.side.size(), nz, plane, nx, nx, 1); - }); - each(result.top.size(), [&] { - ::max_project(volume.data(), result.top.data(), result.top.size(), nx, nx, 1, ny, plane); - }); - each(voxels, [&] { - ::project_oblique(volume.data(), request.x.data(), request.y.data(), request.z.data(), nx, nz, voxels, - view.u[0], view.u[1], view.u[2], view.v[0], view.v[1], view.v[2], view.u0, view.v0, - 1 / view.pixel_m, view.nu, view.nv, oblique.data()); - }); - result.oblique.reserve(oblique.size()); - for (const auto value : oblique) { result.oblique.push_back(std::bit_cast(value)); } - return result; -} -} diff --git a/data_acq_and_processing/tests/sar3d_kernel_regressions.cpp b/data_acq_and_processing/tests/sar3d_kernel_regressions.cpp deleted file mode 100644 index f77c5d6..0000000 --- a/data_acq_and_processing/tests/sar3d_kernel_regressions.cpp +++ /dev/null @@ -1,85 +0,0 @@ -// Kernel arithmetic checks linked only with the host emulation fixture. -#include "../processing/processors/src/sar3d_gpu.hpp" - -#include -#include -#include -#include -#include - -extern "C" void peak_planes(const float*, int, int, float*, unsigned int*); -struct Index { unsigned x; }; -extern Index blockIdx; -auto probe_ascan_upper_bound() -> float; - -namespace { -void rejects_unsigned_padding() { - blockIdx.x = 0x80000000U; - peak_planes(nullptr, 1, 1, nullptr, nullptr); -} - -void rejects_rounded_sample_bound() { - if (probe_ascan_upper_bound() != 0.0F) { - throw std::runtime_error("out-of-range float A-scan position was accepted"); - } -} - -void rejects_nan_peak() { - blockIdx.x = 0; - const float volume[]{std::numeric_limits::quiet_NaN(), 0.0F}; - float level = -1.0F; - unsigned int index = 0U; - peak_planes(volume, 2, 1, &level, &index); - if (std::isfinite(level)) { throw std::runtime_error("peak kernel silently ignored NaN"); } -} - -void keeps_first_joint_peak() { - blockIdx.x = 0; - const float volume[]{0, 1, 0, 1, 0, 0}; - float level = 0; - unsigned int index = 0U; - peak_planes(volume, 6, 1, &level, &index); - if (level != 1.0F || index != 1U) { throw std::runtime_error("joint peak tie chose a nonexistent voxel"); } -} - -void rejects_finite_coherent_overflow() { - using namespace radar::processing::sar3d; - Engine engine(1.5); - const std::vector x{-1, 1}, y{-.5, .5}, z{2, 3}; - std::vector terms(3); - for (auto& term : terms) { term.weight = 1; term.inv_reference_ranges = 1; } - std::vector> scans(3 * 64); - for (int i = 0; i < 64; ++i) { - scans[i] = {1e38F, 0}; - scans[64 + i] = {-1e38F, 0}; - scans[128 + i] = {1, 0}; - } - const VolumeRequest request{ - .x = x, .y = y, .z = z, .min_depth_m = 1, .max_depth_m = 4, - .contributions = terms, .ascans = scans, .samples_per_ascan = 64, .samples_per_metre = 1, - .use_compensation = true, .comp_range_power = 1, .comp_weight_max = 5, .norm_edge_margin_m = .3, - .oblique = {.u = {1, 0, 0}, .v = {0, 0, 1}, .u0 = -1, .v0 = 2, .pixel_m = 1, .nu = 3, .nv = 2}, - }; - try { - (void)engine.project(request); - } catch (const std::runtime_error& error) { - if (std::string_view(error.what()).find("non-finite") == std::string_view::npos) { throw; } - return; - } - throw std::runtime_error("finite coherent overflow was silently accepted as no signal"); -} -} - -int main() { - try { - rejects_unsigned_padding(); - rejects_rounded_sample_bound(); - rejects_nan_peak(); - keeps_first_joint_peak(); - rejects_finite_coherent_overflow(); - std::cout << "SAR CUDA kernel arithmetic regressions passed\n"; - } catch (const std::exception& error) { - std::cerr << error.what() << '\n'; - return 1; - } -} diff --git a/data_acq_and_processing/tests/sar3d_reference.npz b/data_acq_and_processing/tests/sar3d_reference.npz new file mode 100644 index 0000000..d6af539 Binary files /dev/null and b/data_acq_and_processing/tests/sar3d_reference.npz differ diff --git a/data_acq_and_processing/tests/test_sar3d.py b/data_acq_and_processing/tests/test_sar3d.py new file mode 100644 index 0000000..2ffc645 --- /dev/null +++ b/data_acq_and_processing/tests/test_sar3d.py @@ -0,0 +1,139 @@ +"""The sar3d processor on the real CUDA backend: notebook reference and contracts. + +Every case runs build/native_tests/sar3d_fixture (SAR3D_FIXTURE overrides it). +Without a usable GPU the processor publishes sar3d_status 3: then only that +warning is checked and the remaining tests are skipped. The notebook is needed +only to regenerate sar3d_reference.npz with generate_sar3d_reference.py. +""" +from __future__ import annotations + +import copy +import json +import os +from pathlib import Path +import subprocess +import unittest + +import numpy as np + +ROOT = Path(__file__).resolve().parents[2] +FIXTURE = Path(os.environ.get("SAR3D_FIXTURE", ROOT / "build/native_tests/sar3d_fixture")) +REFERENCE = Path(__file__).with_name("sar3d_reference.npz") +VIEWS = ("sar3d_front", "sar3d_side", "sar3d_top", "sar3d_oblique") + + +def run(request: dict) -> dict[str, dict]: + output = subprocess.run([str(FIXTURE)], input=json.dumps(request), text=True, + capture_output=True, timeout=30, check=True) + return {item["name"]: item for item in json.loads(output.stdout)["payloads"]} + + +def status(payloads: dict[str, dict]) -> tuple[int, str]: + table = payloads["sar3d_status"]["table"] + return int(table[0]), bytes(map(int, table[1:])).decode("utf-8") + + +def reference_cases(archive) -> list[dict]: + """Requests of the reference archive, one per notebook parameter case.""" + cases = [] + for case in json.loads(archive["cases"].item()): + request = json.loads(archive["request"].item()) + request.update(case["settings"]) + for member, windows in zip([*request["previous"], request["latest"]], case["windows"], strict=True): + for trace, (start, end) in zip(member["traces"], windows, strict=True): + trace.update(start=start, end=end) + cases.append(request) + return cases + + +class Sar3dTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + with np.load(REFERENCE, allow_pickle=False) as archive: + cls.reference = {key: archive[key] for key in archive.files} + cls.cases = reference_cases(cls.reference) + cls.scene_payloads = run(cls.cases[0]) + code, message = status(cls.scene_payloads) + cls.unavailable = message if code == 3 else None + + def scene(self, **settings) -> dict: + if self.unavailable: + self.skipTest(self.unavailable) + request = copy.deepcopy(self.cases[0]) + request.update(settings) + return request + + def test_unavailable_gpu_is_a_visible_warning(self) -> None: + code, message = status(self.scene_payloads) + if self.unavailable: + self.assertIn("CUDA", message) + self.assertFalse(any(item["image"] for item in self.scene_payloads.values())) + else: + self.assertEqual(code, 0, message) + + def test_cuda_matches_notebook(self) -> None: + if self.unavailable: + self.skipTest(self.unavailable) + for index, request in enumerate(self.cases): + with self.subTest(case=index): + payloads = run(request) + self.assertEqual(status(payloads)[0], 0, status(payloads)) + for view, horizontal, vertical in (("front", "x", "y"), ("side", "z", "y"), ("top", "x", "z")): + payload = payloads[f"sar3d_{view}"] + expected = self.reference[f"case_{index}_{view}"] + np.testing.assert_allclose(payload["x"], self.reference[f"case_{index}_{horizontal}"], + rtol=2e-6, atol=1e-6) + np.testing.assert_allclose(payload["y"], self.reference[f"case_{index}_{vertical}"], + rtol=2e-6, atol=1e-6) + np.testing.assert_allclose(np.asarray(payload["image"]).reshape(expected.shape), + expected, rtol=2e-3, atol=3e-4) + peak = payloads["sar3d_peak"]["table"] + expected_peak = self.reference[f"case_{index}_peak"] + np.testing.assert_allclose(peak[1:], expected_peak[1:], rtol=0, atol=2e-6) + np.testing.assert_allclose(peak[0], expected_peak[0], rtol=2e-3, atol=0) + + def test_invalid_input_is_an_error_with_a_reason(self) -> None: + def no_traces(request): request["latest"]["traces"] = [] + def empty_window(request): request["latest"]["traces"][0]["end"] = request["latest"]["traces"][0]["start"] + def falling_frequency(request): request["latest"]["traces"][0]["frequency"].reverse() + for name, settings, damage in ( + ("no traces", {}, no_traces), + ("empty capture window", {}, empty_window), + ("falling frequency", {}, falling_frequency), + ("grid beyond 32-bit indexing", {"voxel": 1e-8}, None), + ("vertical look angle", {"pitch": 90}, None), + ("result larger than the ring slot", {"slot": 1024}, None), + ): + with self.subTest(name): + request = self.scene(**settings) + if damage: + damage(request) + code, message = status(run(request)) + self.assertEqual(code, 2, message) + self.assertTrue(message) + + def test_empty_signal_is_reported_without_a_peak(self) -> None: + request = self.scene(previous=[], radius=0) + for trace in request["latest"]["traces"]: + trace["s21"] = [[0, 0]] * len(trace["frequency"]) + payloads = run(request) + self.assertEqual(status(payloads)[0], 1) + self.assertNotIn("sar3d_peak", payloads) + for name in VIEWS: + self.assertFalse(np.any(payloads[name]["image"])) + + def test_weak_signal_stays_finite(self) -> None: + request = self.scene(previous=[], radius=0, normalize=False) + for trace in request["latest"]["traces"]: + trace["s21"] = (np.asarray(trace["s21"]) * 1e-38).tolist() + payloads = run(request) + self.assertEqual(status(payloads)[0], 0, status(payloads)) + for name in VIEWS: + values = np.asarray(payloads[name]["image"]) + self.assertTrue(np.isfinite(values).all()) + self.assertGreater(values.max(), 0) + self.assertLess(values.max(), 1e-10) + + +if __name__ == "__main__": + unittest.main() diff --git a/data_acq_and_processing/tests/test_sar3d_oracle.py b/data_acq_and_processing/tests/test_sar3d_oracle.py deleted file mode 100644 index 0a73fed..0000000 --- a/data_acq_and_processing/tests/test_sar3d_oracle.py +++ /dev/null @@ -1,297 +0,0 @@ -"""Compare the native SAR port with notebook cells 0/4, without dataset or GPU fixtures.""" -from __future__ import annotations - -import json -from pathlib import Path -import re -import subprocess -import tempfile -import unittest - -import numpy as np - -ROOT = Path(__file__).resolve().parents[2] -FIXTURE = ROOT / "build/native_tests/sar3d_math_fixture" -CUDA_FIXTURE = ROOT / "build/native_tests/sar3d_fixture" -C = 3e8 - - -def smooth(volume): - # scipy gaussian_filter(sigma=(0,1.5,1.5), mode='reflect', truncate=4). - taps = np.exp(-.5 * (np.arange(-6, 7) / 1.5) ** 2) - taps /= taps.sum() - for axis in (1, 2): - volume = np.apply_along_axis(lambda row: np.convolve( - np.pad(row, 6, mode="symmetric"), taps, mode="valid"), axis, volume) - return volume - - -def native(request, executable=None): - run = subprocess.run([str(executable or FIXTURE)], input=json.dumps(request), text=True, - capture_output=True, timeout=30, check=False) - if not run.stdout: - raise AssertionError(run.stderr) - return json.loads(run.stdout) - - -def fixture(base=2**64 - 3_000_000_000): - request = dict(operation="processor", tx=[[0, -.4, 0, 0], [1, .1, .2, 0], [2, .45, .1, .05]], - rx=[[0, -.2, .1, 0], [1, .3, .35, .02]], f_start=1e9, f_stop=4e9, - min_depth=1., max_depth=3., speed=.4, pitch=13., ground=.3, - voxel=.4, radius=1, power=1.5, compensation=True, normalize=True) - members = [] - frequency = np.linspace(1e9, 4e9, 64, dtype=np.float32) - for frame, offset in enumerate([0, 180_000_000, 510_000_000]): - traces = [] - for tx in range(3): - for rx in range(2): - # Two transmitters interleave in one real capture window; the - # third has a longer independent segment, matching meta timing. - start = base + offset + (0 if tx < 2 else 70_000_000) - end = base + offset + (60_000_000 if tx < 2 else 165_000_001) - samples = ((1 + .1 * rx) * np.exp(-2j * np.pi * frequency * - (4.8 + .08 * tx - .03 * frame) / C) + - .35 * np.exp(-2j * np.pi * frequency * (5.6 - .05 * rx) / C)).astype(np.complex64) - traces.append(dict(input=rx, output=tx, start=start, end=end, - frequency=frequency.tolist(), s21=np.column_stack((samples.real, samples.imag)).tolist())) - members.append(dict(id=frame + 1, timestamp=base + offset, traces=traces)) - request.update(previous=members[:-1], latest=members[-1]) - return request - - -def oracle(request): - """Notebook ground/frequency-domain mode, with actual per-trace capture windows.""" - pitch = np.deg2rad(float(np.float32(request["pitch"]))) - cos_p, sin_p = np.cos(pitch), np.sin(pitch) - speed, voxel, ground = (float(np.float32(request[key])) for key in ("speed", "voxel", "ground")) - axis = lambda lo, hi: np.linspace(lo, hi, max(round((hi - lo) / voxel) + 1, 2)) - x, y = axis(-2, 2), axis(-.5, 2) - limits = [(depth + (height - ground) * sin_p) / cos_p - for height in (y[0], y[-1]) for depth in (request["min_depth"], request["max_depth"])] - z = axis(min(limits), max(limits)) - xx, zz = np.meshgrid(x, z) - frames, centres = [], [] - for member in request["previous"] + [request["latest"]]: - traces = member["traces"] - first, last = min(t["start"] for t in traces), max(t["end"] for t in traces) - centres.append((first + last) / 2 if first < 2**53 else (first, last)) - ascans = [] - for trace in traces: - frequency = np.asarray(trace["frequency"], dtype=np.float32) - samples = np.asarray(trace["s21"]) - samples = samples[:, 0] + 1j * samples[:, 1] - mask = (frequency >= request["f_start"]) & (frequency <= request["f_stop"]) - points = np.flatnonzero(mask) - stages = sorted({t["output"] for t in traces - if (t["start"], t["end"]) == (trace["start"], trace["end"])}) - # Relative integer subtraction retains precision at uint64 limits. - t = ((trace["start"] - first) - .5 * (last - first)) * 1e-9 - t += (len(stages) * points + stages.index(trace["output"])) * ( - (trace["end"] - trace["start"]) * 1e-9 / len(stages) / len(frequency)) - band = samples[mask] * np.exp(-4j * np.pi * speed * cos_p * frequency[mask] * t / C) - df = float(np.median(np.diff(frequency[mask].astype(float)))) - k0, n = round(float(frequency[mask][0]) / df), len(band) - size = 2 ** (2 * (k0 + n - 1) - 1).bit_length() * 8 - spectrum = np.zeros(size, dtype=complex) - spectrum[k0:k0+n] = band * np.hanning(n) - h = np.fft.ifft(spectrum) - dt = 1 / (size * df) - if request["normalize"]: - depth = np.arange(size) * dt * C / 2 - gate = (depth >= request["min_depth"]) & (depth <= request["max_depth"]) - scale = np.percentile(np.abs(h)[gate], 50) - h /= (scale if np.isfinite(scale) and scale > 1e-15 else 1) + 1e-15 - ascans.append(h) - frames.append(ascans) - # All members use identical grids. Native centre trails latest by radius. - centre_index = max(len(frames) - 1 - request["radius"], 0) - def relative(a, b): - if isinstance(a, tuple): - return .5e-9 * ((a[0] - b[0]) + (a[1] - b[1])) - return 1e-9 * (a - b) - offsets = np.array([relative(t, centres[centre_index]) for t in centres]) - farthest = np.max(np.abs(offsets)) - weights = np.ones(len(frames)) if farthest <= 1e-12 else .8 + .1 * (1 + np.cos(np.pi * abs(offsets) / farthest)) - acc = np.zeros((len(y), len(z), len(x)), dtype=complex) - count = np.zeros(acc.shape) - tx = np.asarray(request["tx"], dtype=np.float32)[:, 1:].astype(float) - rx = np.asarray(request["rx"], dtype=np.float32)[:, 1:].astype(float) - for scans, offset, weight in zip(frames, offsets, weights): - shift = np.array([0, speed * offset * sin_p, speed * offset * cos_p]) - for iy, height in enumerate(y): - yr, zr = (height - ground) * cos_p + zz * sin_p, -(height - ground) * sin_p + zz * cos_p - depth_ok = (zr >= request["min_depth"]) & (zr <= request["max_depth"]) - for it, antenna_t in enumerate(tx): - for ir, antenna_r in enumerate(rx): - def distance(antenna): - return np.sqrt((xx-antenna[0])**2 + (yr-antenna[1])**2 + (zr-antenna[2])**2) - rt, rr = distance(antenna_t + shift), distance(antenna_r + shift) - position = (rt + rr) / (C * dt) - h = scans[it * len(rx) + ir] - valid = depth_ok & (position <= len(h) - 1) - sample = np.interp(position, np.arange(len(h)), h) - if request["compensation"]: - point = np.array([.5 * (antenna_t[0] + antenna_r[0]), - (-ground + 5 * sin_p) / cos_p, 5]) - ref = 1 / (np.linalg.norm(point-antenna_t) * np.linalg.norm(point-antenna_r) + 1e-12) + 1e-30 - ratio = (1 / (rt * rr + 1e-12) + 1e-30) / ref - sample *= np.clip(1 / (ratio ** request["power"] + 1e-12), 0, 5) - acc[iy] += weight * np.where(valid, sample, 0) - count[iy] += weight * valid - eligible = (y >= y[0]+.3) & (y <= y[-1]-.3) - raw_volume = np.abs(acc / (count + 1e-12)) - raw_level = raw_volume[eligible].max() + 1e-12 - volume = smooth(raw_volume / raw_level) - smooth_level = volume[eligible].max() + 1e-12 - level = raw_level * smooth_level - peak = np.unravel_index(np.argmax(np.where(eligible[:, None, None], volume, -1)), volume.shape) - volume /= smooth_level - iy, iz, ix = peak - eye = np.array([1.7, -2.2, .85]) - forward = -eye / np.linalg.norm(eye) - right = np.cross(forward, [0, 0, 1]) - right /= np.linalg.norm(right) - up = np.cross(right, forward) - u, v = right[[0, 2, 1]], up[[0, 2, 1]] - points = np.array(np.meshgrid(x, y, z, indexing="ij")).reshape(3, -1).T - us, vs = points @ u, points @ v - u0, v0 = us.min(), vs.min() - nu, nv = round((us.max() - u0) / voxel) + 1, round((vs.max() - v0) / voxel) + 1 - oblique = np.zeros((nv, nu)) - pu, pv = np.rint((us-u0)/voxel).astype(int), np.rint((vs-v0)/voxel).astype(int) - valid = (pu >= 0) & (pu < nu) & (pv >= 0) & (pv < nv) - np.maximum.at(oblique, (pv[valid], pu[valid]), volume.transpose(2, 0, 1).ravel()[valid]) - return dict(x=x, y=y, z=z, volume=volume, level=level, peak=[x[ix], y[iy], z[iz]], - peak_index=peak, oblique=oblique, u=u0 + np.arange(nu)*voxel, v=v0 + np.arange(nv)*voxel) - - -class Sar3dOracleTest(unittest.TestCase): - def test_cuda_branch_cpp_syntax(self): - # Hardware-free name/type structure check. Variadic API placeholders do - # not validate the NVIDIA ABI or replace testing NVRTC on an actual GPU. - source_path = ROOT / "data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp" - source = source_path.read_text() - with tempfile.TemporaryDirectory(prefix="radar_cuda_syntax_") as directory: - path = Path(directory) - functions = set(re.findall(r"\b(cu[A-Z]\w*)\s*\(", source)) - constants = set(re.findall(r"\bCU_DEVICE_ATTRIBUTE_\w+", source)) - (path / "cuda.h").write_text( - "#pragma once\n#include \nenum CUresult {CUDA_SUCCESS=0};\n" - "using CUdevice=int;using CUdeviceptr=std::uintptr_t;using CUcontext=struct Context*;" - "using CUmodule=struct Module*;using CUfunction=struct Function*;\n" + - "".join(f"constexpr int {name}=0;\n" for name in constants) + - "".join(f"inline CUresult {name}(...) {{return CUDA_SUCCESS;}}\n" for name in functions)) - functions = set(re.findall(r"\b(nvrtc[A-Z]\w*)\s*\(", source)) | { - "nvrtcGetCUBINSize", "nvrtcGetPTXSize", "nvrtcGetCUBIN", "nvrtcGetPTX"} - (path / "nvrtc.h").write_text( - "#pragma once\nenum nvrtcResult {NVRTC_SUCCESS=0};using nvrtcProgram=struct Program*;\n" - 'inline const char* nvrtcGetErrorString(...) {return "stub";}\n' + - "".join(f"inline nvrtcResult {name}(...) {{return NVRTC_SUCCESS;}}\n" - for name in functions if name != "nvrtcGetErrorString")) - result = subprocess.run(["g++", "-std=c++20", "-pthread", "-fsyntax-only", "-Wall", "-Wextra", - "-Wpedantic", "-Werror", "-DSAR3D_WITH_CUDA", f"-I{path}", str(source_path)], - text=True, capture_output=True, timeout=30, check=False) - self.assertEqual(result.returncode, 0, result.stderr) - - def ready(self, result): - payloads = {p["name"]: p for p in result["payloads"]} - status = payloads["sar3d_status"]["table"] - self.assertEqual(payloads["sar3d_status"]["columns"], len(status)) - self.assertEqual(status[0], 0) - return payloads - - def test_inverse_fft_matches_numpy(self): - random = np.random.default_rng(209) - bands = random.normal(size=(2, 48)) + 1j * random.normal(size=(2, 48)) - result = native(dict(operation="ifft", bands=np.column_stack((bands.ravel().real, bands.ravel().imag)).tolist(), - points=48, first_bin=17, size=512)) - actual = np.asarray(result["ascans"]) - spectrum = np.zeros((2, 512), dtype=complex) - spectrum[:, 17:65] = bands.astype(np.complex64) - expected = np.fft.ifft(spectrum).ravel() - np.testing.assert_allclose(actual[:, 0] + 1j * actual[:, 1], expected, rtol=3e-4, atol=2e-7) - - def test_production_backend_reports_cuda_warning(self): - result = native(fixture(), CUDA_FIXTURE) - payloads = {p["name"]: p for p in result["payloads"]} - status = payloads["sar3d_status"]["table"] - if status[0] == 3: - self.assertIn("CUDA", bytes(map(int, status[1:])).decode("utf-8")) - self.assertFalse(any(item["image"] for item in payloads.values())) - else: - # On a CUDA-equipped test host, also run the real compiled backend. - self.ready(result) - - def test_motion_volume_matches_notebook(self): - for compensation, normalize in ((True, True), (True, False), (False, True), (False, False)): - with self.subTest(compensation=compensation, normalize=normalize): - request = fixture() - request.update(compensation=compensation, normalize=normalize) - payloads = self.ready(native(request)) - expected = oracle(request) - for name, image, x, y in ( - ("sar3d_front", expected["volume"].max(axis=1), expected["x"], expected["y"]), - ("sar3d_side", expected["volume"].max(axis=2), expected["z"], expected["y"]), - ("sar3d_top", expected["volume"].max(axis=0), expected["x"], expected["z"]), - ("sar3d_oblique", expected["oblique"], expected["u"], expected["v"])): - np.testing.assert_allclose(payloads[name]["x"], x, rtol=2e-6, atol=1e-6) - np.testing.assert_allclose(payloads[name]["y"], y, rtol=2e-6, atol=1e-6) - np.testing.assert_allclose(np.asarray(payloads[name]["image"]).reshape(image.shape), image, - rtol=2e-3, atol=3e-4) - np.testing.assert_allclose(payloads["sar3d_peak"]["table"][1:], expected["peak"], atol=2e-6) - np.testing.assert_allclose(payloads["sar3d_peak"]["table"][0], expected["level"], rtol=2e-3) - - def test_errors_are_published(self): - for malformed in ("empty", "capture", "frequency", "axis", "angle"): - with self.subTest(malformed=malformed): - request = fixture() - if malformed == "empty": - request["latest"]["traces"] = [] - elif malformed == "capture": - request["latest"]["traces"][0]["end"] = request["latest"]["traces"][0]["start"] - elif malformed == "frequency": - request["latest"]["traces"][0]["frequency"].reverse() - elif malformed == "axis": - request["voxel"] = 1e-8 - else: - request["pitch"] = 90 - result = native(request) - payloads = {p["name"]: p for p in result["payloads"]} - status = payloads["sar3d_status"]["table"] - self.assertEqual(payloads["sar3d_status"]["columns"], len(status)) - self.assertEqual(status[0], 2) - self.assertTrue(bytes(map(int, status[1:])).decode()) - - def test_empty_signal_is_reported(self): - request = fixture() - request.update(previous=[], radius=0) - for trace in request["latest"]["traces"]: - trace["s21"] = [[0, 0]] * len(trace["frequency"]) - payloads = {p["name"]: p for p in native(request)["payloads"]} - self.assertEqual(payloads["sar3d_status"]["table"][0], 1) - self.assertNotIn("sar3d_peak", payloads) - for name in ("sar3d_front", "sar3d_side", "sar3d_top", "sar3d_oblique"): - self.assertTrue(np.all(np.asarray(payloads[name]["image"]) == 0)) - - def test_invalid_fft_dimensions_are_rejected(self): - for points, first_bin, size in ((0, 0, 8), (1, 0, 7), (1, 8, 8), (2, 7, 8), (1, 0, 2**31)): - with self.subTest(points=points, first_bin=first_bin, size=size): - result = native(dict(operation="ifft", bands=[[1, 0]], - points=points, first_bin=first_bin, size=size)) - self.assertIn("error", result) - - def test_weak_signal_stays_finite(self): - request = fixture() - request.update(previous=[], radius=0, normalize=False) - for trace in request["latest"]["traces"]: - trace["s21"] = (np.asarray(trace["s21"]) * 1e-38).tolist() - payloads = self.ready(native(request)) - for name in ("sar3d_front", "sar3d_side", "sar3d_top", "sar3d_oblique"): - values = np.asarray(payloads[name]["image"]) - self.assertTrue(np.isfinite(values).all()) - self.assertGreater(values.max(), 0) - self.assertLess(values.max(), 1e-10) - - -if __name__ == "__main__": - unittest.main() diff --git a/deploy/pack-usb.sh b/deploy/pack-usb.sh index 4d613c0..17f7820 100755 --- a/deploy/pack-usb.sh +++ b/deploy/pack-usb.sh @@ -11,7 +11,7 @@ set -euo pipefail PROJECT_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" # What runs on the Pi or Jetson; device_firmware/ stays on this machine. HOST_PATHS=(data_acq_and_processing deploy python_app run_config_examples Makefile - "requirements*.txt" run_config.json start.sh locator_test_client.py) + "requirements.txt" run_config.json start.sh) drive="${1:-}" if [[ -z "${drive}" ]]; then diff --git a/deploy/update.sh b/deploy/update.sh index a555c49..d177550 100755 --- a/deploy/update.sh +++ b/deploy/update.sh @@ -152,7 +152,7 @@ if [[ -n "${backup}" ]]; then for entry in "${backup}"/python_app/data*; do data+=("python_app/${entry##*/}"); done for entry in "${backup}"/*; do name="${entry##*/}" - case "${name}" in build|graphify-out|device_firmware|docs) continue ;; esac + case "${name}" in build|graphify-out|device_firmware|docs|RELEASE) continue ;; esac [[ -e "${target}/${name}" ]] || data+=("${name}") done for entry in "${data[@]}"; do diff --git a/device_firmware/librevna_protocol.md b/device_firmware/librevna_protocol.md index cf7c96f..212f150 100644 --- a/device_firmware/librevna_protocol.md +++ b/device_firmware/librevna_protocol.md @@ -208,16 +208,69 @@ ref» — сигнал на первом приёмнике. отсчёты приёмников. Считает хост, в `librevna_multi_device_driver/cycle_collection.py`. -Нормировка (`calculate_last_cycle_s_parameters`, строка 783): каждый канал -делится на опорный приёмник **мастера**, а не на локальный опорный приёмник -слейва — потому что излучает именно мастер. Опорные измерения хранятся по -номеру выходного канала (`master_reference_measurements_by_output`), так как с -переключением стимула выходов больше, чем портов мастера. +Нормировка (`calculate_s_parameters`): каждый канал прохождения +делится на опорный приёмник **мастера**, а не на локальную опору слейва, +потому что излучает мастер. Опорные измерения хранятся по выходному каналу +(`master_reference_measurements_by_output`): при переключении стимула выходов +больше, чем физических портов мастера. -Имена S-параметров: отражение мастера — `master_reflection_s_parameter_name` -(строка 760), прямые каналы — `build_forward_s_parameter_names` (строка 765). -Нумерация приёмных портов слейвов начинается **после** всех выходов, чтобы -`S33`/`S44` отражения не столкнулись с прямыми каналами. +Отражения мастера приложение не собирает и не использует для отбраковки циклов. +Ключ канала — пара `(input_pos, output_pos)` с нумерацией от нуля. Входы слейвов +идут подряд: два порта первого слейва, затем два порта второго. Выход равен +стадии мастера. Эти же физические номера используются в `ComboKey` коллекции. + +Одиночный LibreVNA использует Python-драйвер `librevna_driver`: порт 1 излучает +на стадии 0, порт 2 выключен (`port2Stage = 7`), `port3Stage = 0` означает +отсутствие под-стадий. Хост сохраняет S21 = отсчёт порта 2 / опорный отсчёт +той же стадии. Опора распознаётся по биту Reference даже при маске `0x13`; +она никогда не подменяет сигнальный отсчёт из-за одновременно выставленных +битов портов. Физические номера антенных ключей остаются заданными конфигом. + +Бюджет ожидания одиночного свипа зависит от числа точек, `1 / IFBW` и dwell, +с запасом на установление частоты и обмен по USB. Отдельный тайм-аут отсутствия +новых точек прерывает зависший прибор независимо от общего бюджета свипа. + +В multi-device `collect_complete_running_sweep_cycle` собирает один цикл. +Окно коллекции и всех её трасс берётся по получению от мастера точки 0 и +последней принятой точки этого цикла. Метка точки 0 фиксируется до ожидания +слейвов на барьере синхронизации. Слив очередей, ожидание начала цикла и +повторные попытки сбора в окно не входят. + +### Формат коллекций между процессами хоста + +Это отдельный формат от USB-пакетов прибора. `RAW3` и `PRP3` — четыре ASCII-байта +magic (`0x33574152` и `0x33505250` little-endian). В каждой трассе идут массив +float32 частот и массив complex64 S21: 12 байт на точку. Окна съёма коллекции +и отдельных трасс обязательны: после массивов идут два u64 окна коллекции, +u32 числа трасс и по два u64 на каждую трассу. Усечённый хвост отвергается, +нулевые окна разрешены в бандлах калибровки/референса. Полный размер кадра: +`44 + число_пар × (28 + 12 × число_точек)` байт. Старые RAW2/PPR2 не принимаются; +runtime-бандлы пересобираются из NPZ при Start. Старые NPZ с лишними S11-массивами +читаются по S21 без изменения файлов. Формат пакетов LibreVNA этим не меняется. + +Кадр результата — `RSL3` (`0x334c5352`, ASCII little-endian). Его заголовок +занимает 64 байта, все поля little-endian, без выравнивающих промежутков: + +| Смещение | Поле | Тип | +| --- | --- | --- | +| 0 | magic | u32 | +| 4 | collection_id | u64 | +| 12 | monotonic_ns (завершение обработки) | u64 | +| 20 | processing_duration_ns | u64 | +| 28 | settings_revision | u64 | +| 36 | capture_start_ns | u64 | +| 44 | capture_end_ns | u64 | +| 52 | speed_m_s | f32 | +| 56 | число payload коллекции | u32 | +| 60 | число блоков пар | u32 | + +После заголовка идут payload'ы коллекции и блоки пар. Блок содержит номера +входа и выхода, число payload'ов (три u32), затем сами payload'ы. Каждый payload +начинается с типа u8 и имени (длина u16 и байты UTF-8); далее идут данные типа: +комплексная трасса, скаляр f32, изображение f32 или таблица f32. Идентичность +свипа на всех стадиях — `(collection_id, capture_start_ns)`, время стадии её +не меняет. +RSL2 отвергается; Python и C++ обновляются одновременно после остановки процессов. ## 7. Почему поле мощности читается, но не используется diff --git a/device_firmware/per_point_switching.md b/device_firmware/per_point_switching.md index 3db3a76..ffb68eb 100644 --- a/device_firmware/per_point_switching.md +++ b/device_firmware/per_point_switching.md @@ -18,12 +18,12 @@ ## Раскладка стадий -| Стадия | Порт мастера | Ключ | Выход | Отражение | -|---|---|---|---|---| -| 0 | 1 | A | 0 | `S11` | -| 1 | 1 | B | 1 | `S22` | -| 2 | 2 | A | 2 | `S33` | -| 3 | 2 | B | 3 | `S44` | +| Стадия | Порт мастера | Ключ | Выход | +|---|---|---|---| +| 0 | 1 | A | 0 | +| 1 | 1 | B | 1 | +| 2 | 2 | A | 2 | +| 3 | 2 | B | 3 | И стадии, и выходы нумеруются **порт-мажорно**: старшие биты счётчика стадий выбирают порт, младший — положение ключа. **Номер выхода равен номеру стадии**, @@ -35,8 +35,8 @@ > два оказывались переставлены относительно физического порядка — слоты 0,1,2,3 > попадали на выходы 0,2,1,3. Физический порядок подтверждён замером. Данные, > снятые до смены, **несопоставимы** с новыми: выходы 1 и 2 поменялись смыслом, -> поэтому калибровки и референсы надо переснять. Имена S-параметров те же, но -> `S22` теперь означает «порт 1, положение B», а не «порт 2, положение A». +> поэтому калибровки и референсы надо переснять. Выход 1 теперь означает +> «порт 1, положение B», а не «порт 2, положение A». Из этого следует главное свойство: **положение ключа — это младший бит номера стадии**, а период 2 нацело делит 4 стадии. Значит через границу точки чередование @@ -44,6 +44,11 @@ цикл триггера — абсолютный счётчик стадий не нужен, нужна только правильная чётность, а её контроллер восстанавливает на каждой точке по длине паузы. +Приложение сохраняет только прохождение от мастера к приёмникам слейвов. +Отражения мастера не собираются. Канал хранится как физическая пара +`(input, output)`: входы 0 и 1 — порты первого слейва, 2 и 3 — второго; +выход — номер стадии из таблицы. + ## Контракт по проводу Хост отправляет это в `build_sweep_settings_payload`, прошивка читает — обе diff --git a/locator_test_client.py b/locator_test_client.py deleted file mode 100644 index dc1b89e..0000000 --- a/locator_test_client.py +++ /dev/null @@ -1,139 +0,0 @@ -import json -import random -import socket -import struct -import threading -from contextlib import suppress -from typing import Any - -HOST = "127.0.0.1" -PORT = 8888 -CLIENT_DEVICE_ID = 0 -MIN_TEST_VLC = 5.0 -MAX_TEST_VLC = 6.0 -SEND_INTERVAL_SECONDS = 1.0 -RECV_TIMEOUT_SECONDS = 1.0 -CONNECT_TIMEOUT_SECONDS = 5.0 -MAX_PAYLOAD_BYTES = 64 * 1024 -HEADER_STRUCT = struct.Struct(" bytes: - """Encode a JSON payload using the protocol binary header.""" - payload_bytes = json.dumps( - payload, - ensure_ascii=True, - separators=(",", ":"), - ).encode("utf-8") - return HEADER_STRUCT.pack(device_id, len(payload_bytes)) + payload_bytes - - -def recv_exactly(sock: socket.socket, size: int, stop_event: threading.Event | None = None) -> bytes: - """Receive an exact number of bytes from the socket.""" - chunks = bytearray() - - while len(chunks) < size: - if stop_event is not None and stop_event.is_set(): - raise ConnectionError("Client stopped.") - try: - chunk = sock.recv(size - len(chunks)) - except socket.timeout: - continue # Preserve a partial header/payload across timeout polls. - if not chunk: - raise ConnectionError("Connection closed by peer.") - chunks.extend(chunk) - - return bytes(chunks) - - -def read_packet(sock: socket.socket, stop_event: threading.Event | None = None) -> tuple[int, Any]: - """Read and decode a single packet from the server.""" - header_bytes = recv_exactly(sock, HEADER_STRUCT.size, stop_event) - device_id, payload_length = HEADER_STRUCT.unpack(header_bytes) - - if payload_length > MAX_PAYLOAD_BYTES: - raise ValueError( - "Payload length %d exceeds the %d byte limit." - % (payload_length, MAX_PAYLOAD_BYTES) - ) - - payload_bytes = recv_exactly(sock, payload_length, stop_event) - payload = json.loads(payload_bytes.decode("utf-8")) - return device_id, payload - - -def send_vlc_message(sock: socket.socket, device_id: int, vlc: float) -> None: - """Send one test client payload to the server.""" - payload = {"vlc": vlc} - sock.sendall(encode_packet(payload, device_id=device_id)) - print(f'>>> sent {json.dumps(payload, ensure_ascii=False)} device_id={device_id}') - - -def receive_loop(sock: socket.socket, stop_event: threading.Event) -> None: - """Print all packets received from the locator server.""" - while not stop_event.is_set(): - try: - device_id, payload = read_packet(sock, stop_event) - except (OSError, ValueError) as error: - if not stop_event.is_set(): - print("Receiver stopped:", error) - stop_event.set() - return - - print(f"<<< received device_id={device_id}") - print(json.dumps(payload, ensure_ascii=False, indent=2)) - - -def send_loop(sock: socket.socket, stop_event: threading.Event) -> None: - """Send test vlc packets until the client is stopped.""" - send_vlc_message( - sock, - CLIENT_DEVICE_ID, - round(random.uniform(MIN_TEST_VLC, MAX_TEST_VLC), 2), - ) - - if SEND_INTERVAL_SECONDS <= 0: - stop_event.wait() - return - - while not stop_event.wait(SEND_INTERVAL_SECONDS): - send_vlc_message( - sock, - CLIENT_DEVICE_ID, - round(random.uniform(MIN_TEST_VLC, MAX_TEST_VLC), 2), - ) - - -def main() -> None: - """Run the client until interrupted or disconnected.""" - stop_event = threading.Event() - - with socket.create_connection( - (HOST, PORT), - timeout=CONNECT_TIMEOUT_SECONDS, - ) as sock: - sock.settimeout(RECV_TIMEOUT_SECONDS) - print(f"Connected to {HOST}:{PORT}") - - receiver = threading.Thread( - target=receive_loop, - args=(sock, stop_event), - daemon=True, - ) - receiver.start() - - try: - send_loop(sock, stop_event) - except KeyboardInterrupt: - print("Stopping client.") - except OSError as error: - print("Sender stopped:", error) - finally: - stop_event.set() - with suppress(OSError): - sock.shutdown(socket.SHUT_RDWR) - receiver.join(RECV_TIMEOUT_SECONDS + 1.0) - - -if __name__ == "__main__": - main() diff --git a/mimo_calibration.py b/mimo_calibration.py deleted file mode 100644 index 571308f..0000000 --- a/mimo_calibration.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 -"""Фазовая калибровка MIMO-радара по точечному отражателю (шару) и применение её к данным. - -Пример использует общий расчёт с интерфейсом программы. Формат файлов калибровки -(cal_*.npy, cal_meta.json) и папки postprocessed сохранён. - -Запуск целиком: python mimo_calibration.py (работает по параметрам из блока ниже) -Из другого кода: - import mimo_calibration as mc - cal = mc.compute_calibration(cal_sets, empty_scene_path, mc.Settings(...)) - mc.save_calibration(cal, cal_dir) - cal = mc.load_calibration(cal_dir) - s21_corr = mc.apply_calibration(s21, freq, cal) # массив (TX, RX, Nf) в памяти - mc.apply_to_measurement(meas_dir, cal) # preprocessed -> postprocessed -""" -from pathlib import Path -from statistics import mean - -from python_app.workflows.mimo_calibration import ( - Settings, apply_calibration, apply_to_measurement, calibration_on_grid, - compute_calibration, find_config_profile, load_calibration, load_geometry, - load_s21, save_calibration, -) - -# --- что делать ------------------------------------------------------------------------- -RUN_CALIBRATION = True # True -> посчитать калибровку по CAL_SETS и записать в CAL_DIR; - # False -> взять готовую калибровку из CAL_DIR -APPLY_TO_DATA = False # True -> записать исправленные данные в <измерение>/postprocessed - -# --- калибровочные измерения ------------------------------------------------------------ -# Каждая запись — одно положение отражателя. snapshots: список префиксов снимков ('0003') -# или None (все снимки папки). Несколько снимков одной записи усредняются. -# target_scene_m = (X, Y, Z) в осях сцены, метры; для шара — координаты его ЦЕНТРА, -# измеренные физически (а не взятые с некалиброванного изображения): -# X — поперёк, вправо, от начала координат антенн (config_profile, x = 0); -# Y — высота центра отражателя над землёй; -# Z — расстояние вдоль земли от точки под началом координат антенн до отражателя. -CAL_ROOT = Path('/Users/ivan_root/Downloads/Telegram_dwnld/2026-09-31 balls calib/' - 'librevna_multi_0.5-6GHz_1031pts_-3dBm_big_ball_h_0cm_z_5m_x_0cm/preprocessed') -CAL_SETS = [ - dict(path=CAL_ROOT, snapshots=[f'{index:04d}'], target_scene_m=(0.0, 0.16, 5.20)) - for index in range(5) -] -SPHERE_RADIUS_M = 0.16 # радиус шара-отражателя [м]; 0 -> точечная цель -EMPTY_SCENE_PATH = Path('/Users/ivan_root/Downloads/Telegram_dwnld/2026-09-31 balls calib/' - 'librevna_multi_0.5-6GHz_1031pts_-3dBm_bkgd_h_0cm/preprocessed') - # Дополнительная пустая сцена для архивного примера. - # Если референс уже удалил фон, задайте None. - -# --- геометрия (как в софте построения изображений) --------------------------------------- -INPUT_IDX = [0, 1, 2, 3] # физические входы (Rx), input_pos -OUTPUT_IDX = [0, 1, 2, 3] # физические выходы (Tx), output_pos -ANTENNA_HEIGHT_M = 0.92 # высота начала координат антенн (нижний ряд) над землёй [м] -PITCH_DEG = 10.0 # наклон оси радара ниже горизонта [град] - -# --- оценка ----------------------------------------------------------------------------- -CAL_F_MIN = 2.0e9 # полоса, по которой оценивается калибровка [Гц] -CAL_F_MAX = 6.0e9 -GATE_HALF_M = 0.15 # окно вокруг эха цели, ± по бистатическому пути [м] -GATE_TAPER_M = 0.03 # плавные края окна [м] -COARSE_RANGE_SEARCH_M = 0.6 # грубый поиск эха по дальности перед точной подгонкой, ± по пути [м]; - # 0 -> выключен -REFINE_AXES = ('x', 'y', 'z') # какие координаты цели уточнять по данным; если X и Y измерены - # точно — ('z',) -REFINE_RANGE_M = 0.10 # уточнение не дальше этого от найденной точки [м] -APPLY_AMPLITUDE = False # True -> поправка выравнивает и амплитуды антенн -PHASE_MODEL = 'linear' # модель фазы антенны по частоте: 'const' | 'linear' | 'poly3' | 'raw' - -# --- куда писать калибровку --------------------------------------------------------------- -CAL_NAME = 'cal_20260931_big_ball_h0_script' -CAL_DIR = Path(__file__).resolve().parent / 'results' / 'calibration' / CAL_NAME -OVERWRITE = True # True -> перезаписать файлы калибровки в CAL_DIR - -# --- к каким измерениям применять --------------------------------------------------------- -# Список папок измерений (тех, где лежат preprocessed, raw, results). -APPLY_TO_MEASUREMENTS = [ - # '/путь/к/измерению', -] -OVERWRITE_POSTPROCESSED = False # True -> пересоздать postprocessed (только созданный этой калибровкой: - # папка без calibration_applied.json не удаляется никогда) - -def main() -> None: - if RUN_CALIBRATION: - cal = compute_calibration(CAL_SETS, EMPTY_SCENE_PATH, Settings( - sphere_radius_m=SPHERE_RADIUS_M, input_idx=tuple(INPUT_IDX), output_idx=tuple(OUTPUT_IDX), - antenna_height_m=ANTENNA_HEIGHT_M, pitch_deg=PITCH_DEG, f_min=CAL_F_MIN, f_max=CAL_F_MAX, - gate_half_m=GATE_HALF_M, gate_taper_m=GATE_TAPER_M, coarse_range_search_m=COARSE_RANGE_SEARCH_M, - refine_axes=tuple(REFINE_AXES), refine_range_m=REFINE_RANGE_M, - apply_amplitude=APPLY_AMPLITUDE, phase_model=PHASE_MODEL)) - save_calibration(cal, CAL_DIR, overwrite=OVERWRITE) - q = cal['meta']['quality'] - print(f'Калибровка записана в {CAL_DIR}: сфазированность пар на отражателе ' - f'{mean(q["coherence_before"]):.2f} -> {mean(q["coherence_final"]):.2f}') - if APPLY_TO_DATA: - cal = load_calibration(CAL_DIR) - targets = APPLY_TO_MEASUREMENTS - if isinstance(targets, (str, Path)): # один путь вместо списка - targets = [targets] - for meas in targets: - post, n_snap, n_fix, _ = apply_to_measurement(meas, cal, OVERWRITE_POSTPROCESSED) - print(f'{Path(meas).name}: {n_snap} снимков, исправлено S21 {n_fix} -> {post}') - - -if __name__ == '__main__': - main() diff --git a/python_app/file_io.py b/python_app/file_io.py index df0609c..0997551 100644 --- a/python_app/file_io.py +++ b/python_app/file_io.py @@ -1,12 +1,66 @@ -"""Atomic writes for JSON settings shared by GUI and pipeline processes.""" +"""Filesystem-safe names and atomic writes shared by storage and pipeline settings.""" from __future__ import annotations import json import os +import ctypes from contextlib import suppress from pathlib import Path -import tempfile +import stat +import re +from uuid import uuid4 + + +_renameat2 = ctypes.CDLL(None, use_errno=True).renameat2 +_renameat2.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] +_renameat2.restype = ctypes.c_int + + +def rename_no_replace(source: Path, destination: Path) -> None: + """Publish a file or directory atomically, refusing to replace an existing name.""" + # Linux AT_FDCWD=-100 and RENAME_NOREPLACE=1; also supported by FAT filesystems. + if _renameat2(-100, os.fsencode(source), -100, os.fsencode(destination), 1): + error = ctypes.get_errno() + raise OSError(error, os.strerror(error), str(destination)) + + +def temporary_file(destination: Path) -> Path: + """Create a sibling file using the destination's permissions or the current umask.""" + path = destination.with_name(f".{destination.name}.{uuid4().hex}.tmp") + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666) + try: + try: + mode = stat.S_IMODE(destination.stat().st_mode) + except FileNotFoundError: + pass + else: + os.fchmod(descriptor, mode) + except BaseException: + path.unlink() + raise + finally: + os.close(descriptor) + return path + + +def sync_directory(path: Path) -> None: + """Make directory entry changes durable after replacing or removing files.""" + descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def make_directory(path: Path, *, durable: bool = False) -> None: + """Create missing parents, optionally syncing each newly created directory entry.""" + if not durable: + path.mkdir(parents=True, exist_ok=True) + elif not path.is_dir(): + make_directory(path.parent, durable=True) + path.mkdir(exist_ok=True) + sync_directory(path.parent) def atomic_write_json(path: Path, payload: object, *, durable: bool = False) -> Path: @@ -21,27 +75,27 @@ def atomic_write_text(path: Path, text: str, *, durable: bool = False) -> Path: rename one another's work. Durable writes also sync the file and directory; frequent live settings can omit those syncs to avoid unnecessary disk I/O. """ - path.parent.mkdir(parents=True, exist_ok=True) + make_directory(path.parent, durable=durable) temporary = None try: - with tempfile.NamedTemporaryFile( - mode="w", encoding="utf-8", dir=path.parent, - prefix=f".{path.name}.", suffix=".tmp", delete=False, - ) as handle: - temporary = Path(handle.name) + temporary = temporary_file(path) + with temporary.open("w", encoding="utf-8") as handle: handle.write(text) if durable: handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) if durable: - directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) - try: - os.fsync(directory_fd) - finally: - os.close(directory_fd) + sync_directory(path.parent) finally: if temporary is not None: with suppress(OSError): temporary.unlink(missing_ok=True) return path + + +def sanitize_path_component(name: str) -> str: + """Convert arbitrary string into filesystem-safe component.""" + cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", name.strip()) + cleaned = cleaned.strip("._") + return cleaned or "snapshot" diff --git a/python_app/generator_sweep/__init__.py b/python_app/generator_sweep/__init__.py deleted file mode 100644 index aaeb340..0000000 --- a/python_app/generator_sweep/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Standalone LibreVNA signal-generator sweep support.""" - -__all__ = [ - "GeneratorSweepConfig", - "GeneratorSweepRunner", - "HardwarePwmGate", - "ResolvedGeneratorSweepConfig", - "resolve_generator_sweep_config", -] - - -def __getattr__(name: str) -> object: - """Load generator sweep helpers lazily to avoid heavy hardware imports.""" - if name in {"GeneratorSweepConfig", "ResolvedGeneratorSweepConfig", "resolve_generator_sweep_config"}: - from .config import GeneratorSweepConfig, ResolvedGeneratorSweepConfig, resolve_generator_sweep_config - - exports = { - "GeneratorSweepConfig": GeneratorSweepConfig, - "ResolvedGeneratorSweepConfig": ResolvedGeneratorSweepConfig, - "resolve_generator_sweep_config": resolve_generator_sweep_config, - } - return exports[name] - if name == "HardwarePwmGate": - from .pwm import HardwarePwmGate - - return HardwarePwmGate - if name == "GeneratorSweepRunner": - from .runner import GeneratorSweepRunner - - return GeneratorSweepRunner - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python_app/generator_sweep/config.py b/python_app/generator_sweep/config.py deleted file mode 100644 index 5214d10..0000000 --- a/python_app/generator_sweep/config.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Configuration resolution for the standalone generator sweep.""" - -from __future__ import annotations - -from dataclasses import dataclass -import logging -import math - -logger = logging.getLogger(__name__) - - -@dataclass(slots=True, frozen=True) -class GeneratorSweepConfig: - """User-edited generator sweep configuration stored next to the script.""" - - log_level: str = "INFO" - serial: str | None = None - strict_protocol_version: int = 14 - start_hz: float = 1_000_000.0 - stop_hz: float = 6_000_000.0 - step_hz: float = 0.0 - points: int = 0 - hold_time_ms: float = 50.0 - loop: bool = True - port: int = 1 - power_dbm: float = -10.0 - amplitude_correction: bool = False - settle_timeout_ms: int = 1_000 - status_poll_ms: float = 10.0 - post_lock_delay_us: int = 0 - gpio_chip: str = "/dev/gpiochip0" - sweep_start_pin: int = 5 - curr_step_pin: int = 6 - curr_step_initial_level: int = 0 - pwm_pin: int = 12 - pwm_frequency_hz: int = 2_000_000 - pwm_duty_cycle: float = 0.5 - - -@dataclass(slots=True, frozen=True) -class ResolvedGeneratorSweepConfig: - """Fully validated generator sweep configuration.""" - - log_level: int - serial: str | None - strict_protocol_version: int - frequencies_hz: tuple[int, ...] - hold_time_s: float - loop: bool - port: int - power_dbm: float - amplitude_correction: bool - settle_timeout_s: float - status_poll_s: float - post_lock_delay_s: float - gpio_chip: str - sweep_start_pin: int - curr_step_pin: int - curr_step_initial_level: int - pwm_pin: int - pwm_frequency_hz: int - pwm_duty_cycle: float - - -def _require(condition: bool, message: str) -> None: - """Raise ``ValueError`` when a configuration invariant is violated.""" - if not condition: - raise ValueError(message) - - -def _validate_positive_int(value: int, label: str) -> None: - """Validate positive integer setting.""" - _require(value > 0, f"{label} must be > 0") - - -def _validate_non_negative_int(value: int, label: str) -> None: - """Validate non-negative integer setting.""" - _require(value >= 0, f"{label} must be >= 0") - - -def _validate_non_negative_float(value: float, label: str) -> None: - """Validate non-negative float setting.""" - _require(value >= 0.0, f"{label} must be >= 0") - - -def _validate_positive_float(value: float, label: str) -> None: - """Validate positive float setting.""" - _require(value > 0.0, f"{label} must be > 0") - - -def _resolve_log_level(value: str) -> int: - """Normalize logging level name to ``logging`` module integer constant.""" - normalized = value.strip().upper() - if not normalized: - raise ValueError("generator_sweep.log_level must not be empty") - - level = logging.getLevelName(normalized) - if isinstance(level, int): - return level - raise ValueError("generator_sweep.log_level must be one of DEBUG, INFO, WARNING, ERROR, CRITICAL") - - -def _build_frequency_grid(model: GeneratorSweepConfig) -> tuple[int, ...]: - """Build the ordered frequency grid with inclusive stop frequency.""" - start_hz = int(round(model.start_hz)) - stop_hz = int(round(model.stop_hz)) - step_hz = int(round(model.step_hz)) - points = int(model.points) - - _validate_positive_int(start_hz, "generator_sweep.start_hz") - _validate_positive_int(stop_hz, "generator_sweep.stop_hz") - _require(stop_hz >= start_hz, "generator_sweep.stop_hz must be >= generator_sweep.start_hz") - - has_step = step_hz > 0 - has_points = points > 0 - _require(has_step != has_points, "generator_sweep must define exactly one of step_hz or points") - - if start_hz == stop_hz: - if has_points: - _require(points == 1, "generator_sweep.points must be 1 when start_hz == stop_hz") - return (start_hz,) - - if has_step: - _validate_positive_int(step_hz, "generator_sweep.step_hz") - _require((stop_hz - start_hz + step_hz - 1) // step_hz + 1 <= 1_000_000, - "generator sweep grid must not exceed 1000000 frequencies") - return (*range(start_hz, stop_hz, step_hz), stop_hz) - - _validate_positive_int(points, "generator_sweep.points") - _require(points >= 2, "generator_sweep.points must be >= 2 when start_hz != stop_hz") - _require(points <= 1_000_000, "generator sweep grid must not exceed 1000000 frequencies") - span_hz = stop_hz - start_hz - frequencies = tuple( - int(round(start_hz + (span_hz * index) / float(points - 1))) - for index in range(points) - ) - _require(frequencies[0] == start_hz, "generator_sweep.points grid does not start at start_hz") - _require(frequencies[-1] == stop_hz, "generator_sweep.points grid does not end at stop_hz") - _require( - all(left < right for left, right in zip(frequencies, frequencies[1:])), - "generator sweep grid contains duplicate or non-monotonic frequencies", - ) - return frequencies - - -def resolve_generator_sweep_config(model: GeneratorSweepConfig) -> ResolvedGeneratorSweepConfig: - """Validate and resolve runtime configuration for generator sweep mode.""" - for name in ("points", "settle_timeout_ms", "post_lock_delay_us", "pwm_frequency_hz", - "sweep_start_pin", "curr_step_pin", "pwm_pin", "curr_step_initial_level", - "strict_protocol_version", "port"): - value = getattr(model, name) - _require(type(value) is int, f"generator_sweep.{name} must be an integer") - for name in ("start_hz", "stop_hz", "step_hz", "hold_time_ms", "status_poll_ms", "power_dbm", "pwm_duty_cycle"): - value = getattr(model, name) - _require(type(value) in (int, float) and math.isfinite(value), - f"generator_sweep.{name} must be a finite number") - _require(model.points >= 0 and model.step_hz >= 0, "generator_sweep points and step_hz must be >= 0") - log_level = _resolve_log_level(model.log_level) - frequencies_hz = _build_frequency_grid(model) - hold_time_ms = float(model.hold_time_ms) - settle_timeout_ms = int(model.settle_timeout_ms) - status_poll_ms = float(model.status_poll_ms) - post_lock_delay_us = int(model.post_lock_delay_us) - pwm_frequency_hz = int(model.pwm_frequency_hz) - sweep_start_pin = int(model.sweep_start_pin) - curr_step_pin = int(model.curr_step_pin) - pwm_pin = int(model.pwm_pin) - curr_step_initial_level = int(model.curr_step_initial_level) - - _validate_non_negative_float(hold_time_ms, "generator_sweep.hold_time_ms") - _validate_positive_int(settle_timeout_ms, "generator_sweep.settle_timeout_ms") - _validate_positive_float(status_poll_ms, "generator_sweep.status_poll_ms") - _validate_non_negative_int(post_lock_delay_us, "generator_sweep.post_lock_delay_us") - _validate_positive_int(pwm_frequency_hz, "generator_sweep.pwm_frequency_hz") - _validate_positive_int(int(model.strict_protocol_version), "generator_sweep.strict_protocol_version") - _require(model.port in {1, 2}, "generator_sweep.port must be 1 or 2") - _require(curr_step_initial_level in {0, 1}, "generator_sweep.curr_step_initial_level must be 0 or 1") - _require(bool(model.gpio_chip.strip()), "generator_sweep.gpio_chip must not be empty") - _validate_non_negative_int(sweep_start_pin, "generator_sweep.sweep_start_pin") - _validate_non_negative_int(curr_step_pin, "generator_sweep.curr_step_pin") - _validate_non_negative_int(pwm_pin, "generator_sweep.pwm_pin") - _require( - len({sweep_start_pin, curr_step_pin, pwm_pin}) == 3, - "generator_sweep sweep_start_pin, curr_step_pin, and pwm_pin must be distinct", - ) - _require( - 0.0 < float(model.pwm_duty_cycle) <= 1.0, - "generator_sweep.pwm_duty_cycle must be within (0, 1]", - ) - - logger.debug( - "Resolved generator sweep config: points=%d start=%dHz stop=%dHz loop=%s port=%d", - len(frequencies_hz), - frequencies_hz[0], - frequencies_hz[-1], - bool(model.loop), - int(model.port), - ) - - return ResolvedGeneratorSweepConfig( - log_level=log_level, - serial=(model.serial.strip() or None) if isinstance(model.serial, str) else model.serial, - strict_protocol_version=int(model.strict_protocol_version), - frequencies_hz=frequencies_hz, - hold_time_s=hold_time_ms / 1000.0, - loop=bool(model.loop), - port=int(model.port), - power_dbm=float(model.power_dbm), - amplitude_correction=bool(model.amplitude_correction), - settle_timeout_s=settle_timeout_ms / 1000.0, - status_poll_s=status_poll_ms / 1000.0, - post_lock_delay_s=post_lock_delay_us / 1_000_000.0, - gpio_chip=model.gpio_chip.strip(), - sweep_start_pin=sweep_start_pin, - curr_step_pin=curr_step_pin, - curr_step_initial_level=curr_step_initial_level, - pwm_pin=pwm_pin, - pwm_frequency_hz=pwm_frequency_hz, - pwm_duty_cycle=float(model.pwm_duty_cycle), - ) diff --git a/python_app/generator_sweep/pwm.py b/python_app/generator_sweep/pwm.py deleted file mode 100644 index 619125a..0000000 --- a/python_app/generator_sweep/pwm.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Hardware-PWM backend for Raspberry Pi 5 generator sweep gating.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -import logging -from pathlib import Path - -logger = logging.getLogger(__name__) - - -_PWM_CHANNEL_BY_PIN = { - 12: 0, - 13: 1, - 18: 2, - 19: 3, -} - - -@dataclass(slots=True) -class HardwarePwmGate: - """Thin wrapper around Raspberry Pi hardware PWM output.""" - - pin: int - frequency_hz: int - duty_cycle: float - _pwm: object | None = field(init=False, default=None, repr=False) - _running: bool = field(init=False, default=False, repr=False) - - def __post_init__(self) -> None: - """Validate static PWM configuration.""" - if self.pin not in _PWM_CHANNEL_BY_PIN: - raise ValueError( - "generator_sweep.pwm_pin must be one of GPIO12, GPIO13, GPIO18, or GPIO19 on Raspberry Pi 5" - ) - if self.frequency_hz <= 0: - raise ValueError("generator_sweep.pwm_frequency_hz must be > 0") - if not (0.0 < self.duty_cycle <= 1.0): - raise ValueError("generator_sweep.pwm_duty_cycle must be within (0, 1]") - - def open(self) -> None: - """Initialize hardware PWM handle.""" - if self._pwm is not None: - return - - try: - from rpi_hardware_pwm import HardwarePWM - except ImportError as exc: - raise RuntimeError( - "rpi_hardware_pwm is required for generator_sweep PWM output. " - "Install project dependencies on the Raspberry Pi before running this script." - ) from exc - - channel = _PWM_CHANNEL_BY_PIN[self.pin] - try: - self._pwm = HardwarePWM(pwm_channel=channel, hz=self.frequency_hz, chip=0) - except Exception as exc: - config_hint = _boot_config_hint() - logger.exception("Failed to initialize hardware PWM on GPIO%d (channel %d)", self.pin, channel) - raise RuntimeError( - "Failed to initialize Raspberry Pi hardware PWM. " - f"Enable the PWM overlay in {config_hint} by adding " - "'dtoverlay=pwm-2chan', then reboot the Raspberry Pi." - ) from exc - self._running = False - logger.info("Hardware PWM opened on GPIO%d at %d Hz", self.pin, self.frequency_hz) - - def enable(self) -> None: - """Start PWM output when not already running.""" - if self._pwm is None: - raise RuntimeError("Hardware PWM gate is not open") - if self._running: - return - - duty_percent = self.duty_cycle * 100.0 - self._pwm.start(duty_percent) - self._running = True - - def disable(self) -> None: - """Stop PWM output when active.""" - if self._pwm is None or not self._running: - return - self._pwm.stop() - self._running = False - - def close(self) -> None: - """Stop PWM and release runtime state.""" - self.disable() - if self._pwm is not None: - logger.info("Hardware PWM closed on GPIO%d", self.pin) - self._pwm = None - - -def _boot_config_hint() -> str: - """Return the most likely active Raspberry Pi boot config path.""" - firmware_config = Path("/boot/firmware/config.txt") - if firmware_config.exists(): - return str(firmware_config) - return "/boot/config.txt" diff --git a/python_app/generator_sweep/runner.py b/python_app/generator_sweep/runner.py deleted file mode 100644 index 117dd2b..0000000 --- a/python_app/generator_sweep/runner.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Orchestration for standalone LibreVNA signal-generator sweeps.""" - -from __future__ import annotations - -from dataclasses import replace -import logging -import time - -from python_app.generator_sweep.config import ( - GeneratorSweepConfig, - ResolvedGeneratorSweepConfig, - resolve_generator_sweep_config, -) -from python_app.generator_sweep.pwm import HardwarePwmGate -from python_app.hardware_full.librevna_driver import GeneratorSettings, LibreVNADevice -from python_app.hardware_full.switch_drivers.gpio_uapi import GpioOutputLines - -logger = logging.getLogger(__name__) - - -class _MarkerOutputs: - """Own two GPIO marker signals used during generator sweeps.""" - - def __init__( - self, - *, - gpio_chip: str, - sweep_start_pin: int, - curr_step_pin: int, - curr_step_initial_level: int, - ) -> None: - """Store GPIO metadata and initialize marker state.""" - self._lines = GpioOutputLines( - chip=gpio_chip, - offsets=[sweep_start_pin, curr_step_pin], - consumer="generator_sweep", - ) - self._curr_step_initial_level = curr_step_initial_level - self._sweep_start_level = 0 - self._curr_step_level = curr_step_initial_level - self._is_open = False - - def open(self) -> None: - """Request GPIO lines and apply idle levels.""" - self._lines.open() - self._is_open = True - self.reset_to_idle() - logger.debug("Generator sweep marker GPIO lines opened") - - def close(self) -> None: - """Release GPIO lines.""" - self._is_open = False - self._lines.close() - logger.debug("Generator sweep marker GPIO lines closed") - - def begin_sweep(self) -> None: - """Raise sweep marker while preserving current step level.""" - time.sleep(0.001) - self._sweep_start_level = 1 - self._apply() - - def end_sweep(self) -> None: - """Drop sweep marker after a completed pass.""" - self._sweep_start_level = 0 - self._apply() - - def toggle_step(self) -> None: - """Invert current-step marker.""" - self._curr_step_level = 1 - self._curr_step_level - self._apply() - - def reset_to_idle(self) -> None: - """Return both markers to their configured idle state.""" - self._sweep_start_level = 0 - self._curr_step_level = self._curr_step_initial_level - self._apply() - - def _apply(self) -> None: - """Write current marker levels to GPIO.""" - if not self._is_open: - return - self._lines.set_values([self._sweep_start_level, self._curr_step_level]) - - -class GeneratorSweepRunner: - """Execute a signal-generator frequency sweep defined in run config.""" - - def __init__(self, config: GeneratorSweepConfig) -> None: - """Resolve configuration and prepare runtime helpers.""" - self._config: ResolvedGeneratorSweepConfig = resolve_generator_sweep_config(config) - self._device: LibreVNADevice | None = None - self._markers = _MarkerOutputs( - gpio_chip=self._config.gpio_chip, - sweep_start_pin=self._config.sweep_start_pin, - curr_step_pin=self._config.curr_step_pin, - curr_step_initial_level=self._config.curr_step_initial_level, - ) - self._pwm = HardwarePwmGate( - pin=self._config.pwm_pin, - frequency_hz=self._config.pwm_frequency_hz, - duty_cycle=self._config.pwm_duty_cycle, - ) - self._base_generator_settings = GeneratorSettings( - frequency_hz=float(self._config.frequencies_hz[0]), - power_dbm=self._config.power_dbm, - active_port=self._config.port, - apply_amplitude_correction=self._config.amplitude_correction, - ) - - def run(self) -> None: - """Open resources and execute one-shot or looping sweeps.""" - try: - self._open() - while True: - self._run_single_sweep() - if not self._config.loop: - break - except KeyboardInterrupt: - logger.info("Generator sweep interrupted by user") - finally: - self._close() - - def _open(self) -> None: - """Open GPIO, PWM, and LibreVNA device connection.""" - self._markers.open() - self._pwm.open() - self._device = LibreVNADevice() - self._device.connect( - serial=self._config.serial, - strict_protocol_version=self._config.strict_protocol_version, - timeout_s=2.0, - ) - self._device.generator.set_idle(timeout_s=1.0) - - logger.info( - "Generator sweep ready: points=%d start=%dHz stop=%dHz loop=%s", - len(self._config.frequencies_hz), - self._config.frequencies_hz[0], - self._config.frequencies_hz[-1], - self._config.loop, - ) - - def _close(self) -> None: - """Best-effort cleanup of PWM, GPIO, and device state.""" - device, self._device = self._device, None - cleanup = [self._pwm.disable] - if device is not None: - cleanup.extend([ - lambda: device.generator.set_idle(timeout_s=1.0) if device.is_connected else None, - device.disconnect, - ]) - cleanup.extend([self._markers.reset_to_idle, self._markers.close, self._pwm.close]) - for action in cleanup: - try: - action() - except Exception as exc: - logger.warning("Generator sweep cleanup failed: %s", exc) - - def _run_single_sweep(self) -> None: - """Execute one pass from the first configured frequency to the last.""" - self._markers.begin_sweep() - try: - for index, frequency_hz in enumerate(self._config.frequencies_hz): - self._pwm.disable() - self._set_generator_frequency(frequency_hz) - if index > 0: - self._markers.toggle_step() - self._wait_for_generator_ready() - self._emit_pwm_window() - logger.debug("Completed generator sweep pass over %d frequencies", len(self._config.frequencies_hz)) - finally: - self._pwm.disable() - self._markers.end_sweep() - - def _set_generator_frequency(self, frequency_hz: int) -> None: - """Transmit next generator frequency to LibreVNA.""" - device = self._require_device() - settings = replace(self._base_generator_settings, frequency_hz=float(frequency_hz)) - device.generator.configure(settings, timeout_s=max(self._config.settle_timeout_s, 1.0)) - logger.debug("Generator frequency set to %d Hz", frequency_hz) - - def _wait_for_generator_ready(self) -> None: - """Wait until generator lock telemetry reports ready.""" - device = self._require_device() - device.generator.wait_until_ready( - timeout_s=self._config.settle_timeout_s, - poll_interval_s=self._config.status_poll_s, - ) - if self._config.post_lock_delay_s > 0.0: - time.sleep(self._config.post_lock_delay_s) - - def _emit_pwm_window(self) -> None: - """Enable PWM only during the configured hold window.""" - if self._config.hold_time_s <= 0.0: - return - self._pwm.enable() - try: - time.sleep(self._config.hold_time_s) - finally: - self._pwm.disable() - - def _require_device(self) -> LibreVNADevice: - """Return connected device or fail fast if runner is not open.""" - if self._device is None: - raise RuntimeError("Generator sweep runner is not open") - return self._device diff --git a/python_app/gui/antenna_geometry_editor.py b/python_app/gui/antenna_geometry_editor.py index 4fd4f6b..30b12fb 100644 --- a/python_app/gui/antenna_geometry_editor.py +++ b/python_app/gui/antenna_geometry_editor.py @@ -2,22 +2,24 @@ from __future__ import annotations -from dataclasses import asdict, is_dataclass -import math +from collections.abc import Callable +from typing import cast -from PyQt6.QtCore import QLocale, QSignalBlocker, pyqtSignal +from PyQt6.QtCore import QItemSelectionModel, QLocale, QSignalBlocker, pyqtSignal from PyQt6.QtGui import QDoubleValidator, QIntValidator from PyQt6.QtWidgets import ( QAbstractItemView, QHeaderView, QHBoxLayout, QLineEdit, QMessageBox, QPushButton, QStyledItemDelegate, QTableWidget, QTableWidgetItem, QVBoxLayout, QWidget, ) +from python_app.models.run_config_validation import normalize_antenna_geometry + class _CoordinateDelegate(QStyledItemDelegate): def createEditor(self, parent, option, index): editor = QLineEdit(parent) validator = ( - QIntValidator(0, self.parent().property("position_count") - 1, editor) + QIntValidator(0, cast(QTableWidget, self.parent()).property("position_count") - 1, editor) if index.column() == 0 else QDoubleValidator(editor) ) validator.setLocale(QLocale.c()) @@ -30,16 +32,19 @@ class AntennaGeometryEditor(QWidget): changed = pyqtSignal() - def __init__(self, position_field: str, entries, *, position_count: int, parent=None) -> None: + def __init__(self, position_field: str, entries, *, position_count: int, + validate_change: Callable[[], object] | None = None, parent=None) -> None: super().__init__(parent) self.position_field = position_field + self.validate_change = validate_change + self._pending_row: int | None = None self._table = QTableWidget(0, 4, self) self.set_position_count(position_count) self._table.setHorizontalHeaderLabels([ "Output" if position_field == "output_pos" else "Input", "X (m)", "Y (m)", "Z (m)", ]) - self._table.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeMode.Stretch) - self._table.verticalHeader().hide() + cast(QHeaderView, self._table.horizontalHeader()).setSectionResizeMode(QHeaderView.ResizeMode.Stretch) + cast(QHeaderView, self._table.verticalHeader()).hide() self._table.setSelectionBehavior(QAbstractItemView.SelectionBehavior.SelectRows) self._table.setItemDelegate(_CoordinateDelegate(self._table)) self._table.setMinimumHeight(130) @@ -61,73 +66,75 @@ class AntennaGeometryEditor(QWidget): def records(self) -> list[dict]: records = [] for row in range(self._table.rowCount()): - records.append(dict(zip( - (self.position_field, "x_m", "y_m", "z_m"), - (self._table.item(row, column).text() for column in range(4)), - strict=True, - ))) - return self._normalize(records) - - def _normalize(self, entries) -> list[dict]: - records = [] - positions = set() - for entry in entries: - entry = asdict(entry) if is_dataclass(entry) else entry - if any(isinstance(entry.get(key), bool) for key in (self.position_field, "x_m", "y_m", "z_m")): - raise ValueError("Antenna positions and coordinates must be numbers, not booleans") - position = int(entry[self.position_field]) - if float(entry[self.position_field]) != position or not 0 <= position < self._table.property("position_count"): - raise ValueError("Antenna position is outside the configured switch positions") - if position in positions: - raise ValueError(f"Duplicate antenna position: {position}") - positions.add(position) - coordinates = {key: float(entry.get(key, 0.0)) for key in ("x_m", "y_m", "z_m")} - if any(not math.isfinite(value) or abs(value) > 3.4e38 for value in coordinates.values()): - raise ValueError("Antenna coordinates must be finite numbers") - records.append({self.position_field: position, **coordinates}) - return records + if row == self._pending_row and not cast(QTableWidgetItem, self._table.item(row, 1)).text().strip(): + continue + record = {} + for column, key in enumerate((self.position_field, "x_m", "y_m", "z_m")): + text = cast(QTableWidgetItem, self._table.item(row, column)).text() + try: + record[key] = int(text) if column == 0 else float(text) + except ValueError as exc: + raise ValueError(f"Antenna row {row + 1}: {key} must be {'an integer' if column == 0 else 'a number'}") from exc + records.append(record) + return normalize_antenna_geometry(records, self.position_field, self.position_count) def set_position_count(self, count: int) -> None: self.position_count = int(count) self._table.setProperty("position_count", int(count)) def set_records(self, entries) -> None: - records = self._normalize(entries) + records = normalize_antenna_geometry(entries, self.position_field, self.position_count) with QSignalBlocker(self._table): self._table.setRowCount(len(records)) for row, record in enumerate(records): for column, value in enumerate(record.values()): self._table.setItem(row, column, QTableWidgetItem(str(value))) + self._pending_row = None self._valid_records = records self._add_button.setEnabled(len(records) < self._table.property("position_count")) self.changed.emit() def _on_item_changed(self, _item) -> None: try: - self._valid_records = self.records() - except ValueError as exc: + records = self.records() + if records == self._valid_records: + return + if self.validate_change is not None: + self.validate_change() + except (ValueError, RuntimeError) as exc: with QSignalBlocker(self): self.set_records(self._valid_records) QMessageBox.warning(self, "Invalid antenna geometry", str(exc)) return + self._pending_row = None + self._valid_records = records + self._add_button.setEnabled(len(records) < self.position_count) self.changed.emit() def _add_row(self) -> None: - records = self.records() - if len(records) >= self._table.property("position_count"): + row = self._table.rowCount() + if row >= self.position_count or self._pending_row is not None: return - positions = {record[self.position_field] for record in records} - position = next(index for index in range(len(positions) + 1) if index not in positions) - self.set_records([*records, {self.position_field: position}]) - self._table.setCurrentCell(len(records), 1) - self._table.editItem(self._table.item(len(records), 1)) + positions = {record[self.position_field] for record in self.records()} + position = next(index for index in range(self.position_count) if index not in positions) + with QSignalBlocker(self._table): + self._table.insertRow(row) + for column, value in enumerate((str(position), "", "0", "0")): + self._table.setItem(row, column, QTableWidgetItem(value)) + self._pending_row = row + self._add_button.setEnabled(False) + self._table.setCurrentCell(row, 1) + self._table.editItem(self._table.item(row, 1)) def _remove_rows(self) -> None: - rows = {index.row() for index in self._table.selectionModel().selectedRows()} - if rows: - with QSignalBlocker(self._table): - for row in sorted(rows, reverse=True): - self._table.removeRow(row) - self._valid_records = self.records() - self._add_button.setEnabled(True) - self.changed.emit() + rows = {index.row() for index in cast(QItemSelectionModel, self._table.selectionModel()).selectedRows()} + if not rows: + return + with QSignalBlocker(self._table): + for row in sorted(rows, reverse=True): + self._table.removeRow(row) + if self._pending_row is not None: + self._pending_row = (None if self._pending_row in rows else + self._pending_row - sum(row < self._pending_row for row in rows)) + self._on_item_changed(None) + self._add_button.setEnabled(self._pending_row is None and self._table.rowCount() < self.position_count) diff --git a/python_app/gui/app_window.py b/python_app/gui/app_window.py index c494bf2..93c787c 100644 --- a/python_app/gui/app_window.py +++ b/python_app/gui/app_window.py @@ -1,814 +1,131 @@ -"""Main GUI composition root. +"""Assemble GUI controllers and own the lifetime of the application window.""" -This module wires UI/controller mixins together and owns application-level -state shared across them (runtime services, readers, history buffers, timer). -""" - -from __future__ import annotations - -from collections import deque -from contextlib import suppress -from datetime import datetime -import html -import json -import logging -import os from pathlib import Path -import sys -import threading -import traceback -from PyQt6.QtCore import QTimer -from PyQt6.QtGui import QTextCursor -from PyQt6.QtWidgets import QApplication, QMainWindow, QMessageBox +from PyQt6.QtWidgets import QMainWindow -from python_app.gui.controllers.app_window_config_mixin import AppWindowConfigMixin -from python_app.gui.controllers.app_window_config.state_builders import GUI_SAVE_HISTORY_LIMIT -from python_app.gui.controllers.app_window_control_button_mixin import AppWindowControlButtonMixin -from python_app.gui.controllers.app_window_pipeline_mixin import AppWindowPipelineMixin -from python_app.gui.controllers.app_window_plot_mixin import AppWindowPlotMixin -from python_app.gui.controllers.app_window_recording_mixin import AppWindowRecordingMixin -from python_app.gui.controllers.app_window_preprocess_mixin import AppWindowPreprocessMixin -from python_app.gui.controllers.app_window_snapshot_mixin import AppWindowSnapshotMixin -from python_app.gui.controllers.app_window_ui_mixin import AppWindowUiMixin -from python_app.gui.controllers.app_window_web_mixin import AppWindowWebMixin -from python_app.gui.preprocess_dialog import PreprocessDialog -from python_app.models.dataset_model import ResultCollection, SweepCollection -from python_app.models.gui_profile_model import GuiProfileModel +from python_app.gui.controllers.config_controller import ConfigController +from python_app.gui.controllers.control_button_controller import ControlButtonController +from python_app.gui.controllers.history_controller import HistoryController +from python_app.gui.controllers.live_settings_controller import LiveSettingsController +from python_app.gui.controllers.log_controller import LogController +from python_app.gui.controllers.pipeline_controller import PipelineController +from python_app.gui.controllers.plot_controller import PlotController +from python_app.gui.controllers.preprocess_controller import PreprocessController +from python_app.gui.controllers.profile_controller import ProfileController, load_startup +from python_app.gui.controllers.radar_settings_controller import RadarSettingsController +from python_app.gui.controllers.recording_controller import RecordingController +from python_app.gui.controllers.snapshot_controller import SnapshotController +from python_app.gui.controllers.web_controller import WebBridge +from python_app.gui.settings_defaults import cpp_bscan_replay_window_for_config +from python_app.gui.window_view import WindowView from python_app.orchestration.config_writer import ConfigWriter -from python_app.orchestration.gui_session_state import GuiSessionState, GuiSessionStateStore from python_app.orchestration.live_processing_config import ProcessingLiveConfigWriter -from python_app.orchestration.pipeline_metrics import PipelineMetrics -from python_app.orchestration.preprocess_assets import VISIBLE_PREPROCESS_ASSET_KEYS, preprocess_asset_model -from python_app.orchestration.process_supervisor import ProcessSupervisor -from python_app.orchestration.shm_reader import ShmRingReader -from python_app.logging_setup import ( - DEFAULT_LOG_LEVEL, - add_handler, - configure_logging, - get_logger, - set_log_level, -) -from python_app.storage.npz_store import NpzStore -from python_app.workflows.multi_radar_capture_workflow import MultiRadarSequentialCaptureSession -from python_app.workflows.radar_config_variants import RadarConfigScanSummary, RadarConfigVariant -from python_app.workflows.sequential_capture_workflow import SequentialCaptureSession +from python_app.storage.npz.store import NpzStore -def _panel_extra(details: str | None, once_key: str | None) -> dict[str, object]: - """Carry GUI-panel-only fields (details block, once-key dedup) on a log record.""" - return {"panel_details": details, "panel_once_key": once_key} - - -class _PanelLogBuffer: - """Thread-safe bounded buffer between logging handlers and the GUI flush timer. - - A :class:`logging.Handler` can fire on a worker thread (readers, broadcaster) - at a very high rate — e.g. the USB RX threads while the free-running sweep - streams. Posting one queued Qt event per record used to flood the GUI event - queue and keep the interface frozen long after a blocking operation finished - while the backlog rendered. Instead, records land in this bounded buffer and - a periodic GUI-side timer drains them in one batch; overflow drops the oldest - records and reports how many were lost. - """ - - _CAPACITY = 2000 - - def __init__(self) -> None: - self._lock = threading.Lock() - self._entries: deque[tuple[str, str, str | None, str | None]] = deque(maxlen=self._CAPACITY) - self._dropped_count = 0 - - def append(self, level: str, text: str, details: str | None, once_key: str | None) -> None: - """Store one record, evicting the oldest when full (any thread).""" - with self._lock: - if len(self._entries) == self._CAPACITY: - self._dropped_count += 1 - self._entries.append((level, text, details, once_key)) - - def drain(self) -> tuple[list[tuple[str, str, str | None, str | None]], int]: - """Return and clear all buffered records plus the overflow-drop count.""" - with self._lock: - entries = list(self._entries) - self._entries.clear() - dropped_count = self._dropped_count - self._dropped_count = 0 - return entries, dropped_count - - -class _QtLogPanelHandler(logging.Handler): - """Logging handler that forwards application log records to the GUI log panel.""" - - def __init__(self, buffer: _PanelLogBuffer) -> None: - super().__init__() - self._buffer = buffer - - def emit(self, record: logging.LogRecord) -> None: - """Buffer one record for the panel, mapping WARNING to the short 'WARN'.""" - try: - display_level = "WARN" if record.levelname == "WARNING" else record.levelname - details = getattr(record, "panel_details", None) - once_key = getattr(record, "panel_once_key", None) - self._buffer.append( - display_level, - record.getMessage(), - details if isinstance(details, str) else None, - once_key if isinstance(once_key, str) else None, - ) - except Exception: # noqa: BLE001 - logging must never raise into the caller - self.handleError(record) - - -class AppWindow( - AppWindowUiMixin, - AppWindowConfigMixin, - AppWindowPreprocessMixin, - AppWindowPlotMixin, - AppWindowPipelineMixin, - AppWindowSnapshotMixin, - AppWindowRecordingMixin, - AppWindowControlButtonMixin, - AppWindowWebMixin, - QMainWindow, -): - """Top-level window coordinating GUI state and acquisition runtime.""" - +class AppWindow(QMainWindow): def __init__(self, project_root: Path) -> None: - """Initialize all app subsystems in deterministic order.""" super().__init__() - - self._init_paths(project_root) - self._init_logging() - self._init_runtime_services() - self._init_config_profile_state() - self._init_reader_handles() - self._init_preprocess_state() - self._init_capture_state() - self._init_history_state() - self._init_recording_state() - self._init_runtime_limits() - self._init_polling_timer() - self._init_control_button_state() - self._bootstrap_ui_runtime() - - def _init_paths(self, project_root: Path) -> None: - """Initialize static project paths and startup log queue.""" - self._project_root = project_root - self._root_profile_path = project_root / "run_config.json" - self._active_profile_path = self._root_profile_path - # Run-config profiles the web UI can browse and load (it has no file dialog). - self._run_configs_dir = project_root / "run_configs" - self._pending_startup_log_entries: list[tuple[str, str, str | None]] = [] - # Guards closeEvent against re-entrant teardown (e.g. a second signal). - self._closing = False - self._close_after_stop = False - self._stopping_run = False - self._stop_request_seq = 0 - - def _init_logging(self) -> None: - """Configure the application logger and the bridge that feeds the GUI panel. - - Installs the rotating-file (``runtime/logs/radar.log``) and stderr handlers on - the ``python_app`` logger so every module's logs — and the GUI's own ``_log*`` - calls — share one level-controlled, rotated pipeline. The verbosity floor starts - at the default and is replaced with the configured level once run_config loads; - the GUI panel is wired in once its widget exists (see ``_attach_log_panel``). - """ - log_dir = self._project_root / "python_app/runtime/logs" - configure_logging(level=DEFAULT_LOG_LEVEL, log_dir=log_dir, console=True) - self._gui_logger = get_logger("gui") - self._log_panel_buffer = _PanelLogBuffer() - - def _attach_log_panel(self) -> None: - """Route application log records into the on-screen panel (widget now exists).""" - add_handler(_QtLogPanelHandler(self._log_panel_buffer)) - # One bounded flush per tick instead of one queued event per record: the - # panel can never flood the GUI event queue, no matter how chatty a - # DEBUG-level driver gets. - self._log_flush_timer = QTimer(self) - self._log_flush_timer.setInterval(100) - self._log_flush_timer.timeout.connect(self._flush_log_panel_buffer) - self._log_flush_timer.start() - - def _flush_log_panel_buffer(self) -> None: - """Render every buffered log record into the panel as one batched insert.""" - entries, dropped_count = self._log_panel_buffer.drain() - if (not entries and not dropped_count) or not hasattr(self, "_log_box"): - return - - entry_htmls: list[str] = [] - if dropped_count: - entry_htmls.append( - self._render_log_entry_html( - "WARN", - f"Log panel overflow: {dropped_count} record(s) dropped " - "(they are still in the log file).", - ) - ) - error_seen = False - for level, text, details, once_key in entries: - if once_key is not None: - if once_key in self._logged_once_keys: - continue - self._logged_once_keys.add(once_key) - entry_htmls.append(self._render_log_entry_html(level, text, details)) - error_seen = error_seen or level.upper() == "ERROR" - - if not entry_htmls: - return - cursor = self._log_box.textCursor() - cursor.movePosition(QTextCursor.MoveOperation.End) - self._log_box.setTextCursor(cursor) - self._log_box.insertHtml("".join(entry_htmls)) - self._log_box.insertPlainText("\n") - self._log_box.ensureCursorVisible() - if error_seen and hasattr(self, "_status_label"): - self._status_label.setText("Status: error") - - def _init_runtime_services(self) -> None: - """Initialize long-lived service objects used by mixins.""" - runtime_dir = self._project_root / "python_app/runtime" - self._runtime_dir = runtime_dir - self._store = NpzStore(self._project_root / "python_app/data") - self._config_writer = ConfigWriter(runtime_dir) - self._supervisor = ProcessSupervisor(self._project_root) - self._live_config_writer = ProcessingLiveConfigWriter(runtime_dir / "processing_live.json") - self._gui_session_state_store = GuiSessionStateStore(runtime_dir / "gui_session_state.json") - # `log_sink` is attached after the runtime log widget exists. - self._pipeline_metrics = PipelineMetrics( - report_every=self._resolve_metrics_report_every() + self._closed = False + self._log = LogController(self, project_root, lambda text: self._view.status.setText(text)) + state = load_startup(project_root, self._log) + runtime = project_root / "python_app/runtime" + store = NpzStore(project_root / "python_app/data") + published: dict[int, dict] = {} + self._history = HistoryController() + self._plots = PlotController( + self._history, lambda: self._config.processing_settings(), lambda: self._config.geometry(), + lambda: self._pipeline.running, self._log.warning, ) + self._plots.live_window = cpp_bscan_replay_window_for_config(state.config) + self._view = WindowView(self, state.config, state.gui, + self._plots.stack, self._log.panel, project_root=project_root) + fields = {key: widget for section in self._view.sections for key, widget in section.fields.items()} + fields["_log_level_combo"] = self._log.level + self._config = ConfigController(fields, self._view.switches.select_single.isChecked) + self._plots.bind_display_controls(self._view.switches.display_combo, + self._view.switches.next_combo) - @staticmethod - def _resolve_metrics_report_every() -> int: - """Read the metrics flush threshold from env, falling back to 50.""" - raw = os.environ.get("RADAR_SYSTEM_METRICS_REPORT_EVERY", "").strip() - if not raw: - return 50 - try: - value = int(raw) - except ValueError: - value = 0 - if value >= 1: - return value - # A non-empty but invalid value is an operator mistake — say so instead of - # silently swallowing it (stderr is captured by journald in headless mode). - print( - f"[radar] Ignoring invalid RADAR_SYSTEM_METRICS_REPORT_EVERY={raw!r}; using default 50.", - file=sys.stderr, - flush=True, + def write_profile(path: Path, *, allow_overwrite: bool = True, dataset: bool = False): + return self._profile.write(path, allow_overwrite=allow_overwrite, dataset=dataset) + + self._recording = RecordingController( + store, published, write_profile, + self._log.info, self._log.show_error, ) - return 50 - - def _init_config_profile_state(self) -> None: - """Resolve the startup profile path, load the active profile, and queue any - fallback notices for replay once the log panel exists. - - On failure to load a non-root profile, falls back to the root run_config.json; - a failure to load the root profile itself is fatal and re-raised. - """ - active_profile_path = self._resolve_startup_profile_path() - try: - profile = GuiProfileModel.load_from_path(active_profile_path) - except Exception as exc: - if active_profile_path == self._root_profile_path: - raise - self._queue_startup_log_entry( - "WARN", - "Failed to load the last selected config profile; falling back to root run_config.json.", - details=self._exception_details(exc), - ) - profile = GuiProfileModel.load_from_path(self._root_profile_path) - active_profile_path = self._root_profile_path - - self._active_profile_path = active_profile_path - self._defaults_config = profile.run_config.clone() - set_log_level(self._defaults_config.logging.level) - if profile.gui is not None: - self._gui_defaults = profile.gui - else: - self._gui_defaults = self._default_gui_state_for_config(self._defaults_config) - if active_profile_path != self._root_profile_path: - self._queue_startup_log_entry( - "INFO", - f"Loaded legacy run config without GUI defaults: {active_profile_path}", - ) - self._remember_active_profile_path(active_profile_path, startup=True) - - def _init_reader_handles(self) -> None: - """Initialize SHM readers as detached (not connected) handles.""" - self._raw_reader: ShmRingReader | None = None - self._pre_reader: ShmRingReader | None = None - self._result_reader: ShmRingReader | None = None - - def _init_preprocess_state(self) -> None: - """Initialize preprocessing dialog and selected set names.""" - self._preprocess_dialog: PreprocessDialog | None = None - self._preprocess_set_name = str(self._gui_defaults.preprocess_dialog.set_name) - self._preprocess_radar_config_dir = str(self._gui_defaults.preprocess_dialog.radar_config_dir) - self._preprocess_use_all_radar_configs = bool(self._gui_defaults.preprocess_dialog.use_all_radar_configs) - self._preprocess_median_sweep_count = max( - 1, int(self._gui_defaults.preprocess_dialog.median_sweep_count) + self._pipeline = PipelineController( + project_root, store, ConfigWriter(runtime), self._history, self._plots, self._recording, + build_config=lambda: self._profile.build_config(), live_config=lambda: self._live.build(), + publish_live=lambda: self._live.publish(), capture_guard=lambda: self._preprocess.busy_reason(), + refresh_radar_limits=lambda: self._radar.refresh_limits(), status=self._view.status.setText, + history_changed=lambda: self._view.update_history(self._history), + poll_ui=lambda: self._web.refresh_snapshot(), show_exception=self._log.show_exception, + deferred_close=self.close, event_scope=self._log.suspend_error_capture, ) - self._preprocess_radar_variants: list[RadarConfigVariant] = [] - self._preprocess_radar_scan_summary = RadarConfigScanSummary( - directory_path=self._preprocess_radar_config_dir, - json_file_count=0, - valid_variant_count=0, - skipped_file_count=0, - duplicate_variant_count=0, - issues=(), + self._preprocess = PreprocessController( + self, state.config, state.gui.preprocess_dialog, store, self._pipeline, self._plots, + build_config=lambda: self._profile.build_config(), current_radar_key=lambda: self._profile.radar_key(), + geometry=self._config.geometry, summary=self._view.update_preprocess, error=self._log.show_exception, ) - self._selected_preprocess_sets = { - key: str(preprocess_asset_model(self._defaults_config, key).set_name) - for key in VISIBLE_PREPROCESS_ASSET_KEYS - } - self._selected_preprocess_radar_key = self._radar_key(self._defaults_config) - - def _init_capture_state(self) -> None: - """Initialize one-shot capture and sequence-control flags.""" - self._capture_session: SequentialCaptureSession | MultiRadarSequentialCaptureSession | None = None - # Guards the blocking per-combo capture against duplicate requests, and keeps - # the dialog's action buttons disabled until the post-capture input backlog - # is dropped (see AppWindowPreprocessMixin._begin/_end_preprocess_capture). - self._preprocess_capture_busy = False - self._resume_pipeline_after_capture = False - self._single_capture_active = False - self._single_capture_start_ns: int | None = None - self._single_capture_seen_raw = False - self._single_capture_target_collection_id: int | None = None - - def _init_history_state(self) -> None: - """Initialize runtime history buffers and render-cache state.""" - bscan_cpp_replay_window = self._cpp_bscan_replay_window_for_config(self._defaults_config) - self._raw_history: deque[SweepCollection] = deque(maxlen=GUI_SAVE_HISTORY_LIMIT) - self._pre_history: deque[SweepCollection] = deque(maxlen=GUI_SAVE_HISTORY_LIMIT) - self._result_history: deque[ResultCollection] = deque(maxlen=GUI_SAVE_HISTORY_LIMIT) - - # Sequence id must survive GUI restarts so history commands stay monotonic. - self._history_command_seq = self._load_history_command_seq(self._live_config_writer.path) - self._bscan_cpp_replay_window = bscan_cpp_replay_window - self._bscan_history_by_combo = {} - self._bscan_depth_axis_by_combo = {} - self._bscan_render_signature = None - # Writer into the processor's input ring, used to re-feed retained sweeps so the - # whole visible B-scan is recomputed. Opened lazily, only while acquisition is - # stopped (see AppWindowBscanReplayMixin). - self._replay_ring_writer = None - self._bscan_replay_active = False - # Results from the last completed replay. While non-empty they, not - # `_result_history`, are what the B-scan renders (see AppWindowBscanReplayMixin). - self._bscan_replay_results = [] - self._bscan_reprocess_timer = QTimer(self) - self._bscan_reprocess_timer.setSingleShot(True) - self._bscan_reprocess_timer.timeout.connect(self._reprocess_history_through_processor) - self._gpr_lookup_table = None - self._gpr_image_item = None - self._gpr_tx_item = None - self._gpr_rx_item = None - self._gpr_points_item = None - self._gpr_region_centers_item = None - self._gpr_point_labels = [] - self._gpr_region_center_labels = [] - self._gpr_region_mask_items = [] - self._gpr_region_contours = [] - self._gpr_geometry_signature = None - self._gpr_selected_geometry = None - self._history_run_signature = None - self._processor_run_signature = None - self._active_processing_mode = "pass_through" - self._radar_limits: dict[str, float | int] | None = None - - def _init_runtime_limits(self) -> None: - """Initialize read/drain loop limits used by polling and snapshot code.""" - self._max_pop_per_poll = 256 - self._max_pop_per_snapshot_drain = 4096 - self._last_reader_error_signature: tuple[str, str] | None = None - self._logged_once_keys: set[str] = set() - - def _init_polling_timer(self) -> None: - """Create periodic timer that polls SHM rings for new data.""" - self._timer = QTimer(self) - self._timer.setInterval(50) - self._timer.timeout.connect(self._poll_rings) - - def _bootstrap_ui_runtime(self) -> None: - """Build the UI and apply initial runtime-bound state once widgets exist. - - This is the first point at which the log panel is wired in, so it also - replays any startup entries buffered during the headless init phase. - """ - self._build_ui() - self._attach_log_panel() - self._flush_pending_startup_log_entries() - self._log(f"Active config profile: {self._active_profile_path}") - self._refresh_preprocess_summary_labels() - self._apply_initial_radar_limits() - self._on_processing_mode_changed(self._processing_mode.currentText()) - self._write_live_processing_config() - # Defer the log sink wiring until the runtime log widget exists. - self._pipeline_metrics.set_log_sink(self._log) - self._timer.start() - self._maybe_auto_start_pipeline() - self._start_control_button_watcher() - self._init_web_ui() - if self._is_truthy_env("RADAR_SYSTEM_HEADLESS"): - self._install_headless_watchdog() - self._log_debug( - f"Window bootstrap complete (mode={self._active_processing_mode}, " - f"ring poll interval={self._timer.interval()}ms)." + self._snapshots = SnapshotController( + self._pipeline, store, write_profile, + destination=self._config.dataset_destination, build_config=lambda: self._profile.build_config(), + processing_settings=published, allowed=lambda deletion: self._preprocess.data_action_allowed(deletion=deletion), + log=self._log.info, error=self._log.show_exception, ) - - def _resolve_startup_profile_path(self) -> Path: - """Resolve active profile path from session-state or root fallback path.""" - env_profile_path = os.environ.get("RADAR_SYSTEM_PROFILE", "").strip() - if env_profile_path: - profile_path = Path(env_profile_path).expanduser() - if not profile_path.is_absolute(): - profile_path = (self._project_root / profile_path).resolve(strict=False) - return profile_path - - try: - session_state = self._gui_session_state_store.load() - except Exception as exc: - self._queue_startup_log_entry( - "WARN", - "Failed to read GUI session-state; using root run_config.json.", - details=self._exception_details(exc), - ) - return self._root_profile_path - - raw_path = session_state.last_profile_path.strip() - if not raw_path: - return self._root_profile_path - - profile_path = Path(raw_path).expanduser() - if not profile_path.is_absolute(): - profile_path = (self._project_root / profile_path).resolve(strict=False) - return profile_path - - def _maybe_auto_start_pipeline(self) -> None: - """Schedule pipeline start when requested by launcher environment. - - With `RADAR_SYSTEM_AUTO_APPLY_RADAR=1` the launcher also reproduces the - "Apply Radar" click before "Start". This is the headless deployment - recipe: the GUI configures the device exactly as a human operator - would, then starts the capture pipeline. - """ - auto_start = self._is_truthy_env("RADAR_SYSTEM_AUTO_START") - if not auto_start: - return - self._log("Auto-start requested by launcher.") - if self._is_truthy_env("RADAR_SYSTEM_AUTO_APPLY_RADAR"): - QTimer.singleShot(500, self._auto_apply_radar_then_start) - else: - QTimer.singleShot(500, self._auto_start_pipeline_step) - - def _auto_apply_radar_then_start(self) -> None: - """Apply current radar settings then start the pipeline (headless boot).""" - try: - self._apply_radar_settings() - except Exception as exc: # noqa: BLE001 - self._log_exception("Auto apply-radar failed", exc, level="WARN") - # Hand control back to the event loop so widget updates from - # _apply_radar_settings can flush, then wait one second before the - # start takes over so the device settles after apply-radar. - QTimer.singleShot(1000, self._auto_start_pipeline_step) - - def _auto_start_pipeline_step(self) -> None: - """Run the launcher-requested pipeline start. - - In headless mode a start that does not bring the pipeline up is fatal: we - exit non-zero so `systemd Restart=on-failure` restarts the unit instead of - leaving an idle daemon producing nothing. (The producer itself waits for - the device forever, so a live-but-deviceless producer counts as running.) - """ - self._start_run() - if self._is_truthy_env("RADAR_SYSTEM_HEADLESS") and not self._supervisor.is_running(): - self._headless_fatal("Headless auto-start did not bring the pipeline up") - - def _install_headless_watchdog(self) -> None: - """Self-heal a headless daemon: if a managed pipeline process crashes (exits - without us stopping it), exit non-zero so the service restarts clean. - - Intentional stops drop processes from the supervisor first, so a normal - stop/start or tmp-reference transition never trips this. - """ - self._headless_watchdog = QTimer(self) - self._headless_watchdog.setInterval(2000) - self._headless_watchdog.timeout.connect(self._headless_watchdog_tick) - self._headless_watchdog.start() - self._log_debug("Headless watchdog armed (interval=2000ms).") - - def _headless_watchdog_tick(self) -> None: - """Escalate any unexpected managed-process exit to a fatal headless restart.""" - crashed = [ - report - for report in self._supervisor.collect_exit_reports() - if not report.expected_clean_exit - ] - if crashed: - names = ", ".join(report.name for report in crashed) - details = "\n\n".join(report.format() for report in crashed) - self._headless_fatal(f"Pipeline process exited unexpectedly: {names}", details=details) - - def _headless_fatal(self, reason: str, *, details: str | None = None) -> None: - """Log loudly to stderr and exit non-zero so systemd restarts the service. - - Headless deployments have no operator and the in-app log only reaches an - offscreen widget, so a dead pipeline would otherwise go unnoticed. - """ - self._log_error(reason, details=details) - print(f"[radar] FATAL (headless): {reason}", file=sys.stderr, flush=True) - if details: - print(details, file=sys.stderr, flush=True) - app = QApplication.instance() - if app is not None: - app.exit(1) - - @staticmethod - def _is_truthy_env(name: str) -> bool: - """Return True when an environment variable is set to a truthy literal.""" - return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"} - - def _normalize_profile_path(self, path: Path) -> Path: - """Return normalized absolute profile path.""" - return path.expanduser().resolve(strict=False) - - def _remember_active_profile_path(self, path: Path, *, startup: bool = False) -> None: - """Persist the last successfully used config profile path to GUI session-state. - - A write failure is non-fatal: it is queued during startup or logged as a - warning afterwards, since it only affects which profile reopens next launch. - """ - normalized_path = self._normalize_profile_path(path) - self._active_profile_path = normalized_path - try: - self._gui_session_state_store.write(GuiSessionState(last_profile_path=str(normalized_path))) - except Exception as exc: - if startup: - self._queue_startup_log_entry( - "WARN", - "Failed to update GUI session-state with the active config profile path.", - details=self._exception_details(exc), - ) - else: - self._log_exception( - "Failed to update GUI session-state with the active config profile path", - exc, - level="WARN", - ) - - def _queue_startup_log_entry(self, level: str, text: str, *, details: str | None = None) -> None: - """Queue startup log entry until log widget exists.""" - self._pending_startup_log_entries.append((level.upper(), text, details)) - - def _flush_pending_startup_log_entries(self) -> None: - """Replay queued startup entries through the logger now that every sink exists. - - Entries logged before the panel widget existed were buffered; routing them - through the logger here delivers them to the file/console and the panel at once. - """ - levels = {"WARN": logging.WARNING, "ERROR": logging.ERROR} - for level, text, details in self._pending_startup_log_entries: - self._gui_logger.log(levels.get(level, logging.INFO), text, extra=_panel_extra(details, None)) - self._pending_startup_log_entries.clear() - - def _apply_initial_radar_limits(self) -> None: - """Apply the startup radar-limits strategy according to the selected radar mode. - - In ``native`` mode the limits are queried from the connected device; otherwise - the UI is populated with no device-imposed limits. - """ - if self._defaults_config.radar.driver_mode == "native": - self._log_debug("Querying radar limits from device (native driver mode).") - self._refresh_radar_limits_from_device() - return - self._log_debug( - f"Skipping device radar-limit query (driver mode={self._defaults_config.radar.driver_mode})." + self._radar = RadarSettingsController( + self._view.radar, self._view.switches, self._view.processing, lambda: state.config, self._log, + reset_preprocess=self._preprocess.reset_selection, live_changed=lambda: self._live.changed(), ) - self._apply_radar_limits_to_ui(None) - - @staticmethod - def _escape_log_text(text: str) -> str: - """Escape log text for insertion into rich-text log widget.""" - return html.escape(text).replace("\n", "
") - - @staticmethod - def _exception_summary(exc: Exception) -> str: - """Build compact one-line exception summary.""" - message = str(exc).strip() - if message: - return f"{type(exc).__name__}: {message}" - return type(exc).__name__ - - @staticmethod - def _exception_details(exc: Exception) -> str: - """Return full chained traceback for error dialogs and log details.""" - return "".join(traceback.TracebackException.from_exception(exc).format(chain=True)).strip() - - def _render_log_entry_html(self, level: str, text: str, details: str | None = None) -> str: - """Render one log entry as the panel's HTML block.""" - level_upper = level.upper() - palette = { - "DEBUG": ("#6c7b8d", "#52627a", "#8a97a8"), - "INFO": ("#1d5fbf", "#1f2937", "#526277"), - "WARN": ("#9a5b00", "#5c4300", "#7a6640"), - "ERROR": ("#c43d4d", "#6b1f2a", "#8b5d66"), - } - accent_color, message_color, detail_color = palette.get(level_upper, palette["INFO"]) - timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] - header_html = ( - f"{html.escape(level_upper)}" - f" {html.escape(timestamp)}" - f" {self._escape_log_text(text)}" + self._live = LiveSettingsController( + self._view.processing, self._config, self._pipeline, self._preprocess, self._radar, + ProcessingLiveConfigWriter(runtime / "processing_live.json"), published, self._log, + build_config=lambda: self._profile.build_config(), ) - - body_parts = [header_html] - if details: - body_parts.append( - "
{html.escape(details)}
" - ) - return "
" + "".join(body_parts) + "
" - - def _on_log_level_selected(self, level_text: str) -> None: - """Apply the chosen log level immediately (sub-level logs stop being generated). - - The value lives in run_config like any other field: it is recorded in the active - config here and saved with the config through the normal path — no separate write. - """ - level = level_text.strip().lower() - set_log_level(level) - self._defaults_config.logging.level = level - self._log(f"Log level set to {level.upper()}.") - - def _log_debug(self, text: str, *, once_key: str | None = None) -> None: - """Log a diagnostic message (emitted only while the level is DEBUG).""" - self._gui_logger.debug(text, extra=_panel_extra(None, once_key)) - - def _log(self, text: str, *, once_key: str | None = None) -> None: - """Log an informational message to the panel, file, and console.""" - self._gui_logger.info(text, extra=_panel_extra(None, once_key)) - - def _log_warning(self, text: str, *, details: str | None = None, once_key: str | None = None) -> None: - """Log a warning to the panel, file, and console.""" - self._gui_logger.warning(text, extra=_panel_extra(details, once_key)) - - def _log_error(self, text: str, *, details: str | None = None, once_key: str | None = None) -> None: - """Log an error to the panel, file, and console.""" - self._gui_logger.error(text, extra=_panel_extra(details, once_key)) - - def _log_exception(self, context: str, exc: Exception, *, level: str = "ERROR") -> tuple[str, str]: - """Log exception with detailed traceback and return `(message, details)`.""" - message = f"{context}: {self._exception_summary(exc)}" - details = self._exception_details(exc) - if level.upper() == "WARN": - self._log_warning(message, details=details) - else: - self._log_error(message, details=details) - return message, details - - def _process_state_details(self) -> str: - """Return formatted summary of managed pipeline process state.""" - if not hasattr(self, "_supervisor"): - return "Managed processes: unavailable" - pid_map = self._supervisor.pids() - if not pid_map: - return "Managed processes: none" - return "Managed processes:\n" + "\n".join( - f"- {name}: pid={pid}" - for name, pid in sorted(pid_map.items()) + self._profile = ProfileController(self, project_root, state, self._config, self._preprocess, + self._pipeline, self._radar, self._live, self._log) + self._control_button = ControlButtonController(self, lambda: state.config.control_button, + self._preprocess.capture_tmp_reference, self._log) + self._web = WebBridge( + self, project_root / "run_configs", self._pipeline, self._preprocess, self._snapshots, + self._profile, self._live, self._log, + save_path=self._view.data.save_path, + save_name=self._view.data.save_name, + record_count=self._view.data.record_count, ) - - def _runtime_history_details(self) -> str: - """Return formatted summary of buffered runtime history counts.""" - return ( - "Runtime history:\n" - f"- raw={len(getattr(self, '_raw_history', []))}\n" - f"- preprocessed={len(getattr(self, '_pre_history', []))}\n" - f"- results={len(getattr(self, '_result_history', []))}" - ) - - def _capture_state_details(self) -> str: - """Return formatted summary of active preprocess capture state.""" - session = getattr(self, "_capture_session", None) - if session is None: - return "Capture session: none" - state = session.state() - lines = [ - "Capture session:", - f"- kind={state.kind}", - f"- progress={state.captured_count}/{state.total_count}", - ] - if state.current_combo is not None: - lines.append( - f"- current_combo=input={state.current_combo.input}, output={state.current_combo.output}" - ) - return "\n".join(lines) - - @staticmethod - def _load_history_command_seq(config_path: Path) -> int: - """Load previously used live-command sequence from runtime config file.""" - try: - payload = json.loads(config_path.read_text(encoding="utf-8")) - except Exception: # noqa: BLE001 - # Missing or malformed file should not block startup. - return 0 - - raw_value = payload.get("history_command_seq", 0) if isinstance(payload, dict) else 0 - return raw_value if type(raw_value) is int and 0 <= raw_value < (1 << 64) - 1 else 0 - - def _capture_web_action_error(self, message: str) -> bool: - """Record the first error of an in-flight web action so the browser can show it. - - Returns ``True`` if a web-triggered action is currently running (see - ``AppWindowWebMixin._run_web_action``). Callers use that to skip the blocking - desktop modal for web errors — the browser shows the message instead, and the - operator at the browser must not have to dismiss a popup on the (often headless) - host before the HTTP response returns. Desktop-only errors are unaffected. - """ - capture = getattr(self, "_web_action_error_capture", None) - if capture is None: - return False - if not capture: - capture.append(message) - return True - - def _show_error(self, message: str, *, details: str | None = None) -> None: - """Log and present an error in a modal dialog with optional detail text.""" - self._log_error(message, details=details) - if self._capture_web_action_error(message): - return # web-triggered: surfaced to the browser; no blocking desktop modal - if self._is_truthy_env("RADAR_SYSTEM_HEADLESS"): - return - dialog = QMessageBox(self) - dialog.setIcon(QMessageBox.Icon.Critical) - dialog.setWindowTitle("Error") - dialog.setText(message) - if details: - dialog.setDetailedText(details) - dialog.exec() - - def _show_exception(self, context: str, exc: Exception) -> None: - """Log full exception details and show modal dialog with expandable traceback.""" - message, details = self._log_exception(context, exc, level="ERROR") - if self._capture_web_action_error(message): - return # web-triggered: surfaced to the browser; no blocking desktop modal - if self._is_truthy_env("RADAR_SYSTEM_HEADLESS"): - return - dialog = QMessageBox(self) - dialog.setIcon(QMessageBox.Icon.Critical) - dialog.setWindowTitle("Error") - dialog.setText(message) - dialog.setDetailedText(details) - dialog.exec() + self._pipeline.bind_actions(self._view.primary) + self._snapshots.bind_actions(self._view.primary, self._view.data) + self._preprocess.bind_actions(self._view.primary, self._view.data) + self._profile.bind_actions(self._view.primary) + self._radar.sync_profile(state.gui.switches.combo_mode) + self._live.sync_controls() + self._view.update_preprocess(self._preprocess.selected_sets) + self._radar.initial_limits() + self._live.mode_changed(state.gui.processing.selected_mode) + self._live.publish() + self._pipeline.timer.start() + self._pipeline.schedule_startup() + self._control_button.start() + self._web.start_server() + self._log.info(f"Active config profile: {state.path}") def closeEvent(self, event) -> None: # noqa: N802 - """Tear down workers, readers, and dialogs before the window is destroyed. - - Guarded against re-entrancy so a second close signal (or a ``window.close()`` - after the event loop has already returned) does not run teardown twice. - """ - if self._stopping_run: - # A drain pumps Qt events. Finish ingesting/finalizing before teardown - # can close the same readers and recording writer beneath that drain. - self._close_after_stop = True - self._stop_all_requested = True + if self._closed: + super().closeEvent(event) + return + if not self._pipeline.begin_close(): event.ignore() return - if self._closing: - # Re-entrant close (second signal, or window.close() after the event loop - # already returned): teardown is in progress or done — do nothing more. - super().closeEvent(event) - return - self._closing = True - self._timer.stop() - self._log("Window closing; shutting down runtime.") + self._closed = True + self._log.info("Window closing; shutting down runtime") try: - # 0) Stop the web server first so a late request cannot start work. - self._shutdown_web_ui() - # 0) Stop the GPIO button watcher so a late press cannot start work. - self._stop_control_button_watcher() - self._resume_pipeline_after_capture = False - controller = getattr(self, "_object_calibration_controller", None) - if controller is not None: - controller.shutdown() - # 0) Drop the B-scan replay writer so a pending debounce cannot push into a - # ring we are about to tear down. - self._bscan_reprocess_timer.stop() - self._close_replay_ring_writer() - # 1) Abort active capture first (releases exclusive hardware resources). - self._abort_capture_sequence(resume_pipeline=False) - # Drain acquisition stages before finalizing the last recording chunk. - if self._recording_active: - self._stop_run() - # 2) Stop all managed processes/readers. - self._stop_all_processes() - # 3) Close auxiliary dialog windows. - if self._preprocess_dialog is not None: - self._preprocess_dialog.close() + for close in (self._web.shutdown, self._control_button.close, + self._preprocess.shutdown, self._pipeline.close, self._recording.finish): + try: + close() + except Exception as exc: + self._log.exception("Shutdown operation failed", exc) finally: - self._shutdown_recording() - self._log_debug("Window teardown finished.") + self._log.close() super().closeEvent(event) diff --git a/python_app/gui/controllers/__init__.py b/python_app/gui/controllers/__init__.py index 77adf32..cf79f6f 100644 --- a/python_app/gui/controllers/__init__.py +++ b/python_app/gui/controllers/__init__.py @@ -1 +1 @@ -"""Controller mixins and orchestration helpers for GUI windows.""" +"""Controllers with explicit GUI and runtime dependencies.""" diff --git a/python_app/gui/controllers/app_window_config/__init__.py b/python_app/gui/controllers/app_window_config/__init__.py deleted file mode 100644 index c31a79a..0000000 --- a/python_app/gui/controllers/app_window_config/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Focused mixins used by the AppWindow config facade.""" - -from python_app.gui.controllers.app_window_config.live_processing_mixin import ( - AppWindowLiveProcessingMixin, -) -from python_app.gui.controllers.app_window_config.profile_io_mixin import ( - AppWindowConfigProfileIOMixin, -) -from python_app.gui.controllers.app_window_config.radar_limits_mixin import ( - AppWindowRadarLimitsMixin, -) -from python_app.gui.controllers.app_window_config.state_builders import ( - AppWindowConfigStateBuildersMixin, -) - -__all__ = [ - "AppWindowConfigProfileIOMixin", - "AppWindowConfigStateBuildersMixin", - "AppWindowLiveProcessingMixin", - "AppWindowRadarLimitsMixin", -] diff --git a/python_app/gui/controllers/app_window_config/live_processing_mixin.py b/python_app/gui/controllers/app_window_config/live_processing_mixin.py deleted file mode 100644 index f8ed43d..0000000 --- a/python_app/gui/controllers/app_window_config/live_processing_mixin.py +++ /dev/null @@ -1,675 +0,0 @@ -"""Live-processing bindings and runtime reactions for the main window.""" - -from __future__ import annotations - -from contextlib import ExitStack -import math -import time - -from PyQt6.QtCore import QSignalBlocker -from PyQt6.QtWidgets import ( - QCheckBox, - QComboBox, - QDoubleSpinBox, - QLineEdit, - QSpinBox, -) - -from python_app.gui.runtime.constraints import validate_processing_mode_constraints -from python_app.gui.antenna_geometry_editor import AntennaGeometryEditor -from python_app.models.run_config_model import GprRxGeometryModel, GprTxGeometryModel -from python_app.orchestration.live_processing_config import ProcessingLiveConfig - -_GPR_MODES = ("gpr", "legacy_gpr") -# Modes that consume the shared gpr_* live-config fields (band, depth gate, antenna -# selection, motion): both GPR imagers plus the 3D SAR processor. -_GPR_LIKE_MODES = ("gpr", "legacy_gpr", "sar3d") - - -def _attr(name: str): - """Widget getter for a field backed by a single fixed widget.""" - return lambda window: getattr(window, name) - - -def _dual(default_attr: str, mode_attrs: dict[str, str] | None = None): - """Widget getter for a field with per-mode widgets (gpr / legacy_gpr / sar3d). - - `mode_attrs` maps a processor mode to its dedicated widget attribute; any - other mode (including inactive ones) falls back to `default_attr`. - """ - attrs = dict(mode_attrs or {}) - return lambda window: getattr( - window, attrs.get(window._processing_mode.currentText(), default_attr) - ) - - -# The ONE authoritative field <-> widget map (inverse of `_live_processing_config`), -# plus each field's display group and the processor mode(s) it applies to. Everything -# else the web form needs — input type, combobox options, numeric ranges, the current -# value, the enabled state — is read live FROM these widgets by `_build_web_settings_schema`, -# so the web hardcodes none of it and always mirrors the desktop. `modes=None` => all modes. -# Entries: (field, group, modes, widget_getter). -_WEB_LIVE_SCHEMA = [ - ("processor_mode", "Processor", None, _attr("_processing_mode")), - ("pass_through_fixed_y_enabled", "Pass-through", ("pass_through",), _attr("_pass_through_fixed_y_enabled")), - ("pass_through_y_min_db", "Pass-through", ("pass_through",), _attr("_pass_through_y_min_db")), - ("pass_through_y_max_db", "Pass-through", ("pass_through",), _attr("_pass_through_y_max_db")), - ("bscan_axis", "B-scan", ("bscan",), _attr("_bscan_axis")), - ("bscan_cut_m", "B-scan", ("bscan",), _attr("_bscan_cut_m")), - ("bscan_max_depth_m", "B-scan", ("bscan",), _attr("_bscan_max_depth_m")), - ("bscan_gain", "B-scan", ("bscan",), _attr("_bscan_gain")), - ("bscan_start_freq_mhz", "B-scan", ("bscan",), _attr("_bscan_start_freq_mhz")), - ("bscan_stop_freq_mhz", "B-scan", ("bscan",), _attr("_bscan_stop_freq_mhz")), - ("legacy_gpr_mode", "Mode", ("legacy_gpr",), _attr("_legacy_gpr_config_mode")), - ("relative_permittivity", "Geometry & medium", ("legacy_gpr",), _attr("_gpr_relative_permittivity")), - ("tx_geometry", "Geometry & medium", _GPR_LIKE_MODES, _attr("_gpr_tx_geometry_input")), - ("rx_geometry", "Geometry & medium", _GPR_LIKE_MODES, _attr("_gpr_rx_geometry_input")), - ("gpr_input_positions", "Geometry & depth", _GPR_LIKE_MODES, _dual("_gpr_input_positions_input", {"legacy_gpr": "_legacy_gpr_input_positions_input", "sar3d": "_sar3d_input_positions_input"})), - ("gpr_output_positions", "Geometry & depth", _GPR_LIKE_MODES, _dual("_gpr_output_positions_input", {"legacy_gpr": "_legacy_gpr_output_positions_input", "sar3d": "_sar3d_output_positions_input"})), - ("gpr_min_depth_m", "Geometry & depth", _GPR_LIKE_MODES, _dual("_gpr_min_depth_m", {"legacy_gpr": "_legacy_gpr_min_depth_m", "sar3d": "_sar3d_min_depth_m"})), - ("gpr_max_depth_m", "Geometry & depth", _GPR_LIKE_MODES, _dual("_gpr_max_depth_m", {"legacy_gpr": "_legacy_gpr_max_depth_m", "sar3d": "_sar3d_max_depth_m"})), - ("gpr_start_freq_mhz", "Geometry & depth", _GPR_LIKE_MODES, _dual("_gpr_start_freq_mhz", {"legacy_gpr": "_legacy_gpr_start_freq_mhz", "sar3d": "_sar3d_start_freq_mhz"})), - ("gpr_stop_freq_mhz", "Geometry & depth", _GPR_LIKE_MODES, _dual("_gpr_stop_freq_mhz", {"legacy_gpr": "_legacy_gpr_stop_freq_mhz", "sar3d": "_sar3d_stop_freq_mhz"})), - ("gpr_imaging_plane_y_m", "Geometry & depth", ("gpr",), _attr("_gpr_imaging_plane_y_m")), - ("gpr_range_comp_power", "Imaging", ("gpr",), _attr("_gpr_range_comp_power")), - ("gpr_score_mode", "Imaging", ("gpr",), _attr("_gpr_score_mode")), - ("gpr_background_subtract_enabled", "Imaging", _GPR_MODES, _dual("_gpr_background_subtract_enabled", {"legacy_gpr": "_legacy_gpr_background_subtract_enabled"})), - ("gpr_background_mean_count", "Imaging", _GPR_MODES, _dual("_gpr_background_mean_count", {"legacy_gpr": "_legacy_gpr_background_mean_count"})), - ("gpr_remove_sidelobe_objects_enabled", "Imaging", ("gpr",), _attr("_gpr_remove_sidelobe_objects_enabled")), - ("gpr_object_min_frac", "Detection", ("gpr",), _attr("_gpr_object_min_frac")), - ("gpr_max_detected_objects_to_draw", "Detection", ("gpr",), _attr("_gpr_max_detected_objects_to_draw")), - ("gpr_draw_top_m_objects", "Detection", ("gpr",), _attr("_gpr_draw_top_m_objects")), - ("gpr_object_approach_min_frames", "Detection", ("gpr",), _attr("_gpr_object_approach_min_frames")), - ("gpr_comp_power", "Detection", ("legacy_gpr",), _attr("_legacy_gpr_comp_power")), - ("gpr_snr_thresh", "Detection", ("legacy_gpr",), _attr("_legacy_gpr_snr_thresh")), - ("gpr_snr_comp_max", "Detection", ("legacy_gpr",), _attr("_legacy_gpr_snr_comp_max")), - ("legacy_gpr_min_visible_pair_count", "Detection", ("legacy_gpr",), _attr("_legacy_gpr_min_visible_pair_count")), - ("gpr_visible_x_min_m", "Visible window", _GPR_MODES, _dual("_gpr_visible_x_min_m", {"legacy_gpr": "_legacy_gpr_visible_x_min_m"})), - ("gpr_visible_x_max_m", "Visible window", _GPR_MODES, _dual("_gpr_visible_x_max_m", {"legacy_gpr": "_legacy_gpr_visible_x_max_m"})), - ("gpr_visible_z_min_m", "Visible window", _GPR_MODES, _dual("_gpr_visible_z_min_m", {"legacy_gpr": "_legacy_gpr_visible_z_min_m"})), - ("gpr_visible_z_max_m", "Visible window", _GPR_MODES, _dual("_gpr_visible_z_max_m", {"legacy_gpr": "_legacy_gpr_visible_z_max_m"})), - ("gpr_motion_mode", "Motion", ("gpr",), _attr("_gpr_motion_mode")), - ("gpr_look_angle_deg", "Motion", _GPR_LIKE_MODES, _dual("_gpr_look_angle_deg", {"legacy_gpr": "_legacy_gpr_look_angle_deg", "sar3d": "_sar3d_look_angle_deg"})), - ("gpr_direction_sign", "Motion", ("gpr",), _attr("_gpr_direction_sign")), - ("gpr_apply_freq_phase_correction", "Motion", ("legacy_gpr",), _attr("_legacy_gpr_apply_freq_phase_correction")), - ("gpr_reference_mode", "Motion", ("legacy_gpr",), _attr("_legacy_gpr_reference_mode")), - ("ignore_socket_speed", "Motion", _GPR_LIKE_MODES, _dual("_gpr_ignore_socket_speed_enabled", {"legacy_gpr": "_legacy_gpr_ignore_socket_speed_enabled", "sar3d": "_sar3d_ignore_socket_speed_enabled"})), - ("gpr_speed_m_s", "Motion", _GPR_LIKE_MODES, _dual("_gpr_speed_m_s", {"legacy_gpr": "_legacy_gpr_speed_m_s", "sar3d": "_sar3d_speed_m_s"})), - ("sar3d_frame_radius", "3D SAR", ("sar3d",), _attr("_sar3d_frame_radius")), - ("sar3d_voxel_m", "3D SAR", ("sar3d",), _attr("_sar3d_voxel_m")), - ("sar3d_bp_ground_depth_m", "3D SAR", ("sar3d",), _attr("_sar3d_bp_ground_depth_m")), - ("sar3d_comp_range_power", "3D SAR", ("sar3d",), _attr("_sar3d_comp_range_power")), - ("sar3d_use_compensation", "3D SAR", ("sar3d",), _attr("_sar3d_use_compensation")), - ("sar3d_normalize_each_frame", "3D SAR", ("sar3d",), _attr("_sar3d_normalize_each_frame")), -] -_WEB_LIVE_GETTERS = {field: getter for field, _group, _modes, getter in _WEB_LIVE_SCHEMA} - -# ProcessingLiveConfig fields that are NOT backed by a settings widget and are set -# directly in `_live_processing_config`: the fixed acquisition channels, the runtime -# command fields, and a constant default. Together with the schema's fields these must -# cover every config field (guarded by a unit test), so the desktop config and the web -# form can never drift apart. -_NON_WIDGET_LIVE_FIELDS = frozenset({ - "pass_through_channel", - "bscan_channel", - "reprocess_current_result", - "history_command", - "history_command_seq", - "settings_revision", -}) - -# Display toggles: GUI-only rendering choices (kept in the GUI profile, not the live -# config). They apply immediately through the same redraw as a live edit but write -# nothing to processing_live.json. -_WEB_DISPLAY_SCHEMA = [ - ("render_mode", "Display", _GPR_MODES, _dual("_gpr_render_mode", {"legacy_gpr": "_legacy_gpr_render_mode"})), - ("subtract_mean_ascan", "Display", ("bscan",), _attr("_bscan_subtract_mean_ascan")), - ("combo_filter", "Display", ("pass_through",), _attr("_pass_through_combo_filter_input")), -] - -# Every field the web form renders and may write back. -_WEB_FORM_GETTERS = { - field: getter - for source in (_WEB_LIVE_SCHEMA, _WEB_DISPLAY_SCHEMA) - for field, _group, _modes, getter in source -} - - -def web_apply_field_names() -> frozenset[str]: - """Return the field names a web client may apply (form fields plus runtime commands).""" - return frozenset(_WEB_FORM_GETTERS) | {"history_command"} - - -def _web_field_schema(field: str, group: str, widget) -> dict | None: - """Describe one widget for the web form (type, options/range, value, enabled).""" - base = {"name": field, "group": group, "enabled": bool(widget.isEnabled())} - if isinstance(widget, QComboBox): - return {**base, "kind": "select", "value": widget.currentText(), - "options": [widget.itemText(i) for i in range(widget.count())]} - if isinstance(widget, QCheckBox): - return {**base, "kind": "bool", "value": bool(widget.isChecked())} - if isinstance(widget, QSpinBox): - return {**base, "kind": "int", "value": int(widget.value()), - "min": int(widget.minimum()), "max": int(widget.maximum()), "step": int(widget.singleStep())} - if isinstance(widget, QDoubleSpinBox): - return {**base, "kind": "float", "value": float(widget.value()), - "min": float(widget.minimum()), "max": float(widget.maximum()), - "step": float(widget.singleStep()), "decimals": int(widget.decimals())} - if isinstance(widget, AntennaGeometryEditor): - return {**base, "kind": "geometry", "value": widget.records(), - "position_field": widget.position_field, "position_count": widget.position_count} - if isinstance(widget, QLineEdit): - return {**base, "kind": "text", "value": widget.text()} - return None - - -def _read_live_widget_value(widget): - """Return a widget's current value for the live config, by widget type. - - Shares the same type dispatch as the web schema/setter, so the desktop config - build and the browser form read every widget identically. - """ - if isinstance(widget, QComboBox): - return widget.currentText() - if isinstance(widget, QCheckBox): - return bool(widget.isChecked()) - if isinstance(widget, QSpinBox): - return int(widget.value()) - if isinstance(widget, QDoubleSpinBox): - return float(widget.value()) - if isinstance(widget, QLineEdit): - return widget.text() - if isinstance(widget, AntennaGeometryEditor): - return widget.records() - raise TypeError(f"Unsupported live-settings widget: {type(widget).__name__}") - - -def _build_web_settings_schema(window) -> list[dict]: - """Build the settings schema for the active mode straight from the Qt widgets. - - Covers live settings and GUI display toggles, matching the desktop panel. - """ - mode = window._processing_mode.currentText() - schema: list[dict] = [] - for source in (_WEB_LIVE_SCHEMA, _WEB_DISPLAY_SCHEMA): - for field, group, modes, getter in source: - if modes is not None and mode not in modes: - continue - entry = _web_field_schema(field, group, getter(window)) - if entry is not None: - schema.append(entry) - return schema - - -def _set_widget_value(window, widget, value) -> None: - """Validate and write a web value, also used when rolling back a rejected batch.""" - if isinstance(widget, QComboBox): - if widget.findText(str(value)) < 0: - raise ValueError(f"Unknown setting option: {value}") - window._set_combo_current_text(widget, str(value)) - elif isinstance(widget, QCheckBox): - if not isinstance(value, bool): - raise ValueError("Checkbox value must be true or false") - widget.setChecked(value) - elif isinstance(widget, (QSpinBox, QDoubleSpinBox)): - numeric = float(value) - if not math.isfinite(numeric) or not widget.minimum() <= numeric <= widget.maximum(): - raise ValueError("Numeric setting is outside its allowed range") - if isinstance(widget, QSpinBox) and numeric != int(numeric): - raise ValueError("Setting requires an integer") - widget.setValue(int(numeric) if isinstance(widget, QSpinBox) else numeric) - elif isinstance(widget, AntennaGeometryEditor): - widget.set_records(value) - elif isinstance(widget, QLineEdit): - widget.setText(",".join(str(int(v)) for v in value) if isinstance(value, list) else str(value)) - - -class AppWindowLiveProcessingMixin: - """Handle live processing updates, redraws, and locator republishing.""" - - @staticmethod - def _is_gpr_processing_mode(mode: str) -> bool: - """Return whether a mode uses GPR result payloads and plot surface.""" - return mode in {"gpr", "legacy_gpr"} - - @staticmethod - def _is_gpr_like_processing_mode(mode: str) -> bool: - """Return whether a mode consumes the shared gpr_* fields and Tx/Rx geometry. - - Covers both GPR imagers and the 3D SAR processor. The latter renders on its - own plot surface, so GPR-object/locator branches must keep using - :meth:`_is_gpr_processing_mode`; this predicate gates only the shared - behaviors (result drain on settings changes, geometry page visibility). - """ - return mode in _GPR_LIKE_MODES - - def _validate_processing_mode_selection(self, mode: str) -> None: - """Validate requested processing mode against current stable/live GUI state.""" - validate_processing_mode_constraints( - mode, - self._build_config(), - self._live_processing_config(), - ) - - def _live_processing_config( - self, - *, - history_command: str = "none", - reprocess_current_result: bool = True, - ) -> ProcessingLiveConfig: - """Build the live processing config from the current processing widgets. - - Widget-backed fields are read through the single ``_WEB_LIVE_SCHEMA`` map — the - same source the embedded web form uses — so the desktop config and the browser - form can never expose different settings. The few fields that need - post-processing (position lists, the ordered pass-through Y range, the - normalized visible window) or are not widget-backed (the fixed acquisition - channels, the history command, the reprocess flag) are applied explicitly here. - """ - self._sync_bscan_frequency_limits_with_radar() - self._sync_gpr_frequency_limits_with_radar() - - values = { - field: _read_live_widget_value(getter(self)) - for field, _group, _modes, getter in _WEB_LIVE_SCHEMA - } - - # Switch-position CSV text -> integer lists. - values["gpr_input_positions"] = self._parse_csv_int_list(values["gpr_input_positions"]) - values["gpr_output_positions"] = self._parse_csv_int_list(values["gpr_output_positions"]) - # Order-normalize the pass-through Y range so a reversed entry still works. - y_min_db, y_max_db = values["pass_through_y_min_db"], values["pass_through_y_max_db"] - values["pass_through_y_min_db"] = min(y_min_db, y_max_db) - values["pass_through_y_max_db"] = max(y_min_db, y_max_db) - # Visible window: ordered with a non-zero span, for the active mode's widgets. - ( - values["gpr_visible_x_min_m"], - values["gpr_visible_x_max_m"], - values["gpr_visible_z_min_m"], - values["gpr_visible_z_max_m"], - ) = self._gpr_visible_bounds() - - return ProcessingLiveConfig( - pass_through_channel="s21", - bscan_channel="s21", - reprocess_current_result=bool(reprocess_current_result), - history_command=str(history_command), - history_command_seq=int(self._history_command_seq), - settings_revision=getattr(self, "_processing_settings_revision", 0), - **values, - ) - - def _write_live_processing_config( - self, - *, - history_command: str = "none", - bump_history_seq: bool = False, - ) -> None: - """Persist current live processing config to runtime JSON file. - - Every change of the file makes the processor reprocess its retained sweeps, - except while the GUI recomputes the B-scan itself by feeding its own, longer - history back (AppWindowBscanReplayMixin): the processor's replay would then - only duplicate that work and race it for the results ring. - """ - if bump_history_seq: - self._history_command_seq += 1 - self._processing_settings_revision = time.monotonic_ns() - self._live_config_writer.write( - self._live_processing_config( - history_command=history_command, - reprocess_current_result=not self._can_reprocess_history(), - ) - ) - active = getattr(self, "_active_run_config", None) - if active is not None and self._supervisor.is_running(): - active.gpr.tx_geometry = [GprTxGeometryModel(**entry) for entry in self._gpr_tx_geometry_input.records()] - active.gpr.rx_geometry = [GprRxGeometryModel(**entry) for entry in self._gpr_rx_geometry_input.records()] - if getattr(self, "_await_processing_revision", False): - self._required_processing_revision = self._processing_settings_revision - - def _apply_web_live_settings(self, fields: dict) -> None: - """Apply web-requested live settings through the SAME path as a desktop edit. - - Writes the values into the desktop widgets, then runs the single - ``_on_processing_live_settings_changed`` handler — one writer, one redraw — - so the browser and the desktop never desync and ``processing_live.json`` has - exactly one author. The per-widget change signals are neutralized by a - suppression flag so the final handler runs once instead of dozens of times. - - A destructive history command is not a settings edit: it routes to the very - same handler the desktop's "Clear History" / "Remove Last" button uses, so the - browser drains the ring readers, empties the runtime histories, clears the - render caches and the processor's replay state, and redraws — rather than only - nudging the processor while the on-screen history stays put. - """ - try: - history_command = str(fields.get("history_command", "none")) - if history_command not in {"none", "clear_all", "remove_last"}: - raise ValueError("Unknown history command") - if history_command != "none" and set(fields) != {"history_command"}: - raise ValueError("History commands must be sent separately from settings") - if history_command == "clear_all": - self._clear_all_runtime_history() - return - if history_command == "remove_last": - self._remove_last_runtime_history() - return - - settings = { - name: value - for name, value in fields.items() - if name not in {"history_command", "history_command_seq"} - } - # Log only field names (not values) so remote edits are traceable - # without recording arbitrary client-supplied payloads. - self._log_debug(f"Applying web live settings: fields={sorted(settings)}.") - self._suppress_live_settings_handler = True - try: - with ExitStack() as blockers: - snapshots = [] - try: - # Set the mode under a blocker before resolving per-mode fields. - ordered = sorted(settings, key=lambda name: name != "processor_mode") - for name in ordered: - widget = _WEB_FORM_GETTERS[name](self) - blockers.enter_context(QSignalBlocker(widget)) - snapshots.append((widget, _read_live_widget_value(widget))) - _set_widget_value(self, widget, settings[name]) - self._validate_processing_mode_selection(self._processing_mode.currentText()) - except Exception: - for widget, value in reversed(snapshots): - _set_widget_value(self, widget, value) - raise - finally: - self._suppress_live_settings_handler = False - self._sync_socket_speed_controls() - if {"tx_geometry", "rx_geometry"} & settings.keys(): - self._invalidate_object_calibration_after_geometry_change() - if {"tx_geometry", "rx_geometry", "relative_permittivity"} & settings.keys(): - self._await_processing_revision = True - if "processor_mode" in settings: - self._on_processing_mode_changed(self._processing_mode.currentText()) - else: - self._on_processing_live_settings_changed() - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to apply web live settings", exc) - - def _web_settings_schema(self) -> list[dict]: - """Return the live-settings schema for the active mode (read from widgets).""" - return _build_web_settings_schema(self) - - def _on_gpr_geometry_changed(self) -> None: - if getattr(self, "_suppress_live_settings_handler", False): - return - self._invalidate_object_calibration_after_geometry_change() - self._await_processing_revision = True - self._on_processing_live_settings_changed() - - def _invalidate_object_calibration_after_geometry_change(self) -> None: - """Disable a correction tied to another antenna layout and reload the preprocessor.""" - selected = self._selected_preprocess_sets.get("s21_object_calibration", "") - was_running = self._supervisor.is_running() - active_config = getattr(self, "_active_run_config", None) - active = active_config.preprocess.s21.object_calibration.set_name if was_running and active_config else "" - if not selected and not active: - return - from python_app.workflows.mimo_calibration import validate_for_config - - geometry = {"gpr": { - "tx_geometry": self._gpr_tx_geometry_input.records(), - "rx_geometry": self._gpr_rx_geometry_input.records(), - }} - selections = [(selected, self._selected_preprocess_radar_key)] - if active: - selections.append((active, self._radar_key(active_config))) - invalid = set() - for name, radar_key in dict.fromkeys(selections): - if not name: - continue - try: - validate_for_config(self._store.load_object_calibration(radar_key, name), geometry) - except Exception as exc: # noqa: BLE001 - reject unavailable or incompatible saved corrections - invalid.add(name) - self._log_warning(f"Object calibration {name} disabled for changed antenna geometry: {exc}") - if not invalid: - return - resume_seq = getattr(self, "_stop_request_seq", 0) + 1 - if active in invalid: - self._stop_run() - if selected in invalid: - self._selected_preprocess_sets["s21_object_calibration"] = "" - self._refresh_preprocess_summary_labels() - if self._preprocess_dialog is not None: - self._preprocess_dialog.set_selected_sets(self._selected_preprocess_sets, emit_signal=False) - if active in invalid and self._pipeline_resume_allowed(resume_seq): - self._start_run() - - def _on_processing_live_settings_changed(self, *_args) -> None: - """Handle live-processing setting changes and trigger redraw when needed.""" - # Web-apply sets many widgets at once; their individual change signals are - # suppressed so this writer+redraw runs exactly once at the end. - if getattr(self, "_suppress_live_settings_handler", False): - return - try: - current_mode = self._processing_mode.currentText() - if self._is_gpr_like_processing_mode(current_mode): - self._drain_results_until_quiet(timeout_s=0.05, poll_s=0.005) - - self._write_live_processing_config() - if current_mode == "bscan": - if self._can_reprocess_history(): - # Recompute every column on screen from the GUI's history. Debounced, - # so dragging a spin box sends one burst instead of one per step. - self._schedule_bscan_history_reprocess() - else: - # The processor reprocesses its own newest sweeps; take them in. - self._drain_results_until_quiet(timeout_s=0.25, poll_s=0.01) - self._sync_bscan_history_from_results() - self._draw_bscan_heatmap_from_history() - elif self._is_gpr_like_processing_mode(current_mode): - latest = self._drain_results_until_quiet(timeout_s=0.8, poll_s=0.02) - collection = latest - if collection is None and self._result_history: - collection = self._result_history[-1] - if current_mode == "sar3d": - self._clear_sar3d_plot() - if collection is not None: - self._draw_results(collection) - elif collection is None or not self._draw_results(collection): - self._clear_gpr_plot() - elif self._result_history: - self._draw_results(self._result_history[-1]) - else: - self._clear_trace_plots() - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to update live processing settings", exc) - - def _sync_socket_speed_controls(self) -> None: - """Enable each GPR-like speed field only while its page ignores the socket speed. - - Unchecked, the speed a locator client streams drives the processing and the - field only stands in while no client sends one, so it must not look editable. - """ - for checkbox, speed in ( - (self._gpr_ignore_socket_speed_enabled, self._gpr_speed_m_s), - (self._legacy_gpr_ignore_socket_speed_enabled, self._legacy_gpr_speed_m_s), - (self._sar3d_ignore_socket_speed_enabled, self._sar3d_speed_m_s), - ): - speed.setEnabled(checkbox.isChecked()) - - def _on_ignore_socket_speed_toggled(self, *_args) -> None: - """Switch the speed authority between the socket client and the GUI field.""" - self._sync_socket_speed_controls() - self._on_processing_live_settings_changed() - - def _on_gpr_visual_settings_changed(self, *_args) -> None: - """Redraw current GPR result using updated GUI-only render settings.""" - if not self._is_gpr_processing_mode(self._processing_mode.currentText()): - return - try: - if self._result_history and self._draw_results(self._result_history[-1]): - return - self._clear_gpr_plot() - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to update GPR render settings", exc) - - def _on_gpr_locator_threshold_changed(self, *_args) -> None: - """Redraw GPR view; locator delivery lives in the C++ data_processor.""" - self._on_gpr_visual_settings_changed() - if not self._is_gpr_processing_mode(self._processing_mode.currentText()): - return - try: - self._write_live_processing_config() - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to update locator GPR threshold", exc) - - def _on_gpr_locator_window_changed(self, *_args) -> None: - """Redraw GPR view; locator delivery lives in the C++ data_processor.""" - self._on_gpr_visual_settings_changed() - if not self._is_gpr_processing_mode(self._processing_mode.currentText()): - return - try: - self._write_live_processing_config() - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to update locator GPR window", exc) - - def _set_processing_mode_page(self, mode: str) -> None: - """Show the parameter page for `mode` without changing runtime state.""" - mode_to_page = { - "pass_through": 0, - "bscan": 1, - "gpr": 2, - "legacy_gpr": 3, - "sar3d": 4, - } - self._processing_mode_pages.setCurrentIndex(mode_to_page.get(mode, 0)) - self._gpr_common_page.setVisible(self._is_gpr_like_processing_mode(mode)) - current_page = self._processing_mode_pages.currentWidget() - if current_page is not None: - self._processing_mode_pages.setFixedHeight(current_page.sizeHint().height()) - self._processing_mode_pages.updateGeometry() - - def _on_processing_mode_changed(self, mode: str) -> None: - """Switch processing parameter page and refresh corresponding visualization.""" - previous_mode = getattr(self, "_active_processing_mode", "pass_through") - if mode != previous_mode: - try: - self._validate_processing_mode_selection(mode) - except Exception as exc: # noqa: BLE001 - with QSignalBlocker(self._processing_mode): - self._set_combo_current_text(self._processing_mode, previous_mode) - self._show_error( - f"Cannot switch processing mode to {mode}", - details=str(exc) or type(exc).__name__, - ) - return - - self._active_processing_mode = mode - self._set_plot_mode(mode) - self._set_processing_mode_page(mode) - self._on_processing_live_settings_changed() - if mode == "pass_through": - self._log( - "Processing mode selected: pass_through " - f"(show_magnitude={self._show_magnitude_checkbox.isChecked()}, " - f"show_phase={self._show_phase_checkbox.isChecked()}, " - f"combos={self._pass_through_combo_filter_input.text().strip() or ''}, " - f"fixed_y={self._pass_through_fixed_y_enabled.isChecked()}, " - f"y_range={self._pass_through_y_min_db.value():g}..{self._pass_through_y_max_db.value():g} dB)" - ) - elif mode == "bscan": - self._log( - "Processing mode selected: bscan " - f"(axis={self._bscan_axis.currentText()}, " - f"cut={self._bscan_cut_m.value():g} m, " - f"max_depth={self._bscan_max_depth_m.value():g} m, " - f"gain={self._bscan_gain.value():g}, " - f"freq={self._bscan_start_freq_mhz.value():g}..{self._bscan_stop_freq_mhz.value():g} MHz, " - f"subtract_mean_ascan={self._bscan_subtract_mean_ascan.isChecked()})" - ) - elif mode == "gpr": - self._log( - "Processing mode selected: gpr " - f"(inputs={self._gpr_input_positions_input.text().strip() or ''}, " - f"outputs={self._gpr_output_positions_input.text().strip() or ''}, " - f"depth={self._gpr_min_depth_m.value():g}..{self._gpr_max_depth_m.value():g} m, " - f"freq={self._gpr_start_freq_mhz.value():g}..{self._gpr_stop_freq_mhz.value():g} MHz, " - f"range_comp={self._gpr_range_comp_power.value():g}, " - f"object_min_frac={self._gpr_object_min_frac.value():g}, " - f"score_mode={self._gpr_score_mode.currentText()}, " - f"background_subtract={self._gpr_background_subtract_enabled.isChecked()}, " - f"mean_count={self._gpr_background_mean_count.value()}, " - f"remove_sidelobes={self._gpr_remove_sidelobe_objects_enabled.isChecked()}, " - f"imaging_plane_y={self._gpr_imaging_plane_y_m.value():g} m, " - f"render_mode={self._gpr_render_mode.currentText()}, " - f"max_draw={self._gpr_max_detected_objects_to_draw.value()}, " - f"draw_top={self._gpr_draw_top_m_objects.value()})" - ) - elif mode == "legacy_gpr": - self._log( - "Processing mode selected: legacy_gpr " - f"(mode={self._legacy_gpr_config_mode.currentText()}, " - f"inputs={self._legacy_gpr_input_positions_input.text().strip() or ''}, " - f"outputs={self._legacy_gpr_output_positions_input.text().strip() or ''}, " - f"depth={self._legacy_gpr_min_depth_m.value():g}..{self._legacy_gpr_max_depth_m.value():g} m, " - f"freq={self._legacy_gpr_start_freq_mhz.value():g}..{self._legacy_gpr_stop_freq_mhz.value():g} MHz, " - f"comp={self._legacy_gpr_comp_power.value():g}, " - f"snr={self._legacy_gpr_snr_thresh.value():g}, " - f"snr_comp_max={self._legacy_gpr_snr_comp_max.value():g}, " - f"speed={self._legacy_gpr_speed_m_s.value():g} m/s, " - f"look_angle={self._legacy_gpr_look_angle_deg.value():g} deg, " - f"background_subtract={self._legacy_gpr_background_subtract_enabled.isChecked()}, " - f"mean_count={self._legacy_gpr_background_mean_count.value()}, " - f"render_mode={self._legacy_gpr_render_mode.currentText()}, " - f"min_pairs={self._legacy_gpr_min_visible_pair_count.value()})" - ) - elif mode == "sar3d": - self._log( - "Processing mode selected: sar3d " - f"(INPUT_IDX={self._sar3d_input_positions_input.text().strip() or ''}, " - f"OUTPUT_IDX={self._sar3d_output_positions_input.text().strip() or ''}, " - f"F={self._sar3d_start_freq_mhz.value():g}..{self._sar3d_stop_freq_mhz.value():g} MHz, " - f"DEPTH={self._sar3d_min_depth_m.value():g}..{self._sar3d_max_depth_m.value():g} m, " - f"SAR_VOXEL_M={self._sar3d_voxel_m.value():g}, " - f"LOOK_ANGLE_DEG={self._sar3d_look_angle_deg.value():g}, " - f"SAR_FRAME_RADIUS={self._sar3d_frame_radius.value()})" - ) - - def _clear_history_mode_caches(self) -> None: - """Drop cached render state for pass-through, B-scan, GPR, and 3D SAR views.""" - self._clear_bscan_plot_history() - if hasattr(self, "_bscan_plot"): - self._bscan_plot.clear() - self._configure_bscan_plot_axes() - self._clear_trace_plots() - if hasattr(self, "_gpr_plot"): - self._clear_gpr_plot() - if hasattr(self, "_sar3d_plots"): - self._clear_sar3d_plot() - - def _redraw_after_history_deletion(self) -> None: - """Refresh plot immediately after destructive history deletion.""" - if self._processing_mode.currentText() == "bscan": - if self._result_history: - self._sync_bscan_history_from_results() - if not self._draw_bscan_heatmap_from_history(): - self._bscan_plot.clear() - self._configure_bscan_plot_axes() - return - if self._is_gpr_processing_mode(self._processing_mode.currentText()): - if self._result_history and self._draw_results(self._result_history[-1]): - return - self._clear_gpr_plot() - return - if self._processing_mode.currentText() == "sar3d": - self._clear_sar3d_plot() - if self._result_history: - self._draw_results(self._result_history[-1]) - return - if self._result_history: - self._draw_results(self._result_history[-1]) - return - self._bscan_plot.clear() - self._configure_bscan_plot_axes() - self._clear_trace_plots() diff --git a/python_app/gui/controllers/app_window_config/profile_io_mixin.py b/python_app/gui/controllers/app_window_config/profile_io_mixin.py deleted file mode 100644 index 70197c7..0000000 --- a/python_app/gui/controllers/app_window_config/profile_io_mixin.py +++ /dev/null @@ -1,628 +0,0 @@ -"""Profile load/save workflows for the main window config facade.""" - -from __future__ import annotations - -from collections import deque -from contextlib import ExitStack -from copy import deepcopy -import json -from pathlib import Path - -from PyQt6.QtCore import QSignalBlocker -from PyQt6.QtWidgets import QFileDialog - -from python_app.file_io import atomic_write_json -from python_app.gui.controllers.app_window_config.state_builders import GUI_SAVE_HISTORY_LIMIT -from python_app.models.gui_profile_model import GuiProfileModel -from python_app.models.run_config_model import RunConfigModel -from python_app.orchestration.preprocess_assets import ( - VISIBLE_PREPROCESS_ASSET_KEYS, - preprocess_asset_model, -) - - -class AppWindowConfigProfileIOMixin: - """Persist and restore full GUI profiles without restarting the app.""" - - def _write_gui_profile_to_path( - self, output_path: Path, *, allow_overwrite: bool = True, dataset: bool = False, - ) -> GuiProfileModel: - """Save UI settings, preserving the acquisition provenance for recorded data.""" - if not allow_overwrite and output_path.exists(): - raise FileExistsError(f"Config profile output already exists: {output_path}") - - profile = self._build_gui_profile() - active = getattr(self, "_active_run_config", None) - if dataset and active is not None: - profile.run_config.radar = deepcopy(active.radar) - profile.run_config.input_switch = deepcopy(active.input_switch) - profile.run_config.output_switch = deepcopy(active.output_switch) - profile.run_config.combos = deepcopy(active.combos) - profile.run_config.preprocess = deepcopy(active.preprocess) - profile.run_config.gpr.tx_geometry = deepcopy(active.gpr.tx_geometry) - profile.run_config.gpr.rx_geometry = deepcopy(active.gpr.rx_geometry) - atomic_write_json(output_path, profile.to_dict(), durable=True) - return profile - - def _set_combo_selection_mode(self, mode: str) -> None: - """Highlight current combo mode and enable only the relevant editors.""" - if self._is_matrix_radar_model_selected(): - self._run_combos_select_button.setChecked(True) - self._single_combo_select_button.setChecked(False) - self._combos_text.setText(self._fixed_matrix_radar_combo_text()) - self._combos_text.setEnabled(False) - self._single_combo_output.setEnabled(True) - self._single_combo_input.setEnabled(True) - self._run_combos_select_button.setEnabled(False) - self._single_combo_select_button.setEnabled(False) - self._next_combo_button.setEnabled(True) - return - - text_selected = mode != "single" - self._run_combos_select_button.setChecked(text_selected) - self._single_combo_select_button.setChecked(not text_selected) - self._combos_text.setEnabled(text_selected) - self._single_combo_output.setEnabled(not text_selected) - self._single_combo_input.setEnabled(not text_selected) - self._run_combos_select_button.setEnabled(True) - self._single_combo_select_button.setEnabled(True) - self._next_combo_button.setEnabled(not text_selected) - - def _select_display_combo(self, input_pos: int, output_pos: int) -> None: - """Select an available pair and redraw its cached B-scan immediately.""" - key = (int(input_pos), int(output_pos)) - if key not in self._available_display_combo_keys(): - raise ValueError(f"Unknown channel combination: {key[0]}:{key[1]}") - self._single_combo_input.setText(str(key[0])) - self._single_combo_output.setText(str(key[1])) - self._on_bscan_display_combo_changed() - - def _select_next_combo(self) -> None: - """Advance output first, then input, wrapping after the last configured pair.""" - available = self._available_display_combo_keys() - if not available: - return - try: - current = (int(self._single_combo_input.text()), int(self._single_combo_output.text())) - index = available.index(current) + 1 - except ValueError: - index = 0 - self._select_display_combo(*available[index % len(available)]) - - def _available_display_combo_keys(self) -> list[tuple[int, int]]: - """Return configured pairs ordered by input, then output for display navigation.""" - return sorted((combo.input, combo.output) for combo in self._defaults_config.build_runtime_combos()) - - def _is_matrix_radar_model_selected(self) -> bool: - """Return whether the loaded config targets a matrix-mode radar acquisition.""" - return bool(self._defaults_config.is_matrix_radar) - - def _fixed_matrix_radar_combo_text(self) -> str: - """Return the canonical virtual combo matrix shown for matrix-mode radars.""" - return ",".join( - f"{int(combo.input)}:{int(combo.output)}" - for combo in self._defaults_config.build_runtime_combos() - ) - - def _sync_pass_through_y_controls(self) -> None: - """Enable Y-range editors only when fixed Y mode is active.""" - enabled = bool(self._pass_through_fixed_y_enabled.isChecked()) - self._pass_through_y_min_db.setEnabled(enabled) - self._pass_through_y_max_db.setEnabled(enabled) - - def _set_radar_settings_mode(self, adc_mode: bool) -> None: - """Show the radar settings panel appropriate for the loaded radar model.""" - self._vna_radar_settings_panel.setVisible(not adc_mode) - self._adc_radar_settings_panel.setVisible(adc_mode) - - def _sync_adc_settings_controls(self) -> None: - """Enable optical-board controls according to enabled flag and mode.""" - if not hasattr(self, "_optical_enabled_checkbox"): - return - enabled = bool(self._optical_enabled_checkbox.isChecked()) - mode = self._optical_mode_combo.currentText() - common_widgets = [ - self._optical_port_input, - self._optical_mode_combo, - self._optical_pi_coeff1_p_input, - self._optical_pi_coeff1_i_input, - self._optical_pi_coeff2_p_input, - self._optical_pi_coeff2_i_input, - ] - manual_widgets = [ - self._optical_manual_temp1_input, - self._optical_manual_temp2_input, - self._optical_manual_current1_input, - self._optical_manual_current2_input, - ] - variation_widgets = [ - self._optical_variation_type_combo, - self._optical_static_temp1_input, - self._optical_static_temp2_input, - self._optical_static_current1_input, - self._optical_static_current2_input, - self._optical_min_value_input, - self._optical_max_value_input, - self._optical_step_input, - self._optical_time_step_input, - self._optical_delay_time_input, - ] - for widget in common_widgets: - widget.setEnabled(enabled) - for widget in manual_widgets: - widget.setEnabled(enabled and mode == "manual") - for widget in variation_widgets: - widget.setEnabled(enabled and mode == "variation") - self._optical_manual_panel.setVisible(mode == "manual") - self._optical_variation_panel.setVisible(mode == "variation") - - def _apply_history_limit_from_config(self, config) -> None: - """Resize in-memory history buffers to match the loaded config. - - Save-side deques use a config-independent limit so that processing-side - ring capacities can stay small without truncating the save buffer. The - C++ replay window still follows ring capacities, since it bounds how much - of the history the processor can re-publish coherently. - """ - bscan_cpp_replay_window = self._cpp_bscan_replay_window_for_config(config) - self._raw_history = deque(self._raw_history, maxlen=GUI_SAVE_HISTORY_LIMIT) - self._pre_history = deque(self._pre_history, maxlen=GUI_SAVE_HISTORY_LIMIT) - self._result_history = deque(self._result_history, maxlen=GUI_SAVE_HISTORY_LIMIT) - self._bscan_cpp_replay_window = bscan_cpp_replay_window - self._clear_bscan_plot_history() - - def _save_current_config(self) -> None: - """Persist current full GUI profile to a user-selected JSON file.""" - try: - suggested_path = str(self._active_profile_path) - selected_path, _selected_filter = QFileDialog.getSaveFileName( - self, - "Save Config Profile", - suggested_path, - "JSON Files (*.json);;All Files (*)", - ) - if not selected_path: - return - - output_path = self._normalize_profile_path(Path(selected_path)) - if not output_path.suffix: - output_path = output_path.with_suffix(".json") - - profile = self._write_gui_profile_to_path(output_path) - - self._defaults_config = profile.run_config.clone() - self._gui_defaults = profile.gui - self._remember_active_profile_path(output_path) - points_text = ( - "points=from Kamil ADC stream" - if profile.run_config.is_kamil_adc - else f"points={profile.run_config.radar.sweep.points}" - ) - self._log( - f"Config profile saved: path={output_path}, " - f"combos={len(profile.run_config.combos)}, " - f"sweep={profile.run_config.radar.sweep.start_hz:g}.." - f"{profile.run_config.radar.sweep.stop_hz:g} Hz, " - f"{points_text}, " - f"ifbw={profile.run_config.radar.sweep.if_bandwidth_hz:g} Hz, " - f"power={profile.run_config.radar.sweep.power_dbm:g} dBm, " - f"processing_mode={profile.gui.processing.selected_mode}" - ) - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to save config profile", exc) - - def _ensure_ready_to_load_config(self) -> bool: - """Return whether a config load may proceed, reporting the blocker if not. - - Loading rebuilds every widget and resizes history, so it is refused while a - capture sequence or the pipeline is active — the operator must stop first. - Shared by the desktop dialog and the web loader so both honor the same rule. - """ - if getattr(self, "_object_capture_job", None) is not None: - self._show_error("Wait for object-calibration capture to finish") - return False - if self._capture_session is not None: - self._show_error( - "Cannot load config during active capture sequence", - details=self._capture_state_details(), - ) - return False - if self._supervisor.is_running(): - self._show_error( - "Stop all pipeline processes before loading a config profile", - details=self._process_state_details(), - ) - return False - return True - - def _load_config_from_path(self, config_path: Path) -> None: - """Load a full GUI profile from ``config_path`` and apply it to the UI. - - This is the single authoritative load path: the desktop "Load Config" dialog - and the web config picker both funnel here, differing only in how they obtain - the path. The pipeline is left untouched; live settings reach a running - data_processor immediately while stable settings stage for the next Start. - """ - if not self._ensure_ready_to_load_config(): - return - try: - normalized_path = self._normalize_profile_path(config_path) - profile = GuiProfileModel.load_from_path(normalized_path) - processor_running = self._supervisor.is_processor_running() - self._apply_loaded_profile(profile, normalized_path) - profile_kind = "legacy run config" if profile.gui is None else "full GUI profile" - message = ( - f"Config profile loaded: path={normalized_path}, " - f"kind={profile_kind}, " - f"combos={len(self._defaults_config.combos)}, " - f"processing_mode={self._processing_mode.currentText()}" - ) - if processor_running: - message += ( - "; data_processor is still running, so live processing settings were applied immediately " - "and stable settings are now staged in the UI for the next Start" - ) - self._log(message) - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to load config profile", exc) - - def _load_config_from_dialog(self) -> None: - """Load a full GUI profile from a user-selected JSON file (desktop button).""" - if not self._ensure_ready_to_load_config(): - return - selected_path, _selected_filter = QFileDialog.getOpenFileName( - self, - "Load Config Profile", - str(self._active_profile_path), - "JSON Files (*.json);;All Files (*)", - ) - if not selected_path: - return - self._load_config_from_path(Path(selected_path)) - - def _apply_loaded_profile(self, profile: GuiProfileModel, profile_path: Path) -> None: - """Apply an already parsed profile to GUI state without restarting the pipeline. - - Repopulates every radar/processing/preprocess widget under signal blockers, - resizes history buffers, and refreshes derived state. The pipeline is left - untouched; callers handle user-facing logging and error reporting. - """ - self._log_debug( - f"Applying loaded profile: path={profile_path}, " - f"has_gui_state={profile.gui is not None}." - ) - config = profile.run_config.clone() - gui_state = profile.gui if profile.gui is not None else self._default_gui_state_for_config(config) - self._defaults_config = config - selected_preprocess_sets = { - key: str(preprocess_asset_model(config, key).set_name) - for key in VISIBLE_PREPROCESS_ASSET_KEYS - } - - radio_widgets = ( - self._start_hz_input, - self._stop_hz_input, - self._points_input, - self._ifbw_input, - self._power_input, - self._settling_ms, - self._combos_text, - self._single_combo_output, - self._single_combo_input, - self._run_combos_select_button, - self._single_combo_select_button, - self._processing_mode, - self._show_magnitude_checkbox, - self._show_phase_checkbox, - self._pass_through_combo_filter_input, - self._pass_through_fixed_y_enabled, - self._pass_through_y_min_db, - self._pass_through_y_max_db, - self._bscan_axis, - self._bscan_cut_m, - self._bscan_max_depth_m, - self._bscan_gain, - self._bscan_start_freq_mhz, - self._bscan_stop_freq_mhz, - self._bscan_subtract_mean_ascan, - self._bscan_history_window, - self._gpr_relative_permittivity, - self._gpr_tx_geometry_input, - self._gpr_rx_geometry_input, - self._gpr_input_positions_input, - self._gpr_output_positions_input, - self._gpr_min_depth_m, - self._gpr_max_depth_m, - self._gpr_range_comp_power, - self._gpr_object_min_frac, - self._gpr_score_mode, - self._gpr_motion_mode, - self._gpr_look_angle_deg, - self._gpr_direction_sign, - self._gpr_ignore_socket_speed_enabled, - self._gpr_speed_m_s, - self._gpr_max_detected_objects_to_draw, - self._gpr_draw_top_m_objects, - self._gpr_object_approach_min_frames, - self._gpr_start_freq_mhz, - self._gpr_stop_freq_mhz, - self._gpr_background_subtract_enabled, - self._gpr_background_mean_count, - self._gpr_remove_sidelobe_objects_enabled, - self._gpr_imaging_plane_y_m, - self._gpr_render_mode, - self._gpr_visible_x_min_m, - self._gpr_visible_x_max_m, - self._gpr_visible_z_min_m, - self._gpr_visible_z_max_m, - self._legacy_gpr_config_mode, - self._legacy_gpr_input_positions_input, - self._legacy_gpr_output_positions_input, - self._legacy_gpr_min_depth_m, - self._legacy_gpr_max_depth_m, - self._legacy_gpr_comp_power, - self._legacy_gpr_snr_thresh, - self._legacy_gpr_snr_comp_max, - self._legacy_gpr_start_freq_mhz, - self._legacy_gpr_stop_freq_mhz, - self._legacy_gpr_speed_m_s, - self._legacy_gpr_ignore_socket_speed_enabled, - self._legacy_gpr_look_angle_deg, - self._legacy_gpr_apply_freq_phase_correction, - self._legacy_gpr_reference_mode, - self._legacy_gpr_background_subtract_enabled, - self._legacy_gpr_background_mean_count, - self._legacy_gpr_render_mode, - self._legacy_gpr_min_visible_pair_count, - self._legacy_gpr_visible_x_min_m, - self._legacy_gpr_visible_x_max_m, - self._legacy_gpr_visible_z_min_m, - self._legacy_gpr_visible_z_max_m, - self._sar3d_input_positions_input, - self._sar3d_output_positions_input, - self._sar3d_min_depth_m, - self._sar3d_max_depth_m, - self._sar3d_start_freq_mhz, - self._sar3d_stop_freq_mhz, - self._sar3d_speed_m_s, - self._sar3d_ignore_socket_speed_enabled, - self._sar3d_look_angle_deg, - self._sar3d_frame_radius, - self._sar3d_voxel_m, - self._sar3d_bp_ground_depth_m, - self._sar3d_comp_range_power, - self._sar3d_use_compensation, - self._sar3d_normalize_each_frame, - self._record_count, - self._save_path_input, - self._save_name_input, - self._adc_project_dir_input, - self._adc_executable_path_input, - self._adc_tty_path_input, - self._adc_args_input, - self._adc_env_input, - self._adc_startup_timeout_s_input, - self._adc_sweep_timeout_s_input, - self._adc_stop_timeout_s_input, - self._optical_enabled_checkbox, - self._optical_port_input, - self._optical_mode_combo, - self._optical_pi_coeff1_p_input, - self._optical_pi_coeff1_i_input, - self._optical_pi_coeff2_p_input, - self._optical_pi_coeff2_i_input, - self._optical_manual_temp1_input, - self._optical_manual_temp2_input, - self._optical_manual_current1_input, - self._optical_manual_current2_input, - self._optical_variation_type_combo, - self._optical_static_temp1_input, - self._optical_static_temp2_input, - self._optical_static_current1_input, - self._optical_static_current2_input, - self._optical_min_value_input, - self._optical_max_value_input, - self._optical_step_input, - self._optical_time_step_input, - self._optical_delay_time_input, - ) - - with ExitStack() as blockers: - for widget in radio_widgets: - blockers.enter_context(QSignalBlocker(widget)) - - self._start_hz_input.setText(f"{config.radar.sweep.start_hz:g}") - self._stop_hz_input.setText(f"{config.radar.sweep.stop_hz:g}") - self._points_input.setText(str(int(config.radar.sweep.points))) - self._ifbw_input.setText(f"{config.radar.sweep.if_bandwidth_hz:g}") - self._power_input.setText(f"{config.radar.sweep.power_dbm:g}") - self._settling_ms.setText(str(int(config.runtime.settling_ms))) - self._set_radar_settings_mode(config.is_kamil_adc) - - adc = config.radar.kamil_adc - optical = config.radar.laser_control - manual = optical.manual - variation = optical.variation - self._adc_project_dir_input.setText(str(adc.project_dir)) - self._adc_executable_path_input.setText(str(adc.executable_path)) - self._adc_tty_path_input.setText(str(adc.tty_path)) - self._adc_args_input.setPlainText("\n".join(adc.args)) - self._adc_env_input.setPlainText(json.dumps(adc.env, indent=2, sort_keys=True) if adc.env else "{}") - self._adc_startup_timeout_s_input.setText(f"{float(adc.startup_timeout_s):g}") - self._adc_sweep_timeout_s_input.setText(f"{float(adc.sweep_timeout_s):g}") - self._adc_stop_timeout_s_input.setText(f"{float(adc.stop_timeout_s):g}") - self._optical_enabled_checkbox.setChecked(bool(optical.enabled)) - self._optical_port_input.setText(str(optical.port)) - self._set_combo_current_text(self._optical_mode_combo, str(optical.mode)) - self._optical_pi_coeff1_p_input.setText(str(int(optical.pi_coeff1_p))) - self._optical_pi_coeff1_i_input.setText(str(int(optical.pi_coeff1_i))) - self._optical_pi_coeff2_p_input.setText(str(int(optical.pi_coeff2_p))) - self._optical_pi_coeff2_i_input.setText(str(int(optical.pi_coeff2_i))) - self._optical_manual_temp1_input.setText(f"{float(manual.temp1):g}") - self._optical_manual_temp2_input.setText(f"{float(manual.temp2):g}") - self._optical_manual_current1_input.setText(f"{float(manual.current1):g}") - self._optical_manual_current2_input.setText(f"{float(manual.current2):g}") - self._set_combo_current_text(self._optical_variation_type_combo, str(variation.variation_type)) - self._optical_static_temp1_input.setText(f"{float(variation.static_temp1):g}") - self._optical_static_temp2_input.setText(f"{float(variation.static_temp2):g}") - self._optical_static_current1_input.setText(f"{float(variation.static_current1):g}") - self._optical_static_current2_input.setText(f"{float(variation.static_current2):g}") - self._optical_min_value_input.setText(f"{float(variation.min_value):g}") - self._optical_max_value_input.setText(f"{float(variation.max_value):g}") - self._optical_step_input.setText(f"{float(variation.step):g}") - self._optical_time_step_input.setText(str(int(variation.time_step))) - self._optical_delay_time_input.setText(str(int(variation.delay_time))) - self._sync_adc_settings_controls() - - self._combos_text.setText(str(gui_state.switches.combos_text)) - self._single_combo_output.setText(str(gui_state.switches.single_output)) - self._single_combo_input.setText(str(gui_state.switches.single_input)) - self._set_combo_selection_mode(gui_state.switches.combo_mode) - - self._set_combo_current_text(self._processing_mode, gui_state.processing.selected_mode) - self._show_magnitude_checkbox.setChecked(bool(gui_state.processing.pass_through.show_magnitude)) - self._show_phase_checkbox.setChecked(bool(gui_state.processing.pass_through.show_phase)) - self._unwrap_phase_checkbox.setChecked(bool(gui_state.processing.pass_through.unwrap_phase)) - self._pass_through_combo_filter_input.setText(str(gui_state.processing.pass_through.combo_filter)) - self._pass_through_fixed_y_enabled.setChecked(bool(gui_state.processing.pass_through.fixed_y_enabled)) - self._pass_through_y_min_db.setValue(float(gui_state.processing.pass_through.y_min_db)) - self._pass_through_y_max_db.setValue(float(gui_state.processing.pass_through.y_max_db)) - - self._set_combo_current_text(self._bscan_axis, gui_state.processing.bscan.axis) - self._bscan_cut_m.setValue(float(gui_state.processing.bscan.cut_m)) - self._bscan_max_depth_m.setValue(float(gui_state.processing.bscan.max_depth_m)) - self._bscan_gain.setValue(float(gui_state.processing.bscan.gain)) - self._bscan_start_freq_mhz.setValue(float(gui_state.processing.bscan.start_freq_mhz)) - self._bscan_stop_freq_mhz.setValue(float(gui_state.processing.bscan.stop_freq_mhz)) - self._bscan_subtract_mean_ascan.setChecked(bool(gui_state.processing.bscan.subtract_mean_ascan)) - self._bscan_history_window.setValue(int(gui_state.processing.bscan.history_window_scans)) - - self._gpr_relative_permittivity.setValue(float(config.gpr.relative_permittivity)) - self._gpr_tx_geometry_input.set_position_count(config.output_switch.positions) - self._gpr_rx_geometry_input.set_position_count(config.input_switch.positions) - self._gpr_tx_geometry_input.set_records(config.gpr.tx_geometry) - self._gpr_rx_geometry_input.set_records(config.gpr.rx_geometry) - self._gpr_input_positions_input.setText(str(gui_state.processing.gpr.input_positions)) - self._gpr_output_positions_input.setText(str(gui_state.processing.gpr.output_positions)) - self._gpr_min_depth_m.setValue(float(gui_state.processing.gpr.min_depth_m)) - self._gpr_max_depth_m.setValue(float(gui_state.processing.gpr.max_depth_m)) - self._gpr_range_comp_power.setValue(float(gui_state.processing.gpr.range_comp_power)) - self._gpr_object_min_frac.setValue(float(gui_state.processing.gpr.object_min_frac)) - self._set_combo_current_text(self._gpr_score_mode, gui_state.processing.gpr.score_mode) - self._set_combo_current_text(self._gpr_motion_mode, gui_state.processing.gpr.motion_mode) - self._gpr_look_angle_deg.setValue(float(gui_state.processing.gpr.look_angle_deg)) - self._gpr_direction_sign.setValue(float(gui_state.processing.gpr.direction_sign)) - self._gpr_ignore_socket_speed_enabled.setChecked( - bool(gui_state.processing.gpr.ignore_socket_speed_enabled) - ) - self._gpr_speed_m_s.setValue(float(gui_state.processing.gpr.speed_m_s)) - self._gpr_max_detected_objects_to_draw.setValue( - int(gui_state.processing.gpr.max_detected_objects_to_draw) - ) - self._gpr_draw_top_m_objects.setValue(int(gui_state.processing.gpr.draw_top_m_objects)) - self._gpr_object_approach_min_frames.setValue(int(gui_state.processing.gpr.object_approach_min_frames)) - self._gpr_start_freq_mhz.setValue(float(gui_state.processing.gpr.start_freq_mhz)) - self._gpr_stop_freq_mhz.setValue(float(gui_state.processing.gpr.stop_freq_mhz)) - self._gpr_background_subtract_enabled.setChecked( - bool(gui_state.processing.gpr.background_subtract_enabled) - ) - self._gpr_background_mean_count.setValue(int(gui_state.processing.gpr.background_mean_count)) - self._gpr_remove_sidelobe_objects_enabled.setChecked( - bool(gui_state.processing.gpr.remove_sidelobe_objects_enabled) - ) - self._gpr_imaging_plane_y_m.setValue(float(gui_state.processing.gpr.imaging_plane_y_m)) - self._set_combo_current_text(self._gpr_render_mode, gui_state.processing.gpr.render_mode) - self._gpr_visible_x_min_m.setValue(float(gui_state.processing.gpr.visible_x_min_m)) - self._gpr_visible_x_max_m.setValue(float(gui_state.processing.gpr.visible_x_max_m)) - self._gpr_visible_z_min_m.setValue(float(gui_state.processing.gpr.visible_z_min_m)) - self._gpr_visible_z_max_m.setValue(float(gui_state.processing.gpr.visible_z_max_m)) - - self._set_combo_current_text(self._legacy_gpr_config_mode, gui_state.processing.legacy_gpr.mode) - self._legacy_gpr_input_positions_input.setText(str(gui_state.processing.legacy_gpr.input_positions)) - self._legacy_gpr_output_positions_input.setText(str(gui_state.processing.legacy_gpr.output_positions)) - self._legacy_gpr_min_depth_m.setValue(float(gui_state.processing.legacy_gpr.min_depth_m)) - self._legacy_gpr_max_depth_m.setValue(float(gui_state.processing.legacy_gpr.max_depth_m)) - self._legacy_gpr_comp_power.setValue(float(gui_state.processing.legacy_gpr.comp_power)) - self._legacy_gpr_snr_thresh.setValue(float(gui_state.processing.legacy_gpr.snr_thresh)) - self._legacy_gpr_snr_comp_max.setValue(float(gui_state.processing.legacy_gpr.snr_comp_max)) - self._legacy_gpr_start_freq_mhz.setValue(float(gui_state.processing.legacy_gpr.start_freq_mhz)) - self._legacy_gpr_stop_freq_mhz.setValue(float(gui_state.processing.legacy_gpr.stop_freq_mhz)) - self._legacy_gpr_speed_m_s.setValue(float(gui_state.processing.legacy_gpr.speed_m_s)) - self._legacy_gpr_ignore_socket_speed_enabled.setChecked( - bool(gui_state.processing.legacy_gpr.ignore_socket_speed_enabled) - ) - self._legacy_gpr_look_angle_deg.setValue(float(gui_state.processing.legacy_gpr.look_angle_deg)) - self._legacy_gpr_apply_freq_phase_correction.setChecked( - bool(gui_state.processing.legacy_gpr.apply_freq_phase_correction) - ) - self._set_combo_current_text( - self._legacy_gpr_reference_mode, - gui_state.processing.legacy_gpr.reference_mode, - ) - self._legacy_gpr_background_subtract_enabled.setChecked( - bool(gui_state.processing.legacy_gpr.background_subtract_enabled) - ) - self._legacy_gpr_background_mean_count.setValue(int(gui_state.processing.legacy_gpr.background_mean_count)) - self._set_combo_current_text(self._legacy_gpr_render_mode, gui_state.processing.legacy_gpr.render_mode) - self._legacy_gpr_min_visible_pair_count.setValue(int(gui_state.processing.legacy_gpr.min_visible_pair_count)) - self._legacy_gpr_visible_x_min_m.setValue(float(gui_state.processing.legacy_gpr.visible_x_min_m)) - self._legacy_gpr_visible_x_max_m.setValue(float(gui_state.processing.legacy_gpr.visible_x_max_m)) - self._legacy_gpr_visible_z_min_m.setValue(float(gui_state.processing.legacy_gpr.visible_z_min_m)) - self._legacy_gpr_visible_z_max_m.setValue(float(gui_state.processing.legacy_gpr.visible_z_max_m)) - - sar3d_state = gui_state.processing.sar3d - self._sar3d_input_positions_input.setText(str(sar3d_state.input_positions)) - self._sar3d_output_positions_input.setText(str(sar3d_state.output_positions)) - self._sar3d_min_depth_m.setValue(float(sar3d_state.min_depth_m)) - self._sar3d_max_depth_m.setValue(float(sar3d_state.max_depth_m)) - self._sar3d_start_freq_mhz.setValue(float(sar3d_state.start_freq_mhz)) - self._sar3d_stop_freq_mhz.setValue(float(sar3d_state.stop_freq_mhz)) - self._sar3d_speed_m_s.setValue(float(sar3d_state.speed_m_s)) - self._sar3d_ignore_socket_speed_enabled.setChecked(bool(sar3d_state.ignore_socket_speed_enabled)) - self._sar3d_look_angle_deg.setValue(float(sar3d_state.look_angle_deg)) - self._sar3d_frame_radius.setValue(int(sar3d_state.frame_radius)) - self._sar3d_voxel_m.setValue(float(sar3d_state.voxel_m)) - self._sar3d_bp_ground_depth_m.setValue(float(sar3d_state.bp_ground_depth_m)) - self._sar3d_comp_range_power.setValue(float(sar3d_state.comp_range_power)) - self._sar3d_use_compensation.setChecked(bool(sar3d_state.use_compensation)) - self._sar3d_normalize_each_frame.setChecked(bool(sar3d_state.normalize_each_frame)) - self._sync_socket_speed_controls() - - self._record_count.setValue(int(gui_state.data_actions.record_count)) - self._save_path_input.setText(str(gui_state.data_actions.save_path)) - self._save_name_input.setText(str(gui_state.data_actions.save_name)) - - self._gui_defaults = gui_state - self._selected_preprocess_sets = selected_preprocess_sets - self._selected_preprocess_radar_key = self._radar_key(config) - self._preprocess_set_name = str(gui_state.preprocess_dialog.set_name) - self._preprocess_radar_config_dir = str(gui_state.preprocess_dialog.radar_config_dir) - self._preprocess_use_all_radar_configs = bool(gui_state.preprocess_dialog.use_all_radar_configs) - self._preprocess_median_sweep_count = max(1, int(gui_state.preprocess_dialog.median_sweep_count)) - self._apply_history_limit_from_config(config) - self._gpr_geometry_signature = None - self._gpr_selected_geometry = None - self._sync_pass_through_y_controls() - self._set_processing_mode_page(gui_state.processing.selected_mode) - self._refresh_preprocess_summary_labels() - - if self._preprocess_dialog is not None: - with ExitStack() as dialog_blockers: - dialog_blockers.enter_context(QSignalBlocker(self._preprocess_dialog._set_name_input)) - dialog_blockers.enter_context(QSignalBlocker(self._preprocess_dialog._radar_config_dir_input)) - dialog_blockers.enter_context(QSignalBlocker(self._preprocess_dialog._use_all_radar_configs_checkbox)) - for combo in self._preprocess_dialog._set_combos.values(): - dialog_blockers.enter_context(QSignalBlocker(combo)) - self._preprocess_dialog.set_set_name(self._preprocess_set_name) - self._preprocess_dialog.set_radar_config_dir(self._preprocess_radar_config_dir) - self._preprocess_dialog.set_use_all_radar_configs(self._preprocess_use_all_radar_configs) - self._preprocess_dialog.set_median_sweep_count(self._preprocess_median_sweep_count) - self._preprocess_dialog.set_selected_sets(self._selected_preprocess_sets, emit_signal=False) - - self._apply_initial_radar_limits() - if self._preprocess_dialog is not None: - self._refresh_sets() - self._await_processing_revision = True - self._on_processing_mode_changed(gui_state.processing.selected_mode) - self._update_history_indicator() - self._remember_active_profile_path(profile_path) diff --git a/python_app/gui/controllers/app_window_config/radar_limits_mixin.py b/python_app/gui/controllers/app_window_config/radar_limits_mixin.py deleted file mode 100644 index 27a3ba0..0000000 --- a/python_app/gui/controllers/app_window_config/radar_limits_mixin.py +++ /dev/null @@ -1,237 +0,0 @@ -"""Radar device limit sync and clamping logic for the main window.""" - -from __future__ import annotations - -from python_app.hardware_full.librevna_service import LibreVnaService -from python_app.hardware_full.matrix_radar_service import create_matrix_radar_service -from python_app.hardware_full.single_radar_service import create_single_radar_service - - -class AppWindowRadarLimitsMixin: - """Handle radar capability probing and dependent UI clamping.""" - - def _on_radar_sweep_limits_changed(self) -> None: - """Clamp processing frequency bounds after sweep start/stop edits.""" - self._reset_preprocess_selection_after_radar_key_change() - if self._sync_processing_frequency_limits_with_radar(): - self._on_processing_live_settings_changed() - - def _refresh_radar_limits_from_device(self) -> bool: - """Query native radar limits and apply them to GUI fields.""" - config = self._defaults_config - if config.is_kamil_adc: - return self._apply_radar_limits_to_ui(None) - if config.is_multi_device: - radar_service = LibreVnaService(serial=config.radar.serial or None) - elif config.is_matrix_radar: - radar_service = create_matrix_radar_service(config) - else: - radar_service = create_single_radar_service(config) - - limits = radar_service.read_device_limits() - return self._apply_radar_limits_to_ui(limits) - - def _apply_radar_limits_to_ui(self, limits: dict[str, float | int] | None) -> bool: - """Apply optional radar limits and clamp dependent GUI fields.""" - previous_limits = dict(self._radar_limits) if self._radar_limits is not None else None - if limits is None: - self._radar_limits = None - self._radar_start_label.setText("Start Hz") - self._radar_stop_label.setText("Stop Hz") - self._radar_points_label.setText("Points") - self._radar_ifbw_label.setText("IF BW Hz") - self._radar_power_label.setText("Stimulus Power dBm") - self._radar_limits_hint.setText("Device limits are not available.") - self._power_input.setToolTip("Stimulus power configured in the active profile.") - return False - - min_freq_hz = float(limits["min_frequency_hz"]) - max_freq_hz = float(limits["max_frequency_hz"]) - min_ifbw_hz = float(limits["min_ifbw_hz"]) - max_ifbw_hz = float(limits["max_ifbw_hz"]) - max_points = int(limits["max_points"]) - min_power_dbm = float(limits["min_power_dbm"]) - max_power_dbm = float(limits["max_power_dbm"]) - - self._radar_limits = limits - - self._radar_start_label.setText(f"Start Hz ({min_freq_hz:g}..{max_freq_hz:g})") - self._radar_stop_label.setText(f"Stop Hz ({min_freq_hz:g}..{max_freq_hz:g})") - self._radar_points_label.setText(f"Points (1..{max_points:d})") - self._radar_ifbw_label.setText(f"IF BW Hz ({min_ifbw_hz:g}..{max_ifbw_hz:g})") - self._radar_power_label.setText(f"Stimulus Power dBm ({min_power_dbm:g}..{max_power_dbm:g})") - self._radar_limits_hint.setText( - f"Limits: Freq {min_freq_hz:g}..{max_freq_hz:g} Hz, Points 1..{max_points:d}, " - f"IF BW {min_ifbw_hz:g}..{max_ifbw_hz:g} Hz, Power {min_power_dbm:g}..{max_power_dbm:g} dBm." - ) - - changed = False - prev_start = self._start_hz_input.text().strip() - prev_stop = self._stop_hz_input.text().strip() - prev_points = self._points_input.text().strip() - prev_ifbw = self._ifbw_input.text().strip() - prev_power = self._power_input.text().strip() - - start_hz = self._clamp_line_edit_float(self._start_hz_input, min_freq_hz, max_freq_hz) - stop_hz = self._clamp_line_edit_float(self._stop_hz_input, min_freq_hz, max_freq_hz) - if start_hz > stop_hz: - stop_hz = start_hz - self._stop_hz_input.setText(f"{stop_hz:g}") - changed = True - - self._clamp_line_edit_int(self._points_input, 1, max_points) - self._clamp_line_edit_float(self._ifbw_input, min_ifbw_hz, max_ifbw_hz) - self._clamp_line_edit_float(self._power_input, min_power_dbm, max_power_dbm) - self._power_input.setToolTip(f"Device range: {min_power_dbm:g}..{max_power_dbm:g} dBm") - - changed = ( - changed - or prev_start != self._start_hz_input.text().strip() - or prev_stop != self._stop_hz_input.text().strip() - or prev_points != self._points_input.text().strip() - or prev_ifbw != self._ifbw_input.text().strip() - or prev_power != self._power_input.text().strip() - ) - - applied_limits_changed = previous_limits != limits - if applied_limits_changed: - self._log( - "Applied radar device limits: " - f"freq={min_freq_hz:g}..{max_freq_hz:g} Hz, " - f"points=1..{max_points}, " - f"ifbw={min_ifbw_hz:g}..{max_ifbw_hz:g} Hz, " - f"power={min_power_dbm:g}..{max_power_dbm:g} dBm" - ) - - adjustments: list[str] = [] - current_start = self._start_hz_input.text().strip() - current_stop = self._stop_hz_input.text().strip() - current_points = self._points_input.text().strip() - current_ifbw = self._ifbw_input.text().strip() - current_power = self._power_input.text().strip() - if prev_start != current_start: - adjustments.append(f"Start Hz: {prev_start or ''} -> {current_start}") - if prev_stop != current_stop: - adjustments.append(f"Stop Hz: {prev_stop or ''} -> {current_stop}") - if prev_points != current_points: - adjustments.append(f"Points: {prev_points or ''} -> {current_points}") - if prev_ifbw != current_ifbw: - adjustments.append(f"IF BW Hz: {prev_ifbw or ''} -> {current_ifbw}") - if prev_power != current_power: - adjustments.append(f"Stimulus Power dBm: {prev_power or ''} -> {current_power}") - if adjustments: - self._log_warning( - "Radar fields were adjusted to satisfy device limits.", - details="\n".join(adjustments), - ) - - self._sync_processing_frequency_limits_with_radar() - return changed - - @staticmethod - def _clamp_line_edit_float(widget, min_value: float, max_value: float) -> float: - """Clamp float line-edit value to inclusive range and rewrite widget text.""" - try: - value = float(widget.text().strip()) - except ValueError: - value = min_value - value = min(max(value, min_value), max_value) - widget.setText(f"{value:g}") - return value - - @staticmethod - def _clamp_line_edit_int(widget, min_value: int, max_value: int) -> int: - """Clamp integer line-edit value to inclusive range and rewrite widget text.""" - try: - value = int(float(widget.text().strip())) - except ValueError: - value = min_value - value = min(max(value, min_value), max_value) - widget.setText(str(value)) - return value - - def _sync_processing_frequency_limits_with_radar(self) -> bool: - """Synchronize all processing frequency widgets with radar sweep bounds.""" - bscan_changed = self._sync_bscan_frequency_limits_with_radar() - gpr_changed = self._sync_gpr_frequency_limits_with_radar() - return bscan_changed or gpr_changed - - def _sync_frequency_spinboxes_with_radar(self, widget_names: tuple[str, ...]) -> bool: - """Synchronize one or more MHz spin boxes with current radar sweep bounds.""" - required_widgets = ("_start_hz_input", "_stop_hz_input", *widget_names) - if not all(hasattr(self, widget_name) for widget_name in required_widgets): - return False - - try: - radar_start_hz = float(self._start_hz_input.text().strip()) - radar_stop_hz = float(self._stop_hz_input.text().strip()) - except ValueError: - return False - - radar_min_mhz = min(radar_start_hz, radar_stop_hz) / 1_000_000.0 - radar_max_mhz = max(radar_start_hz, radar_stop_hz) / 1_000_000.0 - - changed = False - widget_labels = { - "_bscan_start_freq_mhz": "B-scan Start MHz", - "_bscan_stop_freq_mhz": "B-scan Stop MHz", - "_gpr_start_freq_mhz": "GPR Start MHz", - "_gpr_stop_freq_mhz": "GPR Stop MHz", - "_legacy_gpr_start_freq_mhz": "Legacy GPR Start MHz", - "_legacy_gpr_stop_freq_mhz": "Legacy GPR Stop MHz", - "_sar3d_start_freq_mhz": "3D SAR Start MHz", - "_sar3d_stop_freq_mhz": "3D SAR Stop MHz", - } - widgets = [getattr(self, widget_name) for widget_name in widget_names] - previous_values = {widget_name: getattr(self, widget_name).value() for widget_name in widget_names} - for widget in widgets: - if widget.minimum() != radar_min_mhz or widget.maximum() != radar_max_mhz: - changed = True - widget.blockSignals(True) - widget.setRange(radar_min_mhz, radar_max_mhz) - widget.blockSignals(False) - - for widget in widgets: - clamped_value = min(max(widget.value(), radar_min_mhz), radar_max_mhz) - if clamped_value != widget.value(): - changed = True - widget.blockSignals(True) - widget.setValue(clamped_value) - widget.blockSignals(False) - clamped_fields = [] - for widget_name in widget_names: - current_value = getattr(self, widget_name).value() - previous_value = previous_values[widget_name] - if current_value == previous_value: - continue - clamped_fields.append( - f"{widget_labels.get(widget_name, widget_name)}: {previous_value:g} -> {current_value:g}" - ) - if clamped_fields: - self._log_warning( - "Processing frequency limits were clamped to the active radar sweep.", - details="\n".join(clamped_fields), - ) - return changed - - def _sync_bscan_frequency_limits_with_radar(self) -> bool: - """Synchronize B-scan start/stop MHz widget ranges with radar sweep bounds.""" - return self._sync_frequency_spinboxes_with_radar( - ( - "_bscan_start_freq_mhz", - "_bscan_stop_freq_mhz", - ) - ) - - def _sync_gpr_frequency_limits_with_radar(self) -> bool: - """Synchronize GPR start/stop MHz widget ranges with radar sweep bounds.""" - return self._sync_frequency_spinboxes_with_radar( - ( - "_gpr_start_freq_mhz", - "_gpr_stop_freq_mhz", - "_legacy_gpr_start_freq_mhz", - "_legacy_gpr_stop_freq_mhz", - "_sar3d_start_freq_mhz", - "_sar3d_stop_freq_mhz", - ) - ) diff --git a/python_app/gui/controllers/app_window_config/state_builders.py b/python_app/gui/controllers/app_window_config/state_builders.py deleted file mode 100644 index 7a2708d..0000000 --- a/python_app/gui/controllers/app_window_config/state_builders.py +++ /dev/null @@ -1,472 +0,0 @@ -"""State builders and config parsing helpers for the main window.""" - -from __future__ import annotations - -import json - -from python_app.models.gui_profile_model import ( - GuiBscanStateModel, - GuiDataActionsStateModel, - GuiGprStateModel, - GuiLegacyGprStateModel, - GuiPassThroughStateModel, - GuiPreprocessDialogStateModel, - GuiProcessingStateModel, - GuiProfileModel, - GuiSar3dStateModel, - GuiStateModel, - GuiSwitchStateModel, -) -from python_app.models.run_config_model import ComboModel, GprRxGeometryModel, GprTxGeometryModel, RunConfigModel -from python_app.models.run_config_validation import validate_gpr_model -from python_app.orchestration.config_writer import parse_combos_from_text -from python_app.orchestration.preprocess_assets import ( - PREPROCESS_ASSET_KEYS, - VISIBLE_PREPROCESS_ASSET_KEYS, - preprocess_asset_model, -) -from python_app.storage.npz_store import radar_key_from_config - - -# GUI snapshot-save deques (`_raw_history`, `_pre_history`, `_result_history`) -# are intentionally decoupled from SHM ring capacities: rings are sized for the -# C++ processing pipeline, while save buffers only retain what the GUI poll -# loop already read out. Growing this number lets the user save more recent -# history without touching the processing-side ring sizes. -GUI_SAVE_HISTORY_LIMIT: int = 1000 - -# Mirror of `kBscanReplayWindow` in -# data_acq_and_processing/processing/data_processor/src/data_processor.cpp. When a -# live B-scan setting changes, the C++ processor re-processes and re-publishes only -# this many of the newest collections; anything older keeps the payload it was first -# computed with. Keep the two constants in sync. -CPP_BSCAN_REPLAY_WINDOW: int = 50 - - -class AppWindowConfigStateBuildersMixin: - """Build stable and GUI-only config models from current widget state.""" - - @staticmethod - def _parse_csv_int_list(text: str) -> list[int]: - """Parse comma-separated integer selection list.""" - cleaned = text.strip() - if not cleaned: - return [] - values: list[int] = [] - for part in cleaned.split(","): - token = part.strip() - if not token: - continue - values.append(int(token)) - return values - - @staticmethod - def _parse_multiline_string_list(text: str) -> list[str]: - """Parse one command argument per non-empty line.""" - return [line.strip() for line in text.splitlines() if line.strip()] - - @staticmethod - def _parse_string_env_json(text: str) -> dict[str, str]: - """Parse strict string-to-string environment JSON.""" - cleaned = text.strip() - if not cleaned: - return {} - payload = json.loads(cleaned) - if not isinstance(payload, dict): - raise ValueError("Environment JSON must be an object") - env: dict[str, str] = {} - for key, value in payload.items(): - if not isinstance(key, str) or not isinstance(value, str): - raise ValueError("Environment JSON must contain only string keys and values") - env[key] = value - return env - - @staticmethod - def _format_combos_text_from_config(config: RunConfigModel) -> str: - """Render configured combos for UI text editor, keeping full matrix as empty.""" - if config.is_matrix_radar: - return ",".join( - f"{int(combo.input)}:{int(combo.output)}" - for combo in config.build_runtime_combos() - ) - combos = list(config.combos) - full_combos = config.build_full_combos(config.input_switch.positions, config.output_switch.positions) - if len(combos) == len(full_combos) and all( - int(left.input) == int(right.input) and int(left.output) == int(right.output) - for left, right in zip(combos, full_combos, strict=True) - ): - return "" - return ",".join(f"{int(combo.input)}:{int(combo.output)}" for combo in combos) - - @staticmethod - def _default_gpr_input_positions_from_config(config: RunConfigModel) -> str: - """Build default live GPR input-position selection from stable config.""" - geometry_values = {int(entry.input_pos) for entry in config.gpr.rx_geometry} - combo_values = {int(combo.input) for combo in config.combos} - values = sorted(geometry_values & combo_values) or sorted(geometry_values) - return ",".join(str(value) for value in values) - - @staticmethod - def _default_gpr_output_positions_from_config(config: RunConfigModel) -> str: - """Build default live GPR output-position selection from stable config.""" - geometry_values = {int(entry.output_pos) for entry in config.gpr.tx_geometry} - combo_values = {int(combo.output) for combo in config.combos} - values = sorted(geometry_values & combo_values) or sorted(geometry_values) - return ",".join(str(value) for value in values) - - @staticmethod - def _default_gpr_visible_x_bounds_from_config(config: RunConfigModel) -> tuple[float, float]: - """Build default visible X-range for GPR object-only rendering.""" - x_values = [float(entry.x_m) for entry in config.gpr.tx_geometry] - x_values.extend(float(entry.x_m) for entry in config.gpr.rx_geometry) - if not x_values: - return (-2.0, 2.0) - margin_m = 2.0 - return (min(x_values) - margin_m, max(x_values) + margin_m) - - @staticmethod - def _cpp_bscan_replay_window_for_config(config: RunConfigModel) -> int: - """Return how many newest collections the C++ processor re-processes on a - live B-scan settings change. - - Deliberately reproduces `replay_history_limit()` in `data_processor.cpp` - formula-for-formula. The ring capacities matter because `ShmRing` overwrites - the oldest unread slot on overflow, so a replay burst must fit in the results - ring for the GUI to receive all of it. - - This is the single place to change if the replay window ever becomes - configurable on the C++ side. - """ - return max( - 1, - min( - int(config.rings.preprocessed.capacity), - int(config.rings.results.capacity), - CPP_BSCAN_REPLAY_WINDOW, - ), - ) - - def _default_gui_state_for_config(self, config: RunConfigModel) -> GuiStateModel: - """Build fallback GUI-only defaults for a stable run config.""" - default_combo = config.combos[0] if config.combos else ComboModel(input=0, output=0) - default_mode = "single" if len(config.combos) == 1 else "text" - default_gpr_x_min_m, default_gpr_x_max_m = self._default_gpr_visible_x_bounds_from_config(config) - return GuiStateModel( - switches=GuiSwitchStateModel( - combo_mode=default_mode, - combos_text=self._format_combos_text_from_config(config), - single_input=str(int(default_combo.input)), - single_output=str(int(default_combo.output)), - ), - processing=GuiProcessingStateModel( - gpr=GuiGprStateModel( - input_positions=self._default_gpr_input_positions_from_config(config), - output_positions=self._default_gpr_output_positions_from_config(config), - visible_x_min_m=default_gpr_x_min_m, - visible_x_max_m=default_gpr_x_max_m, - ), - legacy_gpr=GuiLegacyGprStateModel( - input_positions=self._default_gpr_input_positions_from_config(config), - output_positions=self._default_gpr_output_positions_from_config(config), - visible_x_min_m=default_gpr_x_min_m, - visible_x_max_m=default_gpr_x_max_m, - ), - sar3d=GuiSar3dStateModel( - input_positions=self._default_gpr_input_positions_from_config(config), - output_positions=self._default_gpr_output_positions_from_config(config), - ), - ), - data_actions=GuiDataActionsStateModel( - save_path=str(self._project_root / "python_app/data/snapshots"), - ), - ) - - def _current_preprocess_set_name(self) -> str: - """Return current preprocess dialog set name, even when dialog is still closed.""" - if self._preprocess_dialog is not None: - self._preprocess_set_name = self._preprocess_dialog.set_name() - return self._preprocess_set_name - - def _current_preprocess_radar_config_dir(self) -> str: - """Return current preprocess dialog radar-config directory path.""" - if self._preprocess_dialog is not None: - self._preprocess_radar_config_dir = self._preprocess_dialog.radar_config_dir() - return self._preprocess_radar_config_dir - - def _current_preprocess_use_all_radar_configs(self) -> bool: - """Return whether multi-radar preprocessing is enabled in the dialog.""" - if self._preprocess_dialog is not None: - self._preprocess_use_all_radar_configs = self._preprocess_dialog.use_all_radar_configs() - return bool(self._preprocess_use_all_radar_configs) - - def _current_preprocess_median_sweep_count(self) -> int: - """Return current per-combo median sweep count from the dialog.""" - if self._preprocess_dialog is not None: - self._preprocess_median_sweep_count = self._preprocess_dialog.median_sweep_count() - return max(1, int(self._preprocess_median_sweep_count)) - - def _build_gui_state(self) -> GuiStateModel: - """Build GUI-only persistent state from current widget values.""" - return GuiStateModel( - switches=GuiSwitchStateModel( - combo_mode=( - "text" - if self._is_matrix_radar_model_selected() - else "single" - if self._single_combo_select_button.isChecked() - else "text" - ), - combos_text=( - self._fixed_matrix_radar_combo_text() - if self._is_matrix_radar_model_selected() - else self._combos_text.text().strip() - ), - single_input=self._single_combo_input.text().strip(), - single_output=self._single_combo_output.text().strip(), - ), - processing=GuiProcessingStateModel( - selected_mode=self._processing_mode.currentText(), - pass_through=GuiPassThroughStateModel( - show_magnitude=bool(self._show_magnitude_checkbox.isChecked()), - show_phase=bool(self._show_phase_checkbox.isChecked()), - unwrap_phase=bool(self._unwrap_phase_checkbox.isChecked()), - combo_filter=self._pass_through_combo_filter_input.text().strip(), - fixed_y_enabled=bool(self._pass_through_fixed_y_enabled.isChecked()), - y_min_db=float(self._pass_through_y_min_db.value()), - y_max_db=float(self._pass_through_y_max_db.value()), - ), - bscan=GuiBscanStateModel( - axis=self._bscan_axis.currentText(), - cut_m=float(self._bscan_cut_m.value()), - max_depth_m=float(self._bscan_max_depth_m.value()), - gain=float(self._bscan_gain.value()), - start_freq_mhz=float(self._bscan_start_freq_mhz.value()), - stop_freq_mhz=float(self._bscan_stop_freq_mhz.value()), - subtract_mean_ascan=bool(self._bscan_subtract_mean_ascan.isChecked()), - history_window_scans=int(self._bscan_history_window.value()), - ), - gpr=GuiGprStateModel( - input_positions=self._gpr_input_positions_input.text().strip(), - output_positions=self._gpr_output_positions_input.text().strip(), - min_depth_m=float(self._gpr_min_depth_m.value()), - max_depth_m=float(self._gpr_max_depth_m.value()), - range_comp_power=float(self._gpr_range_comp_power.value()), - object_min_frac=float(self._gpr_object_min_frac.value()), - score_mode=self._gpr_score_mode.currentText(), - motion_mode=self._gpr_motion_mode.currentText(), - look_angle_deg=float(self._gpr_look_angle_deg.value()), - direction_sign=float(self._gpr_direction_sign.value()), - speed_m_s=float(self._gpr_speed_m_s.value()), - ignore_socket_speed_enabled=bool(self._gpr_ignore_socket_speed_enabled.isChecked()), - max_detected_objects_to_draw=int(self._gpr_max_detected_objects_to_draw.value()), - draw_top_m_objects=int(self._gpr_draw_top_m_objects.value()), - object_approach_min_frames=int(self._gpr_object_approach_min_frames.value()), - start_freq_mhz=float(self._gpr_start_freq_mhz.value()), - stop_freq_mhz=float(self._gpr_stop_freq_mhz.value()), - background_subtract_enabled=bool(self._gpr_background_subtract_enabled.isChecked()), - background_mean_count=int(self._gpr_background_mean_count.value()), - remove_sidelobe_objects_enabled=bool(self._gpr_remove_sidelobe_objects_enabled.isChecked()), - imaging_plane_y_m=float(self._gpr_imaging_plane_y_m.value()), - render_mode=self._gpr_render_mode.currentText(), - visible_x_min_m=float(self._gpr_visible_x_min_m.value()), - visible_x_max_m=float(self._gpr_visible_x_max_m.value()), - visible_z_min_m=float(self._gpr_visible_z_min_m.value()), - visible_z_max_m=float(self._gpr_visible_z_max_m.value()), - ), - legacy_gpr=GuiLegacyGprStateModel( - mode=self._legacy_gpr_config_mode.currentText(), - input_positions=self._legacy_gpr_input_positions_input.text().strip(), - output_positions=self._legacy_gpr_output_positions_input.text().strip(), - min_depth_m=float(self._legacy_gpr_min_depth_m.value()), - max_depth_m=float(self._legacy_gpr_max_depth_m.value()), - comp_power=float(self._legacy_gpr_comp_power.value()), - start_freq_mhz=float(self._legacy_gpr_start_freq_mhz.value()), - stop_freq_mhz=float(self._legacy_gpr_stop_freq_mhz.value()), - speed_m_s=float(self._legacy_gpr_speed_m_s.value()), - ignore_socket_speed_enabled=bool(self._legacy_gpr_ignore_socket_speed_enabled.isChecked()), - look_angle_deg=float(self._legacy_gpr_look_angle_deg.value()), - apply_freq_phase_correction=bool( - self._legacy_gpr_apply_freq_phase_correction.isChecked() - ), - reference_mode=self._legacy_gpr_reference_mode.currentText(), - snr_thresh=float(self._legacy_gpr_snr_thresh.value()), - snr_comp_max=float(self._legacy_gpr_snr_comp_max.value()), - background_subtract_enabled=bool(self._legacy_gpr_background_subtract_enabled.isChecked()), - background_mean_count=int(self._legacy_gpr_background_mean_count.value()), - render_mode=self._legacy_gpr_render_mode.currentText(), - min_visible_pair_count=int(self._legacy_gpr_min_visible_pair_count.value()), - visible_x_min_m=float(self._legacy_gpr_visible_x_min_m.value()), - visible_x_max_m=float(self._legacy_gpr_visible_x_max_m.value()), - visible_z_min_m=float(self._legacy_gpr_visible_z_min_m.value()), - visible_z_max_m=float(self._legacy_gpr_visible_z_max_m.value()), - ), - sar3d=GuiSar3dStateModel( - input_positions=self._sar3d_input_positions_input.text().strip(), - output_positions=self._sar3d_output_positions_input.text().strip(), - min_depth_m=float(self._sar3d_min_depth_m.value()), - max_depth_m=float(self._sar3d_max_depth_m.value()), - start_freq_mhz=float(self._sar3d_start_freq_mhz.value()), - stop_freq_mhz=float(self._sar3d_stop_freq_mhz.value()), - speed_m_s=float(self._sar3d_speed_m_s.value()), - ignore_socket_speed_enabled=bool(self._sar3d_ignore_socket_speed_enabled.isChecked()), - look_angle_deg=float(self._sar3d_look_angle_deg.value()), - frame_radius=int(self._sar3d_frame_radius.value()), - voxel_m=float(self._sar3d_voxel_m.value()), - bp_ground_depth_m=float(self._sar3d_bp_ground_depth_m.value()), - comp_range_power=float(self._sar3d_comp_range_power.value()), - use_compensation=bool(self._sar3d_use_compensation.isChecked()), - normalize_each_frame=bool(self._sar3d_normalize_each_frame.isChecked()), - ), - ), - data_actions=GuiDataActionsStateModel( - save_path=self._save_path_input.text().strip(), - save_name=self._save_name_input.text().strip(), - record_count=int(self._record_count.value()), - ), - preprocess_dialog=GuiPreprocessDialogStateModel( - set_name=self._current_preprocess_set_name(), - radar_config_dir=self._current_preprocess_radar_config_dir(), - use_all_radar_configs=self._current_preprocess_use_all_radar_configs(), - median_sweep_count=self._current_preprocess_median_sweep_count(), - ), - ) - - def _build_gui_profile(self) -> GuiProfileModel: - """Build full GUI config profile from current window state.""" - return GuiProfileModel( - run_config=self._build_config(), - gui=self._build_gui_state(), - ) - - def _build_config(self) -> RunConfigModel: - """Build `RunConfigModel` from current GUI widget values.""" - config = self._defaults_config.clone() - - config.radar.sweep.start_hz = float(self._start_hz_input.text().strip()) - config.radar.sweep.stop_hz = float(self._stop_hz_input.text().strip()) - config.radar.sweep.points = int(self._points_input.text().strip()) - config.radar.sweep.if_bandwidth_hz = float(self._ifbw_input.text().strip()) - config.radar.sweep.power_dbm = float(self._power_input.text().strip()) - if config.is_kamil_adc: - self._apply_adc_settings_to_config(config) - - config.runtime.settling_ms = int(self._settling_ms.text().strip()) - config.runtime.processing_live_config_path = str(self._live_config_writer.path) - config.logging.level = self._log_level_combo.currentText().strip().lower() - - if config.is_matrix_radar: - if config.is_multi_device and len(config.radar.multi_device.slave_serials) != 2: - raise ValueError("LibreVNA multi-device mode requires exactly two slave serials") - config.apply_device_model_constraints() - else: - if self._single_combo_select_button.isChecked(): - config.combos = [ - ComboModel( - input=int(self._single_combo_input.text().strip()), - output=int(self._single_combo_output.text().strip()), - ) - ] - else: - combo_text = self._combos_text.text() - config.combos = parse_combos_from_text(combo_text) - config.ensure_combos() - if self._processing_mode.currentText() not in {"gpr", "legacy_gpr", "sar3d"} and self._switches_are_effectively_static(config): - config.combos = [ComboModel(input=0, output=0)] - - for key in PREPROCESS_ASSET_KEYS: - preprocess_asset_model(config, key).bundle_path = "" - for key in VISIBLE_PREPROCESS_ASSET_KEYS: - preprocess_asset_model(config, key).set_name = self._selected_preprocess_sets.get(key, "") - config.gpr.relative_permittivity = float(self._gpr_relative_permittivity.value()) - config.gpr.tx_geometry = [GprTxGeometryModel(**entry) for entry in self._gpr_tx_geometry_input.records()] - config.gpr.rx_geometry = [GprRxGeometryModel(**entry) for entry in self._gpr_rx_geometry_input.records()] - validate_gpr_model( - config.gpr, - input_switch_positions=config.input_switch.positions, - output_switch_positions=config.output_switch.positions, - sweep=config.radar.sweep, - ) - return config - - def _radar_key(self, config: RunConfigModel) -> str: - """Build radar key used by preprocess-set storage lookup.""" - return radar_key_from_config( - model_name=config.radar.model, - serial=config.radar.serial, - sweep_start_hz=config.radar.sweep.start_hz, - sweep_stop_hz=config.radar.sweep.stop_hz, - sweep_points=config.radar.sweep.points, - ifbw_hz=config.radar.sweep.if_bandwidth_hz, - power_dbm=config.radar.sweep.power_dbm, - extra_serials=config.radar_key_extra_parts() or None, - ) - - def _radar_key_from_ui(self) -> str: - """Build current radar key directly from radar widgets only.""" - model_name = self._defaults_config.radar.model or RunConfigModel.LIBREVNA_MODEL - extra_parts = self._defaults_config.radar_key_extra_parts() - if self._defaults_config.is_kamil_adc: - config = self._defaults_config.clone() - config.radar.sweep.start_hz = float(self._start_hz_input.text().strip()) - config.radar.sweep.stop_hz = float(self._stop_hz_input.text().strip()) - config.radar.sweep.points = int(self._points_input.text().strip()) - config.radar.sweep.if_bandwidth_hz = float(self._ifbw_input.text().strip()) - config.radar.sweep.power_dbm = float(self._power_input.text().strip()) - self._apply_adc_settings_to_config(config) - extra_parts = config.radar_key_extra_parts() - return radar_key_from_config( - model_name=model_name, - serial=self._defaults_config.radar.serial, - sweep_start_hz=float(self._start_hz_input.text().strip()), - sweep_stop_hz=float(self._stop_hz_input.text().strip()), - sweep_points=int(self._points_input.text().strip()), - ifbw_hz=float(self._ifbw_input.text().strip()), - power_dbm=float(self._power_input.text().strip()), - extra_serials=extra_parts or None, - ) - - def _apply_adc_settings_to_config(self, config: RunConfigModel) -> None: - """Copy visible ADC collector and optical-board settings into config.""" - adc = config.radar.kamil_adc - adc.project_dir = self._adc_project_dir_input.text().strip() - adc.executable_path = self._adc_executable_path_input.text().strip() - adc.tty_path = self._adc_tty_path_input.text().strip() - adc.args = self._parse_multiline_string_list(self._adc_args_input.toPlainText()) - adc.env = self._parse_string_env_json(self._adc_env_input.toPlainText()) - adc.startup_timeout_s = float(self._adc_startup_timeout_s_input.text().strip()) - adc.sweep_timeout_s = float(self._adc_sweep_timeout_s_input.text().strip()) - adc.stop_timeout_s = float(self._adc_stop_timeout_s_input.text().strip()) - - optical = config.radar.laser_control - optical.enabled = bool(self._optical_enabled_checkbox.isChecked()) - optical.port = self._optical_port_input.text().strip() - optical.mode = self._optical_mode_combo.currentText() - optical.pi_coeff1_p = int(self._optical_pi_coeff1_p_input.text().strip()) - optical.pi_coeff1_i = int(self._optical_pi_coeff1_i_input.text().strip()) - optical.pi_coeff2_p = int(self._optical_pi_coeff2_p_input.text().strip()) - optical.pi_coeff2_i = int(self._optical_pi_coeff2_i_input.text().strip()) - - optical.manual.temp1 = float(self._optical_manual_temp1_input.text().strip()) - optical.manual.temp2 = float(self._optical_manual_temp2_input.text().strip()) - optical.manual.current1 = float(self._optical_manual_current1_input.text().strip()) - optical.manual.current2 = float(self._optical_manual_current2_input.text().strip()) - - optical.variation.variation_type = self._optical_variation_type_combo.currentText() - optical.variation.static_temp1 = float(self._optical_static_temp1_input.text().strip()) - optical.variation.static_temp2 = float(self._optical_static_temp2_input.text().strip()) - optical.variation.static_current1 = float(self._optical_static_current1_input.text().strip()) - optical.variation.static_current2 = float(self._optical_static_current2_input.text().strip()) - optical.variation.min_value = float(self._optical_min_value_input.text().strip()) - optical.variation.max_value = float(self._optical_max_value_input.text().strip()) - optical.variation.step = float(self._optical_step_input.text().strip()) - optical.variation.time_step = int(self._optical_time_step_input.text().strip()) - optical.variation.delay_time = int(self._optical_delay_time_input.text().strip()) - - @staticmethod - def _switches_are_effectively_static(config: RunConfigModel) -> bool: - """Return `True` when non-GPR switch setup effectively yields one fixed combo.""" - if config.is_multi_device: - return False - has_single_position = config.input_switch.positions <= 1 and config.output_switch.positions <= 1 - both_mock = config.input_switch.driver_mode == "mock" and config.output_switch.driver_mode == "mock" - return has_single_position or both_mock diff --git a/python_app/gui/controllers/app_window_config_mixin.py b/python_app/gui/controllers/app_window_config_mixin.py deleted file mode 100644 index 16d1554..0000000 --- a/python_app/gui/controllers/app_window_config_mixin.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Facade mixin composing focused config/profile/runtime controller mixins.""" - -from python_app.gui.controllers.app_window_config import ( - AppWindowConfigProfileIOMixin, - AppWindowConfigStateBuildersMixin, - AppWindowLiveProcessingMixin, - AppWindowRadarLimitsMixin, -) - - -class AppWindowConfigMixin( - AppWindowConfigStateBuildersMixin, - AppWindowConfigProfileIOMixin, - AppWindowLiveProcessingMixin, - AppWindowRadarLimitsMixin, -): - """Backward-compatible facade for AppWindow config-related behavior.""" - - pass diff --git a/python_app/gui/controllers/app_window_control_button_mixin.py b/python_app/gui/controllers/app_window_control_button_mixin.py deleted file mode 100644 index e87475a..0000000 --- a/python_app/gui/controllers/app_window_control_button_mixin.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Mixin wiring a physical GPIO control button to a runtime action. - -Both GUI and headless launches construct :class:`AppWindow`, so attaching the -watcher here makes the button behave identically in both modes. The button -reuses the existing "Capture Tmp Reference" flow (stop the pipeline, capture a -fresh tmp reference, then restart the pipeline if it had been running), so no -acquisition logic is duplicated for the hardware trigger. -""" - -from __future__ import annotations - -from python_app.gui.control_button import ControlButtonWatcher -from python_app.models.run_config_schema import ControlButtonModel - - -class AppWindowControlButtonMixin: - """Start and stop a background GPIO button watcher bound to a runtime action.""" - - def _init_control_button_state(self) -> None: - """Initialize the watcher handle before the watcher is started.""" - self._control_button_watcher: ControlButtonWatcher | None = None - # Re-entrancy guard: ignore presses that arrive while a capture is in progress. - self._control_button_busy = False - - def _start_control_button_watcher(self) -> None: - """Open the configured GPIO button line and begin watching for presses. - - Any failure is logged and swallowed: the same build runs on developer - machines and non-Pi hosts where the GPIO chip is absent, and a missing - button must never abort startup. - """ - config = getattr(self._defaults_config, "control_button", None) - if config is None or not config.enabled: - return - if config.pin < 0 or not config.gpio_chip: - self._log_warning( - "Control button enabled but gpio_chip/pin are unset; watcher not started." - ) - return - if config.action != ControlButtonModel.ACTION_CAPTURE_TMP_REFERENCE: - self._log_warning( - f"Control button action '{config.action}' is not supported; watcher not started." - ) - return - - try: - watcher = ControlButtonWatcher( - chip=config.gpio_chip, - pin=config.pin, - active_low=config.active_low, - bias=config.bias, - debounce_ms=config.debounce_ms, - parent=self, - ) - watcher.pressed.connect(self._on_control_button_pressed) - watcher.failed.connect(self._on_control_button_failed) - watcher.start() - except Exception as exc: # noqa: BLE001 - self._log_exception("Failed to start GPIO control button watcher", exc, level="WARN") - return - - self._control_button_watcher = watcher - self._log( - "GPIO control button watcher started: " - f"chip={config.gpio_chip}, pin={config.pin}, active_low={config.active_low}, " - f"debounce_ms={config.debounce_ms}, action={config.action}" - ) - - def _on_control_button_pressed(self) -> None: - """Run the configured action for a physical press on the Qt main thread. - - Delivered as a queued signal from the watcher thread, so this executes - on the GUI thread exactly like a click on "Capture Tmp Reference". - """ - # The shared re-entrancy guard lives in _capture_tmp_reference, so this GPIO - # trigger and the GUI "Capture Tmp Reference" button are protected by one - # mechanism (a press during an in-progress capture is ignored there). - self._log("GPIO control button pressed: capturing tmp reference.") - self._capture_tmp_reference() - - def _on_control_button_failed(self, message: str) -> None: - """Log an unrecoverable watcher error reported from the background thread.""" - self._log_warning("GPIO control button watcher stopped", details=message) - - def _stop_control_button_watcher(self) -> None: - """Stop the watcher and release its GPIO line during shutdown.""" - watcher = getattr(self, "_control_button_watcher", None) - if watcher is None: - return - # Disconnect first so a press queued before teardown cannot run a slot afterwards. - self._disconnect_control_button_signals(watcher) - try: - watcher.stop() - except Exception as exc: # noqa: BLE001 - self._log_warning("Error stopping GPIO control button watcher", details=str(exc)) - finally: - # Disconnect again in case stop() re-emitted, then drop and schedule deletion. - self._disconnect_control_button_signals(watcher) - self._control_button_watcher = None - watcher.deleteLater() - - @staticmethod - def _disconnect_control_button_signals(watcher: ControlButtonWatcher) -> None: - """Detach the watcher's signals from their slots, tolerating already-disconnected.""" - for signal in (watcher.pressed, watcher.failed): - try: - signal.disconnect() - except (TypeError, RuntimeError): - # No connections left (or the C++ object is already gone): nothing to do. - pass diff --git a/python_app/gui/controllers/app_window_pipeline_mixin.py b/python_app/gui/controllers/app_window_pipeline_mixin.py deleted file mode 100644 index bf02e50..0000000 --- a/python_app/gui/controllers/app_window_pipeline_mixin.py +++ /dev/null @@ -1,781 +0,0 @@ -"""Pipeline runtime lifecycle mixin for the main GUI window.""" - -from __future__ import annotations - -import time - -from PyQt6.QtCore import QCoreApplication, QEventLoop - -from python_app.gui.runtime.constraints import validate_processing_mode_constraints -from python_app.gui.controllers.app_window_plot.sar3d_plot_mixin import sar3d_status -from python_app.gui.runtime.history import ( - build_processor_run_signature, - build_run_history_signature, - record_result_history, -) -from python_app.hardware_full.kamil_adc import apply_kamil_adc_laser_control -from python_app.hardware_full.single_radar_service import create_single_radar_service -from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection -from python_app.models.run_config_model import RunConfigModel -from python_app.models.run_config_validation import validate_acquisition_config -from python_app.orchestration.preprocess_assets import ( - PREPROCESS_ASSET_SPECS, - REQUIRED_PREPROCESS_ASSET_KEYS, - preprocess_asset_model, - runtime_preprocess_asset_keys, -) -from python_app.orchestration.restart_policy import RestartPolicy -from python_app.orchestration.shm_reader import ShmRingReader - - -class AppWindowPipelineMixin: - """Controls start/stop, readers, and periodic polling of pipeline rings.""" - - # A repeated identical reader error is deduped from the log, but must still be - # re-logged every this-many polls so a persistent failure stays visible. - _READER_ERROR_RELOG_EVERY = 200 - # After this many consecutive identical reader errors, attempt one reader - # reconnect; if it keeps failing past the next threshold, stop the pipeline so - # a wedged reader cannot stay silently broken forever. - _READER_ERROR_RECONNECT_AT = 40 - _READER_ERROR_STOP_AT = 400 - # If a pipeline child exits unexpectedly while the run should be live, relaunch - # the whole pipeline from the last-written runtime config. It retries FOREVER with - # capped back-off (this is an unattended appliance — it must keep trying to come - # back, never permanently stop); the failure streak resets once a healthy poll sees - # data. The "if it breaks it comes back up" contract holds in BOTH GUI and headless. - _RESTART_POLICY = RestartPolicy(min_interval_s=3.0, max_interval_s=60.0) - # Safety backstop so a wedged/dropping processor cannot leave a single capture - # polling forever with no completion and no error. Generous, not a tight deadline. - _SINGLE_CAPTURE_TIMEOUT_S = 300.0 - - def _processor_requires_restart(self, run_signature: tuple[object, ...]) -> bool: - """Return whether alive `data_processor` was started with different stable run settings.""" - return self._supervisor.is_processor_running() and self._processor_run_signature != run_signature - - def _start_single_capture(self) -> None: - """Start acquisition in single-capture mode.""" - self._start_run(single_capture=True) - - def _start_run(self, *, single_capture: bool = False, config_override: RunConfigModel | None = None) -> None: - """Start pipeline processes and ring readers.""" - if getattr(self, "_closing", False) or getattr(self, "_close_after_stop", False): - return - if getattr(self, "_stopping_run", False): - self._show_error("Wait for pipeline shutdown before starting again") - return - if config_override is None and getattr(self, "_object_capture_job", None) is not None: - self._show_error("Wait for object-calibration capture to finish") - return - if self._capture_session is not None: - self._show_error( - "Cannot start pipeline during active capture sequence", - details=self._capture_state_details(), - ) - return - if self._supervisor.is_running(): - # A single-shot capture from a running continuous pipeline must - # restart acquisition with `runtime.continuous=false`; refusing - # here would leave the previous run streaming and the user could - # never reach the single-capture termination state. - if not single_capture: - self._show_error( - "Pipeline is already running", - details=self._process_state_details(), - ) - return - self._log("Stopping continuous pipeline before single capture") - self._stop_run() - if getattr(self, "_closing", False): - return - - try: - processor_was_running = self._supervisor.is_processor_running() - config = config_override.clone() if config_override is not None else self._build_config() - validate_acquisition_config(config) - run_signature = self._build_run_history_signature(config) - processor_signature = self._build_processor_run_signature(config) - if self._processor_requires_restart(processor_signature): - self._log("Restarting data_processor because stable run settings changed") - self._stop_all_processes() - if getattr(self, "_closing", False): - return - processor_was_running = False - if not processor_was_running: - self._reset_runtime_history() - - if config_override is None: - self._validate_processing_mode_constraints(config) - radar_key = self._radar_key(config) - - missing_assets = [ - PREPROCESS_ASSET_SPECS[key].display_name - for key in REQUIRED_PREPROCESS_ASSET_KEYS - if not preprocess_asset_model(config, key).set_name - ] - if missing_assets: - raise RuntimeError( - "Select all required preprocess sets in Preprocessing Panel before Start: " - + ", ".join(missing_assets) - ) - - combo_keys = [ComboKey(input=combo.input, output=combo.output) for combo in config.combos] - active_preprocess_keys = runtime_preprocess_asset_keys(config) - preprocess_summary = "; ".join( - f"{PREPROCESS_ASSET_SPECS[key].display_name}={preprocess_asset_model(config, key).set_name}" - for key in active_preprocess_keys - ) - self._log(f"Active preprocess assets for run: {preprocess_summary}") - - for key in active_preprocess_keys: - spec = PREPROCESS_ASSET_SPECS[key] - asset = preprocess_asset_model(config, key) - if not self._store.has_combo_coverage(spec.set_kind, radar_key, asset.set_name, combo_keys): - raise RuntimeError(f"Selected {spec.display_name} set does not cover requested run combos") - - self._config_writer.prepare_preprocess_bundles(self._store, radar_key, config) - config.runtime.continuous = not single_capture - - if not single_capture and not config.is_kamil_adc: - self._prepare_radar_for_native_acquisition(config) - - config_path = self._config_writer.write(config, self._project_root / "python_app/runtime/run_config.json") - combo_preview = ", ".join(f"in{combo.input}/out{combo.output}" for combo in config.combos[:6]) - if len(config.combos) > 6: - combo_preview += ", ..." - self._log( - f"Starting pipeline: mode={'single_capture' if single_capture else 'continuous'}, " - f"config={config_path}, combos={len(config.combos)}" - f"{', ' + combo_preview if combo_preview else ''}, radar_key={radar_key}" - ) - - if self._history_run_signature is not None and self._history_run_signature != run_signature: - if processor_was_running: - self._reset_runtime_history() - self._write_live_processing_config(history_command="clear_all", bump_history_seq=True) - self._log("History reset because run settings changed") - self._history_run_signature = run_signature - - # Open the single-capture acquisition window BEFORE the producer can emit: - # any collection the one-shot orchestrator publishes is then strictly newer - # than this timestamp and is recognized as the target. Reading the clock - # AFTER start() raced a fast (simulator) orchestrator that had already - # published — its collection.monotonic_ns < start_ns, so the capture never - # completed and the GUI hung. - single_capture_start_ns = time.monotonic_ns() if single_capture else None - - # `data_preprocessor` is about to become the owner of the preprocessed ring - # again, so the GUI must stop holding a writer into it. A queued replay would - # otherwise interleave with the real producer. - self._bscan_reprocess_timer.stop() - self._close_replay_ring_writer() - self._discard_bscan_replay_results() - - self._supervisor.start(config_path, allow_clean_orchestrator_exit=single_capture) - self._close_readers() - self._raw_reader = ShmRingReader(config.rings.raw_tap.name) - self._pre_reader = ShmRingReader(config.rings.preprocessed_tap.name) - self._result_reader = ShmRingReader(config.rings.results.name) - self._processor_run_signature = processor_signature - self._single_capture_active = single_capture - self._single_capture_start_ns = single_capture_start_ns - self._single_capture_seen_raw = False - self._single_capture_target_collection_id = None - - if not single_capture: - # Continuous start: discard any stale tail so rendering begins fresh. - # Single capture must NOT drop here — the drop races with (and can - # discard) the one-shot result; it instead filters stale data by the - # start timestamp recorded above. - self._drop_pending_ring_payloads(include_results=True) - self._last_reader_error_signature = None - self._reader_error_repeat_count = 0 - # Record what to relaunch if a child later dies unexpectedly. Only a - # continuous run auto-restarts; a single capture is bounded by its deadline. - self._active_run_config = config - self._active_run_config_path = config_path - self._pipeline_should_run = not single_capture - self._pipeline_restart_count = 0 - - pid_map = self._supervisor.pids() - pid_text = ", ".join(f"{name}={pid}" for name, pid in sorted(pid_map.items())) or "none" - if single_capture: - self._status_label.setText("Status: single capture running") - self._log(f"Single capture started; managed processes: {pid_text}") - else: - self._status_label.setText("Status: running") - self._log(f"Pipeline started; managed processes: {pid_text}") - except Exception as exc: # noqa: BLE001 - self._single_capture_active = False - self._single_capture_start_ns = None - self._stop_all_processes() - self._show_exception("Failed to start pipeline", exc) - - def _apply_radar_settings(self) -> None: - """Apply current radar settings by preconfiguring native device.""" - if getattr(self, "_object_capture_job", None) is not None: - self._show_error("Wait for object-calibration capture to finish") - return - if self._capture_session is not None: - self._show_error( - "Finish or abort capture sequence before applying radar settings", - details=self._capture_state_details(), - ) - return - - was_running = self._supervisor.is_running() - resume_seq = getattr(self, "_stop_request_seq", 0) + 1 - processor_only_running = self._supervisor.is_processor_running() and not was_running - if was_running: - self._stop_run() - if not self._pipeline_resume_allowed(resume_seq): - return - - try: - config = self._build_config() - validate_acquisition_config(config) - if config.radar.driver_mode == "native" and not config.radar.replay.dataset_dir: - self._refresh_radar_limits_from_device() - processor_signature = self._build_processor_run_signature(config) - if processor_only_running and self._processor_requires_restart(processor_signature): - self._stop_all_processes() - self._reset_runtime_history() - self._history_run_signature = None - self._log( - "Stable run settings changed; stopped data_processor because cached history no longer " - "matches the current config" - ) - self._prepare_radar_for_native_acquisition(config) - points_text = ( - "points=from Kamil ADC stream" - if config.is_kamil_adc - else f"points={config.radar.sweep.points}" - ) - self._log( - "Radar settings applied: " - f"start={config.radar.sweep.start_hz:g} Hz, " - f"stop={config.radar.sweep.stop_hz:g} Hz, " - f"{points_text}, " - f"ifbw={config.radar.sweep.if_bandwidth_hz:g} Hz, " - f"power={config.radar.sweep.power_dbm:g} dBm" - ) - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to apply radar settings", exc) - finally: - if was_running and self._pipeline_resume_allowed(resume_seq): - self._start_run() - - def _prepare_radar_for_native_acquisition(self, config: RunConfigModel) -> None: - """Preconfigure native single-radar hardware using current sweep settings.""" - if config.radar.replay.dataset_dir: - self._log("Radar pre-configuration skipped (dataset replay)") - return - if config.radar.driver_mode != "native": - self._log("Radar pre-configuration skipped (mock mode)") - return - if config.is_multi_device: - self._log("Multi-device raw producer will configure all LibreVNA devices") - return - if config.is_matrix_radar: - self._log("Matrix raw producer will configure the matrix radar") - return - if config.is_kamil_adc: - if apply_kamil_adc_laser_control(config): - self._log("Kamil ADC laser_control applied via Apply Radar") - else: - self._log("Kamil ADC laser_control skipped because it is disabled") - return - - radar_service = create_single_radar_service(config) - try: - radar_service.open() - radar_service.configure(config.radar.sweep) - finally: - radar_service.close() - - self._log(f"Radar pre-configured via Python driver: model={config.radar.model}") - - def _stop_run(self) -> None: - """Stop acquisition-side processes and close readers as needed.""" - self._stop_request_seq = getattr(self, "_stop_request_seq", 0) + 1 - self._resume_pipeline_after_capture = False - self._stop_pipeline(keep_processor=True) - - def _stop_all_processes(self) -> None: - """Stop all managed pipeline processes and close all readers.""" - self._stop_pipeline(keep_processor=False) - - def _stop_pipeline(self, *, keep_processor: bool) -> None: - """Share shutdown/drain/finalization, protecting event-loop re-entry.""" - self._pipeline_should_run = False - if getattr(self, "_stopping_run", False): - self._stop_all_requested |= not keep_processor - return - self._stopping_run = True - self._stop_all_requested = not keep_processor - try: - drain_error = None - try: - was_running = self._supervisor.is_running() - if keep_processor and was_running: - self._supervisor.stop_orchestrator() - self._drain_rings_until_quiet(timeout_s=0.35, poll_s=0.02) - self._supervisor.stop_preprocessor() - self._drain_rings_until_quiet(timeout_s=0.25, poll_s=0.02) - elif keep_processor: - self._supervisor.stop() - self._drain_rings_once_for_history() - if self._stop_all_requested: - self._supervisor.stop_all() - self._drain_rings_until_quiet(timeout_s=0.25, poll_s=0.02) - except Exception as exc: # noqa: BLE001 - corrupt data must not prevent Stop - drain_error = exc - self._stop_request_seq = getattr(self, "_stop_request_seq", 0) + 1 - if self._stop_all_requested: - self._supervisor.stop_all() - else: - self._supervisor.stop() - if self._stop_all_requested: - self._processor_run_signature = None - self._finalize_recording_on_stop() - keep_results = not self._stop_all_requested and self._supervisor.is_processor_running() - result_ring_name = getattr(self._result_reader, "_ring_name", None) if drain_error else None - self._close_readers(keep_results=keep_results and drain_error is None) - self._single_capture_active = False - self._single_capture_start_ns = None - self._single_capture_seen_raw = False - self._single_capture_target_collection_id = None - self._update_history_indicator() - self._status_label.setText("Status: idle") - self._log("Acquisition stopped (data_processor kept running)" if keep_results else "Pipeline stopped") - if drain_error is not None: - self._status_label.setText("Status: error") - self._show_exception("Pipeline stopped, but final ring data could not be read", drain_error) - if keep_results and result_ring_name: - self._result_reader = ShmRingReader(result_ring_name) - except Exception as exc: # noqa: BLE001 - this is also a direct Qt button slot - self._stop_request_seq = getattr(self, "_stop_request_seq", 0) + 1 - self._status_label.setText("Status: error") - self._show_exception("Failed to stop pipeline", exc) - finally: - self._stopping_run = False - if getattr(self, "_close_after_stop", False): - self._close_after_stop = False - self.close() - - def _pipeline_resume_allowed(self, stop_seq: int) -> bool: - """A Stop or Close received during a temporary pause cancels its restart.""" - return not getattr(self, "_closing", False) and not getattr(self, "_close_after_stop", False) and ( - getattr(self, "_stop_request_seq", 0) == stop_seq - ) - - def _close_readers(self, *, keep_results: bool = False) -> None: - """Close active ring readers; keep the results reader when ``keep_results``.""" - closed = [] - if self._raw_reader is not None: - self._raw_reader.close() - self._raw_reader = None - closed.append("raw") - if self._pre_reader is not None: - self._pre_reader.close() - self._pre_reader = None - closed.append("preprocessed") - if not keep_results and self._result_reader is not None: - self._result_reader.close() - self._result_reader = None - closed.append("results") - if closed: - self._log_debug(f"Closed ring readers: {', '.join(closed)}.") - - def _poll_rings(self) -> None: - """Poll readers, ingest history, and trigger rendering.""" - if getattr(self, "_closing", False) or getattr(self, "_stopping_run", False): - return - self._web_update_snapshot() # guarded internally; never raises - self._handle_process_exit_reports() - - try: - if self._raw_reader is not None: - self._read_all_raw() - self._read_all_preprocessed() - result_latest = self._read_all_results() if self._result_reader is not None else None - self._poll_recording_writer() # finalize a disk recording once its writer drains - self._update_history_indicator() - self._last_reader_error_signature = None - self._reader_error_repeat_count = 0 - - if self._single_capture_active: - if self._finish_single_capture_if_ready(): - return - self._check_single_capture_deadline() - return - - if result_latest is not None: - # Genuine data flowed: the pipeline is healthy, so reset the - # crash-storm budget (it only caps consecutive crash-restarts). - self._pipeline_restart_count = 0 - render_started_ns = time.monotonic_ns() - self._draw_results(result_latest) - self._pipeline_metrics.record( - "rendering", time.monotonic_ns() - render_started_ns - ) - except Exception as exc: # noqa: BLE001 - self._handle_reader_poll_error(exc) - - def _handle_process_exit_reports(self) -> None: - """Log child exits and auto-restart the pipeline on an unexpected death. - - Runs first in the poll tick and is fully guarded: it must never raise, or - it would abort the Qt slot. An unexpected (non-clean) exit while the run is - meant to be live triggers a bounded relaunch — in both GUI and headless. - """ - unexpected = False - try: - for report in self._supervisor.collect_exit_reports(): - if report.level == "INFO": - self._log(report.format()) - continue - self._status_label.setText("Status: error") - self._log_error(report.format()) - unexpected = True - except Exception as exc: # noqa: BLE001 - the poll tick must survive this - self._log_exception("Failed to collect process exit reports", exc, level="ERROR") - return - - if getattr(self, "_pipeline_should_run", False) and ( - unexpected - or not {"data_processor", "data_preprocessor", "sweep_orchestrator"}.issubset(self._supervisor.pids()) - ): - # Exit reports are consumed once. A missing child must keep triggering - # recovery after a back-off delay or a failed start with no reports left. - self._recover_pipeline_after_crash() - - def _recover_pipeline_after_crash(self) -> None: - """Relaunch the pipeline after an unexpected child exit (GUI and headless). - - Re-spawns from the already-written runtime config — no widgets, no dialogs, - no re-validation — so it is safe to call from the poll tick. Retries forever - with capped back-off (never gives up); a healthy poll resets the failure streak. - """ - now = time.monotonic() - consecutive_failures = getattr(self, "_pipeline_restart_count", 0) - if not self._RESTART_POLICY.should_restart_now( - now_s=now, - last_restart_s=getattr(self, "_last_pipeline_restart_s", 0.0), - consecutive_failures=consecutive_failures, - ): - return # still inside the current back-off window; let it settle - self._last_pipeline_restart_s = now - self._pipeline_restart_count = consecutive_failures + 1 - - config = getattr(self, "_active_run_config", None) - config_path = getattr(self, "_active_run_config_path", None) - if config is None or config_path is None: - self._pipeline_should_run = False - self._log_error("Cannot auto-restart pipeline: no active run configuration recorded.") - return - - self._log_error( - "Pipeline process exited unexpectedly; restarting " - f"(attempt {self._pipeline_restart_count}, next back-off " - f"{self._RESTART_POLICY.backoff_for(self._pipeline_restart_count):.0f}s)." - ) - try: - # Note: do NOT drain rings here — draining pumps the Qt event loop, which - # would re-enter _poll_rings mid-restart (and reset the crash-storm count). - self._supervisor.stop_all() - self._close_readers(keep_results=False) - self._supervisor.start(config_path, allow_clean_orchestrator_exit=False) - self._raw_reader = ShmRingReader(config.rings.raw_tap.name) - self._pre_reader = ShmRingReader(config.rings.preprocessed_tap.name) - self._result_reader = ShmRingReader(config.rings.results.name) - self._processor_run_signature = self._build_processor_run_signature(config) - self._drop_pending_ring_payloads(include_results=True) - self._status_label.setText("Status: running") - self._log("Pipeline auto-restarted after crash.") - except Exception as exc: # noqa: BLE001 - retry on subsequent polls - self._log_exception("Pipeline auto-restart failed; will retry", exc, level="ERROR") - - def _check_single_capture_deadline(self) -> None: - """Fail a single capture that never completes so it cannot hang forever.""" - start = self._single_capture_start_ns - if start is None: - return - if time.monotonic_ns() - start <= int(self._SINGLE_CAPTURE_TIMEOUT_S * 1e9): - return - self._log_error( - f"Single capture timed out after {self._SINGLE_CAPTURE_TIMEOUT_S:.0f}s " - "with no result; stopping." - ) - self._stop_run() - - def _handle_reader_poll_error(self, exc: Exception) -> None: - """Surface a reader-poll failure without spamming the log. - - Distinct errors log once; an identical recurring error is deduped from the - log but still drives the status label to error, is periodically re-logged, - and after escalating thresholds triggers a reader reconnect and finally a - pipeline stop so a wedged reader cannot fail silently forever. - """ - signature = (type(exc).__name__, str(exc)) - # Always reflect a reader failure in the status label, even when deduped. - self._status_label.setText("Status: error") - - if self._last_reader_error_signature == signature: - self._reader_error_repeat_count = getattr(self, "_reader_error_repeat_count", 0) + 1 - else: - self._last_reader_error_signature = signature - self._reader_error_repeat_count = 0 - self._log_exception("Reader poll failed", exc, level="ERROR") - - repeats = self._reader_error_repeat_count - # Periodically re-log a persistent identical failure so it stays visible. - if repeats and repeats % self._READER_ERROR_RELOG_EVERY == 0: - self._log_exception( - f"Reader poll still failing (repeat #{repeats})", exc, level="ERROR" - ) - - if repeats >= self._READER_ERROR_STOP_AT: - # The reader stayed wedged through a reconnect attempt; stop the - # pipeline so the failure is unmistakable instead of an endless retry. - self._log_error( - f"Stopping pipeline after {repeats} consecutive reader poll failures" - ) - self._reader_error_repeat_count = 0 - self._last_reader_error_signature = None - self._stop_all_processes() - elif repeats == self._READER_ERROR_RECONNECT_AT: - self._reconnect_readers_after_error() - - def _reconnect_readers_after_error(self) -> None: - """Re-open active ring readers in place to recover from a wedged reader.""" - self._log_warning("Attempting ring reader reconnect after repeated poll failures") - try: - for attr in ("_raw_reader", "_pre_reader", "_result_reader"): - reader = getattr(self, attr) - if reader is None: - continue - ring_name = reader._ring_name # noqa: SLF001 - reuse the reader's own ring name - reader.close() - setattr(self, attr, ShmRingReader(ring_name)) - # A successful reconnect clears the error state so the next failure - # logs fresh rather than being swallowed by the stale signature. - self._last_reader_error_signature = None - self._reader_error_repeat_count = 0 - self._status_label.setText("Status: running") - self._log("Ring readers reconnected after repeated poll failures") - except Exception as exc: # noqa: BLE001 - # Leave the error signature intact so escalation to a stop still fires. - self._log_exception("Ring reader reconnect failed", exc, level="ERROR") - - def _finish_single_capture_if_ready(self) -> bool: - """Finalize single capture when the exact target result becomes available.""" - if not self._single_capture_active: - return False - if self._single_capture_start_ns is None: - return False - if not self._single_capture_seen_raw: - return False - target_result = self._find_single_capture_target_result() - if target_result is None: - return False - - self._draw_results(target_result) - status = sar3d_status(target_result) if self._processing_mode.currentText() == "sar3d" else None - self._stop_run() - if status is not None and status[0] in (1, 2, 3): - state = {1: "no signal in selected range", 2: "error", 3: "warning"}[status[0]] - self._status_label.setText(f"Status: {state}") - log = self._log_error if status[0] == 2 else self._log_warning - log(f"Single capture: SAR3D {state}: {status[1]}") - else: - self._log("Single capture completed") - return True - - def _find_single_capture_target_result(self) -> ResultCollection | None: - """Return the exact result collection corresponding to the captured target raw sweep.""" - if self._single_capture_target_collection_id is None: - return None - if self._single_capture_start_ns is None: - return None - - for collection in reversed(self._result_history): - if collection.collection_id != self._single_capture_target_collection_id: - continue - if collection.monotonic_ns < self._single_capture_start_ns: - continue - if not self._result_collection_has_trace(collection): - continue - return collection - - return None - - def _read_all_raw(self, *, max_collections: int | None = None) -> SweepCollection | None: - """Read available raw collections from raw ring.""" - if self._raw_reader is None: - raise RuntimeError("Raw ring reader is not initialised") - latest: SweepCollection | None = None - for _ in range(self._max_pop_per_poll if max_collections is None else max_collections): - collection = self._raw_reader.pop_raw_collection() - if collection is None: - break - self._raw_history.append(collection) - self._record_collection("raw", collection) - latest = collection - # `capture_*_ns` are populated by the C++ sweep_orchestrator with - # wallclocks captured around the actual device read. Pre-orchestrator - # producers leave them zero, in which case PipelineMetrics drops it. - acquisition_ns = int(collection.capture_end_ns) - int(collection.capture_start_ns) - self._pipeline_metrics.record("acquisition", acquisition_ns) - if self._single_capture_active and self._single_capture_start_ns is not None: - if collection.monotonic_ns >= self._single_capture_start_ns: - self._single_capture_seen_raw = True - if self._single_capture_target_collection_id is None: - self._single_capture_target_collection_id = collection.collection_id - return latest - - def _read_all_preprocessed(self, *, max_collections: int | None = None) -> SweepCollection | None: - """Read available preprocessed collections from preprocessed ring.""" - if self._pre_reader is None: - return - - latest: SweepCollection | None = None - for _ in range(self._max_pop_per_poll if max_collections is None else max_collections): - collection = self._pre_reader.pop_preprocessed_collection() - if collection is None: - break - self._pre_history.append(collection) - self._record_collection("preprocessed", collection) - latest = collection - return latest - - def _read_all_results(self, *, max_collections: int | None = None) -> ResultCollection | None: - """Read available result collections from results ring.""" - if self._result_reader is None: - raise RuntimeError("Result ring reader is not initialised") - latest: ResultCollection | None = None - for _ in range(self._max_pop_per_poll if max_collections is None else max_collections): - collection = self._result_reader.pop_result_collection() - if collection is None: - break - self._pipeline_metrics.record("processing", int(collection.processing_duration_ns)) - # Only a new sweep is a new measurement. Any other result is the processor - # reprocessing retained sweeps: recording it would duplicate a measurement - # or put one out of order. - if record_result_history(self._result_history, collection): - self._record_collection("results", collection) - latest = collection - return latest - - def _pump_events_during_drain(self, pause_s: float) -> None: - """Yield to the Qt event loop for `pause_s` instead of blocking on time.sleep. - - The bounded drain loops run on the GUI thread; a raw time.sleep here freezes - the event loop, stalling the keepalive/headless-watchdog timers and starving - queued signals. Pumping events keeps the daemon responsive while we wait. - """ - app = QCoreApplication.instance() - if app is None: - # No event loop (e.g. unit context); fall back to a plain short sleep. - time.sleep(pause_s) - return - deadline = time.monotonic() + pause_s - while True: - remaining_ms = int((deadline - time.monotonic()) * 1000) - if remaining_ms <= 0: - break - app.processEvents(QEventLoop.ProcessEventsFlag.AllEvents, remaining_ms) - # processEvents returns immediately when the queue empties; sleep the - # residual in tiny slices so we neither busy-spin nor block too long. - time.sleep(min(0.002, max(0.0, deadline - time.monotonic()))) - - def _drain_rings_once_for_history(self) -> bool: - """Perform one non-blocking read pass to extend histories.""" - raw = self._read_all_raw() if self._raw_reader is not None else None - preprocessed = self._read_all_preprocessed() - result = self._read_all_results() if self._result_reader is not None else None - return raw is not None or preprocessed is not None or result is not None - - def _drain_rings_until_quiet(self, *, timeout_s: float, poll_s: float) -> None: - """Drain rings until readers are quiet or timeout expires, even at full history capacity.""" - deadline = time.monotonic() + timeout_s - stable_rounds = 0 - while time.monotonic() < deadline and stable_rounds < 2: - stable_rounds = 0 if self._drain_rings_once_for_history() else stable_rounds + 1 - # Event-loop-friendly wait so timers/signals keep firing during drain. - self._pump_events_during_drain(poll_s) - - def _drain_results_until_quiet(self, *, timeout_s: float, poll_s: float) -> ResultCollection | None: - """Drain results until at least one result arrives and the ring becomes quiet.""" - if self._result_reader is None: - return None - - deadline = time.monotonic() + timeout_s - stable_rounds = 0 - latest_seen: ResultCollection | None = None - - while time.monotonic() < deadline and (latest_seen is None or stable_rounds < 2): - latest = self._read_all_results() - if latest is None: - if latest_seen is not None: - stable_rounds += 1 - else: - latest_seen = latest - stable_rounds = 0 - # Event-loop-friendly wait so timers/signals keep firing during drain. - self._pump_events_during_drain(poll_s) - return latest_seen - - def _update_history_indicator(self) -> None: - """Update UI label with current history buffer sizes.""" - self._history_label.setText( - f"History: raw={len(self._raw_history)}, " - f"preprocessed={len(self._pre_history)}, " - f"results={len(self._result_history)}" - ) - - def _reset_runtime_history(self) -> None: - """Reset runtime history and B-scan caches.""" - self._replace_runtime_history(retained_raw=[], retained_pre=[], retained_result=[]) - self._discard_bscan_replay_results() - self._clear_history_mode_caches() - self._update_history_indicator() - - def _replace_runtime_history( - self, - *, - retained_raw: list[SweepCollection], - retained_pre: list[SweepCollection], - retained_result: list[ResultCollection], - ) -> None: - """Replace history deques with provided retained tails.""" - raw_tail = retained_raw[-self._raw_history.maxlen :] if self._raw_history.maxlen is not None else retained_raw - pre_tail = retained_pre[-self._pre_history.maxlen :] if self._pre_history.maxlen is not None else retained_pre - result_tail = ( - retained_result[-self._result_history.maxlen :] - if self._result_history.maxlen is not None - else retained_result - ) - - self._raw_history.clear() - self._pre_history.clear() - self._result_history.clear() - self._raw_history.extend(raw_tail) - self._pre_history.extend(pre_tail) - self._result_history.extend(result_tail) - - def _build_run_history_signature(self, config: RunConfigModel) -> tuple[object, ...]: - """Build signature used to decide when display history should be reset.""" - return build_run_history_signature(config) - - def _build_processor_run_signature(self, config: RunConfigModel) -> tuple[object, ...]: - """Build signature used to decide when the data_processor must be restarted.""" - return build_processor_run_signature(config) - - def _validate_processing_mode_constraints(self, config: RunConfigModel) -> None: - """Validate processing-mode constraints for run start.""" - validate_processing_mode_constraints( - self._processing_mode.currentText(), - config, - self._live_processing_config(), - ) diff --git a/python_app/gui/controllers/app_window_plot/__init__.py b/python_app/gui/controllers/app_window_plot/__init__.py deleted file mode 100644 index af2fa86..0000000 --- a/python_app/gui/controllers/app_window_plot/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Plot-rendering mixins split by rendering mode.""" - -from python_app.gui.controllers.app_window_plot.bscan_plot_mixin import AppWindowBscanPlotMixin -from python_app.gui.controllers.app_window_plot.bscan_replay_mixin import AppWindowBscanReplayMixin -from python_app.gui.controllers.app_window_plot.gpr_plot_mixin import AppWindowGprPlotMixin -from python_app.gui.controllers.app_window_plot.sar3d_plot_mixin import AppWindowSar3dPlotMixin -from python_app.gui.controllers.app_window_plot.trace_plot_mixin import AppWindowTracePlotMixin - -__all__ = [ - "AppWindowBscanPlotMixin", - "AppWindowBscanReplayMixin", - "AppWindowGprPlotMixin", - "AppWindowSar3dPlotMixin", - "AppWindowTracePlotMixin", -] diff --git a/python_app/gui/controllers/app_window_plot/bscan_plot_mixin.py b/python_app/gui/controllers/app_window_plot/bscan_plot_mixin.py deleted file mode 100644 index 77cfc05..0000000 --- a/python_app/gui/controllers/app_window_plot/bscan_plot_mixin.py +++ /dev/null @@ -1,470 +0,0 @@ -"""B-scan rendering and cache helpers.""" - -from __future__ import annotations - -from collections import deque - -from PyQt6.QtCore import QRectF -import numpy as np -import pyqtgraph as pg - -from python_app.models.dataset_model import ResultCollection -from python_app.orchestration.live_processing_config import ProcessingLiveConfig - - -def _result_tail( - *, - result_history: list[ResultCollection], - history_limit: int, -) -> list[ResultCollection]: - """Return the de-duplicated newest `history_limit` entries for B-scan usage. - - Selection is purely positional. An earlier version also dropped entries below a - `collection_id` floor, which cannot work here: ids are neither dense (the results - ring overwrites unread slots) nor monotonic across a run boundary (the C++ side - numbers from 1 again). Given the floor was derived from the first entry of this - very slice, the comparison provably removed nothing when ids ascend, and removed - exactly the newest frames when they do not. - """ - unique_reversed_tail: list[ResultCollection] = [] - seen_keys: set[tuple[int, int]] = set() - for collection in reversed(result_history[-history_limit:]): - key = (int(collection.collection_id), int(collection.monotonic_ns)) - if key in seen_keys: - continue - seen_keys.add(key) - unique_reversed_tail.append(collection) - - unique_reversed_tail.reverse() - return unique_reversed_tail - - -def build_bscan_signature( - live_config: ProcessingLiveConfig, - *, - subtract_mean_ascan_enabled: bool, - result_history: list[ResultCollection], - history_limit: int, -) -> tuple[object, ...]: - """Build deterministic signature used to detect B-scan cache invalidation.""" - result_tail = _result_tail( - result_history=result_history, - history_limit=history_limit, - ) - return ( - int(history_limit), - str(live_config.bscan_axis), - str(live_config.bscan_channel), - float(live_config.bscan_cut_m), - float(live_config.bscan_max_depth_m), - float(live_config.bscan_gain), - float(live_config.bscan_start_freq_mhz), - float(live_config.bscan_stop_freq_mhz), - bool(subtract_mean_ascan_enabled), - tuple((int(collection.collection_id), int(collection.monotonic_ns), len(collection.blocks)) for collection in result_tail), - ) - - -def apply_mean_ascan_subtraction( - history: deque[np.ndarray], - *, - enabled: bool, -) -> np.ndarray: - """Subtract mean A-scan across sweeps when enabled.""" - sweeps = np.vstack(history).astype(np.float32, copy=False) - if not enabled: - return sweeps - mean_trace = np.mean(sweeps, axis=0, dtype=np.float32) - return sweeps - mean_trace - - -def rebuild_bscan_history_from_results( - result_history: list[ResultCollection], - history_limit: int, - stats: dict[str, object] | None = None, -) -> tuple[dict[tuple[int, int], deque[np.ndarray]], dict[tuple[int, int], np.ndarray]]: - """Rebuild B-scan history and depth axes from processed result payloads. - - Pass `stats` to receive a breakdown of why the rebuilt image may hold fewer - columns than `history_limit`. The three causes are independent and only - distinguishable here: too little history, frames carrying no `bscan` payload, and - depth-axis changes that reset the accumulated deque (see below). - """ - history_by_combo: dict[tuple[int, int], deque[np.ndarray]] = {} - depth_axis_by_combo: dict[tuple[int, int], np.ndarray] = {} - - result_tail = _result_tail( - result_history=result_history, - history_limit=history_limit, - ) - axis_resets = 0 - without_bscan = 0 - - for collection in result_tail: - carried_bscan = False - for block in collection.blocks: - key = (block.combo.input, block.combo.output) - for payload in block.payloads: - if payload.kind != 1 or payload.processing_name != "bscan": - continue - if payload.frequency_hz.size == 0 or payload.trace.size == 0: - continue - if payload.frequency_hz.size != payload.trace.size: - continue - - depth_axis = np.asarray(payload.frequency_hz, dtype=np.float32) - amplitudes = np.asarray(np.real(payload.trace), dtype=np.float32) - if depth_axis.size == 0 or amplitudes.size == 0: - continue - - carried_bscan = True - history = history_by_combo.get(key) - stored_axis = depth_axis_by_combo.get(key) - if ( - history is None - or stored_axis is None - or stored_axis.shape != depth_axis.shape - or not np.allclose(stored_axis, depth_axis, rtol=1e-4, atol=1e-6) - ): - # A changed depth axis makes previously accumulated columns - # un-stackable, so the deque restarts and everything gathered so far - # for this combo is dropped. Frames computed with different - # bscan_max_depth_m / frequency bounds land here — which is exactly - # what a partially replayed history looks like. - if history is not None: - axis_resets += 1 - history = deque(maxlen=history_limit) - history_by_combo[key] = history - depth_axis_by_combo[key] = depth_axis.copy() - - history.append(amplitudes.copy()) - if not carried_bscan: - without_bscan += 1 - - if stats is not None: - stats["tail"] = len(result_tail) - stats["without_bscan"] = without_bscan - stats["axis_resets"] = axis_resets - - return history_by_combo, depth_axis_by_combo - - -def pick_bscan_display_key( - history_by_combo: dict[tuple[int, int], deque[np.ndarray]], - preferred_key: tuple[int, int] | None = None, -) -> tuple[int, int] | None: - """Choose combo key to display when multiple histories are present.""" - if not history_by_combo: - return None - if preferred_key is not None and preferred_key in history_by_combo: - return preferred_key - return next(iter(history_by_combo.keys())) - - -def bscan_lookup_table(axis_mode: str) -> np.ndarray: - """Build B-scan colormap table for selected axis mode.""" - if axis_mode == "abs": - return build_lut(["#440154", "#31688e", "#35b779", "#fde725"]) - return build_lut(["#2166ac", "#67a9cf", "#f7f7f7", "#ef8a62", "#b2182b"]) - - -def build_lut(stops: list[str], *, size: int = 256) -> np.ndarray: - """Interpolate hex color stops into 8-bit RGB LUT array.""" - stop_positions = np.linspace(0.0, 1.0, num=len(stops), dtype=np.float32) - sample_positions = np.linspace(0.0, 1.0, num=size, dtype=np.float32) - stop_colors = np.asarray([pg.mkColor(value).getRgb()[:3] for value in stops], dtype=np.float32) - - lut = np.empty((size, 3), dtype=np.uint8) - for channel in range(3): - lut[:, channel] = np.interp(sample_positions, stop_positions, stop_colors[:, channel]).astype(np.uint8) - return lut - - -def bscan_levels(sweeps: np.ndarray, axis_mode: str) -> tuple[float, float]: - """Compute image levels for B-scan data based on axis mode.""" - min_value = float(np.min(sweeps)) - max_value = float(np.max(sweeps)) - if axis_mode == "abs": - if max_value <= min_value: - return min_value, min_value + 1e-6 - return min_value, max_value - - max_abs = max(abs(min_value), abs(max_value), 1e-6) - return -max_abs, max_abs - - -class AppWindowBscanPlotMixin: - """Renders B-scan heatmaps and maintains B-scan history caches.""" - - def _draw_bscan_heatmap(self, _collection: ResultCollection) -> bool: - """Draw B-scan image rebuilt from processed result history.""" - self._trace_phase_plot.clear() - self._sync_bscan_history_from_results() - return self._draw_bscan_heatmap_from_history() - - def _on_bscan_display_combo_changed(self) -> None: - """Show the selected combo's B-scan; the histories of all combos are already cached.""" - if self._processing_mode.currentText() != "bscan": - return - try: - self._sync_bscan_history_from_results() - self._draw_bscan_heatmap_from_history() - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to redraw the B-scan", exc) - - def _configure_bscan_plot_axes(self) -> None: - """Apply persistent B-scan plot axis labels and base view settings.""" - plot = self._bscan_plot - plot_item = plot.getPlotItem() - plot_item.showAxis("left", show=True) - plot_item.showAxis("bottom", show=True) - plot.setLabel("bottom", "Sweep #") - plot.setLabel("left", "Range", units="m") - view_box = plot.getViewBox() - view_box.invertY(False) - view_box.enableAutoRange(x=False, y=False) - - def _draw_bscan_heatmap_from_history(self) -> bool: - """Render B-scan heatmap from currently cached history arrays.""" - display_key = self._pick_bscan_display_key() - if display_key is None: - return False - - history = self._bscan_history_by_combo.get(display_key) - depth_axis = self._bscan_depth_axis_by_combo.get(display_key) - if not history or depth_axis is None: - return False - - sweeps = apply_mean_ascan_subtraction( - history, - enabled=bool(self._bscan_subtract_mean_ascan.isChecked()), - ) - if sweeps.size == 0: - return False - - depth_min = float(np.min(depth_axis)) - depth_max = float(np.max(depth_axis)) - depth_span = max(depth_max - depth_min, 1e-6) - sweep_count = sweeps.shape[0] - self._warn_if_bscan_window_underfilled() - sweep_width = float(max(sweep_count, 1)) - x_min = 0.5 - x_max = x_min + sweep_width - - image_item = pg.ImageItem(axisOrder="row-major") - image_item.setImage(sweeps.T, autoLevels=False) - image_item.setRect(QRectF(x_min, depth_min, sweep_width, depth_span)) - - axis_mode = self._bscan_axis.currentText() - image_item.setLookupTable(bscan_lookup_table(axis_mode)) - image_item.setLevels(bscan_levels(sweeps, axis_mode)) - - self._bscan_plot.clear() - self._configure_bscan_plot_axes() - self._bscan_plot.addItem(image_item) - self._bscan_plot.setXRange(x_min, x_max, padding=0.02) - self._bscan_plot.setYRange(depth_min, depth_max, padding=0.02) - bscan_channel = "S21" - self._bscan_plot.setTitle( - f"B-scan {bscan_channel} in{display_key[0]}/out{display_key[1]} | sweeps={sweep_count}" - ) - return True - - def _warn_if_bscan_window_underfilled(self) -> None: - """Warn once when there is less retained raw material than the operator asked for. - - Settings edits are recomputed across the whole image by re-feeding - `_pre_history` to the processor (see :class:`AppWindowBscanReplayMixin`), so a - wide window is no longer a coherency problem. What it can still be is an empty - promise: asking for more sweeps than were ever captured simply shows fewer. - """ - window = self._bscan_display_window_scans() - retained = len(self._pre_history) - if retained >= window: - return - - self._log_warning( - f"B-scan is set to show {window} sweeps but only {retained} are retained; " - "the image shows what history there is.", - details=( - "The GUI keeps a bounded history of preprocessed sweeps, so a window " - "wider than the run itself cannot be filled.\n" - f"Capture more sweeps, or set 'Scans to show (stopped)' to {retained} or less." - ), - # Keyed on the operator-controlled window rather than the live retained count, - # so that repeated "Remove Last" does not re-warn on every click. - once_key=f"bscan_window_underfilled_{window}", - ) - - def _bscan_display_window_scans(self) -> int: - """Return how many past sweeps the B-scan should render right now. - - While acquisition runs the window stays at the C++ replay window: results - arrive continuously, the whole history is rebuilt on every new one, and a - 1000-wide rebuild on the live path would cost ~20x per frame. - - Once stopped, the operator reviews a frozen history, so the user-configured - window applies and may reach back over the whole GUI result deque. - """ - if self._supervisor.is_running(): - return int(self._bscan_cpp_replay_window) - return max(1, int(self._bscan_history_window.value())) - - def _sync_bscan_history_from_results(self) -> None: - """Rebuild B-scan history cache when live params or inputs changed.""" - signature = self._bscan_signature() - if signature == self._bscan_render_signature: - return - self._rebuild_bscan_history_from_results() - self._bscan_render_signature = signature - - def _bscan_signature(self) -> tuple[object, ...]: - """Build state signature for B-scan history cache invalidation.""" - live_config = self._live_processing_config() - # Must read the same source the rebuild will, or the cache decides nothing - # changed while the image would in fact be built from different collections. - result_history, _from_replay = self._bscan_source_collections() - return build_bscan_signature( - live_config=live_config, - subtract_mean_ascan_enabled=bool(self._bscan_subtract_mean_ascan.isChecked()), - result_history=result_history, - history_limit=self._bscan_display_window_scans(), - ) - - def _bscan_source_collections(self) -> tuple[list[ResultCollection], bool]: - """Return the collections the image is built from, and whether they are replayed. - - A completed replay is the better source: it is exactly `window` long and every - entry went through the processor with the same settings. The runtime history is - not: it can also hold results of sweeps the GUI never received preprocessed, - which were not re-fed and keep their old settings. - """ - replayed = getattr(self, "_bscan_replay_results", None) - if replayed: - return list(replayed), True - return list(self._result_history), False - - def _rebuild_bscan_history_from_results(self) -> None: - """Recompute B-scan history cache from results history buffer.""" - result_history, from_replay = self._bscan_source_collections() - window = self._bscan_display_window_scans() - stats: dict[str, object] = {} - history_by_combo, depth_axis_by_combo = rebuild_bscan_history_from_results( - result_history=result_history, - history_limit=window, - stats=stats, - ) - self._bscan_history_by_combo = history_by_combo - self._bscan_depth_axis_by_combo = depth_axis_by_combo - self._log_bscan_window_shortfall( - window=window, - result_history_len=len(result_history), - from_replay=from_replay, - history_by_combo=history_by_combo, - stats=stats, - ) - - def _log_bscan_window_shortfall( - self, - *, - window: int, - result_history_len: int, - from_replay: bool, - history_by_combo: dict[tuple[int, int], deque[np.ndarray]], - stats: dict[str, object], - ) -> None: - """Explain at DEBUG why the image holds fewer columns than were requested. - - Three independent causes produce the same symptom, so each is reported as its - own number rather than a single verdict: - * `tail` — the run simply produced fewer sweeps than were asked for; - * `without_bscan` — frames processed in another mode carry no bscan payload; - * `axis_resets` — a changed depth axis restarted the deque, dropping every - column gathered before it (the usual cause after a partial replay); - * per-combo counts — the image renders one combo at a time. - """ - rendered = max((len(history) for history in history_by_combo.values()), default=0) - if rendered >= window: - return - - per_combo = ", ".join( - f"in{input_pos}/out{output_pos}={len(history)}" - for (input_pos, output_pos), history in sorted(history_by_combo.items()) - ) - self._log_debug( - f"B-scan window not filled: rendered={rendered} of requested={window}. " - f"source={'replay' if from_replay else 'result history'} len={result_history_len}, " - f"newest-{window} tail={stats.get('tail')}, " - f"of those without a bscan payload={stats.get('without_bscan')}, " - f"depth-axis resets={stats.get('axis_resets')}. " - f"Per combo: {per_combo or 'none'}." - ) - - def _pick_bscan_display_key(self) -> tuple[int, int] | None: - """Choose combo history key to render.""" - requested_key = self._requested_bscan_display_key() - display_key = pick_bscan_display_key( - self._bscan_history_by_combo, - preferred_key=requested_key, - ) - available_keys = sorted(self._bscan_history_by_combo.keys()) - if requested_key is not None and requested_key != display_key and available_keys: - combo_signature = ",".join(f"{input_pos}:{output_pos}" for input_pos, output_pos in available_keys) - details = "\n".join( - f"- in{input_pos}/out{output_pos}" - for input_pos, output_pos in available_keys - ) - self._log_warning( - f"B-scan display combo in{requested_key[0]}/out{requested_key[1]} is not available; " - f"rendering in{display_key[0]}/out{display_key[1]} instead.", - details=details, - once_key=( - f"bscan_requested_combo_missing_{requested_key[0]}_{requested_key[1]}_" - f"{combo_signature}" - ), - ) - return display_key - if display_key is not None and len(available_keys) > 1: - if requested_key == display_key: - return display_key - combo_signature = ",".join(f"{input_pos}:{output_pos}" for input_pos, output_pos in available_keys) - details = "\n".join( - f"- in{input_pos}/out{output_pos}" - for input_pos, output_pos in available_keys - ) - self._log( - f"B-scan auto-selected combo in{display_key[0]}/out{display_key[1]} because multiple combos are available.", - once_key=f"bscan_auto_display_{combo_signature}", - ) - self._log_warning( - "B-scan has multiple combo histories but the UI currently renders only one at a time.", - details=details, - once_key=f"bscan_multi_combo_warning_{combo_signature}", - ) - return display_key - - def _requested_bscan_display_key(self) -> tuple[int, int] | None: - """Return the matrix-radar B-scan display combo requested by the GUI.""" - if not ( - hasattr(self, "_is_matrix_radar_model_selected") - and self._is_matrix_radar_model_selected() - and hasattr(self, "_single_combo_input") - and hasattr(self, "_single_combo_output") - ): - return None - - input_text = self._single_combo_input.text().strip() - output_text = self._single_combo_output.text().strip() - if not input_text or not output_text: - return None - - try: - return int(input_text), int(output_text) - except ValueError: - return None - - def _clear_bscan_plot_history(self) -> None: - """Drop cached B-scan history and invalidate cache signature.""" - self._bscan_history_by_combo.clear() - self._bscan_depth_axis_by_combo.clear() - self._bscan_render_signature = None diff --git a/python_app/gui/controllers/app_window_plot/bscan_replay_mixin.py b/python_app/gui/controllers/app_window_plot/bscan_replay_mixin.py deleted file mode 100644 index eb21ab0..0000000 --- a/python_app/gui/controllers/app_window_plot/bscan_replay_mixin.py +++ /dev/null @@ -1,243 +0,0 @@ -"""Re-feed retained sweeps to the processor so it recomputes the whole B-scan. - -The processor keeps only ~50 preprocessed sweeps of its own (`kBscanReplayWindow` in -`data_processor.cpp`), so changing a live B-scan setting used to refresh just the newest -50 columns while everything older kept the parameters it was captured with — one image -stitched from two parameter sets. - -The GUI already holds up to 1000 preprocessed sweeps in `_pre_history`, byte-identical -to what the processor consumes: `data_preprocessor` pushes the very same serialized -buffer into both the working ring and the tap the GUI reads. So instead of making the -processor hoard sweeps, we hand its own raw material back to it and let the ordinary -`pop -> process -> publish` path recompute every column with the current settings. -The re-fed sweeps keep their timestamps, by which the processor tells them from new -ones and keeps them out of its own history. -""" - -from __future__ import annotations - -import time - -from python_app.gui.runtime.history import collection_key, record_result_history -from python_app.models.dataset_model import ResultCollection -from python_app.orchestration.shm import ShmRingWriter -from python_app.storage.npz.serialize import PREPROC_MAGIC, serialize_trace_collection - -# The results ring overwrites unread slots, so a burst must never exceed what the GUI -# drains between chunks. Comfortably below the (typically 50-slot) ring capacity. -_REPLAY_CHUNK_SIZE = 12 - -# Per-chunk budget. Generous: the processor may be busy, and overshooting only costs -# a redraw that reflects fewer columns. -_REPLAY_CHUNK_TIMEOUT_S = 2.0 - -# Restarting the timer on every edit collapses a spin-box drag into one replay. -_BSCAN_REPROCESS_DEBOUNCE_MS = 300 - - -class AppWindowBscanReplayMixin: - """Recomputes the full visible B-scan by re-feeding sweeps to the processor.""" - - def _reprocess_history_through_processor(self) -> bool: - """Re-feed the visible tail of `_pre_history` so every column is recomputed. - - Returns True when a replay actually ran. Refuses (returning False) whenever the - preconditions for safely writing into the processor's input ring do not hold. - """ - # `_pump_events_during_drain` runs the event loop, so a settings edit made while - # a long replay is in flight can re-arm the debounce and fire this method inside - # itself — two nested bursts sharing one ring and one result count. - if getattr(self, "_bscan_replay_active", False): - return False - if not self._can_reprocess_history(): - return False - - window = self._bscan_display_window_scans() - tail = list(self._pre_history)[-window:] - keys = [collection_key(collection) for collection in tail] - # `_pre_history` and `_result_history` are fed by two different rings, each - # dropping independently when the producer outruns the GUI. A low overlap means - # many replayed results are of sweeps the result history never held. - result_keys = {collection_key(collection) for collection in self._result_history} - overlap = sum(1 for key in keys if key in result_keys) - self._log_debug( - f"B-scan replay starting: window={window}, preprocessed history=" - f"{len(self._pre_history)}, result history={len(self._result_history)}, " - f"to re-send={len(tail)}, of those already in result history={overlap}." - ) - if not tail: - return False - - writer = self._ensure_replay_ring_writer() - if writer is None: - return False - - # Our own event pumping would otherwise let the poll timer fire and consume the - # replayed results through `_read_all_results`, which records them to disk and - # feeds the pipeline metrics. Restart in `finally`: losing the ring poll on an - # exception would leave the GUI permanently blind. - self._timer.stop() - self._bscan_replay_active = True - sent = 0 - replayed_keys = set(keys) - replayed: dict[tuple[int, int], ResultCollection] = {} - try: - for start in range(0, len(tail), _REPLAY_CHUNK_SIZE): - chunk = tail[start : start + _REPLAY_CHUNK_SIZE] - pushed = 0 - for collection in chunk: - payload = serialize_trace_collection(collection, PREPROC_MAGIC) - if not writer.push(payload): - # Only fails when the payload exceeds the slot size, which is a - # config problem rather than a transient one: stop the burst. - self._log_warning( - "B-scan replay stopped: a preprocessed sweep does not fit the ring slot.", - details=( - f"payload={len(payload)} bytes, " - f"slot={writer.slot_size_bytes} bytes" - ), - once_key="bscan_replay_payload_too_large", - ) - break - pushed += 1 - sent += pushed - if not self._collect_replayed_results( - keys[start : start + pushed], replayed_keys, replayed, timeout_s=_REPLAY_CHUNK_TIMEOUT_S - ): - # The processor did not answer in time; keep going, but say which - # chunk so a systematic stall is visible. - self._log_debug(f"B-scan replay chunk at offset {start} came back incomplete.") - if pushed != len(chunk): - break - finally: - self._bscan_replay_active = False - self._timer.start() - - # Render exactly the re-fed sweeps, in acquisition order, rather than the newest - # `window` of `_result_history`: the two histories are fed by rings that drop - # independently, so the latter can also hold results of sweeps the GUI never - # received preprocessed, which were not re-fed and keep their old settings. - self._bscan_replay_results = [replayed[key] for key in keys if key in replayed] - received = sum(1 for key in keys[:sent] if key in replayed) - - # Replayed collections keep their original ids and timestamps, so the render - # signature is unchanged even though the payload values are not. Drop it or the - # cache would decide nothing needs rebuilding. - self._bscan_render_signature = None - self._sync_bscan_history_from_results() - self._draw_bscan_heatmap_from_history() - - if received < sent: - self._log_warning( - f"B-scan replay recovered {received} of {sent} re-sent sweeps; " - "some columns may still show their captured settings.", - once_key=f"bscan_replay_incomplete_{sent}_{received}", - ) - self._log_debug(f"B-scan replay finished: sent={sent}, recovered={received}.") - return True - - def _can_reprocess_history(self) -> bool: - """Return whether re-feeding the processor's input ring is safe right now.""" - if self._processing_mode.currentText() != "bscan": - return False - # `data_preprocessor` owns the preprocessed ring while acquisition runs; writing - # into it concurrently would corrupt the sequence counters. - if self._supervisor.is_running(): - return False - if not self._supervisor.is_processor_running(): - return False - return self._result_reader is not None - - def _collect_replayed_results( - self, - waiting_for: list[tuple[int, int]], - replayed_keys: set[tuple[int, int]], - into: dict[tuple[int, int], ResultCollection], - *, - timeout_s: float, - ) -> bool: - """Pop results until those of the sweeps `waiting_for` are back; return whether they are. - - Every result goes into the runtime history, deliberately not through - `_read_all_results`: that path also feeds the pipeline metrics and the disk - recorder, and a replay is neither new acquisition nor something that should be - written to disk a second time. Results of re-fed sweeps (`replayed_keys`) also - go into `into` by sweep, the latest winning, so whatever else the ring delivers - meanwhile, and in whichever order, cannot get into the image or reorder it. - """ - if self._result_reader is None: - return False - - deadline = time.monotonic() + timeout_s - while any(key not in into for key in waiting_for) and time.monotonic() < deadline: - collection = self._result_reader.pop_result_collection() - if collection is None: - self._pump_events_during_drain(0.005) - continue - record_result_history(self._result_history, collection) - key = collection_key(collection) - if key in replayed_keys: - into[key] = collection - return all(key in into for key in waiting_for) - - def _ensure_replay_ring_writer(self) -> ShmRingWriter | None: - """Return a writer attached to the processor's input ring, creating it lazily. - - Geometry comes from `_active_run_config` — the config the C++ side actually - started with — never from the editable `_defaults_config`: `ShmRingWriter` owns - the rings it opens and *recreates* a segment whose geometry disagrees, which - would destroy the ring under a live processor. - """ - existing = getattr(self, "_replay_ring_writer", None) - if existing is not None: - return existing - - config = getattr(self, "_active_run_config", None) - if config is None: - # Processor outlived the GUI that started it: its ring geometry is unknown, - # and guessing risks recreating the segment underneath it. - self._log_warning( - "Cannot recompute the full B-scan: this GUI session did not start the " - "pipeline, so the processor's ring geometry is unknown.", - once_key="bscan_replay_no_active_run_config", - ) - return None - - ring = config.rings.preprocessed - try: - self._replay_ring_writer = ShmRingWriter( - ring.name, int(ring.capacity), int(ring.slot_size_bytes) - ) - except Exception as exc: # noqa: BLE001 - self._log_exception("Failed to open the B-scan replay ring writer", exc, level="WARN") - self._replay_ring_writer = None - return self._replay_ring_writer - - def _discard_bscan_replay_results(self) -> None: - """Fall back to the runtime history as the render source. - - Called whenever the replayed set stops describing what should be on screen: - acquisition resuming (live frames must win) or the history being edited. - """ - if not getattr(self, "_bscan_replay_results", None): - return - self._bscan_replay_results = [] - self._bscan_render_signature = None - - def _close_replay_ring_writer(self) -> None: - """Detach from the processor's input ring (safe to call repeatedly).""" - writer = getattr(self, "_replay_ring_writer", None) - if writer is None: - return - try: - writer.close() - except Exception as exc: # noqa: BLE001 - self._log_exception("Failed to close the B-scan replay ring writer", exc, level="WARN") - finally: - self._replay_ring_writer = None - - def _schedule_bscan_history_reprocess(self) -> None: - """Debounce a full recompute so dragging a spin box does not send hundreds of sweeps.""" - if not self._can_reprocess_history(): - return - self._bscan_reprocess_timer.start(_BSCAN_REPROCESS_DEBOUNCE_MS) diff --git a/python_app/gui/controllers/app_window_plot/gpr_plot_mixin.py b/python_app/gui/controllers/app_window_plot/gpr_plot_mixin.py deleted file mode 100644 index e59c5ba..0000000 --- a/python_app/gui/controllers/app_window_plot/gpr_plot_mixin.py +++ /dev/null @@ -1,576 +0,0 @@ -"""GPR heatmap and object-only rendering helpers.""" - -from __future__ import annotations - -from PyQt6.QtCore import QRectF -import numpy as np -import pyqtgraph as pg - -from python_app.gui.controllers.app_window_plot.bscan_plot_mixin import build_lut -from python_app.models.dataset_model import ResultCollection -from python_app.orchestration.gpr_locator import ( - collection_payload_by_name as gpr_collection_payload_by_name, - collection_payloads_by_prefix as gpr_collection_payloads_by_prefix, - filter_object_rows as gpr_filter_object_rows, - gpr_object_rows as extract_gpr_object_rows, -) - - -class AppWindowGprPlotMixin: - """Renders GPR accumulator heatmaps and detected object overlays.""" - - def _gpr_plot_title(self, *, objects_only: bool = False) -> str: - """Return title for the active GPR-like processing mode.""" - if self._processing_mode.currentText() == "legacy_gpr": - title = f"Legacy GPR {self._legacy_gpr_config_mode.currentText()}" - else: - title = "GPR coherent BP" - return f"{title} Objects Only" if objects_only else title - - def _clear_gpr_plot(self) -> None: - """Clear latest GPR plot surface.""" - if not hasattr(self, "_gpr_plot"): - return - self._configure_gpr_plot_axes() - self._clear_gpr_point_labels() - self._clear_gpr_region_labels() - self._clear_gpr_region_masks() - if self._gpr_image_item is not None: - self._gpr_image_item.hide() - if self._gpr_tx_item is not None: - self._gpr_tx_item.setData(x=[], y=[]) - self._gpr_tx_item.hide() - if self._gpr_rx_item is not None: - self._gpr_rx_item.setData(x=[], y=[]) - self._gpr_rx_item.hide() - if self._gpr_points_item is not None: - self._gpr_points_item.setData(x=[], y=[]) - self._gpr_points_item.hide() - if self._gpr_region_centers_item is not None: - self._gpr_region_centers_item.setData(x=[], y=[]) - self._gpr_region_centers_item.hide() - self._gpr_plot.setTitle(self._gpr_plot_title()) - - def _configure_gpr_plot_axes(self) -> None: - """Apply persistent GPR plot axis labels and base view settings.""" - plot = self._gpr_plot - plot_item = plot.getPlotItem() - plot_item.showAxis("left", show=True) - plot_item.showAxis("bottom", show=True) - plot_item.setClipToView(True) - plot.setLabel("bottom", "X", units="m") - plot.setLabel("left", "Range", units="m") - view_box = plot.getViewBox() - view_box.invertY(False) - view_box.enableAutoRange(x=False, y=False) - self._disable_gpr_plot_interaction(plot, view_box) - - @staticmethod - def _disable_gpr_plot_interaction(plot: pg.PlotWidget, view_box: pg.ViewBox) -> None: - """Disable mouse-driven GPR pan/zoom; ranges are controlled by widgets.""" - AppWindowGprPlotMixin._call_if_present(plot, "setMouseEnabled", x=False, y=False) - AppWindowGprPlotMixin._call_if_present(plot, "setMenuEnabled", False) - AppWindowGprPlotMixin._call_if_present(plot.getPlotItem(), "hideButtons") - AppWindowGprPlotMixin._call_if_present(view_box, "setMouseEnabled", x=False, y=False) - AppWindowGprPlotMixin._call_if_present(view_box, "setMenuEnabled", False) - for axis_name in ("bottom", "left"): - AppWindowGprPlotMixin._call_if_present( - plot.getPlotItem().getAxis(axis_name), - "setMouseEnabled", - False, - ) - - @staticmethod - def _call_if_present(obj: object, method_name: str, *args, **kwargs) -> None: - """Call optional pyqtgraph API when available in the installed version.""" - method = getattr(obj, method_name, None) - if method is not None: - method(*args, **kwargs) - - def _ensure_gpr_plot_items(self) -> None: - """Create persistent GPR plot items once and reuse them on redraw.""" - if self._gpr_image_item is not None: - return - - plot = self._gpr_plot - self._configure_gpr_plot_axes() - - if self._gpr_lookup_table is None: - self._gpr_lookup_table = build_lut(["#081c15", "#1b4332", "#ffd166", "#f94144"]) - - self._gpr_image_item = pg.ImageItem(axisOrder="row-major") - self._gpr_image_item.setZValue(0) - self._gpr_image_item.hide() - plot.addItem(self._gpr_image_item) - - self._gpr_tx_item = pg.ScatterPlotItem() - self._gpr_tx_item.setZValue(20) - self._gpr_tx_item.hide() - plot.addItem(self._gpr_tx_item) - - self._gpr_rx_item = pg.ScatterPlotItem() - self._gpr_rx_item.setZValue(20) - self._gpr_rx_item.hide() - plot.addItem(self._gpr_rx_item) - - self._gpr_points_item = pg.ScatterPlotItem() - self._gpr_points_item.setZValue(30) - self._gpr_points_item.hide() - plot.addItem(self._gpr_points_item) - - self._gpr_region_centers_item = pg.ScatterPlotItem() - self._gpr_region_centers_item.setZValue(30) - self._gpr_region_centers_item.hide() - plot.addItem(self._gpr_region_centers_item) - - def _clear_gpr_point_labels(self) -> None: - """Remove dynamic point-score labels from GPR plot.""" - for item in self._gpr_point_labels: - try: - self._gpr_plot.removeItem(item) - except Exception as exc: # noqa: BLE001 - self._log_warning( - f"Failed to remove GPR point label: {type(exc).__name__}: {exc}", - once_key="gpr_remove_point_label_failed", - ) - self._gpr_point_labels.clear() - - def _clear_gpr_region_labels(self) -> None: - """Remove dynamic region labels from GPR plot.""" - for item in self._gpr_region_center_labels: - try: - self._gpr_plot.removeItem(item) - except Exception as exc: # noqa: BLE001 - self._log_warning( - f"Failed to remove GPR region label: {type(exc).__name__}: {exc}", - once_key="gpr_remove_region_label_failed", - ) - self._gpr_region_center_labels.clear() - - def _clear_gpr_region_masks(self) -> None: - """Remove dynamic region contour carriers from GPR plot.""" - for item in self._gpr_region_mask_items: - try: - self._gpr_plot.removeItem(item) - except Exception as exc: # noqa: BLE001 - self._log_warning( - f"Failed to remove GPR region mask: {type(exc).__name__}: {exc}", - once_key="gpr_remove_region_mask_failed", - ) - self._gpr_region_mask_items.clear() - self._gpr_region_contours.clear() - - @staticmethod - def _collection_payload_by_name(collection: ResultCollection, name: str, kind: int | None = None): - """Return first collection payload matching name and optional kind.""" - return gpr_collection_payload_by_name(collection, name, kind) - - @staticmethod - def _collection_payloads_by_prefix(collection: ResultCollection, prefix: str, kind: int | None = None): - """Return collection payloads matching processing-name prefix.""" - return gpr_collection_payloads_by_prefix(collection, prefix, kind) - - def _selected_gpr_geometry(self) -> tuple[np.ndarray, np.ndarray]: - """Resolve selected Tx/Rx geometry arrays for current GPR selection.""" - if self._processing_mode.currentText() == "legacy_gpr": - input_positions_text = self._legacy_gpr_input_positions_input.text() - output_positions_text = self._legacy_gpr_output_positions_input.text() - else: - input_positions_text = self._gpr_input_positions_input.text() - output_positions_text = self._gpr_output_positions_input.text() - requested_inputs = tuple(self._parse_csv_int_list(input_positions_text)) - requested_outputs = tuple(self._parse_csv_int_list(output_positions_text)) - tx_entries = self._gpr_tx_geometry_input.records() - rx_entries = self._gpr_rx_geometry_input.records() - signature = ( - tuple(tuple(entry.values()) for entry in tx_entries), - tuple(tuple(entry.values()) for entry in rx_entries), - requested_inputs, - requested_outputs, - ) - if signature == self._gpr_geometry_signature and self._gpr_selected_geometry is not None: - return self._gpr_selected_geometry - - requested_input_set = set(requested_inputs) - requested_output_set = set(requested_outputs) - - rx_entries = sorted(rx_entries, key=lambda entry: entry["input_pos"]) - tx_entries = sorted(tx_entries, key=lambda entry: entry["output_pos"]) - if requested_input_set: - rx_entries = [entry for entry in rx_entries if entry["input_pos"] in requested_input_set] - if requested_output_set: - tx_entries = [entry for entry in tx_entries if entry["output_pos"] in requested_output_set] - - x_tx = np.asarray([entry["x_m"] for entry in tx_entries], dtype=np.float32) - x_rx = np.asarray([entry["x_m"] for entry in rx_entries], dtype=np.float32) - self._gpr_geometry_signature = signature - self._gpr_selected_geometry = (x_tx, x_rx) - return self._gpr_selected_geometry - - def _draw_gpr_map(self, collection: ResultCollection) -> bool: - """Draw latest collection-level GPR plot according to current render mode.""" - if self._gpr_render_mode_text() == "objects_only": - return self._draw_gpr_objects_only(collection) - return self._draw_gpr_heatmap(collection) - - def _draw_gpr_heatmap(self, collection: ResultCollection) -> bool: - """Draw latest collection-level GPR accumulator and annotations.""" - accumulator_payload = self._collection_payload_by_name(collection, "gpr_accumulator", kind=3) - if accumulator_payload is None: - self._clear_gpr_plot() - return False - - image = np.asarray(accumulator_payload.image, dtype=np.float32) - x_axis = np.asarray(accumulator_payload.image_x_axis, dtype=np.float32) - y_axis = np.asarray(accumulator_payload.image_y_axis, dtype=np.float32) - if image.ndim != 2 or image.size == 0 or x_axis.size == 0 or y_axis.size == 0: - self._clear_gpr_plot() - return False - - x_min = float(x_axis[0]) - x_max = float(x_axis[-1]) - y_min = float(y_axis[0]) - y_max = float(y_axis[-1]) - rect = QRectF(x_min, y_min, max(x_max - x_min, 1e-6), max(y_max - y_min, 1e-6)) - - plot = self._gpr_plot - plot.setUpdatesEnabled(False) - try: - self._ensure_gpr_plot_items() - plot.getViewBox().invertY(False) - self._clear_gpr_point_labels() - self._clear_gpr_region_labels() - self._clear_gpr_region_masks() - - self._gpr_image_item.setImage(image, autoLevels=False) - self._gpr_image_item.setRect(rect) - self._gpr_image_item.setLookupTable(self._gpr_lookup_table) - self._gpr_image_item.setLevels((float(np.min(image)), float(np.max(image) + 1e-6))) - self._gpr_image_item.show() - - visible_x_min, visible_x_max, visible_z_min, visible_z_max = self._gpr_visible_bounds() - plot.setXRange(visible_x_min, visible_x_max, padding=0.0) - plot.setYRange(self._gpr_display_y_min(visible_z_min, visible_z_max), visible_z_max, padding=0.0) - - self._draw_gpr_geometry_markers() - - # Both gpr and legacy_gpr filter heatmap object markers by their own - # threshold + visible X/Z window (legacy has the same controls), so the - # markers match the objects-only view instead of showing raw detections. - points_payload = self._collection_payload_by_name(collection, "gpr_points", kind=4) - if points_payload is not None and np.asarray(points_payload.table).size > 0: - points = self._filtered_gpr_object_rows(collection) - else: - points = np.zeros((0, 3), dtype=np.float32) - - if points.size > 0: - self._gpr_points_item.setData( - x=points[:, 0], - y=points[:, 1], - symbol="d", - size=11, - brush=pg.mkBrush("#ffffff"), - pen=pg.mkPen("#111111", width=1.1), - ) - self._gpr_points_item.show() - for x_value, y_value, score in points: - label_text = f"{float(score):.0f}" if self._processing_mode.currentText() == "legacy_gpr" else f"{float(score):.2f}" - label = pg.TextItem(text=label_text, color="#ffffff", anchor=(0.0, 1.0)) - label.setZValue(40) - label.setPos(float(x_value), float(y_value)) - plot.addItem(label) - self._gpr_point_labels.append(label) - else: - self._gpr_points_item.setData(x=[], y=[]) - self._gpr_points_item.hide() - - region_centers_payload = self._collection_payload_by_name(collection, "gpr_region_centers", kind=4) - if region_centers_payload is not None and np.asarray(region_centers_payload.table).size > 0: - centers = np.asarray(region_centers_payload.table, dtype=np.float32) - self._gpr_region_centers_item.setData( - x=centers[:, 0], - y=centers[:, 1], - symbol="o", - size=10, - brush=pg.mkBrush("#80ed99"), - pen=pg.mkPen("#081c15", width=1.1), - ) - self._gpr_region_centers_item.show() - for row in centers: - label = pg.TextItem(text=f"{float(row[2]):.0f}", color="#d8f3dc", anchor=(0.0, 1.0)) - label.setZValue(40) - label.setPos(float(row[0]), float(row[1])) - plot.addItem(label) - self._gpr_region_center_labels.append(label) - else: - self._gpr_region_centers_item.setData(x=[], y=[]) - self._gpr_region_centers_item.hide() - - for payload in self._collection_payloads_by_prefix(collection, "gpr_region_mask_", kind=3): - mask = np.asarray(payload.image, dtype=np.float32) - if mask.ndim != 2 or mask.size == 0: - continue - mask_image = pg.ImageItem(axisOrder="row-major") - mask_image.setZValue(5) - mask_image.setImage(mask, autoLevels=False) - mask_image.setRect(rect) - mask_image.setOpacity(0.0) - plot.addItem(mask_image) - contour = pg.IsocurveItem(data=mask, level=0.5, pen=pg.mkPen("#4cc9f0", width=1.3)) - contour.setParentItem(mask_image) - self._gpr_region_mask_items.append(mask_image) - self._gpr_region_contours.append(contour) - - plot.setTitle(self._gpr_plot_title()) - finally: - plot.setUpdatesEnabled(True) - return True - - @staticmethod - def _normalized_display_range(start: float, stop: float, *, minimum_span: float = 0.1) -> tuple[float, float]: - """Return ordered display bounds with a non-zero span.""" - lower = min(float(start), float(stop)) - upper = max(float(start), float(stop)) - if upper - lower >= minimum_span: - return lower, upper - center = 0.5 * (lower + upper) - half_span = 0.5 * minimum_span - return center - half_span, center + half_span - - def _gpr_visible_bounds(self) -> tuple[float, float, float, float]: - """Return normalized GPR visible X/Z bounds from GUI controls.""" - if self._processing_mode.currentText() == "legacy_gpr": - x_min_widget = self._legacy_gpr_visible_x_min_m - x_max_widget = self._legacy_gpr_visible_x_max_m - z_min_widget = self._legacy_gpr_visible_z_min_m - z_max_widget = self._legacy_gpr_visible_z_max_m - else: - x_min_widget = self._gpr_visible_x_min_m - x_max_widget = self._gpr_visible_x_max_m - z_min_widget = self._gpr_visible_z_min_m - z_max_widget = self._gpr_visible_z_max_m - x_min, x_max = self._normalized_display_range( - float(x_min_widget.value()), - float(x_max_widget.value()), - minimum_span=0.1, - ) - z_min, z_max = self._normalized_display_range( - float(z_min_widget.value()), - float(z_max_widget.value()), - minimum_span=0.1, - ) - return x_min, x_max, z_min, z_max - - def _gpr_visible_object_bounds(self) -> tuple[float, float, float, float]: - """Return normalized object/locator visible X/Z bounds from GUI controls.""" - return self._gpr_visible_bounds() - - def _gpr_render_mode_text(self) -> str: - """Return render mode for the active GPR-like processing mode.""" - if self._processing_mode.currentText() == "legacy_gpr": - return self._legacy_gpr_render_mode.currentText() - return self._gpr_render_mode.currentText() - - @staticmethod - def _gpr_display_y_min(z_min: float, z_max: float) -> float: - """Return lower display bound, preserving surface markers only when surface is visible.""" - lower = float(z_min) - if lower > 0.0: - return lower - span = max(float(z_max) - lower, 1e-6) - marker_margin = max(span * 0.03, 0.06) - return lower - marker_margin - - def _draw_gpr_geometry_markers(self) -> None: - """Render selected Tx/Rx geometry markers on current GPR plot.""" - x_tx, x_rx = self._selected_gpr_geometry() - if x_tx.size > 0: - self._gpr_tx_item.setData( - x=x_tx, - y=np.zeros_like(x_tx), - symbol="t", - size=13, - brush=pg.mkBrush("#ff595e"), - pen=pg.mkPen("#ffca3a", width=1.0), - ) - self._gpr_tx_item.show() - else: - self._gpr_tx_item.setData(x=[], y=[]) - self._gpr_tx_item.hide() - - if x_rx.size > 0: - self._gpr_rx_item.setData( - x=x_rx, - y=np.zeros_like(x_rx), - symbol="t1", - size=13, - brush=pg.mkBrush("#4cc9f0"), - pen=pg.mkPen("#e0fbfc", width=1.0), - ) - self._gpr_rx_item.show() - else: - self._gpr_rx_item.setData(x=[], y=[]) - self._gpr_rx_item.hide() - - def _format_gpr_object_label(self, x_m: float, z_m: float, score: float) -> str: - """Format object-only annotation text with normalized BP score and coordinates.""" - if self._processing_mode.currentText() == "legacy_gpr": - return f"{int(round(score))} | x={x_m:.1f} | z={z_m:.1f}" - return f"{score:.2f} | x={x_m:.1f} | z={z_m:.1f}" - - @staticmethod - def _expanded_scene_rect(rect: QRectF, *, padding_px: float = 4.0) -> QRectF: - """Return scene rect padded to keep labels visually separated.""" - return rect.adjusted(-padding_px, -padding_px, padding_px, padding_px) - - @staticmethod - def _scene_rect_intersects_any(rect: QRectF, occupied_rects: list[QRectF]) -> bool: - """Return whether candidate label rect intersects any already placed label.""" - return any(rect.intersects(occupied_rect) for occupied_rect in occupied_rects) - - @staticmethod - def _gpr_object_label_candidates( - x_m: float, - z_m: float, - *, - x_span: float, - z_span: float, - ) -> list[tuple[float, float, tuple[float, float]]]: - """Return candidate label placements around one object.""" - x_offset = max(x_span * 0.015, 0.02) - z_offset = max(z_span * 0.02, 0.02) - return [ - (x_m + x_offset, z_m - z_offset, (0.0, 1.0)), - (x_m + x_offset, z_m + z_offset, (0.0, 0.0)), - (x_m - x_offset, z_m - z_offset, (1.0, 1.0)), - (x_m - x_offset, z_m + z_offset, (1.0, 0.0)), - (x_m, z_m - (z_offset * 1.35), (0.5, 1.0)), - (x_m, z_m + (z_offset * 1.35), (0.5, 0.0)), - (x_m + (x_offset * 2.2), z_m - (z_offset * 1.5), (0.0, 1.0)), - (x_m + (x_offset * 2.2), z_m + (z_offset * 1.5), (0.0, 0.0)), - (x_m - (x_offset * 2.2), z_m - (z_offset * 1.5), (1.0, 1.0)), - (x_m - (x_offset * 2.2), z_m + (z_offset * 1.5), (1.0, 0.0)), - ] - - def _place_gpr_object_label( - self, - *, - label: pg.TextItem, - x_m: float, - z_m: float, - x_span: float, - z_span: float, - occupied_scene_rects: list[QRectF], - ) -> None: - """Place one object label using the first non-overlapping candidate position.""" - last_rect: QRectF | None = None - for label_x, label_z, anchor in self._gpr_object_label_candidates( - x_m, - z_m, - x_span=x_span, - z_span=z_span, - ): - label.setAnchor(anchor) - label.setPos(label_x, label_z) - candidate_rect = self._expanded_scene_rect(label.sceneBoundingRect()) - last_rect = candidate_rect - if not self._scene_rect_intersects_any(candidate_rect, occupied_scene_rects): - occupied_scene_rects.append(candidate_rect) - return - - if last_rect is not None: - occupied_scene_rects.append(last_rect) - - def _gpr_object_rows(self, collection: ResultCollection) -> np.ndarray: - """Return object rows as `[x_m, z_m, score]` from current GPR result payload.""" - return extract_gpr_object_rows(collection) - - def _filtered_gpr_object_rows(self, collection: ResultCollection) -> np.ndarray: - """Return the object rows to draw for the active GPR mode. - - Coherent BP is already finalized by the processor (visible window + N/M draw - limits, no score threshold — exactly Horns_motion_3libre.py), so its rows are - drawn verbatim. Legacy GPR is still filtered here by its pair-count threshold - and the visible window. - """ - rows = self._gpr_object_rows(collection) - if rows.size == 0 or self._processing_mode.currentText() != "legacy_gpr": - return rows - - x_min, x_max, z_min, z_max = self._gpr_visible_object_bounds() - return gpr_filter_object_rows( - rows, - min_score=float(self._legacy_gpr_min_visible_pair_count.value()), - x_bounds=(x_min, x_max), - z_bounds=(z_min, z_max), - draw_limits=None, - ) - - def _draw_gpr_objects_only(self, collection: ResultCollection) -> bool: - """Draw only detected GPR objects inside configured X/Z bounds.""" - accumulator_payload = self._collection_payload_by_name(collection, "gpr_accumulator", kind=3) - object_rows = self._filtered_gpr_object_rows(collection) - if accumulator_payload is None and object_rows.size == 0: - self._clear_gpr_plot() - return False - - x_min, x_max, z_min, z_max = self._gpr_visible_object_bounds() - plot = self._gpr_plot - plot.setUpdatesEnabled(False) - try: - self._ensure_gpr_plot_items() - plot.getViewBox().invertY(False) - self._clear_gpr_point_labels() - self._clear_gpr_region_labels() - self._clear_gpr_region_masks() - - self._gpr_image_item.hide() - self._draw_gpr_geometry_markers() - self._gpr_region_centers_item.setData(x=[], y=[]) - self._gpr_region_centers_item.hide() - - if object_rows.size > 0: - self._gpr_points_item.setData( - x=object_rows[:, 0], - y=object_rows[:, 1], - symbol="o", - size=18, - brush=pg.mkBrush("#ff4d4f"), - pen=pg.mkPen("#ff4d4f", width=1.6), - ) - self._gpr_points_item.show() - - occupied_scene_rects: list[QRectF] = [] - x_span = x_max - x_min - z_span = z_max - z_min - for x_value, z_value, score in object_rows: - label = pg.TextItem( - text=self._format_gpr_object_label( - float(x_value), - float(z_value), - float(score), - ), - color="#ffd6d9", - anchor=(0.0, 1.0), - ) - label.setZValue(40) - plot.addItem(label) - self._place_gpr_object_label( - label=label, - x_m=float(x_value), - z_m=float(z_value), - x_span=x_span, - z_span=z_span, - occupied_scene_rects=occupied_scene_rects, - ) - self._gpr_point_labels.append(label) - else: - self._gpr_points_item.setData(x=[], y=[]) - self._gpr_points_item.hide() - - plot.setXRange(x_min, x_max, padding=0.0) - plot.setYRange(self._gpr_display_y_min(z_min, z_max), z_max, padding=0.0) - plot.setTitle(self._gpr_plot_title(objects_only=True)) - finally: - plot.setUpdatesEnabled(True) - return True diff --git a/python_app/gui/controllers/app_window_plot/trace_plot_mixin.py b/python_app/gui/controllers/app_window_plot/trace_plot_mixin.py deleted file mode 100644 index 57b119c..0000000 --- a/python_app/gui/controllers/app_window_plot/trace_plot_mixin.py +++ /dev/null @@ -1,458 +0,0 @@ -"""Trace plot rendering helpers for pass-through and single-trace views.""" - -from __future__ import annotations - -from PyQt6.QtCore import Qt -import numpy as np -import pyqtgraph as pg - -from python_app.models.dataset_model import ResultCollection, TraceData -from python_app.models.run_config_model import parse_combos_from_text - - -class AppWindowTracePlotMixin: - """Renders pass-through trace plots on stacked magnitude/phase widgets.""" - - def _show_magnitude_curves(self) -> bool: - """Return whether magnitude curves should be rendered.""" - return self._show_magnitude_checkbox.isChecked() - - def _show_phase_curves(self) -> bool: - """Return whether phase curves should be rendered.""" - return self._show_phase_checkbox.isChecked() - - def _unwrap_phase_enabled(self) -> bool: - """Return whether the phase trace should be unwrapped (cumulative).""" - return self._unwrap_phase_checkbox.isChecked() - - def _pass_through_fixed_y_range(self) -> tuple[bool, float, float]: - """Return normalized magnitude Y-range override for pass-through mode.""" - y_min = float(self._pass_through_y_min_db.value()) - y_max = float(self._pass_through_y_max_db.value()) - return bool(self._pass_through_fixed_y_enabled.isChecked()), min(y_min, y_max), max(y_min, y_max) - - def _pass_through_combo_filter(self) -> set[tuple[int, int]] | None: - """Return selected pass-through switch combos, or `None` when all are visible.""" - text = self._pass_through_combo_filter_input.text().strip() - if not text: - return None - try: - return { - (int(combo.input), int(combo.output)) - for combo in parse_combos_from_text(text) - } - except Exception as exc: # noqa: BLE001 - self._log_warning( - "Invalid pass-through switch-combo filter.", - details=f"{exc}\nExpected format: input:output,input:output", - once_key=f"pass_through_combo_filter_invalid_{text}", - ) - # Fail open: an unparseable filter is ignored (all traces stay visible), - # not turned into an empty allow-set that silently hides every trace. - return None - - def _configure_pass_through_magnitude_axis(self, plot: pg.PlotWidget) -> None: - """Apply the magnitude Y-axis: a fixed window or autorange. - - The X axis is set explicitly from the data band by the caller, so this - never re-enables X autorange (which would scan all curves every frame). - """ - fixed_y_enabled, y_min, y_max = self._pass_through_fixed_y_range() - view_box = plot.getViewBox() - view_box.invertY(False) - if fixed_y_enabled: - view_box.enableAutoRange(y=False) - plot.setYRange(y_min, y_max, padding=0.0) - else: - view_box.enableAutoRange(y=True) - - def _configure_pass_through_phase_axis(self, plot: pg.PlotWidget) -> None: - """Apply the phase Y-axis: fixed ±180° when wrapped, autorange when unwrapped.""" - view_box = plot.getViewBox() - view_box.invertY(False) - if self._unwrap_phase_enabled(): - view_box.enableAutoRange(y=True) - else: - view_box.enableAutoRange(y=False) - plot.setYRange(-180.0, 180.0, padding=0.02) - - def _on_trace_visibility_changed(self, *_args) -> None: - """Redraw pass-through traces when magnitude/phase toggles changed.""" - if self._processing_mode.currentText() in {"bscan", "gpr", "legacy_gpr", "sar3d"}: - return - if self._result_history: - self._draw_results(self._result_history[-1]) - return - self._clear_trace_plots() - - def _clear_trace_plots(self) -> None: - """Clear pass-through magnitude and phase plots.""" - self._trace_magnitude_plot.clear() - self._trace_phase_plot.clear() - self._clear_trace_legends() - self._trace_magnitude_curves.clear() - self._trace_phase_curves.clear() - - def _clear_trace_legends(self) -> None: - """Remove trace plot legends to avoid stale combo-color mappings.""" - mag_legend = self._trace_magnitude_legend - if mag_legend is not None: - try: - self._trace_magnitude_plot.getPlotItem().removeItem(mag_legend) - except Exception as exc: # noqa: BLE001 - self._log_warning( - f"Failed to remove pass-through magnitude legend: {type(exc).__name__}: {exc}", - once_key="plot_remove_magnitude_legend_failed", - ) - self._trace_magnitude_legend = None - self._trace_magnitude_legend_combo_keys.clear() - - phase_legend = self._trace_phase_legend - if phase_legend is not None: - try: - self._trace_phase_plot.getPlotItem().removeItem(phase_legend) - except Exception as exc: # noqa: BLE001 - self._log_warning( - f"Failed to remove pass-through phase legend: {type(exc).__name__}: {exc}", - once_key="plot_remove_phase_legend_failed", - ) - self._trace_phase_legend = None - self._trace_phase_legend_combo_keys.clear() - - def _draw_trace_lines(self, collection: ResultCollection) -> bool: - """Draw result payload traces as stacked magnitude/phase plots.""" - show_magnitude = self._show_magnitude_curves() - show_phase = self._show_phase_curves() - magnitude_plot = self._trace_magnitude_plot - phase_plot = self._trace_phase_plot - pass_through_channel = "S21" - - magnitude_plot.setVisible(show_magnitude) - phase_plot.setVisible(show_phase) - if not show_magnitude and not show_phase: - self._clear_trace_plots() - return False - combo_filter = self._pass_through_combo_filter() - - if show_magnitude: - mag_item = magnitude_plot.getPlotItem() - mag_item.showAxis("left", show=True) - mag_item.showAxis("bottom", show=not show_phase) - magnitude_plot.setLabel("left", "Magnitude", units="dB") - magnitude_plot.setTitle(f"Pass-Through {pass_through_channel}") - if not show_phase: - magnitude_plot.setLabel("bottom", "Frequency", units="Hz") - - if show_phase: - phase_item = phase_plot.getPlotItem() - phase_item.showAxis("left", show=True) - phase_item.showAxis("bottom", show=True) - phase_plot.setLabel("left", "Phase", units="deg") - phase_plot.setLabel("bottom", "Frequency", units="Hz") - phase_plot.setTitle(f"Pass-Through {pass_through_channel}") - - palette = [ - "#4cc9f0", - "#f72585", - "#b8f2e6", - "#ffd166", - "#90be6d", - "#ff595e", - "#6a4c93", - "#1982c4", - ] - - combo_colors: dict[tuple[int, int], str] = {} - legend_source_magnitude: dict[tuple[int, int], pg.PlotCurveItem] = {} - legend_source_phase: dict[tuple[int, int], pg.PlotCurveItem] = {} - active_magnitude_keys: set[tuple[int, int, int, str]] = set() - active_phase_keys: set[tuple[int, int, int, str]] = set() - has_data = False - x_min = np.inf - x_max = -np.inf - for block in collection.blocks: - combo_key = (int(block.combo.input), int(block.combo.output)) - if combo_filter is not None and combo_key not in combo_filter: - continue - if combo_key not in combo_colors: - combo_colors[combo_key] = palette[len(combo_colors) % len(palette)] - color = combo_colors[combo_key] - - for payload_index, payload in enumerate(block.payloads): - if payload.kind != 1 or payload.trace.size == 0: - continue - if payload.frequency_hz.size == 0 or payload.frequency_hz.size != payload.trace.size: - continue - curve_key = ( - combo_key[0], - combo_key[1], - int(payload_index), - str(payload.processing_name), - ) - - local_x_min = float(np.min(payload.frequency_hz)) - local_x_max = float(np.max(payload.frequency_hz)) - x_min = min(x_min, local_x_min) - x_max = max(x_max, local_x_max) - - if show_magnitude: - mag_x, magnitude_values = self._magnitude_display_arrays( - payload.frequency_hz, payload.trace - ) - active_magnitude_keys.add(curve_key) - magnitude_curve = self._trace_magnitude_curves.get(curve_key) - if magnitude_curve is None: - magnitude_curve = pg.PlotCurveItem(pen=pg.mkPen(color, width=1.4), antialias=False) - self._trace_magnitude_curves[curve_key] = magnitude_curve - magnitude_plot.addItem(magnitude_curve) - else: - magnitude_curve.setPen(pg.mkPen(color, width=1.4)) - magnitude_curve.setData(mag_x, magnitude_values) - legend_source_magnitude.setdefault(combo_key, magnitude_curve) - has_data = True - - if show_phase: - active_phase_keys.add(curve_key) - phase_curve = self._trace_phase_curves.get(curve_key) - if phase_curve is None: - phase_curve = pg.PlotCurveItem(pen=pg.mkPen(color, width=1.2), antialias=False) - self._trace_phase_curves[curve_key] = phase_curve - phase_plot.addItem(phase_curve) - else: - phase_curve.setPen(pg.mkPen(color, width=1.2)) - phase_x, phase_values = self._phase_display_arrays(payload.frequency_hz, payload.trace) - phase_curve.setData(phase_x, phase_values) - legend_source_phase.setdefault(combo_key, phase_curve) - has_data = True - - if show_magnitude: - self._remove_inactive_trace_curves( - plot=magnitude_plot, - cache=self._trace_magnitude_curves, - active_keys=active_magnitude_keys, - ) - else: - self._remove_all_trace_curves(plot=magnitude_plot, cache=self._trace_magnitude_curves) - - if show_phase: - self._remove_inactive_trace_curves( - plot=phase_plot, - cache=self._trace_phase_curves, - active_keys=active_phase_keys, - ) - else: - self._remove_all_trace_curves(plot=phase_plot, cache=self._trace_phase_curves) - - self._sync_trace_legends( - show_magnitude=show_magnitude, - show_phase=show_phase, - magnitude_sources=legend_source_magnitude, - phase_sources=legend_source_phase, - ) - - # X is the (constant) frequency band: set it explicitly rather than - # autoranging every frame. Y is configured once per draw. - if has_data and np.isfinite(x_min) and np.isfinite(x_max): - if show_magnitude: - magnitude_plot.setXRange(x_min, x_max, padding=0.02) - if show_phase: - phase_plot.setXRange(x_min, x_max, padding=0.02) - if show_magnitude: - self._configure_pass_through_magnitude_axis(magnitude_plot) - if show_phase: - self._configure_pass_through_phase_axis(phase_plot) - return has_data - - @staticmethod - def _remove_inactive_trace_curves( - *, - plot: pg.PlotWidget, - cache: dict[tuple[int, int, int, str], pg.PlotCurveItem], - active_keys: set[tuple[int, int, int, str]], - ) -> None: - """Delete curve items no longer present in latest result collection.""" - for key in list(cache.keys()): - if key in active_keys: - continue - curve = cache.pop(key) - plot.removeItem(curve) - - @staticmethod - def _remove_all_trace_curves( - *, - plot: pg.PlotWidget, - cache: dict[tuple[int, int, int, str], pg.PlotCurveItem], - ) -> None: - """Delete all cached curves from selected plot.""" - for curve in cache.values(): - plot.removeItem(curve) - cache.clear() - - def _render_decimation_step(self, size: int) -> int: - """Stride that caps a rendered trace at the configured maximum point count. - - Beyond ~1 point per screen pixel there is no visual gain, so decimating - keeps the line plots responsive even with many combos on screen. - """ - max_points = int(getattr(self, "_trace_render_max_points", 800)) - if max_points > 0 and size > max_points: - return max(1, int(np.ceil(size / max_points))) - return 1 - - def _magnitude_display_arrays( - self, frequency_hz: np.ndarray, trace: np.ndarray - ) -> tuple[np.ndarray, np.ndarray]: - """Return decimated (frequency, magnitude-dB) arrays for fast rendering.""" - step = self._render_decimation_step(int(trace.size)) - frequency_hz = frequency_hz[::step] - magnitude_db = 20.0 * np.log10(np.maximum(np.abs(trace[::step]), 1e-12)) - return frequency_hz, magnitude_db - - def _phase_display_arrays( - self, frequency_hz: np.ndarray, trace: np.ndarray - ) -> tuple[np.ndarray, np.ndarray]: - """Return decimated (frequency, phase-deg) arrays, unwrapped when enabled. - - Unwrapping runs at full resolution (before decimation) so the cumulative - phase is correct, then both arrays are decimated together for rendering. - """ - phase = np.angle(trace) - if self._unwrap_phase_enabled(): - phase = np.unwrap(phase) - phase_deg = np.degrees(phase) - step = self._render_decimation_step(int(trace.size)) - return frequency_hz[::step], phase_deg[::step] - - def _sync_trace_legends( - self, - *, - show_magnitude: bool, - show_phase: bool, - magnitude_sources: dict[tuple[int, int], pg.PlotCurveItem], - phase_sources: dict[tuple[int, int], pg.PlotCurveItem], - ) -> None: - """Rebuild legends only when active combo set changes.""" - self._sync_single_trace_legend( - show=show_magnitude, - plot=self._trace_magnitude_plot, - legend_attr="_trace_magnitude_legend", - legend_keys_attr="_trace_magnitude_legend_combo_keys", - sources=magnitude_sources, - ) - self._sync_single_trace_legend( - show=show_phase, - plot=self._trace_phase_plot, - legend_attr="_trace_phase_legend", - legend_keys_attr="_trace_phase_legend_combo_keys", - sources=phase_sources, - ) - - def _sync_single_trace_legend( - self, - *, - show: bool, - plot: pg.PlotWidget, - legend_attr: str, - legend_keys_attr: str, - sources: dict[tuple[int, int], pg.PlotCurveItem], - ) -> None: - """Rebuild one legend from provided combo->curve mapping when needed.""" - legend = getattr(self, legend_attr) - existing_keys = getattr(self, legend_keys_attr) - active_keys = set(sources.keys()) - if not show or not active_keys: - if legend is not None: - try: - plot.getPlotItem().removeItem(legend) - except Exception as exc: # noqa: BLE001 - self._log_warning( - f"Failed to clear plot legend: {type(exc).__name__}: {exc}", - once_key=f"{legend_attr}_clear_failed", - ) - setattr(self, legend_attr, None) - existing_keys.clear() - return - - if legend is not None and existing_keys == active_keys: - return - - if legend is not None: - try: - plot.getPlotItem().removeItem(legend) - except Exception as exc: # noqa: BLE001 - self._log_warning( - f"Failed to replace plot legend: {type(exc).__name__}: {exc}", - once_key=f"{legend_attr}_replace_failed", - ) - - legend = plot.addLegend(offset=(8, 8)) - for combo_key in sorted(active_keys): - curve = sources[combo_key] - legend.addItem(curve, f"in{combo_key[0]}/out{combo_key[1]}") - setattr(self, legend_attr, legend) - existing_keys.clear() - existing_keys.update(active_keys) - - def _draw_single_trace(self, trace: TraceData, title: str, *, channel: str = "s21") -> None: - """Draw one trace on stacked magnitude/phase plots.""" - show_magnitude = self._show_magnitude_curves() - show_phase = self._show_phase_curves() - magnitude_plot = self._trace_magnitude_plot - phase_plot = self._trace_phase_plot - samples = trace.s11 if channel == "s11" else trace.s21 - - magnitude_plot.setVisible(show_magnitude) - phase_plot.setVisible(show_phase) - self._clear_trace_plots() - if not show_magnitude and not show_phase: - return - - if show_magnitude: - magnitude_plot.getPlotItem().showAxis("bottom", show=not show_phase) - magnitude_plot.setLabel("left", "Magnitude", units="dB") - magnitude_plot.setTitle(title) - if not show_phase: - magnitude_plot.setLabel("bottom", "Frequency", units="Hz") - if show_phase: - phase_plot.getPlotItem().showAxis("bottom", show=True) - phase_plot.setLabel("left", "Phase", units="deg") - phase_plot.setLabel("bottom", "Frequency", units="Hz") - phase_plot.setTitle(title) - - if show_magnitude: - mag_x, magnitude_db = self._magnitude_display_arrays(trace.frequency_hz, samples) - magnitude_curve = pg.PlotCurveItem( - mag_x, - magnitude_db, - pen=pg.mkPen("#ffd166", width=1.8), - antialias=False, - ) - magnitude_plot.addItem(magnitude_curve) - self._trace_magnitude_curves[ - (int(trace.combo.input), int(trace.combo.output), 0, "__single_trace__") - ] = magnitude_curve - - if show_phase: - phase_x, phase_values = self._phase_display_arrays(trace.frequency_hz, samples) - phase_curve = pg.PlotCurveItem( - phase_x, - phase_values, - pen=pg.mkPen("#80ed99", width=1.4, style=Qt.PenStyle.DashLine), - antialias=False, - ) - phase_plot.addItem(phase_curve) - self._trace_phase_curves[ - (int(trace.combo.input), int(trace.combo.output), 0, "__single_trace__") - ] = phase_curve - - if np.size(trace.frequency_hz) > 1: - x_min = float(np.min(trace.frequency_hz)) - x_max = float(np.max(trace.frequency_hz)) - if show_magnitude: - magnitude_plot.setXRange(x_min, x_max, padding=0.02) - if show_phase: - phase_plot.setXRange(x_min, x_max, padding=0.02) - if show_magnitude: - self._configure_pass_through_magnitude_axis(magnitude_plot) - if show_phase: - self._configure_pass_through_phase_axis(phase_plot) diff --git a/python_app/gui/controllers/app_window_plot_mixin.py b/python_app/gui/controllers/app_window_plot_mixin.py deleted file mode 100644 index 13e21c5..0000000 --- a/python_app/gui/controllers/app_window_plot_mixin.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Facade mixin for processed-radar plot rendering.""" - -from __future__ import annotations - -from python_app.gui.controllers.app_window_plot import ( - AppWindowBscanPlotMixin, - AppWindowBscanReplayMixin, - AppWindowGprPlotMixin, - AppWindowSar3dPlotMixin, - AppWindowTracePlotMixin, -) -from python_app.models.dataset_model import ResultCollection - - -class AppWindowPlotMixin( - AppWindowTracePlotMixin, - AppWindowBscanPlotMixin, - AppWindowBscanReplayMixin, - AppWindowGprPlotMixin, - AppWindowSar3dPlotMixin, -): - """Routes plotting to trace, B-scan, GPR, or 3D SAR-specific mixins.""" - - def _draw_results(self, collection: ResultCollection) -> bool: - """Draw collection based on currently selected processing mode.""" - expected_revision = getattr(self, "_required_processing_revision", None) - if expected_revision is not None and self._is_gpr_like_processing_mode(self._processing_mode.currentText()): - revision = next((payload for payload in collection.collection_payloads - if payload.processing_name == "processing_revision" and payload.kind == 4), None) - if revision is None or revision.table.shape != (1, 4): - return False - if any(not 0 <= word <= 0xffff or word != int(word) for word in revision.table[0]): - return False - actual_revision = sum(int(word) << (index * 16) for index, word in enumerate(revision.table[0])) - if actual_revision != expected_revision: - return False - if self._processing_mode.currentText() == "bscan": - return self._draw_bscan_heatmap(collection) - if self._processing_mode.currentText() in {"gpr", "legacy_gpr"}: - return self._draw_gpr_map(collection) - if self._processing_mode.currentText() == "sar3d": - return self._draw_sar3d_projections(collection) - return self._draw_trace_lines(collection) - - def _result_collection_has_trace(self, collection: ResultCollection) -> bool: - """Recognize processed output, including an empty trace after a valid depth/frequency cut.""" - if any(payload.processing_name != "processing_revision" for payload in collection.collection_payloads): - return True - for block in collection.blocks: - for payload in block.payloads: - if payload.kind == 1: - return True - return False diff --git a/python_app/gui/controllers/app_window_preprocess_mixin.py b/python_app/gui/controllers/app_window_preprocess_mixin.py deleted file mode 100644 index 973b1ef..0000000 --- a/python_app/gui/controllers/app_window_preprocess_mixin.py +++ /dev/null @@ -1,921 +0,0 @@ -"""Preprocessing-set selection and sequential capture workflow mixin.""" - -from __future__ import annotations - -from PyQt6.QtCore import QTimer - -from python_app.gui.preprocess_dialog import PreprocessDialog -from python_app.gui.controllers.object_calibration_controller import ObjectCalibrationController -from python_app.gui.trace_png_export import export_trace_png -from python_app.orchestration.preprocess_assets import ( - PREPROCESS_ASSET_SPECS, - VISIBLE_PREPROCESS_ASSET_KEYS, - preprocess_asset_channel, - preprocess_asset_display_name, -) -from python_app.workflows.kamil_adc_neutral_preprocess import ( - build_neutral_s21_sets, - supports_neutral_preprocess_sets, -) -from python_app.workflows.multi_radar_capture_workflow import ( - MultiRadarCaptureBatch, - MultiRadarSequentialCaptureSession, -) -from python_app.workflows.radar_config_variants import scan_radar_config_variants -from python_app.workflows.reference_workflow import capture_reference_set -from python_app.workflows.sequential_capture_workflow import SequentialCaptureSession - - -TMP_REFERENCE_SET_NAME = "tmp_reference" - - -class AppWindowPreprocessMixin: - """Handles preprocess set management and sequential capture workflows.""" - - @staticmethod - def _capture_log_entries_for_session(session: SequentialCaptureSession | MultiRadarSequentialCaptureSession) -> list[str]: - """Build capture-log rows from current session traces.""" - display_name = preprocess_asset_display_name(session.kind) - total_count = session.state().total_count - if isinstance(session, MultiRadarSequentialCaptureSession): - return [ - f"{display_name}: {index}/{total_count} | " - f"input={batch.combo.input} output={batch.combo.output} | " - f"radar_configs={session.radar_variant_count()}" - for index, batch in enumerate(session.captured_batches(), start=1) - ] - - entries: list[str] = [] - for index, trace in enumerate(session.captured_traces(), start=1): - entries.append( - f"{display_name}: {index}/{total_count} | " - f"input={trace.combo.input} output={trace.combo.output}" - ) - return entries - - def _available_preprocess_sets_for_radar_key(self, radar_key: str) -> dict[str, list[str]]: - """Load available preprocess-set names for one radar key.""" - return { - key: self._store.list_sets(PREPROCESS_ASSET_SPECS[key].set_kind, radar_key) - for key in VISIBLE_PREPROCESS_ASSET_KEYS - } - - def _reset_preprocess_selection_after_radar_key_change(self) -> None: - """Clear selected preprocess sets when radar-key-defining settings change.""" - try: - radar_key = self._radar_key_from_ui() - except Exception as exc: # noqa: BLE001 - # Radar fields can be mid-edit (empty/partial) while signals fire; the - # key cannot be computed yet, so skip until the inputs are valid again. - self._log_debug( - f"Skipping preprocess-selection reset; radar key unavailable: " - f"{type(exc).__name__}: {exc}" - ) - return - - previous_radar_key = getattr(self, "_selected_preprocess_radar_key", radar_key) - if radar_key == previous_radar_key: - return - - self._selected_preprocess_radar_key = radar_key - cleared = { - key: value - for key, value in self._selected_preprocess_sets.items() - if value - } - if not cleared: - if self._preprocess_dialog is not None: - self._refresh_sets() - return - - self._selected_preprocess_sets = { - key: "" - for key in VISIBLE_PREPROCESS_ASSET_KEYS - } - self._refresh_preprocess_summary_labels() - - if self._preprocess_dialog is not None: - self._preprocess_dialog.set_selected_sets(self._selected_preprocess_sets, emit_signal=False) - self._refresh_sets() - - cleared_summary = ", ".join( - f"{preprocess_asset_display_name(key)}={value}" - for key, value in cleared.items() - ) - self._log( - "Preprocess selection reset after radar settings changed: " - f"{previous_radar_key} -> {radar_key}; cleared {cleared_summary}" - ) - - def _open_preprocess_panel(self) -> None: - """Open preprocessing dialog and refresh available sets.""" - dialog = self._ensure_preprocess_dialog() - try: - self._refresh_sets() - self._update_capture_dialog_state() - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to open preprocessing panel", exc) - return - - dialog.showMaximized() - dialog.raise_() - dialog.activateWindow() - - def _capture_tmp_reference(self) -> None: - """Capture, save, and select a temporary S21 reference with current sweep settings.""" - if getattr(self, "_closing", False) or getattr(self, "_stopping_run", False): - return - if getattr(self, "_object_capture_job", None) is not None: - self._show_error("Wait for object-calibration capture to finish") - return - if self._capture_session is not None: - self._show_error( - "Cannot capture tmp reference during active capture sequence", - details=self._capture_state_details(), - ) - return - # One shared re-entrancy guard for BOTH the GUI button and the GPIO trigger: - # this capture spins the Qt event loop (stop/start run, dialogs), so a second - # request from either source must not start a nested capture. - if self._control_button_busy: - self._log("Tmp reference capture already in progress; ignoring duplicate request.") - return - self._control_button_busy = True - - pipeline_was_running = self._supervisor.is_running() - pipeline_was_paused = False - resume_seq = getattr(self, "_stop_request_seq", 0) + 1 - - try: - if pipeline_was_running: - self._log("Pipeline paused for tmp reference capture") - self._stop_run() - pipeline_was_paused = True - if not self._pipeline_resume_allowed(resume_seq): - return - - config = self._build_config() - radar_key = self._radar_key(config) - self._log( - "Tmp S21 Reference capture started: " - f"set={TMP_REFERENCE_SET_NAME}, radar_key={radar_key}" - ) - - radar_key, collection = capture_reference_set( - config, - TMP_REFERENCE_SET_NAME, - self._store, - median_sweep_count=self._preprocess_median_sweep_count, - ) - self._selected_preprocess_sets["s21_reference"] = TMP_REFERENCE_SET_NAME - self._selected_preprocess_radar_key = radar_key - # Do NOT restart data_processor for a reference change: it does not consume - # the S21 reference (the data_preprocessor does), and restarting it would tear - # down the locator server it hosts and drop every connected client. The - # preprocessor reloads the new reference when acquisition restarts below; the - # processor keeps running. We only re-baseline the on-screen display history. - self._history_run_signature = None - self._reset_runtime_history() - # Tell the still-running data_processor to drop its accumulated background/ - # history so the new reference takes effect cleanly (no old/new-reference - # blend) without restarting the process. Reuses the live-config clear_all - # command, applied on the processor's next poll. - if self._supervisor.is_processor_running(): - self._write_live_processing_config(history_command="clear_all", bump_history_seq=True) - self._refresh_preprocess_summary_labels() - if collection.traces: - self._draw_single_trace( - collection.traces[-1], - title="Tmp S21 Reference last trace", - channel="s21", - ) - if self._preprocess_dialog is not None: - self._refresh_sets() - self._preprocess_dialog.set_status( - f"S21 Reference set saved: {TMP_REFERENCE_SET_NAME} ({len(collection.traces)} traces)" - ) - - self._log( - "Tmp S21 Reference captured and selected: " - f"set={TMP_REFERENCE_SET_NAME}, key={radar_key}, traces={len(collection.traces)}; " - "runtime history reset" - ) - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to capture tmp reference", exc) - finally: - if pipeline_was_paused and self._pipeline_resume_allowed(resume_seq): - self._start_run() - self._control_button_busy = False - - def _ensure_preprocess_dialog(self) -> PreprocessDialog: - """Create preprocessing dialog lazily and wire its signals once.""" - if self._preprocess_dialog is not None: - return self._preprocess_dialog - - dialog = PreprocessDialog(self) - self._preprocess_dialog = dialog - dialog.set_set_name(self._preprocess_set_name) - dialog.set_radar_config_dir(self._preprocess_radar_config_dir) - dialog.set_use_all_radar_configs(self._preprocess_use_all_radar_configs) - dialog.set_median_sweep_count(self._preprocess_median_sweep_count) - dialog.set_selected_sets(self._selected_preprocess_sets, emit_signal=False) - dialog.refresh_requested.connect(self._refresh_sets) - dialog.selection_changed.connect(self._on_preprocess_selection_changed) - dialog.radar_config_dir_changed.connect(self._on_preprocess_radar_config_inputs_changed) - dialog.multi_radar_option_changed.connect(self._on_preprocess_radar_config_inputs_changed) - dialog.median_sweep_count_changed.connect(self._on_preprocess_median_sweep_count_changed) - dialog.start_sequence_requested.connect(self._start_capture_sequence) - dialog.capture_next_requested.connect(self._capture_next_combo) - dialog.capture_all_requested.connect(self._capture_all_remaining) - dialog.undo_last_requested.connect(self._undo_last_capture) - dialog.finalize_sequence_requested.connect(self._finalize_capture_sequence) - dialog.abort_sequence_requested.connect(self._abort_capture_sequence) - dialog.create_neutral_sets_requested.connect(self._create_neutral_sets) - self._object_calibration_controller = ObjectCalibrationController(self) - dialog.import_object_calibration_requested.connect(self._object_calibration_controller.import_calibration) - dialog.create_object_calibration_requested.connect(self._object_calibration_controller.open_creation) - dialog.set_neutral_sets_visible(supports_neutral_preprocess_sets(self._defaults_config)) - dialog.set_radar_config_summary( - directory_path=self._preprocess_radar_scan_summary.directory_path, - json_file_count=self._preprocess_radar_scan_summary.json_file_count, - valid_variant_count=self._preprocess_radar_scan_summary.valid_variant_count, - skipped_file_count=self._preprocess_radar_scan_summary.skipped_file_count, - duplicate_variant_count=self._preprocess_radar_scan_summary.duplicate_variant_count, - ) - self._update_capture_dialog_state() - return dialog - - def _on_preprocess_radar_config_inputs_changed(self) -> None: - """Persist preprocess radar-config directory options and refresh the scan result.""" - dialog = self._ensure_preprocess_dialog() - self._preprocess_radar_config_dir = dialog.radar_config_dir() - self._preprocess_use_all_radar_configs = dialog.use_all_radar_configs() - self._refresh_sets() - - def _on_preprocess_median_sweep_count_changed(self) -> None: - """Persist updated per-combo median sweep count from the preprocessing dialog.""" - dialog = self._ensure_preprocess_dialog() - self._preprocess_median_sweep_count = dialog.median_sweep_count() - - def _on_preprocess_selection_changed(self) -> None: - """Persist selected preprocessing set names from dialog.""" - dialog = self._ensure_preprocess_dialog() - previous_selection = dict(self._selected_preprocess_sets) - self._selected_preprocess_sets = dialog.selection_snapshot() - self._refresh_preprocess_summary_labels() - changes = [] - for key in VISIBLE_PREPROCESS_ASSET_KEYS: - previous_value = previous_selection.get(key, "") - current_value = self._selected_preprocess_sets.get(key, "") - if previous_value == current_value: - continue - changes.append( - f"{preprocess_asset_display_name(key)}: " - f"{previous_value or ''} -> {current_value or ''}" - ) - if changes: - self._log("Preprocess selection changed: " + "; ".join(changes)) - - def _refresh_preprocess_summary_labels(self) -> None: - """Update compact summary labels in the main window.""" - for key in VISIBLE_PREPROCESS_ASSET_KEYS: - self._selected_preprocess_labels[key].setText(self._selected_preprocess_sets.get(key, "") or "") - - def _refresh_sets(self) -> None: - """Refresh preprocess set lists for current radar key.""" - radar_key = self._radar_key_from_ui() - self._selected_preprocess_radar_key = radar_key - dialog = self._ensure_preprocess_dialog() - self._preprocess_radar_config_dir = dialog.radar_config_dir() - self._preprocess_use_all_radar_configs = dialog.use_all_radar_configs() - - available_sets = self._available_preprocess_sets_for_radar_key(radar_key) - dialog.set_available_sets(available_sets) - self._refresh_preprocess_radar_variants() - - unavailable_selections: list[str] = [] - for key, names in available_sets.items(): - current_value = self._selected_preprocess_sets.get(key, "") - if not current_value or current_value in names: - continue - unavailable_selections.append( - f"{preprocess_asset_display_name(key)}: " - f"{current_value} (not available for current radar key)" - ) - - dialog.set_selected_sets(self._selected_preprocess_sets) - self._refresh_preprocess_summary_labels() - available_counts = ", ".join( - f"{preprocess_asset_display_name(key)}={len(names)}" - for key, names in available_sets.items() - ) - dialog.set_neutral_sets_visible(supports_neutral_preprocess_sets(self._defaults_config)) - self._log(f"Preprocess set lists refreshed: radar_key={radar_key}, {available_counts}") - if unavailable_selections: - self._log_warning( - "Some selected preprocess sets are not currently available for this radar key.", - details="\n".join(unavailable_selections), - ) - - def _refresh_preprocess_radar_variants(self) -> None: - """Reload valid radar sweep variants from the preprocess dialog directory.""" - config = self._build_config() - variants, summary = scan_radar_config_variants( - self._preprocess_radar_config_dir, - base_config=config, - ) - self._preprocess_radar_variants = variants - self._preprocess_radar_scan_summary = summary - - if self._preprocess_dialog is not None: - self._preprocess_dialog.set_radar_config_summary( - directory_path=summary.directory_path, - json_file_count=summary.json_file_count, - valid_variant_count=summary.valid_variant_count, - skipped_file_count=summary.skipped_file_count, - duplicate_variant_count=summary.duplicate_variant_count, - ) - - if summary.issues: - self._log_warning( - "Some radar config JSON files were skipped during preprocess scan.", - details="\n".join(summary.issues), - once_key=f"preprocess_radar_config_scan_{summary.directory_path}", - ) - - def _start_capture_sequence(self, kind: str) -> None: - """Start sequential capture session for requested preprocess asset.""" - if getattr(self, "_closing", False) or getattr(self, "_stopping_run", False): - return - if getattr(self, "_object_capture_job", None) is not None: - self._show_error("Wait for object-calibration capture to finish") - return - if self._capture_session is not None: - self._show_error("Another capture sequence is already active", details=self._capture_state_details()) - return - - dialog = self._ensure_preprocess_dialog() - set_name = dialog.set_name() - if not set_name: - self._show_error("Set name is required") - return - - was_running = self._supervisor.is_running() - resume_seq = getattr(self, "_stop_request_seq", 0) + 1 - if was_running: - self._log("Pipeline paused for exclusive hardware capture") - self._stop_run() - if not self._pipeline_resume_allowed(resume_seq): - return - - self._resume_pipeline_after_capture = was_running - - try: - config = self._build_config() - display_name = preprocess_asset_display_name(kind) - median_sweep_count = dialog.median_sweep_count() - self._preprocess_median_sweep_count = median_sweep_count - if self._preprocess_use_all_radar_configs: - session = self._build_multi_radar_capture_session( - config=config, - kind=kind, - set_name=set_name, - median_sweep_count=median_sweep_count, - ) - radar_summary = ( - f"radar_variants={session.radar_variant_count()}, " - f"combos={session.state().total_count}, " - f"median_sweep_count={median_sweep_count}" - ) - else: - session = self._build_single_radar_capture_session( - config=config, - kind=kind, - set_name=set_name, - median_sweep_count=median_sweep_count, - ) - radar_summary = ( - f"radar_key={self._radar_key(config)}, " - f"combos={session.state().total_count}, " - f"median_sweep_count={median_sweep_count}" - ) - - session.open() - self._capture_session = session - - dialog.clear_capture_log() - dialog.reset_preview() - dialog.set_status(f"{display_name} sequence started") - self._clear_trace_plots() - self._update_capture_dialog_state() - self._log( - f"{display_name} sequence started: set={set_name}, {radar_summary}" - ) - except Exception as exc: # noqa: BLE001 - self._cleanup_capture_session() - self._show_exception(f"Failed to start {kind} sequence", exc) - self._resume_pipeline_if_needed() - - def _create_neutral_sets(self) -> None: - """Save neutral S21 calibration/reference sets for the current radar settings.""" - if getattr(self, "_closing", False) or getattr(self, "_stopping_run", False): - return - if getattr(self, "_object_capture_job", None) is not None: - self._show_error("Wait for object-calibration capture to finish") - return - if self._capture_session is not None: - self._show_error( - "Cannot create neutral sets during active capture sequence", - details=self._capture_state_details(), - ) - return - - dialog = self._ensure_preprocess_dialog() - set_name = dialog.set_name() - if not set_name: - self._show_error("Set name is required") - return - - pipeline_was_paused = False - resume_seq = getattr(self, "_stop_request_seq", 0) + 1 - try: - config = self._build_config() - if not supports_neutral_preprocess_sets(config): - self._show_error( - "Neutral S21 sets are not supported for this radar configuration" - ) - return - - radar_key = self._radar_key(config) - duplicate_assets = [ - preprocess_asset_display_name(key) - for key in ("s21_calibration", "s21_reference") - if set_name in self._store.list_sets(PREPROCESS_ASSET_SPECS[key].set_kind, radar_key) - ] - if duplicate_assets: - raise RuntimeError( - f"Set '{set_name}' already exists for: " + ", ".join(duplicate_assets) - ) - - if self._supervisor.is_running(): - self._log("Pipeline paused for neutral-set creation") - self._stop_run() - pipeline_was_paused = True - if not self._pipeline_resume_allowed(resume_seq): - return - - calibration, reference = build_neutral_s21_sets(config) - point_count = int(calibration.traces[0].frequency_hz.size) - self._store.save_sets(radar_key, set_name, { - "s21_calibration": calibration, "s21_reference": reference, - }) - - self._selected_preprocess_sets["s21_calibration"] = set_name - self._selected_preprocess_sets["s21_reference"] = set_name - self._selected_preprocess_radar_key = radar_key - self._processor_run_signature = None - self._history_run_signature = None - self._reset_runtime_history() - self._refresh_sets() - dialog.set_status( - f"Neutral S21 sets saved: {set_name} ({len(calibration.traces)} combos, {point_count} points)" - ) - self._log( - "Neutral S21 sets saved: " - f"set={set_name}, radar_key={radar_key}, combos={len(calibration.traces)}, points={point_count}" - ) - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to create neutral S21 sets", exc) - finally: - if pipeline_was_paused and self._pipeline_resume_allowed(resume_seq): - self._start_run() - - def _build_single_radar_capture_session( - self, - *, - config, - kind: str, - set_name: str, - median_sweep_count: int, - ) -> SequentialCaptureSession: - """Validate and create the existing single-radar capture session.""" - radar_key = self._radar_key(config) - existing_sets = self._store.list_sets(PREPROCESS_ASSET_SPECS[kind].set_kind, radar_key) - display_name = preprocess_asset_display_name(kind) - if set_name in existing_sets: - raise RuntimeError(f"Set '{set_name}' already exists for {display_name} and cannot be overwritten") - return SequentialCaptureSession( - config=config, - kind=kind, - set_name=set_name, - median_sweep_count=median_sweep_count, - ) - - def _build_multi_radar_capture_session( - self, - *, - config, - kind: str, - set_name: str, - median_sweep_count: int, - ) -> MultiRadarSequentialCaptureSession: - """Validate and create the multi-radar capture session.""" - self._refresh_preprocess_radar_variants() - if not self._preprocess_radar_variants: - raise RuntimeError("No valid radar config variants were found in the selected directory") - - display_name = preprocess_asset_display_name(kind) - duplicate_set_keys: list[str] = [] - for variant in self._preprocess_radar_variants: - existing_sets = self._store.list_sets(PREPROCESS_ASSET_SPECS[kind].set_kind, variant.radar_key) - if set_name in existing_sets: - duplicate_set_keys.append(f"{variant.display_name} -> {variant.radar_key}") - if duplicate_set_keys: - raise RuntimeError( - f"Set '{set_name}' already exists for {display_name} in these radar variants:\n" - + "\n".join(duplicate_set_keys) - ) - - return MultiRadarSequentialCaptureSession( - base_config=config, - kind=kind, - set_name=set_name, - radar_variants=self._preprocess_radar_variants, - median_sweep_count=median_sweep_count, - ) - - def _capture_next_combo(self) -> None: - """Capture next combo in active sequential capture session.""" - session = self._capture_session - if session is None: - self._show_error("No active capture sequence") - return - # The capture blocks the event loop, so clicks made during it are delivered - # only after it finishes. `_begin_preprocess_capture` disables the action - # buttons for that whole window (re-enabled via a posted event), so a queued - # click lands on a disabled button instead of silently starting — and - # advancing the combo cursor of — another capture. - if not self._begin_preprocess_capture(): - return - - try: - try: - capture_result = session.capture_current_combo() - except Exception as exc: # noqa: BLE001 - self._on_capture_combo_failed(session, exc) - return - self._record_preprocess_capture(session, capture_result) - finally: - self._end_preprocess_capture() - - def _capture_all_remaining(self) -> None: - """Capture all remaining combos for the active preprocess session.""" - session = self._capture_session - if session is None: - self._show_error("No active capture sequence") - return - if session.is_complete(): - self._show_error( - "Capture sequence is already complete", - details=self._capture_state_details(), - ) - return - if not self._begin_preprocess_capture(): - return - - display_name = preprocess_asset_display_name(session.kind) - dialog = self._ensure_preprocess_dialog() - stop_seq = getattr(self, "_stop_request_seq", 0) - dialog.set_status(f"{display_name} batch capture started") - self._log( - f"{display_name} batch capture started: remaining=" - f"{session.state().total_count - session.state().captured_count}" - ) - try: - while self._capture_session is session and self._pipeline_resume_allowed(stop_seq) and not session.is_complete(): - try: - capture_result = session.capture_current_combo() - except Exception as exc: # noqa: BLE001 - self._on_capture_combo_failed(session, exc) - return - self._record_preprocess_capture(session, capture_result) - if not session.is_complete(): - self._pump_events_during_drain(0.01) - if self._capture_session is session and not session.is_complete(): - dialog.set_status(f"{display_name} batch capture paused; previous captures kept") - finally: - self._end_preprocess_capture() - - def _begin_preprocess_capture(self) -> bool: - """Mark a blocking combo capture as running; refuse when one already is. - - Returns False for a duplicate request (e.g. a click delivered while an - error dialog inside a capture pumps the event loop). - """ - if getattr(self, "_closing", False) or self._preprocess_capture_busy: - self._log("Preprocess combo capture already in progress; ignoring duplicate request.") - return False - self._preprocess_capture_busy = True - # Disable the sequence action buttons for the whole blocked window. - self._update_capture_dialog_state() - return True - - def _end_preprocess_capture(self) -> None: - """Re-enable capture actions after the pending input backlog is discarded. - - The zero-delay timer fires only after Qt has dispatched the window-system - events queued while the capture blocked the loop; those clicks hit the - still-disabled buttons and are dropped, then the buttons come back. - """ - self._preprocess_capture_busy = False - QTimer.singleShot(0, self._update_capture_dialog_state) - - def _on_capture_combo_failed( - self, - session: SequentialCaptureSession | MultiRadarSequentialCaptureSession, - exc: BaseException, - ) -> None: - """Report a failed combo capture while preserving the session and prior captures.""" - state = session.state() - combo = state.current_combo - combo_text = ( - f"input={combo.input}, output={combo.output}" if combo is not None else "" - ) - self._show_exception( - f"Failed to capture combo {combo_text}; previous captures kept, retry when ready", - exc, - ) - display_name = preprocess_asset_display_name(session.kind) - dialog = self._ensure_preprocess_dialog() - dialog.set_status( - f"{display_name} capture failed at {combo_text}: " - f"{state.captured_count}/{state.total_count} kept, ready to retry" - ) - self._update_capture_dialog_state() - - def _record_preprocess_capture( - self, - session: SequentialCaptureSession | MultiRadarSequentialCaptureSession, - capture_result, - ) -> None: - """Update UI, preview, and logs after one successful preprocess capture.""" - dialog = self._ensure_preprocess_dialog() - state = session.state() - display_name = preprocess_asset_display_name(session.kind) - channel = preprocess_asset_channel(session.kind) - if isinstance(session, MultiRadarSequentialCaptureSession): - assert isinstance(capture_result, MultiRadarCaptureBatch) - trace = capture_result.display_trace - input_pos = capture_result.combo.input - output_pos = capture_result.combo.output - extra_details = f" | radar_configs={len(capture_result.traces)}" - else: - trace = capture_result - input_pos = trace.combo.input - output_pos = trace.combo.output - extra_details = "" - - dialog.append_capture_log_entry( - kind=display_name, - captured_count=state.captured_count, - total_count=state.total_count, - input_pos=input_pos, - output_pos=output_pos, - extra_details=extra_details, - ) - dialog.draw_last_trace(trace, title=f"{display_name} captured", channel=channel) - self._draw_single_trace(trace, title=f"{display_name} last trace", channel=channel) - self._log( - f"{display_name} capture: {state.captured_count}/{state.total_count} | " - f"input={input_pos} output={output_pos}{extra_details}" - ) - - self._update_capture_dialog_state() - if session.is_complete(): - dialog.set_status(f"{display_name} sequence complete. Review captures or save the set.") - self._log( - f"{display_name} sequence capture complete: " - f"{state.captured_count}/{state.total_count}; waiting for save or undo" - ) - - def _undo_last_capture(self) -> None: - """Remove the most recently captured combo and rewind capture cursor.""" - session = self._capture_session - if session is None: - self._show_error("No active capture sequence") - return - - dialog = self._ensure_preprocess_dialog() - try: - removed_capture = session.undo_last_capture() - state = session.state() - display_name = preprocess_asset_display_name(session.kind) - channel = preprocess_asset_channel(session.kind) - if isinstance(session, MultiRadarSequentialCaptureSession): - assert isinstance(removed_capture, MultiRadarCaptureBatch) - removed_input = removed_capture.combo.input - removed_output = removed_capture.combo.output - extra_details = f" | radar_configs={len(removed_capture.traces)}" - else: - removed_input = removed_capture.combo.input - removed_output = removed_capture.combo.output - extra_details = "" - - dialog.set_capture_log_entries(self._capture_log_entries_for_session(session)) - last_trace = session.last_captured_trace() - if last_trace is None: - dialog.reset_preview() - dialog.set_status(f"{display_name} last capture removed. No captured combos remain.") - self._clear_trace_plots() - else: - dialog.draw_last_trace(last_trace, title=f"{display_name} last trace", channel=channel) - dialog.set_status(f"{display_name} last capture removed. Ready to recapture.") - self._draw_single_trace(last_trace, title=f"{display_name} last trace", channel=channel) - - self._update_capture_dialog_state() - self._log( - f"{display_name} undo last capture: removed input={removed_input} " - f"output={removed_output}{extra_details}; remaining={state.captured_count}/{state.total_count}" - ) - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to undo last preprocess capture", exc) - - def _finalize_capture_sequence(self) -> None: - """Persist completed capture session into preprocess-set storage.""" - session = self._capture_session - if session is None: - self._show_error("No active capture sequence") - return - if not session.is_complete(): - self._show_error( - "Capture sequence is not complete", - details=( - f"Captured {session.state().captured_count}/{session.state().total_count} combos. " - "Finish the remaining captures before saving." - ), - ) - return - - dialog = self._ensure_preprocess_dialog() - try: - set_name = session.set_name - kind = session.kind - display_name = preprocess_asset_display_name(kind) - if isinstance(session, MultiRadarSequentialCaptureSession): - saved_sets = session.finalize(self._store) - else: - radar_key, collection = session.finalize(self._store) - self._cleanup_capture_session() - - self._selected_preprocess_sets[kind] = set_name - self._refresh_sets() - if isinstance(session, MultiRadarSequentialCaptureSession): - assert isinstance(saved_sets, list) - dialog.set_status( - f"{display_name} set saved: {set_name} ({len(saved_sets)} radar variants)" - ) - saved_summary = ", ".join( - f"{saved.display_name}:{saved.trace_count}" - for saved in saved_sets - ) - self._log( - f"{display_name} sequence completed and saved: set={set_name}, " - f"radar_variants={len(saved_sets)} [{saved_summary}]" - ) - self._export_multi_radar_preview_pngs( - session=session, - saved_sets=saved_sets, - kind=kind, - set_name=set_name, - ) - else: - dialog.set_status(f"{display_name} set saved: {set_name} ({len(collection.traces)} traces)") - self._log(f"{display_name} sequence completed and saved: set={set_name}, key={radar_key}") - self._resume_pipeline_if_needed() - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to save preprocess set", exc) - - def _export_multi_radar_preview_pngs( - self, - *, - session: MultiRadarSequentialCaptureSession, - saved_sets: list, - kind: str, - set_name: str, - ) -> None: - """Save amplitude/phase PNGs for every captured combo across all radar configs. - - The live preview only shows one trace per config, so this persists a graph for - every config and every combo (port) under the store's ``preview_png/`` tree. - Iterating per config over its full trace set (not the per-batch preview trace) - is what makes matrix radars save all ports instead of only the last one. The - set is already saved by the time we get here, so any rendering failure is - logged but never aborts the save. - """ - channel = preprocess_asset_channel(kind) - display_name = preprocess_asset_display_name(kind) - saved_count = 0 - for saved in saved_sets: - for trace in session.traces_for_radar_key(saved.radar_key): - combo_label = f"input={trace.combo.input} output={trace.combo.output}" - try: - png_path = self._store.preview_png_dir(kind, set_name, saved.radar_key) / ( - f"i{trace.combo.input}_o{trace.combo.output}.png" - ) - export_trace_png( - trace, - png_path, - channel=channel, - title=f"{display_name} | {saved.display_name} | {combo_label}", - ) - saved_count += 1 - except Exception as exc: # noqa: BLE001 - self._log( - f"Failed to save preview PNG for {saved.display_name} ({combo_label}): {exc}" - ) - if saved_count and saved_sets: - png_root = self._store.preview_png_dir(kind, set_name, saved_sets[0].radar_key).parent - self._log( - f"Saved {saved_count} preview PNG(s) for {len(saved_sets)} radar config(s) under {png_root}" - ) - - def _abort_capture_sequence(self, *, resume_pipeline: bool = True) -> None: - """Abort active capture session and optionally resume pipeline.""" - if self._capture_session is None: - return - - display_name = preprocess_asset_display_name(self._capture_session.kind) - self._cleanup_capture_session() - dialog = self._ensure_preprocess_dialog() - dialog.set_status(f"{display_name} sequence aborted") - self._log(f"{display_name} sequence aborted") - if resume_pipeline: - self._resume_pipeline_if_needed() - - def _update_capture_dialog_state(self) -> None: - """Sync dialog state widgets with active capture session.""" - if self._preprocess_dialog is None: - return - - if self._capture_session is None: - self._preprocess_dialog.set_capture_state( - kind=None, - captured_count=0, - total_count=0, - next_input=None, - next_output=None, - can_undo=False, - can_finalize=False, - can_capture_all=False, - variant_count=1, - ) - return - - state = self._capture_session.state() - next_input = None - next_output = None - if state.current_combo is not None: - next_input = state.current_combo.input - next_output = state.current_combo.output - - self._preprocess_dialog.set_capture_state( - kind=state.kind, - captured_count=state.captured_count, - total_count=state.total_count, - next_input=next_input, - next_output=next_output, - can_undo=state.can_undo, - can_finalize=state.is_complete, - can_capture_all=( - state.supports_batch_capture - and not state.is_complete - and state.current_combo is not None - ), - variant_count=state.variant_count, - # While a blocking capture is executing, every action stays disabled no - # matter what the session state allows: clicks queued during the freeze - # must land on disabled buttons (see `_end_preprocess_capture`). - actions_enabled=not self._preprocess_capture_busy, - ) - - def _cleanup_capture_session(self) -> None: - """Close and clear the current capture session object.""" - if self._capture_session is not None: - self._log_debug( - f"Closing capture session: kind={self._capture_session.kind}." - ) - self._capture_session.close() - self._capture_session = None - self._update_capture_dialog_state() - - def _resume_pipeline_if_needed(self) -> None: - """Resume acquisition pipeline if it was paused for capture session.""" - should_resume = self._resume_pipeline_after_capture - self._resume_pipeline_after_capture = False - if not should_resume: - return - - try: - self._start_run() - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to resume pipeline after capture", exc) diff --git a/python_app/gui/controllers/app_window_recording_mixin.py b/python_app/gui/controllers/app_window_recording_mixin.py deleted file mode 100644 index 9bba847..0000000 --- a/python_app/gui/controllers/app_window_recording_mixin.py +++ /dev/null @@ -1,332 +0,0 @@ -"""Record the next N runtime measurements to disk, in chunks, on a background thread. - -The manual "Save Dataset" button persists what is *already* in history (capped at the -GUI history limit). This mixin adds the complementary "Start + Record" action: arm a -recording, then stream the measurements that arrive *after* arming — up to the -configured count — to disk without ever blocking acquisition. - -Key properties: - -* **Flat memory.** Measurements are flushed in chunks of :data:`_RECORDING_CHUNK_SIZE` - and freed, and at most :data:`_RECORDING_QUEUE_CHUNKS` chunks are ever in flight, so - RAM does not grow with the recording length. -* **Off the GUI thread.** A single background writer thread does all disk I/O; the GUI - poll tick only hands it chunks. If - the disk genuinely cannot keep up, the bounded queue applies brief backpressure - rather than growing memory without limit. -* **Complete received stages.** Raw, preprocessed and result collections are saved - independently with their original timestamps, including sweeps without a result. -* **Only new sweeps.** Collections produced before the arm instant are ignored (gated - on ``monotonic_ns``), so a pre-existing ring backlog is not recorded. -* **No double-arm.** Arming is refused while a recording is still in progress. - -The on-disk layout is identical to "Save Dataset" (one streaming dataset directory). -""" - -from __future__ import annotations - -from contextlib import suppress -import logging -from pathlib import Path -import queue -import shutil -import threading -import time - -logger = logging.getLogger(__name__) - -# Flush to disk every this many results, then drop them from memory. -_RECORDING_CHUNK_SIZE = 200 -# Max chunks queued for the writer thread before the producer applies backpressure; -# bounds in-flight memory to roughly this many chunks of measurements. -_RECORDING_QUEUE_CHUNKS = 8 - - -class AppWindowRecordingMixin: - """Stream the next N arriving measurements to a snapshot dataset on a worker thread.""" - - def _init_recording_state(self) -> None: - """Initialise the (idle) disk-recording state. Called once from ``__init__``.""" - self._recording_active = False - self._recording_collecting = False - self._recording_target = 0 - self._recording_since_ns = 0 - self._recording_enqueued = 0 - self._recording_received_count = 0 - self._recording_writer = None - self._recording_queue: queue.Queue | None = None - self._recording_thread: threading.Thread | None = None - self._recording_stop = threading.Event() - self._recording_write_error: str | None = None - self._recording_chunk_raw: list = [] - self._recording_chunk_pre: list = [] - self._recording_chunk_results: list = [] - - # -- arming --------------------------------------------------------------- - - def _start_run_with_recording(self) -> None: - """Arm a streamed recording of the next N measurements, starting the run if stopped. - - ``N`` comes from the record-count field; the destination is the shared save - path/name. Refuses to start a second recording while one is in progress, and - reports a name clash up front (before any data is collected). - """ - if getattr(self, "_object_capture_job", None) is not None: - self._show_error("Wait for object-calibration capture to finish") - return - if self._recording_active: - self._show_error( - "A recording is already in progress; wait for it to finish before starting another", - details=f"progress={self._recording_collected_count()}/{self._recording_target}", - ) - return - - target = int(self._record_count.value()) - if target <= 0: - self._show_error("Recording count must be greater than zero") - return - - destination = self._snapshot_destination_dir() - if destination.exists(): - self._show_error( - "Recording destination already exists; choose a new name or path", - details=f"destination={destination}", - ) - return - - writer = None - try: - writer = self._store.create_snapshot_stream( - Path(self._save_path_input.text().strip()).expanduser(), - self._save_name_input.text().strip(), - name_prefix=self._radar_config_name_prefix(), - ) - # Adjacent config profile, mirroring the manual save (fail if it clashes). - self._write_gui_profile_to_path( - self._snapshot_config_profile_path(writer.directory), allow_overwrite=False, - dataset=self._supervisor.is_running(), - ) - except Exception as exc: # noqa: BLE001 - if writer is not None: - with suppress(OSError): - shutil.rmtree(writer.directory) - self._show_exception("Failed to start disk recording", exc) - return - - self._recording_writer = writer - self._recording_target = target - self._recording_since_ns = time.monotonic_ns() - self._recording_enqueued = 0 - self._recording_received_count = 0 - self._recording_write_error = None - self._recording_chunk_raw = [] - self._recording_chunk_pre = [] - self._recording_chunk_results = [] - self._recording_queue = queue.Queue(maxsize=_RECORDING_QUEUE_CHUNKS) - self._recording_stop = threading.Event() - try: - self._recording_thread = threading.Thread( - target=self._recording_writer_loop, - args=(writer, self._recording_queue, self._recording_stop), - name="disk-recorder", - daemon=False, - ) - self._recording_thread.start() - except Exception as exc: # noqa: BLE001 - thread resource exhaustion must reach the operator - self._init_recording_state() - with suppress(OSError): - shutil.rmtree(writer.directory) - self._show_exception("Failed to start disk recording", exc) - return - self._recording_active = True - self._recording_collecting = True - - if not self._supervisor.is_running(): - self._start_run() - if not self._supervisor.is_running(): - # Startup can fail after producing data. Flush it, and release the - # destination only when no collection was ever accepted. - self._finalize_recording_on_stop() - if not self._recording_active and not self._recording_received_count: - with suppress(OSError): - shutil.rmtree(writer.directory) - self._log("Disk recording canceled because acquisition did not start") - return - try: - self._write_gui_profile_to_path( - self._snapshot_config_profile_path(writer.directory), dataset=True, - ) - except Exception as exc: # noqa: BLE001 - keep any accepted data and the initial profile - self._finalize_recording_on_stop() - self._show_exception("Failed to update disk recording config profile", exc) - return - - self._log(f"Disk recording armed: streaming the next {target} measurement(s) to {writer.directory}") - - # -- background writer ---------------------------------------------------- - - def _recording_writer_loop( - self, writer, work_queue: queue.Queue, stop: threading.Event - ) -> None: - """Write queued chunks to disk until a sentinel, a stop request, or an error. - - Runs on a worker thread; touches only the writer (file I/O) and logging — never - Qt. A write failure is recorded for the GUI thread to surface and ends the loop. - """ - while not stop.is_set(): - try: - chunk = work_queue.get(timeout=0.2) - except queue.Empty: - continue - if chunk is None: # sentinel: target reached, all chunks drained - return - raw, preprocessed, results = chunk - try: - writer.append(raw, preprocessed, results) - except Exception as exc: # noqa: BLE001 - reported to the GUI thread - logger.exception("Disk recording chunk write failed") - self._recording_write_error = f"{type(exc).__name__}: {exc}" - return - - # -- capture (called from the ring-read hooks on the poll tick) ----------- - - def _record_collection(self, stage: str, collection) -> None: - """Save each received stage, retaining raw/preprocessed sweeps without a result.""" - if not self._recording_collecting: - return - produced_ns = int(getattr(collection, "monotonic_ns", 0) or 0) - if produced_ns and produced_ns < self._recording_since_ns: - return - - if stage == "raw": - self._recording_chunk_raw.append(collection) - elif stage == "preprocessed": - self._recording_chunk_pre.append(collection) - elif stage == "results": - if self._recording_collected_count() >= self._recording_target: - return - self._recording_chunk_results.append(collection) - else: - raise ValueError(f"Unknown recording stage: {stage}") - self._recording_received_count += 1 - - chunk_size = max(len(self._recording_chunk_raw), len(self._recording_chunk_pre), len(self._recording_chunk_results)) - if chunk_size >= _RECORDING_CHUNK_SIZE and not self._flush_recording_chunk(): - return - if self._recording_collected_count() >= self._recording_target: - if self._flush_recording_chunk(): - self._stop_collecting() - - def _enqueue_recording_work(self, work) -> bool: - """Apply bounded backpressure while checking that the consumer remains healthy.""" - while self._recording_thread is not None: - if self._recording_write_error is not None or not self._recording_thread.is_alive(): - self._abort_recording_after_write_error() - return False - try: - self._recording_queue.put(work, timeout=0.1) - return True - except queue.Full: - continue - return False - - def _flush_recording_chunk(self) -> bool: - """Queue all received stages, including a chunk containing no results yet.""" - if not (self._recording_chunk_raw or self._recording_chunk_pre or self._recording_chunk_results): - return True - chunk = (self._recording_chunk_raw, self._recording_chunk_pre, self._recording_chunk_results) - if not self._enqueue_recording_work(chunk): - return False - self._recording_enqueued += len(self._recording_chunk_results) - self._recording_chunk_raw = [] - self._recording_chunk_pre = [] - self._recording_chunk_results = [] - return True - - def _stop_collecting(self) -> None: - """Queue an exit sentinel after every accepted chunk, including a full queue.""" - self._recording_collecting = False - self._enqueue_recording_work(None) - - # -- completion / teardown (GUI thread) ----------------------------------- - - def _poll_recording_writer(self) -> None: - """Finalize a recording once the writer thread has drained (or failed). - - Called every GUI poll tick. Cheap no-op while idle or still writing. - """ - if not self._recording_active: - return - if self._recording_write_error is not None: - self._abort_recording_after_write_error() - return - if not self._recording_collecting and not self._recording_thread.is_alive(): - directory = self._recording_writer.directory if self._recording_writer is not None else "?" - written = self._recording_enqueued - self._reset_recording_state() - self._log(f"Disk recording complete: {written} measurement(s) written to {directory}") - - def _finalize_recording_on_stop(self) -> None: - """Flush accepted data on Stop and retain the writer until its queue is drained.""" - if not self._recording_active: - return - if self._recording_collecting: - if not self._flush_recording_chunk(): - return - self._stop_collecting() - thread = self._recording_thread - if thread is not None: - thread.join(timeout=3.0) - self._poll_recording_writer() - if self._recording_active: - self._log("Disk recording is finishing queued writes; the dataset is not complete yet") - - def _abort_recording_after_write_error(self) -> None: - """Surface a writer-thread failure on the GUI thread and disarm.""" - message = self._recording_write_error or "unknown error" - directory = self._recording_writer.directory if self._recording_writer is not None else "?" - self._reset_recording_state() - self._show_error( - "Disk recording failed and was stopped", - details=f"dataset={directory}\nerror={message}", - ) - - def _reset_recording_state(self) -> None: - """Disarm recording, stop the writer thread, and release all buffers.""" - self._recording_stop.set() - if self._recording_queue is not None: - with suppress(Exception): - self._recording_queue.put_nowait(None) - self._recording_active = False - self._recording_collecting = False - self._recording_writer = None - self._recording_queue = None - self._recording_thread = None - self._recording_write_error = None - self._recording_enqueued = 0 - self._recording_chunk_raw = [] - self._recording_chunk_pre = [] - self._recording_chunk_results = [] - - def _shutdown_recording(self) -> None: - """Flush on close; a non-daemon writer also drains after the GUI event loop exits.""" - self._finalize_recording_on_stop() - if self._recording_active: - # Do not reset a live writer: setting its stop flag would discard its queue. - # Python waits for this non-daemon thread during process exit. - return - self._reset_recording_state() - - # -- status --------------------------------------------------------------- - - def _recording_collected_count(self) -> int: - """Measurements accepted so far (handed to the writer + still buffered).""" - return self._recording_enqueued + len(self._recording_chunk_results) - - def _recording_status(self) -> dict: - """A compact snapshot of recording progress for the status feed / web UI.""" - return { - "active": self._recording_active, - "collected": self._recording_collected_count(), - "target": self._recording_target, - } diff --git a/python_app/gui/controllers/app_window_snapshot_mixin.py b/python_app/gui/controllers/app_window_snapshot_mixin.py deleted file mode 100644 index 2296483..0000000 --- a/python_app/gui/controllers/app_window_snapshot_mixin.py +++ /dev/null @@ -1,347 +0,0 @@ -"""Snapshot and ring-drain helper mixin for the main GUI window.""" - -from __future__ import annotations - -from pathlib import Path -import time - -from PyQt6.QtWidgets import QFileDialog -from python_app.gui.controllers.app_window_config.state_builders import GUI_SAVE_HISTORY_LIMIT -from python_app.gui.runtime.history import remove_last_aligned_histories -from python_app.storage.npz.paths import radar_config_filename_prefix - - -class AppWindowSnapshotMixin: - """Saves runtime data snapshots and maintains ring-reader freshness.""" - - def _radar_config_name_prefix(self) -> str: - """Filename prefix encoding the current radar setup (model, sweep span, points, power). - - Prepended to every saved dataset name so the file records the configuration it was - captured with — identical for desktop and web saves, which share these handlers. - """ - radar = self._build_config().radar - return radar_config_filename_prefix( - radar.model, - radar.sweep.start_hz, - radar.sweep.stop_hz, - radar.sweep.points, - radar.sweep.power_dbm, - ) - - @staticmethod - def _snapshot_config_profile_path(snapshot_dir: Path) -> Path: - """Return companion config-profile path inside a saved snapshot directory.""" - return snapshot_dir / "config_profile.json" - - @staticmethod - def _vna_json_config_profile_path(output_dir: Path) -> Path: - """Return companion config-profile path for one VNA-history JSON export batch.""" - return output_dir / "config_profile.json" - - def _save_snapshot(self) -> None: - """Save up to 1000 recent runtime measurements as a numpy snapshot.""" - if getattr(self, "_object_capture_job", None) is not None: - self._show_error("Wait for object-calibration capture to finish") - return - self._drain_runtime_rings_for_snapshot() - if getattr(self, "_closing", False): - return - - if not self._raw_history and not self._pre_history and not self._result_history: - self._show_error("No runtime data is available for save", details=self._runtime_history_details()) - return - - self._write_snapshot_dataset( - list(self._raw_history), - list(self._pre_history), - list(self._result_history), - GUI_SAVE_HISTORY_LIMIT, - include_unaligned=True, - ) - - def _snapshot_destination_dir(self) -> Path: - """The directory a save/record would create now, from the current path/name fields. - - Used to surface a name clash *before* doing work (manual save and the disk - recorder both create this directory and fail if it already exists). - """ - output_root = Path(self._save_path_input.text().strip()).expanduser() - return self._store.snapshot_directory( - output_root, - self._save_name_input.text().strip(), - name_prefix=self._radar_config_name_prefix(), - ) - - def _write_snapshot_dataset( - self, - raw_history: list, - preprocessed_history: list, - result_history: list, - last_n: int, - *, - include_unaligned: bool = False, - ) -> None: - """Save the given histories as a numpy snapshot + adjacent config profile. - - Shared by the manual "Save Last 1000" button (which passes the runtime history) - and the on-the-fly disk recorder (which passes the freshly recorded buffers), - so both produce byte-identical dataset layouts and identical error reporting. - """ - try: - output_root = Path(self._save_path_input.text().strip()).expanduser() - snapshot_name = self._save_name_input.text().strip() - snapshot_dir, summary = self._store.save_runtime_snapshot_numpy( - output_root, - snapshot_name, - raw_history, - preprocessed_history, - result_history, - last_n, - name_prefix=self._radar_config_name_prefix(), - include_unaligned=include_unaligned, - ) - config_profile_path = self._snapshot_config_profile_path(snapshot_dir) - try: - self._write_gui_profile_to_path(config_profile_path, allow_overwrite=False, dataset=True) - except Exception as exc: # noqa: BLE001 - self._show_error( - "Snapshot data was saved, but the adjacent config profile could not be written", - details=( - f"snapshot_dir={snapshot_dir}\n" - f"config_profile_path={config_profile_path}\n\n" - f"{self._exception_details(exc)}" - ), - ) - return - self._log( - f"Saved numpy snapshot: {snapshot_dir} " - f"(raw={summary.get('raw_count', 0)}, " - f"preprocessed={summary.get('preprocessed_count', 0)}, " - f"results={summary.get('result_count', 0)}, " - f"mode={summary.get('selection_mode', 'unknown')}, " - f"anchor={summary.get('anchor_stage', 'unknown')}, " - f"aligned_keys={summary.get('aligned_key_count', 0)}, " - f"raw_missing={summary.get('raw_missing_count', 0)}, " - f"pre_missing={summary.get('preprocessed_missing_count', 0)}, " - f"result_missing={summary.get('result_missing_count', 0)}, " - f"requested_last_n={last_n}, " - f"config_profile={config_profile_path})" - ) - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to save snapshot", exc) - - def _save_vna_history_json(self) -> None: - """Save one VNA-history JSON per available combo in runtime history.""" - if getattr(self, "_object_capture_job", None) is not None: - self._show_error("Wait for object-calibration capture to finish") - return - self._drain_runtime_rings_for_snapshot() - if getattr(self, "_closing", False): - return - - if not self._raw_history and not self._pre_history and not self._result_history: - self._show_error("No runtime data is available for save", details=self._runtime_history_details()) - return - - try: - last_n = GUI_SAVE_HISTORY_LIMIT - channel = "s21" - output_root = Path(self._save_path_input.text().strip()).expanduser() - output_name = self._save_name_input.text().strip() - output_paths, summary = self._store.save_runtime_vna_history_json_batch( - output_root, - output_name, - list(self._raw_history), - list(self._pre_history), - list(self._result_history), - last_n, - channel=channel, - primary_stage="preprocessed", - name_prefix=self._radar_config_name_prefix(), - include_unaligned=True, - ) - output_stem = str(summary.get("output_stem", "")).strip() - if not output_stem: - raise RuntimeError("VNA history JSON export did not report output_stem for config companion save") - output_dir = Path(str(summary.get("output_dir", "")).strip() or output_paths[0].parent) - - config_profile_path = self._vna_json_config_profile_path(output_dir) - try: - # Snapshot and VNA JSON exports intentionally share the same output directory stem. - # Refresh the companion GUI profile in place so a prior dataset save does not block JSON export. - self._write_gui_profile_to_path(config_profile_path, allow_overwrite=True, dataset=True) - except Exception as exc: # noqa: BLE001 - exported_preview = "\n".join(str(path) for path in output_paths[:8]) - if len(output_paths) > 8: - exported_preview += "\n..." - self._show_error( - "VNA history JSON files were saved, but the adjacent config profile could not be written", - details=( - f"output_root={output_root}\n" - f"output_dir={output_dir}\n" - f"config_profile_path={config_profile_path}\n" - f"saved_json_files={len(output_paths)}\n" - f"{exported_preview}\n\n" - f"{self._exception_details(exc)}" - ), - ) - return - combos = summary.get("combos", []) - combo_preview = ", ".join(f"in{input_pos}/out{output_pos}" for input_pos, output_pos in combos[:6]) - if len(combos) > 6: - combo_preview += ", ..." - self._log( - f"Saved VNA history JSON batch: files={len(output_paths)} " - f"(combos={summary.get('combo_count', 0)}" - f"{', ' + combo_preview if combo_preview else ''}, " - f"preprocessed_records={summary.get('preprocessed_record_count', 0)}, " - f"raw={summary.get('raw_count', 0)}, " - f"preprocessed={summary.get('preprocessed_count', 0)}, " - f"results={summary.get('result_count', 0)}, " - f"mode={summary.get('selection_mode', 'unknown')}, " - f"anchor={summary.get('anchor_stage', 'unknown')}, " - f"channel={channel}, " - f"requested_last_n={last_n}, " - f"config_profile={config_profile_path})" - ) - except Exception as exc: # noqa: BLE001 - self._show_exception("Failed to save VNA history JSON", exc) - - def _remove_last_runtime_history(self) -> None: - """Remove the newest runtime measurement from all stages and processor replay state.""" - self._apply_runtime_history_deletion(remove_last_only=True) - - def _clear_all_runtime_history(self) -> None: - """Clear all runtime histories, ring backlogs, and processor replay state.""" - self._apply_runtime_history_deletion(remove_last_only=False) - - def _apply_runtime_history_deletion(self, *, remove_last_only: bool) -> None: - """Apply destructive runtime-history deletion across readers, caches, and processor replay state.""" - if getattr(self, "_object_capture_job", None) is not None: - self._show_error("Wait for object-calibration capture to finish") - return - if self._capture_session is not None: - self._show_error( - "Cannot modify runtime history during active capture sequence", - details=f"{self._capture_state_details()}\n\n{self._runtime_history_details()}", - ) - return - - resume_acquisition = self._supervisor.is_running() - resume_seq = getattr(self, "_stop_request_seq", 0) + 1 - dropped_raw = 0 - dropped_pre = 0 - dropped_results = 0 - history_command = "remove_last" if remove_last_only else "clear_all" - action_label = "last runtime measurement removed" if remove_last_only else "runtime history fully cleared" - error_action = "remove last runtime measurement" if remove_last_only else "clear runtime history" - - try: - if resume_acquisition: - self._stop_run() - if not self._pipeline_resume_allowed(resume_seq): - return - - dropped_raw = self._raw_reader.drop_all() if self._raw_reader is not None else 0 - dropped_pre = self._pre_reader.drop_all() if self._pre_reader is not None else 0 - dropped_results = self._result_reader.drop_all() if self._result_reader is not None else 0 - - if remove_last_only: - retained_raw, retained_pre, retained_result = remove_last_aligned_histories( - list(self._raw_history), - list(self._pre_history), - list(self._result_history), - ) - else: - retained_raw = [] - retained_pre = [] - retained_result = [] - - self._replace_runtime_history( - retained_raw=retained_raw, - retained_pre=retained_pre, - retained_result=retained_result, - ) - self._discard_bscan_replay_results() - self._clear_history_mode_caches() - - self._write_live_processing_config(history_command=history_command, bump_history_seq=True) - - if self._supervisor.is_processor_running(): - self._drain_results_until_quiet(timeout_s=0.35, poll_s=0.01) - if self._result_reader is not None: - dropped_results += self._result_reader.drop_all() - - self._update_history_indicator() - self._redraw_after_history_deletion() - self._log( - f"{action_label}: " - f"dropped raw={dropped_raw}, " - f"preprocessed={dropped_pre}, " - f"results={dropped_results}" - ) - - if resume_acquisition and self._pipeline_resume_allowed(resume_seq): - self._start_run() - except Exception as exc: # noqa: BLE001 - self._show_exception(f"Failed to {error_action}", exc) - - def _browse_save_path(self) -> None: - """Open directory picker for snapshot output path.""" - selected = QFileDialog.getExistingDirectory( - self, - "Select Snapshot Directory", - self._save_path_input.text().strip() or str(self._project_root), - ) - if selected: - self._save_path_input.setText(selected) - - def _drain_runtime_rings_for_snapshot(self) -> None: - """Drain readers before snapshot to reduce partial-history races.""" - if self._raw_reader is None and self._pre_reader is None and self._result_reader is None: - return - try: - readers = (self._raw_reader, self._pre_reader, self._result_reader) - observed = [False, False, False] - deadline = time.monotonic() + 1.2 - - while time.monotonic() < deadline and not getattr(self, "_closing", False): - limit = self._max_pop_per_snapshot_drain - raw = self._read_all_raw(max_collections=limit) if self._raw_reader is not None else None - preprocessed = self._read_all_preprocessed(max_collections=limit) - result = self._read_all_results(max_collections=limit) if self._result_reader is not None else None - latest = (raw, preprocessed, result) - observed = [seen or collection is not None for seen, collection in zip(observed, latest)] - - if any(collection is not None for collection in latest): - continue - - # Event-loop-friendly waits so the keepalive/watchdog timers and - # queued signals keep firing instead of freezing the headless daemon. - missing_stage = any(reader is not None and not seen for reader, seen in zip(readers, observed)) - awaiting_results = self._result_reader is not None and ( - max(len(self._raw_history), len(self._pre_history)) > len(self._result_history) - ) - if (any(observed) and missing_stage) or awaiting_results: - self._pump_events_during_drain(0.01) - continue - break - except Exception as exc: # noqa: BLE001 - self._log_exception("Snapshot drain warning", exc, level="WARN") - - def _drop_pending_ring_payloads(self, *, include_results: bool = True) -> None: - """Drop unread payloads from active readers.""" - dropped_raw = self._raw_reader.drop_all() if self._raw_reader is not None else 0 - dropped_pre = self._pre_reader.drop_all() if self._pre_reader is not None else 0 - dropped_results = 0 - if include_results and self._result_reader is not None: - dropped_results = self._result_reader.drop_all() - - if dropped_raw or dropped_pre or dropped_results: - self._log( - "Single capture ring reset: " - f"dropped raw={dropped_raw}, " - f"preprocessed={dropped_pre}, " - f"results={dropped_results}" - ) diff --git a/python_app/gui/controllers/app_window_ui_mixin.py b/python_app/gui/controllers/app_window_ui_mixin.py deleted file mode 100644 index ad3baa4..0000000 --- a/python_app/gui/controllers/app_window_ui_mixin.py +++ /dev/null @@ -1,338 +0,0 @@ -"""UI construction mixin for the main radar control window. - -Layout is intentionally split into two independent plot surfaces: -- single `PlotWidget` for B-scan heatmap rendering; -- single `PlotWidget` for GPR accumulator/annotation rendering; -- stacked magnitude/phase `PlotWidget`s for pass-through traces. - -`_set_plot_mode()` switches between these surfaces via `QStackedWidget`. -""" - -from __future__ import annotations - -from PyQt6.QtCore import Qt -from PyQt6.QtWidgets import ( - QComboBox, - QFrame, - QGridLayout, - QGroupBox, - QHBoxLayout, - QLabel, - QPushButton, - QScrollArea, - QStackedWidget, - QTextEdit, - QVBoxLayout, - QWidget, -) -import pyqtgraph as pg - -from python_app.gui.controllers.app_window_plot.sar3d_plot_mixin import SAR3D_VIEWS -from python_app.gui.controllers.sections import ( - build_data_actions_group, - build_primary_actions_group, - build_preprocess_summary_group, - build_processing_group, - build_radar_group, - build_switch_group, -) -from python_app.logging_setup import LOG_LEVELS - - -class AppWindowUiMixin: - """Build and wire all static widgets used by :class:`AppWindow`. - - This mixin only creates UI objects and connects lightweight UI signals. - Runtime logic (pipeline start/stop, polling, drawing, snapshot saving) - is implemented in other mixins. - """ - - def _build_ui(self) -> None: - """Build complete main-window layout. - - High-level structure: - 1. Left: plot stack (`B-scan` page + `trace magnitude/phase` page). - 2. Middle: narrow toggle button to collapse/expand settings. - 3. Right: scrollable settings panel + status/history + runtime log. - """ - self.setWindowTitle("Radar System Control") - root = QWidget(self) - self.setCentralWidget(root) - - layout = QHBoxLayout(root) - layout.setContentsMargins(12, 12, 12, 12) - layout.setSpacing(14) - - self._build_plot_area(layout, root) - self._build_settings_toggle(layout) - self._build_settings_panel(layout, root) - - self._toggle_settings_panel(visible=True) - self.resize(1650, 940) - - def _build_plot_area(self, root_layout: QHBoxLayout, root: QWidget) -> None: - """Build left plot area with stacked B-scan and trace pages.""" - # Plot stack holds mutually exclusive visualization surfaces. - # We create both upfront and only switch active page at runtime. - self._plot_stack = QStackedWidget(root) - self._build_bscan_plot_page() - self._build_gpr_plot_page() - self._build_sar3d_plot_page() - self._build_trace_plot_page() - - # Default view on startup is pass-through traces. - self._plot_stack.setCurrentWidget(self._trace_plots_container) - root_layout.addWidget(self._plot_stack, stretch=11) - - @staticmethod - def _create_plot_widget(*, background: str) -> pg.PlotWidget: - """Create PlotWidget with pyqtgraph context menu disabled for PyQt6 compatibility.""" - return pg.PlotWidget(background=background, enableMenu=False) - - def _build_bscan_plot_page(self) -> None: - """Create B-scan page in plot stack.""" - # B-scan surface: one PlotWidget used as canvas for ImageItem heatmap. - self._bscan_plot = self._create_plot_widget(background="#0f141c") - self._bscan_plot.showGrid(x=True, y=True, alpha=0.2) - self._configure_bscan_plot_axes() - self._plot_stack.addWidget(self._bscan_plot) - - def _build_trace_plot_page(self) -> None: - """Create pass-through page with magnitude and phase plots.""" - # Pass-through surface: container with two synchronized line plots. - # Upper plot shows magnitude, lower plot shows phase. - self._trace_plots_container = QWidget() - trace_layout = QVBoxLayout(self._trace_plots_container) - trace_layout.setContentsMargins(0, 0, 0, 0) - trace_layout.setSpacing(6) - - self._trace_magnitude_plot = self._create_plot_widget(background="#0f141c") - self._trace_magnitude_plot.showGrid(x=True, y=True, alpha=0.2) - self._trace_magnitude_plot.setLabel("left", "Magnitude", units="dB") - self._trace_magnitude_plot.getPlotItem().showAxis("bottom", show=False) - self._trace_magnitude_plot.getPlotItem().setDownsampling(mode="peak") - self._trace_magnitude_plot.getPlotItem().setClipToView(True) - trace_layout.addWidget(self._trace_magnitude_plot, stretch=1) - - self._trace_phase_plot = self._create_plot_widget(background="#0f141c") - self._trace_phase_plot.showGrid(x=True, y=True, alpha=0.2) - self._trace_phase_plot.setLabel("left", "Phase", units="deg") - self._trace_phase_plot.setLabel("bottom", "Frequency", units="Hz") - self._trace_phase_plot.getPlotItem().setDownsampling(mode="peak") - self._trace_phase_plot.getPlotItem().setClipToView(True) - trace_layout.addWidget(self._trace_phase_plot, stretch=1) - - # Legends are created lazily only when trace mode draws line series. - self._trace_magnitude_legend = None - self._trace_phase_legend = None - self._trace_magnitude_legend_combo_keys = set() - self._trace_phase_legend_combo_keys = set() - self._trace_magnitude_curves = {} - self._trace_phase_curves = {} - # Cap rendered points per trace (magnitude and phase alike): beyond ~1 - # point/pixel there is no visual gain, and decimation keeps pass-through - # responsive with many combos on screen. - self._trace_render_max_points = 800 - - self._plot_stack.addWidget(self._trace_plots_container) - - def _build_gpr_plot_page(self) -> None: - """Create GPR page in plot stack.""" - self._gpr_plot = self._create_plot_widget(background="#0f141c") - self._gpr_plot.showGrid(x=True, y=True, alpha=0.2) - self._configure_gpr_plot_axes() - self._plot_stack.addWidget(self._gpr_plot) - - def _build_sar3d_plot_page(self) -> None: - """Create the 3D SAR page: front, side, top and 3D views in a 2x2 grid.""" - self._sar3d_plots_container = QWidget() - layout = QVBoxLayout(self._sar3d_plots_container) - layout.setContentsMargins(0, 0, 0, 0) - layout.setSpacing(6) - self._sar3d_status_label = QLabel() - self._sar3d_status_label.setTextFormat(Qt.TextFormat.PlainText) - self._sar3d_status_label.setWordWrap(True) - layout.addWidget(self._sar3d_status_label) - self._sar3d_plot_grid = QWidget() - grid = QGridLayout(self._sar3d_plot_grid) - grid.setContentsMargins(0, 0, 0, 0) - grid.setSpacing(6) - layout.addWidget(self._sar3d_plot_grid, stretch=1) - self._sar3d_plots = {} - for index, name in enumerate(SAR3D_VIEWS): - self._sar3d_plots[name] = self._create_plot_widget(background="#0f141c") - grid.addWidget(self._sar3d_plots[name], index // 2, index % 2) - self._init_sar3d_views() - self._clear_sar3d_plot() - self._plot_stack.addWidget(self._sar3d_plots_container) - - def _build_settings_toggle(self, root_layout: QHBoxLayout) -> None: - """Create narrow button used to collapse or show settings panel.""" - self._settings_toggle_button = QPushButton("<") - self._settings_toggle_button.setObjectName("settingsToggleButton") - self._settings_toggle_button.setFixedWidth(26) - self._settings_toggle_button.clicked.connect(lambda: self._toggle_settings_panel()) - root_layout.addWidget(self._settings_toggle_button, stretch=0) - - def _build_settings_panel(self, root_layout: QHBoxLayout, root: QWidget) -> None: - """Build right settings panel with controls, history summary, and log.""" - self._settings_panel = QWidget(root) - self._settings_panel.setMinimumWidth(610) - right_layout = QVBoxLayout(self._settings_panel) - right_layout.setContentsMargins(0, 0, 0, 0) - right_layout.setSpacing(10) - - # Build log early so `_show_error()` can append text even during - # subsequent group construction if something fails. - self._log_panel = self._build_log_panel() - - right_layout.addWidget(build_primary_actions_group(self), stretch=0) - right_layout.addWidget(self._build_settings_scroll(), stretch=1) - - self._status_label = QLabel("Status: idle", self._settings_panel) - self._status_label.setObjectName("statusLabel") - right_layout.addWidget(self._status_label) - - self._history_label = QLabel("History: raw=0, preprocessed=0, results=0", self._settings_panel) - self._history_label.setObjectName("hintLabel") - right_layout.addWidget(self._history_label) - - right_layout.addWidget(self._log_panel, stretch=0) - root_layout.addWidget(self._settings_panel, stretch=7) - - def _build_log_panel(self) -> QWidget: - """Build runtime log block with collapsible log body.""" - self._log_panel_title = QLabel("Runtime Log", self._settings_panel) - self._log_panel_title.setObjectName("hintLabel") - - self._log_toggle_button = QPushButton(self._settings_panel) - self._log_toggle_button.setObjectName("sectionToggleButton") - self._log_toggle_button.clicked.connect(lambda: self._toggle_log_panel()) - - # Log-level selector: live verbosity control, persisted to run_config. - self._log_level_label = QLabel("Level", self._settings_panel) - self._log_level_label.setObjectName("hintLabel") - self._log_level_combo = QComboBox(self._settings_panel) - self._log_level_combo.setObjectName("logLevelCombo") - self._log_level_combo.addItems([name.capitalize() for name in LOG_LEVELS]) - self._log_level_combo.setToolTip("Logging verbosity — applied live and saved to run_config") - current_level = self._defaults_config.logging.level.capitalize() - current_index = self._log_level_combo.findText(current_level) - if current_index >= 0: - self._log_level_combo.setCurrentIndex(current_index) - # Connect AFTER seeding the value so reflecting the config does not save it back. - self._log_level_combo.currentTextChanged.connect(self._on_log_level_selected) - - self._log_box = QTextEdit(self._settings_panel) - self._log_box.setObjectName("runtimeLogBox") - self._log_box.setReadOnly(True) - self._log_box.setUndoRedoEnabled(False) - self._log_box.setMinimumHeight(170) - self._log_box.document().setMaximumBlockCount(1200) - - panel = QWidget(self._settings_panel) - panel_layout = QVBoxLayout(panel) - panel_layout.setContentsMargins(0, 0, 0, 0) - panel_layout.setSpacing(6) - - header_row = QWidget(panel) - header_layout = QHBoxLayout(header_row) - header_layout.setContentsMargins(0, 0, 0, 0) - header_layout.setSpacing(8) - header_layout.addWidget(self._log_panel_title) - header_layout.addStretch(1) - header_layout.addWidget(self._log_level_label) - header_layout.addWidget(self._log_level_combo) - header_layout.addWidget(self._log_toggle_button) - - panel_layout.addWidget(header_row) - panel_layout.addWidget(self._log_box) - - self._toggle_log_panel(visible=True) - return panel - - def _build_settings_scroll(self) -> QScrollArea: - """Build scroll area with all control groups in display order.""" - # All control groups are placed into a scroll area so right panel - # remains usable on smaller screens and with future extra controls. - controls = QWidget(self._settings_panel) - controls_layout = QVBoxLayout(controls) - controls_layout.setContentsMargins(0, 0, 12, 0) - controls_layout.setSpacing(10) - for group in self._build_control_groups(): - controls_layout.addWidget(group) - controls_layout.addStretch(1) - - scroll = QScrollArea(self._settings_panel) - scroll.setWidgetResizable(True) - scroll.setFrameShape(QFrame.Shape.NoFrame) - scroll.setWidget(controls) - return scroll - - def _build_control_groups(self) -> list[QGroupBox]: - """Create all settings groups in top-to-bottom order.""" - return [ - build_data_actions_group(self), - build_switch_group(self), - build_preprocess_summary_group(self), - build_processing_group(self), - build_radar_group(self), - ] - - def _toggle_settings_panel(self, *, visible: bool | None = None) -> None: - """Toggle settings panel visibility or force a specific state. - - When `visible` is `None`, state is toggled. - When `visible` is set, panel visibility is forced to that value. - """ - if visible is None: - visible = not self._settings_panel.isVisible() - self._settings_panel.setVisible(visible) - if visible: - self._settings_toggle_button.setText(">") - self._settings_toggle_button.setToolTip("Hide settings panel") - else: - self._settings_toggle_button.setText("<") - self._settings_toggle_button.setToolTip("Show settings panel") - - def _toggle_log_panel(self, *, visible: bool | None = None) -> None: - """Toggle runtime log body visibility or force a specific state.""" - if visible is None: - visible = not self._log_box.isVisible() - - self._log_box.setVisible(visible) - if visible: - self._log_toggle_button.setText("Collapse") - self._log_toggle_button.setToolTip("Hide runtime log entries") - else: - self._log_toggle_button.setText("Expand") - self._log_toggle_button.setToolTip("Show runtime log entries") - - def _set_plot_mode(self, mode: str) -> None: - """Switch visible plot page according to processing mode. - - `bscan` -> show `self._bscan_plot` - `gpr` and `legacy_gpr` -> show `self._gpr_plot` - `sar3d` -> show `self._sar3d_plots_container` - otherwise -> show `self._trace_plots_container` - """ - if mode == "bscan": - self._plot_stack.setCurrentWidget(self._bscan_plot) - return - if mode in {"gpr", "legacy_gpr"}: - self._plot_stack.setCurrentWidget(self._gpr_plot) - return - if mode == "sar3d": - self._plot_stack.setCurrentWidget(self._sar3d_plots_container) - return - self._plot_stack.setCurrentWidget(self._trace_plots_container) - - @staticmethod - def _set_combo_current_text(combo: QComboBox, value: str) -> None: - """Select combo item by text, appending it when missing.""" - index = combo.findText(value) - if index >= 0: - combo.setCurrentIndex(index) - return - combo.addItem(value) - combo.setCurrentIndex(combo.count() - 1) diff --git a/python_app/gui/controllers/app_window_web_mixin.py b/python_app/gui/controllers/app_window_web_mixin.py deleted file mode 100644 index c088ec6..0000000 --- a/python_app/gui/controllers/app_window_web_mixin.py +++ /dev/null @@ -1,503 +0,0 @@ -"""Embed the browser UI inside the AppWindow without duplicating any rendering. - -The desktop already owns the hardware and draws every plot with pyqtgraph, so the -web view simply streams a snapshot of the *current Qt plot widget* — the browser -shows exactly what the desktop shows, for every processing mode, with zero -re-implemented rendering. Controls cross back to the Qt main thread through queued -signals (the GPIO-button pattern) and invoke the AppWindow's existing buttons. - -Two roles, kept separate from the Qt-free :mod:`python_app.webui` package: - -* :class:`AppWindowWebController` — the Qt bridge implementing the web contract. -* :class:`AppWindowWebMixin` — wiring that grabs the plot, refreshes snapshots, - and starts/stops the server. -""" - -from __future__ import annotations - -import base64 -import contextlib -import os -import threading -import time -from pathlib import Path - -from PyQt6.QtCore import QBuffer, QIODevice, QObject, pyqtSignal - -from python_app.gui.controllers.app_window_config.live_processing_mixin import web_apply_field_names -from python_app.webui.controller import WebActionError - -_WEBUI_PORT_ENV = "RADAR_SYSTEM_WEBUI_PORT" -_DEFAULT_PORT = 8080 -# Upper bound on how long a browser control call waits for the GUI thread to run and -# report the action. Comfortably above a real save/start, but bounded so a wedged GUI -# thread surfaces as an error instead of hanging the HTTP worker forever. -_WEB_ACTION_TIMEOUT_S = 30.0 - - -class _WebActionCall: - """One synchronous web action: the GUI thread fills the result, the web thread waits. - - The web (uvicorn) thread emits a control signal carrying this object and blocks on - :attr:`done`; the GUI thread runs the desktop action, records any surfaced error in - :attr:`error`, and sets the event. This turns the fire-and-forget signal bridge into - a request/response so failures reach the browser. - """ - - __slots__ = ("done", "error", "_lock", "_started") - - def __init__(self) -> None: - self.done = threading.Event() - self.error: str | None = None - self._lock = threading.Lock() - self._started = False - - def begin(self) -> bool: - with self._lock: - if self.done.is_set() or self._started: - return False - self._started = True - return True - - def finish(self, error: str | None = None) -> None: - with self._lock: - if not self.done.is_set(): - self.error = error - self.done.set() - - def cancel(self, error: str, *, queued_only: bool = True) -> bool: - with self._lock: - if self.done.is_set(): - return True - if queued_only and self._started: - return False - self.error = error - self.done.set() - return True -# Headless has no shown window, so give the offscreen window a usable size for the -# grabbed plot. In GUI mode the user's real (shown) window size is used as-is. -_HEADLESS_PLOT_SIZE = (1600, 900) -# Grab/encode the plot at most this often (the plot only changes per result/redraw). -_GRAB_INTERVAL_S = 0.2 -# Field names a web client may apply: the form's live/display fields plus the -# history command. Derived from the single schema so it can never drift from the form. -_LIVE_FIELD_NAMES = web_apply_field_names() - - -def _safe_run_config_path(run_configs_dir: Path, name: str) -> Path | None: - """Resolve a client-supplied config name to a file strictly inside ``run_configs_dir``. - - The web sends only a bare file name; this is the trust boundary. The name must be a - plain ``.json`` file living directly in the directory — ``name != Path(name).name`` - rejects any separator or ``..`` traversal. Returns the path, or ``None`` if invalid. - """ - if not name or name != Path(name).name or not name.lower().endswith(".json"): - return None - candidate = run_configs_dir / name - return candidate if candidate.is_file() else None - - -def _list_run_config_names(run_configs_dir: Path) -> list[str]: - """Return the sorted ``.json`` file names directly inside ``run_configs_dir``.""" - if not run_configs_dir.is_dir(): - return [] - return sorted(entry.name for entry in run_configs_dir.glob("*.json") if entry.is_file()) - - -class AppWindowWebController(QObject): - """Qt bridge satisfying the web contract: streams the plot, forwards controls. - - Mutating calls (made on the web thread) emit queued signals the AppWindow - connects to its existing slots. Read calls return immutable snapshots the - AppWindow refreshes on its poll tick — replaced atomically, never mutated in - place, so the web thread reads a consistent value without locking. - """ - - # Control signals carry a trailing _WebActionCall the GUI slot fills in, so the web - # thread can block on the real outcome and receive the updated state. - start_requested = pyqtSignal(object) - stop_requested = pyqtSignal(object) - remove_last_requested = pyqtSignal(object) - single_capture_requested = pyqtSignal(object) - capture_requested = pyqtSignal(object) - start_recording_requested = pyqtSignal(str, str, int, object) - load_config_requested = pyqtSignal(str, object) - save_dataset_requested = pyqtSignal(str, str, object) - select_combo_requested = pyqtSignal(int, int, object) - next_combo_requested = pyqtSignal(object) - apply_settings_requested = pyqtSignal(dict, object) - - def __init__(self, run_configs_dir: Path, parent: QObject | None = None) -> None: - super().__init__(parent) - self._run_configs_dir = run_configs_dir - self._status: dict = {} - self._live_settings: list = [] - self._frame: dict | None = None - self._pending_lock = threading.Lock() - self._pending_calls: set[_WebActionCall] = set() - self._closed = False - - # -- Snapshot refresh (Qt main thread) ----------------------------------- - - def update_snapshot(self, *, status: dict, live_settings: list, frame: dict | None) -> None: - """Replace the served snapshots; ``frame`` only when a new one was grabbed.""" - self._status = status - self._live_settings = live_settings - if frame is not None: - self._frame = frame - - # -- WebController reads (web thread) ------------------------------------ - - def status(self) -> dict: - return dict(self._status) - - def current_live_settings(self) -> list: - return list(self._live_settings) - - def list_configs(self) -> list[str]: - """Return the run-config file names available to load (a plain directory scan).""" - return _list_run_config_names(self._run_configs_dir) - - def peek_frame(self) -> dict | None: - """Return the most recent rendered-plot frame (or None before the first).""" - return self._frame - - # -- WebController controls (web thread -> Qt main thread) --------------- - - def _dispatch(self, signal, *args) -> None: - """Emit a control signal and block until the GUI thread reports the outcome. - - Runs on the web worker thread (the routes call this via a thread pool, so the - event loop is never blocked). Raises :class:`WebActionError` if the desktop - action surfaced an error or did not finish within the timeout. - """ - call = _WebActionCall() - with self._pending_lock: - if self._closed: - raise WebActionError("The application is closing") - self._pending_calls.add(call) - try: - signal.emit(*args, call) - if not call.done.wait(_WEB_ACTION_TIMEOUT_S): - if not call.cancel("The desktop did not start the action in time; the request was cancelled"): - raise WebActionError("The desktop action is still running; check its status before retrying") - if call.error is not None: - raise WebActionError(call.error) - finally: - with self._pending_lock: - self._pending_calls.discard(call) - - def shutdown(self) -> None: - """Release waiting HTTP workers and reject queued controls during teardown.""" - with self._pending_lock: - self._closed = True - pending = tuple(self._pending_calls) - for call in pending: - call.cancel("The application is closing", queued_only=False) - - def start(self) -> None: - self._dispatch(self.start_requested) - - def stop(self) -> None: - self._dispatch(self.stop_requested) - - def single_capture(self) -> None: - self._dispatch(self.single_capture_requested) - - def capture_tmp_reference(self) -> None: - self._dispatch(self.capture_requested) - - def remove_last_measurement(self) -> None: - self._dispatch(self.remove_last_requested) - - def start_recording(self, path: str, name: str, count: int) -> None: - """Arm a run + disk recording of the next ``count`` measurements (the desktop button).""" - if type(count) is not int or count < 1: - raise ValueError("Recording count must be a positive integer") - self._dispatch(self.start_recording_requested, path, name, count) - - def apply_live_settings(self, fields: dict) -> list: - unknown = set(fields) - _LIVE_FIELD_NAMES - if unknown: - raise ValueError(f"Unknown live-settings fields: {', '.join(sorted(unknown))}") - self._dispatch(self.apply_settings_requested, dict(fields)) - return self.current_live_settings() - - def load_config(self, name: str) -> None: - """Request loading the run-config named ``name`` (the desktop "Load Config" action). - - Validates the name against the directory here — on the web thread — so an invalid - or unsafe name fails the HTTP request immediately; the load itself then runs - synchronously on the Qt side and any load error is surfaced too. - """ - if _safe_run_config_path(self._run_configs_dir, name) is None: - raise ValueError(f"Unknown run config: {name}") - self._dispatch(self.load_config_requested, name) - - def save_dataset(self, path: str, name: str) -> None: - """Save the last 1000 runtime measurements to ``path``/``name``.""" - self._dispatch(self.save_dataset_requested, path, name) - - def select_combo(self, input_pos: int, output_pos: int) -> None: - """Select the matrix B-scan channel through the desktop display controls.""" - if any(type(pos) is not int or pos < 0 for pos in (input_pos, output_pos)): - raise ValueError("Channel positions must be nonnegative integers") - self._dispatch(self.select_combo_requested, input_pos, output_pos) - - def next_combo(self) -> None: - """Advance the matrix B-scan display to the next configured channel pair.""" - self._dispatch(self.next_combo_requested) - - -class AppWindowWebMixin: - """Start/stop the embedded web server and feed it the live plot + settings.""" - - def _init_web_ui(self) -> None: - """Start the web server (default-on, both modes), wired to existing buttons.""" - self._web_controller: AppWindowWebController | None = None - self._web_server = None - self._web_frame_seq = 0 - self._web_last_png: str | None = None - self._web_last_grab_s = 0.0 - - try: - from python_app.webui.server import WebUiServer - - # Headless never shows the window; size it so the grabbed plot is usable. - if self._is_truthy_env("RADAR_SYSTEM_HEADLESS"): - self.resize(*_HEADLESS_PLOT_SIZE) - - # The web picker browses this directory; ensure it exists on fresh deploys. - self._run_configs_dir.mkdir(parents=True, exist_ok=True) - - # Capture slot for errors a web-triggered action surfaces (None = no web - # action in flight). Read default-safe by `_show_error`/`_show_exception`. - self._web_action_error_capture: list[str] | None = None - - controller = AppWindowWebController(self._run_configs_dir, parent=self) - # Each control signal carries a _WebActionCall the wrapper finalizes, so the - # browser learns whether the desktop action actually succeeded. - controller.start_requested.connect( - lambda call: self._run_web_action(call, self._start_run) - ) - controller.stop_requested.connect( - lambda call: self._run_web_action(call, self._stop_run) - ) - controller.single_capture_requested.connect( - lambda call: self._run_web_action(call, self._start_single_capture) - ) - controller.capture_requested.connect( - lambda call: self._run_web_action(call, self._capture_tmp_reference) - ) - controller.remove_last_requested.connect( - lambda call: self._run_web_action(call, self._remove_last_runtime_history) - ) - controller.start_recording_requested.connect( - lambda path, name, count, call: self._run_web_action( - call, self._start_web_recording, path, name, count - ) - ) - controller.load_config_requested.connect( - lambda name, call: self._run_web_action(call, self._load_web_config, name) - ) - controller.save_dataset_requested.connect( - lambda path, name, call: self._run_web_action(call, self._save_web_dataset, path, name) - ) - controller.select_combo_requested.connect( - lambda inp, out, call: self._run_web_action(call, self._select_display_combo, inp, out) - ) - controller.next_combo_requested.connect( - lambda call: self._run_web_action(call, self._select_next_combo) - ) - controller.apply_settings_requested.connect( - lambda fields, call: self._run_web_action(call, self._apply_web_live_settings, fields) - ) - - self._web_controller = controller - self._web_update_snapshot() # seed snapshots before the first request - - port = self._web_ui_port() - self._web_server = WebUiServer(controller, port=port) - self._web_server.start() - self._log(f"Web UI started on http://0.0.0.0:{port}") - except Exception as exc: # noqa: BLE001 - a missing dependency must not abort startup - self._log_exception("Failed to start web UI", exc, level="WARN") - self._web_controller = None - self._web_server = None - - def _run_web_action(self, call: _WebActionCall, action, *args) -> None: - """Run a web-triggered desktop action on the GUI thread, capturing its outcome. - - Errors the action reports through ``_show_error``/``_show_exception`` are - captured into the call (and still logged/shown on the desktop) instead of - vanishing from the browser's view. Re-entrancy-safe: a nested action — e.g. a - modal error dialog pumping the event loop in GUI mode — saves and restores the - capture slot, so each action only sees its own first error. - """ - if getattr(self, "_closing", False): - call.cancel("The application is closing", queued_only=False) - return - if not call.begin(): - return - previous_capture = self._web_action_error_capture - capture: list[str] = [] - self._web_action_error_capture = capture - error = None - try: - action(*args) - error = capture[0] if capture else None - self._web_update_snapshot() - except Exception as exc: # noqa: BLE001 - handlers self-report; this is a backstop - error = self._exception_summary(exc) - finally: - self._web_action_error_capture = previous_capture - call.finish(error) - - def _load_web_config(self, name: str) -> None: - """Load a run config chosen in the browser through the shared desktop load path. - - Runs on the Qt main thread (queued from the web thread). The name is re-resolved - against the directory here as the authoritative trust boundary before any file - access, then handed to the same loader the desktop "Load Config" button uses. - """ - config_path = _safe_run_config_path(self._run_configs_dir, name) - if config_path is None: - raise ValueError(f"Unknown run config: {name}") - self._load_config_from_path(config_path) - - def _save_web_dataset(self, path: str, name: str) -> None: - """Save the last 1000 measurements via the shared desktop handler. - - The web fields are remote editors of the shared save-path/name widgets: the name - is mirrored as-is (blank is valid — it yields a timestamped file), while a blank - path is ignored to avoid wiping the configured destination. The save itself — and - the radar-config filename prefix — is the unchanged desktop action. - """ - if path.strip(): - self._save_path_input.setText(path) - self._save_name_input.setText(name) - self._save_snapshot() - - def _start_web_recording(self, path: str, name: str, count: int) -> None: - """Arm disk recording from the browser via the same handler as the desktop button. - - The save path/name fields are mirrored exactly like ``_save_web_dataset`` (blank - path keeps the configured destination), the record count is applied to the shared - spinbox, then the unchanged desktop arming action runs. - """ - if type(count) is not int or not self._record_count.minimum() <= count <= self._record_count.maximum(): - raise ValueError( - f"Recording count must be an integer from {self._record_count.minimum()} " - f"to {self._record_count.maximum()}" - ) - if path.strip(): - self._save_path_input.setText(path) - self._save_name_input.setText(name) - self._record_count.setValue(count) - self._start_run_with_recording() - - def _web_update_snapshot(self) -> None: - """Refresh the snapshots the bridge serves (called on the Qt poll tick). - - Must NEVER raise: it runs inside the periodic render tick, where an escaping - exception would abort the Qt slot (qFatal) and kill the app. - """ - controller = getattr(self, "_web_controller", None) - if controller is None: - return - try: - controller.update_snapshot( - status={ - "running": self._supervisor.is_running(), - "processor_running": self._supervisor.is_processor_running(), - "ring_name": self._defaults_config.rings.results.name, - # File name of the currently loaded config profile, so the web picker - # can show what is active (the desktop tracks the full path). - "active_config": self._active_profile_path.name, - # Current save path/name, so the web fields can prefill the desktop values. - "save_path": self._save_path_input.text(), - "save_name": self._save_name_input.text(), - # Default record count + live disk-recording progress for the web UI. - "record_count": int(self._record_count.value()), - "recording": self._recording_status(), - # Per-stage capture counts, identical to the desktop history - # label (raw -> preprocessed -> results), mirrored to the browser. - "raw_count": len(self._raw_history), - "preprocessed_count": len(self._pre_history), - "result_count": len(self._result_history), - "sar3d_status": self._sar3d_status_label.text() - if self._processing_mode.currentText() == "sar3d" else "", - "display_combo": { - "enabled": self._is_matrix_radar_model_selected() - and self._processing_mode.currentText() == "bscan", - "input": self._single_combo_input.text(), - "output": self._single_combo_output.text(), - "options": self._available_display_combo_keys() - if self._is_matrix_radar_model_selected() else [], - }, - }, - live_settings=self._web_settings_schema(), - frame=self._web_grab_frame_if_due(), - ) - except Exception as exc: # noqa: BLE001 - the render loop must survive this - self._web_snapshot_errors = getattr(self, "_web_snapshot_errors", 0) + 1 - if self._web_snapshot_errors % 200 == 1: - self._log_exception("Web UI snapshot refresh failed", exc, level="WARN") - - def _web_grab_frame_if_due(self) -> dict | None: - """Grab the current plot as a PNG frame, throttled and change-gated.""" - now = time.monotonic() - if now - self._web_last_grab_s < _GRAB_INTERVAL_S: - return None - self._web_last_grab_s = now - png_b64 = self._grab_plot_png_b64() - if png_b64 is None or png_b64 == self._web_last_png: - return None # nothing rendered yet, or the plot is unchanged - self._web_last_png = png_b64 - self._web_frame_seq += 1 - return { - "type": "frame", - "seq": self._web_frame_seq, - "mode": self._processing_mode.currentText(), - "png_b64": png_b64, - } - - def _grab_plot_png_b64(self) -> str | None: - """Render the active plots; SAR3D status is sent separately as web text.""" - widget = self._plot_stack.currentWidget() - if widget is self._sar3d_plots_container: - widget = self._sar3d_plot_grid - if widget is None or widget.width() <= 0 or widget.height() <= 0: - return None - pixmap = widget.grab() - if pixmap.isNull(): - return None - buffer = QBuffer() - buffer.open(QIODevice.OpenModeFlag.WriteOnly) - pixmap.save(buffer, "PNG") - return base64.b64encode(bytes(buffer.data())).decode() - - def _shutdown_web_ui(self) -> None: - """Stop the server (and its broadcaster) during teardown.""" - controller = getattr(self, "_web_controller", None) - if controller is not None: - controller.shutdown() - server = getattr(self, "_web_server", None) - if server is not None: - with contextlib.suppress(Exception): - server.stop() - self._web_server = None - self._log("Web UI stopped.") - self._web_controller = None - - @staticmethod - def _web_ui_port() -> int: - """Resolve the web UI port from the environment, defaulting to 8080.""" - raw = os.environ.get(_WEBUI_PORT_ENV, "").strip() - if not raw: - return _DEFAULT_PORT - try: - port = int(raw) - except ValueError: - return _DEFAULT_PORT - return port if 1 <= port <= 65535 else _DEFAULT_PORT diff --git a/python_app/gui/controllers/bscan_replay.py b/python_app/gui/controllers/bscan_replay.py new file mode 100644 index 0000000..397ea03 --- /dev/null +++ b/python_app/gui/controllers/bscan_replay.py @@ -0,0 +1,197 @@ +"""Re-feed the GUI's retained sweeps to recompute the full stopped B-scan. + +The processor retains a shorter history than the GUI. Replay preserves acquisition +keys, bounds bursts by both ring capacities, and bypasses recording and live metrics. +Only the caller decides when the processor's input ring is safe to write. +""" + +from __future__ import annotations + +from collections.abc import Callable +import time +from typing import Protocol + +from PyQt6.QtCore import QTimer + +from python_app.gui.controllers.history_controller import HistoryController +from python_app.models.collection_identity import collection_key +from python_app.models.dataset_model import ResultCollection +from python_app.models.run_config_schema import RingEndpointModel +from python_app.orchestration.shm.decoder import PREPROC_MAGIC +from python_app.orchestration.shm.ring_reader import ShmRingReader +from python_app.orchestration.shm.ring_writer import ShmRingWriter +from python_app.storage.npz.serialize import serialize_trace_collection + +_REPLAY_CHUNK_SIZE = 12 +_REPLAY_CHUNK_TIMEOUT_S = 2.0 +_REPROCESS_DEBOUNCE_MS = 300 + + +class WarningSink(Protocol): + def __call__( + self, text: str, *, details: str | None = None, once_key: str | None = None, + ) -> None: ... + + +class BscanReplay: + def __init__( + self, history: HistoryController, on_timeout: Callable[[], None], + warning: WarningSink, debug: Callable[[str], None], + ) -> None: + self.history = history + self._warning = warning + self._debug = debug + self._writer: ShmRingWriter | None = None + self._generation = 0 + self.active = False + self.timer = QTimer() + self.timer.setSingleShot(True) + self.timer.timeout.connect(on_timeout) + + def schedule(self, enabled: bool) -> None: + if enabled: + if self.active: + self.discard() + self.timer.start(_REPROCESS_DEBOUNCE_MS) + + def run( + self, *, window: int, revision: int, ring: RingEndpointModel | None, + reader: ShmRingReader | None, pump_events: Callable[[float], None], + continue_replay: Callable[[], bool], + ) -> bool: + if self.active or reader is None: + return False + generation = self._generation + if not self._continues(generation, continue_replay): + return False + tail = list(self.history.preprocessed)[-window:] + if not tail: + return False + writer = self._ensure_writer(ring) + if writer is None or ring is None: + return False + + keys = [collection_key(collection) for collection in tail] + result_keys = {collection_key(collection) for collection in self.history.results} + self._debug( + f"B-scan replay starting: window={window}, preprocessed history=" + f"{len(self.history.preprocessed)}, result history={len(self.history.results)}, " + f"to re-send={len(tail)}, of those already in result history=" + f"{sum(key in result_keys for key in keys)}." + ) + replayed: dict[tuple[int, int], ResultCollection] = {} + replayed_keys = set(keys) + sent = 0 + chunk_size = min(_REPLAY_CHUNK_SIZE, ring.capacity, reader.capacity) + # The caller suspends result polling. Event pumping may still change settings + # or close the pipeline, so each boundary checks this session remains valid. + self.active = True + try: + for start in range(0, len(tail), chunk_size): + chunk = tail[start:start + chunk_size] + pushed = 0 + for collection in chunk: + if not self._continues(generation, continue_replay): + return False + payload = serialize_trace_collection(collection, PREPROC_MAGIC) + if not writer.push(payload): + self._warning( + "B-scan replay stopped: a preprocessed sweep does not fit the ring slot.", + details=f"payload={len(payload)} bytes, slot={writer.slot_size_bytes} bytes", + once_key="bscan_replay_payload_too_large", + ) + break + pushed += 1 + sent += pushed + waiting_for = keys[start:start + pushed] + complete = self._collect( + reader, waiting_for, replayed_keys, replayed, pump_events, generation, continue_replay, revision, + ) + if not self._continues(generation, continue_replay): + return False + if not complete: + self._debug(f"B-scan replay chunk at offset {start} came back incomplete.") + if pushed != len(chunk): + break + finally: + self.active = False + + # Tap and results rings drop independently: only these replayed keys belong + # in the refreshed image; unrelated runtime results may still use old settings. + self.history.replay_results[:] = [replayed[key] for key in keys if key in replayed] + received = sum(key in replayed for key in keys[:sent]) + if received < sent: + self._warning( + f"B-scan replay recovered {received} of {sent} re-sent sweeps; " + "some columns may still show their captured settings.", + once_key=f"bscan_replay_incomplete_{sent}_{received}", + ) + self._debug(f"B-scan replay finished: sent={sent}, recovered={received}.") + return True + + def _collect( + self, reader: ShmRingReader, waiting_for: list[tuple[int, int]], + replayed_keys: set[tuple[int, int]], into: dict[tuple[int, int], ResultCollection], + pump_events: Callable[[float], None], generation: int, continue_replay: Callable[[], bool], + revision: int, + ) -> bool: + deadline = time.monotonic() + _REPLAY_CHUNK_TIMEOUT_S + while any(key not in into for key in waiting_for) and time.monotonic() < deadline: + if not self._continues(generation, continue_replay): + return False + collection = reader.pop_result_collection() + if collection is None: + pump_events(0.005) + if not self._continues(generation, continue_replay): + return False + continue + if self.history.is_removed(collection): + continue + self.history.record_result(collection) + key = collection_key(collection) + # An old processor replay can return this exact sweep before the fresh + # result. Its identity alone does not acknowledge the current request. + if key in replayed_keys and collection.settings_revision == revision: + into[key] = collection + return all(key in into for key in waiting_for) + + def _continues(self, generation: int, continue_replay: Callable[[], bool]) -> bool: + if generation != self._generation: + return False + if not continue_replay(): + self.discard() + return False + return True + + def _ensure_writer(self, ring: RingEndpointModel | None) -> ShmRingWriter | None: + if self._writer is not None: + return self._writer + # Geometry must come from the active run. Guessing from an editable profile + # could recreate the segment underneath the processor. + if ring is None: + self._warning( + "Cannot recompute the full B-scan: this GUI session did not start the " + "pipeline, so the processor's ring geometry is unknown.", + once_key="bscan_replay_no_active_run_config", + ) + return None + try: + self._writer = ShmRingWriter(ring.name, ring.capacity, ring.slot_size_bytes) + except Exception as exc: # noqa: BLE001 - report boundary failures through the GUI + self._warning("Failed to open the B-scan replay ring writer", details=str(exc)) + return self._writer + + def discard(self) -> None: + self._generation += 1 + self.timer.stop() + self.history.replay_results.clear() + + def close(self) -> None: + self._generation += 1 + self.timer.stop() + writer, self._writer = self._writer, None + if writer is not None: + try: + writer.close() + except Exception as exc: # noqa: BLE001 - cleanup must leave no attached writer + self._warning("Failed to close the B-scan replay ring writer", details=str(exc)) diff --git a/python_app/gui/controllers/config_controller.py b/python_app/gui/controllers/config_controller.py new file mode 100644 index 0000000..a237a3b --- /dev/null +++ b/python_app/gui/controllers/config_controller.py @@ -0,0 +1,375 @@ +"""Bind configuration models to settings widgets without owning runtime state.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from contextlib import ExitStack +from copy import deepcopy +import json +import math +from pathlib import Path + +from PyQt6.QtCore import QSignalBlocker +from PyQt6.QtWidgets import QCheckBox, QComboBox, QDoubleSpinBox, QSpinBox + +from python_app.storage.npz.paths import radar_config_filename_prefix +from python_app.gui.antenna_geometry_editor import AntennaGeometryEditor +from python_app.gui.controllers.sections.views import ScalarWidget, SettingWidget, read_setting, write_setting +from python_app.models.gui_profile_schema import GuiPreprocessDialogStateModel, GuiProcessingStateModel, GuiStateModel, Scalar +from python_app.models.run_config_schema import ComboModel, GprModel, GprRxGeometryModel, GprTxGeometryModel, RunConfigModel +from python_app.models.run_config_validation import parse_combos_from_text, parse_position_csv, validate_gpr_model +from python_app.models.setting_fields import GUI_FIELDS, active_live_fields, active_web_fields +from python_app.orchestration.live_processing_config import ProcessingLiveConfig +from python_app.orchestration.preprocess_assets import PREPROCESS_ASSET_NAMES, preprocess_asset_model + +# Stable fields have one mapping for both applying a profile and reading edits. +_SWEEP_FIELDS = ( + ("radar.sweep.start_hz", "_start_hz_input"), + ("radar.sweep.stop_hz", "_stop_hz_input"), + ("radar.sweep.points", "_points_input"), + ("radar.sweep.if_bandwidth_hz", "_ifbw_input"), + ("radar.sweep.power_dbm", "_power_input"), +) +_ADC_FIELDS = ( + ("radar.kamil_adc.project_dir", "_adc_project_dir_input"), + ("radar.kamil_adc.executable_path", "_adc_executable_path_input"), + ("radar.kamil_adc.tty_path", "_adc_tty_path_input"), + ("radar.kamil_adc.args", "_adc_args_input"), + ("radar.kamil_adc.env", "_adc_env_input"), + ("radar.kamil_adc.startup_timeout_s", "_adc_startup_timeout_s_input"), + ("radar.kamil_adc.sweep_timeout_s", "_adc_sweep_timeout_s_input"), + ("radar.kamil_adc.stop_timeout_s", "_adc_stop_timeout_s_input"), + ("radar.laser_control.enabled", "_optical_enabled_checkbox"), + ("radar.laser_control.port", "_optical_port_input"), + ("radar.laser_control.mode", "_optical_mode_combo"), + ("radar.laser_control.pi_coeff1_p", "_optical_pi_coeff1_p_input"), + ("radar.laser_control.pi_coeff1_i", "_optical_pi_coeff1_i_input"), + ("radar.laser_control.pi_coeff2_p", "_optical_pi_coeff2_p_input"), + ("radar.laser_control.pi_coeff2_i", "_optical_pi_coeff2_i_input"), + ("radar.laser_control.manual.temp1", "_optical_manual_temp1_input"), + ("radar.laser_control.manual.temp2", "_optical_manual_temp2_input"), + ("radar.laser_control.manual.current1", "_optical_manual_current1_input"), + ("radar.laser_control.manual.current2", "_optical_manual_current2_input"), + ("radar.laser_control.variation.variation_type", "_optical_variation_type_combo"), + ("radar.laser_control.variation.static_temp1", "_optical_static_temp1_input"), + ("radar.laser_control.variation.static_temp2", "_optical_static_temp2_input"), + ("radar.laser_control.variation.static_current1", "_optical_static_current1_input"), + ("radar.laser_control.variation.static_current2", "_optical_static_current2_input"), + ("radar.laser_control.variation.min_value", "_optical_min_value_input"), + ("radar.laser_control.variation.max_value", "_optical_max_value_input"), + ("radar.laser_control.variation.step", "_optical_step_input"), + ("radar.laser_control.variation.time_step", "_optical_time_step_input"), + ("radar.laser_control.variation.delay_time", "_optical_delay_time_input"), +) +_GEOMETRY_FIELDS = { + "relative_permittivity": "_gpr_relative_permittivity", + "tx_geometry": "_gpr_tx_geometry_input", + "rx_geometry": "_gpr_rx_geometry_input", +} + + +def web_apply_field_names() -> frozenset[str]: + return frozenset(binding.spec.web for binding in GUI_FIELDS if binding.spec.web) | frozenset(_GEOMETRY_FIELDS) + + +def _model_field(config: RunConfigModel, path: str) -> tuple[object, str]: + owner: object = config + *parents, name = path.split(".") + for parent in parents: + owner = getattr(owner, parent) + return owner, name + + +def _scalar_widget(widget: SettingWidget) -> ScalarWidget: + if isinstance(widget, AntennaGeometryEditor): + raise TypeError("Geometry is not a scalar setting") + return widget + + +def _read_widget(widget: SettingWidget) -> Scalar | list[dict]: + return widget.records() if isinstance(widget, AntennaGeometryEditor) else read_setting(widget) + + +def _set_web_widget(widget: SettingWidget, value: object) -> None: + """Validate untrusted web input before Qt can coerce or clamp it.""" + if isinstance(widget, AntennaGeometryEditor): + widget.set_records(value) + return + if isinstance(widget, QCheckBox): + if not isinstance(value, bool): + raise ValueError("Checkbox value must be true or false") + elif isinstance(widget, (QSpinBox, QDoubleSpinBox)): + if isinstance(value, bool) or not isinstance(value, (int, float, str)): + raise ValueError("Numeric setting requires a number") + numeric = float(value) + if not math.isfinite(numeric) or not widget.minimum() <= numeric <= widget.maximum(): + raise ValueError("Numeric setting is outside its allowed range") + if isinstance(widget, QSpinBox) and numeric != int(numeric): + raise ValueError("Setting requires an integer") + value = int(numeric) if isinstance(widget, QSpinBox) else numeric + elif isinstance(widget, QComboBox): + if not isinstance(value, str) or widget.findText(value) < 0: + raise ValueError(f"Unknown setting option: {value}") + elif isinstance(value, list): + if any(isinstance(item, bool) or not isinstance(item, int) for item in value): + raise ValueError("Position selection requires integer values") + value = ",".join(str(item) for item in value) + if not isinstance(value, (str, int, float, bool)): + raise ValueError("Setting requires a scalar value") + write_setting(widget, value) + + +class ConfigController: + """Read and apply editor values; callers own persistence and runtime effects.""" + + def __init__(self, fields: Mapping[str, SettingWidget], single_combo_selected: Callable[[], bool]) -> None: + self.fields = fields + self._single_combo_selected = single_combo_selected + + def _scalar(self, name: str) -> Scalar: + value = read_setting(_scalar_widget(self.fields[name])) + return value.strip() if isinstance(value, str) else value + + def dataset_destination(self, config: RunConfigModel) -> tuple[Path, str, str]: + radar = config.radar + prefix = radar_config_filename_prefix(radar.model, radar.sweep.start_hz, radar.sweep.stop_hz, + radar.sweep.points, radar.sweep.power_dbm) + return Path(str(self._scalar("_save_path_input"))).expanduser(), str(self._scalar("_save_name_input")), prefix + + def _read_radar(self, config: RunConfigModel) -> None: + for path, widget_id in (*_SWEEP_FIELDS, *(_ADC_FIELDS if config.is_kamil_adc else ())): + owner, name = _model_field(config, path) + original = getattr(owner, name) + value = self._scalar(widget_id) + if isinstance(original, bool): + if not isinstance(value, bool): + raise ValueError(f"{path} must be a boolean") + parsed = value + elif isinstance(original, int): + parsed = int(str(value)) + elif isinstance(original, float): + parsed = float(value) + elif isinstance(original, list): + parsed = [line.strip() for line in str(value).splitlines() if line.strip()] + elif isinstance(original, dict): + parsed = json.loads(str(value) or "{}") + if not isinstance(parsed, dict) or any( + not isinstance(key, str) or not isinstance(item, str) for key, item in parsed.items() + ): + raise ValueError("Environment JSON must contain only string keys and values") + else: + parsed = str(value) + setattr(owner, name, parsed) + + def build_gui_state(self, defaults: RunConfigModel, preprocess: GuiPreprocessDialogStateModel) -> GuiStateModel: + gui = GuiStateModel(preprocess_dialog=deepcopy(preprocess)) + for binding in GUI_FIELDS: + if binding.spec.widget: + binding.write(gui, self._scalar(binding.spec.widget)) + gui.switches.combo_mode = "single" if not defaults.is_multi_device and self._single_combo_selected() else "text" + if defaults.is_multi_device: + gui.switches.combos_text = ",".join(f"{combo.input}:{combo.output}" for combo in defaults.build_runtime_combos()) + return gui + + def build_config( + self, defaults: RunConfigModel, selected_sets: Mapping[str, str], live_path: Path, log_level: str, + ) -> RunConfigModel: + config = defaults.clone() + self._read_radar(config) + config.runtime.settling_ms = int(str(self._scalar("_settling_ms"))) + config.runtime.processing_live_config_path = str(live_path) + config.logging.level = log_level.strip().lower() + if config.is_multi_device: + if len(config.radar.multi_device.slave_serials) != 2: + raise ValueError("LibreVNA multi-device mode requires exactly two slave serials") + config.apply_device_model_constraints() + else: + config.combos = ( + [ComboModel(int(str(self._scalar("_single_combo_input"))), int(str(self._scalar("_single_combo_output"))))] + if self._single_combo_selected() else parse_combos_from_text(str(self._scalar("_combos_text"))) + ) + config.ensure_combos() + static = ( + config.input_switch.positions <= 1 and config.output_switch.positions <= 1 + or config.input_switch.driver_mode == config.output_switch.driver_mode == "mock" + ) + if self._scalar("_processing_mode") not in {"gpr", "legacy_gpr", "sar3d"} and static: + config.combos = [ComboModel(0, 0)] + for key in PREPROCESS_ASSET_NAMES: + asset = preprocess_asset_model(config, key) + asset.bundle_path = "" + asset.set_name = selected_sets.get(key, "") + config.gpr = self.geometry() + validate_gpr_model(config.gpr, input_switch_positions=config.input_switch.positions, + output_switch_positions=config.output_switch.positions, sweep=config.radar.sweep) + return config + + def radar_key(self, defaults: RunConfigModel) -> str: + config = defaults.clone() + self._read_radar(config) + return config.radar_key() + + def _geometry(self, name: str) -> AntennaGeometryEditor: + widget = self.fields[_GEOMETRY_FIELDS[name]] + if not isinstance(widget, AntennaGeometryEditor): + raise TypeError(f"{name} requires an antenna geometry editor") + return widget + + def processing_settings(self) -> GuiProcessingStateModel: + gui = GuiStateModel() + gui.processing.selected_mode = str(self._scalar("_processing_mode")) + for binding in GUI_FIELDS: + if binding.mode in {gui.processing.selected_mode, "pass_through"} and binding.spec.widget: + binding.write(gui, self._scalar(binding.spec.widget)) + return gui.processing + + def geometry(self) -> GprModel: + return GprModel( + relative_permittivity=float(self._scalar("_gpr_relative_permittivity")), + tx_geometry=[GprTxGeometryModel(**entry) for entry in self._geometry("tx_geometry").records()], + rx_geometry=[GprRxGeometryModel(**entry) for entry in self._geometry("rx_geometry").records()], + ) + + def build_live_config( + self, *, reprocess_current_result: bool = True, history_clear_before_ns: int = 0, + history_removed: Sequence[dict[str, int]] = (), settings_revision: int = 0, + ) -> ProcessingLiveConfig: + config = ProcessingLiveConfig( + reprocess_current_result=reprocess_current_result, history_clear_before_ns=history_clear_before_ns, + history_removed=deepcopy(list(history_removed)), settings_revision=settings_revision, + ) + for binding in active_live_fields(str(self._scalar("_processing_mode"))): + value = self._scalar(binding.spec.widget) + if binding.spec.live in {"gpr_input_positions", "gpr_output_positions"}: + value = parse_position_csv(str(value)) + setattr(config, binding.spec.live, value) + config.relative_permittivity = float(self._scalar("_gpr_relative_permittivity")) + config.tx_geometry = self._geometry("tx_geometry").records() + config.rx_geometry = self._geometry("rx_geometry").records() + for axis in ("x", "z"): + lower_name, upper_name = f"gpr_visible_{axis}_min_m", f"gpr_visible_{axis}_max_m" + lower, upper = sorted((getattr(config, lower_name), getattr(config, upper_name))) + if upper - lower < .1: + center = .5 * (lower + upper) + lower, upper = center - .05, center + .05 + setattr(config, lower_name, lower) + setattr(config, upper_name, upper) + return config + + def apply_profile_widgets(self, config: RunConfigModel, gui: GuiStateModel) -> None: + with ExitStack() as blockers: + for widget in self.fields.values(): + blockers.enter_context(QSignalBlocker(widget)) + for path, widget_id in (*_SWEEP_FIELDS, *_ADC_FIELDS, ("runtime.settling_ms", "_settling_ms")): + owner, name = _model_field(config, path) + value = getattr(owner, name) + if isinstance(value, dict): + value = json.dumps(value, indent=2, sort_keys=True) + elif isinstance(value, list): + value = "\n".join(value) + elif isinstance(value, float): + value = f"{value:g}" + write_setting(_scalar_widget(self.fields[widget_id]), value) + # Set frequency ranges before values so a previously narrow profile + # cannot silently clamp the newly loaded processing band. + for binding in GUI_FIELDS: + if binding.path[-1] in {"start_freq_mhz", "stop_freq_mhz"}: + widget = self.fields[binding.spec.widget] + if isinstance(widget, QDoubleSpinBox): + widget.setRange(config.radar.sweep.start_hz / 1e6, config.radar.sweep.stop_hz / 1e6) + if binding.spec.widget: + write_setting(_scalar_widget(self.fields[binding.spec.widget]), binding.read(gui)) + write_setting(_scalar_widget(self.fields["_log_level_combo"]), config.logging.level.capitalize()) + write_setting(_scalar_widget(self.fields["_gpr_relative_permittivity"]), config.gpr.relative_permittivity) + for name, count, records in ( + ("tx_geometry", config.output_switch.positions, config.gpr.tx_geometry), + ("rx_geometry", config.input_switch.positions, config.gpr.rx_geometry), + ): + editor = self._geometry(name) + editor.set_position_count(count) + editor.set_records(records) + + def _web_widgets(self, mode: str) -> dict[str, tuple[str, SettingWidget]]: + widgets = {binding.spec.web: (binding.spec.group, self.fields[binding.spec.widget]) + for binding in active_web_fields(mode)} + for name, widget_id in _GEOMETRY_FIELDS.items(): + if mode in ({"legacy_gpr"} if name == "relative_permittivity" else {"gpr", "legacy_gpr", "sar3d"}): + widgets[name] = ("Geometry & medium", self.fields[widget_id]) + return widgets + + def web_schema(self) -> list[dict[str, object]]: + schema = [] + for name, (group, widget) in self._web_widgets(str(self._scalar("_processing_mode"))).items(): + entry: dict[str, object] = {"name": name, "group": group, "enabled": widget.isEnabled(), "value": _read_widget(widget)} + if isinstance(widget, QComboBox): + entry.update(kind="select", options=[widget.itemText(i) for i in range(widget.count())]) + elif isinstance(widget, QCheckBox): + entry.update(kind="bool") + elif isinstance(widget, (QSpinBox, QDoubleSpinBox)): + entry.update(kind="int" if isinstance(widget, QSpinBox) else "float", + min=widget.minimum(), max=widget.maximum(), step=widget.singleStep()) + if isinstance(widget, QDoubleSpinBox): + entry["decimals"] = widget.decimals() + elif isinstance(widget, AntennaGeometryEditor): + entry.update(kind="geometry", position_field=widget.position_field, position_count=widget.position_count) + else: + entry.update(kind="text") + schema.append(entry) + return schema + + def restore_live_config(self, config: Mapping[str, object]) -> None: + """Restore desktop edits from the last successfully published revision.""" + with ExitStack() as blockers: + for widget in self.fields.values(): + blockers.enter_context(QSignalBlocker(widget)) + for binding in active_live_fields(str(config["processor_mode"])): + value = config[binding.spec.live] + if isinstance(value, list): + value = ",".join(map(str, value)) + if not isinstance(value, (str, int, float, bool)): + raise TypeError(f"Invalid published setting: {binding.spec.live}") + write_setting(_scalar_widget(self.fields[binding.spec.widget]), value) + for name, widget_id in _GEOMETRY_FIELDS.items(): + _set_web_widget(self.fields[widget_id], config[name]) + + def apply_web_fields(self, fields: Mapping[str, object], commit: Callable[[], None]) -> None: + """Apply one batch without signals; restore editors if validation or publication fails.""" + with ExitStack() as blockers: + scalar_snapshots = [] + geometry_snapshots = [] + combo_snapshots = [] + ranges = [] + for widget in self.fields.values(): + blockers.enter_context(QSignalBlocker(widget)) + if isinstance(widget, QComboBox): + combo_snapshots.append((widget, widget.currentIndex())) + elif isinstance(widget, AntennaGeometryEditor): + geometry_snapshots.append((widget, widget.records())) + else: + scalar_snapshots.append((widget, read_setting(widget))) + if isinstance(widget, (QSpinBox, QDoubleSpinBox)): + ranges.append((widget, widget.minimum(), widget.maximum())) + try: + if "processor_mode" in fields: + _set_web_widget(self.fields["_processing_mode"], fields["processor_mode"]) + bindings = self._web_widgets(str(self._scalar("_processing_mode"))) + for name, value in fields.items(): + if name not in bindings: + raise ValueError(f"Unknown setting for current mode: {name}") + _set_web_widget(bindings[name][1], value) + commit() + except Exception: + for widget, lower, upper in ranges: + widget.setRange(lower, upper) + for widget, index in combo_snapshots: + widget.setCurrentIndex(index) + for widget, records in geometry_snapshots: + widget.set_records(records) + for widget, value in scalar_snapshots: + write_setting(widget, value) + raise + + def mode_summary(self, mode: str) -> str: + values = (f"{binding.spec.web}={self._scalar(binding.spec.widget)}" + for binding in active_web_fields(mode) if binding.spec.web != "processor_mode") + return "(" + ", ".join(values) + ")" diff --git a/python_app/gui/controllers/control_button_controller.py b/python_app/gui/controllers/control_button_controller.py new file mode 100644 index 0000000..4e3f51c --- /dev/null +++ b/python_app/gui/controllers/control_button_controller.py @@ -0,0 +1,77 @@ +"""Bind the optional GPIO watcher to the shared reference-capture action.""" + +from collections.abc import Callable +import logging + +from PyQt6.QtCore import QObject + +from python_app.gui.control_button import ControlButtonWatcher +from python_app.gui.controllers.log_controller import LogController +from python_app.models.run_config_schema import ControlButtonModel + +logger = logging.getLogger(__name__) + + +class ControlButtonController(QObject): + def __init__(self, parent: QObject, current_config: Callable[[], ControlButtonModel], + capture_reference: Callable[[], None], log: LogController) -> None: + super().__init__(parent) + self._config, self._capture_reference = current_config, capture_reference + self._log = log + self._watcher: ControlButtonWatcher | None = None + self._closed = False + + def start(self) -> None: + if self._closed or self._watcher is not None: + return + config = self._config() + if not config.enabled: + return + if config.pin < 0 or not config.gpio_chip or config.action != ControlButtonModel.ACTION_CAPTURE_TMP_REFERENCE: + logger.warning("GPIO control button configuration is incomplete or unsupported; watcher not started") + return + watcher = None + try: + watcher = ControlButtonWatcher(chip=config.gpio_chip, pin=config.pin, active_low=config.active_low, + bias=config.bias, debounce_ms=config.debounce_ms, parent=self) + watcher.pressed.connect(self._pressed) + watcher.failed.connect(self._failed) + watcher.start() + self._watcher = watcher + logger.info("GPIO control button watching %s pin=%d action=%s", config.gpio_chip, config.pin, config.action) + except Exception: + logger.warning("Failed to start GPIO control button watcher", exc_info=True) + if watcher is not None: + self._disconnect(watcher) + watcher.deleteLater() + + def _pressed(self) -> None: + if not self._closed and self._watcher is not None: + logger.info("GPIO control button pressed: capturing tmp reference") + with self._log.capture_errors(): + self._capture_reference() + + def _failed(self, message: str) -> None: + logger.warning("GPIO control button watcher stopped: %s", message) + + def close(self) -> None: + self._closed = True + watcher, self._watcher = self._watcher, None + if watcher is None: + return + self._disconnect(watcher) + try: + watcher.stop() + except Exception: + logger.warning("Error stopping GPIO control button watcher", exc_info=True) + finally: + self._disconnect(watcher) + watcher.deleteLater() + + @staticmethod + def _disconnect(watcher: ControlButtonWatcher) -> None: + for signal in (watcher.pressed, watcher.failed): + try: + signal.disconnect() + except (TypeError, RuntimeError): + pass diff --git a/python_app/gui/controllers/history_controller.py b/python_app/gui/controllers/history_controller.py new file mode 100644 index 0000000..104dae4 --- /dev/null +++ b/python_app/gui/controllers/history_controller.py @@ -0,0 +1,71 @@ +"""Retained acquisitions and pending processor-history deletions.""" + +from __future__ import annotations + +from collections import deque +from itertools import chain +import time +from typing import TypeVar + +from python_app.gui.settings_defaults import GUI_SAVE_HISTORY_LIMIT +from python_app.gui.runtime.history import record_result_history +from python_app.models.collection_identity import CollectionIdentity, collection_key, history_collection_removed +from python_app.models.dataset_model import ResultCollection, SweepCollection + +_Collection = TypeVar("_Collection", SweepCollection, ResultCollection) + + +def _remove_acquisition(history: deque[_Collection], key: tuple[int, int]) -> None: + retained = [item for item in history if collection_key(item) != key] + history.clear() + history.extend(retained) + + +class HistoryController: + """Own GUI history without opening rings, starting workers, or drawing plots.""" + + def __init__(self, limit: int = GUI_SAVE_HISTORY_LIMIT) -> None: + self.raw: deque[SweepCollection] = deque(maxlen=limit) + self.preprocessed: deque[SweepCollection] = deque(maxlen=limit) + self.results: deque[ResultCollection] = deque(maxlen=limit) + self.replay_results: list[ResultCollection] = [] + self.run_signature: tuple[object, ...] | None = None + self.clear_before_ns = 0 + self.removed: list[dict[str, int]] = [] + + def is_removed(self, collection: CollectionIdentity) -> bool: + return history_collection_removed(collection, self.clear_before_ns, self.removed) + + def record_result(self, collection: ResultCollection) -> bool: + """Replace a recomputed sweep or insert a new one in acquisition order.""" + return not self.is_removed(collection) and record_result_history(self.results, collection) + + def apply_deletion(self, command: str, target: dict[str, int] | None = None) -> None: + """Stage deletions before publishing settings; a failed write must retain them.""" + if command == "clear_all": + self.clear_before_ns = time.monotonic_ns() + self.removed.clear() + elif command == "remove_last" and target is not None: + self.removed.append(dict(target)) + + def delete(self, *, remove_last: bool) -> None: + """Mutate retained stages and stage their deletion before any external action.""" + self.replay_results.clear() + if not remove_last: + self.apply_deletion("clear_all") + self.raw.clear() + self.preprocessed.clear() + self.results.clear() + return + newest = max(chain(self.raw, self.preprocessed, self.results), key=lambda item: item.capture_start_ns, default=None) + if newest is None: + return + key = collection_key(newest) + self.apply_deletion("remove_last", {"collection_id": key[0], "capture_start_ns": key[1]}) + _remove_acquisition(self.raw, key) + _remove_acquisition(self.preprocessed, key) + _remove_acquisition(self.results, key) + + def bscan_source(self) -> list[ResultCollection]: + """A completed replay contains only the re-fed sweeps with current settings.""" + return list(self.replay_results or self.results) diff --git a/python_app/gui/controllers/live_settings_controller.py b/python_app/gui/controllers/live_settings_controller.py new file mode 100644 index 0000000..bdd37f4 --- /dev/null +++ b/python_app/gui/controllers/live_settings_controller.py @@ -0,0 +1,203 @@ +"""Publish live processing revisions and apply desktop or web editor changes.""" + +from collections.abc import Callable, Mapping +import time + +from PyQt6.QtCore import QSignalBlocker +from python_app.gui.antenna_geometry_editor import AntennaGeometryEditor + +from python_app.gui.controllers.config_controller import ConfigController +from python_app.gui.controllers.log_controller import LogController +from python_app.gui.controllers.pipeline_controller import PipelineController +from python_app.gui.controllers.preprocess_controller import PreprocessController +from python_app.gui.controllers.radar_settings_controller import RadarSettingsController +from python_app.gui.controllers.sections.views import ProcessingView, connect_setting, write_setting +from python_app.gui.runtime.constraints import validate_processing_mode_constraints +from python_app.models.run_config_schema import RunConfigModel +from python_app.models.setting_fields import active_live_fields +from python_app.orchestration.live_processing_config import ProcessingLiveConfig, ProcessingLiveConfigWriter + +_GPR_MODES = {"gpr", "legacy_gpr"} +_GEOMETRY_MODES = _GPR_MODES | {"sar3d"} + + +class LiveSettingsController: + def __init__( + self, processing: ProcessingView, config: ConfigController, pipeline: PipelineController, + preprocess: PreprocessController, radar: RadarSettingsController, + writer: ProcessingLiveConfigWriter, processing_settings: dict[int, dict], log: LogController, + *, build_config: Callable[[], RunConfigModel], + ) -> None: + self.config, self.writer, self.processing_settings = config, writer, processing_settings + self._processing, self._pipeline, self._preprocess = processing, pipeline, preprocess + self._radar, self._log, self._build_config = radar, log, build_config + self._mode = processing.mode + self.revision = 0 + self.active_mode = "pass_through" + self._mode.currentTextChanged.connect(self.mode_changed) + special = {self._mode, *processing.trace_controls, *processing.visual_controls, + *processing.locator_controls, *(toggle for toggle, _ in processing.speed_controls)} + for widgets, callback in ((processing.trace_controls, self.trace_changed), + (processing.visual_controls, self.visual_changed), + (processing.locator_controls, self.locator_changed)): + for widget in widgets: + connect_setting(widget, callback) + for toggle, _ in processing.speed_controls: + toggle.toggled.connect(self.speed_changed) + for widget in processing.fields.values(): + if isinstance(widget, AntennaGeometryEditor): + widget.validate_change = lambda: self.validate_mode(self._mode.currentText()) + widget.changed.connect(self.geometry_changed) + elif widget not in special: + connect_setting(widget, self.changed) + + def build(self, *, reprocess_current_result: bool = True) -> ProcessingLiveConfig: + self._radar.sync_frequency_limits() + history = self._pipeline.history + return self.config.build_live_config( + reprocess_current_result=reprocess_current_result, + history_clear_before_ns=history.clear_before_ns, history_removed=history.removed, + settings_revision=self.revision, + ) + + def publish(self) -> None: + revision = time.monotonic_ns() + config = self.build(reprocess_current_result=not self._pipeline.can_reprocess_history(mode=self._mode.currentText())) + config.settings_revision = revision + self.writer.write(config) + self.revision = revision + self.processing_settings[revision] = config.to_dict() + active = self._pipeline.active_config + if active is not None and self._pipeline.running: + geometry = self.config.geometry() + active.gpr.tx_geometry, active.gpr.rx_geometry = geometry.tx_geometry, geometry.rx_geometry + self._pipeline.plots.required_revision = revision + + def validate_mode(self, mode: str) -> None: + validate_processing_mode_constraints(mode, self._build_config(), self.build()) + + def apply_web(self, fields: Mapping[str, object]) -> None: + try: + settings = dict(fields) + self._log.debug(f"Applying web live settings: fields={sorted(settings)}.") + + def commit() -> None: + self.validate_mode(self._mode.currentText()) + live_fields = {binding.spec.web for binding in active_live_fields(self._mode.currentText())} + if settings.keys() & (live_fields | {"tx_geometry", "rx_geometry", "relative_permittivity"}): + self.publish() + + self.config.apply_web_fields(settings, commit) + self.sync_controls() + if "processor_mode" in settings: + self._show_mode(self._mode.currentText()) + if {"tx_geometry", "rx_geometry"} & settings.keys(): + self._preprocess.invalidate_object_calibration_after_geometry_change() + self._redraw() + except Exception as exc: + self._log.show_exception("Failed to apply web live settings", exc) + + def _restore_published(self) -> None: + previous = self.processing_settings.get(self.revision) + if previous is not None: + self.config.restore_live_config(previous) + else: + with QSignalBlocker(self._mode): + write_setting(self._mode, self.active_mode) + self.sync_controls() + + def _publish_changes(self) -> bool: + try: + self.publish() + except Exception as exc: + self._restore_published() + self._log.show_exception("Failed to update live processing settings", exc) + return False + return True + + def changed(self, *_args: object) -> None: + if self._publish_changes(): + self._redraw() + + def _redraw(self) -> None: + try: + mode = self._mode.currentText() + pipeline, plots, history = self._pipeline, self._pipeline.plots, self._pipeline.history + if mode == "bscan": + if pipeline.can_reprocess_history(): + pipeline.schedule_replay() + else: + pipeline.drain_results(timeout_s=0.25, poll_s=0.01) + plots.draw_bscan() + elif mode in _GEOMETRY_MODES: + latest = pipeline.drain_results(timeout_s=0.8, poll_s=0.02) + collection = latest if latest is not None else history.results[-1] if history.results else None + if mode == "sar3d": + plots.sar3d.clear() + if collection is not None: + plots.draw(collection) + elif collection is None or not plots.draw(collection): + plots.clear_gpr() + elif history.results: + plots.draw(history.results[-1]) + else: + plots.trace.clear() + except Exception as exc: + self._log.show_exception("Failed to render processing settings", exc) + + def geometry_changed(self) -> None: + if self._publish_changes(): + self._preprocess.invalidate_object_calibration_after_geometry_change() + self._redraw() + + def sync_controls(self) -> None: + for widget in self._processing.y_bounds: + widget.setEnabled(self._processing.fixed_y.isChecked()) + for toggle, speed in self._processing.speed_controls: + speed.setEnabled(toggle.isChecked()) + + def trace_changed(self, *_args: object) -> None: + self.sync_controls() + self.visual_changed() + + def speed_changed(self, *_args: object) -> None: + self.sync_controls() + self.changed() + + def visual_changed(self, *_args: object) -> None: + try: + self._pipeline.plots.redraw() + except Exception as exc: + self._log.show_exception("Failed to update plot display settings", exc) + + def locator_changed(self, *_args: object) -> None: + if self._mode.currentText() in _GPR_MODES and self._publish_changes(): + self.visual_changed() + + def set_mode_page(self, mode: str) -> None: + pages = self._processing.pages + pages.setCurrentIndex({"pass_through": 0, "bscan": 1, "gpr": 2, "legacy_gpr": 3, "sar3d": 4}.get(mode, 0)) + self._processing.geometry_page.setVisible(mode in _GEOMETRY_MODES) + page = pages.currentWidget() + if page is not None: + pages.setFixedHeight(page.sizeHint().height()) + pages.updateGeometry() + self._pipeline.plots.sync_display_combos() + + def mode_changed(self, mode: str) -> None: + try: + if mode != self.active_mode: + self.validate_mode(mode) + self.publish() + except Exception as exc: + self._restore_published() + self._log.show_exception(f"Cannot switch processing mode to {mode}", exc) + return + self._show_mode(mode) + self._redraw() + + def _show_mode(self, mode: str) -> None: + self.active_mode = mode + self._pipeline.plots.set_mode(mode) + self.set_mode_page(mode) + self._log.info(f"Processing mode selected: {mode} {self.config.mode_summary(mode)}") diff --git a/python_app/gui/controllers/log_controller.py b/python_app/gui/controllers/log_controller.py new file mode 100644 index 0000000..e103339 --- /dev/null +++ b/python_app/gui/controllers/log_controller.py @@ -0,0 +1,226 @@ +"""Bounded GUI logging, error presentation and request-scoped error capture.""" + +from __future__ import annotations + +from collections import deque +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from datetime import datetime +import html +import logging +from pathlib import Path +import threading +import traceback + +from PyQt6.QtCore import QSignalBlocker, QTimer +from PyQt6.QtGui import QTextCursor +from PyQt6.QtWidgets import QComboBox, QHBoxLayout, QLabel, QMessageBox, QPushButton, QTextEdit, QVBoxLayout, QWidget + +from python_app.gui.settings_defaults import env_enabled +from python_app.logging_setup import LOG_LEVELS, add_handler, configure_logging, get_logger, package_logger, set_log_level + + +class _PanelLogBuffer: + """Worker threads enqueue records; the GUI flushes one bounded batch per tick.""" + + _CAPACITY = 2000 + + def __init__(self) -> None: + self._lock = threading.Lock() + self._entries: deque[tuple[str, str, str | None, str | None]] = deque(maxlen=self._CAPACITY) + self._dropped_count = 0 + + def append(self, level: str, text: str, details: str | None, once_key: str | None) -> None: + with self._lock: + if len(self._entries) == self._CAPACITY: + self._dropped_count += 1 + self._entries.append((level, text, details, once_key)) + + def drain(self) -> tuple[list[tuple[str, str, str | None, str | None]], int]: + with self._lock: + entries, dropped = list(self._entries), self._dropped_count + self._entries.clear() + self._dropped_count = 0 + return entries, dropped + + +class _PanelHandler(logging.Handler): + def __init__(self, buffer: _PanelLogBuffer) -> None: + super().__init__() + self._buffer = buffer + + def emit(self, record: logging.LogRecord) -> None: + try: + details = getattr(record, "panel_details", None) + once_key = getattr(record, "panel_once_key", None) + self._buffer.append( + "WARN" if record.levelname == "WARNING" else record.levelname, record.getMessage(), + details if isinstance(details, str) else None, once_key if isinstance(once_key, str) else None, + ) + except Exception: + self.handleError(record) + + +def _entry_html(level: str, text: str, details: str | None = None) -> str: + palette = {"DEBUG": ("#6c7b8d", "#52627a", "#8a97a8"), "INFO": ("#1d5fbf", "#1f2937", "#526277"), + "WARN": ("#9a5b00", "#5c4300", "#7a6640"), "ERROR": ("#c43d4d", "#6b1f2a", "#8b5d66")} + accent, message, detail = palette.get(level.upper(), palette["INFO"]) + timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] + body = (f"{html.escape(level)}" + f" {timestamp}" + f" {html.escape(text).replace(chr(10), '
')}") + if details: + body += f"
{html.escape(details)}
" + return f"
{body}
" + + +class LogController: + def __init__(self, parent: QWidget, project_root: Path, status: Callable[[str], None]) -> None: + self._parent, self._status = parent, status + configure_logging(log_dir=project_root / "python_app/runtime/logs") + self._logger = get_logger("gui") + self._buffer = _PanelLogBuffer() + self._seen: set[str] = set() + self._capture: list[str] | None = None + self.panel = QWidget(parent) + layout = QVBoxLayout(self.panel) + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(6) + header = QHBoxLayout() + header.setSpacing(8) + title = QLabel("Runtime Log") + title.setObjectName("hintLabel") + header.addWidget(title) + header.addStretch(1) + label = QLabel("Level") + label.setObjectName("hintLabel") + header.addWidget(label) + self.level = QComboBox() + self.level.setObjectName("logLevelCombo") + self.level.addItems([name.capitalize() for name in LOG_LEVELS]) + self.level.setCurrentText("Info") + self.level.setToolTip("Logging verbosity — applied live and saved to run_config") + self.level.currentTextChanged.connect(self.set_level) + header.addWidget(self.level) + self._toggle = QPushButton("Collapse") + self._toggle.setObjectName("sectionToggleButton") + self._toggle.setToolTip("Hide runtime log entries") + self._toggle.clicked.connect(self.toggle) + header.addWidget(self._toggle) + layout.addLayout(header) + self._box = QTextEdit() + self._box.setObjectName("runtimeLogBox") + self._box.setReadOnly(True) + self._box.setUndoRedoEnabled(False) + self._box.setMinimumHeight(170) + document = self._box.document() + assert document is not None + document.setMaximumBlockCount(1200) + layout.addWidget(self._box) + self._handler = _PanelHandler(self._buffer) + add_handler(self._handler) + self._timer = QTimer(self.panel) + self._timer.setInterval(100) + self._timer.timeout.connect(self.flush) + self._timer.start() + + def set_level(self, text: str) -> None: + with QSignalBlocker(self.level): + self.level.setCurrentText(text.capitalize()) + set_log_level(text) + self.info(f"Log level set to {text.upper()}.") + + def toggle(self) -> None: + visible = self._box.isHidden() + self._box.setVisible(visible) + self._toggle.setText("Collapse" if visible else "Expand") + self._toggle.setToolTip("Hide runtime log entries" if visible else "Show runtime log entries") + + def flush(self) -> None: + entries, dropped = self._buffer.drain() + rendered: list[str] = [] + if dropped: + rendered.append(_entry_html("WARN", f"Log panel overflow: {dropped} record(s) dropped (they are still in the log file).")) + for level, text, details, once_key in entries: + if once_key is not None: + if once_key in self._seen: + continue + self._seen.add(once_key) + rendered.append(_entry_html(level, text, details)) + if level == "ERROR": + self._status("Status: error") + if rendered: + cursor = self._box.textCursor() + cursor.movePosition(QTextCursor.MoveOperation.End) + self._box.setTextCursor(cursor) + self._box.insertHtml("".join(rendered)) + self._box.insertPlainText("\n") + self._box.ensureCursorVisible() + + def debug(self, text: str, *, once_key: str | None = None) -> None: + self._logger.debug(text, extra={"panel_once_key": once_key}) + + def info(self, text: str, *, once_key: str | None = None) -> None: + self._logger.info(text, extra={"panel_once_key": once_key}) + + def warning(self, text: str, *, details: str | None = None, once_key: str | None = None) -> None: + self._logger.warning(text, extra={"panel_details": details, "panel_once_key": once_key}) + + def error(self, text: str, *, details: str | None = None, once_key: str | None = None) -> None: + self._logger.error(text, extra={"panel_details": details, "panel_once_key": once_key}) + + @staticmethod + def exception_summary(exc: Exception) -> str: + message = str(exc).strip() + return f"{type(exc).__name__}: {message}" if message else type(exc).__name__ + + @staticmethod + def exception_details(exc: Exception) -> str: + return "".join(traceback.TracebackException.from_exception(exc).format(chain=True)).strip() + + def exception(self, context: str, exc: Exception, *, level: str = "ERROR") -> tuple[str, str]: + message, details = f"{context}: {self.exception_summary(exc)}", self.exception_details(exc) + (self.warning if level.upper() in {"WARN", "WARNING"} else self.error)(message, details=details) + return message, details + + @contextmanager + def capture_errors(self) -> Iterator[list[str]]: + previous, captured = self._capture, [] + self._capture = captured + try: + yield captured + finally: + self._capture = previous + + @contextmanager + def suspend_error_capture(self) -> Iterator[None]: + """Keep unrelated Qt callbacks out of the active web or GPIO action.""" + previous, self._capture = self._capture, None + try: + yield + finally: + self._capture = previous + + def show_error(self, message: str, *, details: str | None = None) -> None: + self.error(message, details=details) + if self._capture is not None: + if not self._capture: + self._capture.append(message) + return + if env_enabled("RADAR_SYSTEM_HEADLESS"): + return + dialog = QMessageBox(self._parent) + dialog.setIcon(QMessageBox.Icon.Critical) + dialog.setWindowTitle("Error") + dialog.setText(message) + if details: + dialog.setDetailedText(details) + dialog.exec() + + def show_exception(self, context: str, exc: Exception) -> None: + self.show_error(f"{context}: {self.exception_summary(exc)}", details=self.exception_details(exc)) + + def close(self) -> None: + self._timer.stop() + package_logger().removeHandler(self._handler) + self._handler.close() diff --git a/python_app/gui/controllers/object_calibration_capture.py b/python_app/gui/controllers/object_calibration_capture.py index a4dfbb6..3b7d75a 100644 --- a/python_app/gui/controllers/object_calibration_capture.py +++ b/python_app/gui/controllers/object_calibration_capture.py @@ -1,4 +1,4 @@ -"""Acquire fresh object-calibration sweeps through the existing native preprocessor.""" +"""Acquire fresh object-calibration sweeps through the native preprocessor.""" from __future__ import annotations @@ -6,88 +6,75 @@ import time from PyQt6.QtCore import QObject, QTimer, pyqtSignal +from python_app.gui.controllers.pipeline_controller import PipelineController +from python_app.gui.object_calibration_dialog import ObjectCalibrationDialog +from python_app.gui.runtime.pipeline_pause import PipelinePause +from python_app.models.collection_identity import collection_key from python_app.models.dataset_model import SweepCollection -from python_app.models.run_config_model import ComboModel +from python_app.models.run_config_schema import RunConfigModel class ObjectSweepCapture(QObject): - """Temporarily acquire the full antenna matrix without applying an object correction.""" + """Hold a pipeline pause while collecting a fresh matrix without object correction.""" finished = pyqtSignal() + TIMEOUT_S = 300 - def __init__(self, owner, dialog) -> None: - if getattr(owner, "_object_capture_job", None) is not None: - raise RuntimeError("Object-calibration capture is already in progress") + def __init__(self, pipeline: PipelineController, dialog: ObjectCalibrationDialog, + config: RunConfigModel, count: int, provenance: dict) -> None: super().__init__(dialog) - owner._object_capture_job = self - self._owner = owner - self._dialog = dialog + self._pipeline, self._dialog = pipeline, dialog + self._config, self._count, self._provenance = config, count, provenance self._timer = QTimer(self) self._timer.timeout.connect(self._poll) - self._done = False - self._paused = False - self._started = False - self._resume = False + self._done = self._started = self._starting = False + self._pending_error: str | None = None + self._pause: PipelinePause | None = None self._collections: list[SweepCollection] = [] self._seen: set[tuple[int, int]] = set() - self._stop_seq = getattr(owner, "_stop_request_seq", 0) + self._stop_generation = pipeline.stop_generation + self._start_ns = 0 + dialog.set_capture_pending(True) QTimer.singleShot(0, self._start) def _start(self) -> None: if self._done: return - owner = self._owner + self._starting = True try: - if not owner._pipeline_resume_allowed(self._stop_seq): + if not self._pipeline.can_resume(self._stop_generation): self.cancel() return - if getattr(owner, "_closing", False) or owner._capture_session is not None or owner._preprocess_capture_busy: - raise RuntimeError("Finish the active capture before taking object-calibration data") - if owner._single_capture_active or getattr(owner, "_recording_active", False): - raise RuntimeError("Finish the single capture or disk recording before taking calibration data") - self._config = owner._build_config() - if self._config.radar.replay.dataset_dir.strip(): - raise RuntimeError("Object-calibration capture requires live acquisition; disable archive replay") - self._config.preprocess.s21.object_calibration.set_name = "" - self._config.preprocess.s21.object_calibration.bundle_path = "" - tx = sorted({entry.output_pos for entry in self._config.gpr.tx_geometry}) - rx = sorted({entry.input_pos for entry in self._config.gpr.rx_geometry}) - if not tx or not rx: - raise RuntimeError("Configure Tx/Rx antenna positions before taking calibration data") - self._config.combos = [ComboModel(input=i, output=o) for o in tx for i in rx] - self._count = owner._ensure_preprocess_dialog().median_sweep_count() - self._resume = owner._supervisor.is_running() - if self._resume: - self._paused = True - owner._stop_run() - self._stop_seq += 1 - if not owner._pipeline_resume_allowed(self._stop_seq): - self.cancel() - return - self._source_config = self._config.to_dict() + self._pause = self._pipeline.pause() + if self._pending_error is not None: + return self._start_ns = time.monotonic_ns() - owner._start_run(config_override=self._config) - self._started = owner._supervisor.is_running() - if not self._started: + self._started = self._pipeline.start(config_override=self._config, pause=self._pause) + if self._pending_error is not None: + return + if not self._started or not self._pipeline.acquisition_healthy: raise RuntimeError("Calibration acquisition did not start; check the preprocessing sets and log") self._dialog.set_status(f"Taking calibration data: 0 / {self._count} sweeps") self._timer.start(50) except Exception as exc: - self._finish(error=str(exc)) + self._pending_error = str(exc) + finally: + self._starting = False + if self._pending_error is not None: + self._finish(error=self._pending_error) def _poll(self) -> None: - owner = self._owner + if self._done: + return try: - if not owner._pipeline_resume_allowed(self._stop_seq): + if self._pause is None or not self._pause.valid: self.cancel() return - if not owner._supervisor.is_running(): + if not self._pipeline.acquisition_healthy: raise RuntimeError("Calibration acquisition stopped before all sweeps were received") - for collection in owner._pre_history: - identity = (collection.collection_id, collection.monotonic_ns) - starts = [collection.capture_start_ns, *(trace.capture_start_ns for trace in collection.traces)] - acquired_ns = min((start for start in starts if start > 0), default=collection.monotonic_ns) - if acquired_ns < self._start_ns or identity in self._seen: + for collection in self._pipeline.history.preprocessed: + identity = collection_key(collection) + if collection.capture_start_ns < self._start_ns or identity in self._seen: continue self._seen.add(identity) self._collections.append(collection) @@ -95,46 +82,41 @@ class ObjectSweepCapture(QObject): self._finish() return self._dialog.set_status(f"Taking calibration data: {len(self._collections)} / {self._count} sweeps") - if time.monotonic_ns() - self._start_ns > owner._SINGLE_CAPTURE_TIMEOUT_S * 1e9: + if time.monotonic_ns() - self._start_ns > self.TIMEOUT_S * 1e9: raise TimeoutError("Timed out waiting for preprocessed calibration sweeps") except Exception as exc: self._finish(error=str(exc)) def cancel(self) -> None: - """Cancel this capture and restore the previous acquisition when it is still allowed.""" self._finish(error="Calibration capture cancelled") def _finish(self, *, error: str | None = None) -> None: if self._done: return + if self._starting: + # Stop/start drains dispatch Qt events. Keep the lease until startup + # returns, so Cancel cannot start a second run inside the first one. + self._pending_error = error or "Calibration capture cancelled" + return self._done = True self._timer.stop() - owner = self._owner - resume_allowed = owner._pipeline_resume_allowed(self._stop_seq) try: - if self._started and resume_allowed: - owner._stop_run() - self._stop_seq += 1 - if not owner._pipeline_resume_allowed(self._stop_seq): + if self._started: + self._pipeline.stop_pipeline(keep_processor=True) + if self._pause is not None and not self._pause.valid: error = "Calibration capture cancelled" if error is None: - self._dialog.add_capture(self._collections, self._source_config) + self._dialog.add_capture(self._collections, self._config.to_dict(), self._provenance) else: self._dialog.set_status(error) except Exception as exc: self._dialog.set_status(f"Calibration capture failed: {exc}") finally: - if getattr(owner, "_object_capture_job", None) is self: - owner._object_capture_job = None - if self._resume and (self._paused or self._started) and resume_allowed and owner._pipeline_resume_allowed(self._stop_seq): - owner._start_run() self._dialog.set_capture_pending(False) + # The owner clears its job before finish() can start normal acquisition. self.finished.emit() - self.deleteLater() - - -def capture_object_sweeps(owner, dialog) -> ObjectSweepCapture: - """Start an asynchronous capture; the dialog receives sweeps or a visible error.""" - capture = ObjectSweepCapture(owner, dialog) - dialog.set_capture_pending(True) - return capture + try: + if self._pause is not None: + self._pause.finish() + finally: + self.deleteLater() diff --git a/python_app/gui/controllers/object_calibration_controller.py b/python_app/gui/controllers/object_calibration_controller.py index 18f929b..bbbe0f1 100644 --- a/python_app/gui/controllers/object_calibration_controller.py +++ b/python_app/gui/controllers/object_calibration_controller.py @@ -1,183 +1,181 @@ -"""GUI coordination for importing and estimating optional object calibration.""" +"""GUI coordination for estimating and saving optional object calibration.""" from __future__ import annotations +from collections.abc import Callable +from dataclasses import dataclass +import logging import queue import threading -from pathlib import Path from PyQt6.QtCore import QObject, QTimer -from PyQt6.QtWidgets import QFileDialog +from PyQt6.QtWidgets import QWidget +from python_app.gui.controllers.object_calibration_capture import ObjectSweepCapture from python_app.gui.object_calibration_dialog import ObjectCalibrationDialog +from python_app.models.dataset_model import SweepCollection +from python_app.models.run_config_schema import RunConfigModel +from python_app.storage.npz.store import NpzStore +from python_app.workflows.object_calibration import validate_object_set + +logger = logging.getLogger(__name__) -def _calculate(groups, config, settings, cancelled, messages, result) -> None: - """The worker handles only Python data; every Qt action stays on the GUI thread.""" +@dataclass +class _CalculationResult: + calibration: tuple[SweepCollection, dict] | None = None + error: Exception | None = None + cancelled: bool = False + + +def _calculate(groups: list[dict], config: dict, settings: dict, provenance: dict, + cancelled: threading.Event, messages: queue.Queue[str], result: queue.Queue[_CalculationResult]) -> None: + """Only immutable inputs and queues cross the worker boundary, never Qt objects.""" try: from python_app.workflows.mimo_calibration import Settings, compute_from_collections calibration = compute_from_collections( - groups, config, settings=Settings(**settings), + groups, config, settings=Settings(**settings), provenance=provenance, progress=messages.put, cancelled=cancelled.is_set, ) - result.put(("cancelled", None) if cancelled.is_set() else ("ready", calibration)) + result.put(_CalculationResult(calibration=calibration, cancelled=cancelled.is_set())) except InterruptedError: - result.put(("cancelled", None)) - except Exception as exc: # noqa: BLE001 - report worker failures on the GUI thread - result.put(("error", exc)) + result.put(_CalculationResult(cancelled=True)) + except Exception as exc: + result.put(_CalculationResult(error=exc)) class ObjectCalibrationController(QObject): - """Own one modeless editor and cancellable calculation for an AppWindow.""" + """Own one modeless editor, acquisition job and cancellable calculation.""" - def __init__(self, owner) -> None: - super().__init__(owner) - self._owner = owner - self._dialog: ObjectCalibrationDialog | None = None - self._capture = None + def __init__(self, parent: QWidget, store: NpzStore, build_config: Callable[[], RunConfigModel], + capture: Callable[[ObjectCalibrationDialog], ObjectSweepCapture], select_saved: Callable[[str], None], + error: Callable[[str, Exception], None]) -> None: + super().__init__(parent) + self._parent, self._store, self._build_config = parent, store, build_config + self._start_capture, self._select_saved, self._error = capture, select_saved, error + self.dialog: ObjectCalibrationDialog | None = None + self.capture: ObjectSweepCapture | None = None + self._closed = False self._worker: threading.Thread | None = None self._cancel = threading.Event() - self._messages: queue.Queue = queue.Queue() - self._result: queue.Queue = queue.Queue(maxsize=1) + self._messages: queue.Queue[str] = queue.Queue() + self._result: queue.Queue[_CalculationResult] = queue.Queue(maxsize=1) self._poll = QTimer(self) self._poll.setInterval(100) self._poll.timeout.connect(self._poll_calculation) self._save_name = "" + self._provenance: dict = {} - def import_calibration(self) -> None: - """Validate and copy a legacy cal_*.npy folder into the app's own store.""" - if getattr(self._owner, "_closing", False): + def open_creation(self, set_name: str) -> None: + if self._closed: return - if getattr(self._owner, "_object_capture_job", None) is not None or self._worker is not None: - self._owner._show_error("Wait for the object-calibration operation to finish") - return - panel = self._owner._ensure_preprocess_dialog() - name = panel.set_name() - if not name: - self._owner._show_error("Set name is required") - return - directory = QFileDialog.getExistingDirectory(panel, "Import MIMO calibration") - if not directory: + if self.dialog is None: + self.dialog = ObjectCalibrationDialog(set_name, self._parent) + self.dialog.capture_requested.connect(self.capture_data) + self.dialog.calculate_requested.connect(self._start_calculation) + self.dialog.cancel_requested.connect(self.cancel) + self.dialog.show() + self.dialog.raise_() + self.dialog.activateWindow() + + def capture_data(self) -> None: + if self._closed or self.capture is not None or self._worker is not None or self.dialog is None: return try: - from python_app.workflows.mimo_calibration import load_calibration, validate_for_config - - config = self._owner._build_config() - calibration = load_calibration(Path(directory)) - validate_for_config(calibration, config.to_dict()) - radar_key = self._owner._radar_key(config) - self._owner._store.save_object_calibration(radar_key, name, calibration) - self._select_saved(name) - panel.set_status(f"Object calibration imported: {name}. Applies on the next Start.") - except Exception as exc: # noqa: BLE001 - self._owner._show_exception("Failed to import object calibration", exc) - - def open_creation(self) -> None: - if getattr(self._owner, "_closing", False): - return - if self._dialog is None: - self._dialog = ObjectCalibrationDialog(self._owner._ensure_preprocess_dialog().set_name(), self._owner) - self._dialog.capture_requested.connect(self._capture_data) - self._dialog.calculate_requested.connect(self._start_calculation) - self._dialog.cancel_requested.connect(self.cancel) - self._dialog.show() - self._dialog.raise_() - self._dialog.activateWindow() - - def _capture_data(self) -> None: - if (getattr(self._owner, "_closing", False) or self._capture is not None - or self._worker is not None or self._dialog is None): - return - from python_app.gui.controllers.object_calibration_capture import capture_object_sweeps - - try: - self._capture = capture_object_sweeps(self._owner, self._dialog) - self._capture.finished.connect(self._capture_finished) - except Exception as exc: # noqa: BLE001 - self._dialog.set_capture_pending(False) - self._dialog.set_status(str(exc)) - self._owner._show_exception("Failed to capture object-calibration data", exc) + self.capture = self._start_capture(self.dialog) + self.capture.finished.connect(self._capture_finished) + except Exception as exc: + self.dialog.set_capture_pending(False) + self.dialog.set_status(str(exc)) + self._error("Failed to capture object-calibration data", exc) def _capture_finished(self) -> None: - self._capture = None + self.capture = None def _start_calculation(self) -> None: - if getattr(self._owner, "_closing", False) or self._worker is not None or self._capture is not None or self._dialog is None: + if self._closed or self._worker is not None or self.capture is not None or self.dialog is None: return try: - name = self._dialog.set_name() + name = self.dialog.set_name() if not name: raise ValueError("Set name is required") - groups, config, settings = self._dialog.calculation_inputs() - self._save_name = name + groups, config, settings, provenance = self.dialog.calculation_inputs() + if name in self._store.list_sets("s21_object_calibration", provenance["radar_key"]): + raise FileExistsError(f"Object calibration already exists: {name}") + self._provenance, self._save_name = provenance, name self._cancel = threading.Event() - self._messages = queue.Queue() - self._result = queue.Queue(maxsize=1) + self._messages, self._result = queue.Queue(), queue.Queue(maxsize=1) self._worker = threading.Thread( - target=_calculate, - args=(groups, config, settings, self._cancel, self._messages, self._result), + target=_calculate, args=(groups, config, settings, provenance, self._cancel, self._messages, self._result), name="object-calibration", daemon=True, ) - self._dialog.set_calculating(True) - self._dialog.set_status("Расчёт калибровки…") + self.dialog.set_calculating(True) + self.dialog.set_status("Calculating calibration…") self._worker.start() self._poll.start() - except Exception as exc: # noqa: BLE001 + except Exception as exc: self._worker = None - self._dialog.set_calculating(False) - self._dialog.set_status(f"Ошибка расчёта: {exc}") - self._owner._show_exception("Failed to calculate object calibration", exc) + self.dialog.set_calculating(False) + self.dialog.set_status(f"Calculation failed: {exc}") + self._error("Failed to calculate object calibration", exc) def _poll_calculation(self) -> None: - if self._dialog is None or getattr(self._owner, "_closing", False): + if self.dialog is None or self._closed: self._poll.stop() return while not self._messages.empty(): - self._dialog.set_status(self._messages.get_nowait()) + self.dialog.set_status(self._messages.get_nowait()) try: - state, payload = self._result.get_nowait() + outcome = self._result.get_nowait() except queue.Empty: return self._poll.stop() self._worker = None - self._dialog.set_calculating(False) - if self._cancel.is_set() or state == "cancelled": - self._dialog.set_status("Расчёт отменён. Снятые данные сохранены в этом окне.") + self.dialog.set_calculating(False) + if self._cancel.is_set() or outcome.cancelled: + self.dialog.set_status("Calculation cancelled. Captured data remain in this window.") return - if state == "error": - self._dialog.set_status(f"Ошибка расчёта: {payload}") - self._owner._show_exception("Failed to calculate object calibration", payload) + if outcome.error is not None: + self.dialog.set_status(f"Calculation failed: {outcome.error}") + self._error("Failed to calculate object calibration", outcome.error) return + assert outcome.calibration is not None try: - from python_app.workflows.mimo_calibration import validate_for_config - - current_config = self._owner._build_config() - validate_for_config(payload, current_config.to_dict()) - radar_key = self._owner._radar_key(current_config) - self._owner._store.save_object_calibration(radar_key, self._save_name, payload) - self._select_saved(self._save_name) - self._owner._ensure_preprocess_dialog().set_set_name(self._save_name) - self._dialog.show_calibration(payload) - self._dialog.set_status(f"Калибровка сохранена: {self._save_name}. Применится при следующем Start.") - except Exception as exc: # noqa: BLE001 - self._dialog.set_status(f"Калибровка не сохранена: {exc}") - self._owner._show_exception("Failed to save object calibration", exc) - - def _select_saved(self, name: str) -> None: - self._owner._selected_preprocess_sets["s21_object_calibration"] = name - self._owner._refresh_sets() - self._owner._log(f"Selected object calibration: {name}") + # Save under captured provenance even when the pending GUI profile changed. + radar_key = self._provenance["radar_key"] + if self._save_name in self._store.list_sets("s21_object_calibration", radar_key): + raise FileExistsError(f"Object calibration already exists: {self._save_name}") + collection, metadata = outcome.calibration + self._store.save_set("s21_object_calibration", radar_key, self._save_name, collection, metadata=metadata) + self.dialog.show_calibration(metadata) + status = f"Calibration saved: {self._save_name} (capture radar: {radar_key})." + try: + current = self._build_config() + current_key = current.radar_key() + through = self._store.load_set("s21_calibration", current_key, current.preprocess.s21.calibration.set_name) + validate_object_set(collection, metadata, current.to_dict(), current_key, through) + except Exception as exc: + status += f" Not selected for the current configuration: {exc}" + else: + self._select_saved(self._save_name) + status += " Applies on the next Start." + self.dialog.set_status(status) + logger.info("%s", status) + except Exception as exc: + self.dialog.set_status(f"Calibration could not be saved: {exc}") + self._error("Failed to save object calibration", exc) def cancel(self) -> None: self._cancel.set() - if self._capture is not None: - self._capture.cancel() - if self._worker is not None and self._dialog is not None: - self._dialog.set_status("Отмена расчёта…") + if self.capture is not None: + self.capture.cancel() + if self._worker is not None and self.dialog is not None: + self.dialog.set_status("Cancelling calculation…") def shutdown(self) -> None: + self._closed = True self.cancel() self._poll.stop() - if self._dialog is not None: - self._dialog.close() + if self.dialog is not None: + self.dialog.close() diff --git a/python_app/gui/controllers/pipeline_controller.py b/python_app/gui/controllers/pipeline_controller.py new file mode 100644 index 0000000..72ec218 --- /dev/null +++ b/python_app/gui/controllers/pipeline_controller.py @@ -0,0 +1,762 @@ +"""Own pipeline processes, shared ingestion, recovery, and exclusive operations.""" + +from __future__ import annotations + +from collections.abc import Callable +from contextlib import AbstractContextManager, nullcontext +import logging +import os +from pathlib import Path +import time + +from PyQt6.QtCore import QCoreApplication, QEventLoop, QTimer + +from python_app.gui.controllers.bscan_replay import BscanReplay +from python_app.gui.controllers.history_controller import HistoryController +from python_app.gui.controllers.sections.views import PrimaryView +from python_app.gui.controllers.plot_controller import PlotController +from python_app.gui.controllers.recording_controller import RecordingController +from python_app.gui.controllers.renderers.sar3d import SAR3D_STATES, sar3d_status +from python_app.gui.runtime.constraints import validate_processing_mode_constraints +from python_app.gui.runtime.history import build_processor_run_signature, build_run_history_signature +from python_app.gui.runtime.pipeline_pause import PipelinePause +from python_app.gui.settings_defaults import env_enabled +from python_app.hardware_full.kamil_adc import apply_kamil_adc_laser_control +from python_app.hardware_full.single_radar_service import create_single_radar_service +from python_app.models.collection_identity import collection_key, index_collections +from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection +from python_app.models.run_config_schema import RunConfigModel +from python_app.models.run_config_validation import validate_acquisition_config +from python_app.orchestration.config_writer import ConfigWriter +from python_app.orchestration.live_processing_config import ProcessingLiveConfig +from python_app.orchestration.pipeline_metrics import PipelineMetrics +from python_app.orchestration.preprocess_assets import ( + PREPROCESS_ASSET_NAMES, REQUIRED_PREPROCESS_ASSET_KEYS, + preprocess_asset_model, runtime_preprocess_asset_keys, +) +from python_app.orchestration.process_supervisor import ProcessSupervisor +from python_app.orchestration.restart_policy import RestartPolicy +from python_app.orchestration.shm.ring_reader import ShmRingReader +from python_app.storage.npz.store import NpzStore +from python_app.workflows.object_calibration import validate_object_set + +logger = logging.getLogger(__name__) +Readers = tuple[ShmRingReader | None, ShmRingReader | None, ShmRingReader | None] + + +class PipelineController: + _READER_ERROR_RELOG_EVERY = 200 + _READER_ERROR_RECONNECT_AT = 40 + _READER_ERROR_STOP_AT = 400 + _RESTART_POLICY = RestartPolicy(min_interval_s=3.0, max_interval_s=60.0) + SINGLE_CAPTURE_TIMEOUT_S = 300.0 + + def __init__( + self, project_root: Path, store: NpzStore, config_writer: ConfigWriter, + history: HistoryController, plots: PlotController, recording: RecordingController, + *, build_config: Callable[[], RunConfigModel], live_config: Callable[[], ProcessingLiveConfig], + publish_live: Callable[[], None], capture_guard: Callable[[], str | None], + refresh_radar_limits: Callable[[], bool], status: Callable[[str], None], + history_changed: Callable[[], None], poll_ui: Callable[[], None], + show_exception: Callable[[str, Exception], None], deferred_close: Callable[[], object], + metrics_every: int | None = None, + event_scope: Callable[[], AbstractContextManager[object]] = nullcontext, + ) -> None: + self._project_root, self._store, self._config_writer = project_root, store, config_writer + self.history, self.plots, self.recording = history, plots, recording + self._build_config, self._live_config, self.publish_live = build_config, live_config, publish_live + self._capture_guard, self._refresh_radar_limits = capture_guard, refresh_radar_limits + self._status, self.history_changed, self._poll_ui = status, history_changed, poll_ui + self._show_exception, self._deferred_close = show_exception, deferred_close + self._event_scope = event_scope + self.supervisor = ProcessSupervisor(project_root) + self.metrics = PipelineMetrics(report_every=self._metrics_every() if metrics_every is None else metrics_every, + log_sink=logger.info) + self.timer = QTimer() + self.timer.setInterval(50) + self.timer.timeout.connect(self.poll) + self.replay = BscanReplay(history, self.reprocess_history, self._warning, logger.debug) + self._warnings: set[str] = set() + self._raw_reader: ShmRingReader | None = None + self._pre_reader: ShmRingReader | None = None + self._result_reader: ShmRingReader | None = None + self.active_config: RunConfigModel | None = None + self.active_config_path: Path | None = None + self.stop_generation = 0 + self.should_run = False + self._pause: PipelinePause | None = None + self._stopping = self._closing = self._close_deferred = self._stop_all_requested = False + self._processor_signature: tuple[object, ...] | None = None + self.single_capture_active = False + self._single_capture_start_ns: int | None = None + self._single_capture_target_key: tuple[int, int] | None = None + self.max_pop_per_poll, self.max_pop_per_snapshot_drain = 256, 4096 + self._last_reader_error_signature: tuple[str, str] | None = None + self._reader_error_repeat_count = self._restart_count = 0 + self._last_restart_s = 0.0 + self._ui_errors: set[tuple[str, str, str]] = set() + + def bind_actions(self, view: PrimaryView) -> None: + view.start.clicked.connect(lambda: self.start()) + view.single_capture.clicked.connect(lambda: self.start(single_capture=True)) + view.stop.clicked.connect(self.stop) + view.apply_radar.clicked.connect(self.apply_radar) + + @staticmethod + def _metrics_every() -> int: + raw = os.environ.get("RADAR_SYSTEM_METRICS_REPORT_EVERY", "").strip() + try: + value = int(raw) if raw else 50 + if value >= 1: + return value + except ValueError: + pass + logger.warning("Ignoring invalid RADAR_SYSTEM_METRICS_REPORT_EVERY=%r; using default 50.", raw) + return 50 + + @property + def running(self) -> bool: + return self.supervisor.is_running() + + @property + def processor_running(self) -> bool: + return self.supervisor.is_processor_running() + + @property + def acquisition_healthy(self) -> bool: + return {"acquisition", "data_preprocessor"}.issubset(self.supervisor.pids()) + + @property + def paused(self) -> bool: + return self._pause is not None + + @property + def stopping(self) -> bool: + return self._stopping + + @property + def closing(self) -> bool: + return self._closing + + @property + def readers(self) -> Readers: + return self._raw_reader, self._pre_reader, self._result_reader + + def _warning(self, text: str, *, details: str | None = None, once_key: str | None = None) -> None: + if once_key is not None: + if once_key in self._warnings: + return + self._warnings.add(once_key) + logger.warning("%s%s", text, f"\n{details}" if details else "") + + def can_resume(self, generation: int) -> bool: + return not self._closing and self.stop_generation == generation + + def operation_cancelled(self) -> None: + if not self._closing: + logger.info("Pipeline operation cancelled by Stop") + + def schedule_startup(self) -> None: + if not env_enabled("RADAR_SYSTEM_AUTO_START"): + return + generation = self.stop_generation + + def start() -> None: + if not self.can_resume(generation): + return + started = self.start() + if not started and env_enabled("RADAR_SYSTEM_HEADLESS") and self.can_resume(generation): + logger.error("Headless auto-start did not bring the pipeline up") + app = QCoreApplication.instance() + if app is not None: + app.exit(1) + + def apply_then_start() -> None: + if self.can_resume(generation): + self.apply_radar() + QTimer.singleShot(1000, start) + + logger.info("Auto-start requested by launcher") + QTimer.singleShot(500, apply_then_start if env_enabled("RADAR_SYSTEM_AUTO_APPLY_RADAR") else start) + + def pause(self) -> PipelinePause: + if self.paused or self._stopping or self._closing: + raise RuntimeError("Wait for the current pipeline operation to finish") + generation, resume = self.stop_generation, self.should_run + + def release(requested: bool) -> None: + if self._pause is operation: + self._pause = None + if requested and resume and self.can_resume(generation): + if not self.start() and self.can_resume(generation): + self.should_run = True + self._last_restart_s = time.monotonic() + self._restart_count += 1 + logger.error("Pipeline resume failed; automatic restart will retry") + + operation = PipelinePause(lambda: self._pause is operation and self.can_resume(generation), release) + # Ownership precedes shutdown: drain pumps Qt and may receive Stop or Close. + self._pause = operation + try: + error = self.stop_pipeline(keep_processor=True) + if error is not None: + raise error + if not operation.valid: + raise InterruptedError("Pipeline operation cancelled") + return operation + except BaseException: + operation.finish(resume=False) + raise + + def _validate_assets(self, config: RunConfigModel) -> None: + missing = [PREPROCESS_ASSET_NAMES[key] for key in REQUIRED_PREPROCESS_ASSET_KEYS + if not preprocess_asset_model(config, key).set_name] + if missing: + raise RuntimeError("Select all required preprocess sets in Preprocessing Panel before Start: " + ", ".join(missing)) + radar_key = config.radar_key() + combos = [ComboKey(input=combo.input, output=combo.output) for combo in config.combos] + keys = runtime_preprocess_asset_keys(config) + logger.info("Active preprocess assets for run: %s", "; ".join( + f"{PREPROCESS_ASSET_NAMES[key]}={preprocess_asset_model(config, key).set_name}" for key in keys)) + for key in keys: + asset = preprocess_asset_model(config, key) + if not self._store.has_combo_coverage(key, radar_key, asset.set_name, combos): + raise RuntimeError(f"Selected {PREPROCESS_ASSET_NAMES[key]} set does not cover requested run combos") + name = config.preprocess.s21.object_calibration.set_name + if name: + collection, metadata = self._store.load_set_with_metadata("s21_object_calibration", radar_key, name) + through = self._store.load_set("s21_calibration", radar_key, config.preprocess.s21.calibration.set_name) + validate_object_set(collection, metadata, config.to_dict(), radar_key, through) + + def start(self, *, single_capture: bool = False, config_override: RunConfigModel | None = None, + pause: PipelinePause | None = None) -> bool: + if self._closing: + return False + generation = self.stop_generation + starting = False + try: + if self._stopping: + raise RuntimeError("Wait for pipeline shutdown before starting again") + if config_override is None or pause is None: + reason = self._capture_guard() + if reason: + raise RuntimeError(reason) + if self._pause is not pause or (pause is not None and not pause.valid): + raise RuntimeError("Wait for the current pipeline operation to finish") + config = config_override.clone() if config_override is not None else self._build_config() + validate_acquisition_config(config) + if config_override is None: + live = self._live_config() + validate_processing_mode_constraints(live.processor_mode, config, live) + self._validate_assets(config) + run_signature, processor_signature = build_run_history_signature(config), build_processor_run_signature(config) + if self.running: + if not single_capture: + raise RuntimeError("Pipeline is already running") + starting = True + logger.info("Stopping continuous pipeline before single capture") + self.stop_pipeline(keep_processor=True) + if not self.can_resume(generation): + return False + starting = True + processor_was_running = self.processor_running + if self._processor_requires_restart(processor_signature): + logger.info("Restarting data_processor because stable run settings changed") + self.stop_all() + if not self.can_resume(generation): + return False + processor_was_running = False + self._config_writer.prepare_preprocess_bundles(self._store, config.radar_key(), config) + if not processor_was_running: + self.reset_history() + config.runtime.continuous = not single_capture + config_path = self._config_writer.write(config, self._project_root / "python_app/runtime/run_config.json") + logger.info("Starting pipeline: mode=%s, config=%s, combos=%d, radar_key=%s", + "single_capture" if single_capture else "continuous", config_path, len(config.combos), config.radar_key()) + if self.history.run_signature is not None and self.history.run_signature != run_signature: + if processor_was_running: + self.reset_history() + self.history.apply_deletion("clear_all") + self.publish_live() + logger.info("History reset because run settings changed") + self.history.run_signature = run_signature + # Timestamp before spawn; a fast one-shot producer can publish inside start(). + started_ns = time.monotonic_ns() if single_capture else None + self.replay.close() + self.discard_replay() + if not self.can_resume(generation) or (pause is not None and not pause.valid): + return False + self.supervisor.start(config_path, allow_clean_acquisition_exit=single_capture) + self._open_readers(config) + self._processor_signature = processor_signature + self.single_capture_active, self._single_capture_start_ns = single_capture, started_ns + self._single_capture_target_key = None + if not single_capture: + self.drop_pending() + self._last_reader_error_signature, self._reader_error_repeat_count = None, 0 + self.active_config, self.active_config_path = config, config_path + self.should_run, self._restart_count = not single_capture and pause is None, 0 + self._status("Status: single capture running" if single_capture else "Status: running") + logger.info("Pipeline started; managed processes: %s", self.supervisor.pids()) + return self.can_resume(generation) and self.running + except Exception as exc: + # Refused actions must not stop another operation or an existing run. + if starting: + self.stop_all() + self._show_exception("Failed to start pipeline", exc) + return False + + def _processor_requires_restart(self, signature: tuple[object, ...]) -> bool: + return self.processor_running and self._processor_signature != signature + + def apply_radar(self) -> None: + try: + reason = self._capture_guard() + if reason: + raise RuntimeError(reason) + config = self._build_config() + validate_acquisition_config(config) + processor_only = self.processor_running and not self.running + with self.pause() as operation: + if config.radar.driver_mode == "native" and not config.radar.replay.dataset_dir: + self._refresh_radar_limits() + if not operation.valid: + return + if processor_only and self._processor_requires_restart(build_processor_run_signature(config)): + self.stop_all() + if not operation.valid: + return + self.reset_history() + self.history.run_signature = None + self._prepare_radar(config) + logger.info("Radar settings applied: start=%g Hz, stop=%g Hz, points=%s, ifbw=%g Hz, power=%g dBm", + config.radar.sweep.start_hz, config.radar.sweep.stop_hz, + "from Kamil ADC stream" if config.is_kamil_adc else config.radar.sweep.points, + config.radar.sweep.if_bandwidth_hz, config.radar.sweep.power_dbm) + except InterruptedError: + self.operation_cancelled() + except Exception as exc: + self._show_exception("Failed to apply radar settings", exc) + + @staticmethod + def _prepare_radar(config: RunConfigModel) -> None: + if config.radar.replay.dataset_dir or config.radar.driver_mode != "native": + logger.info("Radar pre-configuration skipped (replay or mock mode)") + elif config.is_multi_device: + logger.info("Multi-device raw producer will configure all LibreVNA devices") + elif config.is_kamil_adc: + applied = apply_kamil_adc_laser_control(config) + logger.info("Kamil ADC laser_control %s", "applied via Apply Radar" if applied else "disabled") + else: + service = create_single_radar_service(config) + try: + service.open() + service.configure(config.radar.sweep) + finally: + service.close() + logger.info("Radar pre-configured via Python driver: model=%s", config.radar.model) + + def stop(self) -> None: + self.stop_generation += 1 + self.stop_pipeline(keep_processor=True) + + def stop_all(self) -> None: + self.stop_pipeline(keep_processor=False) + + def stop_pipeline(self, *, keep_processor: bool) -> Exception | None: + self.should_run = False + self.replay.close() + if self._stopping: + self._stop_all_requested |= not keep_processor + return + self._stopping, self._stop_all_requested = True, not keep_processor + failure: tuple[str, Exception] | None = None + try: + drain_error: Exception | None = None + try: + if keep_processor and self.running: + self.supervisor.stop_acquisition() + self.drain_rings(timeout_s=0.35, poll_s=0.02) + self.supervisor.stop_preprocessor() + self.drain_rings(timeout_s=0.25, poll_s=0.02) + elif keep_processor: + self.supervisor.stop() + self.drain_once() + if self._stop_all_requested: + self.supervisor.stop_all() + self.drain_rings(timeout_s=0.25, poll_s=0.02) + except Exception as exc: + drain_error = exc + self.stop_generation += 1 + (self.supervisor.stop_all if self._stop_all_requested else self.supervisor.stop)() + if self._stop_all_requested: + self._processor_signature = None + self.recording.finish(reason="pipeline pause" if self.paused else "pipeline closed" if self._closing else "pipeline stopped") + keep_results = not self._stop_all_requested and self.processor_running + ring_name = self._result_reader.name if self._result_reader is not None else None + self._close_readers(keep_results=keep_results and drain_error is None) + self.single_capture_active, self._single_capture_start_ns, self._single_capture_target_key = False, None, None + self.history_changed() + self._status("Status: idle") + logger.info("Acquisition stopped (data_processor kept running)" if keep_results else "Pipeline stopped") + if drain_error is not None: + self._status("Status: error") + failure = "Pipeline stopped, but final ring data could not be read", drain_error + if keep_results and ring_name and not self._closing: + self._result_reader = ShmRingReader(ring_name) + except Exception as exc: + self.stop_generation += 1 + self._status("Status: error") + failure = "Failed to stop pipeline", exc + finally: + self._stopping = False + if self._close_deferred: + self._close_deferred = False + self._deferred_close() + if failure is not None: + if self.paused or self._closing: + logger.error(failure[0], exc_info=failure[1]) + else: + self._show_exception(*failure) + return failure[1] + return None + + def begin_close(self) -> bool: + if not self._closing: + self.stop_generation += 1 + self._closing = True + self.timer.stop() + self.replay.close() + if self._stopping: + self._stop_all_requested = self._close_deferred = True + return False + return True + + def close(self) -> None: + if not self.begin_close(): + return + if self.recording.active: + self.stop_pipeline(keep_processor=True) + self.stop_all() + + def _close_readers(self, *, keep_results: bool = False) -> None: + raw, pre, result = self.readers + self._raw_reader = self._pre_reader = None + if not keep_results: + self._result_reader = None + for reader in (raw, pre, None if keep_results else result): + if reader is not None: + reader.close() + + def _open_readers(self, config: RunConfigModel) -> None: + self._close_readers() + self._raw_reader = ShmRingReader(config.rings.raw_tap.name) if config.rings.raw_tap.name else None + self._pre_reader = ShmRingReader(config.rings.preprocessed_tap.name) if config.rings.preprocessed_tap.name else None + self._result_reader = ShmRingReader(config.rings.results.name) + + def poll(self) -> None: + if self._closing or self._stopping: + return + self._poll_ui() + self._handle_process_exit_reports() + try: + latest = self.read_raw(), self.read_preprocessed(), self.read_results() + except Exception as exc: + self._handle_reader_poll_error(exc) + return + if any(collection is not None for collection in latest): + self._last_reader_error_signature, self._reader_error_repeat_count = None, 0 + for action in (self.recording.poll, self.history_changed): + try: + action() + except Exception as exc: + self._report_ui_error("Runtime display/recording", exc) + try: + if self.single_capture_active: + try: + self._finish_single_capture_if_ready() + finally: + if self.single_capture_active: + self._check_single_capture_deadline() + elif latest[2] is not None: + self._restart_count = 0 + started = time.monotonic_ns() + self.plots.draw(latest[2]) + self.metrics.record("rendering", time.monotonic_ns() - started) + except Exception as exc: + self._report_ui_error("Result display", exc) + + def _report_ui_error(self, context: str, exc: Exception) -> None: + self._status(f"Status: {context.lower()} error (acquisition continues)") + signature = context, type(exc).__name__, str(exc) + if signature not in self._ui_errors: + self._ui_errors.add(signature) + logger.error("%s failed; acquisition continues", context, exc_info=exc) + + def _handle_process_exit_reports(self) -> None: + unexpected = False + try: + for report in self.supervisor.collect_exit_reports(): + if report.level == "INFO": + logger.info(report.format()) + else: + self._status("Status: error") + logger.error(report.format()) + unexpected = True + if not self.paused and self.should_run and (unexpected or not + {"data_processor", "data_preprocessor", "acquisition"}.issubset(self.supervisor.pids())): + self._recover_pipeline_after_crash() + except Exception: + logger.exception("Failed to collect process exit reports") + + def _recover_pipeline_after_crash(self) -> None: + if self.paused or self._closing: + return + now = time.monotonic() + if not self._RESTART_POLICY.should_restart_now(now_s=now, last_restart_s=self._last_restart_s, + consecutive_failures=self._restart_count): + return + self._last_restart_s, self._restart_count = now, self._restart_count + 1 + config, path = self.active_config, self.active_config_path + if config is None or path is None: + self.should_run = False + logger.error("Cannot auto-restart pipeline: no active run configuration recorded.") + return + logger.error("Pipeline process exited unexpectedly; restarting (attempt %d, next back-off %.0fs).", + self._restart_count, self._RESTART_POLICY.backoff_for(self._restart_count)) + try: + # No event-pumping drain in recovery: polling must not re-enter this restart. + self.supervisor.stop_all() + self._close_readers() + self.supervisor.start(path, allow_clean_acquisition_exit=False) + self._open_readers(config) + self._processor_signature = build_processor_run_signature(config) + self.drop_pending() + self._status("Status: running") + logger.info("Pipeline auto-restarted after crash.") + except Exception: + logger.exception("Pipeline auto-restart failed; will retry") + + def _handle_reader_poll_error(self, exc: Exception) -> None: + signature = type(exc).__name__, str(exc) + self._status("Status: error") + self._reader_error_repeat_count += 1 + if self._last_reader_error_signature != signature: + self._last_reader_error_signature = signature + logger.error("Reader poll failed", exc_info=exc) + repeats = self._reader_error_repeat_count + if repeats % self._READER_ERROR_RELOG_EVERY == 0: + logger.error("Reader poll still failing (repeat #%d)", repeats, exc_info=exc) + if repeats >= self._READER_ERROR_STOP_AT: + logger.error("Stopping pipeline after %d consecutive reader poll failures", repeats) + self._last_reader_error_signature, self._reader_error_repeat_count = None, 0 + self.stop_all() + elif repeats == self._READER_ERROR_RECONNECT_AT: + try: + self._raw_reader = self._reconnect(self._raw_reader) + self._pre_reader = self._reconnect(self._pre_reader) + self._result_reader = self._reconnect(self._result_reader) + logger.info("Ring readers reconnected after repeated poll failures") + except Exception: + logger.exception("Ring reader reconnect failed") + + @staticmethod + def _reconnect(reader: ShmRingReader | None) -> ShmRingReader | None: + if reader is None: + return None + name = reader.name + reader.close() + return ShmRingReader(name) + + def _finish_single_capture_if_ready(self) -> bool: + if not self.single_capture_active or self._single_capture_target_key is None: + return False + result = index_collections(self.history.results).get(self._single_capture_target_key) + if result is None or not self.plots.has_output(result): + return False + try: + self.plots.draw(result) + except Exception as exc: + self._report_ui_error("Result display", exc) + status = sar3d_status(result) if self.plots.stack.currentWidget() is self.plots.sar3d.widget else None + self.stop() + if status is not None and status[0] != 0: + state = SAR3D_STATES[status[0]] + self._status(f"Status: {state}") + (logger.error if status[0] == 2 else logger.warning)("Single capture: SAR3D %s: %s", state, status[1]) + else: + logger.info("Single capture completed") + return True + + def _check_single_capture_deadline(self) -> None: + start = self._single_capture_start_ns + if start is not None and time.monotonic_ns() - start > self.SINGLE_CAPTURE_TIMEOUT_S * 1e9: + logger.error("Single capture timed out after %.0fs with no result; stopping.", self.SINGLE_CAPTURE_TIMEOUT_S) + self.stop() + + def read_raw(self, *, max_collections: int | None = None) -> SweepCollection | None: + latest = None + if self._raw_reader is not None: + for _ in range(self.max_pop_per_poll if max_collections is None else max_collections): + collection = self._raw_reader.pop_raw_collection() + if collection is None: + break + self.history.raw.append(collection) + self.recording.record("raw", collection) + self.metrics.record("acquisition", collection.capture_end_ns - collection.capture_start_ns) + self._select_single_target(collection) + latest = collection + return latest + + def read_preprocessed(self, *, max_collections: int | None = None) -> SweepCollection | None: + latest = None + if self._pre_reader is not None: + for _ in range(self.max_pop_per_poll if max_collections is None else max_collections): + collection = self._pre_reader.pop_preprocessed_collection() + if collection is None: + break + self.history.preprocessed.append(collection) + self.recording.record("preprocessed", collection) + latest = collection + return latest + + def read_results(self, *, max_collections: int | None = None) -> ResultCollection | None: + latest = None + if self._result_reader is not None: + for _ in range(self.max_pop_per_poll if max_collections is None else max_collections): + collection = self._result_reader.pop_result_collection() + if collection is None: + break + if self.history.is_removed(collection): + continue + if self._raw_reader is None: + self._select_single_target(collection) + self.metrics.record("processing", collection.processing_duration_ns) + if self.history.record_result(collection): + self.recording.record("results", collection) + latest = collection + return latest + + def _select_single_target(self, collection: SweepCollection | ResultCollection) -> None: + if (self.single_capture_active and self._single_capture_target_key is None + and self._single_capture_start_ns is not None and collection.capture_start_ns >= self._single_capture_start_ns): + self._single_capture_target_key = collection_key(collection) + + def pump_events(self, pause_s: float) -> None: + app = QCoreApplication.instance() + if app is None: + time.sleep(pause_s) + return + deadline = time.monotonic() + pause_s + while (remaining_ms := int((deadline - time.monotonic()) * 1000)) > 0: + with self._event_scope(): + app.processEvents(QEventLoop.ProcessEventsFlag.AllEvents, remaining_ms) + time.sleep(min(0.002, max(0.0, deadline - time.monotonic()))) + + def drain_once(self) -> bool: + return any(collection is not None for collection in (self.read_raw(), self.read_preprocessed(), self.read_results())) + + def drain_rings(self, *, timeout_s: float, poll_s: float) -> None: + deadline, stable = time.monotonic() + timeout_s, 0 + while time.monotonic() < deadline and stable < 2: + stable = 0 if self.drain_once() else stable + 1 + self.pump_events(poll_s) + + def drain_results(self, *, timeout_s: float, poll_s: float) -> ResultCollection | None: + deadline, stable = time.monotonic() + timeout_s, 0 + latest_seen = None + while self._result_reader is not None and time.monotonic() < deadline and (latest_seen is None or stable < 2): + latest = self.read_results() + if latest is not None: + latest_seen, stable = latest, 0 + elif latest_seen is not None: + stable += 1 + self.pump_events(poll_s) + return latest_seen + + def drain_snapshot(self) -> None: + initial_readers = self.readers + if all(reader is None for reader in initial_readers): + return + try: + observed = [False, False, False] + deadline = time.monotonic() + 1.2 + while time.monotonic() < deadline and not self._closing: + latest = (self.read_raw(max_collections=self.max_pop_per_snapshot_drain), + self.read_preprocessed(max_collections=self.max_pop_per_snapshot_drain), + self.read_results(max_collections=self.max_pop_per_snapshot_drain)) + observed = [seen or value is not None for seen, value in zip(observed, latest)] + if any(value is not None for value in latest): + continue + missing = any(reader is not None and not seen for reader, seen in zip(initial_readers, observed)) + awaiting = self._result_reader is not None and max(len(self.history.raw), len(self.history.preprocessed)) > len(self.history.results) + if (any(observed) and missing) or awaiting: + self.pump_events(0.01) + else: + break + except Exception: + logger.warning("Snapshot drain warning", exc_info=True) + + def drop_pending(self, *, include_results: bool = True) -> tuple[int, int, int]: + raw, pre, result = self.readers + dropped = (raw.drop_all() if raw is not None else 0, pre.drop_all() if pre is not None else 0, + result.drop_all() if include_results and result is not None else 0) + if any(dropped): + logger.info("Ring reset: raw=%d, preprocessed=%d, results=%d", *dropped) + return dropped + + def settle_deleted_results(self) -> None: + if self.processor_running: + self.drain_results(timeout_s=0.35, poll_s=0.01) + if self._result_reader is not None: + self._result_reader.drop_all() + + def reset_history(self) -> None: + self.history.raw.clear() + self.history.preprocessed.clear() + self.history.results.clear() + self.discard_replay() + self.plots.clear() + self.history_changed() + + def preprocessing_changed(self, *, restart_processor: bool = False) -> None: + self.history.run_signature = None + if restart_processor: + self._processor_signature = None + try: + if self.processor_running: + self.history.apply_deletion("clear_all") + self.publish_live() + finally: + self.reset_history() + + def can_reprocess_history(self, *, mode: str | None = None) -> bool: + bscan = mode == "bscan" if mode is not None else self.plots.stack.currentWidget() is self.plots.bscan.widget + return (not self._closing and bscan + and bool(self.history.preprocessed) and not self.running and self.processor_running + and self._result_reader is not None) + + def schedule_replay(self) -> None: + self.replay.schedule(self.can_reprocess_history()) + + def discard_replay(self) -> None: + self.replay.discard() + self.plots.bscan.signature = None + + def reprocess_history(self) -> None: + if self.replay.active or not self.can_reprocess_history(): + return + config, reader, generation = self.active_config, self._result_reader, self.stop_generation + polling = self.timer.isActive() + self.timer.stop() + try: + if self.replay.run(window=self.plots.window_scans(), revision=self.plots.required_revision or 0, + ring=config.rings.preprocessed if config is not None else None, reader=reader, + pump_events=self.pump_events, continue_replay=lambda: self.can_resume(generation) + and not self._stopping and self.can_reprocess_history() and self._result_reader is reader): + self.plots.redraw() + except Exception as exc: + self._show_exception("Failed to recompute the B-scan", exc) + finally: + if polling and not self._closing: + self.timer.start() diff --git a/python_app/gui/controllers/plot_controller.py b/python_app/gui/controllers/plot_controller.py new file mode 100644 index 0000000..d08ed04 --- /dev/null +++ b/python_app/gui/controllers/plot_controller.py @@ -0,0 +1,169 @@ +"""Select radar views and supply independent renderers with settings and history.""" + +from collections.abc import Callable + +from PyQt6.QtCore import QSignalBlocker +from PyQt6.QtWidgets import QComboBox, QPushButton, QStackedWidget, QWidget + +from python_app.gui.controllers.renderers.sar3d import Sar3dRenderer +from python_app.gui.controllers.history_controller import HistoryController +from python_app.gui.controllers.renderers.bscan import BscanRenderer +from python_app.gui.controllers.renderers.gpr import GprRenderer +from python_app.gui.controllers.renderers.trace import TraceRenderer +from python_app.models.dataset_model import ResultCollection, TraceData +from python_app.models.gui_profile_schema import GuiProcessingStateModel +from python_app.models.run_config_schema import GprModel + + +class PlotController: + def __init__( + self, history: HistoryController, settings: Callable[[], GuiProcessingStateModel], + geometry: Callable[[], GprModel], running: Callable[[], bool], + warning: Callable[..., None], + ) -> None: + self.history = history + self._settings = settings + self._geometry = geometry + self._running = running + self._warning = warning + self.required_revision: int | None = None + self.live_window = 50 + self.bscan = BscanRenderer() + self.gpr = GprRenderer() + self.sar3d = Sar3dRenderer() + self.trace = TraceRenderer(warning) + self.stack = QStackedWidget() + for renderer in (self.bscan, self.gpr, self.sar3d, self.trace): + self.stack.addWidget(renderer.widget) + self.stack.setCurrentWidget(self.trace.widget) + self._combo: QComboBox | None = None + self._next: QPushButton | None = None + + def bind_display_controls(self, combo: QComboBox, next_button: QPushButton) -> None: + self._combo, self._next = combo, next_button + combo.currentIndexChanged.connect(self.on_display_combo_changed) + next_button.clicked.connect(self.select_next_combo) + self.sync_display_combos() + + @property + def display_key(self) -> tuple[int, int] | None: + return self._combo.currentData() if self._combo is not None else None + + @property + def capture_widget(self) -> QWidget | None: + current = self.stack.currentWidget() + return self.sar3d.grid if current is self.sar3d.widget else current + + def set_mode(self, mode: str) -> None: + renderer = {"bscan": self.bscan, "gpr": self.gpr, "legacy_gpr": self.gpr, "sar3d": self.sar3d}.get(mode, self.trace) + self.stack.setCurrentWidget(renderer.widget) + self.sync_display_combos() + + def select_combo(self, input_pos: int, output_pos: int) -> None: + key = input_pos, output_pos + keys = sorted(self.bscan.history_by_combo) + if key not in keys or self._combo is None: + raise ValueError(f"Unknown channel combination: {input_pos}:{output_pos}") + self._combo.setCurrentIndex(keys.index(key)) + + def select_next_combo(self) -> None: + combo = self._combo + if combo is not None and combo.count(): + combo.setCurrentIndex((combo.currentIndex() + 1) % combo.count()) + + def sync_display_combos(self) -> None: + combo, next_button = self._combo, self._next + if combo is None or next_button is None: + return + keys = sorted(self.bscan.history_by_combo) + current = combo.currentData() + if keys != [combo.itemData(i) for i in range(combo.count())]: + with QSignalBlocker(combo): + combo.clear() + for key in keys: + combo.addItem(f"{key[0]} : {key[1]}", key) + combo.setCurrentIndex(keys.index(current) if current in keys else 0) + enabled = bool(keys) and self._settings().selected_mode == "bscan" + combo.setEnabled(enabled) + next_button.setEnabled(enabled and len(keys) > 1) + + def window_scans(self) -> int: + return self.live_window if self._running() else max(1, self._settings().bscan.history_window_scans) + + def sync_bscan(self) -> None: + results = self.history.bscan_source() + self.bscan.sync(results, self.window_scans()) + self.sync_display_combos() + + def draw_bscan(self) -> bool: + self.sync_bscan() + window, retained = self.window_scans(), len(self.history.preprocessed) + if retained < window and self.bscan.history_by_combo: + self._warning( + f"B-scan is set to show {window} sweeps but only {retained} are retained; " + "the image shows what history there is.", + details="Capture more sweeps, or reduce 'Scans to show (stopped)'.", + once_key=f"bscan_window_underfilled_{window}", + ) + return self.bscan.draw(self.display_key, self._settings().bscan) + + def on_display_combo_changed(self, *_args: object) -> None: + if self._settings().selected_mode == "bscan": + self.draw_bscan() + + def draw(self, collection: ResultCollection) -> bool: + settings = self._settings() + mode = settings.selected_mode + if mode in {"gpr", "legacy_gpr", "sar3d"} and self.required_revision is not None: + if collection.settings_revision != self.required_revision: + return False + if mode == "bscan": + return self.draw_bscan() + if mode in {"gpr", "legacy_gpr"}: + return self.gpr.draw(collection, settings.legacy_gpr if mode == "legacy_gpr" else settings.gpr, self._geometry()) + if mode == "sar3d": + return self.sar3d.draw(collection) + return self.trace.draw(collection, settings.pass_through) + + def redraw(self, *_args: object) -> None: + mode = self._settings().selected_mode + if mode == "bscan": + self.draw_bscan() + elif mode == "sar3d": + self.sar3d.clear() + if self.history.results: + self.draw(self.history.results[-1]) + elif not self.history.results or not self.draw(self.history.results[-1]): + self.clear_current() + + def draw_trace(self, trace: TraceData, title: str) -> None: + self.trace.draw_trace(trace, title, self._settings().pass_through) + + def clear_bscan(self) -> None: + self.bscan.clear() + self.sync_display_combos() + + def clear_gpr(self) -> None: + settings = self._settings() + self.gpr.clear(settings.legacy_gpr if settings.selected_mode == "legacy_gpr" else settings.gpr) + + def clear_current(self) -> None: + mode = self._settings().selected_mode + if mode == "bscan": + self.clear_bscan() + elif mode in {"gpr", "legacy_gpr"}: + self.clear_gpr() + elif mode == "sar3d": + self.sar3d.clear() + else: + self.trace.clear() + + def clear(self) -> None: + self.clear_bscan() + self.clear_gpr() + self.sar3d.clear() + self.trace.clear() + + @staticmethod + def has_output(collection: ResultCollection) -> bool: + return bool(collection.collection_payloads) or any(p.kind == 1 for b in collection.blocks for p in b.payloads) diff --git a/python_app/gui/controllers/preprocess_controller.py b/python_app/gui/controllers/preprocess_controller.py new file mode 100644 index 0000000..b7e8e15 --- /dev/null +++ b/python_app/gui/controllers/preprocess_controller.py @@ -0,0 +1,580 @@ +"""Select preprocessing assets and own paused hardware capture sessions.""" + +from __future__ import annotations + +from collections.abc import Callable +from contextlib import nullcontext +from copy import deepcopy +from dataclasses import asdict +import logging + +from PyQt6.QtCore import QTimer +from PyQt6.QtWidgets import QWidget + +from python_app.gui.controllers.object_calibration_capture import ObjectSweepCapture +from python_app.gui.controllers.object_calibration_controller import ObjectCalibrationController +from python_app.gui.controllers.pipeline_controller import PipelineController +from python_app.gui.controllers.plot_controller import PlotController +from python_app.gui.controllers.sections.views import DataView, PrimaryView +from python_app.gui.object_calibration_dialog import ObjectCalibrationDialog +from python_app.gui.preprocess_dialog import PreprocessDialog +from python_app.gui.runtime.pipeline_pause import PipelinePause +from python_app.gui.trace_png_export import export_trace_png +from python_app.models.gui_profile_schema import GuiPreprocessDialogStateModel +from python_app.models.run_config_schema import ComboModel, GprModel, RunConfigModel +from python_app.orchestration.preprocess_assets import ( + PREPROCESS_ASSET_NAMES, + preprocess_asset_display_name, preprocess_asset_model, +) +from python_app.storage.npz.store import NpzStore +from python_app.workflows.neutral_preprocess import build_neutral_s21_sets, supports_neutral_preprocess_sets +from python_app.workflows.object_calibration import capture_provenance, geometry_signature +from python_app.workflows.radar_config_variants import RadarConfigVariant, scan_radar_config_variants +from python_app.workflows.reference_workflow import capture_reference_set +from python_app.workflows.sequential_capture_workflow import CaptureBatch, SavedCaptureSet, SequentialCaptureSession + +logger = logging.getLogger(__name__) +TMP_REFERENCE_SET_NAME = "tmp_reference" + + +class PreprocessController: + def __init__( + self, parent: QWidget, config: RunConfigModel, state: GuiPreprocessDialogStateModel, + store: NpzStore, pipeline: PipelineController, plots: PlotController, *, + build_config: Callable[[], RunConfigModel], current_radar_key: Callable[[], str], + geometry: Callable[[], GprModel], summary: Callable[[dict[str, str]], None], + error: Callable[[str, Exception], None], + ) -> None: + self._parent, self._store, self._pipeline, self._plots = parent, store, pipeline, plots + self._build_config, self._current_radar_key, self._geometry = build_config, current_radar_key, geometry + self._summary, self._error = summary, error + self._defaults, self._settings = config, deepcopy(state) + self.selected_sets = {key: preprocess_asset_model(config, key).set_name for key in PREPROCESS_ASSET_NAMES} + self.radar_key = config.radar_key() + self.dialog: PreprocessDialog | None = None + self.capture_session: SequentialCaptureSession | None = None + self.capture_busy = False + self._tmp_reference_busy = False + self._capture_pause: PipelinePause | None = None + self._radar_variants: list[RadarConfigVariant] = [] + self.object_calibration = ObjectCalibrationController( + parent, store, build_config, self._capture_object, self._select_object, error, + ) + + def bind_actions(self, primary: PrimaryView, data: DataView) -> None: + primary.preprocess.clicked.connect(self.open_panel) + data.capture_tmp.clicked.connect(self.capture_tmp_reference) + + @property + def object_capture(self) -> ObjectSweepCapture | None: + return self.object_calibration.capture + + def settings(self) -> GuiPreprocessDialogStateModel: + if self.dialog is not None: + self._settings = GuiPreprocessDialogStateModel( + set_name=self.dialog.set_name(), radar_config_dir=self.dialog.radar_config_dir(), + use_all_radar_configs=self.dialog.use_all_radar_configs(), + median_sweep_count=self.dialog.median_sweep_count(), + ) + return deepcopy(self._settings) + + def apply_profile(self, config: RunConfigModel, state: GuiPreprocessDialogStateModel) -> None: + self._defaults, self._settings = config, deepcopy(state) + self.selected_sets.update({key: preprocess_asset_model(config, key).set_name for key in PREPROCESS_ASSET_NAMES}) + self.radar_key = config.radar_key() + self._summary(self.selected_sets) + if self.dialog is not None: + self._apply_dialog_settings() + + def _apply_dialog_settings(self) -> None: + assert self.dialog is not None + self.dialog.apply_settings(self._settings, self.selected_sets, + neutral_visible=supports_neutral_preprocess_sets(self._defaults)) + + def ensure_dialog(self) -> PreprocessDialog: + if self.dialog is None: + self.dialog = PreprocessDialog(self._parent) + self._apply_dialog_settings() + self.dialog.refresh_requested.connect(self._refresh_requested) + self.dialog.selection_changed.connect(self._selection_changed) + self.dialog.radar_config_dir_changed.connect(self._refresh_requested) + self.dialog.multi_radar_option_changed.connect(self._refresh_requested) + self.dialog.start_sequence_requested.connect(self.start_sequence) + self.dialog.capture_next_requested.connect(self.capture_next) + self.dialog.capture_all_requested.connect(self.capture_all) + self.dialog.undo_last_requested.connect(self.undo) + self.dialog.finalize_sequence_requested.connect(self.finalize) + self.dialog.abort_sequence_requested.connect(self.abort) + self.dialog.create_neutral_sets_requested.connect(self.create_neutral_sets) + self.dialog.create_object_calibration_requested.connect( + lambda: self.object_calibration.open_creation(self.settings().set_name)) + self._update_capture_state() + return self.dialog + + def open_panel(self) -> None: + try: + dialog = self.ensure_dialog() + self.refresh_sets() + self._update_capture_state() + dialog.showMaximized() + dialog.raise_() + dialog.activateWindow() + except Exception as exc: + self._error("Failed to open preprocessing panel", exc) + + def busy_reason(self) -> str | None: + if self._pipeline.stopping: + return "Wait for pipeline shutdown to finish" + if self.object_capture is not None: + return "Wait for object-calibration capture to finish" + if self.capture_session is not None: + return "Finish the active capture sequence first" + if self.capture_busy or self._tmp_reference_busy: + return "Wait for preprocessing capture to finish" + return None + + def data_action_allowed(self, *, deletion: bool = False) -> bool: + if self._pipeline.closing: + return False + if self._pipeline.stopping: + self._error("Cannot perform data action", RuntimeError(self.busy_reason())) + return False + if self.object_capture is not None: + self._error("Wait for object-calibration capture to finish", RuntimeError("Calibration capture is active")) + return False + if deletion and self.capture_session is not None: + self._error("Cannot modify runtime history during active capture sequence", RuntimeError(self.capture_state_details())) + return False + return True + + def _can_capture(self) -> bool: + if self._pipeline.closing: + return False + reason = self.busy_reason() + if reason: + self._error(reason, RuntimeError(self.capture_state_details())) + return False + return True + + def capture_state_details(self) -> str: + if self.capture_session is None: + return "Capture session: none" + state = self.capture_session.state() + lines = ["Capture session:", f"- kind={state.kind}", f"- progress={state.captured_count}/{state.total_count}"] + if state.current_combo is not None: + lines.append(f"- current_combo=input={state.current_combo.input}, output={state.current_combo.output}") + return "\n".join(lines) + + def reset_selection(self) -> None: + try: + radar_key = self._current_radar_key() + except Exception: + logger.debug("Skipping preprocess-selection reset; radar key unavailable", exc_info=True) + return + if radar_key == self.radar_key: + return + previous_key, self.radar_key = self.radar_key, radar_key + cleared = {key: value for key, value in self.selected_sets.items() if value} + self.selected_sets.update(dict.fromkeys(PREPROCESS_ASSET_NAMES, "")) + self._summary(self.selected_sets) + if self.dialog is not None: + self.dialog.set_selected_sets(self.selected_sets, emit_signal=False) + self._refresh_requested() + if cleared: + logger.info("Preprocess selection reset: %s -> %s; cleared %s", previous_key, radar_key, cleared) + + def _selection_changed(self) -> None: + previous = dict(self.selected_sets) + self.selected_sets.update(self.ensure_dialog().selection_snapshot()) + self._summary(self.selected_sets) + changes = [f"{preprocess_asset_display_name(key)}: {previous.get(key) or ''} -> " + f"{self.selected_sets.get(key) or ''}" for key in PREPROCESS_ASSET_NAMES + if previous.get(key) != self.selected_sets.get(key)] + if changes: + logger.info("Preprocess selection changed: %s", "; ".join(changes)) + + def refresh_sets(self) -> None: + self.radar_key = self._current_radar_key() + dialog = self.ensure_dialog() + self.settings() + available = {key: self._store.list_sets(key, self.radar_key) + for key in PREPROCESS_ASSET_NAMES} + dialog.set_available_sets(available) + self._refresh_variants() + for key, names in available.items(): + selected = self.selected_sets.get(key, "") + if selected and selected not in names: + logger.warning("%s set %s is unavailable for radar key %s", key, selected, self.radar_key) + dialog.set_selected_sets(self.selected_sets) + dialog.set_neutral_sets_visible(supports_neutral_preprocess_sets(self._defaults)) + self._summary(self.selected_sets) + logger.info("Preprocess set lists refreshed: radar_key=%s, counts=%s", self.radar_key, + {key: len(names) for key, names in available.items()}) + + def _refresh_requested(self) -> None: + try: + self.refresh_sets() + except Exception as exc: + self._error("Failed to refresh preprocessing sets", exc) + + def _refresh_variants(self) -> None: + self._radar_variants, summary = scan_radar_config_variants( + self.settings().radar_config_dir, base_config=self._build_config()) + if self.dialog is not None: + values = asdict(summary) + values.pop("issues") + self.dialog.set_radar_config_summary(**values) + if summary.issues: + logger.warning("Some radar config JSON files were skipped:\n%s", "\n".join(summary.issues)) + + def capture_tmp_reference(self) -> None: + if self._tmp_reference_busy: + logger.info("Tmp reference capture already in progress; ignoring duplicate request") + return + if not self._can_capture(): + return + self._tmp_reference_busy = True + try: + with self._pipeline.pause(): + try: + config = self._build_config() + radar_key, collection = capture_reference_set( + config, TMP_REFERENCE_SET_NAME, self._store, + median_sweep_count=self.settings().median_sweep_count, + ) + self.selected_sets["s21_reference"] = TMP_REFERENCE_SET_NAME + self.radar_key = radar_key + # Reference reload must keep the processor and its TCP locator alive. + self._pipeline.preprocessing_changed() + self._summary(self.selected_sets) + if collection.traces: + self._plots.draw_trace(collection.traces[-1], "Tmp S21 Reference last trace") + if self.dialog is not None: + self.refresh_sets() + self.dialog.set_status(f"S21 Reference set saved: {TMP_REFERENCE_SET_NAME} ({len(collection.traces)} traces)") + logger.info("Tmp S21 Reference captured: set=%s, key=%s, traces=%d; runtime history reset", + TMP_REFERENCE_SET_NAME, radar_key, len(collection.traces)) + finally: + self._tmp_reference_busy = False + except InterruptedError: + self._pipeline.operation_cancelled() + except Exception as exc: + self._error("Failed to capture tmp reference", exc) + finally: + self._tmp_reference_busy = False + + def create_neutral_sets(self) -> None: + if not self._can_capture(): + return + try: + dialog = self.ensure_dialog() + name, config = dialog.set_name(), self._build_config() + if not name: + raise ValueError("Set name is required") + if not supports_neutral_preprocess_sets(config): + raise ValueError("Neutral S21 sets are not supported for this radar configuration") + radar_key = config.radar_key() + for kind in ("s21_calibration", "s21_reference"): + if name in self._store.list_sets(kind, radar_key): + raise FileExistsError(f"Set '{name}' already exists for {preprocess_asset_display_name(kind)}") + with self._pipeline.pause(): + calibration, reference = build_neutral_s21_sets(config) + self._store.save_sets(radar_key, name, {"s21_calibration": calibration, "s21_reference": reference}) + self.selected_sets.update(s21_calibration=name, s21_reference=name) + self.radar_key = radar_key + self._pipeline.preprocessing_changed(restart_processor=True) + self.refresh_sets() + dialog.set_status(f"Neutral S21 sets saved: {name} ({len(calibration.traces)} combos, " + f"{calibration.traces[0].frequency_hz.size} points)") + logger.info("Neutral S21 sets saved: set=%s, radar_key=%s", name, radar_key) + except InterruptedError: + self._pipeline.operation_cancelled() + except Exception as exc: + self._error("Failed to create neutral S21 sets", exc) + + + def start_sequence(self, kind: str) -> None: + if not self._can_capture(): + return + pause = None + try: + dialog = self.ensure_dialog() + name = dialog.set_name() + if not name: + raise ValueError("Set name is required") + pause = self._pipeline.pause() + self._capture_pause = pause + config, settings = self._build_config(), self.settings() + if settings.use_all_radar_configs: + self._refresh_variants() + if not self._radar_variants: + raise ValueError("No valid radar config variants were found in the selected directory") + keys = [variant.radar_key for variant in self._radar_variants] + else: + keys = [config.radar_key()] + duplicates = [key for key in keys if name in self._store.list_sets(kind, key)] + if duplicates: + raise FileExistsError(f"Set '{name}' already exists for {kind}: {', '.join(duplicates)}") + session = SequentialCaptureSession( + config, kind, name, median_sweep_count=settings.median_sweep_count, + radar_variants=self._radar_variants if settings.use_all_radar_configs else None, + ) + session.open() + self.capture_session = session + dialog.clear_capture_log() + dialog.reset_preview() + dialog.set_status(f"{preprocess_asset_display_name(kind)} sequence started") + self._plots.trace.clear() + self._update_capture_state() + logger.info("%s sequence started: set=%s, combos=%d, radar_variants=%d, median_sweeps=%d", + kind, name, session.state().total_count, len(keys), settings.median_sweep_count) + except InterruptedError: + self._pipeline.operation_cancelled() + self._release_pause() + except Exception as exc: + if pause is not None: + self._cleanup_session() + if pause is not None: + self._release_pause() + self._error(f"Failed to start {kind} sequence", exc) + + def _begin_capture(self) -> bool: + if self._pipeline.closing or self.capture_busy: + logger.info("Preprocess combo capture already in progress; ignoring duplicate request") + return False + self.capture_busy = True + self._update_capture_state() + return True + + def capture_next(self) -> None: + self._capture_remaining(all_remaining=False) + + def capture_all(self) -> None: + self._capture_remaining(all_remaining=True) + + def _capture_remaining(self, *, all_remaining: bool) -> None: + session = self.capture_session + if session is None or session.is_complete(): + self._error("Cannot capture preprocess data", RuntimeError("No incomplete capture sequence")) + return + if not self._begin_capture(): + return + sequence = self._pipeline.stop_generation + if all_remaining: + self.ensure_dialog().set_status(f"{preprocess_asset_display_name(session.kind)} batch capture started") + try: + while self.capture_session is session and self._pipeline.can_resume(sequence): + try: + captured = session.capture_current_combo() + except Exception as exc: + state = session.state() + self._error("Failed to capture combo; previous captures kept, retry when ready", exc) + self.ensure_dialog().set_status( + f"{preprocess_asset_display_name(session.kind)} capture failed: " + f"{state.captured_count}/{state.total_count} kept, ready to retry") + return + self._record_capture(session, captured) + if not all_remaining or session.is_complete(): + break + self._pipeline.pump_events(0.01) + if all_remaining and self.capture_session is session and not session.is_complete(): + self.ensure_dialog().set_status( + f"{preprocess_asset_display_name(session.kind)} batch capture paused; previous captures kept") + finally: + self.capture_busy = False + # Queued clicks must reach disabled buttons before they are re-enabled. + QTimer.singleShot(0, self._update_capture_state) + + def _record_capture(self, session: SequentialCaptureSession, captured: CaptureBatch) -> None: + dialog, state = self.ensure_dialog(), session.state() + display = preprocess_asset_display_name(session.kind) + trace = captured.display_trace + details = f" | radar_configs={len(captured.traces)}" if state.variant_count > 1 else "" + dialog.append_capture_log_entry(kind=display, captured_count=state.captured_count, total_count=state.total_count, + input_pos=trace.combo.input, output_pos=trace.combo.output, extra_details=details) + dialog.draw_last_trace(trace, title=f"{display} captured") + self._plots.draw_trace(trace, f"{display} last trace") + logger.info("%s capture: %d/%d | input=%d output=%d%s", display, state.captured_count, + state.total_count, trace.combo.input, trace.combo.output, details) + self._update_capture_state() + if session.is_complete(): + dialog.set_status(f"{display} sequence complete. Review captures or save the set.") + + def undo(self) -> None: + session = self.capture_session + if session is None: + self._error("No active capture sequence", RuntimeError(self.capture_state_details())) + return + try: + session.undo_last_capture() + dialog, state = self.ensure_dialog(), session.state() + display = preprocess_asset_display_name(session.kind) + captures = session.captured_batches() + extra = f" | radar_configs={state.variant_count}" if state.variant_count > 1 else "" + dialog.set_capture_log_entries([ + f"{display}: {index}/{state.total_count} | input={item.combo.input} output={item.combo.output}{extra}" + for index, item in enumerate(captures, start=1)]) + trace = session.last_captured_trace() + if trace is None: + dialog.reset_preview() + self._plots.trace.clear() + else: + dialog.draw_last_trace(trace, title=f"{display} last trace") + self._plots.draw_trace(trace, f"{display} last trace") + dialog.set_status(f"{display} last capture removed. Ready to recapture.") + self._update_capture_state() + logger.info("%s undo last capture; remaining=%d/%d", display, state.captured_count, state.total_count) + except Exception as exc: + self._error("Failed to undo last preprocess capture", exc) + + def finalize(self) -> None: + session = self.capture_session + if session is None or not session.is_complete(): + self._error("Capture sequence is not complete", RuntimeError(self.capture_state_details())) + return + try: + try: + dialog = self.ensure_dialog() + kind, name = session.kind, session.set_name + saved = session.finalize(self._store) + status = (f"{len(saved)} radar variants" if len(saved) > 1 + else f"{len(saved[0].collection.traces)} traces") + if self.settings().use_all_radar_configs: + self._export_previews(session, saved) + self._cleanup_session() + self.selected_sets[kind] = name + self._summary(self.selected_sets) + dialog.set_status(f"{preprocess_asset_display_name(kind)} set saved: {name} ({status})") + logger.info("%s sequence saved: set=%s (%s)", kind, name, status) + self.refresh_sets() + finally: + if self.capture_session is None: + self._release_pause() + except Exception as exc: + self._error("Failed to save preprocess set", exc) + + def _export_previews(self, session: SequentialCaptureSession, saved: list[SavedCaptureSet]) -> None: + for item in saved: + for trace in item.collection.traces: + try: + path = self._store.preview_png_dir(session.kind, session.set_name, item.radar_key) / f"i{trace.combo.input}_o{trace.combo.output}.png" + export_trace_png(trace, path, title=f"{preprocess_asset_display_name(session.kind)} | {item.display_name} | " + f"input={trace.combo.input} output={trace.combo.output}") + except Exception: + # Data are already saved; rendering failure must not abort completion. + logger.warning("Failed to save preview PNG for %s", item.display_name, exc_info=True) + + def abort(self, *, resume_pipeline: bool = True) -> None: + if self.capture_session is None: + return + try: + try: + kind = self.capture_session.kind + self._cleanup_session() + self.ensure_dialog().set_status(f"{preprocess_asset_display_name(kind)} sequence aborted") + logger.info("%s sequence aborted", kind) + finally: + self._release_pause(resume=resume_pipeline) + except Exception as exc: + self._error("Failed to abort capture sequence", exc) + + def _cleanup_session(self) -> None: + session, self.capture_session = self.capture_session, None + try: + if session is not None: + session.close() + except Exception: + self._release_pause(resume=False) + raise + finally: + self._update_capture_state() + + def _release_pause(self, *, resume: bool = True) -> None: + pause, self._capture_pause = self._capture_pause, None + if pause is not None: + pause.finish(resume=resume) + + def _update_capture_state(self) -> None: + # A queued post-capture callback can outlive the window and its Qt children. + if self._pipeline.closing or self.dialog is None: + return + state = self.capture_session.state() if self.capture_session is not None else None + combo = state.current_combo if state else None + self.dialog.set_capture_state( + kind=state.kind if state else None, captured_count=state.captured_count if state else 0, + total_count=state.total_count if state else 0, next_input=combo.input if combo else None, + next_output=combo.output if combo else None, can_undo=state.can_undo if state else False, + can_finalize=state.is_complete if state else False, + can_capture_all=bool(state and state.supports_batch_capture and not state.is_complete and combo), + variant_count=state.variant_count if state else 1, actions_enabled=not self.capture_busy, + ) + + def _capture_object(self, dialog: ObjectCalibrationDialog) -> ObjectSweepCapture: + reason = self.busy_reason() + if reason or self._pipeline.closing or self._pipeline.stopping: + raise RuntimeError(reason or "Pipeline is stopping") + if self._pipeline.single_capture_active or self._pipeline.recording.active: + raise RuntimeError("Finish the single capture or disk recording before taking calibration data") + config = self._build_config() + if not config.rings.preprocessed_tap.name: + raise ValueError("Object calibration requires rings.preprocessed_tap; enable it in the run profile") + if config.radar.replay.dataset_dir.strip(): + raise ValueError("Object-calibration capture requires live acquisition; disable archive replay") + config.preprocess.s21.object_calibration.set_name = "" + config.preprocess.s21.object_calibration.bundle_path = "" + tx, rx = sorted({entry.output_pos for entry in config.gpr.tx_geometry}), sorted({entry.input_pos for entry in config.gpr.rx_geometry}) + if not tx or not rx: + raise ValueError("Configure Tx/Rx antenna positions before taking calibration data") + config.combos = [ComboModel(input=i, output=o) for o in tx for i in rx] + radar_key = config.radar_key() + through = self._store.load_set("s21_calibration", radar_key, config.preprocess.s21.calibration.set_name) + provenance = capture_provenance(config.to_dict(), radar_key, through) + return ObjectSweepCapture(self._pipeline, dialog, config, self.settings().median_sweep_count, provenance) + + def _select_object(self, name: str) -> None: + self.selected_sets["s21_object_calibration"] = name + self.ensure_dialog().set_set_name(name) + self._summary(self.selected_sets) + self._refresh_requested() + logger.info("Selected object calibration: %s", name) + + def invalidate_object_calibration_after_geometry_change(self) -> None: + selected = self.selected_sets.get("s21_object_calibration", "") + active_config = self._pipeline.active_config if self._pipeline.running else None + active = active_config.preprocess.s21.object_calibration.set_name if active_config else "" + if not selected and not active: + return + geometry = geometry_signature({"gpr": asdict(self._geometry())}) + selections = [(selected, self.radar_key)] + if active_config is not None: + selections.append((active, active_config.radar_key())) + invalid: set[str] = set() + for name, radar_key in dict.fromkeys(selections): + if not name: + continue + try: + _, metadata = self._store.load_set_with_metadata("s21_object_calibration", radar_key, name) + if metadata["provenance"]["geometry"] != geometry: + raise ValueError("Antenna geometry differs from the captured calibration") + except Exception as exc: + invalid.add(name) + logger.warning("Object calibration %s disabled for changed antenna geometry: %s", name, exc) + if not invalid: + return + try: + with self._pipeline.pause() if active in invalid else nullcontext(): + if selected in invalid: + self.selected_sets["s21_object_calibration"] = "" + self._summary(self.selected_sets) + if self.dialog is not None: + self.dialog.set_selected_sets(self.selected_sets, emit_signal=False) + except InterruptedError: + self._pipeline.operation_cancelled() + except Exception as exc: + self._error("Failed to update object calibration after geometry change", exc) + + def shutdown(self) -> None: + self.object_calibration.shutdown() + self.abort(resume_pipeline=False) + if self.dialog is not None: + self.dialog.close() diff --git a/python_app/gui/controllers/profile_controller.py b/python_app/gui/controllers/profile_controller.py new file mode 100644 index 0000000..d505bcb --- /dev/null +++ b/python_app/gui/controllers/profile_controller.py @@ -0,0 +1,198 @@ +"""GUI profile persistence, startup selection and application to concrete editors.""" + +from copy import deepcopy +from dataclasses import dataclass +import os +from pathlib import Path + +from PyQt6.QtWidgets import QFileDialog, QWidget + +from python_app.gui.controllers.config_controller import ConfigController +from python_app.gui.controllers.sections.views import PrimaryView +from python_app.file_io import atomic_write_json +from python_app.gui.controllers.live_settings_controller import LiveSettingsController +from python_app.gui.controllers.log_controller import LogController +from python_app.gui.controllers.pipeline_controller import PipelineController +from python_app.gui.controllers.preprocess_controller import PreprocessController +from python_app.gui.controllers.radar_settings_controller import RadarSettingsController +from python_app.gui.settings_defaults import cpp_bscan_replay_window_for_config, default_gui_state_for_config +from python_app.models.gui_profile_schema import GuiProfileModel, GuiStateModel +from python_app.models.run_config_schema import RunConfigModel +from python_app.orchestration.gui_session_state import GuiSessionState, GuiSessionStateStore + + +@dataclass(slots=True) +class ProfileState: + config: RunConfigModel + gui: GuiStateModel + path: Path + session: GuiSessionStateStore + + +def _remember(state: ProfileState, path: Path, store: GuiSessionStateStore, log: LogController) -> None: + state.path = path.expanduser().resolve(strict=False) + try: + store.write(GuiSessionState(last_profile_path=str(state.path))) + except Exception as exc: + log.exception("Failed to update GUI session-state with the active config profile path", exc, level="WARN") + + +def load_startup(project_root: Path, log: LogController) -> ProfileState: + store = GuiSessionStateStore(project_root / "python_app/runtime/gui_session_state.json") + root_path = (project_root / "run_config.json").resolve(strict=False) + requested = os.environ.get("RADAR_SYSTEM_PROFILE", "").strip() + if not requested: + try: + requested = store.load().last_profile_path.strip() + except Exception as exc: + log.warning("Failed to read GUI session-state; using root run_config.json.", details=log.exception_details(exc)) + path = Path(requested).expanduser() if requested else root_path + if not path.is_absolute(): + path = project_root / path + path = path.resolve(strict=False) + try: + profile = GuiProfileModel.load_from_path(path) + except Exception as exc: + if path == root_path: + raise + log.warning("Failed to load the last selected config profile; falling back to root run_config.json.", + details=log.exception_details(exc)) + profile, path = GuiProfileModel.load_from_path(root_path), root_path + config = profile.run_config.clone() + log.set_level(config.logging.level) + gui = profile.gui or default_gui_state_for_config(config, project_root) + if profile.gui is None and path != root_path: + log.info(f"Loaded legacy run config without GUI defaults: {path}") + state = ProfileState(config, gui, path, store) + _remember(state, path, store, log) + return state + + +class ProfileController: + def __init__( + self, parent: QWidget, project_root: Path, state: ProfileState, config: ConfigController, + preprocess: PreprocessController, pipeline: PipelineController, radar: RadarSettingsController, + live: LiveSettingsController, log: LogController, + ) -> None: + self.state = state + self._parent, self._project_root, self._config = parent, project_root, config + self._preprocess, self._pipeline, self._radar, self._live, self._log = preprocess, pipeline, radar, live, log + self._session = state.session + + @staticmethod + def write_profile( + path: Path, profile: GuiProfileModel, active: RunConfigModel | None = None, + *, allow_overwrite: bool = True, dataset: bool = False, + ) -> GuiProfileModel: + """Atomically save settings or a dataset profile with its acquisition provenance.""" + if not allow_overwrite and path.exists(): + raise FileExistsError(f"Config profile output already exists: {path}") + if dataset and active is not None: + profile = profile.clone() + profile.run_config.radar = deepcopy(active.radar) + profile.run_config.input_switch = deepcopy(active.input_switch) + profile.run_config.output_switch = deepcopy(active.output_switch) + profile.run_config.combos = deepcopy(active.combos) + profile.run_config.preprocess = deepcopy(active.preprocess) + profile.run_config.gpr.tx_geometry = deepcopy(active.gpr.tx_geometry) + profile.run_config.gpr.rx_geometry = deepcopy(active.gpr.rx_geometry) + atomic_write_json(path, profile.to_dict(), durable=True) + return profile + + def bind_actions(self, actions: PrimaryView) -> None: + actions.load.clicked.connect(self.load_dialog) + actions.save.clicked.connect(self.save_dialog) + + def build_config(self) -> RunConfigModel: + return self._config.build_config(self.state.config, self._preprocess.selected_sets, + self._live.writer.path, self._log.level.currentText().lower()) + + def build_gui_state(self) -> GuiStateModel: + return self._config.build_gui_state(self.state.config, self._preprocess.settings()) + + def build_profile(self) -> GuiProfileModel: + return GuiProfileModel(run_config=self.build_config(), gui=self.build_gui_state()) + + def radar_key(self) -> str: + return self._config.radar_key(self.state.config) + + def write(self, path: Path, *, allow_overwrite: bool = True, dataset: bool = False) -> GuiProfileModel: + return self.write_profile(path, self.build_profile(), self._pipeline.active_config, + allow_overwrite=allow_overwrite, dataset=dataset) + + def save_dialog(self) -> None: + try: + selected, _ = QFileDialog.getSaveFileName(self._parent, "Save Config Profile", str(self.state.path), + "JSON Files (*.json);;All Files (*)") + if not selected: + return + path = Path(selected).expanduser().resolve(strict=False) + if not path.suffix: + path = path.with_suffix(".json") + profile = self.write(path) + self.state.config = profile.run_config.clone() + self.state.gui = profile.gui or default_gui_state_for_config(profile.run_config, self._project_root) + _remember(self.state, path, self._session, self._log) + config, sweep = profile.run_config, profile.run_config.radar.sweep + points = "from Kamil ADC stream" if config.is_kamil_adc else str(sweep.points) + self._log.info( + f"Config profile saved: path={path}, combos={len(config.combos)}, " + f"sweep={sweep.start_hz:g}..{sweep.stop_hz:g} Hz, points={points}, " + f"ifbw={sweep.if_bandwidth_hz:g} Hz, power={sweep.power_dbm:g} dBm, " + f"processing_mode={self.state.gui.processing.selected_mode}" + ) + except Exception as exc: + self._log.show_exception("Failed to save config profile", exc) + + def _ready_to_load(self) -> bool: + if self._pipeline.closing: + return False + reason = self._preprocess.busy_reason() + if reason: + self._log.show_error(reason, details=self._preprocess.capture_state_details()) + return False + if self._pipeline.running or self._pipeline.should_run or self._pipeline.paused: + self._log.show_error("Stop all pipeline processes before loading a config profile") + return False + return True + + def load(self, path: Path) -> None: + if not self._ready_to_load(): + return + try: + path = path.expanduser().resolve(strict=False) + profile = GuiProfileModel.load_from_path(path) + self.apply(profile, path) + kind = "legacy run config" if profile.gui is None else "full GUI profile" + self._log.info(f"Config profile loaded: path={path}, kind={kind}, combos={len(self.state.config.combos)}, " + f"processing_mode={self._live.active_mode}") + except Exception as exc: + self._log.show_exception("Failed to load config profile", exc) + + def load_dialog(self) -> None: + if not self._ready_to_load(): + return + selected, _ = QFileDialog.getOpenFileName(self._parent, "Load Config Profile", str(self.state.path), + "JSON Files (*.json);;All Files (*)") + if selected: + self.load(Path(selected)) + + def apply(self, profile: GuiProfileModel, path: Path) -> None: + self._log.debug(f"Applying loaded profile: path={path}, has_gui_state={profile.gui is not None}.") + config = profile.run_config.clone() + gui = profile.gui or default_gui_state_for_config(config, self._project_root) + self.state.config, self.state.gui = config, gui + self._config.apply_profile_widgets(config, gui) + self._radar.sync_profile(gui.switches.combo_mode) + self._live.sync_controls() + self._log.set_level(self._log.level.currentText()) + self._preprocess.apply_profile(config, gui.preprocess_dialog) + self._pipeline.plots.live_window = cpp_bscan_replay_window_for_config(config) + self._pipeline.plots.clear_bscan() + self._live.set_mode_page(gui.processing.selected_mode) + self._radar.initial_limits() + if self._preprocess.dialog is not None: + self._preprocess.refresh_sets() + self._live.mode_changed(gui.processing.selected_mode) + self._pipeline.history_changed() + _remember(self.state, path, self._session, self._log) diff --git a/python_app/gui/controllers/radar_settings_controller.py b/python_app/gui/controllers/radar_settings_controller.py new file mode 100644 index 0000000..408e702 --- /dev/null +++ b/python_app/gui/controllers/radar_settings_controller.py @@ -0,0 +1,159 @@ +"""Radar-specific editor constraints and device limits.""" + +from collections.abc import Callable + +from PyQt6.QtCore import QSignalBlocker +from PyQt6.QtWidgets import QCheckBox, QComboBox, QLineEdit, QPlainTextEdit + +from python_app.gui.controllers.log_controller import LogController +from python_app.gui.controllers.sections.views import RadarView, SwitchView, ProcessingView, connect_setting +from python_app.hardware_full.librevna_service import LibreVnaService +from python_app.hardware_full.single_radar_service import create_single_radar_service +from python_app.models.run_config_schema import RunConfigModel + + +class RadarSettingsController: + def __init__( + self, radar: RadarView, switches: SwitchView, processing: ProcessingView, + defaults: Callable[[], RunConfigModel], log: LogController, *, + reset_preprocess: Callable[[], None], live_changed: Callable[[], None], + ) -> None: + self._radar, self._switches, self._processing = radar, switches, processing + self._defaults, self._log = defaults, log + self._reset_preprocess, self._live_changed = reset_preprocess, live_changed + self.limits: dict[str, float | int] | None = None + switches.select_text.clicked.connect(lambda: self.set_combo_mode("text")) + switches.select_single.clicked.connect(lambda: self.set_combo_mode("single")) + for widget in radar.sweep_edits: + widget.editingFinished.connect(self.sweep_changed) + for name, widget in radar.fields.items(): + if name.startswith(("_adc_", "_optical_")) and isinstance(widget, (QLineEdit, QPlainTextEdit, QComboBox, QCheckBox)): + connect_setting(widget, self._optical_changed if widget in (radar.optical_enabled, radar.optical_mode) + else self._reset_preprocess) + + def set_combo_mode(self, mode: str) -> None: + config = self._defaults() + matrix = config.is_multi_device + text_selected = matrix or mode != "single" + for button, selected in ((self._switches.select_text, text_selected), + (self._switches.select_single, not text_selected)): + button.setChecked(selected) + button.setEnabled(not matrix) + combos = self._switches.combos + if matrix: + combos.setText(",".join(f"{combo.input}:{combo.output}" for combo in config.build_runtime_combos())) + combos.setEnabled(text_selected and not matrix) + for widget in (self._switches.single_input, self._switches.single_output): + widget.setEnabled(not text_selected and not matrix) + + def sync_profile(self, combo_mode: str) -> None: + self.set_combo_mode(combo_mode) + adc = self._defaults().is_kamil_adc + self._radar.vna_panel.setVisible(not adc) + self._radar.adc_panel.setVisible(adc) + self.sync_adc_controls() + + def sync_adc_controls(self) -> None: + enabled = self._radar.optical_enabled.isChecked() + mode = self._radar.optical_mode.currentText() + for active, widgets in ((enabled, self._radar.optical_common), + (enabled and mode == "manual", self._radar.optical_manual), + (enabled and mode == "variation", self._radar.optical_variation)): + for widget in widgets: + widget.setEnabled(active) + self._radar.manual_panel.setVisible(mode == "manual") + self._radar.variation_panel.setVisible(mode == "variation") + + def _optical_changed(self) -> None: + self.sync_adc_controls() + self._reset_preprocess() + + def initial_limits(self) -> None: + mode = self._defaults().radar.driver_mode + if mode == "native": + self._log.debug("Querying radar limits from device (native driver mode).") + self.refresh_limits() + else: + self._log.debug(f"Skipping device radar-limit query (driver mode={mode}).") + self.apply_limits(None) + + def refresh_limits(self) -> bool: + config = self._defaults() + if config.is_kamil_adc: + return self.apply_limits(None) + service = (LibreVnaService(serial=config.radar.serial or None) if config.is_multi_device + else create_single_radar_service(config)) + return self.apply_limits(service.read_device_limits()) + + def apply_limits(self, limits: dict[str, float | int] | None) -> bool: + previous, self.limits = self.limits, limits + labels = ("Start Hz", "Stop Hz", "Points", "IF BW Hz", "Stimulus Power dBm") + edits = self._radar.sweep_edits + hint = self._radar.limits_hint + if limits is None: + for widget, label in zip(self._radar.sweep_labels, labels, strict=True): + widget.setText(label) + hint.setText("Device limits are not available.") + edits[-1].setToolTip("Stimulus power configured in the active profile.") + return False + + frequency = (float(limits["min_frequency_hz"]), float(limits["max_frequency_hz"])) + ifbw = (float(limits["min_ifbw_hz"]), float(limits["max_ifbw_hz"])) + power = (float(limits["min_power_dbm"]), float(limits["max_power_dbm"])) + max_points = int(limits["max_points"]) + ranges = (frequency, frequency, (1, max_points), ifbw, power) + before = [edit.text().strip() for edit in edits] + for index, (widget, label, edit, bounds) in enumerate(zip(self._radar.sweep_labels, labels, edits, ranges, strict=True)): + minimum, maximum = bounds + widget.setText(f"{label} ({minimum:g}..{maximum:g})") + self._clamp(edit, minimum, maximum, integer=index == 2) + if float(edits[0].text()) > float(edits[1].text()): + edits[1].setText(edits[0].text()) + edits[-1].setToolTip(f"Device range: {power[0]:g}..{power[1]:g} dBm") + summary = (f"freq={frequency[0]:g}..{frequency[1]:g} Hz, points=1..{max_points}, " + f"ifbw={ifbw[0]:g}..{ifbw[1]:g} Hz, power={power[0]:g}..{power[1]:g} dBm") + hint.setText(f"Limits: {summary}") + if previous != limits: + self._log.info(f"Applied radar device limits: {summary}") + after = [edit.text().strip() for edit in edits] + adjustments = [f"{label}: {old or ''} -> {new}" + for label, old, new in zip(labels, before, after, strict=True) if old != new] + if adjustments: + self._log.warning("Radar fields were adjusted to satisfy device limits.", details="\n".join(adjustments)) + self.sync_frequency_limits() + return before != after + + @staticmethod + def _clamp(widget: QLineEdit, minimum: float, maximum: float, *, integer: bool = False) -> None: + try: + value = float(widget.text().strip()) + if integer: + value = int(value) + except ValueError: + value = minimum + value = min(max(value, minimum), maximum) + widget.setText(str(int(value)) if integer else f"{value:g}") + + def sync_frequency_limits(self) -> bool: + try: + bounds = sorted(float(widget.text()) / 1e6 for widget in self._radar.sweep_edits[:2]) + except ValueError: + return False + changed = False + adjustments: list[str] = [] + for index, widget in enumerate(self._processing.frequency_controls): + before = widget.value() + changed |= (widget.minimum(), widget.maximum()) != tuple(bounds) + with QSignalBlocker(widget): + widget.setRange(*bounds) + if before != widget.value(): + adjustments.append(f"Frequency control {index + 1}: {before:g} -> {widget.value():g}") + if adjustments: + self._log.warning("Processing frequency limits were clamped to the active radar sweep.", + details="\n".join(adjustments)) + return changed or bool(adjustments) + + def sweep_changed(self) -> None: + self._reset_preprocess() + if self.sync_frequency_limits(): + self._live_changed() diff --git a/python_app/gui/controllers/recording_controller.py b/python_app/gui/controllers/recording_controller.py new file mode 100644 index 0000000..af54381 --- /dev/null +++ b/python_app/gui/controllers/recording_controller.py @@ -0,0 +1,233 @@ +"""Bounded recording of independent pipeline stages on a non-daemon writer.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from contextlib import suppress +from dataclasses import dataclass, field +import logging +from pathlib import Path +import queue +import shutil +import threading +import time +import traceback +from typing import Protocol + +from python_app.models.gui_profile_schema import ProfileWriter +from python_app.models.dataset_model import ResultCollection, SweepCollection +from python_app.storage.npz.store import NpzStore + +logger = logging.getLogger(__name__) +_RECORDING_CHUNK_SIZE = 200 +_RECORDING_QUEUE_CHUNKS = 8 +_Chunk = tuple[list[SweepCollection], list[SweepCollection], list[ResultCollection], dict[int, dict]] + + +class SnapshotWriter(Protocol): + @property + def directory(self) -> Path: ... + + def append(self, raw: list[SweepCollection], preprocessed: list[SweepCollection], + results: list[ResultCollection], *, processing_settings: dict[int, dict] | None = None) -> None: ... + + +@dataclass(slots=True) +class _Session: + writer: SnapshotWriter + starting: bool + since_ns: int = field(default_factory=time.monotonic_ns) + queue: queue.Queue[_Chunk] = field(default_factory=lambda: queue.Queue(maxsize=_RECORDING_QUEUE_CHUNKS)) + finished: threading.Event = field(default_factory=threading.Event) + raw: list[SweepCollection] = field(default_factory=list) + preprocessed: list[SweepCollection] = field(default_factory=list) + results: list[ResultCollection] = field(default_factory=list) + tail: _Chunk | None = None + thread: threading.Thread | None = None + error: str | None = None + capacity_exceeded: bool = False + received: int = 0 + enqueued: int = 0 + stop_reason: str = "" + + +def _write_chunks(session: _Session) -> None: + """Own disk I/O only; no callbacks or GUI objects cross the worker boundary.""" + while True: + try: + chunk = session.queue.get(timeout=0.05) + except queue.Empty: + if not session.finished.is_set(): + continue + try: # A final enqueue may have raced get() before finished.set(). + chunk = session.queue.get_nowait() + except queue.Empty: + if session.tail is None: + return + chunk, session.tail = session.tail, None + try: + raw, preprocessed, results, settings = chunk + session.writer.append(raw, preprocessed, results, processing_settings=settings) + except Exception as exc: # noqa: BLE001 - report on the GUI thread + logger.exception("Disk recording write failed") + session.error = f"{type(exc).__name__}: {exc}" + session.finished.set() + session.tail = None + while True: + try: + session.queue.get_nowait() + except queue.Empty: + return + + +class RecordingController: + def __init__(self, store: NpzStore, processing_settings: Mapping[int, dict], + write_profile: ProfileWriter, log: Callable[[str], None], error: Callable[..., None]) -> None: + self._store, self._settings = store, processing_settings + self._write_profile, self._log, self._error = write_profile, log, error + self._session: _Session | None = None + self._target = 0 + + @property + def active(self) -> bool: + return self._session is not None + + def start(self, root: Path, name: str, prefix: str, target: int, *, + running: bool, start_acquisition: Callable[[], bool]) -> None: + if self.active: + self._error("A recording is already in progress; wait for it to finish before starting another", + details=f"progress={self.status()['collected']}/{self._target}") + return + if target <= 0: + self._error("Recording count must be greater than zero") + return + destination = self._store.snapshot_directory(root, name, name_prefix=prefix) + if destination.exists(): + self._error("Recording destination already exists; choose a new name or path", + details=f"destination={destination}") + return + + writer: SnapshotWriter | None = None + try: + writer = self._store.create_snapshot_stream(root, name, name_prefix=prefix) + self._write_profile(writer.directory / "config_profile.json", allow_overwrite=False, dataset=running) + session = _Session(writer, starting=not running) + session.thread = threading.Thread(target=_write_chunks, args=(session,), name="disk-recorder", daemon=False) + self._session, self._target = session, target + session.thread.start() + except Exception as exc: # noqa: BLE001 - includes thread resource exhaustion + self._session, self._target = None, 0 + if writer is not None: + with suppress(OSError): + shutil.rmtree(writer.directory) + self._error(f"Failed to start disk recording: {exc}", details=traceback.format_exc()) + return + + start_error: Exception | None = None + try: + started = running or start_acquisition() + except Exception as exc: # noqa: BLE001 - retain anything accepted during startup + started, start_error = False, exc + finally: + session.starting = False + if not started: + self.finish() + if not session.received: + with suppress(OSError): + shutil.rmtree(writer.directory) + if start_error is not None: + self._error(f"Failed to start acquisition for disk recording: {start_error}", + details="".join(traceback.format_exception(start_error))) + else: + self._log("Disk recording canceled because acquisition did not start") + return + if not running: + try: + self._write_profile(writer.directory / "config_profile.json", dataset=True) + except Exception as exc: # noqa: BLE001 - keep accepted data and the initial profile + self.finish() + self._error(f"Failed to update disk recording config profile: {exc}", details=traceback.format_exc()) + return + self._log(f"Disk recording armed: streaming the next {target} measurement(s) to {writer.directory}") + self.poll() + + def record(self, stage: str, collection: SweepCollection | ResultCollection) -> None: + session = self._session + if session is None or session.finished.is_set() or collection.capture_start_ns < session.since_ns: + return + if stage == "raw" and isinstance(collection, SweepCollection): + session.raw.append(collection) + elif stage == "preprocessed" and isinstance(collection, SweepCollection): + session.preprocessed.append(collection) + elif stage == "results" and isinstance(collection, ResultCollection): + session.results.append(collection) + else: + raise ValueError(f"Invalid collection for recording stage: {stage}") + session.received += 1 + if max(len(session.raw), len(session.preprocessed), len(session.results)) >= _RECORDING_CHUNK_SIZE: + if not self._flush(session): + return + if session.enqueued + len(session.results) >= self._target and self._flush(session): + session.finished.set() + + def _flush(self, session: _Session) -> bool: + if not (session.raw or session.preprocessed or session.results): + return True + if session.error is not None or session.thread is None or not session.thread.is_alive(): + session.error = session.error or "Disk writer stopped unexpectedly" + session.finished.set() + self.poll() + return False + chunk = (session.raw, session.preprocessed, session.results, + {result.settings_revision: self._settings[result.settings_revision] + for result in session.results if result.settings_revision in self._settings}) + try: + session.queue.put_nowait(chunk) + except queue.Full: + session.tail, session.capacity_exceeded = chunk, True + session.finished.set() + self._log("Disk writer cannot keep up; finishing accepted data and stopping recording") + session.enqueued += len(session.results) + session.raw, session.preprocessed, session.results = [], [], [] + return True + + def poll(self) -> None: + session = self._session + if session is None or session.starting: + return + if session.error is not None: + self._session = None + self._error("Disk recording failed and was stopped", + details=f"dataset={session.writer.directory}\nerror={session.error}") + elif session.finished.is_set() and session.thread is not None and not session.thread.is_alive(): + self._session = None + if session.enqueued >= self._target: + self._log(f"Disk recording complete: {session.enqueued} measurement(s) written to {session.writer.directory}") + else: + reason = session.stop_reason or "writer queue full" + self._log(f"Disk recording stopped: {session.enqueued} of {self._target} measurement(s); " + f"reason: {reason}. Accepted data saved in {session.writer.directory}") + if session.capacity_exceeded: + self._error("Disk recording stopped because the writer could not keep up", + details=f"Accepted data saved in {session.writer.directory}; results={session.enqueued}") + + def finish(self, *, reason: str = "pipeline stopped") -> None: + """Stop/Close share a nonblocking flush; startup's technical Stop is deferred.""" + session = self._session + if session is None or session.starting: + return + if not session.finished.is_set(): + session.stop_reason = reason + if self._flush(session): + session.finished.set() + if session.enqueued < self._target: + self._log(f"Disk recording stopped: {session.enqueued} of {self._target} measurement(s); " + f"reason: {reason}. Finishing accepted data.") + self.poll() + if self.active: + self._log("Disk recording is finishing queued writes; the dataset is not complete yet") + + def status(self) -> dict[str, int | bool]: + session = self._session + return {"active": self.active, "collected": session.enqueued + len(session.results) if session else 0, + "target": self._target} diff --git a/python_app/gui/controllers/renderers/__init__.py b/python_app/gui/controllers/renderers/__init__.py new file mode 100644 index 0000000..cdd0e02 --- /dev/null +++ b/python_app/gui/controllers/renderers/__init__.py @@ -0,0 +1 @@ +"""Independent renderers for processed radar results.""" diff --git a/python_app/gui/controllers/renderers/bscan.py b/python_app/gui/controllers/renderers/bscan.py new file mode 100644 index 0000000..f255109 --- /dev/null +++ b/python_app/gui/controllers/renderers/bscan.py @@ -0,0 +1,162 @@ +"""B-scan rendering and cache helpers.""" + +from __future__ import annotations + +from collections import deque + +from PyQt6.QtCore import QRectF +import numpy as np +import pyqtgraph as pg + +from python_app.models.collection_identity import collection_key +from python_app.models.dataset_model import ResultCollection +from python_app.models.gui_profile_schema import GuiBscanStateModel +from python_app.gui.controllers.renderers.common import build_lut, create_plot + + +def _result_tail( + *, + result_history: list[ResultCollection], + history_limit: int, +) -> list[ResultCollection]: + """Choose by acquisition order, keeping the newest result of each exact sweep.""" + unique_reversed_tail: list[ResultCollection] = [] + seen_keys: set[tuple[int, int]] = set() + for collection in reversed(result_history[-history_limit:]): + key = collection_key(collection) + if key in seen_keys: + continue + seen_keys.add(key) + unique_reversed_tail.append(collection) + + unique_reversed_tail.reverse() + return unique_reversed_tail + + +def apply_mean_ascan_subtraction( + history: deque[np.ndarray], + *, + enabled: bool, +) -> np.ndarray: + """Subtract mean A-scan across sweeps when enabled.""" + sweeps = np.vstack(history).astype(np.float32, copy=False) + if not enabled: + return sweeps + mean_trace = np.mean(sweeps, axis=0, dtype=np.float32) + return sweeps - mean_trace + + +def rebuild_bscan_history_from_results( + result_history: list[ResultCollection], + history_limit: int, +) -> tuple[dict[tuple[int, int], deque[np.ndarray]], dict[tuple[int, int], np.ndarray]]: + """Group compatible range profiles; a changed depth axis starts a new image.""" + history_by_combo: dict[tuple[int, int], deque[np.ndarray]] = {} + depth_axis_by_combo: dict[tuple[int, int], np.ndarray] = {} + + result_tail = _result_tail( + result_history=result_history, + history_limit=history_limit, + ) + + for collection in result_tail: + for block in collection.blocks: + key = (block.combo.input, block.combo.output) + for payload in block.payloads: + if payload.kind != 1 or payload.processing_name != "bscan": + continue + if payload.frequency_hz.size == 0 or payload.trace.size == 0: + continue + if payload.frequency_hz.size != payload.trace.size: + continue + + depth_axis = np.asarray(payload.frequency_hz, dtype=np.float32) + amplitudes = np.asarray(np.real(payload.trace), dtype=np.float32) + history = history_by_combo.get(key) + stored_axis = depth_axis_by_combo.get(key) + if ( + history is None + or stored_axis is None + or stored_axis.shape != depth_axis.shape + or not np.allclose(stored_axis, depth_axis, rtol=1e-4, atol=1e-6) + ): + history = deque(maxlen=history_limit) + history_by_combo[key] = history + depth_axis_by_combo[key] = depth_axis.copy() + + history.append(amplitudes.copy()) + + return history_by_combo, depth_axis_by_combo + + +def bscan_lookup_table(axis_mode: str) -> np.ndarray: + """Build B-scan colormap table for selected axis mode.""" + if axis_mode == "abs": + return build_lut(["#440154", "#31688e", "#35b779", "#fde725"]) + return build_lut(["#2166ac", "#67a9cf", "#f7f7f7", "#ef8a62", "#b2182b"]) + + +def bscan_levels(sweeps: np.ndarray, axis_mode: str) -> tuple[float, float]: + """Compute image levels for B-scan data based on axis mode.""" + min_value = float(np.min(sweeps)) + max_value = float(np.max(sweeps)) + if axis_mode == "abs": + if max_value <= min_value: + return min_value, min_value + 1e-6 + return min_value, max_value + + max_abs = max(abs(min_value), abs(max_value), 1e-6) + return -max_abs, max_abs + + +class BscanRenderer: + """Own the heatmap and its per-combination arrays; input history stays external.""" + + def __init__(self) -> None: + self.widget = create_plot() + self.widget.setLabel("bottom", "Sweep #") + self.widget.setLabel("left", "Range", units="m") + item = self.widget.getPlotItem() + assert item is not None + item.getViewBox().invertY(False) + item.getViewBox().enableAutoRange(x=False, y=False) + self.image = pg.ImageItem(axisOrder="row-major") + item.addItem(self.image) + self.history_by_combo: dict[tuple[int, int], deque[np.ndarray]] = {} + self.depth_axis_by_combo: dict[tuple[int, int], np.ndarray] = {} + self.signature: tuple[object, ...] | None = None + + def sync(self, results: list[ResultCollection], window: int) -> bool: + # Completion time distinguishes recomputed payloads for the same acquisition. + signature = (window, tuple((collection_key(c), c.monotonic_ns, c.settings_revision) for c in results[-window:])) + if signature == self.signature: + return False + self.history_by_combo, self.depth_axis_by_combo = rebuild_bscan_history_from_results(results, window) + self.signature = signature + return True + + def draw(self, key: tuple[int, int] | None, settings: GuiBscanStateModel) -> bool: + history = self.history_by_combo.get(key) if key is not None else None + depth = self.depth_axis_by_combo.get(key) if key is not None else None + if not history or depth is None: + self.image.clear() + return False + sweeps = apply_mean_ascan_subtraction(history, enabled=settings.subtract_mean_ascan) + depth_min, depth_max = float(np.min(depth)), float(np.max(depth)) + count = sweeps.shape[0] + self.image.setImage(sweeps.T, autoLevels=False) + self.image.setRect(QRectF(.5, depth_min, float(count), max(depth_max - depth_min, 1e-6))) + self.image.setLookupTable(bscan_lookup_table(settings.axis)) + self.image.setLevels(bscan_levels(sweeps, settings.axis)) + item = self.widget.getPlotItem() + assert item is not None + item.getViewBox().setRange(xRange=(.5, .5 + count), yRange=(depth_min, depth_max), padding=.02) + assert key is not None + self.widget.setTitle(f"B-scan S21 in{key[0]}/out{key[1]} | sweeps={count}") + return True + + def clear(self) -> None: + self.history_by_combo.clear() + self.depth_axis_by_combo.clear() + self.signature = None + self.image.clear() diff --git a/python_app/gui/controllers/renderers/common.py b/python_app/gui/controllers/renderers/common.py new file mode 100644 index 0000000..cbd5778 --- /dev/null +++ b/python_app/gui/controllers/renderers/common.py @@ -0,0 +1,23 @@ +"""Plot construction and color tables shared by radar views.""" + +import numpy as np +import pyqtgraph as pg + + +def create_plot(title: str = "") -> pg.PlotWidget: + plot = pg.PlotWidget(background="#0f141c", enableMenu=False, title=title) + plot.showGrid(x=True, y=True, alpha=0.2) + return plot + + +def build_lut(stops: list[str], *, size: int = 256) -> np.ndarray: + """Interpolate hex color stops into 8-bit RGB LUT array.""" + stop_positions = np.linspace(0.0, 1.0, num=len(stops), dtype=np.float32) + sample_positions = np.linspace(0.0, 1.0, num=size, dtype=np.float32) + stop_colors = np.asarray([pg.mkColor(value).getRgb()[:3] for value in stops], dtype=np.float32) + + lut = np.empty((size, 3), dtype=np.uint8) + for channel in range(3): + lut[:, channel] = np.interp(sample_positions, stop_positions, stop_colors[:, channel]).astype(np.uint8) + return lut + diff --git a/python_app/gui/controllers/renderers/gpr.py b/python_app/gui/controllers/renderers/gpr.py new file mode 100644 index 0000000..0502366 --- /dev/null +++ b/python_app/gui/controllers/renderers/gpr.py @@ -0,0 +1,292 @@ +"""GPR heatmaps and object overlays with renderer-owned graphics and geometry.""" + +from __future__ import annotations + +from PyQt6.QtCore import QRectF +import numpy as np +import pyqtgraph as pg + +from python_app.gui.controllers.renderers.common import build_lut, create_plot +from python_app.gui.result_payloads import ( + collection_payload_by_name, collection_payloads_by_prefix, + filter_object_rows, gpr_object_rows, +) +from python_app.models.run_config_validation import parse_position_csv +from python_app.models.dataset_model import ResultCollection, ResultPayload +from python_app.models.gui_profile_schema import GuiGprStateModel, GuiLegacyGprStateModel +from python_app.models.run_config_schema import GprModel + +GprSettings = GuiGprStateModel | GuiLegacyGprStateModel + + +def normalized_display_range(start: float, stop: float, *, minimum_span: float = 0.1) -> tuple[float, float]: + """Order display bounds and expand an empty interval around its center.""" + lower, upper = sorted((float(start), float(stop))) + if upper - lower >= minimum_span: + return lower, upper + center = 0.5 * (lower + upper) + return center - 0.5 * minimum_span, center + 0.5 * minimum_span + + +def display_y_min(z_min: float, z_max: float) -> float: + """Leave room for antenna markers when the surface is visible.""" + if z_min > 0.0: + return z_min + return z_min - max(max(z_max - z_min, 1e-6) * 0.03, 0.06) + + +def object_label_candidates( + x_m: float, z_m: float, *, x_span: float, z_span: float, +) -> list[tuple[float, float, tuple[float, float]]]: + """Candidate label placements, ordered by proximity to the object.""" + dx, dz = max(x_span * 0.015, 0.02), max(z_span * 0.02, 0.02) + return [ + (x_m + dx, z_m - dz, (0.0, 1.0)), + (x_m + dx, z_m + dz, (0.0, 0.0)), + (x_m - dx, z_m - dz, (1.0, 1.0)), + (x_m - dx, z_m + dz, (1.0, 0.0)), + (x_m, z_m - dz * 1.35, (0.5, 1.0)), + (x_m, z_m + dz * 1.35, (0.5, 0.0)), + (x_m + dx * 2.2, z_m - dz * 1.5, (0.0, 1.0)), + (x_m + dx * 2.2, z_m + dz * 1.5, (0.0, 0.0)), + (x_m - dx * 2.2, z_m - dz * 1.5, (1.0, 1.0)), + (x_m - dx * 2.2, z_m + dz * 1.5, (1.0, 0.0)), + ] + + +def _title(settings: GprSettings, *, objects_only: bool = False) -> str: + title = f"Legacy GPR {settings.mode}" if isinstance(settings, GuiLegacyGprStateModel) else "GPR coherent BP" + return f"{title} Objects Only" if objects_only else title + + +class GprRenderer: + """Draw one collection from explicit display settings and antenna geometry.""" + + def __init__(self) -> None: + self.widget = create_plot("GPR coherent BP") + self.widget.setBackground("#0f141c") + plot = self.widget.getPlotItem() + assert plot is not None + self._plot = plot + self._view = plot.getViewBox() + self._plot.showGrid(x=True, y=True, alpha=0.2) + self._configure_axes() + self._lookup_table = build_lut(["#081c15", "#1b4332", "#ffd166", "#f94144"]) + self._image = pg.ImageItem(axisOrder="row-major") + self._tx = pg.ScatterPlotItem() + self._rx = pg.ScatterPlotItem() + self._points = pg.ScatterPlotItem() + self._region_centers = pg.ScatterPlotItem() + self._markers = (self._tx, self._rx, self._points, self._region_centers) + for item, z_value in zip((self._image, *self._markers), (0, 20, 20, 30, 30)): + item.setZValue(z_value) + item.hide() + self._plot.addItem(item) + self._labels: list[pg.TextItem] = [] + self._masks: list[pg.ImageItem] = [] + self._contours: list[pg.IsocurveItem] = [] + self._geometry_signature: tuple[object, ...] | None = None + self._selected_geometry = (np.zeros(0, dtype=np.float32), np.zeros(0, dtype=np.float32)) + + def _configure_axes(self) -> None: + plot = self._plot + plot.showAxis("left", show=True) + plot.showAxis("bottom", show=True) + plot.setClipToView(True) + plot.setLabel("bottom", "X", units="m") + plot.setLabel("left", "Range", units="m") + plot.setMenuEnabled(False) + plot.hideButtons() + view = self._view + view.invertY(False) + view.enableAutoRange(x=False, y=False) + view.setMouseEnabled(x=False, y=False) + view.setMenuEnabled(False) + + def _clear_annotations(self) -> None: + for item in (*self._labels, *self._masks): + self._plot.removeItem(item) + self._labels.clear() + self._masks.clear() + self._contours.clear() + + def clear(self, settings: GprSettings) -> None: + """Clear displayed data while retaining reusable graphics items.""" + self._configure_axes() + self._clear_annotations() + self._image.hide() + for marker in self._markers: + marker.setData(x=[], y=[]) + marker.hide() + self._plot.setTitle(_title(settings)) + + def _geometry(self, settings: GprSettings, geometry: GprModel) -> tuple[np.ndarray, np.ndarray]: + inputs = tuple(parse_position_csv(settings.input_positions)) + outputs = tuple(parse_position_csv(settings.output_positions)) + signature = ( + tuple((entry.output_pos, entry.x_m, entry.y_m, entry.z_m) for entry in geometry.tx_geometry), + tuple((entry.input_pos, entry.x_m, entry.y_m, entry.z_m) for entry in geometry.rx_geometry), + inputs, outputs, + ) + if signature != self._geometry_signature: + self._selected_geometry = ( + np.asarray([ + entry.x_m for entry in sorted(geometry.tx_geometry, key=lambda entry: entry.output_pos) + if not outputs or entry.output_pos in outputs + ], dtype=np.float32), + np.asarray([ + entry.x_m for entry in sorted(geometry.rx_geometry, key=lambda entry: entry.input_pos) + if not inputs or entry.input_pos in inputs + ], dtype=np.float32), + ) + self._geometry_signature = signature + return self._selected_geometry + + def _draw_geometry(self, settings: GprSettings, geometry: GprModel) -> None: + tx, rx = self._geometry(settings, geometry) + for marker, xs, symbol, brush, pen in ( + (self._tx, tx, "t", "#ff595e", "#ffca3a"), + (self._rx, rx, "t1", "#4cc9f0", "#e0fbfc"), + ): + marker.setData( + x=xs, y=np.zeros_like(xs), symbol=symbol, size=13, + brush=pg.mkBrush(brush), pen=pg.mkPen(pen, width=1.0), + ) + marker.setVisible(xs.size > 0) + + def _label(self, text: str, x: float, y: float, color: str) -> pg.TextItem: + label = pg.TextItem(text=text, color=color, anchor=(0.0, 1.0)) + label.setZValue(40) + label.setPos(x, y) + self._plot.addItem(label) + self._labels.append(label) + return label + + def _draw_regions(self, collection: ResultCollection, rect: QRectF) -> None: + centers_payload = collection_payload_by_name(collection, "gpr_region_centers", kind=4) + if centers_payload is not None and np.asarray(centers_payload.table).size > 0: + centers = np.asarray(centers_payload.table, dtype=np.float32) + self._region_centers.setData( + x=centers[:, 0], y=centers[:, 1], symbol="o", size=10, + brush=pg.mkBrush("#80ed99"), pen=pg.mkPen("#081c15", width=1.1), + ) + self._region_centers.show() + for row in centers: + self._label(f"{float(row[2]):.0f}", float(row[0]), float(row[1]), "#d8f3dc") + for payload in collection_payloads_by_prefix(collection, "gpr_region_mask_", kind=3): + mask = np.asarray(payload.image, dtype=np.float32) + if mask.ndim != 2 or mask.size == 0: + continue + mask_image = pg.ImageItem(axisOrder="row-major") + mask_image.setZValue(5) + mask_image.setImage(mask, autoLevels=False) + mask_image.setRect(rect) + mask_image.setOpacity(0.0) + self._plot.addItem(mask_image) + contour = pg.IsocurveItem(data=mask) + contour.setLevel(0.5) + contour.setPen("#4cc9f0", width=1.3) + contour.setParentItem(mask_image) + self._masks.append(mask_image) + self._contours.append(contour) + + @staticmethod + def _place_object_label( + label: pg.TextItem, x: float, z: float, x_span: float, z_span: float, occupied: list[QRectF], + ) -> None: + for label_x, label_z, anchor in object_label_candidates(x, z, x_span=x_span, z_span=z_span): + label.setAnchor(anchor) + label.setPos(label_x, label_z) + rect = label.sceneBoundingRect().adjusted(-4.0, -4.0, 4.0, 4.0) + if not any(rect.intersects(other) for other in occupied): + occupied.append(rect) + return + occupied.append(label.sceneBoundingRect().adjusted(-4.0, -4.0, 4.0, 4.0)) + + def _draw_points( + self, rows: np.ndarray, settings: GprSettings, objects_only: bool, x_span: float, z_span: float, + ) -> None: + if rows.size == 0: + return + self._points.setData( + x=rows[:, 0], y=rows[:, 1], symbol="o" if objects_only else "d", + size=18 if objects_only else 11, + brush=pg.mkBrush("#ff4d4f" if objects_only else "#ffffff"), + pen=pg.mkPen("#ff4d4f" if objects_only else "#111111", width=1.6 if objects_only else 1.1), + ) + self._points.show() + occupied: list[QRectF] = [] + for x_value, z_value, score in rows: + x, z = float(x_value), float(z_value) + text = f"{float(score):.0f}" if isinstance(settings, GuiLegacyGprStateModel) else f"{float(score):.2f}" + if objects_only: + text += f" | x={x:.1f} | z={z:.1f}" + label = self._label(text, x, z, "#ffd6d9" if objects_only else "#ffffff") + if objects_only: + self._place_object_label(label, x, z, x_span, z_span, occupied) + + def draw(self, collection: ResultCollection, settings: GprSettings, geometry: GprModel) -> bool: + """Render a heatmap or object-only view, returning whether data is present.""" + objects_only = settings.render_mode == "objects_only" + accumulator = collection_payload_by_name(collection, "gpr_accumulator", kind=3) + x_min, x_max = normalized_display_range(settings.visible_x_min_m, settings.visible_x_max_m) + z_min, z_max = normalized_display_range(settings.visible_z_min_m, settings.visible_z_max_m) + rows = gpr_object_rows(collection) + # Coherent BP rows are finalized in C++; only legacy GPR uses GUI filtering. + if isinstance(settings, GuiLegacyGprStateModel): + rows = filter_object_rows( + rows, min_score=float(settings.min_visible_pair_count), + x_bounds=(x_min, x_max), z_bounds=(z_min, z_max), + ) + if objects_only: + if accumulator is None and rows.size == 0: + self.clear(settings) + return False + elif accumulator is None or not self._valid_accumulator(accumulator): + self.clear(settings) + return False + + self.widget.setUpdatesEnabled(False) + try: + self._clear_annotations() + self._image.hide() + for marker in (self._points, self._region_centers): + marker.setData(x=[], y=[]) + marker.hide() + self._view.invertY(False) + self._draw_geometry(settings, geometry) + rect: QRectF | None = None + if not objects_only and accumulator is not None: + image = np.asarray(accumulator.image, dtype=np.float32) + xs = np.asarray(accumulator.image_x_axis, dtype=np.float32) + ys = np.asarray(accumulator.image_y_axis, dtype=np.float32) + rect = QRectF( + float(xs[0]), float(ys[0]), + max(float(xs[-1]) - float(xs[0]), 1e-6), + max(float(ys[-1]) - float(ys[0]), 1e-6), + ) + self._image.setImage(image, autoLevels=False) + self._image.setRect(rect) + self._image.setLookupTable(self._lookup_table) + self._image.setLevels((float(np.min(image)), float(np.max(image) + 1e-6))) + self._image.show() + self._view.setXRange(x_min, x_max, padding=0.0) + self._view.setYRange(display_y_min(z_min, z_max), z_max, padding=0.0) + points = collection_payload_by_name(collection, "gpr_points", kind=4) + if points is None or np.asarray(points.table).size == 0: + rows = np.zeros((0, 3), dtype=np.float32) + self._draw_points(rows, settings, objects_only, x_max - x_min, z_max - z_min) + if rect is not None: + self._draw_regions(collection, rect) + if objects_only: + self._view.setXRange(x_min, x_max, padding=0.0) + self._view.setYRange(display_y_min(z_min, z_max), z_max, padding=0.0) + self._plot.setTitle(_title(settings, objects_only=objects_only)) + finally: + self.widget.setUpdatesEnabled(True) + return True + + @staticmethod + def _valid_accumulator(payload: ResultPayload) -> bool: + image = np.asarray(payload.image) + return image.ndim == 2 and image.size > 0 and np.size(payload.image_x_axis) > 0 and np.size(payload.image_y_axis) > 0 diff --git a/python_app/gui/controllers/app_window_plot/sar3d_plot_mixin.py b/python_app/gui/controllers/renderers/sar3d.py similarity index 55% rename from python_app/gui/controllers/app_window_plot/sar3d_plot_mixin.py rename to python_app/gui/controllers/renderers/sar3d.py index fb22b5d..9e512f5 100644 --- a/python_app/gui/controllers/app_window_plot/sar3d_plot_mixin.py +++ b/python_app/gui/controllers/renderers/sar3d.py @@ -1,4 +1,4 @@ -"""3D SAR views of the volume published by the sar3d processor. +"""Independent 3D SAR views of the volume published by the sar3d processor. Scene axes: X across, Y height above the ground (Y = 0 is the ground), Z along the ground away from the radar. Payloads share normalization by the maximum @@ -21,13 +21,18 @@ import itertools import numpy as np import pyqtgraph as pg from PyQt6.QtCore import QRectF, Qt +from PyQt6.QtWidgets import QGridLayout, QLabel, QVBoxLayout, QWidget -from python_app.gui.controllers.app_window_plot.bscan_plot_mixin import build_lut -from python_app.models.dataset_model import ResultCollection +from python_app.gui.controllers.renderers.common import build_lut, create_plot +from python_app.gui.result_payloads import collection_payload_by_name +from python_app.models.dataset_model import ResultCollection, ResultPayload # Colours start at this fraction of the normalization reference; weaker voxels stay transparent, -# like those under the threshold of the notebook's point cloud. -DISPLAY_THRESHOLD = 0.75 +# like those under the notebook's point-cloud threshold (SAR_CLOUD_THRESHOLD). +DISPLAY_THRESHOLD = 0.70 + +# Status codes published by the sar3d processor in its "sar3d_status" table. +SAR3D_STATES = {0: "ready", 1: "no signal", 2: "error", 3: "warning"} # payload -> (title, x label, y label, scene axes (0 = X, 1 = Y, 2 = Z) along the plot axes) SAR3D_VIEWS: dict[str, tuple[str, str, str, tuple[int, int] | None]] = { @@ -44,19 +49,17 @@ _AXIS_EDGES = {"X": (0, 1), "Y": (1, 3), "Z": (1, 5)} def sar3d_status(collection: ResultCollection) -> tuple[int, str] | None: - """Decode processor status without letting malformed telemetry break a Qt slot.""" - payload = next((item for item in collection.collection_payloads - if item.processing_name == "sar3d_status" and item.kind == 4), None) + """Status code (a SAR3D_STATES key) and message, one UTF-8 byte per table column.""" + payload = collection_payload_by_name(collection, "sar3d_status", kind=4) if payload is None: return None - table = payload.table - if table.ndim != 2 or table.shape[0] != 1 or table.shape[1] == 0: - return 2, "Invalid SAR3D status payload" - row = table[0] - if (not np.isfinite(row).all() or not np.equal(row, np.floor(row)).all() - or row[0] not in (0, 1, 2, 3) or np.any((row[1:] < 0) | (row[1:] > 255))): - return 2, "Invalid SAR3D status payload" - return int(row[0]), bytes(row[1:].astype(np.uint8)).decode("utf-8", errors="replace") + if payload.table.ndim != 2 or not payload.table.size or not np.isfinite(payload.table).all(): + return 2, "Invalid SAR3D status table" + row = payload.table[0] + code = int(row[0]) + if code not in SAR3D_STATES or code != row[0]: + return 2, f"Unknown SAR3D status code: {row[0]}" + return code, bytes(row[1:].astype(np.uint8)).decode("utf-8", errors="replace") def _image_rect(x_axis: np.ndarray, y_axis: np.ndarray) -> QRectF: @@ -66,28 +69,50 @@ def _image_rect(x_axis: np.ndarray, y_axis: np.ndarray) -> QRectF: return QRectF(x_axis[0] - dx / 2, y_axis[0] - dy / 2, x_axis[-1] - x_axis[0] + dx, y_axis[-1] - y_axis[0] + dy) -class AppWindowSar3dPlotMixin: +class Sar3dRenderer: """Draws the 3D SAR views.""" - def _init_sar3d_views(self) -> None: - """Create the persistent items of the four 3D SAR plots.""" + def __init__(self) -> None: + self.widget = QWidget() + layout = QVBoxLayout(self.widget) + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(6) + self.status_label = QLabel() + self.status_label.setTextFormat(Qt.TextFormat.PlainText) + self.status_label.setWordWrap(True) + layout.addWidget(self.status_label) + self.grid = QWidget() + grid_layout = QGridLayout(self.grid) + grid_layout.setContentsMargins(0, 0, 0, 0) + grid_layout.setSpacing(6) + layout.addWidget(self.grid, stretch=1) + self.plots: dict[str, pg.PlotWidget] = {} + for index, (name, (title, *_rest)) in enumerate(SAR3D_VIEWS.items()): + plot = create_plot(title) + grid_layout.addWidget(plot, index // 2, index % 2) + self.plots[name] = plot lut = build_lut(["#081c15", "#1b4332", "#ffd166", "#f94144"]) # the GPR palette lut = np.column_stack([lut, np.full(len(lut), 255, dtype=np.uint8)]) lut[0, 3] = 0 # below DISPLAY_THRESHOLD - self._sar3d_images = {} - self._sar3d_peaks = {} + self.images: dict[str, pg.ImageItem] = {} + self.peaks: dict[str, pg.ScatterPlotItem] = {} for name, (_title, x_label, y_label, _axes) in SAR3D_VIEWS.items(): - plot = self._sar3d_plots[name] - self._disable_gpr_plot_interaction(plot, plot.getViewBox()) - plot.getViewBox().enableAutoRange(x=False, y=False) + plot = self.plots[name] + item = plot.getPlotItem() + assert item is not None + view = item.getViewBox() + view.setMouseEnabled(x=False, y=False) + view.setMenuEnabled(False) + view.enableAutoRange(x=False, y=False) + item.hideButtons() image = pg.ImageItem(axisOrder="row-major", lut=lut) plot.addItem(image) # Hollow, so the response at the maximum stays visible. peak = pg.ScatterPlotItem(symbol="star", size=18, pen=pg.mkPen("w", width=1.5), brush=None) peak.setZValue(3) plot.addItem(peak) - self._sar3d_images[name] = image - self._sar3d_peaks[name] = peak + self.images[name] = image + self.peaks[name] = peak if x_label: plot.showGrid(x=True, y=True, alpha=0.2) plot.setLabel("bottom", x_label, units="m") @@ -97,66 +122,72 @@ class AppWindowSar3dPlotMixin: ground.setZValue(2) plot.addItem(ground) - oblique = self._sar3d_plots["sar3d_oblique"] + oblique = self.plots["sar3d_oblique"] oblique.hideAxis("bottom") oblique.hideAxis("left") oblique.setAspectLocked(True) # The notebook scene's ground, under the image so it shows through weak voxels. - self._sar3d_ground = pg.PlotCurveItem( + self.ground = pg.PlotCurveItem( pen=pg.mkPen(115, 105, 88), brush=pg.mkBrush(115, 105, 88, 110), fillLevel="enclosed" ) - self._sar3d_ground.setZValue(-1) - oblique.addItem(self._sar3d_ground) - self._sar3d_box = pg.PlotCurveItem(pen=pg.mkPen(190, 190, 190, 170), connect="pairs") - self._sar3d_box.setZValue(2) - oblique.addItem(self._sar3d_box) - self._sar3d_axis_labels = {} + self.ground.setZValue(-1) + oblique.addItem(self.ground) + self.box = pg.PlotCurveItem(pen=pg.mkPen(190, 190, 190, 170), connect="pairs") + self.box.setZValue(2) + oblique.addItem(self.box) + self.axis_labels: dict[str, pg.TextItem] = {} for axis_name in _AXIS_EDGES: label = pg.TextItem(axis_name, color=(190, 190, 190), anchor=(0.5, 0.5)) oblique.addItem(label) - self._sar3d_axis_labels[axis_name] = label + self.axis_labels[axis_name] = label + self.clear() - def _clear_sar3d_plot(self, message: str = "SAR3D: waiting for measurements") -> None: + def clear(self, message: str = "SAR3D: waiting for measurements") -> None: """Clear all four 3D SAR views.""" for name, (title, *_rest) in SAR3D_VIEWS.items(): - self._sar3d_images[name].hide() - self._sar3d_peaks[name].setData([], []) - self._sar3d_plots[name].setTitle(title) - self._sar3d_box.setData([], []) - self._sar3d_ground.setData([], []) - for label in self._sar3d_axis_labels.values(): + self.images[name].hide() + self.peaks[name].setData([], []) + self.plots[name].setTitle(title) + self.box.setData([], []) + self.ground.setData([], []) + for label in self.axis_labels.values(): label.hide() - self._sar3d_status_label.setText(message) + self.status_label.setText(message) - def _draw_sar3d_projections(self, collection: ResultCollection) -> bool: + def draw(self, collection: ResultCollection) -> bool: """Draw four maximum projections of the volume; return success.""" status = sar3d_status(collection) - if status is not None and status[0] in (1, 2, 3): - state = {1: "no signal", 2: "error", 3: "warning"}[status[0]] - self._clear_sar3d_plot(f"SAR3D: {state} — {status[1]}") + if status is not None and status[0] != 0: + self.clear(f"SAR3D: {SAR3D_STATES[status[0]]} — {status[1]}") return False - payloads = {name: self._collection_payload_by_name(collection, name, kind=3) for name in SAR3D_VIEWS} - basis = self._collection_payload_by_name(collection, "sar3d_oblique_basis", kind=4) - if basis is None or any(payload is None or payload.image.size == 0 for payload in payloads.values()): - self._clear_sar3d_plot("SAR3D: waiting for projection data") + payloads: dict[str, ResultPayload] = {} + for name in SAR3D_VIEWS: + payload = collection_payload_by_name(collection, name, kind=3) + if payload is None or not payload.image.size: + self.clear("SAR3D: waiting for projection data") + return False + payloads[name] = payload + basis = collection_payload_by_name(collection, "sar3d_oblique_basis", kind=4) + if basis is None: + self.clear("SAR3D: waiting for projection data") return False - peak = self._collection_payload_by_name(collection, "sar3d_peak", kind=4) + peak = collection_payload_by_name(collection, "sar3d_peak", kind=4) level, *peak_xyz = peak.table[0] if peak is not None else (None,) for name, payload in payloads.items(): title, _x_label, _y_label, axes = SAR3D_VIEWS[name] rect = _image_rect(payload.image_x_axis, payload.image_y_axis) - image = self._sar3d_images[name] + image = self.images[name] image.setImage(payload.image, autoLevels=False, levels=(DISPLAY_THRESHOLD, 1.0)) image.setRect(rect) image.show() - plot = self._sar3d_plots[name] + plot = self.plots[name] plot.setRange(rect, padding=0.0) if peak_xyz: position = basis.table @ np.asarray(peak_xyz) if axes is None else [peak_xyz[axis] for axis in axes] - self._sar3d_peaks[name].setData([position[0]], [position[1]]) + self.peaks[name].setData([position[0]], [position[1]]) else: - self._sar3d_peaks[name].setData([], []) + self.peaks[name].setData([], []) plot.setTitle(title) # Scene box and ground on the oblique view, from the extents of the other views. @@ -169,19 +200,19 @@ class AppWindowSar3dPlotMixin: corners = np.array([[bounds[axis][(index >> axis) & 1] for axis in range(3)] for index in range(8)]) projected = corners @ basis.table.T box = projected[np.ravel(_BOX_EDGES)] - self._sar3d_box.setData(box[:, 0], box[:, 1]) + self.box.setData(box[:, 0], box[:, 1]) centre = projected.mean(axis=0) for axis_name, (start, end) in _AXIS_EDGES.items(): middle = (projected[start] + projected[end]) / 2 - self._sar3d_axis_labels[axis_name].setPos(*(middle + 0.08 * (middle - centre))) # just outside - self._sar3d_axis_labels[axis_name].show() + self.axis_labels[axis_name].setPos(*(middle + 0.08 * (middle - centre))) # just outside + self.axis_labels[axis_name].show() (x0, x1), _, (z0, z1) = bounds ground = (np.array([[x0, 0.0, z0], [x1, 0.0, z0], [x1, 0.0, z1], [x0, 0.0, z1], [x0, 0.0, z0]]) @ basis.table.T).T - self._sar3d_ground.setData(ground[0], ground[1]) + self.ground.setData(ground[0], ground[1]) if level is not None: x, y, z = peak_xyz - self._sar3d_plots["sar3d_oblique"].setTitle( + self.plots["sar3d_oblique"].setTitle( f"3D projection | max X {x:+.2f}, Y {y:+.2f}, Z {z:.2f} m | level {level:.3g}" ) - self._sar3d_status_label.setText(f"SAR3D: ready — {status[1]}" if status else "SAR3D: ready") + self.status_label.setText(f"SAR3D: ready — {status[1]}" if status else "SAR3D: ready") return True diff --git a/python_app/gui/controllers/renderers/trace.py b/python_app/gui/controllers/renderers/trace.py new file mode 100644 index 0000000..0911aa5 --- /dev/null +++ b/python_app/gui/controllers/renderers/trace.py @@ -0,0 +1,186 @@ +"""Magnitude and phase rendering with owned plots, curves and legends.""" + +from __future__ import annotations + +from typing import Protocol + +import numpy as np +import pyqtgraph as pg +from PyQt6.QtCore import Qt +from PyQt6.QtWidgets import QVBoxLayout, QWidget + +from python_app.gui.controllers.renderers.common import create_plot +from python_app.models.dataset_model import ComboKey, ResultCollection, TraceData +from python_app.models.gui_profile_schema import GuiPassThroughStateModel +from python_app.models.run_config_validation import parse_combos_from_text + + +class WarningLogger(Protocol): + def __call__(self, text: str, *, details: str | None = None, + once_key: str | None = None) -> None: ... + + +CurveKey = tuple[ComboKey, int, str] +Series = tuple[CurveKey, np.ndarray, np.ndarray, str] +_PALETTE = ("#4cc9f0", "#f72585", "#b8f2e6", "#ffd166", + "#90be6d", "#ff595e", "#6a4c93", "#1982c4") + + +class _TracePanel: + """One persistent curve surface and its combo legend.""" + + def __init__(self, label: str, units: str) -> None: + self.plot = create_plot("Pass-Through S21") + item = self.plot.getPlotItem() + assert item is not None + self.item = item + self.view = item.getViewBox() + self.item.showGrid(x=True, y=True, alpha=0.2) + self.item.setLabel("left", label, units=units) + self.item.setLabel("bottom", "Frequency", units="Hz") + self.item.setDownsampling(mode="peak") + self.item.setClipToView(True) + self.curves: dict[CurveKey, pg.PlotCurveItem] = {} + self.legend: pg.LegendItem | None = None + self._legend_sources: dict[ComboKey, pg.PlotCurveItem] = {} + + def clear(self) -> None: + self.item.clear() + self.curves.clear() + self.sync_legend({}) + + def update_curve(self, key: CurveKey, x: np.ndarray, y: np.ndarray, + color: str, *, width: float, dashed: bool = False) -> pg.PlotCurveItem: + pen = pg.mkPen(color, width=width, + style=Qt.PenStyle.DashLine if dashed else Qt.PenStyle.SolidLine) + curve = self.curves.get(key) + if curve is None: + curve = pg.PlotCurveItem(pen=pen, antialias=False) + self.curves[key] = curve + self.item.addItem(curve) + else: + curve.setPen(pen) + curve.setData(x, y) + return curve + + def retain_curves(self, active_keys: set[CurveKey]) -> None: + for key in self.curves.keys() - active_keys: + self.item.removeItem(self.curves.pop(key)) + + def sync_legend(self, sources: dict[ComboKey, pg.PlotCurveItem]) -> None: + if self._legend_sources == sources: + return + if self.legend is None and sources: + self.legend = self.item.addLegend(offset=(8, 8)) + if self.legend is not None: + self.legend.clear() + for combo in sorted(sources, key=lambda key: (key.input, key.output)): + self.legend.addItem(sources[combo], f"in{combo.input}/out{combo.output}") + self.legend.setVisible(bool(sources)) + self._legend_sources = sources + + +class TraceRenderer: + """Render result traces or a captured trace using explicit display settings.""" + + def __init__(self, log_warning: WarningLogger, max_points: int = 800) -> None: + self._log_warning = log_warning + self.max_points = max_points + self.widget = QWidget() + layout = QVBoxLayout(self.widget) + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(6) + self.magnitude = _TracePanel("Magnitude", "dB") + self.phase = _TracePanel("Phase", "deg") + self.magnitude.item.showAxis("bottom", show=False) + for panel in (self.magnitude, self.phase): + layout.addWidget(panel.plot, stretch=1) + + def clear(self) -> None: + self.magnitude.clear() + self.phase.clear() + + def draw(self, collection: ResultCollection, settings: GuiPassThroughStateModel) -> bool: + combo_filter = self._combo_filter(settings.combo_filter) + colors: dict[ComboKey, str] = {} + series: list[Series] = [] + for block in collection.blocks: + if combo_filter is not None and block.combo not in combo_filter: + continue + color = colors.setdefault(block.combo, _PALETTE[len(colors) % len(_PALETTE)]) + for index, payload in enumerate(block.payloads): + if payload.kind != 1 or not payload.trace.size or payload.frequency_hz.size != payload.trace.size: + continue + series.append(((block.combo, index, payload.processing_name), + payload.frequency_hz, payload.trace, color)) + return self._render(series, settings, "Pass-Through S21") + + def draw_trace(self, trace: TraceData, title: str, settings: GuiPassThroughStateModel) -> None: + self.clear() + self._render([((trace.combo, 0, "__single_trace__"), trace.frequency_hz, + trace.s21, "#ffd166")], settings, title, single=True) + + def _combo_filter(self, text: str) -> set[ComboKey] | None: + text = text.strip() + if not text: + return None + try: + return {ComboKey(combo.input, combo.output) for combo in parse_combos_from_text(text)} + except Exception as exc: # noqa: BLE001 + self._log_warning("Invalid pass-through switch-combo filter.", + details=f"{exc}\nExpected format: input:output,input:output", + once_key=f"pass_through_combo_filter_invalid_{text}") + return None + + def _render(self, series: list[Series], settings: GuiPassThroughStateModel, + title: str, *, single: bool = False) -> bool: + mag, phase = self.magnitude, self.phase + mag.plot.setVisible(settings.show_magnitude) + phase.plot.setVisible(settings.show_phase) + if not settings.show_magnitude and not settings.show_phase: + self.clear() + return False + mag.item.showAxis("bottom", show=not settings.show_phase) + active: set[CurveKey] = set() + mag_sources: dict[ComboKey, pg.PlotCurveItem] = {} + phase_sources: dict[ComboKey, pg.PlotCurveItem] = {} + x_min, x_max = np.inf, -np.inf + for key, frequency, samples, color in series: + if not samples.size or frequency.size != samples.size: + continue + active.add(key) + x_min, x_max = min(x_min, float(np.min(frequency))), max(x_max, float(np.max(frequency))) + step = max(1, int(np.ceil(samples.size / self.max_points))) if self.max_points > 0 else 1 + if settings.show_magnitude: + magnitude = 20.0 * np.log10(np.maximum(np.abs(samples[::step]), 1e-12)) + curve = mag.update_curve(key, frequency[::step], magnitude, color, + width=1.8 if single else 1.4) + mag_sources.setdefault(key[0], curve) + if settings.show_phase: + angles = np.angle(samples) + if settings.unwrap_phase: + angles = np.unwrap(angles) + curve = phase.update_curve(key, frequency[::step], np.degrees(angles[::step]), + "#80ed99" if single else color, + width=1.4 if single else 1.2, dashed=single) + phase_sources.setdefault(key[0], curve) + + for panel, visible, sources in ((mag, settings.show_magnitude, mag_sources), + (phase, settings.show_phase, phase_sources)): + panel.retain_curves(active if visible else set()) + panel.sync_legend(sources if not single else {}) + if visible: + panel.item.setTitle(title) + panel.view.invertY(False) + if active and np.isfinite(x_min) and np.isfinite(x_max): + panel.view.setXRange(x_min, x_max, padding=0.02) + if settings.show_magnitude: + mag.view.enableAutoRange(y=not settings.fixed_y_enabled) + if settings.fixed_y_enabled: + mag.view.setYRange(min(settings.y_min_db, settings.y_max_db), + max(settings.y_min_db, settings.y_max_db), padding=0.0) + if settings.show_phase: + phase.view.enableAutoRange(y=settings.unwrap_phase) + if not settings.unwrap_phase: + phase.view.setYRange(-180.0, 180.0, padding=0.02) + return bool(active) diff --git a/python_app/gui/controllers/sections/__init__.py b/python_app/gui/controllers/sections/__init__.py index 1f9c8b3..a8f2e5e 100644 --- a/python_app/gui/controllers/sections/__init__.py +++ b/python_app/gui/controllers/sections/__init__.py @@ -1,17 +1 @@ -"""Composable UI section builders used by AppWindow UI mixin.""" - -from python_app.gui.controllers.sections.data_actions_section import build_data_actions_group -from python_app.gui.controllers.sections.primary_actions_section import build_primary_actions_group -from python_app.gui.controllers.sections.preprocess_summary_section import build_preprocess_summary_group -from python_app.gui.controllers.sections.processing_section import build_processing_group -from python_app.gui.controllers.sections.radar_section import build_radar_group -from python_app.gui.controllers.sections.switch_section import build_switch_group - -__all__ = [ - "build_data_actions_group", - "build_primary_actions_group", - "build_preprocess_summary_group", - "build_processing_group", - "build_radar_group", - "build_switch_group", -] +"""Typed configuration sections.""" diff --git a/python_app/gui/controllers/sections/data_actions_section.py b/python_app/gui/controllers/sections/data_actions_section.py index 4f459c7..52b0b82 100644 --- a/python_app/gui/controllers/sections/data_actions_section.py +++ b/python_app/gui/controllers/sections/data_actions_section.py @@ -1,7 +1,5 @@ """Builder for snapshot and storage actions section.""" -from __future__ import annotations - from PyQt6.QtWidgets import ( QGridLayout, QGroupBox, @@ -15,39 +13,39 @@ from PyQt6.QtWidgets import ( ) -def build_data_actions_group(owner) -> QGroupBox: +from python_app.gui.controllers.sections.views import DataView, SectionView +from python_app.models.gui_profile_schema import GuiStateModel + + +def build_data_actions_group(gui: GuiStateModel) -> DataView: """Create snapshot save controls section.""" - group = QGroupBox("Data Actions") + view = SectionView(QGroupBox("Data Actions")) + group = view.group layout = QVBoxLayout(group) layout.setSpacing(8) - data_defaults = owner._gui_defaults.data_actions + data_defaults = gui.data_actions save_button = QPushButton("Save Last 1000") save_button.setToolTip("Save up to 1000 most recent measurements with all available data stages.") - save_button.clicked.connect(owner._save_snapshot) save_button.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) save_vna_json_button = QPushButton("Save S21 JSON") save_vna_json_button.setToolTip( - "Export one S21 VNA-history JSON per combo from the preprocessed stage (S11 is not included)." + "Export one S21 VNA-history JSON per combo from the preprocessed stage." ) - save_vna_json_button.clicked.connect(owner._save_vna_history_json) save_vna_json_button.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) remove_last_button = QPushButton("Remove Last Measurement") - remove_last_button.clicked.connect(owner._remove_last_runtime_history) remove_last_button.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) capture_tmp_reference_button = QPushButton("Capture Tmp Reference") - capture_tmp_reference_button.clicked.connect(owner._capture_tmp_reference) capture_tmp_reference_button.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) record_button = QPushButton("Start + Record to Disk") record_button.setToolTip( "Start the run (if stopped) and save the next N measurements to the path/name below." ) - record_button.clicked.connect(owner._start_run_with_recording) record_button.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) - owner._record_count = QSpinBox() - owner._record_count.setMinimum(1) - owner._record_count.setMaximum(1_000_000) - owner._record_count.setValue(int(data_defaults.record_count)) + record_count = view.add("_record_count", QSpinBox()) + record_count.setMinimum(1) + record_count.setMaximum(1_000_000) + record_count.setValue(int(data_defaults.record_count)) button_grid = QGridLayout() button_grid.setHorizontalSpacing(8) @@ -64,24 +62,26 @@ def build_data_actions_group(owner) -> QGroupBox: record_count_row = QHBoxLayout() record_count_row.setSpacing(8) record_count_row.addWidget(QLabel("Number of Sweeps to Record")) - record_count_row.addWidget(owner._record_count) + record_count_row.addWidget(record_count) record_count_row.addStretch(1) layout.addLayout(record_count_row) path_row = QHBoxLayout() path_row.setSpacing(8) - owner._save_path_input = QLineEdit(str(data_defaults.save_path)) + save_path_input = view.add("_save_path_input", QLineEdit(str(data_defaults.save_path))) browse_button = QPushButton("Browse") - browse_button.clicked.connect(owner._browse_save_path) path_row.addWidget(QLabel("Path")) - path_row.addWidget(owner._save_path_input, stretch=1) + path_row.addWidget(save_path_input, stretch=1) path_row.addWidget(browse_button) layout.addLayout(path_row) name_row = QHBoxLayout() name_row.setSpacing(8) - owner._save_name_input = QLineEdit(str(data_defaults.save_name)) + save_name_input = view.add("_save_name_input", QLineEdit(str(data_defaults.save_name))) name_row.addWidget(QLabel("Name")) - name_row.addWidget(owner._save_name_input, stretch=1) + name_row.addWidget(save_name_input, stretch=1) layout.addLayout(name_row) - return group + return DataView(group=group, fields=view.fields, save=save_button, save_json=save_vna_json_button, + remove_last=remove_last_button, capture_tmp=capture_tmp_reference_button, + record=record_button, browse=browse_button, save_path=save_path_input, + save_name=save_name_input, record_count=record_count) diff --git a/python_app/gui/controllers/sections/layout_helpers.py b/python_app/gui/controllers/sections/layout_helpers.py index 24141af..97d781d 100644 --- a/python_app/gui/controllers/sections/layout_helpers.py +++ b/python_app/gui/controllers/sections/layout_helpers.py @@ -1,60 +1,33 @@ -"""Shared layout helpers for section builders.""" - -from __future__ import annotations +"""Shared two-column form layout for settings sections.""" from typing import TypeAlias from PyQt6.QtWidgets import QFormLayout, QHBoxLayout, QWidget - FormLabel: TypeAlias = str | QWidget FormRow: TypeAlias = QWidget | tuple[FormLabel, QWidget] def build_two_column_form_widget( - parent: QWidget, - rows: list[FormRow], - *, - split_index: int | None = None, + parent: QWidget, rows: list[FormRow], *, split_index: int | None = None, ) -> QWidget: - """Build one widget containing two side-by-side form columns.""" panel = QWidget(parent) - panel_layout = QHBoxLayout(panel) - panel_layout.setContentsMargins(0, 0, 0, 0) - panel_layout.setSpacing(12) - - left_column = QWidget(panel) - left_form = _create_form_layout(left_column) - right_column = QWidget(panel) - right_form = _create_form_layout(right_column) - - panel_layout.addWidget(left_column, stretch=1) - panel_layout.addWidget(right_column, stretch=1) - - actual_split_index = (len(rows) + 1) // 2 if split_index is None else split_index - actual_split_index = max(0, min(actual_split_index, len(rows))) - - for row in rows[:actual_split_index]: - _add_form_row(left_form, row) - for row in rows[actual_split_index:]: - _add_form_row(right_form, row) + layout = QHBoxLayout(panel) + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(12) + split = (len(rows) + 1) // 2 if split_index is None else max(0, min(split_index, len(rows))) + for column_rows in (rows[:split], rows[split:]): + column = QWidget(panel) + form = QFormLayout(column) + form.setContentsMargins(0, 0, 0, 0) + form.setHorizontalSpacing(10) + form.setVerticalSpacing(6) + form.setFieldGrowthPolicy(QFormLayout.FieldGrowthPolicy.AllNonFixedFieldsGrow) + for row in column_rows: + if isinstance(row, tuple): + label, widget = row + form.addRow(label, widget) + else: + form.addRow(row) + layout.addWidget(column, stretch=1) return panel - - -def _create_form_layout(parent: QWidget) -> QFormLayout: - """Create a compact growing form layout for one column.""" - form = QFormLayout(parent) - form.setContentsMargins(0, 0, 0, 0) - form.setHorizontalSpacing(10) - form.setVerticalSpacing(6) - form.setFieldGrowthPolicy(QFormLayout.FieldGrowthPolicy.AllNonFixedFieldsGrow) - return form - - -def _add_form_row(form: QFormLayout, row: FormRow) -> None: - """Append one form row, with or without an explicit label.""" - if isinstance(row, tuple): - label, field = row - form.addRow(label, field) - return - form.addRow(row) diff --git a/python_app/gui/controllers/sections/preprocess_summary_section.py b/python_app/gui/controllers/sections/preprocess_summary_section.py index d1c3c51..2da7b01 100644 --- a/python_app/gui/controllers/sections/preprocess_summary_section.py +++ b/python_app/gui/controllers/sections/preprocess_summary_section.py @@ -1,26 +1,26 @@ """Builder for preprocess set summary section.""" -from __future__ import annotations - from PyQt6.QtWidgets import QGroupBox, QLabel, QVBoxLayout -from python_app.gui.controllers.sections.layout_helpers import build_two_column_form_widget -from python_app.orchestration.preprocess_assets import VISIBLE_PREPROCESS_ASSET_KEYS, preprocess_asset_display_name +from python_app.gui.controllers.sections.layout_helpers import FormRow, build_two_column_form_widget +from python_app.orchestration.preprocess_assets import PREPROCESS_ASSET_NAMES, preprocess_asset_display_name + +from python_app.gui.controllers.sections.views import PreprocessView -def build_preprocess_summary_group(owner) -> QGroupBox: +def build_preprocess_summary_group() -> PreprocessView: """Create selected preprocess-set summary section.""" - group = QGroupBox("Selected Preprocess Sets") + view = PreprocessView(group=QGroupBox("Selected Preprocess Sets"), labels={}) + group = view.group layout = QVBoxLayout(group) layout.setContentsMargins(10, 10, 10, 10) layout.setSpacing(8) - owner._selected_preprocess_labels = {} - rows = [] - for key in VISIBLE_PREPROCESS_ASSET_KEYS: + rows: list[FormRow] = [] + for key in PREPROCESS_ASSET_NAMES: label = QLabel("") - owner._selected_preprocess_labels[key] = label + view.labels[key] = label rows.append((preprocess_asset_display_name(key), label)) layout.addWidget(build_two_column_form_widget(group, rows, split_index=(len(rows) + 1) // 2)) - return group + return view diff --git a/python_app/gui/controllers/sections/primary_actions_section.py b/python_app/gui/controllers/sections/primary_actions_section.py index 5a67e56..1081e7d 100644 --- a/python_app/gui/controllers/sections/primary_actions_section.py +++ b/python_app/gui/controllers/sections/primary_actions_section.py @@ -1,57 +1,23 @@ -"""Builder for pinned primary action controls.""" - -from __future__ import annotations +"""Pinned primary actions; controllers connect their own buttons.""" from PyQt6.QtWidgets import QGridLayout, QGroupBox, QPushButton, QSizePolicy - -def _expanding_button(label: str) -> QPushButton: - """Create horizontally expanding action button.""" - button = QPushButton(label) - button.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) - return button +from python_app.gui.controllers.sections.views import PrimaryView -def build_primary_actions_group(owner) -> QGroupBox: - """Create pinned action block combining pipeline and hardware actions.""" - group = QGroupBox("Actions") - layout = QGridLayout(group) +def build_primary_actions_group() -> PrimaryView: + view = PrimaryView(group=QGroupBox("Actions"), start=QPushButton("Start"), + single_capture=QPushButton("Single Capture"), stop=QPushButton("Stop"), + apply_radar=QPushButton("Apply Radar"), load=QPushButton("Load Config"), + save=QPushButton("Save Config"), preprocess=QPushButton("Preprocessing"), + clear=QPushButton("Clear Runtime History")) + layout = QGridLayout(view.group) layout.setHorizontalSpacing(8) layout.setVerticalSpacing(8) - - start_button = _expanding_button("Start") - start_button.clicked.connect(owner._start_run) - layout.addWidget(start_button, 0, 0) - - single_button = _expanding_button("Single Capture") - single_button.clicked.connect(owner._start_single_capture) - layout.addWidget(single_button, 0, 1) - - stop_button = _expanding_button("Stop") - stop_button.clicked.connect(owner._stop_run) - layout.addWidget(stop_button, 0, 2) - - apply_radar_button = _expanding_button("Apply Radar") - apply_radar_button.clicked.connect(owner._apply_radar_settings) - layout.addWidget(apply_radar_button, 1, 0) - - load_config_button = _expanding_button("Load Config") - load_config_button.clicked.connect(owner._load_config_from_dialog) - layout.addWidget(load_config_button, 1, 1) - - save_config_button = _expanding_button("Save Config") - save_config_button.clicked.connect(owner._save_current_config) - layout.addWidget(save_config_button, 1, 2) - - preprocess_button = _expanding_button("Preprocessing") - preprocess_button.clicked.connect(owner._open_preprocess_panel) - layout.addWidget(preprocess_button, 2, 0) - - clear_history_button = _expanding_button("Clear Runtime History") - clear_history_button.clicked.connect(owner._clear_all_runtime_history) - layout.addWidget(clear_history_button, 2, 1, 1, 2) - - layout.setColumnStretch(0, 1) - layout.setColumnStretch(1, 1) - layout.setColumnStretch(2, 1) - return group + for index, button in enumerate((view.start, view.single_capture, view.stop, view.apply_radar, + view.load, view.save, view.preprocess, view.clear)): + button.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) + layout.addWidget(button, index // 3, index % 3, 1, 2 if button is view.clear else 1) + for column in range(3): + layout.setColumnStretch(column, 1) + return view diff --git a/python_app/gui/controllers/sections/processing_section.py b/python_app/gui/controllers/sections/processing_section.py index 12bbbb8..5c3ffac 100644 --- a/python_app/gui/controllers/sections/processing_section.py +++ b/python_app/gui/controllers/sections/processing_section.py @@ -2,24 +2,29 @@ from __future__ import annotations +from PyQt6.QtCore import QRegularExpression +from PyQt6.QtGui import QRegularExpressionValidator from PyQt6.QtWidgets import ( QCheckBox, - QComboBox, QDoubleSpinBox, QFormLayout, QGroupBox, QLineEdit, QSizePolicy, - QSpinBox, QStackedWidget, QVBoxLayout, QWidget, ) -from python_app.gui.controllers.app_window_config.state_builders import GUI_SAVE_HISTORY_LIMIT from python_app.gui.controllers.sections.layout_helpers import FormRow, build_two_column_form_widget from python_app.gui.antenna_geometry_editor import AntennaGeometryEditor - +from python_app.gui.controllers.sections.views import ( + ProcessingView, SectionView, choice, double_spin, integer_spin, write_setting, +) +from python_app.models.gui_profile_schema import GuiStateModel +from python_app.models.run_config_schema import RunConfigModel +from python_app.models.setting_fields import GUI_FIELDS +from python_app.models.run_config_validation import POSITION_CSV_PATTERN # Same semantics on every GPR-like page; the data_processor applies them. _IGNORE_SOCKET_SPEED_TOOLTIP = ( @@ -29,7 +34,7 @@ _IGNORE_SOCKET_SPEED_TOOLTIP = ( ) -def _build_processing_mode_page(parent: QStackedWidget, rows: list[FormRow], *, split_index: int | None = None) -> QWidget: +def _build_processing_mode_page(parent: QWidget, rows: list[FormRow], *, split_index: int | None = None) -> QWidget: """Create one processing-mode page with a two-column form body.""" page = QWidget(parent) page.setSizePolicy(QSizePolicy.Policy.Preferred, QSizePolicy.Policy.Fixed) @@ -40,714 +45,339 @@ def _build_processing_mode_page(parent: QStackedWidget, rows: list[FormRow], *, return page -def build_processing_group(owner) -> QGroupBox: +def build_processing_group(config: RunConfigModel, gui: GuiStateModel) -> ProcessingView: """Create processing mode section with pass-through, B-scan, GPR, and 3D SAR pages.""" - group = QGroupBox("Processing") + view = SectionView(QGroupBox("Processing")) + group = view.group form = QFormLayout(group) form.setFieldGrowthPolicy(QFormLayout.FieldGrowthPolicy.AllNonFixedFieldsGrow) - processing_defaults = owner._gui_defaults.processing - pass_defaults = processing_defaults.pass_through - bscan_defaults = processing_defaults.bscan - gpr_live_defaults = processing_defaults.gpr - legacy_gpr_defaults = processing_defaults.legacy_gpr - sar3d_defaults = processing_defaults.sar3d - - owner._processing_mode = QComboBox() - owner._processing_mode.addItems(["pass_through", "bscan", "gpr", "legacy_gpr", "sar3d"]) - owner._set_combo_current_text(owner._processing_mode, processing_defaults.selected_mode) - - owner._processing_mode_pages = QStackedWidget(group) - owner._processing_mode_pages.setSizePolicy(QSizePolicy.Policy.Preferred, QSizePolicy.Policy.Fixed) - - owner._show_magnitude_checkbox = QCheckBox("Show magnitude") - owner._show_magnitude_checkbox.setChecked(bool(pass_defaults.show_magnitude)) - - owner._show_phase_checkbox = QCheckBox("Show phase") - owner._show_phase_checkbox.setChecked(bool(pass_defaults.show_phase)) - - owner._unwrap_phase_checkbox = QCheckBox("Unwrap phase") - owner._unwrap_phase_checkbox.setToolTip( + processing_mode = view.add("_processing_mode", choice(['pass_through', 'bscan', 'gpr', 'legacy_gpr', 'sar3d'])) + processing_mode_pages = QStackedWidget(group) + processing_mode_pages.setSizePolicy(QSizePolicy.Policy.Preferred, QSizePolicy.Policy.Fixed) + show_magnitude_checkbox = view.add("_show_magnitude_checkbox", QCheckBox("Show magnitude")) + show_phase_checkbox = view.add("_show_phase_checkbox", QCheckBox("Show phase")) + unwrap_phase_checkbox = view.add("_unwrap_phase_checkbox", QCheckBox("Unwrap phase")) + unwrap_phase_checkbox.setToolTip( "Plot the cumulative (unwrapped) phase instead of wrapping to ±180°; " "the phase axis auto-scales to the unwrapped range." ) - owner._unwrap_phase_checkbox.setChecked(bool(pass_defaults.unwrap_phase)) - - owner._pass_through_combo_filter_input = QLineEdit(str(pass_defaults.combo_filter)) - owner._pass_through_combo_filter_input.setPlaceholderText("empty = all, e.g. 0:0,1:0") - - owner._pass_through_fixed_y_enabled = QCheckBox("Fix magnitude Y range") - owner._pass_through_fixed_y_enabled.setChecked(bool(pass_defaults.fixed_y_enabled)) - - owner._pass_through_y_min_db = QDoubleSpinBox() - owner._pass_through_y_min_db.setDecimals(1) - owner._pass_through_y_min_db.setRange(-240.0, 240.0) - owner._pass_through_y_min_db.setSingleStep(1.0) - owner._pass_through_y_min_db.setValue(float(pass_defaults.y_min_db)) - - owner._pass_through_y_max_db = QDoubleSpinBox() - owner._pass_through_y_max_db.setDecimals(1) - owner._pass_through_y_max_db.setRange(-240.0, 240.0) - owner._pass_through_y_max_db.setSingleStep(1.0) - owner._pass_through_y_max_db.setValue(float(pass_defaults.y_max_db)) - - owner._sync_pass_through_y_controls() + pass_through_combo_filter_input = view.add("_pass_through_combo_filter_input", QLineEdit()) + pass_through_combo_filter_input.setPlaceholderText("empty = all, e.g. 0:0,1:0") + pass_through_fixed_y_enabled = view.add("_pass_through_fixed_y_enabled", QCheckBox("Fix magnitude Y range")) + pass_through_y_min_db = view.add("_pass_through_y_min_db", double_spin(-240.0, 240.0, 1, 1.0)) + pass_through_y_max_db = view.add("_pass_through_y_max_db", double_spin(-240.0, 240.0, 1, 1.0)) pass_through_page = _build_processing_mode_page( - owner._processing_mode_pages, + processing_mode_pages, [ - owner._show_magnitude_checkbox, - owner._show_phase_checkbox, - owner._unwrap_phase_checkbox, - ("Switch combos", owner._pass_through_combo_filter_input), - owner._pass_through_fixed_y_enabled, - ("Y min dB", owner._pass_through_y_min_db), - ("Y max dB", owner._pass_through_y_max_db), + show_magnitude_checkbox, + show_phase_checkbox, + unwrap_phase_checkbox, + ("Switch combos", pass_through_combo_filter_input), + pass_through_fixed_y_enabled, + ("Y min dB", pass_through_y_min_db), + ("Y max dB", pass_through_y_max_db), ], split_index=5, ) - owner._processing_mode_pages.addWidget(pass_through_page) - - owner._bscan_axis = QComboBox() - owner._bscan_axis.addItems(["abs", "real", "phase"]) - owner._set_combo_current_text(owner._bscan_axis, bscan_defaults.axis) - - owner._bscan_cut_m = QDoubleSpinBox() - owner._bscan_cut_m.setDecimals(3) - owner._bscan_cut_m.setRange(0.0, 2.0) - owner._bscan_cut_m.setSingleStep(0.001) - owner._bscan_cut_m.setValue(float(bscan_defaults.cut_m)) - - owner._bscan_max_depth_m = QDoubleSpinBox() - owner._bscan_max_depth_m.setDecimals(1) - owner._bscan_max_depth_m.setRange(0.1, 20.0) - owner._bscan_max_depth_m.setSingleStep(0.1) - owner._bscan_max_depth_m.setValue(float(bscan_defaults.max_depth_m)) - - owner._bscan_gain = QDoubleSpinBox() - owner._bscan_gain.setDecimals(1) - owner._bscan_gain.setRange(0.0, 3.0) - owner._bscan_gain.setSingleStep(0.1) - owner._bscan_gain.setValue(float(bscan_defaults.gain)) - - owner._bscan_start_freq_mhz = QDoubleSpinBox() - owner._bscan_start_freq_mhz.setDecimals(1) - owner._bscan_start_freq_mhz.setRange(100.0, 8800.0) - owner._bscan_start_freq_mhz.setSingleStep(10.0) - owner._bscan_start_freq_mhz.setValue(float(bscan_defaults.start_freq_mhz)) - - owner._bscan_stop_freq_mhz = QDoubleSpinBox() - owner._bscan_stop_freq_mhz.setDecimals(1) - owner._bscan_stop_freq_mhz.setRange(100.0, 8800.0) - owner._bscan_stop_freq_mhz.setSingleStep(10.0) - owner._bscan_stop_freq_mhz.setValue(float(bscan_defaults.stop_freq_mhz)) - - owner._bscan_subtract_mean_ascan = QCheckBox("Subtract mean A-scan") - owner._bscan_subtract_mean_ascan.setChecked(bool(bscan_defaults.subtract_mean_ascan)) - - owner._bscan_history_window = QSpinBox() - owner._bscan_history_window.setMinimum(1) - owner._bscan_history_window.setMaximum(GUI_SAVE_HISTORY_LIMIT) - owner._bscan_history_window.setValue(int(bscan_defaults.history_window_scans)) - owner._bscan_history_window.setToolTip( + processing_mode_pages.addWidget(pass_through_page) + bscan_axis = view.add("_bscan_axis", choice(['abs', 'real', 'phase'])) + bscan_cut_m = view.add("_bscan_cut_m", double_spin(0.0, 2.0, 3, 0.001)) + bscan_max_depth_m = view.add("_bscan_max_depth_m", double_spin(0.1, 20.0, 1, 0.1)) + bscan_gain = view.add("_bscan_gain", double_spin(0.0, 3.0, 1, 0.1)) + bscan_start_freq_mhz = view.add("_bscan_start_freq_mhz", double_spin(100.0, 8800.0, 1, 10.0)) + bscan_stop_freq_mhz = view.add("_bscan_stop_freq_mhz", double_spin(100.0, 8800.0, 1, 10.0)) + bscan_subtract_mean_ascan = view.add("_bscan_subtract_mean_ascan", QCheckBox("Subtract mean A-scan")) + bscan_history_window = view.add("_bscan_history_window", integer_spin(1, 1000)) + bscan_history_window.setToolTip( "How many past sweeps the B-scan shows once acquisition is stopped. While " "running, the window stays clamped to the C++ ring capacity." ) bscan_page = _build_processing_mode_page( - owner._processing_mode_pages, + processing_mode_pages, [ - ("Axis", owner._bscan_axis), - ("Cut m", owner._bscan_cut_m), - ("Max depth m", owner._bscan_max_depth_m), - ("Gain", owner._bscan_gain), - ("Start MHz", owner._bscan_start_freq_mhz), - ("Stop MHz", owner._bscan_stop_freq_mhz), - ("Scans to show (stopped)", owner._bscan_history_window), - owner._bscan_subtract_mean_ascan, + ("Axis", bscan_axis), + ("Cut m", bscan_cut_m), + ("Max depth m", bscan_max_depth_m), + ("Gain", bscan_gain), + ("Start MHz", bscan_start_freq_mhz), + ("Stop MHz", bscan_stop_freq_mhz), + ("Scans to show (stopped)", bscan_history_window), + bscan_subtract_mean_ascan, ], split_index=4, ) - owner._processing_mode_pages.addWidget(bscan_page) + processing_mode_pages.addWidget(bscan_page) - gpr_defaults = owner._defaults_config.gpr + gpr_defaults = config.gpr # Medium permittivity is a legacy-GPR-only knob (shown on the legacy page below). - # Coherent BP fixes the medium to eps_r = 1 (Horns_motion_3libre.py), so it is not - # offered there. - owner._gpr_relative_permittivity = QDoubleSpinBox() - owner._gpr_relative_permittivity.setDecimals(4) - owner._gpr_relative_permittivity.setRange(0.0001, 1000.0) - owner._gpr_relative_permittivity.setSingleStep(0.05) - owner._gpr_relative_permittivity.setValue(float(gpr_defaults.relative_permittivity)) - - owner._gpr_tx_geometry_input = AntennaGeometryEditor( - "output_pos", gpr_defaults.tx_geometry, position_count=owner._defaults_config.output_switch.positions, - ) - owner._gpr_rx_geometry_input = AntennaGeometryEditor( - "input_pos", gpr_defaults.rx_geometry, position_count=owner._defaults_config.input_switch.positions, - ) - - owner._gpr_common_page = _build_processing_mode_page( + # Coherent BP fixes the medium to eps_r = 1, so it is not offered there. + gpr_relative_permittivity = view.add("_gpr_relative_permittivity", double_spin(0.0001, 1000.0, 4, 0.05)) + gpr_relative_permittivity.setValue(float(gpr_defaults.relative_permittivity)) + gpr_tx_geometry_input = view.add("_gpr_tx_geometry_input", AntennaGeometryEditor( + "output_pos", gpr_defaults.tx_geometry, position_count=config.output_switch.positions, + )) + gpr_rx_geometry_input = view.add("_gpr_rx_geometry_input", AntennaGeometryEditor( + "input_pos", gpr_defaults.rx_geometry, position_count=config.input_switch.positions, + )) + gpr_common_page = _build_processing_mode_page( group, [ - ("Tx geometry", owner._gpr_tx_geometry_input), - ("Rx geometry", owner._gpr_rx_geometry_input), + ("Tx geometry", gpr_tx_geometry_input), + ("Rx geometry", gpr_rx_geometry_input), ], split_index=1, ) - - owner._gpr_tx_geometry_input.changed.connect(owner._on_gpr_geometry_changed) - owner._gpr_rx_geometry_input.changed.connect(owner._on_gpr_geometry_changed) - owner._gpr_relative_permittivity.valueChanged.connect(owner._on_gpr_geometry_changed) - - owner._gpr_input_positions_input = QLineEdit(str(gpr_live_defaults.input_positions)) - owner._gpr_input_positions_input.setPlaceholderText("0,1,2") - - owner._gpr_output_positions_input = QLineEdit(str(gpr_live_defaults.output_positions)) - owner._gpr_output_positions_input.setPlaceholderText("0,1") - - owner._gpr_min_depth_m = QDoubleSpinBox() - owner._gpr_min_depth_m.setDecimals(2) - owner._gpr_min_depth_m.setRange(0.0, 50.0) - owner._gpr_min_depth_m.setSingleStep(0.1) - owner._gpr_min_depth_m.setValue(float(gpr_live_defaults.min_depth_m)) - - owner._gpr_max_depth_m = QDoubleSpinBox() - owner._gpr_max_depth_m.setDecimals(2) - owner._gpr_max_depth_m.setRange(0.1, 50.0) - owner._gpr_max_depth_m.setSingleStep(0.1) - owner._gpr_max_depth_m.setValue(float(gpr_live_defaults.max_depth_m)) - - owner._gpr_range_comp_power = QDoubleSpinBox() - owner._gpr_range_comp_power.setDecimals(3) - owner._gpr_range_comp_power.setRange(0.0, 5.0) - owner._gpr_range_comp_power.setSingleStep(0.01) - owner._gpr_range_comp_power.setValue(float(gpr_live_defaults.range_comp_power)) - - owner._gpr_object_min_frac = QDoubleSpinBox() - owner._gpr_object_min_frac.setDecimals(2) - owner._gpr_object_min_frac.setRange(0.0, 1.0) - owner._gpr_object_min_frac.setSingleStep(0.05) - owner._gpr_object_min_frac.setToolTip( - "Object detection stops once a peak falls below this fraction of the global " - "maximum (Horns_motion_3libre.py BP_OBJECT_MIN_FRAC)." + gpr_input_positions_input = view.add("_gpr_input_positions_input", QLineEdit()) + gpr_input_positions_input.setPlaceholderText("0,1,2") + gpr_output_positions_input = view.add("_gpr_output_positions_input", QLineEdit()) + gpr_output_positions_input.setPlaceholderText("0,1") + gpr_min_depth_m = view.add("_gpr_min_depth_m", double_spin(0.0, 50.0, 2, 0.1)) + gpr_max_depth_m = view.add("_gpr_max_depth_m", double_spin(0.1, 50.0, 2, 0.1)) + gpr_range_comp_power = view.add("_gpr_range_comp_power", double_spin(0.0, 5.0, 3, 0.01)) + gpr_object_min_frac = view.add("_gpr_object_min_frac", double_spin(0.0, 1.0, 2, 0.05)) + gpr_object_min_frac.setToolTip( + "Object detection stops once a peak falls below this fraction of the global maximum." ) - owner._gpr_object_min_frac.setValue(float(gpr_live_defaults.object_min_frac)) - - owner._gpr_score_mode = QComboBox() - owner._gpr_score_mode.addItems(["peak", "combined"]) - owner._set_combo_current_text(owner._gpr_score_mode, gpr_live_defaults.score_mode) - - owner._gpr_motion_mode = QComboBox() - owner._gpr_motion_mode.addItems(["int_minus", "int_focus"]) - owner._set_combo_current_text(owner._gpr_motion_mode, gpr_live_defaults.motion_mode) - owner._gpr_motion_mode.setToolTip( + gpr_score_mode = view.add("_gpr_score_mode", choice(['peak', 'combined'])) + gpr_motion_mode = view.add("_gpr_motion_mode", choice(['int_minus', 'int_focus'])) + gpr_motion_mode.setToolTip( "Intra-sweep speed correction before the IFFT. int_minus: full correction; " "int_focus: focusing residual only (no net Z shift)." ) - - owner._gpr_look_angle_deg = QDoubleSpinBox() - owner._gpr_look_angle_deg.setDecimals(2) - owner._gpr_look_angle_deg.setRange(-90.0, 90.0) - owner._gpr_look_angle_deg.setSingleStep(0.5) - owner._gpr_look_angle_deg.setValue(float(gpr_live_defaults.look_angle_deg)) - owner._gpr_look_angle_deg.setToolTip( + gpr_look_angle_deg = view.add("_gpr_look_angle_deg", double_spin(-90.0, 90.0, 2, 0.5)) + gpr_look_angle_deg.setToolTip( "Radar look angle vs the range axis (deg). Used by intra-sweep motion correction." ) - - owner._gpr_direction_sign = QDoubleSpinBox() - owner._gpr_direction_sign.setDecimals(0) - owner._gpr_direction_sign.setRange(-1.0, 1.0) - owner._gpr_direction_sign.setSingleStep(2.0) - owner._gpr_direction_sign.setValue(float(gpr_live_defaults.direction_sign)) - owner._gpr_direction_sign.setToolTip( + gpr_direction_sign = view.add("_gpr_direction_sign", double_spin(-1.0, 1.0, 0, 2.0)) + gpr_direction_sign.setToolTip( "Motion direction along range: +1 (later frequencies deeper) or -1 (shallower)." ) - - owner._gpr_ignore_socket_speed_enabled = QCheckBox("Ignore socket speed") - owner._gpr_ignore_socket_speed_enabled.setChecked(bool(gpr_live_defaults.ignore_socket_speed_enabled)) - owner._gpr_ignore_socket_speed_enabled.setToolTip(_IGNORE_SOCKET_SPEED_TOOLTIP) - - owner._gpr_speed_m_s = QDoubleSpinBox() - owner._gpr_speed_m_s.setDecimals(3) - owner._gpr_speed_m_s.setRange(-100.0, 100.0) - owner._gpr_speed_m_s.setSingleStep(0.05) - owner._gpr_speed_m_s.setValue(float(gpr_live_defaults.speed_m_s)) - owner._gpr_speed_m_s.setToolTip( + gpr_ignore_socket_speed_enabled = view.add("_gpr_ignore_socket_speed_enabled", QCheckBox("Ignore socket speed")) + gpr_ignore_socket_speed_enabled.setToolTip(_IGNORE_SOCKET_SPEED_TOOLTIP) + gpr_speed_m_s = view.add("_gpr_speed_m_s", double_spin(-100.0, 100.0, 3, 0.05)) + gpr_speed_m_s.setToolTip( "Radar speed (m/s) used by intra-sweep motion correction when 'Ignore socket speed' is on." ) - - owner._gpr_max_detected_objects_to_draw = QSpinBox() - owner._gpr_max_detected_objects_to_draw.setRange(0, 10_000) - owner._gpr_max_detected_objects_to_draw.setValue(int(gpr_live_defaults.max_detected_objects_to_draw)) - - owner._gpr_draw_top_m_objects = QSpinBox() - owner._gpr_draw_top_m_objects.setRange(0, 10_000) - owner._gpr_draw_top_m_objects.setValue(int(gpr_live_defaults.draw_top_m_objects)) - - owner._gpr_object_approach_min_frames = QSpinBox() - owner._gpr_object_approach_min_frames.setRange(1, 100) - owner._gpr_object_approach_min_frames.setToolTip( + gpr_max_detected_objects_to_draw = view.add("_gpr_max_detected_objects_to_draw", integer_spin(0, 10000)) + gpr_draw_top_m_objects = view.add("_gpr_draw_top_m_objects", integer_spin(0, 10000)) + gpr_object_approach_min_frames = view.add("_gpr_object_approach_min_frames", integer_spin(1, 100)) + gpr_object_approach_min_frames.setToolTip( "Show an object only after it persists as a motion-consistent track this many " "consecutive frames (1 disables the approach filter)." ) - owner._gpr_object_approach_min_frames.setValue(int(gpr_live_defaults.object_approach_min_frames)) - - owner._gpr_start_freq_mhz = QDoubleSpinBox() - owner._gpr_start_freq_mhz.setDecimals(1) - owner._gpr_start_freq_mhz.setRange(100.0, 8800.0) - owner._gpr_start_freq_mhz.setSingleStep(10.0) - owner._gpr_start_freq_mhz.setValue(float(gpr_live_defaults.start_freq_mhz)) - - owner._gpr_stop_freq_mhz = QDoubleSpinBox() - owner._gpr_stop_freq_mhz.setDecimals(1) - owner._gpr_stop_freq_mhz.setRange(100.0, 8800.0) - owner._gpr_stop_freq_mhz.setSingleStep(10.0) - owner._gpr_stop_freq_mhz.setValue(float(gpr_live_defaults.stop_freq_mhz)) - - owner._gpr_background_subtract_enabled = QCheckBox("Subtract mean of previous collections") - owner._gpr_background_subtract_enabled.setChecked(bool(gpr_live_defaults.background_subtract_enabled)) - - owner._gpr_background_mean_count = QSpinBox() - owner._gpr_background_mean_count.setRange(0, 10_000) - owner._gpr_background_mean_count.setValue(int(gpr_live_defaults.background_mean_count)) - - owner._gpr_remove_sidelobe_objects_enabled = QCheckBox("Remove sidelobe objects") - owner._gpr_remove_sidelobe_objects_enabled.setChecked(bool(gpr_live_defaults.remove_sidelobe_objects_enabled)) - - owner._gpr_render_mode = QComboBox() - owner._gpr_render_mode.addItems(["heatmap", "objects_only"]) - owner._set_combo_current_text(owner._gpr_render_mode, gpr_live_defaults.render_mode) - - owner._gpr_visible_x_min_m = QDoubleSpinBox() - owner._gpr_visible_x_min_m.setDecimals(2) - owner._gpr_visible_x_min_m.setRange(-100.0, 100.0) - owner._gpr_visible_x_min_m.setSingleStep(0.1) - owner._gpr_visible_x_min_m.setValue(float(gpr_live_defaults.visible_x_min_m)) - - owner._gpr_visible_x_max_m = QDoubleSpinBox() - owner._gpr_visible_x_max_m.setDecimals(2) - owner._gpr_visible_x_max_m.setRange(-100.0, 100.0) - owner._gpr_visible_x_max_m.setSingleStep(0.1) - owner._gpr_visible_x_max_m.setValue(float(gpr_live_defaults.visible_x_max_m)) - - owner._gpr_visible_z_min_m = QDoubleSpinBox() - owner._gpr_visible_z_min_m.setDecimals(2) - owner._gpr_visible_z_min_m.setRange(0.0, 100.0) - owner._gpr_visible_z_min_m.setSingleStep(0.1) - owner._gpr_visible_z_min_m.setValue(float(gpr_live_defaults.visible_z_min_m)) - - owner._gpr_visible_z_max_m = QDoubleSpinBox() - owner._gpr_visible_z_max_m.setDecimals(2) - owner._gpr_visible_z_max_m.setRange(0.0, 100.0) - owner._gpr_visible_z_max_m.setSingleStep(0.1) - owner._gpr_visible_z_max_m.setValue(float(gpr_live_defaults.visible_z_max_m)) - - owner._gpr_imaging_plane_y_m = QDoubleSpinBox() - owner._gpr_imaging_plane_y_m.setDecimals(3) - owner._gpr_imaging_plane_y_m.setRange(-50.0, 50.0) - owner._gpr_imaging_plane_y_m.setSingleStep(0.05) - owner._gpr_imaging_plane_y_m.setValue(float(gpr_live_defaults.imaging_plane_y_m)) - owner._gpr_imaging_plane_y_m.setToolTip( + gpr_start_freq_mhz = view.add("_gpr_start_freq_mhz", double_spin(100.0, 8800.0, 1, 10.0)) + gpr_stop_freq_mhz = view.add("_gpr_stop_freq_mhz", double_spin(100.0, 8800.0, 1, 10.0)) + gpr_background_subtract_enabled = view.add("_gpr_background_subtract_enabled", QCheckBox("Subtract mean of previous collections")) + gpr_background_mean_count = view.add("_gpr_background_mean_count", integer_spin(0, 10000)) + gpr_remove_sidelobe_objects_enabled = view.add("_gpr_remove_sidelobe_objects_enabled", QCheckBox("Remove sidelobe objects")) + gpr_render_mode = view.add("_gpr_render_mode", choice(['heatmap', 'objects_only'])) + gpr_visible_x_min_m = view.add("_gpr_visible_x_min_m", double_spin(-100.0, 100.0, 2, 0.1)) + gpr_visible_x_max_m = view.add("_gpr_visible_x_max_m", double_spin(-100.0, 100.0, 2, 0.1)) + gpr_visible_z_min_m = view.add("_gpr_visible_z_min_m", double_spin(0.0, 100.0, 2, 0.1)) + gpr_visible_z_max_m = view.add("_gpr_visible_z_max_m", double_spin(0.0, 100.0, 2, 0.1)) + gpr_imaging_plane_y_m = view.add("_gpr_imaging_plane_y_m", double_spin(-50.0, 50.0, 3, 0.05)) + gpr_imaging_plane_y_m.setToolTip( "Y coordinate of the BP imaging slice (m). Use 0 for legacy 1D antenna layouts." ) gpr_page = _build_processing_mode_page( - owner._processing_mode_pages, + processing_mode_pages, [ - ("Input positions", owner._gpr_input_positions_input), - ("Output positions", owner._gpr_output_positions_input), - ("Min depth m", owner._gpr_min_depth_m), - ("Max depth m", owner._gpr_max_depth_m), - ("Range comp power", owner._gpr_range_comp_power), - ("Object min frac", owner._gpr_object_min_frac), - ("Score mode", owner._gpr_score_mode), - ("Motion mode", owner._gpr_motion_mode), - ("Look angle deg", owner._gpr_look_angle_deg), - ("Direction sign", owner._gpr_direction_sign), - owner._gpr_ignore_socket_speed_enabled, - ("Speed m/s", owner._gpr_speed_m_s), - ("Render mode", owner._gpr_render_mode), - ("Max detected objects", owner._gpr_max_detected_objects_to_draw), - ("Draw top M objects", owner._gpr_draw_top_m_objects), - ("Approach min frames", owner._gpr_object_approach_min_frames), - ("Start MHz", owner._gpr_start_freq_mhz), - ("Stop MHz", owner._gpr_stop_freq_mhz), - ("Imaging plane Y m", owner._gpr_imaging_plane_y_m), - ("Visible X min m", owner._gpr_visible_x_min_m), - ("Visible X max m", owner._gpr_visible_x_max_m), - ("Visible Z min m", owner._gpr_visible_z_min_m), - ("Visible Z max m", owner._gpr_visible_z_max_m), - owner._gpr_background_subtract_enabled, - ("Mean count", owner._gpr_background_mean_count), - owner._gpr_remove_sidelobe_objects_enabled, + ("Input positions", gpr_input_positions_input), + ("Output positions", gpr_output_positions_input), + ("Min depth m", gpr_min_depth_m), + ("Max depth m", gpr_max_depth_m), + ("Range comp power", gpr_range_comp_power), + ("Object min frac", gpr_object_min_frac), + ("Score mode", gpr_score_mode), + ("Motion mode", gpr_motion_mode), + ("Look angle deg", gpr_look_angle_deg), + ("Direction sign", gpr_direction_sign), + gpr_ignore_socket_speed_enabled, + ("Speed m/s", gpr_speed_m_s), + ("Render mode", gpr_render_mode), + ("Max detected objects", gpr_max_detected_objects_to_draw), + ("Draw top M objects", gpr_draw_top_m_objects), + ("Approach min frames", gpr_object_approach_min_frames), + ("Start MHz", gpr_start_freq_mhz), + ("Stop MHz", gpr_stop_freq_mhz), + ("Imaging plane Y m", gpr_imaging_plane_y_m), + ("Visible X min m", gpr_visible_x_min_m), + ("Visible X max m", gpr_visible_x_max_m), + ("Visible Z min m", gpr_visible_z_min_m), + ("Visible Z max m", gpr_visible_z_max_m), + gpr_background_subtract_enabled, + ("Mean count", gpr_background_mean_count), + gpr_remove_sidelobe_objects_enabled, ], split_index=8, ) - owner._processing_mode_pages.addWidget(gpr_page) - - owner._legacy_gpr_config_mode = QComboBox() - owner._legacy_gpr_config_mode.addItems(["point", "extended"]) - owner._set_combo_current_text(owner._legacy_gpr_config_mode, legacy_gpr_defaults.mode) - - owner._legacy_gpr_input_positions_input = QLineEdit(str(legacy_gpr_defaults.input_positions)) - owner._legacy_gpr_input_positions_input.setPlaceholderText("0,1,2") - - owner._legacy_gpr_output_positions_input = QLineEdit(str(legacy_gpr_defaults.output_positions)) - owner._legacy_gpr_output_positions_input.setPlaceholderText("0,1") - - owner._legacy_gpr_min_depth_m = QDoubleSpinBox() - owner._legacy_gpr_min_depth_m.setDecimals(2) - owner._legacy_gpr_min_depth_m.setRange(0.0, 50.0) - owner._legacy_gpr_min_depth_m.setSingleStep(0.1) - owner._legacy_gpr_min_depth_m.setValue(float(legacy_gpr_defaults.min_depth_m)) - - owner._legacy_gpr_max_depth_m = QDoubleSpinBox() - owner._legacy_gpr_max_depth_m.setDecimals(2) - owner._legacy_gpr_max_depth_m.setRange(0.1, 50.0) - owner._legacy_gpr_max_depth_m.setSingleStep(0.1) - owner._legacy_gpr_max_depth_m.setValue(float(legacy_gpr_defaults.max_depth_m)) - - owner._legacy_gpr_comp_power = QDoubleSpinBox() - owner._legacy_gpr_comp_power.setDecimals(3) - owner._legacy_gpr_comp_power.setRange(0.0, 5.0) - owner._legacy_gpr_comp_power.setSingleStep(0.05) - owner._legacy_gpr_comp_power.setValue(float(legacy_gpr_defaults.comp_power)) - - owner._legacy_gpr_snr_thresh = QDoubleSpinBox() - owner._legacy_gpr_snr_thresh.setDecimals(2) - owner._legacy_gpr_snr_thresh.setRange(0.0, 1_000.0) - owner._legacy_gpr_snr_thresh.setSingleStep(0.1) - owner._legacy_gpr_snr_thresh.setValue(float(legacy_gpr_defaults.snr_thresh)) - - owner._legacy_gpr_snr_comp_max = QDoubleSpinBox() - owner._legacy_gpr_snr_comp_max.setDecimals(2) - owner._legacy_gpr_snr_comp_max.setRange(0.0, 1_000.0) - owner._legacy_gpr_snr_comp_max.setSingleStep(0.5) - owner._legacy_gpr_snr_comp_max.setValue(float(legacy_gpr_defaults.snr_comp_max)) - - owner._legacy_gpr_start_freq_mhz = QDoubleSpinBox() - owner._legacy_gpr_start_freq_mhz.setDecimals(1) - owner._legacy_gpr_start_freq_mhz.setRange(100.0, 8800.0) - owner._legacy_gpr_start_freq_mhz.setSingleStep(10.0) - owner._legacy_gpr_start_freq_mhz.setValue(float(legacy_gpr_defaults.start_freq_mhz)) - - owner._legacy_gpr_stop_freq_mhz = QDoubleSpinBox() - owner._legacy_gpr_stop_freq_mhz.setDecimals(1) - owner._legacy_gpr_stop_freq_mhz.setRange(100.0, 8800.0) - owner._legacy_gpr_stop_freq_mhz.setSingleStep(10.0) - owner._legacy_gpr_stop_freq_mhz.setValue(float(legacy_gpr_defaults.stop_freq_mhz)) - - owner._legacy_gpr_speed_m_s = QDoubleSpinBox() - owner._legacy_gpr_speed_m_s.setDecimals(3) - owner._legacy_gpr_speed_m_s.setRange(-100.0, 100.0) - owner._legacy_gpr_speed_m_s.setSingleStep(0.01) - owner._legacy_gpr_speed_m_s.setValue(float(legacy_gpr_defaults.speed_m_s)) - - owner._legacy_gpr_ignore_socket_speed_enabled = QCheckBox("Ignore socket speed") - owner._legacy_gpr_ignore_socket_speed_enabled.setChecked( - bool(legacy_gpr_defaults.ignore_socket_speed_enabled) - ) - owner._legacy_gpr_ignore_socket_speed_enabled.setToolTip(_IGNORE_SOCKET_SPEED_TOOLTIP) - - owner._legacy_gpr_look_angle_deg = QDoubleSpinBox() - owner._legacy_gpr_look_angle_deg.setDecimals(2) - owner._legacy_gpr_look_angle_deg.setRange(-90.0, 90.0) - owner._legacy_gpr_look_angle_deg.setSingleStep(0.1) - owner._legacy_gpr_look_angle_deg.setValue(float(legacy_gpr_defaults.look_angle_deg)) + processing_mode_pages.addWidget(gpr_page) + legacy_gpr_config_mode = view.add("_legacy_gpr_config_mode", choice(['point', 'extended'])) + legacy_gpr_input_positions_input = view.add("_legacy_gpr_input_positions_input", QLineEdit()) + legacy_gpr_input_positions_input.setPlaceholderText("0,1,2") + legacy_gpr_output_positions_input = view.add("_legacy_gpr_output_positions_input", QLineEdit()) + legacy_gpr_output_positions_input.setPlaceholderText("0,1") + legacy_gpr_min_depth_m = view.add("_legacy_gpr_min_depth_m", double_spin(0.0, 50.0, 2, 0.1)) + legacy_gpr_max_depth_m = view.add("_legacy_gpr_max_depth_m", double_spin(0.1, 50.0, 2, 0.1)) + legacy_gpr_comp_power = view.add("_legacy_gpr_comp_power", double_spin(0.0, 5.0, 3, 0.05)) + legacy_gpr_snr_thresh = view.add("_legacy_gpr_snr_thresh", double_spin(0.0, 1000.0, 2, 0.1)) + legacy_gpr_snr_comp_max = view.add("_legacy_gpr_snr_comp_max", double_spin(0.0, 1000.0, 2, 0.5)) + legacy_gpr_start_freq_mhz = view.add("_legacy_gpr_start_freq_mhz", double_spin(100.0, 8800.0, 1, 10.0)) + legacy_gpr_stop_freq_mhz = view.add("_legacy_gpr_stop_freq_mhz", double_spin(100.0, 8800.0, 1, 10.0)) + legacy_gpr_speed_m_s = view.add("_legacy_gpr_speed_m_s", double_spin(-100.0, 100.0, 3, 0.01)) + legacy_gpr_ignore_socket_speed_enabled = view.add("_legacy_gpr_ignore_socket_speed_enabled", QCheckBox("Ignore socket speed")) + legacy_gpr_ignore_socket_speed_enabled.setToolTip(_IGNORE_SOCKET_SPEED_TOOLTIP) + legacy_gpr_look_angle_deg = view.add("_legacy_gpr_look_angle_deg", double_spin(-90.0, 90.0, 2, 0.1)) # Intra-sweep phase compensation toggle. When checked, S21 phase is # corrected per frequency point before the IFFT so the radar's motion # during one sweep is removed at the spectrum stage. Only matters when # speed != 0; otherwise has no effect by construction. - owner._legacy_gpr_apply_freq_phase_correction = QCheckBox("Apply intra-sweep phase correction") - owner._legacy_gpr_apply_freq_phase_correction.setChecked( - bool(legacy_gpr_defaults.apply_freq_phase_correction) - ) + legacy_gpr_apply_freq_phase_correction = view.add("_legacy_gpr_apply_freq_phase_correction", QCheckBox("Apply intra-sweep phase correction")) # Anchor point for the motion model's per-event `dt_ref`. `frame_center` # spreads motion symmetrically around the frame midpoint; `first_tx_event` # accumulates it forward from the first event center. - owner._legacy_gpr_reference_mode = QComboBox() - owner._legacy_gpr_reference_mode.addItems(["frame_center", "first_tx_event"]) - owner._set_combo_current_text(owner._legacy_gpr_reference_mode, legacy_gpr_defaults.reference_mode) - - owner._legacy_gpr_background_subtract_enabled = QCheckBox("Subtract mean of previous collections") - owner._legacy_gpr_background_subtract_enabled.setChecked(bool(legacy_gpr_defaults.background_subtract_enabled)) - - owner._legacy_gpr_background_mean_count = QSpinBox() - owner._legacy_gpr_background_mean_count.setRange(0, 10_000) - owner._legacy_gpr_background_mean_count.setValue(int(legacy_gpr_defaults.background_mean_count)) - - owner._legacy_gpr_render_mode = QComboBox() - owner._legacy_gpr_render_mode.addItems(["heatmap", "objects_only"]) - owner._set_combo_current_text(owner._legacy_gpr_render_mode, legacy_gpr_defaults.render_mode) - - owner._legacy_gpr_min_visible_pair_count = QSpinBox() - owner._legacy_gpr_min_visible_pair_count.setRange(1, 10_000) - owner._legacy_gpr_min_visible_pair_count.setValue(int(legacy_gpr_defaults.min_visible_pair_count)) - - owner._legacy_gpr_visible_x_min_m = QDoubleSpinBox() - owner._legacy_gpr_visible_x_min_m.setDecimals(2) - owner._legacy_gpr_visible_x_min_m.setRange(-100.0, 100.0) - owner._legacy_gpr_visible_x_min_m.setSingleStep(0.1) - owner._legacy_gpr_visible_x_min_m.setValue(float(legacy_gpr_defaults.visible_x_min_m)) - - owner._legacy_gpr_visible_x_max_m = QDoubleSpinBox() - owner._legacy_gpr_visible_x_max_m.setDecimals(2) - owner._legacy_gpr_visible_x_max_m.setRange(-100.0, 100.0) - owner._legacy_gpr_visible_x_max_m.setSingleStep(0.1) - owner._legacy_gpr_visible_x_max_m.setValue(float(legacy_gpr_defaults.visible_x_max_m)) - - owner._legacy_gpr_visible_z_min_m = QDoubleSpinBox() - owner._legacy_gpr_visible_z_min_m.setDecimals(2) - owner._legacy_gpr_visible_z_min_m.setRange(0.0, 100.0) - owner._legacy_gpr_visible_z_min_m.setSingleStep(0.1) - owner._legacy_gpr_visible_z_min_m.setValue(float(legacy_gpr_defaults.visible_z_min_m)) - - owner._legacy_gpr_visible_z_max_m = QDoubleSpinBox() - owner._legacy_gpr_visible_z_max_m.setDecimals(2) - owner._legacy_gpr_visible_z_max_m.setRange(0.0, 100.0) - owner._legacy_gpr_visible_z_max_m.setSingleStep(0.1) - owner._legacy_gpr_visible_z_max_m.setValue(float(legacy_gpr_defaults.visible_z_max_m)) + legacy_gpr_reference_mode = view.add("_legacy_gpr_reference_mode", choice(['frame_center', 'first_tx_event'])) + legacy_gpr_background_subtract_enabled = view.add("_legacy_gpr_background_subtract_enabled", QCheckBox("Subtract mean of previous collections")) + legacy_gpr_background_mean_count = view.add("_legacy_gpr_background_mean_count", integer_spin(0, 10000)) + legacy_gpr_render_mode = view.add("_legacy_gpr_render_mode", choice(['heatmap', 'objects_only'])) + legacy_gpr_min_visible_pair_count = view.add("_legacy_gpr_min_visible_pair_count", integer_spin(1, 10000)) + legacy_gpr_visible_x_min_m = view.add("_legacy_gpr_visible_x_min_m", double_spin(-100.0, 100.0, 2, 0.1)) + legacy_gpr_visible_x_max_m = view.add("_legacy_gpr_visible_x_max_m", double_spin(-100.0, 100.0, 2, 0.1)) + legacy_gpr_visible_z_min_m = view.add("_legacy_gpr_visible_z_min_m", double_spin(0.0, 100.0, 2, 0.1)) + legacy_gpr_visible_z_max_m = view.add("_legacy_gpr_visible_z_max_m", double_spin(0.0, 100.0, 2, 0.1)) legacy_gpr_page = _build_processing_mode_page( - owner._processing_mode_pages, + processing_mode_pages, [ - ("Config mode", owner._legacy_gpr_config_mode), - ("Relative permittivity", owner._gpr_relative_permittivity), - ("Input positions", owner._legacy_gpr_input_positions_input), - ("Output positions", owner._legacy_gpr_output_positions_input), - ("Min depth m", owner._legacy_gpr_min_depth_m), - ("Max depth m", owner._legacy_gpr_max_depth_m), - ("Comp power", owner._legacy_gpr_comp_power), - ("SNR thresh", owner._legacy_gpr_snr_thresh), - ("SNR comp max", owner._legacy_gpr_snr_comp_max), - ("Speed m/s", owner._legacy_gpr_speed_m_s), - owner._legacy_gpr_ignore_socket_speed_enabled, - ("Render mode", owner._legacy_gpr_render_mode), - ("Min visible pairs", owner._legacy_gpr_min_visible_pair_count), - ("Start MHz", owner._legacy_gpr_start_freq_mhz), - ("Stop MHz", owner._legacy_gpr_stop_freq_mhz), - ("Look angle deg", owner._legacy_gpr_look_angle_deg), - ("Reference mode", owner._legacy_gpr_reference_mode), - owner._legacy_gpr_apply_freq_phase_correction, - ("Visible X min m", owner._legacy_gpr_visible_x_min_m), - ("Visible X max m", owner._legacy_gpr_visible_x_max_m), - ("Visible Z min m", owner._legacy_gpr_visible_z_min_m), - ("Visible Z max m", owner._legacy_gpr_visible_z_max_m), - owner._legacy_gpr_background_subtract_enabled, - ("Mean count", owner._legacy_gpr_background_mean_count), + ("Config mode", legacy_gpr_config_mode), + ("Relative permittivity", gpr_relative_permittivity), + ("Input positions", legacy_gpr_input_positions_input), + ("Output positions", legacy_gpr_output_positions_input), + ("Min depth m", legacy_gpr_min_depth_m), + ("Max depth m", legacy_gpr_max_depth_m), + ("Comp power", legacy_gpr_comp_power), + ("SNR thresh", legacy_gpr_snr_thresh), + ("SNR comp max", legacy_gpr_snr_comp_max), + ("Speed m/s", legacy_gpr_speed_m_s), + legacy_gpr_ignore_socket_speed_enabled, + ("Render mode", legacy_gpr_render_mode), + ("Min visible pairs", legacy_gpr_min_visible_pair_count), + ("Start MHz", legacy_gpr_start_freq_mhz), + ("Stop MHz", legacy_gpr_stop_freq_mhz), + ("Look angle deg", legacy_gpr_look_angle_deg), + ("Reference mode", legacy_gpr_reference_mode), + legacy_gpr_apply_freq_phase_correction, + ("Visible X min m", legacy_gpr_visible_x_min_m), + ("Visible X max m", legacy_gpr_visible_x_max_m), + ("Visible Z min m", legacy_gpr_visible_z_min_m), + ("Visible Z max m", legacy_gpr_visible_z_max_m), + legacy_gpr_background_subtract_enabled, + ("Mean count", legacy_gpr_background_mean_count), ], split_index=10, ) - owner._processing_mode_pages.addWidget(legacy_gpr_page) + processing_mode_pages.addWidget(legacy_gpr_page) # 3D SAR page. The labels are the names of the reference notebook # Multi_antenna_3D_clean_v1.ipynb; the shared gpr_* fields get their own # widgets here, like on the legacy page. - owner._sar3d_input_positions_input = QLineEdit(str(sar3d_defaults.input_positions)) - owner._sar3d_input_positions_input.setPlaceholderText("0,1,2") - - owner._sar3d_output_positions_input = QLineEdit(str(sar3d_defaults.output_positions)) - owner._sar3d_output_positions_input.setPlaceholderText("0,1") - - owner._sar3d_start_freq_mhz = QDoubleSpinBox() - owner._sar3d_start_freq_mhz.setDecimals(1) - owner._sar3d_start_freq_mhz.setRange(100.0, 8800.0) - owner._sar3d_start_freq_mhz.setSingleStep(10.0) - owner._sar3d_start_freq_mhz.setValue(float(sar3d_defaults.start_freq_mhz)) - - owner._sar3d_stop_freq_mhz = QDoubleSpinBox() - owner._sar3d_stop_freq_mhz.setDecimals(1) - owner._sar3d_stop_freq_mhz.setRange(100.0, 8800.0) - owner._sar3d_stop_freq_mhz.setSingleStep(10.0) - owner._sar3d_stop_freq_mhz.setValue(float(sar3d_defaults.stop_freq_mhz)) - - owner._sar3d_min_depth_m = QDoubleSpinBox() - owner._sar3d_min_depth_m.setDecimals(2) - owner._sar3d_min_depth_m.setRange(0.0, 50.0) - owner._sar3d_min_depth_m.setSingleStep(0.1) - owner._sar3d_min_depth_m.setValue(float(sar3d_defaults.min_depth_m)) - owner._sar3d_min_depth_m.setToolTip("Radar depth gate start (m); the scene grid along Z follows the gate.") - - owner._sar3d_max_depth_m = QDoubleSpinBox() - owner._sar3d_max_depth_m.setDecimals(2) - owner._sar3d_max_depth_m.setRange(0.1, 50.0) - owner._sar3d_max_depth_m.setSingleStep(0.1) - owner._sar3d_max_depth_m.setValue(float(sar3d_defaults.max_depth_m)) - owner._sar3d_max_depth_m.setToolTip("Radar depth gate end (m); the scene grid along Z follows the gate.") - - owner._sar3d_voxel_m = QDoubleSpinBox() - owner._sar3d_voxel_m.setDecimals(3) - owner._sar3d_voxel_m.setRange(0.01, 0.1) - owner._sar3d_voxel_m.setSingleStep(0.005) - owner._sar3d_voxel_m.setValue(float(sar3d_defaults.voxel_m)) - owner._sar3d_voxel_m.setToolTip( + sar3d_input_positions_input = view.add("_sar3d_input_positions_input", QLineEdit()) + sar3d_input_positions_input.setPlaceholderText("0,1,2") + sar3d_output_positions_input = view.add("_sar3d_output_positions_input", QLineEdit()) + sar3d_output_positions_input.setPlaceholderText("0,1") + sar3d_start_freq_mhz = view.add("_sar3d_start_freq_mhz", double_spin(100.0, 8800.0, 1, 10.0)) + sar3d_stop_freq_mhz = view.add("_sar3d_stop_freq_mhz", double_spin(100.0, 8800.0, 1, 10.0)) + sar3d_min_depth_m = view.add("_sar3d_min_depth_m", double_spin(0.0, 50.0, 2, 0.1)) + sar3d_min_depth_m.setToolTip("Radar depth gate start (m); the scene grid along Z follows the gate.") + sar3d_max_depth_m = view.add("_sar3d_max_depth_m", double_spin(0.1, 50.0, 2, 0.1)) + sar3d_max_depth_m.setToolTip("Radar depth gate end (m); the scene grid along Z follows the gate.") + sar3d_voxel_m = view.add("_sar3d_voxel_m", double_spin(0.01, 0.1, 3, 0.005)) + sar3d_voxel_m.setToolTip( "Edge of the cubic voxel (m), the same along X, Y and Z. Smaller is finer and slower: " "halving it makes 8 times more voxels." ) - - owner._sar3d_look_angle_deg = QDoubleSpinBox() - owner._sar3d_look_angle_deg.setDecimals(2) - owner._sar3d_look_angle_deg.setRange(-90.0, 90.0) - owner._sar3d_look_angle_deg.setSingleStep(0.5) - owner._sar3d_look_angle_deg.setValue(float(sar3d_defaults.look_angle_deg)) - owner._sar3d_look_angle_deg.setToolTip( + sar3d_look_angle_deg = view.add("_sar3d_look_angle_deg", double_spin(-90.0, 90.0, 2, 0.5)) + sar3d_look_angle_deg.setToolTip( "Downward tilt of the radar (deg): motion correction along range and the ground scene axes." ) - - owner._sar3d_ignore_socket_speed_enabled = QCheckBox("Ignore socket speed") - owner._sar3d_ignore_socket_speed_enabled.setChecked(bool(sar3d_defaults.ignore_socket_speed_enabled)) - owner._sar3d_ignore_socket_speed_enabled.setToolTip(_IGNORE_SOCKET_SPEED_TOOLTIP) - - owner._sar3d_speed_m_s = QDoubleSpinBox() - owner._sar3d_speed_m_s.setDecimals(3) - owner._sar3d_speed_m_s.setRange(-100.0, 100.0) - owner._sar3d_speed_m_s.setSingleStep(0.05) - owner._sar3d_speed_m_s.setValue(float(sar3d_defaults.speed_m_s)) - - owner._sar3d_frame_radius = QSpinBox() - owner._sar3d_frame_radius.setRange(0, 24) - owner._sar3d_frame_radius.setValue(int(sar3d_defaults.frame_radius)) - owner._sar3d_frame_radius.setToolTip( + sar3d_ignore_socket_speed_enabled = view.add("_sar3d_ignore_socket_speed_enabled", QCheckBox("Ignore socket speed")) + sar3d_ignore_socket_speed_enabled.setToolTip(_IGNORE_SOCKET_SPEED_TOOLTIP) + sar3d_speed_m_s = view.add("_sar3d_speed_m_s", double_spin(-100.0, 100.0, 3, 0.05)) + sar3d_frame_radius = view.add("_sar3d_frame_radius", integer_spin(0, 24)) + sar3d_frame_radius.setToolTip( "Aperture of 2*r+1 consecutive frames. 0: the current frame only (needs a vertical antenna " "baseline); >0 adds aperture from motion and needs an accurate speed. The image lags by r frames." ) - - owner._sar3d_bp_ground_depth_m = QDoubleSpinBox() - owner._sar3d_bp_ground_depth_m.setDecimals(2) - owner._sar3d_bp_ground_depth_m.setRange(0.0, 10.0) - owner._sar3d_bp_ground_depth_m.setSingleStep(0.05) - owner._sar3d_bp_ground_depth_m.setValue(float(sar3d_defaults.bp_ground_depth_m)) - owner._sar3d_bp_ground_depth_m.setToolTip("Height of the lower antenna row above the ground (m); scene Y = 0 is the ground.") - - owner._sar3d_comp_range_power = QDoubleSpinBox() - owner._sar3d_comp_range_power.setDecimals(2) - owner._sar3d_comp_range_power.setRange(0.0, 2.0) - owner._sar3d_comp_range_power.setSingleStep(0.05) - owner._sar3d_comp_range_power.setValue(float(sar3d_defaults.comp_range_power)) - owner._sar3d_comp_range_power.setToolTip("Geometric spreading compensation (Rtx*Rrx)^power, capped at 5.") - - owner._sar3d_use_compensation = QCheckBox("SAR_USE_COMPENSATION") - owner._sar3d_use_compensation.setChecked(bool(sar3d_defaults.use_compensation)) - - owner._sar3d_normalize_each_frame = QCheckBox("SAR_NORMALIZE_EACH_FRAME") - owner._sar3d_normalize_each_frame.setChecked(bool(sar3d_defaults.normalize_each_frame)) - owner._sar3d_normalize_each_frame.setToolTip("Scale every Tx/Rx A-scan by its median inside the depth gate.") + sar3d_bp_ground_depth_m = view.add("_sar3d_bp_ground_depth_m", double_spin(0.0, 10.0, 2, 0.05)) + sar3d_bp_ground_depth_m.setToolTip("Height of the lower antenna row above the ground (m); scene Y = 0 is the ground.") + sar3d_comp_range_power = view.add("_sar3d_comp_range_power", double_spin(0.0, 2.0, 2, 0.05)) + sar3d_comp_range_power.setToolTip("Geometric spreading compensation (Rtx*Rrx)^power, capped at 5.") + sar3d_use_compensation = view.add("_sar3d_use_compensation", QCheckBox("SAR_USE_COMPENSATION")) + sar3d_normalize_each_frame = view.add("_sar3d_normalize_each_frame", QCheckBox("SAR_NORMALIZE_EACH_FRAME")) + sar3d_normalize_each_frame.setToolTip("Scale every Tx/Rx A-scan by its median inside the depth gate.") sar3d_page = _build_processing_mode_page( - owner._processing_mode_pages, + processing_mode_pages, [ - ("INPUT_IDX", owner._sar3d_input_positions_input), - ("OUTPUT_IDX", owner._sar3d_output_positions_input), - ("F_START MHz", owner._sar3d_start_freq_mhz), - ("F_STOP MHz", owner._sar3d_stop_freq_mhz), - ("MIN_DEPTH", owner._sar3d_min_depth_m), - ("MAX_DEPTH", owner._sar3d_max_depth_m), - ("SAR_VOXEL_M", owner._sar3d_voxel_m), - ("LOOK_ANGLE_DEG", owner._sar3d_look_angle_deg), - owner._sar3d_ignore_socket_speed_enabled, - ("SPEED_M_S", owner._sar3d_speed_m_s), - ("SAR_FRAME_RADIUS", owner._sar3d_frame_radius), - ("BP_GROUND_DEPTH_M", owner._sar3d_bp_ground_depth_m), - ("COMP_RANGE_POWER", owner._sar3d_comp_range_power), - owner._sar3d_use_compensation, - owner._sar3d_normalize_each_frame, + ("INPUT_IDX", sar3d_input_positions_input), + ("OUTPUT_IDX", sar3d_output_positions_input), + ("F_START MHz", sar3d_start_freq_mhz), + ("F_STOP MHz", sar3d_stop_freq_mhz), + ("MIN_DEPTH", sar3d_min_depth_m), + ("MAX_DEPTH", sar3d_max_depth_m), + ("SAR_VOXEL_M", sar3d_voxel_m), + ("LOOK_ANGLE_DEG", sar3d_look_angle_deg), + sar3d_ignore_socket_speed_enabled, + ("SPEED_M_S", sar3d_speed_m_s), + ("SAR_FRAME_RADIUS", sar3d_frame_radius), + ("BP_GROUND_DEPTH_M", sar3d_bp_ground_depth_m), + ("COMP_RANGE_POWER", sar3d_comp_range_power), + sar3d_use_compensation, + sar3d_normalize_each_frame, ], split_index=8, ) - owner._processing_mode_pages.addWidget(sar3d_page) + processing_mode_pages.addWidget(sar3d_page) - owner._processing_mode.currentTextChanged.connect(owner._on_processing_mode_changed) - owner._show_magnitude_checkbox.toggled.connect(owner._on_trace_visibility_changed) - owner._show_phase_checkbox.toggled.connect(owner._on_trace_visibility_changed) - owner._unwrap_phase_checkbox.toggled.connect(owner._on_trace_visibility_changed) - owner._pass_through_combo_filter_input.editingFinished.connect(owner._on_trace_visibility_changed) - owner._pass_through_fixed_y_enabled.toggled.connect(owner._sync_pass_through_y_controls) - owner._pass_through_fixed_y_enabled.toggled.connect(owner._on_processing_live_settings_changed) - owner._pass_through_y_min_db.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._pass_through_y_max_db.valueChanged.connect(owner._on_processing_live_settings_changed) - form.addRow("Mode", owner._processing_mode) - form.addRow(owner._processing_mode_pages) - form.addRow(owner._gpr_common_page) + form.addRow("Mode", processing_mode) + form.addRow(processing_mode_pages) + form.addRow(gpr_common_page) - owner._bscan_axis.currentTextChanged.connect(owner._on_processing_live_settings_changed) - owner._bscan_cut_m.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._bscan_max_depth_m.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._bscan_gain.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._bscan_start_freq_mhz.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._bscan_stop_freq_mhz.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._bscan_subtract_mean_ascan.toggled.connect(owner._on_processing_live_settings_changed) - owner._bscan_history_window.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._gpr_input_positions_input.editingFinished.connect(owner._on_processing_live_settings_changed) - owner._gpr_output_positions_input.editingFinished.connect(owner._on_processing_live_settings_changed) - owner._gpr_min_depth_m.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._gpr_max_depth_m.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._gpr_range_comp_power.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._gpr_object_min_frac.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._gpr_score_mode.currentTextChanged.connect(owner._on_processing_live_settings_changed) - owner._gpr_motion_mode.currentTextChanged.connect(owner._on_processing_live_settings_changed) - owner._gpr_look_angle_deg.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._gpr_direction_sign.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._gpr_ignore_socket_speed_enabled.toggled.connect(owner._on_ignore_socket_speed_toggled) - owner._gpr_speed_m_s.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._gpr_start_freq_mhz.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._gpr_stop_freq_mhz.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._gpr_background_subtract_enabled.toggled.connect(owner._on_processing_live_settings_changed) - owner._gpr_background_mean_count.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._gpr_remove_sidelobe_objects_enabled.toggled.connect(owner._on_processing_live_settings_changed) - owner._gpr_imaging_plane_y_m.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._gpr_render_mode.currentTextChanged.connect(owner._on_gpr_visual_settings_changed) - owner._gpr_max_detected_objects_to_draw.valueChanged.connect(owner._on_gpr_locator_threshold_changed) - owner._gpr_draw_top_m_objects.valueChanged.connect(owner._on_gpr_locator_threshold_changed) - owner._gpr_object_approach_min_frames.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._gpr_visible_x_min_m.valueChanged.connect(owner._on_gpr_locator_window_changed) - owner._gpr_visible_x_max_m.valueChanged.connect(owner._on_gpr_locator_window_changed) - owner._gpr_visible_z_min_m.valueChanged.connect(owner._on_gpr_locator_window_changed) - owner._gpr_visible_z_max_m.valueChanged.connect(owner._on_gpr_locator_window_changed) - owner._legacy_gpr_config_mode.currentTextChanged.connect(owner._on_processing_live_settings_changed) - owner._legacy_gpr_input_positions_input.editingFinished.connect(owner._on_processing_live_settings_changed) - owner._legacy_gpr_output_positions_input.editingFinished.connect(owner._on_processing_live_settings_changed) - owner._legacy_gpr_min_depth_m.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._legacy_gpr_max_depth_m.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._legacy_gpr_comp_power.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._legacy_gpr_snr_thresh.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._legacy_gpr_snr_comp_max.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._legacy_gpr_start_freq_mhz.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._legacy_gpr_stop_freq_mhz.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._legacy_gpr_speed_m_s.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._legacy_gpr_ignore_socket_speed_enabled.toggled.connect(owner._on_ignore_socket_speed_toggled) - owner._legacy_gpr_look_angle_deg.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._legacy_gpr_apply_freq_phase_correction.toggled.connect(owner._on_processing_live_settings_changed) - owner._legacy_gpr_reference_mode.currentTextChanged.connect(owner._on_processing_live_settings_changed) - owner._legacy_gpr_background_subtract_enabled.toggled.connect(owner._on_processing_live_settings_changed) - owner._legacy_gpr_background_mean_count.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._legacy_gpr_render_mode.currentTextChanged.connect(owner._on_gpr_visual_settings_changed) - owner._legacy_gpr_min_visible_pair_count.valueChanged.connect(owner._on_gpr_locator_threshold_changed) - owner._legacy_gpr_visible_x_min_m.valueChanged.connect(owner._on_gpr_locator_window_changed) - owner._legacy_gpr_visible_x_max_m.valueChanged.connect(owner._on_gpr_locator_window_changed) - owner._legacy_gpr_visible_z_min_m.valueChanged.connect(owner._on_gpr_locator_window_changed) - owner._legacy_gpr_visible_z_max_m.valueChanged.connect(owner._on_gpr_locator_window_changed) - owner._sar3d_input_positions_input.editingFinished.connect(owner._on_processing_live_settings_changed) - owner._sar3d_output_positions_input.editingFinished.connect(owner._on_processing_live_settings_changed) - for widget in ( - owner._sar3d_start_freq_mhz, - owner._sar3d_stop_freq_mhz, - owner._sar3d_min_depth_m, - owner._sar3d_max_depth_m, - owner._sar3d_voxel_m, - owner._sar3d_look_angle_deg, - owner._sar3d_speed_m_s, - owner._sar3d_frame_radius, - owner._sar3d_bp_ground_depth_m, - owner._sar3d_comp_range_power, - ): - widget.valueChanged.connect(owner._on_processing_live_settings_changed) - owner._sar3d_ignore_socket_speed_enabled.toggled.connect(owner._on_ignore_socket_speed_toggled) - owner._sar3d_use_compensation.toggled.connect(owner._on_processing_live_settings_changed) - owner._sar3d_normalize_each_frame.toggled.connect(owner._on_processing_live_settings_changed) + for binding in GUI_FIELDS: + if binding.path[0] == "processing": + widget = view.fields[binding.spec.widget] + if isinstance(widget, AntennaGeometryEditor): + raise TypeError("Geometry is not a scalar GUI setting") + if binding.path[-1] in {"start_freq_mhz", "stop_freq_mhz"}: + assert isinstance(widget, QDoubleSpinBox) + widget.setRange(config.radar.sweep.start_hz / 1e6, config.radar.sweep.stop_hz / 1e6) + if binding.spec.live in {"gpr_input_positions", "gpr_output_positions"}: + assert isinstance(widget, QLineEdit) + widget.setValidator(QRegularExpressionValidator(QRegularExpression(POSITION_CSV_PATTERN), widget)) + write_setting(widget, binding.read(gui)) - owner._sync_socket_speed_controls() - owner._set_processing_mode_page(owner._processing_mode.currentText()) - return group + return ProcessingView( + group=group, fields=view.fields, mode=processing_mode, pages=processing_mode_pages, + geometry_page=gpr_common_page, geometries=(gpr_tx_geometry_input, gpr_rx_geometry_input), + fixed_y=pass_through_fixed_y_enabled, y_bounds=(pass_through_y_min_db, pass_through_y_max_db), + speed_controls=((gpr_ignore_socket_speed_enabled, gpr_speed_m_s), + (legacy_gpr_ignore_socket_speed_enabled, legacy_gpr_speed_m_s), + (sar3d_ignore_socket_speed_enabled, sar3d_speed_m_s)), + trace_controls=(show_magnitude_checkbox, show_phase_checkbox, unwrap_phase_checkbox, + pass_through_combo_filter_input, pass_through_fixed_y_enabled, + pass_through_y_min_db, pass_through_y_max_db), + visual_controls=(gpr_render_mode, legacy_gpr_render_mode), + locator_controls=(gpr_max_detected_objects_to_draw, gpr_draw_top_m_objects, + legacy_gpr_min_visible_pair_count, gpr_visible_x_min_m, gpr_visible_x_max_m, + gpr_visible_z_min_m, gpr_visible_z_max_m, legacy_gpr_visible_x_min_m, + legacy_gpr_visible_x_max_m, legacy_gpr_visible_z_min_m, legacy_gpr_visible_z_max_m), + frequency_controls=(bscan_start_freq_mhz, bscan_stop_freq_mhz, gpr_start_freq_mhz, gpr_stop_freq_mhz, + legacy_gpr_start_freq_mhz, legacy_gpr_stop_freq_mhz, + sar3d_start_freq_mhz, sar3d_stop_freq_mhz), + ) diff --git a/python_app/gui/controllers/sections/radar_section.py b/python_app/gui/controllers/sections/radar_section.py index 714f193..9d654b5 100644 --- a/python_app/gui/controllers/sections/radar_section.py +++ b/python_app/gui/controllers/sections/radar_section.py @@ -1,7 +1,5 @@ """Builder for radar settings section.""" -from __future__ import annotations - import json from PyQt6.QtWidgets import ( @@ -16,103 +14,109 @@ from PyQt6.QtWidgets import ( ) from python_app.gui.controllers.sections.layout_helpers import build_two_column_form_widget +from python_app.gui.controllers.sections.views import RadarView, SectionView, write_setting +from python_app.models.run_config_schema import RunConfigModel -def build_radar_group(owner) -> QGroupBox: +def build_radar_group(config: RunConfigModel) -> RadarView: """Create radar settings controls and labels.""" - group = QGroupBox("Radar") + view = SectionView(QGroupBox("Radar")) + group = view.group layout = QVBoxLayout(group) layout.setContentsMargins(10, 10, 10, 10) layout.setSpacing(8) - defaults = owner._defaults_config.radar + defaults = config.radar - owner._vna_radar_settings_panel = QWidget(group) - vna_layout = QVBoxLayout(owner._vna_radar_settings_panel) + vna_radar_settings_panel = QWidget(group) + vna_layout = QVBoxLayout(vna_radar_settings_panel) vna_layout.setContentsMargins(0, 0, 0, 0) vna_layout.setSpacing(8) - owner._start_hz_input = QLineEdit(f"{defaults.sweep.start_hz:g}") - owner._stop_hz_input = QLineEdit(f"{defaults.sweep.stop_hz:g}") - owner._points_input = QLineEdit(str(defaults.sweep.points)) - owner._ifbw_input = QLineEdit(f"{defaults.sweep.if_bandwidth_hz:g}") - owner._power_input = QLineEdit(f"{defaults.sweep.power_dbm:g}") - owner._power_input.setToolTip("Stimulus power configured in the active profile.") - owner._start_hz_input.editingFinished.connect(owner._on_radar_sweep_limits_changed) - owner._stop_hz_input.editingFinished.connect(owner._on_radar_sweep_limits_changed) - owner._points_input.editingFinished.connect(owner._on_radar_sweep_limits_changed) - owner._ifbw_input.editingFinished.connect(owner._on_radar_sweep_limits_changed) - owner._power_input.editingFinished.connect(owner._on_radar_sweep_limits_changed) + start_hz_input = view.add("_start_hz_input", QLineEdit(f"{defaults.sweep.start_hz:g}")) + stop_hz_input = view.add("_stop_hz_input", QLineEdit(f"{defaults.sweep.stop_hz:g}")) + points_input = view.add("_points_input", QLineEdit(str(defaults.sweep.points))) + ifbw_input = view.add("_ifbw_input", QLineEdit(f"{defaults.sweep.if_bandwidth_hz:g}")) + power_input = view.add("_power_input", QLineEdit(f"{defaults.sweep.power_dbm:g}")) + power_input.setToolTip("Stimulus power configured in the active profile.") - owner._radar_start_label = QLabel("Start Hz") - owner._radar_stop_label = QLabel("Stop Hz") - owner._radar_points_label = QLabel("Points") - owner._radar_ifbw_label = QLabel("IF BW Hz") - owner._radar_power_label = QLabel("Stimulus Power dBm") + radar_start_label = QLabel("Start Hz") + radar_stop_label = QLabel("Stop Hz") + radar_points_label = QLabel("Points") + radar_ifbw_label = QLabel("IF BW Hz") + radar_power_label = QLabel("Stimulus Power dBm") - owner._radar_limits_hint = QLabel("Device limits are not available.") - owner._radar_limits_hint.setObjectName("hintLabel") + radar_limits_hint = QLabel("Device limits are not available.") + radar_limits_hint.setObjectName("hintLabel") vna_layout.addWidget( build_two_column_form_widget( group, [ - (owner._radar_start_label, owner._start_hz_input), - (owner._radar_stop_label, owner._stop_hz_input), - (owner._radar_points_label, owner._points_input), - (owner._radar_ifbw_label, owner._ifbw_input), - (owner._radar_power_label, owner._power_input), + (radar_start_label, start_hz_input), + (radar_stop_label, stop_hz_input), + (radar_points_label, points_input), + (radar_ifbw_label, ifbw_input), + (radar_power_label, power_input), ], ) ) - vna_layout.addWidget(owner._radar_limits_hint) + vna_layout.addWidget(radar_limits_hint) - owner._adc_radar_settings_panel = _build_adc_settings_panel(owner, group) + (adc_radar_settings_panel, optical_enabled, optical_mode, optical_common, optical_manual, + optical_variation, manual_panel, variation_panel) = _build_adc_settings_panel(view, config) - layout.addWidget(owner._vna_radar_settings_panel) - layout.addWidget(owner._adc_radar_settings_panel) - owner._set_radar_settings_mode(owner._defaults_config.is_kamil_adc) - return group + layout.addWidget(vna_radar_settings_panel) + layout.addWidget(adc_radar_settings_panel) + vna_radar_settings_panel.setVisible(not config.is_kamil_adc) + adc_radar_settings_panel.setVisible(config.is_kamil_adc) + return RadarView(group=group, fields=view.fields, + sweep_edits=(start_hz_input, stop_hz_input, points_input, ifbw_input, power_input), + sweep_labels=(radar_start_label, radar_stop_label, radar_points_label, radar_ifbw_label, radar_power_label), + limits_hint=radar_limits_hint, vna_panel=vna_radar_settings_panel, adc_panel=adc_radar_settings_panel, + optical_enabled=optical_enabled, optical_mode=optical_mode, optical_common=optical_common, + optical_manual=optical_manual, optical_variation=optical_variation, + manual_panel=manual_panel, variation_panel=variation_panel) -def _build_adc_settings_panel(owner, parent: QWidget) -> QWidget: +def _build_adc_settings_panel(view: SectionView, config: RunConfigModel): """Create controls for the external ADC collector and optical board.""" - panel = QWidget(parent) + panel = QWidget(view.group) layout = QVBoxLayout(panel) layout.setContentsMargins(0, 0, 0, 0) layout.setSpacing(8) - adc = owner._defaults_config.radar.kamil_adc - optical = owner._defaults_config.radar.laser_control + adc = config.radar.kamil_adc + optical = config.radar.laser_control manual = optical.manual variation = optical.variation - owner._adc_project_dir_input = QLineEdit(adc.project_dir) - owner._adc_executable_path_input = QLineEdit(adc.executable_path) - owner._adc_tty_path_input = QLineEdit(adc.tty_path) - owner._adc_args_input = _plain_text("\n".join(adc.args)) - owner._adc_env_input = _plain_text(json.dumps(adc.env, indent=2, sort_keys=True) if adc.env else "{}") - owner._adc_startup_timeout_s_input = QLineEdit(f"{adc.startup_timeout_s:g}") - owner._adc_sweep_timeout_s_input = QLineEdit(f"{adc.sweep_timeout_s:g}") - owner._adc_stop_timeout_s_input = QLineEdit(f"{adc.stop_timeout_s:g}") + adc_project_dir_input = view.add("_adc_project_dir_input", QLineEdit(adc.project_dir)) + adc_executable_path_input = view.add("_adc_executable_path_input", QLineEdit(adc.executable_path)) + adc_tty_path_input = view.add("_adc_tty_path_input", QLineEdit(adc.tty_path)) + adc_args_input = view.add("_adc_args_input", _plain_text("\n".join(adc.args))) + adc_env_input = view.add("_adc_env_input", _plain_text(json.dumps(adc.env, indent=2, sort_keys=True) if adc.env else "{}")) + adc_startup_timeout_s_input = view.add("_adc_startup_timeout_s_input", QLineEdit(f"{adc.startup_timeout_s:g}")) + adc_sweep_timeout_s_input = view.add("_adc_sweep_timeout_s_input", QLineEdit(f"{adc.sweep_timeout_s:g}")) + adc_stop_timeout_s_input = view.add("_adc_stop_timeout_s_input", QLineEdit(f"{adc.stop_timeout_s:g}")) - owner._optical_enabled_checkbox = QCheckBox() - owner._optical_enabled_checkbox.setChecked(bool(optical.enabled)) - owner._optical_port_input = QLineEdit(optical.port) - owner._optical_mode_combo = QComboBox() - owner._optical_mode_combo.addItems(["manual", "variation"]) - owner._set_combo_current_text(owner._optical_mode_combo, optical.mode) - owner._optical_pi_coeff1_p_input = QLineEdit(str(optical.pi_coeff1_p)) - owner._optical_pi_coeff1_i_input = QLineEdit(str(optical.pi_coeff1_i)) - owner._optical_pi_coeff2_p_input = QLineEdit(str(optical.pi_coeff2_p)) - owner._optical_pi_coeff2_i_input = QLineEdit(str(optical.pi_coeff2_i)) + optical_enabled_checkbox = view.add("_optical_enabled_checkbox", QCheckBox()) + optical_enabled_checkbox.setChecked(bool(optical.enabled)) + optical_port_input = view.add("_optical_port_input", QLineEdit(optical.port)) + optical_mode_combo = view.add("_optical_mode_combo", QComboBox()) + optical_mode_combo.addItems(["manual", "variation"]) + write_setting(optical_mode_combo, optical.mode) + optical_pi_coeff1_p_input = view.add("_optical_pi_coeff1_p_input", QLineEdit(str(optical.pi_coeff1_p))) + optical_pi_coeff1_i_input = view.add("_optical_pi_coeff1_i_input", QLineEdit(str(optical.pi_coeff1_i))) + optical_pi_coeff2_p_input = view.add("_optical_pi_coeff2_p_input", QLineEdit(str(optical.pi_coeff2_p))) + optical_pi_coeff2_i_input = view.add("_optical_pi_coeff2_i_input", QLineEdit(str(optical.pi_coeff2_i))) - owner._optical_manual_temp1_input = QLineEdit(f"{manual.temp1:g}") - owner._optical_manual_temp2_input = QLineEdit(f"{manual.temp2:g}") - owner._optical_manual_current1_input = QLineEdit(f"{manual.current1:g}") - owner._optical_manual_current2_input = QLineEdit(f"{manual.current2:g}") + optical_manual_temp1_input = view.add("_optical_manual_temp1_input", QLineEdit(f"{manual.temp1:g}")) + optical_manual_temp2_input = view.add("_optical_manual_temp2_input", QLineEdit(f"{manual.temp2:g}")) + optical_manual_current1_input = view.add("_optical_manual_current1_input", QLineEdit(f"{manual.current1:g}")) + optical_manual_current2_input = view.add("_optical_manual_current2_input", QLineEdit(f"{manual.current2:g}")) - owner._optical_variation_type_combo = QComboBox() - owner._optical_variation_type_combo.addItems( + optical_variation_type_combo = view.add("_optical_variation_type_combo", QComboBox()) + optical_variation_type_combo.addItems( [ "CHANGE_CURRENT_LD1", "CHANGE_CURRENT_LD2", @@ -120,118 +124,80 @@ def _build_adc_settings_panel(owner, parent: QWidget) -> QWidget: "CHANGE_TEMPERATURE_LD2", ] ) - owner._set_combo_current_text(owner._optical_variation_type_combo, variation.variation_type) - owner._optical_static_temp1_input = QLineEdit(f"{variation.static_temp1:g}") - owner._optical_static_temp2_input = QLineEdit(f"{variation.static_temp2:g}") - owner._optical_static_current1_input = QLineEdit(f"{variation.static_current1:g}") - owner._optical_static_current2_input = QLineEdit(f"{variation.static_current2:g}") - owner._optical_min_value_input = QLineEdit(f"{variation.min_value:g}") - owner._optical_max_value_input = QLineEdit(f"{variation.max_value:g}") - owner._optical_step_input = QLineEdit(f"{variation.step:g}") - owner._optical_time_step_input = QLineEdit(str(variation.time_step)) - owner._optical_delay_time_input = QLineEdit(str(variation.delay_time)) + write_setting(optical_variation_type_combo, variation.variation_type) + optical_static_temp1_input = view.add("_optical_static_temp1_input", QLineEdit(f"{variation.static_temp1:g}")) + optical_static_temp2_input = view.add("_optical_static_temp2_input", QLineEdit(f"{variation.static_temp2:g}")) + optical_static_current1_input = view.add("_optical_static_current1_input", QLineEdit(f"{variation.static_current1:g}")) + optical_static_current2_input = view.add("_optical_static_current2_input", QLineEdit(f"{variation.static_current2:g}")) + optical_min_value_input = view.add("_optical_min_value_input", QLineEdit(f"{variation.min_value:g}")) + optical_max_value_input = view.add("_optical_max_value_input", QLineEdit(f"{variation.max_value:g}")) + optical_step_input = view.add("_optical_step_input", QLineEdit(f"{variation.step:g}")) + optical_time_step_input = view.add("_optical_time_step_input", QLineEdit(str(variation.time_step))) + optical_delay_time_input = view.add("_optical_delay_time_input", QLineEdit(str(variation.delay_time))) layout.addWidget( build_two_column_form_widget( panel, [ - ("Project dir", owner._adc_project_dir_input), - ("Executable path", owner._adc_executable_path_input), - ("TTY path", owner._adc_tty_path_input), - ("Arguments", owner._adc_args_input), - ("Environment JSON", owner._adc_env_input), - ("Startup timeout s", owner._adc_startup_timeout_s_input), - ("Read timeout s", owner._adc_sweep_timeout_s_input), - ("Stop timeout s", owner._adc_stop_timeout_s_input), - ("Board enabled", owner._optical_enabled_checkbox), - ("Board port", owner._optical_port_input), - ("Board mode", owner._optical_mode_combo), - ("PI 1 P", owner._optical_pi_coeff1_p_input), - ("PI 1 I", owner._optical_pi_coeff1_i_input), - ("PI 2 P", owner._optical_pi_coeff2_p_input), - ("PI 2 I", owner._optical_pi_coeff2_i_input), + ("Project dir", adc_project_dir_input), + ("Executable path", adc_executable_path_input), + ("TTY path", adc_tty_path_input), + ("Arguments", adc_args_input), + ("Environment JSON", adc_env_input), + ("Startup timeout s", adc_startup_timeout_s_input), + ("Read timeout s", adc_sweep_timeout_s_input), + ("Stop timeout s", adc_stop_timeout_s_input), + ("Board enabled", optical_enabled_checkbox), + ("Board port", optical_port_input), + ("Board mode", optical_mode_combo), + ("PI 1 P", optical_pi_coeff1_p_input), + ("PI 1 I", optical_pi_coeff1_i_input), + ("PI 2 P", optical_pi_coeff2_p_input), + ("PI 2 I", optical_pi_coeff2_i_input), ], split_index=8, ) ) - owner._optical_manual_panel = build_two_column_form_widget( + optical_manual_panel = build_two_column_form_widget( panel, [ - ("Temperature 1", owner._optical_manual_temp1_input), - ("Temperature 2", owner._optical_manual_temp2_input), - ("Current 1", owner._optical_manual_current1_input), - ("Current 2", owner._optical_manual_current2_input), + ("Temperature 1", optical_manual_temp1_input), + ("Temperature 2", optical_manual_temp2_input), + ("Current 1", optical_manual_current1_input), + ("Current 2", optical_manual_current2_input), ], ) - owner._optical_variation_panel = build_two_column_form_widget( + optical_variation_panel = build_two_column_form_widget( panel, [ - ("Variation type", owner._optical_variation_type_combo), - ("Static temp 1", owner._optical_static_temp1_input), - ("Static temp 2", owner._optical_static_temp2_input), - ("Static current 1", owner._optical_static_current1_input), - ("Static current 2", owner._optical_static_current2_input), - ("Min value", owner._optical_min_value_input), - ("Max value", owner._optical_max_value_input), - ("Step", owner._optical_step_input), - ("Time step", owner._optical_time_step_input), - ("Delay time", owner._optical_delay_time_input), + ("Variation type", optical_variation_type_combo), + ("Static temp 1", optical_static_temp1_input), + ("Static temp 2", optical_static_temp2_input), + ("Static current 1", optical_static_current1_input), + ("Static current 2", optical_static_current2_input), + ("Min value", optical_min_value_input), + ("Max value", optical_max_value_input), + ("Step", optical_step_input), + ("Time step", optical_time_step_input), + ("Delay time", optical_delay_time_input), ], split_index=5, ) - layout.addWidget(owner._optical_manual_panel) - layout.addWidget(owner._optical_variation_panel) + layout.addWidget(optical_manual_panel) + layout.addWidget(optical_variation_panel) - _connect_adc_settings(owner) - owner._sync_adc_settings_controls() - return panel + return (panel, optical_enabled_checkbox, optical_mode_combo, + (optical_port_input, optical_mode_combo, optical_pi_coeff1_p_input, optical_pi_coeff1_i_input, + optical_pi_coeff2_p_input, optical_pi_coeff2_i_input), + (optical_manual_temp1_input, optical_manual_temp2_input, optical_manual_current1_input, optical_manual_current2_input), + (optical_variation_type_combo, optical_static_temp1_input, optical_static_temp2_input, + optical_static_current1_input, optical_static_current2_input, optical_min_value_input, + optical_max_value_input, optical_step_input, optical_time_step_input, optical_delay_time_input), + optical_manual_panel, optical_variation_panel) def _plain_text(text: str) -> QPlainTextEdit: widget = QPlainTextEdit(text) widget.setMaximumHeight(76) return widget - - -def _connect_adc_settings(owner) -> None: - text_widgets = [ - owner._adc_project_dir_input, - owner._adc_executable_path_input, - owner._adc_tty_path_input, - owner._adc_startup_timeout_s_input, - owner._adc_sweep_timeout_s_input, - owner._adc_stop_timeout_s_input, - owner._optical_port_input, - owner._optical_pi_coeff1_p_input, - owner._optical_pi_coeff1_i_input, - owner._optical_pi_coeff2_p_input, - owner._optical_pi_coeff2_i_input, - owner._optical_manual_temp1_input, - owner._optical_manual_temp2_input, - owner._optical_manual_current1_input, - owner._optical_manual_current2_input, - owner._optical_static_temp1_input, - owner._optical_static_temp2_input, - owner._optical_static_current1_input, - owner._optical_static_current2_input, - owner._optical_min_value_input, - owner._optical_max_value_input, - owner._optical_step_input, - owner._optical_time_step_input, - owner._optical_delay_time_input, - ] - for widget in text_widgets: - widget.editingFinished.connect(owner._reset_preprocess_selection_after_radar_key_change) - owner._adc_args_input.textChanged.connect(owner._reset_preprocess_selection_after_radar_key_change) - owner._adc_env_input.textChanged.connect(owner._reset_preprocess_selection_after_radar_key_change) - owner._optical_variation_type_combo.currentTextChanged.connect( - owner._reset_preprocess_selection_after_radar_key_change - ) - - def sync_and_reset() -> None: - owner._sync_adc_settings_controls() - owner._reset_preprocess_selection_after_radar_key_change() - - owner._optical_enabled_checkbox.toggled.connect(sync_and_reset) - owner._optical_mode_combo.currentTextChanged.connect(sync_and_reset) diff --git a/python_app/gui/controllers/sections/switch_section.py b/python_app/gui/controllers/sections/switch_section.py index 5accb05..d15a8b3 100644 --- a/python_app/gui/controllers/sections/switch_section.py +++ b/python_app/gui/controllers/sections/switch_section.py @@ -1,9 +1,8 @@ """Builder for switch timing and combo-selection settings section.""" -from __future__ import annotations - from PyQt6.QtWidgets import ( QButtonGroup, + QComboBox, QFormLayout, QGroupBox, QHBoxLayout, @@ -13,64 +12,71 @@ from PyQt6.QtWidgets import ( QVBoxLayout, ) -def build_switch_group(owner) -> QGroupBox: +from python_app.gui.controllers.sections.views import SwitchView, SectionView +from python_app.models.gui_profile_schema import GuiStateModel +from python_app.models.run_config_schema import RunConfigModel + + +def build_switch_group(config: RunConfigModel, gui: GuiStateModel) -> SwitchView: """Create switch timing field and two explicit combo-selection modes.""" - group = QGroupBox("Switches") + view = SectionView(QGroupBox("Switches")) + group = view.group layout = QVBoxLayout(group) layout.setSpacing(8) - switch_defaults = owner._gui_defaults.switches + switch_defaults = gui.switches global_form = QFormLayout() global_form.setFieldGrowthPolicy(QFormLayout.FieldGrowthPolicy.AllNonFixedFieldsGrow) - owner._settling_ms = QLineEdit(str(owner._defaults_config.runtime.settling_ms)) - global_form.addRow("Settling ms", owner._settling_ms) + settling_ms = view.add("_settling_ms", QLineEdit(str(config.runtime.settling_ms))) + global_form.addRow("Settling ms", settling_ms) layout.addLayout(global_form) - owner._combos_text = QLineEdit(str(switch_defaults.combos_text)) - owner._combos_text.setPlaceholderText("empty = full matrix, or input:output,input:output") - owner._run_combos_select_button = QPushButton("Select") - owner._run_combos_select_button.setCheckable(True) + combos_text = view.add("_combos_text", QLineEdit(str(switch_defaults.combos_text))) + combos_text.setPlaceholderText("empty = full matrix, or input:output,input:output") + run_combos_select_button = QPushButton("Select") + run_combos_select_button.setCheckable(True) combos_row = QHBoxLayout() combos_row.setSpacing(8) combos_row.addWidget(QLabel("Run combos")) - combos_row.addWidget(owner._combos_text, stretch=1) - combos_row.addWidget(owner._run_combos_select_button) + combos_row.addWidget(combos_text, stretch=1) + combos_row.addWidget(run_combos_select_button) layout.addLayout(combos_row) - owner._single_combo_output = QLineEdit(str(switch_defaults.single_output)) - owner._single_combo_output.setPlaceholderText("0") - owner._single_combo_input = QLineEdit(str(switch_defaults.single_input)) - owner._single_combo_input.setPlaceholderText("0") - owner._single_combo_select_button = QPushButton("Select") - owner._single_combo_select_button.setCheckable(True) - owner._next_combo_button = QPushButton("Next combination") - owner._next_combo_button.setToolTip("Advance through input/output pairs and redraw the selected B-scan.") + single_combo_output = view.add("_single_combo_output", QLineEdit(str(switch_defaults.single_output))) + single_combo_output.setPlaceholderText("0") + single_combo_input = view.add("_single_combo_input", QLineEdit(str(switch_defaults.single_input))) + single_combo_input.setPlaceholderText("0") + single_combo_select_button = QPushButton("Select") + single_combo_select_button.setCheckable(True) single_row = QHBoxLayout() single_row.setSpacing(8) single_row.addWidget(QLabel("Single combo")) single_row.addWidget(QLabel("Input")) - single_row.addWidget(owner._single_combo_input) + single_row.addWidget(single_combo_input) single_row.addWidget(QLabel("Output")) - single_row.addWidget(owner._single_combo_output) - single_row.addWidget(owner._single_combo_select_button) - single_row.addWidget(owner._next_combo_button) + single_row.addWidget(single_combo_output) + single_row.addWidget(single_combo_select_button) layout.addLayout(single_row) - owner._combo_select_group = QButtonGroup(group) - owner._combo_select_group.setExclusive(True) - owner._combo_select_group.addButton(owner._run_combos_select_button) - owner._combo_select_group.addButton(owner._single_combo_select_button) + combo_select_group = QButtonGroup(group) + combo_select_group.setExclusive(True) + combo_select_group.addButton(run_combos_select_button) + combo_select_group.addButton(single_combo_select_button) - owner._run_combos_select_button.clicked.connect(lambda: owner._set_combo_selection_mode("text")) - owner._single_combo_select_button.clicked.connect(lambda: owner._set_combo_selection_mode("single")) - owner._next_combo_button.clicked.connect(owner._select_next_combo) - # The live processing config has no combo: picking another one only changes which - # B-scan is shown (and, outside matrix mode, the combo of the next run). - owner._single_combo_output.editingFinished.connect(owner._on_bscan_display_combo_changed) - owner._single_combo_input.editingFinished.connect(owner._on_bscan_display_combo_changed) - owner._set_combo_selection_mode(str(switch_defaults.combo_mode)) + display_row = QHBoxLayout() + display_row.addWidget(QLabel("B-scan pair")) + display_combo = view.add("_display_combo", QComboBox()) + display_combo.setEnabled(False) + display_row.addWidget(display_combo, stretch=1) + next_combo_button = QPushButton("Next combination") + next_combo_button.setEnabled(False) + display_row.addWidget(next_combo_button) + layout.addLayout(display_row) - return group + return SwitchView(group=group, fields=view.fields, select_text=run_combos_select_button, + select_single=single_combo_select_button, combos=combos_text, + single_input=single_combo_input, single_output=single_combo_output, + display_combo=display_combo, next_combo=next_combo_button) diff --git a/python_app/gui/controllers/sections/views.py b/python_app/gui/controllers/sections/views.py new file mode 100644 index 0000000..1e8f5ef --- /dev/null +++ b/python_app/gui/controllers/sections/views.py @@ -0,0 +1,165 @@ +"""Typed section widgets and the scalar Qt boundary shared by GUI settings.""" + +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import TypeAlias, TypeVar + +from PyQt6.QtWidgets import ( + QCheckBox, QComboBox, QDoubleSpinBox, QGroupBox, QLineEdit, QPlainTextEdit, + QSpinBox, QWidget, QLabel, QPushButton, QStackedWidget, +) + +from python_app.gui.antenna_geometry_editor import AntennaGeometryEditor +from python_app.models.setting_fields import Scalar + +ScalarWidget: TypeAlias = QLineEdit | QPlainTextEdit | QComboBox | QCheckBox | QSpinBox | QDoubleSpinBox +SettingWidget: TypeAlias = ScalarWidget | AntennaGeometryEditor +_Widget = TypeVar("_Widget", bound=SettingWidget) + + +@dataclass(slots=True) +class SectionView: + group: QGroupBox + fields: dict[str, SettingWidget] = field(default_factory=dict) + + def add(self, name: str, widget: _Widget) -> _Widget: + self.fields[name] = widget + return widget + + +@dataclass(slots=True, kw_only=True) +class PreprocessView(SectionView): + labels: dict[str, QLabel] + + +@dataclass(slots=True, kw_only=True) +class PrimaryView(SectionView): + start: QPushButton + single_capture: QPushButton + stop: QPushButton + apply_radar: QPushButton + load: QPushButton + save: QPushButton + preprocess: QPushButton + clear: QPushButton + + +@dataclass(slots=True, kw_only=True) +class DataView(SectionView): + save: QPushButton + save_json: QPushButton + remove_last: QPushButton + capture_tmp: QPushButton + record: QPushButton + browse: QPushButton + save_path: QLineEdit + save_name: QLineEdit + record_count: QSpinBox + + +@dataclass(slots=True, kw_only=True) +class SwitchView(SectionView): + select_text: QPushButton + select_single: QPushButton + combos: QLineEdit + single_input: QLineEdit + single_output: QLineEdit + display_combo: QComboBox + next_combo: QPushButton + + +@dataclass(slots=True, kw_only=True) +class ProcessingView(SectionView): + mode: QComboBox + pages: QStackedWidget + geometry_page: QWidget + geometries: tuple[AntennaGeometryEditor, AntennaGeometryEditor] + fixed_y: QCheckBox + y_bounds: tuple[QDoubleSpinBox, QDoubleSpinBox] + speed_controls: tuple[tuple[QCheckBox, QDoubleSpinBox], ...] + trace_controls: tuple[ScalarWidget, ...] + visual_controls: tuple[ScalarWidget, ...] + locator_controls: tuple[ScalarWidget, ...] + frequency_controls: tuple[QDoubleSpinBox, ...] + + +@dataclass(slots=True, kw_only=True) +class RadarView(SectionView): + sweep_edits: tuple[QLineEdit, ...] + sweep_labels: tuple[QLabel, ...] + limits_hint: QLabel + vna_panel: QWidget + adc_panel: QWidget + optical_enabled: QCheckBox + optical_mode: QComboBox + optical_common: tuple[ScalarWidget, ...] + optical_manual: tuple[ScalarWidget, ...] + optical_variation: tuple[ScalarWidget, ...] + manual_panel: QWidget + variation_panel: QWidget + + +def read_setting(widget: ScalarWidget) -> Scalar: + if isinstance(widget, QCheckBox): + return widget.isChecked() + if isinstance(widget, (QSpinBox, QDoubleSpinBox)): + return widget.value() + if isinstance(widget, QComboBox): + return widget.currentText() + if isinstance(widget, QPlainTextEdit): + return widget.toPlainText() + return widget.text() + + +def write_setting(widget: ScalarWidget, value: Scalar) -> None: + if isinstance(widget, QCheckBox): + widget.setChecked(bool(value)) + elif isinstance(widget, QSpinBox): + widget.setValue(int(value)) + elif isinstance(widget, QDoubleSpinBox): + widget.setValue(float(value)) + elif isinstance(widget, QComboBox): + text = str(value) + index = widget.findText(text) + if index < 0: + widget.addItem(text) + index = widget.count() - 1 + widget.setCurrentIndex(index) + elif isinstance(widget, QPlainTextEdit): + widget.setPlainText(str(value)) + else: + widget.setText(str(value)) + + +def connect_setting(widget: ScalarWidget, callback: Callable[..., object]) -> None: + if isinstance(widget, QCheckBox): + signal = widget.toggled + elif isinstance(widget, (QSpinBox, QDoubleSpinBox)): + signal = widget.valueChanged + elif isinstance(widget, QComboBox): + signal = widget.currentTextChanged + elif isinstance(widget, QPlainTextEdit): + signal = widget.textChanged + else: + signal = widget.editingFinished + signal.connect(callback) + + +def double_spin(minimum: float, maximum: float, decimals: int, step: float) -> QDoubleSpinBox: + widget = QDoubleSpinBox() + widget.setDecimals(decimals) + widget.setRange(minimum, maximum) + widget.setSingleStep(step) + return widget + + +def integer_spin(minimum: int, maximum: int) -> QSpinBox: + widget = QSpinBox() + widget.setRange(minimum, maximum) + return widget + + +def choice(options: list[str]) -> QComboBox: + widget = QComboBox() + widget.addItems(options) + return widget diff --git a/python_app/gui/controllers/snapshot_controller.py b/python_app/gui/controllers/snapshot_controller.py new file mode 100644 index 0000000..f9ca1c8 --- /dev/null +++ b/python_app/gui/controllers/snapshot_controller.py @@ -0,0 +1,119 @@ +"""Manual dataset export and paused deletion of retained acquisitions.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +from python_app.models.gui_profile_schema import ProfileWriter +from python_app.gui.controllers.pipeline_controller import PipelineController +from python_app.gui.controllers.sections.views import PrimaryView, DataView +from python_app.models.run_config_schema import RunConfigModel +from python_app.storage.npz.store import NpzStore + + +class SnapshotController: + """Save stage histories and hold deletion ownership through final redraw.""" + + def __init__( + self, pipeline: PipelineController, store: NpzStore, write_profile: ProfileWriter, + *, destination: Callable[[RunConfigModel], tuple[Path, str, str]], + build_config: Callable[[], RunConfigModel], processing_settings: dict[int, dict], + allowed: Callable[[bool], bool], log: Callable[[str], None], + error: Callable[[str, Exception], None], + ) -> None: + self.pipeline, self.store = pipeline, store + self._write_profile, self._log, self._error = write_profile, log, error + self._destination, self._build_config = destination, build_config + self._settings, self._allowed = processing_settings, allowed + + def bind_actions(self, primary: PrimaryView, data: DataView) -> None: + primary.clear.clicked.connect(lambda: self.delete_history(remove_last=False)) + data.remove_last.clicked.connect(lambda: self.delete_history(remove_last=True)) + data.save.clicked.connect(lambda: self.save()) + data.save_json.clicked.connect(lambda: self.save(json_export=True)) + data.record.clicked.connect(lambda: self.start_recording(data.record_count.value())) + + def start_recording(self, count: int) -> None: + if not self._allowed(False): + return + try: + config = self.pipeline.active_config if self.pipeline.running else None + self.pipeline.recording.start( + *self._destination(config or self._build_config()), count, + running=self.pipeline.running, start_acquisition=self.pipeline.start, + ) + except Exception as exc: + self._error("Failed to start disk recording", exc) + + def save(self, *, json_export: bool = False) -> None: + if not self._allowed(False): + return + action = "VNA history JSON" if json_export else "snapshot" + try: + active = self.pipeline.active_config + root, name, prefix = self._destination(active or self._build_config()) + self.pipeline.drain_snapshot() + if self.pipeline.closing or self.pipeline.active_config is not active: + return + raw = list(self.pipeline.history.raw) + preprocessed = list(self.pipeline.history.preprocessed) + results = list(self.pipeline.history.results) + if not (raw or preprocessed or results): + raise ValueError("No runtime data is available for save") + if json_export: + paths = self.store.save_runtime_vna_history_json_batch( + root, name, raw, preprocessed, results, + primary_stage="preprocessed", name_prefix=prefix, + ) + directory = paths[0].parent + profile_path = directory / "vna_config_profile.json" + else: + directory = self.store.save_runtime_snapshot_numpy( + root, name, raw, preprocessed, results, + name_prefix=prefix, processing_settings=self._settings, + ) + profile_path = directory / "config_profile.json" + try: + self._write_profile(profile_path, allow_overwrite=json_export, dataset=True) + except Exception as exc: + self._error( + f"{action} data was saved in {directory}, but the adjacent config profile " + f"could not be written: {profile_path}", exc, + ) + return + self._log( + f"Saved {action}: {directory} " + f"(raw={len(raw)}, preprocessed={len(preprocessed)}, results={len(results)})" + ) + except Exception as exc: + self._error(f"Failed to save {action}", exc) + + def delete_history(self, *, remove_last: bool) -> None: + if not self._allowed(True): + return + action = "remove last runtime measurement" if remove_last else "clear runtime history" + try: + with self.pipeline.pause() as operation: + if not operation.valid: + return + self.pipeline.discard_replay() + dropped_raw, dropped_pre, dropped_results = self.pipeline.drop_pending() + self.pipeline.history.delete(remove_last=remove_last) + try: + self.pipeline.publish_live() + finally: + self.pipeline.plots.clear() + self.pipeline.settle_deleted_results() + if not operation.valid: + return + self.pipeline.history_changed() + self.pipeline.plots.redraw() + self._log( + f"{action}: dropped raw={dropped_raw}, " + f"preprocessed={dropped_pre}, results={dropped_results}" + ) + except InterruptedError: + self.pipeline.operation_cancelled() + except Exception as exc: + self._error(f"Failed to {action}", exc) diff --git a/python_app/gui/controllers/web_controller.py b/python_app/gui/controllers/web_controller.py new file mode 100644 index 0000000..ef4e512 --- /dev/null +++ b/python_app/gui/controllers/web_controller.py @@ -0,0 +1,334 @@ +"""Serve GUI snapshots and marshal browser controls onto the Qt thread.""" + +from __future__ import annotations + +import base64 +from collections.abc import Callable +from contextlib import suppress +import logging +import os +from pathlib import Path +import threading +import time +from typing import TYPE_CHECKING + +import pyqtgraph as pg +from PyQt6.QtCore import QBuffer, QIODevice, QObject, pyqtSignal +from PyQt6.QtWidgets import QLineEdit, QSpinBox, QWidget + +from python_app.gui.settings_defaults import env_enabled +from python_app.gui.controllers.config_controller import web_apply_field_names +from python_app.webui.controller import WebActionError + +if TYPE_CHECKING: + from python_app.gui.controllers.live_settings_controller import LiveSettingsController + from python_app.gui.controllers.log_controller import LogController + from python_app.gui.controllers.pipeline_controller import PipelineController + from python_app.gui.controllers.preprocess_controller import PreprocessController + from python_app.gui.controllers.profile_controller import ProfileController + from python_app.gui.controllers.snapshot_controller import SnapshotController + from python_app.webui.server import WebUiServer + +logger = logging.getLogger(__name__) +_WEB_ACTION_TIMEOUT_S = 30.0 +_GRAB_INTERVAL_S = 0.2 +_LIVE_FIELD_NAMES = web_apply_field_names() + + +class _WebActionCall: + """A timeout can cancel queued work; running work must never be retried blindly.""" + + def __init__(self, action: Callable[[], object]) -> None: + self.action = action + self.done = threading.Event() + self.error: str | None = None + self._lock = threading.Lock() + self._started = False + + def begin(self) -> bool: + with self._lock: + if self.done.is_set() or self._started: + return False + self._started = True + return True + + def finish(self, error: str | None = None) -> None: + with self._lock: + if not self.done.is_set(): + self.error = error + self.done.set() + + def cancel(self, error: str, *, queued_only: bool = True) -> bool: + with self._lock: + if self.done.is_set(): + return True + if queued_only and self._started: + return False + self.error = error + self.done.set() + return True + + +def web_ui_port() -> int: + try: + port = int(os.environ.get("RADAR_SYSTEM_WEBUI_PORT", "8080")) + return port if 1 <= port <= 65535 else 8080 + except ValueError: + return 8080 + + +class WebBridge(QObject): + """Own web lifecycle, queued actions, immutable snapshots and change-gated PNGs.""" + + _action_requested = pyqtSignal(object) + _streaming_changed = pyqtSignal(bool) + + def __init__( + self, parent: QWidget, run_configs_dir: Path, pipeline: PipelineController, + preprocess: PreprocessController, snapshots: SnapshotController, profile: ProfileController, + live: LiveSettingsController, log: LogController, *, + save_path: QLineEdit, save_name: QLineEdit, record_count: QSpinBox, + ) -> None: + super().__init__(parent) + self._parent, self._run_configs_dir = parent, run_configs_dir + self._pipeline, self._preprocess, self._snapshots = pipeline, preprocess, snapshots + self._profile, self._live, self._log = profile, live, log + self._save_path, self._save_name, self._record_count = save_path, save_name, record_count + self._server: WebUiServer | None = None + self._status: dict = {} + self._live_settings: list[dict] = [] + self._frame: dict | None = None + self._pending_lock = threading.Lock() + self._pending_calls: set[_WebActionCall] = set() + self._closed = self._streaming = self._grabbing = False + self._dirty = True + self._last_png: str | None = None + self._last_grab_s = 0.0 + self._frame_seq = self._snapshot_errors = 0 + self._action_requested.connect(self._run_action) + self._streaming_changed.connect(self._set_streaming) + plots = pipeline.plots + plots.stack.currentChanged.connect(self._mark_dirty) + for plot in plots.stack.findChildren(pg.PlotWidget): + scene = plot.scene() + assert scene is not None + scene.changed.connect(self._mark_dirty) + plot.sigDeviceRangeChanged.connect(self._mark_dirty) + + def start_server(self) -> None: + if self._closed or self._server is not None: + return + try: + from python_app.webui.server import WebUiServer + + if env_enabled("RADAR_SYSTEM_HEADLESS"): + self._parent.resize(1600, 900) + self._run_configs_dir.mkdir(parents=True, exist_ok=True) + self.refresh_snapshot() + self._server = WebUiServer(self, port=web_ui_port()) + self._server.start() + except Exception: + logger.warning("Failed to start web UI", exc_info=True) + if self._server is not None: + with suppress(Exception): + self._server.stop() + self._server = None + + def shutdown(self) -> None: + with self._pending_lock: + self._closed, self._streaming = True, False + pending = tuple(self._pending_calls) + for call in pending: + call.cancel("The application is closing", queued_only=False) + if self._server is not None: + with suppress(Exception): + self._server.stop() + self._server = None + + def _dispatch(self, action: Callable[[], object]) -> None: + call = _WebActionCall(action) + with self._pending_lock: + if self._closed: + raise WebActionError("The application is closing") + self._pending_calls.add(call) + try: + self._action_requested.emit(call) + if not call.done.wait(_WEB_ACTION_TIMEOUT_S): + if not call.cancel("The desktop did not start the action in time; the request was cancelled"): + raise WebActionError("The desktop action is still running; check its status before retrying") + if call.error is not None: + raise WebActionError(call.error) + finally: + with self._pending_lock: + self._pending_calls.discard(call) + + def _run_action(self, call: _WebActionCall) -> None: + if self._closed or self._pipeline.closing: + call.cancel("The application is closing", queued_only=False) + return + if not call.begin(): + return + error = None + try: + with self._log.capture_errors() as captured: + call.action() + error = captured[0] if captured else None + self.refresh_snapshot() + except Exception as exc: + error = self._log.exception_summary(exc) + finally: + call.finish(error) + + # The web thread reads only replaced snapshots; all widget reads happen below. + def status(self) -> dict: + return dict(self._status) + + def current_live_settings(self) -> list[dict]: + return list(self._live_settings) + + def peek_frame(self) -> dict | None: + return self._frame + + def list_configs(self) -> list[str]: + return sorted(path.name for path in self._run_configs_dir.glob("*.json") if path.is_file()) + + def start(self) -> None: + self._dispatch(self._pipeline.start) + + def stop(self) -> None: + self._dispatch(self._pipeline.stop) + + def single_capture(self) -> None: + self._dispatch(lambda: self._pipeline.start(single_capture=True)) + + def capture_tmp_reference(self) -> None: + self._dispatch(self._preprocess.capture_tmp_reference) + + def remove_last_measurement(self) -> None: + self._dispatch(lambda: self._snapshots.delete_history(remove_last=True)) + + def clear_history(self) -> None: + self._dispatch(lambda: self._snapshots.delete_history(remove_last=False)) + + def start_recording(self, path: str, name: str, count: int) -> None: + if type(count) is not int or count < 1: + raise ValueError("Recording count must be a positive integer") + self._dispatch(lambda: self._start_recording(path, name, count)) + + def _start_recording(self, path: str, name: str, count: int) -> None: + low, high = self._record_count.minimum(), self._record_count.maximum() + if not low <= count <= high: + raise ValueError(f"Recording count must be an integer from {low} to {high}") + self._set_destination(path, name) + self._record_count.setValue(count) + self._snapshots.start_recording(count) + + def _set_destination(self, path: str, name: str) -> None: + if path.strip(): + self._save_path.setText(path) + self._save_name.setText(name) + + def save_dataset(self, path: str, name: str) -> None: + self._dispatch(lambda: self._save_dataset(path, name)) + + def _save_dataset(self, path: str, name: str) -> None: + self._set_destination(path, name) + self._snapshots.save() + + def load_config(self, name: str) -> None: + def load() -> None: + path = self._run_configs_dir / name + if not name or name != Path(name).name or path.suffix.lower() != ".json" or not path.is_file(): + raise ValueError(f"Unknown run config: {name}") + self._profile.load(path) + self._dispatch(load) + + def apply_live_settings(self, fields: dict) -> list[dict]: + fields = dict(fields) + unknown = set(fields) - _LIVE_FIELD_NAMES + if unknown: + raise ValueError(f"Unknown live-settings fields: {', '.join(sorted(unknown))}") + self._dispatch(lambda: self._live.apply_web(fields)) + return self.current_live_settings() + + def select_combo(self, input_pos: int, output_pos: int) -> None: + if any(type(pos) is not int or pos < 0 for pos in (input_pos, output_pos)): + raise ValueError("Channel positions must be nonnegative integers") + self._dispatch(lambda: self._pipeline.plots.select_combo(input_pos, output_pos)) + + def next_combo(self) -> None: + self._dispatch(self._pipeline.plots.select_next_combo) + + def refresh_snapshot(self) -> None: + if self._closed or self._pipeline.closing: + return + try: + pipeline, plots = self._pipeline, self._pipeline.plots + input_pos, output_pos = plots.display_key or (0, 0) + mode = self._live.active_mode + self._status = { + "running": pipeline.running, "processor_running": pipeline.processor_running, + "ring_name": self._profile.state.config.rings.results.name, + "active_config": self._profile.state.path.name, + "save_path": self._save_path.text(), "save_name": self._save_name.text(), + "record_count": self._record_count.value(), "recording": pipeline.recording.status(), + "raw_count": len(pipeline.history.raw), "preprocessed_count": len(pipeline.history.preprocessed), + "result_count": len(pipeline.history.results), + "sar3d_status": plots.sar3d.status_label.text() if mode == "sar3d" else "", + "display_combo": {"enabled": mode == "bscan", "input": input_pos, "output": output_pos, + "options": sorted(plots.bscan.history_by_combo)}, + } + self._live_settings = self._live.config.web_schema() + frame = self._grab_frame_if_due() + if frame is not None: + self._frame = frame + except Exception: + self._snapshot_errors += 1 + if self._snapshot_errors % 200 == 1: + logger.warning("Web UI snapshot refresh failed", exc_info=True) + + def set_frame_streaming(self, enabled: bool) -> None: + if not self._closed: + self._streaming_changed.emit(enabled) + + def _set_streaming(self, enabled: bool) -> None: + if not self._closed: + self._streaming = enabled + if enabled: + self._dirty = True + + def _mark_dirty(self, *_args: object) -> None: + if not self._closed and not self._grabbing: + self._dirty = True + + def _grab_frame_if_due(self) -> dict | None: + if self._closed or not self._streaming or not self._dirty: + return None + now = time.monotonic() + if now - self._last_grab_s < _GRAB_INTERVAL_S: + return None + self._last_grab_s, self._dirty, self._grabbing = now, False, True + try: + png = self._grab_png() + except Exception: + self._dirty = True + raise + finally: + self._grabbing = False + if png is None or png == self._last_png: + return None + self._last_png = png + self._frame_seq += 1 + return {"type": "frame", "seq": self._frame_seq, "mode": self._live.active_mode, "png_b64": png} + + def _grab_png(self) -> str | None: + widget = self._pipeline.plots.capture_widget + if widget is None or widget.width() <= 0 or widget.height() <= 0: + return None + pixmap = widget.grab() + if pixmap.isNull(): + return None + buffer = QBuffer() + buffer.open(QIODevice.OpenModeFlag.WriteOnly) + pixmap.save(buffer, "PNG") + return base64.b64encode(buffer.data().data()).decode() diff --git a/python_app/gui/main.py b/python_app/gui/main.py index 96dec5b..fffbc12 100644 --- a/python_app/gui/main.py +++ b/python_app/gui/main.py @@ -3,7 +3,6 @@ from __future__ import annotations from contextlib import suppress -import os from pathlib import Path import signal import sys @@ -17,20 +16,11 @@ PROJECT_ROOT = Path(__file__).resolve().parents[2] if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) +from python_app.gui.settings_defaults import env_enabled from python_app.gui.app_window import AppWindow from python_app.gui.theme import apply_light_theme -def _is_headless() -> bool: - """Return whether the launcher requested a non-interactive deployment.""" - return os.environ.get("RADAR_SYSTEM_HEADLESS", "").strip().lower() in { - "1", - "true", - "yes", - "on", - } - - def _install_unix_signal_handlers(app: QApplication, window: AppWindow) -> None: """Route SIGINT and SIGTERM through the Qt event loop into a clean shutdown. @@ -67,7 +57,7 @@ def main() -> int: # PyQtGraph foreground controls axis lines, tick text, labels, and titles. pg.setConfigOptions(antialias=True, background="#ffffff", foreground="#ffffff") window = AppWindow(PROJECT_ROOT) - if _is_headless(): + if env_enabled("RADAR_SYSTEM_HEADLESS"): _install_unix_signal_handlers(app, window) else: window.showMaximized() diff --git a/python_app/gui/object_calibration_dialog.py b/python_app/gui/object_calibration_dialog.py index aafb85d..d165cc1 100644 --- a/python_app/gui/object_calibration_dialog.py +++ b/python_app/gui/object_calibration_dialog.py @@ -7,11 +7,11 @@ import math import numpy as np import pyqtgraph as pg -from PyQt6.QtCore import Qt, pyqtSignal +from PyQt6.QtCore import QSignalBlocker, QRectF, Qt, pyqtSignal from PyQt6.QtWidgets import ( QCheckBox, QComboBox, QDialog, QDoubleSpinBox, QFormLayout, QGroupBox, QHBoxLayout, QHeaderView, QLabel, QLineEdit, QPushButton, QTableWidget, QTableWidgetItem, - QVBoxLayout, QWidget, + QToolButton, QVBoxLayout, QWidget, ) from python_app.models.dataset_model import SweepCollection @@ -30,6 +30,7 @@ class ObjectCalibrationDialog(QDialog): self.resize(1000, 850) self._captures: list[dict] = [] self._config: dict | None = None + self._provenance: dict | None = None self._capture_pending = self._calculating = False self._calibration: dict | None = None layout = QVBoxLayout(self) @@ -39,26 +40,26 @@ class ObjectCalibrationDialog(QDialog): name_row.addWidget(self._set_name_input) layout.addLayout(name_row) explanation = QLabel( - "Снимите данные, затем введите физически измеренные координаты центра отражателя: " - "X — поперёк, Y — высота над землёй, Z — вдоль земли. " - "Обычные S21-калибровка и выбранный референс уже применены; " - "существующая MIMO-поправка при съёмке отключена.", self, + "Capture data, then enter the measured reflector centre: " + "X across the array, Y above the ground, Z along the ground. " + "S21 through calibration and the selected reference are already applied; " + "object calibration is disabled during capture. Only antenna pairs with configured " + "Tx/Rx geometry are calibrated; other acquired pairs remain unchanged.", self, ) explanation.setWordWrap(True) layout.addWidget(explanation) self._table = QTableWidget(0, 5, self) - self._table.setHorizontalHeaderLabels(["Набор", "Свипов", "X (м)", "Y (м)", "Z (м)"]) + self._table.setHorizontalHeaderLabels(["Target", "Sweeps", "X (m)", "Y (m)", "Z (m)"]) horizontal, vertical = self._table.horizontalHeader(), self._table.verticalHeader() assert horizontal is not None and vertical is not None horizontal.setSectionResizeMode(QHeaderView.ResizeMode.Stretch) vertical.hide() self._table.setSelectionBehavior(QTableWidget.SelectionBehavior.SelectRows) - self._table.itemChanged.connect(self._measurements_changed) layout.addWidget(self._table) actions = QHBoxLayout() - self._capture_button = QPushButton("Снять данные", self) + self._capture_button = QPushButton("Capture data", self) self._capture_button.clicked.connect(self.capture_requested.emit) - self._remove_button = QPushButton("Удалить выбранный набор", self) + self._remove_button = QPushButton("Remove selected target", self) self._remove_button.clicked.connect(self._remove_capture) actions.addWidget(self._capture_button) actions.addWidget(self._remove_button) @@ -67,81 +68,109 @@ class ObjectCalibrationDialog(QDialog): self._settings: dict[str, QDoubleSpinBox | QCheckBox | QComboBox] = {} form = QFormLayout() for key, title, default, low, high in ( - ("sphere_radius_m", "Радиус шара, м (0 — точка)", .16, 0, 100), - ("antenna_height_m", "Высота начала координат антенн над землёй, м", .92, 0, 100), - ("pitch_deg", "Наклон радара вниз, °", 10, -90, 90), + ("sphere_radius_m", "Sphere radius, m (0 = point)", .16, 0, 100), + ("antenna_height_m", "Antenna origin height above ground, m", .92, 0, 100), + ("pitch_deg", "Radar downward tilt, °", 10, -90, 90), ): form.addRow(title, self._number(key, default, low, high)) layout.addLayout(form) - advanced = QGroupBox("Параметры расчёта", self) - advanced.setFlat(True) - advanced.setCheckable(True) - advanced.setChecked(False) - advanced_layout = QVBoxLayout(advanced) - advanced_content = QWidget(advanced) - advanced_form = QFormLayout(advanced_content) + # A disclosure header: only the header shows until the operator opens it. + advanced_toggle = QToolButton(self) + advanced_toggle.setText("Calculation settings") + advanced_toggle.setToolButtonStyle(Qt.ToolButtonStyle.ToolButtonTextBesideIcon) + advanced_toggle.setArrowType(Qt.ArrowType.RightArrow) + advanced_toggle.setAutoRaise(True) + advanced = QGroupBox(self) + advanced_form = QFormLayout(advanced) for key, title, default, low, high in ( - ("f_min", "Нижняя частота оценки, МГц", 2000, 0, 100000), - ("f_max", "Верхняя частота оценки, МГц", 6000, 0, 100000), - ("gate_half_m", "Полуширина окна эха по пути, м", .15, .000001, 100), - ("gate_taper_m", "Плавный край окна, м", .03, .000001, 100), - ("coarse_range_search_m", "Грубый поиск по пути, ±м (0 — выкл.)", .6, 0, 100), - ("refine_range_m", "Уточнение координат, ±м", .1, 0, 100), + ("f_min", "Estimation band start, MHz", 2000, 0, 100000), + ("f_max", "Estimation band stop, MHz", 6000, 0, 100000), + ("gate_half_m", "Echo path gate half-width, m", .15, .000001, 100), + ("gate_taper_m", "Gate taper width, m", .03, .000001, 100), + ("coarse_range_search_m", "Coarse path search, ±m (0 = off)", .6, 0, 100), + ("refine_range_m", "Coordinate refinement, ±m", .1, 0, 100), ): advanced_form.addRow(title, self._number(key, default, low, high)) - axes = QWidget(advanced_content) + axes = QWidget(advanced) axes_layout = QHBoxLayout(axes) axes_layout.setContentsMargins(0, 0, 0, 0) self._refine_axes = {} for axis in "xyz": checkbox = QCheckBox(axis.upper(), axes) checkbox.setChecked(True) - checkbox.toggled.connect(self._measurements_changed) self._refine_axes[axis] = checkbox axes_layout.addWidget(checkbox) - advanced_form.addRow("Уточнять координаты", axes) - phase_model = QComboBox(advanced_content) + axes_layout.addStretch(1) + advanced_form.addRow("Refine coordinates", axes) + phase_model = QComboBox(advanced) phase_model.addItems(["const", "linear", "poly3", "raw"]) phase_model.setCurrentText("linear") - phase_model.currentTextChanged.connect(self._measurements_changed) self._settings["phase_model"] = phase_model - advanced_form.addRow("Модель фазы антенны", phase_model) - amplitude = QCheckBox("Выравнивать также амплитуды", advanced_content) - amplitude.toggled.connect(self._measurements_changed) + advanced_form.addRow("Antenna phase model", phase_model) + amplitude = QCheckBox("Equalize amplitudes", advanced) self._settings["apply_amplitude"] = amplitude advanced_form.addRow(amplitude) - advanced_content.hide() - advanced.toggled.connect(advanced_content.setVisible) - advanced_layout.addWidget(advanced_content) + advanced.hide() + + def toggle_advanced() -> None: + shown = advanced.isHidden() + advanced_toggle.setArrowType(Qt.ArrowType.DownArrow if shown else Qt.ArrowType.RightArrow) + advanced.setVisible(shown) + + advanced_toggle.clicked.connect(toggle_advanced) + layout.addWidget(advanced_toggle) layout.addWidget(advanced) self._target_combo = QComboBox(self) - self._target_combo.currentIndexChanged.connect(self._draw_phase_effect) layout.addWidget(self._target_combo) - self._phase_plot = pg.PlotWidget(background="#0f141c", enableMenu=False) - self._phase_plot.setLabel("left", "Target phase relative to first pair", units="deg") - self._phase_plot.setLabel("bottom", "Tx / Rx pair") - self._phase_plot.showGrid(x=True, y=True, alpha=.2) - self._phase_plot.setMinimumHeight(220) - self._phase_plot.addLegend() - layout.addWidget(self._phase_plot) - self._quality_label = QLabel("Фазы вкладов пар появятся после расчёта.", self) + self._phase_view = pg.GraphicsLayoutWidget(parent=self) + self._phase_view.setBackground("#0f141c") + self._phase_view.setMinimumHeight(270) + self._phase_plots = [] + self._phase_images = [] + self._phase_text = [] + for column, title in enumerate(("Before", "After")): + plot = self._phase_view.ci.addPlot(row=0, col=column, title=title, enableMenu=False) + plot.setLabel("bottom", "Rx") + plot.setLabel("left", "Tx") + plot.getViewBox().setAspectLocked(True) + plot.getViewBox().invertY(True) + plot.getViewBox().setMouseEnabled(x=False, y=False) + image = pg.ImageItem(axisOrder="row-major") + plot.addItem(image) + self._phase_plots.append(plot) + self._phase_images.append(image) + self._phase_colorbar = pg.ColorBarItem( + values=(-180, 180), colorMap=pg.colormap.get("CET-C2"), + label="Phase (deg)", interactive=False, colorMapMenu=False, + ) + self._phase_colorbar.setImageItem(self._phase_images) + self._phase_colorbar.axis.setTicks([[(value, str(value)) for value in (-180, -90, 0, 90, 180)]]) + self._phase_view.ci.addItem(self._phase_colorbar, row=0, col=2) + layout.addWidget(self._phase_view) + self._quality_label = QLabel("Pair phases at the target will appear after calculation.", self) self._quality_label.setWordWrap(True) layout.addWidget(self._quality_label) - self._status_label = QLabel("Готово к съёмке", self) + self._status_label = QLabel("Ready to capture", self) self._status_label.setWordWrap(True) layout.addWidget(self._status_label) footer = QHBoxLayout() - self._calculate_button = QPushButton("Рассчитать и сохранить", self) + self._calculate_button = QPushButton("Calculate and save", self) self._calculate_button.clicked.connect(self.calculate_requested.emit) - self._cancel_button = QPushButton("Отменить операцию", self) + self._cancel_button = QPushButton("Cancel operation", self) self._cancel_button.clicked.connect(self.cancel_requested.emit) - close_button = QPushButton("Закрыть", self) + close_button = QPushButton("Close", self) close_button.clicked.connect(self.reject) footer.addWidget(self._calculate_button) footer.addWidget(self._cancel_button) footer.addStretch(1) footer.addWidget(close_button) layout.addLayout(footer) + self._table.itemChanged.connect(self._measurements_changed) + self._target_combo.currentIndexChanged.connect(self._draw_phase_effect) + for field in [*self._settings.values(), *self._refine_axes.values()]: + signal = (field.valueChanged if isinstance(field, QDoubleSpinBox) else + field.toggled if isinstance(field, QCheckBox) else field.currentTextChanged) + signal.connect(self._measurements_changed) self._update_actions() def _number(self, key: str, value: float, low: float, high: float) -> QDoubleSpinBox: @@ -150,35 +179,40 @@ class ObjectCalibrationDialog(QDialog): field.setRange(low, high) field.setSingleStep(100 if key in {"f_min", "f_max"} else .1 if key == "pitch_deg" else .01) field.setValue(value) - field.valueChanged.connect(self._measurements_changed) self._settings[key] = field return field - def add_capture(self, collections: list[SweepCollection], config: dict) -> None: + def add_capture(self, collections: list[SweepCollection], config: dict, provenance: dict) -> None: """Freeze a captured group before the operator enters its target coordinates.""" if not collections: - raise ValueError("Не получены свипы отражателя") - if self._config is not None and any( - config.get(key) != self._config.get(key) for key in ("radar", "gpr", "preprocess") + raise ValueError("No reflector sweeps were captured") + if self._config is not None and ( + any(config.get(key) != self._config.get(key) for key in ("radar", "gpr", "preprocess")) + or config.get("run", {}).get("combos") != self._config.get("run", {}).get("combos") + or provenance != self._provenance ): - raise ValueError("Параметры радара, геометрия или препроцессинг изменились между наборами") - self._config = copy.deepcopy(config) + raise ValueError("Radar settings, geometry, channels or preprocessing changed between captures") + if self._config is None: + self._config = copy.deepcopy(config) + self._provenance = copy.deepcopy(provenance) self._captures.append({"collections": copy.deepcopy(collections)}) row = self._table.rowCount() - self._table.insertRow(row) - for column, value in enumerate((str(row + 1), str(len(collections)), "", "", "")): - item = QTableWidgetItem(value) - if column < 2: - item.setFlags(item.flags() & ~Qt.ItemFlag.ItemIsEditable) - self._table.setItem(row, column, item) + with QSignalBlocker(self._table): + self._table.insertRow(row) + for column, value in enumerate((str(row + 1), str(len(collections)), "", "", "")): + item = QTableWidgetItem(value) + if column < 2: + item.setFlags(item.flags() & ~Qt.ItemFlag.ItemIsEditable) + self._table.setItem(row, column, item) self._table.setCurrentCell(row, 2) - self.set_status("Данные сняты. Введите координаты центра отражателя для нового набора.") + self._measurements_changed() + self.set_status("Data captured. Enter the reflector centre coordinates for the new target.") self._update_actions() - def calculation_inputs(self) -> tuple[list[dict], dict, dict]: + def calculation_inputs(self) -> tuple[list[dict], dict, dict, dict]: """Validate XYZ after capture; expose a frozen recipe to the worker.""" - if not self._captures or self._config is None: - raise ValueError("Сначала снимите данные отражателя") + if not self._captures or self._config is None or self._provenance is None: + raise ValueError("Capture reflector data first") groups = [] for row, capture in enumerate(self._captures): try: @@ -189,9 +223,9 @@ class ObjectCalibrationDialog(QDialog): coordinates.append(float(item.text().strip())) target = tuple(coordinates) except ValueError as exc: - raise ValueError(f"Введите X, Y и Z центра отражателя для набора {row + 1}") from exc + raise ValueError(f"Enter X, Y and Z of the reflector centre for target {row + 1}") from exc if not all(math.isfinite(value) for value in target): - raise ValueError(f"Координаты набора {row + 1} должны быть конечными числами") + raise ValueError(f"Target {row + 1} coordinates must be finite numbers") groups.append({**capture, "target_scene_m": target}) settings: dict[str, float | bool | str | tuple[str, ...] | tuple[int, ...]] = { key: field.value() * (1e6 if key in {"f_min", "f_max"} else 1) if isinstance(field, QDoubleSpinBox) else @@ -199,10 +233,10 @@ class ObjectCalibrationDialog(QDialog): for key, field in self._settings.items() } settings["refine_axes"] = tuple(axis for axis, field in self._refine_axes.items() if field.isChecked()) - traces = self._captures[0]["collections"][0].traces - settings["input_idx"] = tuple(sorted({trace.combo.input for trace in traces})) - settings["output_idx"] = tuple(sorted({trace.combo.output for trace in traces})) - return groups, copy.deepcopy(self._config), settings + geometry = self._config["gpr"] + settings["input_idx"] = tuple(sorted(entry["input_pos"] for entry in geometry["rx_geometry"])) + settings["output_idx"] = tuple(sorted(entry["output_pos"] for entry in geometry["tx_geometry"])) + return groups, copy.deepcopy(self._config), settings, copy.deepcopy(self._provenance) def set_name(self) -> str: return self._set_name_input.text().strip() @@ -218,60 +252,74 @@ class ObjectCalibrationDialog(QDialog): def set_status(self, message: str) -> None: self._status_label.setText(message) - def show_calibration(self, calibration: dict) -> None: - self._calibration = calibration - self._target_combo.clear() - for index, target in enumerate(calibration.get("meta", {}).get("targets", [])): - xyz = target.get("nominal_scene_m", []) - self._target_combo.addItem(f"Набор {index + 1}: {xyz}") + def show_calibration(self, metadata: dict) -> None: + """Display phase diagnostics stored with the calculated calibration set.""" + self._calibration = metadata + with QSignalBlocker(self._target_combo): + self._target_combo.clear() + for index, target in enumerate(metadata["targets"]): + xyz = target["nominal_scene_m"] + self._target_combo.addItem(f"Target {index + 1}: {xyz}") self._draw_phase_effect() + def _clear_phase_effect(self) -> None: + for image in self._phase_images: + image.clear() + for plot, text in self._phase_text: + plot.removeItem(text) + self._phase_text.clear() + for plot, title in zip(self._phase_plots, ("Before", "After")): + plot.setTitle(title) + def _draw_phase_effect(self, *_args) -> None: - self._phase_plot.clear() + """Render notebook cell 5: Tx-by-Rx phase maps relative to each matrix sum.""" + self._clear_phase_effect() if self._calibration is None: return - diagnostics = self._calibration.get("meta", {}).get("diagnostics", []) + diagnostics = self._calibration["diagnostics"] index = self._target_combo.currentIndex() if not 0 <= index < len(diagnostics): - self._quality_label.setText("У этого набора нет диагностики фаз в точке цели.") + self._quality_label.setText("This target has no phase diagnostics.") return diagnostic = diagnostics[index] - phases = np.array([diagnostic["phase_before_deg"], diagnostic["phase_after_deg"]]).reshape(2, -1) - before, after = (phases - phases[0, 0] + 180) % 360 - 180 - pairs = [(output, input_pos) for output in self._calibration["tx_output_pos"] - for input_pos in self._calibration["rx_input_pos"]] - xx = np.arange(len(pairs)) - for values, color, label in ((before, "#ed7d31", "До"), (after, "#42c7a5", "После")): - self._phase_plot.plot(xx, values, pen=pg.mkPen(color, width=1.5), symbol="o", - symbolBrush=color, symbolPen=color, name=label) - step = max(1, len(pairs) // 16) - self._phase_plot.getAxis("bottom").setTicks([[ - (k, f"o{pairs[k][0]}/i{pairs[k][1]}") for k in range(0, len(pairs), step) - ]]) - self._quality_label.setText( - f"Согласованность фаз в точке цели: {diagnostic['coherence_before']:.3f} " - f"→ {diagnostic['coherence_after']:.3f} (1 — все вклады в фазе)." - ) + tx, rx = self._calibration["tx_output_pos"], self._calibration["rx_input_pos"] + for key, plot, image in zip(("before", "after"), self._phase_plots, self._phase_images): + matrix = np.asarray(diagnostic[f"phase_{key}_deg"], dtype=float) + image.setImage(matrix, autoLevels=False, levels=(-180, 180)) + image.setRect(QRectF(-.5, -.5, len(rx), len(tx))) + plot.getAxis("bottom").setTicks([[(i, f"Rx{port}") for i, port in enumerate(rx)]]) + plot.getAxis("left").setTicks([[(i, f"Tx{port}") for i, port in enumerate(tx)]]) + plot.setRange(xRange=(-.5, len(rx) - .5), yRange=(-.5, len(tx) - .5), padding=0) + plot.setTitle(f"{key.title()}: pair phase at target (STD {diagnostic[f'phase_std_{key}_deg']:.0f}°)") + for (row, column), value in np.ndenumerate(matrix): + text = pg.TextItem(f"{value:+.0f}", color="w", anchor=(.5, .5), fill=pg.mkBrush(0, 0, 0, 90)) + text.setPos(column, row) + plot.addItem(text) + self._phase_text.append((plot, text)) + self._quality_label.setText("Each matrix shows phase relative to its own coherent sum, in degrees.") def _remove_capture(self) -> None: row = self._table.currentRow() if row >= 0: self._captures.pop(row) - self._table.removeRow(row) - for index in range(self._table.rowCount()): - item = self._table.item(index, 0) - assert item is not None - item.setText(str(index + 1)) + with QSignalBlocker(self._table): + self._table.removeRow(row) + for index in range(self._table.rowCount()): + item = self._table.item(index, 0) + assert item is not None + item.setText(str(index + 1)) if not self._captures: self._config = None + self._provenance = None self._measurements_changed() + self._update_actions() def _measurements_changed(self, *_args) -> None: self._calibration = None - if hasattr(self, "_phase_plot"): - self._phase_plot.clear() + with QSignalBlocker(self._target_combo): self._target_combo.clear() - self._quality_label.setText("Параметры изменены — требуется новый расчёт.") + self._clear_phase_effect() + self._quality_label.setText("Measurements or settings changed. Recalculate to update the phase maps.") def _update_actions(self) -> None: busy = self._capture_pending or self._calculating @@ -287,7 +335,3 @@ class ObjectCalibrationDialog(QDialog): def reject(self) -> None: self.cancel_requested.emit() super().reject() - - def closeEvent(self, event) -> None: # noqa: N802 - self.cancel_requested.emit() - super().closeEvent(event) diff --git a/python_app/gui/preprocess_dialog.py b/python_app/gui/preprocess_dialog.py index 1fb2982..7815112 100644 --- a/python_app/gui/preprocess_dialog.py +++ b/python_app/gui/preprocess_dialog.py @@ -2,6 +2,8 @@ from __future__ import annotations +from contextlib import ExitStack + from PyQt6.QtCore import QSignalBlocker, pyqtSignal from PyQt6.QtWidgets import ( QCheckBox, @@ -24,12 +26,11 @@ from PyQt6.QtWidgets import ( import numpy as np import pyqtgraph as pg +from python_app.models.gui_profile_schema import GuiPreprocessDialogStateModel from python_app.models.dataset_model import TraceData from python_app.orchestration.preprocess_assets import ( - CAPTURABLE_PREPROCESS_ASSET_KEYS, - PREPROCESS_ASSET_SPECS, + PREPROCESS_ASSET_NAMES, REQUIRED_PREPROCESS_ASSET_KEYS, - VISIBLE_PREPROCESS_ASSET_KEYS, preprocess_asset_display_name, ) @@ -49,7 +50,6 @@ class PreprocessDialog(QDialog): finalize_sequence_requested = pyqtSignal() abort_sequence_requested = pyqtSignal() create_neutral_sets_requested = pyqtSignal() - import_object_calibration_requested = pyqtSignal() create_object_calibration_requested = pyqtSignal() def __init__(self, parent=None) -> None: @@ -113,7 +113,7 @@ class PreprocessDialog(QDialog): layout.addLayout(self._build_median_sweep_row(group)) layout.addWidget(self._build_radar_config_group(group)) - layout.addWidget(self._build_selector_group("S21", VISIBLE_PREPROCESS_ASSET_KEYS, group)) + layout.addWidget(self._build_selector_group("S21", tuple(PREPROCESS_ASSET_NAMES), group)) return group def _build_median_sweep_row(self, parent: QGroupBox) -> QHBoxLayout: @@ -179,13 +179,9 @@ class PreprocessDialog(QDialog): row = QHBoxLayout(field) row.setContentsMargins(0, 0, 0, 0) row.addWidget(combo, stretch=1) - self._import_object_calibration_button = QPushButton("Import…", field) - self._import_object_calibration_button.setToolTip("Import an existing cal_*.npy MIMO calibration folder.") - self._import_object_calibration_button.clicked.connect(self.import_object_calibration_requested.emit) self._create_object_calibration_button = QPushButton("Create…", field) self._create_object_calibration_button.setToolTip("Calculate a MIMO calibration from reflector measurements.") self._create_object_calibration_button.clicked.connect(self.create_object_calibration_requested.emit) - row.addWidget(self._import_object_calibration_button) row.addWidget(self._create_object_calibration_button) form.addRow(self._asset_row_label(key), field) return group @@ -218,7 +214,7 @@ class PreprocessDialog(QDialog): def _build_sequence_button_grid(self, parent: QGroupBox) -> QGridLayout: """Build per-asset capture start buttons.""" layout = QGridLayout() - for index, key in enumerate(CAPTURABLE_PREPROCESS_ASSET_KEYS): + for index, key in enumerate(REQUIRED_PREPROCESS_ASSET_KEYS): button = QPushButton(f"Start {preprocess_asset_display_name(key)}", parent) button.clicked.connect(lambda _checked=False, asset_key=key: self.start_sequence_requested.emit(asset_key)) layout.addWidget(button, index // 2, index % 2) @@ -285,6 +281,19 @@ class PreprocessDialog(QDialog): """Return requested target set name.""" return self._set_name_input.text().strip() + def apply_settings(self, state: GuiPreprocessDialogStateModel, selected: dict[str, str], *, neutral_visible: bool) -> None: + """Apply profile values atomically, without emitting selection/edit signals.""" + with ExitStack() as blockers: + for widget in (self._set_name_input, self._radar_config_dir_input, + self._use_all_radar_configs_checkbox, *self._set_combos.values()): + blockers.enter_context(QSignalBlocker(widget)) + self.set_set_name(state.set_name) + self.set_radar_config_dir(state.radar_config_dir) + self.set_use_all_radar_configs(state.use_all_radar_configs) + self.set_median_sweep_count(state.median_sweep_count) + self.set_selected_sets(selected, emit_signal=False) + self.set_neutral_sets_visible(neutral_visible) + def set_set_name(self, value: str) -> None: """Replace requested target set name.""" self._set_name_input.setText(value) @@ -335,7 +344,7 @@ class PreprocessDialog(QDialog): def selection_snapshot(self) -> dict[str, str]: """Return currently selected set names keyed by preprocess asset key.""" - return {key: self._set_combos[key].currentData() or "" for key in VISIBLE_PREPROCESS_ASSET_KEYS} + return {key: self._set_combos[key].currentData() or "" for key in PREPROCESS_ASSET_NAMES} def clear_capture_log(self) -> None: """Clear capture history text box.""" @@ -394,7 +403,7 @@ class PreprocessDialog(QDialog): return actions_enabled = bool(actions_enabled) - active_label = preprocess_asset_display_name(kind) if kind in PREPROCESS_ASSET_SPECS else kind + active_label = preprocess_asset_display_name(kind) if kind in PREPROCESS_ASSET_NAMES else kind self._active_kind_label.setText(active_label) self._progress_label.setText(f"{captured_count} / {total_count}") self._undo_last_button.setEnabled(bool(can_undo) and actions_enabled) @@ -414,14 +423,14 @@ class PreprocessDialog(QDialog): def set_available_sets(self, available_sets: dict[str, list[str]]) -> None: """Replace combo-box choices for all preprocess assets.""" - for key in VISIBLE_PREPROCESS_ASSET_KEYS: + for key in PREPROCESS_ASSET_NAMES: combo = self._set_combos[key] with QSignalBlocker(combo): self._set_combo_items(combo, available_sets.get(key, []), combo.currentData() or "") def set_selected_sets(self, selected_sets: dict[str, str], *, emit_signal: bool = True) -> None: """Apply selected set names to all comboboxes and optionally emit update.""" - for key in VISIBLE_PREPROCESS_ASSET_KEYS: + for key in PREPROCESS_ASSET_NAMES: selected_value = selected_sets.get(key, "") combo = self._set_combos[key] with QSignalBlocker(combo): @@ -451,9 +460,9 @@ class PreprocessDialog(QDialog): if self._preview_placeholder is not None: self._preview_placeholder.setText("Preview will appear after the first successful capture.") - def draw_last_trace(self, trace: TraceData, title: str, *, channel: str) -> None: + def draw_last_trace(self, trace: TraceData, title: str) -> None: """Draw amplitude (dB) and wrapped phase (deg, ±180) panes for the latest capture.""" - samples = trace.s11 if channel == "s11" else trace.s21 + samples = trace.s21 magnitude_db = 20.0 * np.log10(np.maximum(np.abs(samples), 1e-12)) phase_deg = np.degrees(np.angle(samples)) diff --git a/python_app/orchestration/gpr_locator.py b/python_app/gui/result_payloads.py similarity index 56% rename from python_app/orchestration/gpr_locator.py rename to python_app/gui/result_payloads.py index 2097285..b9c8677 100644 --- a/python_app/orchestration/gpr_locator.py +++ b/python_app/gui/result_payloads.py @@ -1,8 +1,4 @@ -"""Helpers for extracting GPR objects from result collections. - -Locator TCP delivery now lives in the C++ data_processor. This module retains -only the inspection helpers that the GUI uses for plotting. -""" +"""Result payload lookup and GPR display filtering.""" from __future__ import annotations @@ -42,14 +38,6 @@ def collection_payloads_by_prefix( return payloads -def collection_has_gpr_payloads(collection: ResultCollection) -> bool: - """Return whether collection carries GPR-specific collection payloads.""" - return any( - str(payload.processing_name).startswith("gpr_") - for payload in collection.collection_payloads - ) - - def gpr_object_rows(collection: ResultCollection) -> np.ndarray: """Return object rows as `[x_m, z_m, score]` from a GPR collection.""" points_payload = collection_payload_by_name(collection, "gpr_points", kind=4) @@ -67,40 +55,14 @@ def gpr_object_rows(collection: ResultCollection) -> np.ndarray: return np.zeros((0, 3), dtype=np.float32) -def apply_object_draw_limits( - rows: np.ndarray, - limits: tuple[int, int] | None, -) -> np.ndarray: - """Apply the object count/top-M drawing rules to already-filtered `[x, z, score]` rows. - - `limits` is `(max_detected_objects, draw_top_objects)`, or `None` to disable - (legacy GPR). When more than ``max_detected_objects`` survive, ALL are hidden - (the scene is too cluttered to be meaningful); otherwise the top ``draw_top_objects`` - rows are kept (rows arrive already sorted by score descending). - """ - if limits is None or rows.size == 0: - return rows - max_detected_objects, draw_top_objects = limits - if rows.shape[0] > int(max_detected_objects): - return np.zeros((0, rows.shape[1]), dtype=rows.dtype) - return rows[: max(0, int(draw_top_objects))] - - def filter_object_rows( rows: np.ndarray, *, min_score: float, x_bounds: tuple[float, float], z_bounds: tuple[float, float], - draw_limits: tuple[int, int] | None, ) -> np.ndarray: - """Filter `[x_m, z_m, score]` object rows for display/broadcast. - - Drops non-finite rows, rows below ``min_score`` (the caller passes the mode's own - threshold — a normalized float for coherent GPR, a pair count for legacy GPR; the - comparison is identical either way), and rows outside the visible X/Z window, then - applies ``draw_limits``. Mode-agnostic: all semantics enter through the parameters. - """ + """Keep finite objects inside the visible window and above the score threshold.""" if rows.size == 0: return rows x_min, x_max = x_bounds @@ -113,4 +75,4 @@ def filter_object_rows( & (rows[:, 1] >= z_min) & (rows[:, 1] <= z_max) ) - return apply_object_draw_limits(rows[visible_mask], draw_limits) + return rows[visible_mask] diff --git a/python_app/gui/runtime/__init__.py b/python_app/gui/runtime/__init__.py index ef2a80a..75ad3c8 100644 --- a/python_app/gui/runtime/__init__.py +++ b/python_app/gui/runtime/__init__.py @@ -1,13 +1 @@ """Runtime helpers for GUI polling, history management, and constraints.""" - -from python_app.gui.runtime.constraints import validate_processing_mode_constraints -from python_app.gui.runtime.history import ( - build_run_history_signature, - record_result_history, -) - -__all__ = [ - "build_run_history_signature", - "record_result_history", - "validate_processing_mode_constraints", -] diff --git a/python_app/gui/runtime/constraints.py b/python_app/gui/runtime/constraints.py index 9c36e84..41c28b4 100644 --- a/python_app/gui/runtime/constraints.py +++ b/python_app/gui/runtime/constraints.py @@ -2,7 +2,7 @@ from __future__ import annotations -from python_app.models.run_config_model import RunConfigModel +from python_app.models.run_config_schema import RunConfigModel from python_app.orchestration.live_processing_config import ProcessingLiveConfig @@ -38,6 +38,8 @@ def validate_processing_mode_constraints( if not available_inputs or not available_outputs: raise RuntimeError("GPR requires non-empty Tx/Rx geometry in run_config") + assert live_config.gpr_input_positions is not None + assert live_config.gpr_output_positions is not None requested_inputs = sorted({int(value) for value in live_config.gpr_input_positions}) requested_outputs = sorted({int(value) for value in live_config.gpr_output_positions}) diff --git a/python_app/gui/runtime/history.py b/python_app/gui/runtime/history.py index 598f8a3..01f37b3 100644 --- a/python_app/gui/runtime/history.py +++ b/python_app/gui/runtime/history.py @@ -3,22 +3,12 @@ from __future__ import annotations from collections import deque -from typing import TypeVar +from dataclasses import astuple -from python_app.models.dataset_model import ResultCollection, SweepCollection -from python_app.models.run_config_model import RunConfigModel -from python_app.orchestration.preprocess_assets import PREPROCESS_ASSET_KEYS, preprocess_asset_model - -THistoryCollection = TypeVar("THistoryCollection", SweepCollection, ResultCollection) - - -def collection_key(collection: SweepCollection | ResultCollection) -> tuple[int, int]: - """Identity of a sweep through the pipeline: its result keeps the same id and timestamp. - - `monotonic_ns` is stamped by `data_preprocessor` on the system monotonic clock, so - it also orders sweeps by acquisition, across runs too (ids restart from 1). - """ - return int(collection.collection_id), int(collection.monotonic_ns) +from python_app.models.collection_identity import collection_key +from python_app.models.dataset_model import ResultCollection +from python_app.models.run_config_schema import RunConfigModel +from python_app.orchestration.preprocess_assets import PREPROCESS_ASSET_NAMES, preprocess_asset_model def record_result_history( @@ -33,6 +23,8 @@ def record_result_history( delivers sweeps whose first result the GUI missed, and appending those would show them out of order. """ + if result_history.maxlen == 0: + return False key = collection_key(collection) position = len(result_history) while position > 0: @@ -40,7 +32,7 @@ def record_result_history( if collection_key(existing) == key: result_history[position - 1] = collection return False - if existing.monotonic_ns < collection.monotonic_ns: + if existing.capture_start_ns < collection.capture_start_ns: break position -= 1 @@ -53,48 +45,19 @@ def record_result_history( result_history.popleft() position -= 1 result_history.insert(position, collection) - return False - - -def remove_last_aligned_histories( - raw_history: list[SweepCollection], - preprocessed_history: list[SweepCollection], - result_history: list[ResultCollection], -) -> tuple[list[SweepCollection], list[SweepCollection], list[ResultCollection]]: - """Remove the newest aligned history entry using results as preferred anchor.""" - retained_raw = list(raw_history) - retained_preprocessed = list(preprocessed_history) - retained_results = list(result_history) - - if retained_results: - target_key = _tail_occurrence_key(retained_results, len(retained_results) - 1) - retained_results.pop() - _remove_by_tail_occurrence_key(retained_raw, target_key) - _remove_by_tail_occurrence_key(retained_preprocessed, target_key) - return retained_raw, retained_preprocessed, retained_results - - if retained_preprocessed: - target_key = _tail_occurrence_key(retained_preprocessed, len(retained_preprocessed) - 1) - retained_preprocessed.pop() - _remove_by_tail_occurrence_key(retained_raw, target_key) - return retained_raw, retained_preprocessed, retained_results - - if retained_raw: - retained_raw.pop() - - return retained_raw, retained_preprocessed, retained_results + return True def build_processor_run_signature( config: RunConfigModel, ) -> tuple[object, ...]: - """Build signature of settings that change the data SHAPE the data_processor parses. + """Settings consumed at processor startup, rather than through the live file. Excludes the preprocess set names on purpose: the data_processor does not consume calibration/reference sets (those are applied upstream by the data_preprocessor), so changing a set must NOT restart the processor. Restarting it would also tear down the - locator TCP server it hosts and drop every connected client. Only genuine shape changes - (radar model, sweep, switch layout, combos) require a processor restart. + locator TCP server it hosts and drop every connected client. Shape, ring endpoints, + locator settings, live-file path and polling interval require a restart. """ combos_signature = tuple((int(combo.input), int(combo.output)) for combo in config.combos) sweep_points_signature: object = "adc" if config.is_kamil_adc else int(config.radar.sweep.points) @@ -110,49 +73,30 @@ def build_processor_run_signature( float(config.radar.sweep.if_bandwidth_hz), float(config.radar.sweep.power_dbm), str(config.input_switch.driver_mode), - str(config.input_switch.driver), int(config.input_switch.positions), - bool(config.input_switch.invert_logic), str(config.output_switch.driver_mode), - str(config.output_switch.driver), int(config.output_switch.positions), - bool(config.output_switch.invert_logic), combos_signature, + astuple(config.rings), + astuple(config.runtime.locator_server), + str(config.runtime.processing_live_config_path), + int(config.runtime.idle_sleep_ms), ) def build_run_history_signature( config: RunConfigModel, ) -> tuple[object, ...]: - """Build full signature for GUI display-history reset (processor shape + preprocess sets). + """Reset display history when startup settings or preprocessing change. Display history still resets when the reference/calibration set changes (the on-screen B-scan would otherwise mix old- and new-reference frames), even though the processor process itself is intentionally kept alive across that change. """ - preprocess_signature = tuple(preprocess_asset_model(config, key).set_name for key in PREPROCESS_ASSET_KEYS) - return build_processor_run_signature(config) + (preprocess_signature,) - - -def _tail_occurrence_key(history: list[THistoryCollection], index: int) -> tuple[int, int]: - """Return `(collection_id, occurrence_from_tail)` for the item at `index`.""" - collection_id = int(history[index].collection_id) - occurrence_from_tail = 0 - for cursor in range(len(history) - 1, index, -1): - if int(history[cursor].collection_id) == collection_id: - occurrence_from_tail += 1 - return collection_id, occurrence_from_tail - - -def _remove_by_tail_occurrence_key(history: list[THistoryCollection], key: tuple[int, int]) -> bool: - """Remove newest matching entry identified by `(collection_id, occurrence_from_tail)`.""" - collection_id, target_occurrence = key - occurrence_from_tail = 0 - for index in range(len(history) - 1, -1, -1): - if int(history[index].collection_id) != collection_id: - continue - if occurrence_from_tail == target_occurrence: - history.pop(index) - return True - occurrence_from_tail += 1 - return False + preprocess_signature = tuple(astuple(preprocess_asset_model(config, key)) for key in PREPROCESS_ASSET_NAMES) + notch = config.preprocess.notch + notch_signature = ( + bool(notch.enabled), tuple(tuple(band) for band in notch.bands_hz), + float(notch.taper_width_hz), str(notch.taper_type), + ) + return build_processor_run_signature(config) + (preprocess_signature, notch_signature) diff --git a/python_app/gui/runtime/pipeline_pause.py b/python_app/gui/runtime/pipeline_pause.py new file mode 100644 index 0000000..e6b891a --- /dev/null +++ b/python_app/gui/runtime/pipeline_pause.py @@ -0,0 +1,31 @@ +"""Exclusive pipeline pause held until a synchronous or asynchronous operation ends.""" + +from __future__ import annotations + +from collections.abc import Callable + + +class PipelinePause: + """Suspend acquisition and restart intent; an explicit Stop cancels resumption.""" + + def __init__(self, valid: Callable[[], bool], release: Callable[[bool], None]) -> None: + self._valid = valid + self._release = release + self._finished = False + + @property + def valid(self) -> bool: + return not self._finished and self._valid() + + def finish(self, *, resume: bool = True) -> None: + if self._finished: + return + should_resume = resume and self.valid + self._finished = True + self._release(should_resume) + + def __enter__(self) -> PipelinePause: + return self + + def __exit__(self, *_exc: object) -> None: + self.finish() diff --git a/python_app/gui/settings_defaults.py b/python_app/gui/settings_defaults.py new file mode 100644 index 0000000..a01b3bd --- /dev/null +++ b/python_app/gui/settings_defaults.py @@ -0,0 +1,108 @@ +"""Qt-free defaults derived from a radar profile and GUI history capacities.""" + +import os +from pathlib import Path +from python_app.models.run_config_schema import RunConfigModel, ComboModel +from python_app.models.gui_profile_schema import ( + GuiStateModel, GuiSwitchStateModel, GuiProcessingStateModel, GuiGprStateModel, + GuiLegacyGprStateModel, GuiSar3dStateModel, GuiDataActionsStateModel, +) + +GUI_SAVE_HISTORY_LIMIT = 1000 +# Mirrors data_processor replay_history_limit, bounded further by SHM capacities. +CPP_BSCAN_REPLAY_WINDOW = 50 + + +def format_combos_text_from_config(config: RunConfigModel) -> str: + """Render configured combos for UI text editor, keeping full matrix as empty.""" + if config.is_multi_device: + return ",".join( + f"{int(combo.input)}:{int(combo.output)}" + for combo in config.build_runtime_combos() + ) + combos = list(config.combos) + full_combos = config.build_full_combos(config.input_switch.positions, config.output_switch.positions) + if len(combos) == len(full_combos) and all( + int(left.input) == int(right.input) and int(left.output) == int(right.output) + for left, right in zip(combos, full_combos, strict=True) + ): + return "" + return ",".join(f"{int(combo.input)}:{int(combo.output)}" for combo in combos) + + +def env_enabled(name: str) -> bool: + return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"} + + +def _positions(geometry: set[int], combos: set[int]) -> str: + return ",".join(map(str, sorted(geometry & combos or geometry))) + + +def default_gpr_visible_x_bounds_from_config(config: RunConfigModel) -> tuple[float, float]: + """Build default visible X-range for GPR object-only rendering.""" + x_values = [float(entry.x_m) for entry in config.gpr.tx_geometry] + x_values.extend(float(entry.x_m) for entry in config.gpr.rx_geometry) + if not x_values: + return (-2.0, 2.0) + margin_m = 2.0 + return (min(x_values) - margin_m, max(x_values) + margin_m) + + +def cpp_bscan_replay_window_for_config(config: RunConfigModel) -> int: + """Return how many newest collections the C++ processor re-processes on a + live B-scan settings change. + + Deliberately reproduces `replay_history_limit()` in `data_processor.cpp` + formula-for-formula. The ring capacities matter because `ShmRing` overwrites + the oldest unread slot on overflow, so a replay burst must fit in the results + ring for the GUI to receive all of it. + + This is the single place to change if the replay window ever becomes + configurable on the C++ side. + """ + return max( + 1, + min( + int(config.rings.preprocessed.capacity), + int(config.rings.results.capacity), + CPP_BSCAN_REPLAY_WINDOW, + ), + ) + + +def default_gui_state_for_config(config: RunConfigModel, project_root: Path) -> GuiStateModel: + """Build fallback GUI-only defaults for a stable run config.""" + default_combo = config.combos[0] if config.combos else ComboModel(input=0, output=0) + default_mode = "single" if len(config.combos) == 1 else "text" + inputs = _positions({entry.input_pos for entry in config.gpr.rx_geometry}, {combo.input for combo in config.combos}) + outputs = _positions({entry.output_pos for entry in config.gpr.tx_geometry}, {combo.output for combo in config.combos}) + default_gpr_x_min_m, default_gpr_x_max_m = default_gpr_visible_x_bounds_from_config(config) + return GuiStateModel( + switches=GuiSwitchStateModel( + combo_mode=default_mode, + combos_text=format_combos_text_from_config(config), + single_input=str(int(default_combo.input)), + single_output=str(int(default_combo.output)), + ), + processing=GuiProcessingStateModel( + gpr=GuiGprStateModel( + input_positions=inputs, + output_positions=outputs, + visible_x_min_m=default_gpr_x_min_m, + visible_x_max_m=default_gpr_x_max_m, + ), + legacy_gpr=GuiLegacyGprStateModel( + input_positions=inputs, + output_positions=outputs, + visible_x_min_m=default_gpr_x_min_m, + visible_x_max_m=default_gpr_x_max_m, + ), + sar3d=GuiSar3dStateModel( + input_positions=inputs, + output_positions=outputs, + ), + ), + data_actions=GuiDataActionsStateModel( + save_path=str(project_root / "python_app/data/snapshots"), + ), + ) diff --git a/python_app/gui/trace_png_export.py b/python_app/gui/trace_png_export.py index ebb1c92..16c27a6 100644 --- a/python_app/gui/trace_png_export.py +++ b/python_app/gui/trace_png_export.py @@ -32,7 +32,6 @@ def export_trace_png( trace: TraceData, output_path: Path, *, - channel: str, title: str, ) -> None: """Render one trace's amplitude (dB) and wrapped phase (deg) panes to a PNG. @@ -41,22 +40,22 @@ def export_trace_png( curves as the live preview, and exports the scene as a PNG. Requires a running ``QApplication`` (always true inside the GUI). """ - samples = trace.s11 if channel == "s11" else trace.s21 + samples = trace.s21 magnitude_db = 20.0 * np.log10(np.maximum(np.abs(samples), 1e-12)) phase_deg = np.degrees(np.angle(samples)) layout = pg.GraphicsLayoutWidget(show=False, size=_EXPORT_SIZE) layout.setBackground(_BACKGROUND) try: - layout.addLabel(title, row=0, col=0, colspan=2) + layout.ci.addLabel(title, row=0, col=0, colspan=2) - magnitude_plot = layout.addPlot(row=1, col=0) + magnitude_plot = layout.ci.addPlot(row=1, col=0) magnitude_plot.showGrid(x=True, y=True, alpha=0.2) magnitude_plot.setLabel("bottom", "Frequency", units="Hz") magnitude_plot.setLabel("left", "Magnitude", units="dB") magnitude_plot.plot(trace.frequency_hz, magnitude_db, pen=_MAGNITUDE_PEN) - phase_plot = layout.addPlot(row=1, col=1) + phase_plot = layout.ci.addPlot(row=1, col=1) phase_plot.showGrid(x=True, y=True, alpha=0.2) phase_plot.setLabel("bottom", "Frequency", units="Hz") phase_plot.setLabel("left", "Phase", units="deg") diff --git a/python_app/gui/window_view.py b/python_app/gui/window_view.py new file mode 100644 index 0000000..5a03a9a --- /dev/null +++ b/python_app/gui/window_view.py @@ -0,0 +1,93 @@ +"""Main-window layout and typed section views, without acquisition state.""" + +from pathlib import Path + +from PyQt6.QtWidgets import ( + QFileDialog, QFrame, QHBoxLayout, QLabel, QMainWindow, QPushButton, + QScrollArea, QVBoxLayout, QWidget, +) + +from python_app.gui.controllers.history_controller import HistoryController +from python_app.gui.controllers.sections.data_actions_section import build_data_actions_group +from python_app.gui.controllers.sections.preprocess_summary_section import build_preprocess_summary_group +from python_app.gui.controllers.sections.primary_actions_section import build_primary_actions_group +from python_app.gui.controllers.sections.processing_section import build_processing_group +from python_app.gui.controllers.sections.radar_section import build_radar_group +from python_app.gui.controllers.sections.switch_section import build_switch_group +from python_app.models.gui_profile_schema import GuiStateModel +from python_app.models.run_config_schema import RunConfigModel + + +class WindowView: + def __init__(self, parent: QMainWindow, config: RunConfigModel, gui: GuiStateModel, + plot_stack: QWidget, log_panel: QWidget, *, project_root: Path) -> None: + self._parent = parent + self.root = QWidget(parent) + parent.setCentralWidget(self.root) + parent.setWindowTitle("Radar System Control") + layout = QHBoxLayout(self.root) + layout.setContentsMargins(12, 12, 12, 12) + layout.setSpacing(14) + layout.addWidget(plot_stack, stretch=11) + self._toggle = QPushButton(">") + self._toggle.setObjectName("settingsToggleButton") + self._toggle.setFixedWidth(26) + self._toggle.setToolTip("Hide settings panel") + self._toggle.clicked.connect(self.toggle_settings) + layout.addWidget(self._toggle) + self.settings_panel = QWidget(self.root) + self.settings_panel.setMinimumWidth(610) + settings_layout = QVBoxLayout(self.settings_panel) + settings_layout.setContentsMargins(0, 0, 0, 0) + settings_layout.setSpacing(10) + self.primary = build_primary_actions_group() + settings_layout.addWidget(self.primary.group) + self.data = build_data_actions_group(gui) + self.switches = build_switch_group(config, gui) + self.preprocess = build_preprocess_summary_group() + self.processing = build_processing_group(config, gui) + self.radar = build_radar_group(config) + self.data.browse.clicked.connect(lambda: self.browse_save_path(project_root)) + self.sections = (self.data, self.switches, self.preprocess, self.processing, self.radar) + controls = QWidget() + controls_layout = QVBoxLayout(controls) + controls_layout.setContentsMargins(0, 0, 12, 0) + controls_layout.setSpacing(10) + for section in self.sections: + controls_layout.addWidget(section.group) + controls_layout.addStretch(1) + scroll = QScrollArea(self.settings_panel) + scroll.setWidgetResizable(True) + scroll.setFrameShape(QFrame.Shape.NoFrame) + scroll.setWidget(controls) + settings_layout.addWidget(scroll, stretch=1) + self.status = QLabel("Status: idle") + self.status.setObjectName("statusLabel") + self._history = QLabel("History: raw=0, preprocessed=0, results=0") + self._history.setObjectName("hintLabel") + settings_layout.addWidget(self.status) + settings_layout.addWidget(self._history) + settings_layout.addWidget(log_panel) + layout.addWidget(self.settings_panel, stretch=7) + parent.resize(1650, 940) + + def toggle_settings(self) -> None: + visible = self.settings_panel.isHidden() + self.settings_panel.setVisible(visible) + self._toggle.setText(">" if visible else "<") + self._toggle.setToolTip("Hide settings panel" if visible else "Show settings panel") + + def update_history(self, history: HistoryController) -> None: + self._history.setText(f"History: raw={len(history.raw)}, preprocessed={len(history.preprocessed)}, " + f"results={len(history.results)}") + + def update_preprocess(self, selected: dict[str, str]) -> None: + for key, label in self.preprocess.labels.items(): + label.setText(selected.get(key, "") or "") + + def browse_save_path(self, project_root: Path) -> None: + field = self.data.save_path + selected = QFileDialog.getExistingDirectory(self._parent, "Select Snapshot Directory", + field.text().strip() or str(project_root)) + if selected: + field.setText(selected) diff --git a/python_app/hardware_full/kamil_adc/laser.py b/python_app/hardware_full/kamil_adc/laser.py index 9fcaf17..4d19d7d 100644 --- a/python_app/hardware_full/kamil_adc/laser.py +++ b/python_app/hardware_full/kamil_adc/laser.py @@ -10,7 +10,7 @@ from __future__ import annotations import logging -from python_app.models.run_config_model import RunConfigModel +from python_app.models.run_config_schema import RunConfigModel logger = logging.getLogger(__name__) diff --git a/python_app/hardware_full/kamil_adc/processing.py b/python_app/hardware_full/kamil_adc/processing.py index b0e5492..0727bf4 100644 --- a/python_app/hardware_full/kamil_adc/processing.py +++ b/python_app/hardware_full/kamil_adc/processing.py @@ -44,6 +44,7 @@ from __future__ import annotations from dataclasses import dataclass import numpy as np +from numpy.typing import NDArray from python_app.models.run_config_schema import KamilAdcModel @@ -189,7 +190,7 @@ class KamilAdcSweepProcessor: return phase + shift, anchor + shift return phase, anchor - def process(self, main: np.ndarray, reference: np.ndarray) -> np.ndarray | None: + def process(self, main: np.ndarray, reference: np.ndarray) -> NDArray[np.complex64] | None: """Return the S21 trace resampled onto the fixed grid, or ``None`` to reject. ``main`` and ``reference`` are equal-length complex arrays ordered by diff --git a/python_app/hardware_full/kamil_adc/protocol.py b/python_app/hardware_full/kamil_adc/protocol.py index c6db4bb..e1f0dd0 100644 --- a/python_app/hardware_full/kamil_adc/protocol.py +++ b/python_app/hardware_full/kamil_adc/protocol.py @@ -24,6 +24,7 @@ from dataclasses import dataclass import struct import numpy as np +from numpy.typing import NDArray # Wire-format constants. FRAME_BYTES = 8 @@ -55,9 +56,9 @@ class RawSweep: samples at those steps. All three arrays share the same length. """ - steps: np.ndarray - main: np.ndarray - reference: np.ndarray + steps: NDArray[np.int32] + main: NDArray[np.complex64] + reference: NDArray[np.complex64] combo: tuple[int, int] | None = None dirty: bool = False diff --git a/python_app/hardware_full/kamil_adc/service.py b/python_app/hardware_full/kamil_adc/service.py index 6d1ffd8..f303509 100644 --- a/python_app/hardware_full/kamil_adc/service.py +++ b/python_app/hardware_full/kamil_adc/service.py @@ -30,7 +30,8 @@ from python_app.hardware_full.kamil_adc.tty_reader import ( raise_if_process_exited, ) from python_app.hardware_full.librevna_driver.models import SweepResult -from python_app.models.run_config_model import RadarSweepModel, RunConfigModel +from python_app.models.dataset_model import ComboKey +from python_app.models.run_config_schema import RadarSweepModel, RunConfigModel logger = logging.getLogger(__name__) @@ -178,12 +179,15 @@ class KamilAdcService: return SweepResult( x=grid.copy(), traces={ - "s11": np.zeros(points, dtype=np.complex64), "s21": s21, }, ) self._log_rejected_sweep(raw) + def prepare_combo(self, combo: ComboKey) -> None: + """Discard buffered sweeps that predate or straddle the physical switch.""" + self.drain_after_switch() + def drain_after_switch(self, sweeps: int = _SWITCH_DRAIN_SWEEPS) -> None: """Discard sweeps captured before / across a just-applied switch change. @@ -362,9 +366,10 @@ class KamilAdcService: self._rejected_count += 1 if self._rejected_count != 1 and self._rejected_count % _REJECT_LOG_EVERY != 0: return - params = self._processor.params # type: ignore[union-attr] + assert self._processor is not None + params = self._processor.params try: - freqs = self._processor.reference_frequency_axis(raw.reference) # type: ignore[union-attr] + freqs = self._processor.reference_frequency_axis(raw.reference) covered = f"[{float(np.min(freqs)):.6g}, {float(np.max(freqs)):.6g}]" except Exception: # noqa: BLE001 — diagnostics must never raise covered = "" diff --git a/python_app/hardware_full/laser_control/constants.py b/python_app/hardware_full/laser_control/constants.py index f3b81fe..e07f7a8 100644 --- a/python_app/hardware_full/laser_control/constants.py +++ b/python_app/hardware_full/laser_control/constants.py @@ -14,13 +14,11 @@ SEND_PARAMS_TOTAL_LENGTH = 30 # bytes in DECODE_ENABLE command TASK_ENABLE_COMMAND_LENGTH = 32 # bytes in TASK_ENABLE command WAIT_AFTER_SEND_SEC = 0.15 # delay after sending a command -GUI_POLL_INTERVAL_MS = 5 # GUI event loop timeout # ---- Command codes (as sent to device, already flipped to LE) CMD_DECODE_ENABLE = 0x1111 # Set control parameters CMD_DEFAULT_ENABLE = 0x2222 # Reset device -CMD_TRANSS_ENABLE = 0x3333 # Request all saved data (not implemented) CMD_TRANS_ENABLE = 0x4444 # Request last data CMD_REMOVE_FILE = 0x5555 # Delete saved data CMD_STATE = 0x6666 # Request state @@ -54,7 +52,6 @@ VREF = 2.5 # Reference voltage, Volts # Bridge resistors for temperature measurement R1 = 10000 # Ohm -R2 = 2200 # Ohm R3 = 27000 # Ohm R4 = 30000 # Ohm R5 = 27000 # Ohm @@ -105,17 +102,3 @@ TIME_STEP_MAX_US = 100 # Maximum time step, microseconds DELAY_TIME_MIN_MS = 3 # Minimum delay between pulses, milliseconds DELAY_TIME_MAX_MS = 10 # Maximum delay between pulses, milliseconds - -# ---- Acceptable voltage tolerances for power rail health check - -VOLT_3V3_MIN = 3.1 -VOLT_3V3_MAX = 3.5 -VOLT_5V_MIN = 4.8 -VOLT_5V_MAX = 5.3 -VOLT_7V_MIN = 6.5 -VOLT_7V_MAX = 7.5 - -# ---- Data buffer limits - -MAX_DATA_POINTS = 1000 # Max stored measurement points -PLOT_POINTS = 100 # Points shown in real-time plots \ No newline at end of file diff --git a/python_app/hardware_full/laser_control/controller.py b/python_app/hardware_full/laser_control/controller.py index 4220d87..c9ded45 100644 --- a/python_app/hardware_full/laser_control/controller.py +++ b/python_app/hardware_full/laser_control/controller.py @@ -13,18 +13,12 @@ from typing import Optional, Callable from .protocol import Protocol, TaskType as ProtoTaskType from .validators import ParameterValidator from .models import ( - ManualModeParams, - VariationParams, VariationType, Measurements, - DeviceStatus, - DeviceState, ) from .exceptions import ( - ValidationError, CommunicationError, DeviceNotRespondingError, - DeviceStateError, ) from .constants import WAIT_AFTER_SEND_SEC @@ -301,38 +295,6 @@ class LaserController: return measurements - def get_status(self) -> DeviceStatus: - """ - Request and return the current device status. - - Returns: - :class:`DeviceStatus` with state and latest measurements. - - Raises: - CommunicationError: On transport errors. - """ - cmd = Protocol.encode_state() - self._send(cmd) - - raw = self._protocol.receive_raw(2) - if not raw or len(raw) < 2: - raise DeviceNotRespondingError() - - state_code = Protocol.decode_state(raw) - - # Try to get measurements as well - measurements = self._last_measurements - - return DeviceStatus( - state=DeviceState(state_code) if state_code in DeviceState._value2member_map_ - else DeviceState.ERROR, - measurements=measurements, - is_connected=self.is_connected, - last_command_id=self._message_id, - error_message=Protocol.state_to_description(f"{state_code:04x}") - if state_code != 0 else None, - ) - def reset(self) -> None: """Send a hardware reset command to the device.""" cmd = Protocol.encode_default_enable() diff --git a/python_app/hardware_full/laser_control/exceptions.py b/python_app/hardware_full/laser_control/exceptions.py index 1d86787..00489a6 100644 --- a/python_app/hardware_full/laser_control/exceptions.py +++ b/python_app/hardware_full/laser_control/exceptions.py @@ -60,7 +60,7 @@ class CommunicationError(LaserControlError): class PortNotFoundError(CommunicationError): """Exception raised when serial port cannot be found.""" - def __init__(self, port: str = None): + def __init__(self, port: str | None = None): if port: message = f"Serial port '{port}' not found" else: @@ -77,7 +77,7 @@ class PortBusyError(CommunicationError): of silently sharing the bus and corrupting both sides' traffic. """ - def __init__(self, port: str = None): + def __init__(self, port: str | None = None): target = f"'{port}'" if port else "the laser controller port" super().__init__( f"Serial port {target} is busy — another program is already using " @@ -89,7 +89,7 @@ class PortBusyError(CommunicationError): class DeviceNotRespondingError(CommunicationError): """Exception raised when device doesn't respond to commands.""" - def __init__(self, timeout: float = None): + def __init__(self, timeout: float | None = None): if timeout: message = f"Device did not respond within {timeout} seconds" else: @@ -100,7 +100,7 @@ class DeviceNotRespondingError(CommunicationError): class CRCError(CommunicationError): """Exception raised when CRC check fails.""" - def __init__(self, expected: int = None, received: int = None): + def __init__(self, expected: int | None = None, received: int | None = None): if expected is not None and received is not None: message = f"CRC check failed. Expected: 0x{expected:04X}, Received: 0x{received:04X}" else: @@ -113,45 +113,3 @@ class ProtocolError(CommunicationError): def __init__(self, message: str): super().__init__(f"Protocol error: {message}") - - -class DeviceError(LaserControlError): - """Base exception for device-level errors.""" - pass - - -class DeviceOverheatingError(DeviceError): - """Exception raised when device reports overheating.""" - - def __init__(self, laser_id: int = None, temperature: float = None): - if laser_id and temperature: - message = f"Laser {laser_id} overheating: {temperature}°C" - else: - message = "Device overheating detected" - super().__init__(message) - - -class PowerSupplyError(DeviceError): - """Exception raised when power supply issues are detected.""" - - def __init__(self, rail: str = None, voltage: float = None, expected: float = None): - if rail and voltage is not None: - if expected: - message = f"Power supply {rail}: {voltage}V (expected ~{expected}V)" - else: - message = f"Power supply {rail}: abnormal voltage {voltage}V" - else: - message = "Power supply error detected" - super().__init__(message) - - -class DeviceStateError(DeviceError): - """Exception raised when device is in an error state.""" - - def __init__(self, state_code: int, state_name: str = None): - self.state_code = state_code - if state_name: - message = f"Device error state: {state_name} (0x{state_code:04X})" - else: - message = f"Device error state: 0x{state_code:04X}" - super().__init__(message) \ No newline at end of file diff --git a/python_app/hardware_full/laser_control/models.py b/python_app/hardware_full/laser_control/models.py index df547d5..030113a 100644 --- a/python_app/hardware_full/laser_control/models.py +++ b/python_app/hardware_full/laser_control/models.py @@ -20,77 +20,6 @@ class VariationType(IntEnum): CHANGE_TEMPERATURE_LD2 = 0x04 -class DeviceState(IntEnum): - """Device operational states.""" - IDLE = 0x0000 - RUNNING = 0x0001 - BUSY = 0x0002 - ERROR = 0x00FF - ERROR_OVERHEAT = 0x0100 - ERROR_POWER = 0x0200 - ERROR_COMMUNICATION = 0x0400 - ERROR_INVALID_COMMAND = 0x0800 - - -@dataclass -class ManualModeParams: - """Parameters for manual control mode.""" - temp1: float # Temperature for laser 1 (°C) - temp2: float # Temperature for laser 2 (°C) - current1: float # Current for laser 1 (mA) - current2: float # Current for laser 2 (mA) - pi_coeff1_p: float = 1.0 # PI controller proportional coefficient for laser 1 - pi_coeff1_i: float = 0.5 # PI controller integral coefficient for laser 1 - pi_coeff2_p: float = 1.0 # PI controller proportional coefficient for laser 2 - pi_coeff2_i: float = 0.5 # PI controller integral coefficient for laser 2 - - def to_dict(self) -> Dict[str, float]: - """Convert to dictionary representation.""" - return { - 'temp1': self.temp1, - 'temp2': self.temp2, - 'current1': self.current1, - 'current2': self.current2, - 'pi_coeff1_p': self.pi_coeff1_p, - 'pi_coeff1_i': self.pi_coeff1_i, - 'pi_coeff2_p': self.pi_coeff2_p, - 'pi_coeff2_i': self.pi_coeff2_i - } - - -@dataclass -class VariationParams: - """Parameters for variation mode.""" - variation_type: VariationType - # Static parameters (fixed during variation) - static_temp1: float - static_temp2: float - static_current1: float - static_current2: float - # Variation range - min_value: float # Minimum value for varied parameter - max_value: float # Maximum value for varied parameter - step: float # Step size for variation - # Time parameters - time_step: int # Time step in microseconds (20-100) - delay_time: int # Delay between measurements in milliseconds (3-10) - - def to_dict(self) -> Dict[str, Any]: - """Convert to dictionary representation.""" - return { - 'variation_type': self.variation_type.value, - 'static_temp1': self.static_temp1, - 'static_temp2': self.static_temp2, - 'static_current1': self.static_current1, - 'static_current2': self.static_current2, - 'min_value': self.min_value, - 'max_value': self.max_value, - 'step': self.step, - 'time_step': self.time_step, - 'delay_time': self.delay_time - } - - @dataclass class Measurements: """Real-time measurements from the device.""" @@ -134,86 +63,3 @@ class Measurements: 'timestamp': self.timestamp.isoformat() if self.timestamp else None, 'message_id': self.message_id } - - def check_power_rails(self) -> Dict[str, bool]: - """Check if power supply voltages are within acceptable range.""" - return { - '3v3': 3.1 <= self.voltage_3v3 <= 3.5, - '5v1': 4.8 <= self.voltage_5v1 <= 5.3, - '5v2': 4.8 <= self.voltage_5v2 <= 5.3, - '7v0': 6.5 <= self.voltage_7v0 <= 7.5 - } - - -@dataclass -class DeviceStatus: - """Complete device status information.""" - state: DeviceState - measurements: Optional[Measurements] = None - is_connected: bool = False - last_command_id: Optional[int] = None - error_message: Optional[str] = None - - @property - def is_idle(self) -> bool: - """Check if device is idle.""" - return self.state == DeviceState.IDLE - - @property - def is_running(self) -> bool: - """Check if device is running a task.""" - return self.state == DeviceState.RUNNING - - @property - def has_error(self) -> bool: - """Check if device has any error.""" - return self.state >= DeviceState.ERROR - - @property - def error_type(self) -> Optional[str]: - """Get human-readable error type.""" - if not self.has_error: - return None - - error_map = { - DeviceState.ERROR_OVERHEAT: "Overheating", - DeviceState.ERROR_POWER: "Power supply issue", - DeviceState.ERROR_COMMUNICATION: "Communication error", - DeviceState.ERROR_INVALID_COMMAND: "Invalid command" - } - return error_map.get(self.state, "Unknown error") - - def to_dict(self) -> Dict[str, Any]: - """Convert to dictionary representation.""" - return { - 'state': self.state.value, - 'state_name': self.state.name, - 'measurements': self.measurements.to_dict() if self.measurements else None, - 'is_connected': self.is_connected, - 'last_command_id': self.last_command_id, - 'error_message': self.error_message, - 'is_idle': self.is_idle, - 'is_running': self.is_running, - 'has_error': self.has_error, - 'error_type': self.error_type - } - - -@dataclass -class CalibrationData: - """Calibration data for device sensors.""" - # Temperature calibration coefficients - temp1_offset: float = 0.0 - temp1_scale: float = 1.0 - temp2_offset: float = 0.0 - temp2_scale: float = 1.0 - # Current calibration coefficients - current1_offset: float = 0.0 - current1_scale: float = 1.0 - current2_offset: float = 0.0 - current2_scale: float = 1.0 - # Voltage calibration - voltage_3v3_scale: float = 1.0 - voltage_5v1_scale: float = 1.0 - voltage_5v2_scale: float = 1.0 - voltage_7v0_scale: float = 1.0 \ No newline at end of file diff --git a/python_app/hardware_full/laser_control/monitoring/README.md b/python_app/hardware_full/laser_control/monitoring/README.md index 38362f9..7028e94 100644 --- a/python_app/hardware_full/laser_control/monitoring/README.md +++ b/python_app/hardware_full/laser_control/monitoring/README.md @@ -21,7 +21,7 @@ ## Архитектура ``` -[starter] ── TASK_ENABLE ──► плата кратко открыл порт, послал, закрыл +[starter] ── TASK_ENABLE ──► плата запускает вариацию │ пишет session.json (target temp1/2, tolerance, variation_type) ▼ [monitor] ── TRANS_ENABLE ──► плата владеет портом всё время работы @@ -32,19 +32,21 @@ сверяет temp1↔target_temp1 и temp2↔target_temp2, |Δ|>tol ─► WARNING в консоль ``` -- **Порт лазера эксклюзивен.** `starter` трогает его кратко, затем `monitor` - владеет им всё время. `checker` порт не трогает вовсе — читает только файлы. +- **Порт лазера эксклюзивен.** При `--start` запуск и мониторинг используют одно + соединение. Штатный пайплайн закрывает порт после запуска вариации, и его + может открыть отдельный монитор. `checker` читает только файлы. - **Связь через файлы** (JSONL + JSON), а не сокеты, — процессы стартуют, останавливаются и перезапускаются независимо, без рукопожатия. - **Сверяются оба лазера** по внутренним `temp1`/`temp2` (не по внешним - термисторам `temp_ext*`), каждый со своим допуском (по умолчанию `0.03 °C`). + термисторам `temp_ext*`), с общим допуском (по умолчанию `0.03 °C`). ## Быстрый старт (CLI, два процесса) -Терминал 1 — стартовать вариацию и мониторить температуру: +Команды запускаются из корня проекта. Терминал 1 — стартовать вариацию и +мониторить температуру: ```bash -python -m python_app.scripts.laser_temp_monitor \ +.venv/bin/python -m python_app.scripts.laser_temp_monitor \ --config run_config.json \ --start ``` @@ -52,15 +54,15 @@ python -m python_app.scripts.laser_temp_monitor \ Терминал 2 — валидировать температуру и печатать предупреждения: ```bash -python -m python_app.scripts.laser_temp_checker +.venv/bin/python -m python_app.scripts.laser_temp_checker ``` Пример вывода чекера при расхождении и возврате в допуск: ``` -WARNING laser_temp_checker: Laser 1 temperature off target: measured 28.100 °C, +WARNING python_app.hardware_full.laser_control.monitoring.checker: Laser 1 temperature off target: measured 28.100 °C, target 28.000 °C, Δ=+0.100 °C exceeds tolerance ±0.030 °C [seq=1] -INFO laser_temp_checker: Laser 1 temperature back within tolerance: +INFO python_app.hardware_full.laser_control.monitoring.checker: Laser 1 temperature back within tolerance: 28.000 °C (target 28.000, |Δ|=0.000 ≤ 0.030) [seq=2] ``` @@ -69,7 +71,7 @@ INFO laser_temp_checker: Laser 1 temperature back within tolerance: ## Конфигурация Параметры берутся из `run_config.json`, секция `radar.laser_control`. Мониторинг -использует блок `variation` и новое поле `temp_tolerance_c`: +использует блок `variation`, включая `temp_tolerance_c`: ```json { @@ -105,7 +107,7 @@ INFO laser_temp_checker: Laser 1 temperature back within tolerance: | Поле | Смысл | |---|---| -| `port` | Серийный порт лазерной платы (пусто → автоопределение) | +| `port` | Серийный порт лазерной платы; CLI-монитор допускает пустое значение для автоопределения, пайплайн требует явный порт | | `static_temp1` / `static_temp2` | Целевые статичные температуры лазеров 1/2, °C | | `min_value` / `max_value` / `step` | Диапазон и шаг свипа тока, мА — из них считается период свипа | | `time_step` / `delay_time` | Тайминги точки (мкс / мс) — тоже входят в период свипа | @@ -126,13 +128,13 @@ INFO laser_temp_checker: Laser 1 temperature back within tolerance: Мониторить уже запущенную из GUI/пайплайна вариацию (без повторного старта): ```bash -python -m python_app.scripts.laser_temp_monitor --config run_config.json +.venv/bin/python -m python_app.scripts.laser_temp_monitor --config run_config.json ``` Фиксированный период вместо расчётного (напр. раз в 500 мс): ```bash -python -m python_app.scripts.laser_temp_monitor \ +.venv/bin/python -m python_app.scripts.laser_temp_monitor \ --config run_config.json --strategy interval:500 ``` @@ -150,10 +152,10 @@ python -m python_app.scripts.laser_temp_monitor \ ```bash # монитор -python -m python_app.scripts.laser_temp_monitor --config cfg.json --start \ +.venv/bin/python -m python_app.scripts.laser_temp_monitor --config cfg.json --start \ --readings /tmp/run7.jsonl --session /tmp/run7.session.json # чекер -python -m python_app.scripts.laser_temp_checker \ +.venv/bin/python -m python_app.scripts.laser_temp_checker \ --readings /tmp/run7.jsonl --session /tmp/run7.session.json --reminder-every 20 ``` @@ -161,9 +163,10 @@ python -m python_app.scripts.laser_temp_checker \ Когда вариацию стартует штатный пайплайн ([`apply_kamil_adc_laser_control`](../../kamil_adc/laser.py)), снимок сессии -`session.json` пишется автоматически. Достаточно запустить только чекер -(и, при желании, монитор без `--start`, чтобы он опрашивал плату). Так консоль -получит предупреждения о рассинхроне температуры прямо во время захвата. +`/laser_variation_session.json` пишется автоматически. Для предупреждений +во время захвата нужны монитор без `--start`, который создаёт показания, и чекер. +Если монитор уже работает, достаточно добавить чекер. Он загружает сессию +один раз при запуске; после запуска новой вариации чекер нужно перезапустить. ## Встраивание в свой код (без CLI) @@ -187,15 +190,22 @@ period = resolve_period_s("computed", min_value=33.0, max_value=60.0, step=0.05, stop = threading.Event() with LaserController(port="/dev/ttyUSB0") as ctrl, ReadingWriter("readings.jsonl") as w: monitor = LaserTemperatureMonitor(ctrl, w, period_s=period) - threading.Thread(target=monitor.run, args=(stop,), daemon=True).start() + worker = threading.Thread(target=monitor.run, args=(stop,)) + worker.start() # 3. Чекер: тайлить показания и валидировать оба лазера checker = LaserTemperatureChecker.from_session(session) reader = ReadingReader("readings.jsonl") - while not stop.is_set(): - for reading in reader.poll(): - checker.process(reading) # печатает WARNING при |Δ| > tolerance - stop.wait(0.2) + try: + while not stop.is_set(): + for reading in reader.poll(): + checker.process(reading) # печатает WARNING при |Δ| > tolerance + stop.wait(0.2) + except KeyboardInterrupt: + pass + finally: + stop.set() + worker.join() # завершить опрос до закрытия порта и файла ``` `LaserTemperatureChecker.evaluate(reading)` возвращает список @@ -216,7 +226,7 @@ with LaserController(port="/dev/ttyUSB0") as ctrl, ReadingWriter("readings.jsonl } ``` -`readings.jsonl` (по одной строке-объекту на свип): +`readings.jsonl` (по одной строке-объекту на успешный опрос): ```json {"seq":0,"mono_ns":123456789,"temp1":28.0,"temp2":28.9,"temp_ext1":22.0,"temp_ext2":23.0,"current1":33.0,"current2":35.0} @@ -232,14 +242,16 @@ per_point_s = delay_time / 1000 + time_step / 1_000_000 sweep_period = num_steps × per_point_s ``` -Монитор публикует одно показание за такой период. Если нужен другой темп — +Монитор опрашивает плату сразу после запуска, затем ждёт такой период после +каждого опроса; это оценка, без синхронизации с границами аппаратных свипов. +Если нужен другой темп — `--strategy interval:`. (Внутренний счётчик `TO6` платы существует, но его семантика не гарантирована, поэтому для тайминга он не используется.) ## Тесты ```bash -python -m pytest python_app/tests/test_laser_temp_monitoring.py -q +.venv/bin/python -m unittest python_app.tests.test_laser_temp_monitoring ``` Покрыто: round-trip сессии, tail JSONL (включая усечённую последнюю строку), diff --git a/python_app/hardware_full/laser_control/monitoring/monitor.py b/python_app/hardware_full/laser_control/monitoring/monitor.py index e44e5df..1091f5f 100644 --- a/python_app/hardware_full/laser_control/monitoring/monitor.py +++ b/python_app/hardware_full/laser_control/monitoring/monitor.py @@ -23,10 +23,15 @@ from .readings_channel import ReadingWriter, TemperatureReading logger = logging.getLogger(__name__) +class _Temperatures(Protocol): + temp1: float + temp2: float + + class _MeasurementSource(Protocol): """Minimal controller surface the monitor depends on (eases testing).""" - def get_measurements(self) -> object: ... + def get_measurements(self) -> _Temperatures | None: ... def compute_sweep_period_s( @@ -138,5 +143,5 @@ class LaserTemperatureMonitor: logger.info("Temperature monitor stopped after %d readings", seq) -def _opt(value: object) -> Optional[float]: +def _opt(value: float | None) -> Optional[float]: return None if value is None else float(value) diff --git a/python_app/hardware_full/laser_control/monitoring/readings_channel.py b/python_app/hardware_full/laser_control/monitoring/readings_channel.py index 26f21cc..158d07d 100644 --- a/python_app/hardware_full/laser_control/monitoring/readings_channel.py +++ b/python_app/hardware_full/laser_control/monitoring/readings_channel.py @@ -58,7 +58,7 @@ class TemperatureReading: ) -def _opt_float(value: object) -> Optional[float]: +def _opt_float(value: str | float | None) -> Optional[float]: return None if value is None else float(value) diff --git a/python_app/hardware_full/laser_control/protocol.py b/python_app/hardware_full/laser_control/protocol.py index 8ed2f3f..1b20031 100644 --- a/python_app/hardware_full/laser_control/protocol.py +++ b/python_app/hardware_full/laser_control/protocol.py @@ -6,7 +6,6 @@ serial port connection to the laser control board. import errno import logging -import struct from typing import Optional from enum import IntEnum from datetime import datetime @@ -30,7 +29,7 @@ from .conversions import ( current_ma_to_n, current_n_to_ma, voltage_3v3_n_to_v, voltage_5v_n_to_v, voltage_7v_n_to_v, ) -from .models import Measurements, VariationType +from .models import Measurements from .exceptions import ( CommunicationError, PortNotFoundError, @@ -42,16 +41,6 @@ from .exceptions import ( logger = logging.getLogger(__name__) -# Re-export enums so tests can import from protocol module -class CommandCode(IntEnum): - DECODE_ENABLE = CMD_DECODE_ENABLE - DEFAULT_ENABLE = CMD_DEFAULT_ENABLE - TRANS_ENABLE = CMD_TRANS_ENABLE - REMOVE_FILE = CMD_REMOVE_FILE - STATE = CMD_STATE - TASK_ENABLE = CMD_TASK_ENABLE - - class TaskType(IntEnum): MANUAL = 0x00 CHANGE_CURRENT_LD1 = 0x01 @@ -60,17 +49,6 @@ class TaskType(IntEnum): CHANGE_TEMPERATURE_LD2 = 0x04 -class DeviceState(IntEnum): - IDLE = 0x0000 - RUNNING = 0x0001 - BUSY = 0x0002 - ERROR = 0x00FF - ERROR_OVERHEAT = 0x0100 - ERROR_POWER = 0x0200 - ERROR_COMMUNICATION = 0x0400 - ERROR_INVALID_COMMAND = 0x0800 - - # ---- Low-level helpers -------------------------------------------------- def _int_to_hex4(value: int) -> str: @@ -156,20 +134,6 @@ class Response: ) -# ---- Message builder -------------------------------------------------- - -class Message: - """Named container for an encoded command byte array.""" - def __init__(self, data: bytearray): - self._data = data - - def to_bytes(self) -> bytes: - return bytes(self._data) - - def __len__(self): - return len(self._data) - - # ---- Protocol class -------------------------------------------------- class Protocol: @@ -364,17 +328,17 @@ class Protocol: @staticmethod def encode_trans_enable(message_id: int = 0) -> bytes: """Build TRANS_ENABLE command (0x4444) — request last data.""" - return bytearray.fromhex(_flipfour(_int_to_hex4(CMD_TRANS_ENABLE))) + return bytes.fromhex(_flipfour(_int_to_hex4(CMD_TRANS_ENABLE))) @staticmethod def encode_state(message_id: int = 0) -> bytes: """Build STATE command (0x6666) — request device state.""" - return bytearray.fromhex(_flipfour(_int_to_hex4(CMD_STATE))) + return bytes.fromhex(_flipfour(_int_to_hex4(CMD_STATE))) @staticmethod def encode_default_enable(message_id: int = 0) -> bytes: """Build DEFAULT_ENABLE command (0x2222) — reset device.""" - return bytearray.fromhex(_flipfour(_int_to_hex4(CMD_DEFAULT_ENABLE))) + return bytes.fromhex(_flipfour(_int_to_hex4(CMD_DEFAULT_ENABLE))) # ---- Response decoders ----------------------------------------------- @@ -433,7 +397,7 @@ class Protocol: Decode a 2-byte STATE response from the device. Returns: - Integer state code (compare with DeviceState enum). + Integer state code; see STATE_* constants. """ if len(data) < 2: raise ProtocolError(f"STATE response too short: {len(data)} bytes") diff --git a/python_app/hardware_full/librevna_backends.py b/python_app/hardware_full/librevna_backends.py deleted file mode 100644 index 9c2a813..0000000 --- a/python_app/hardware_full/librevna_backends.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Backend adapters used by :mod:`python_app.hardware_full.librevna_service`.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from contextlib import suppress -from typing import Any - -import numpy as np - -from python_app.hardware_full.librevna_driver.models import SweepResult -from python_app.models.run_config_model import RadarSweepModel - - -@dataclass(slots=True) -class NativeLibreVnaBackend: - """Native backend using direct USB LibreVNA Python driver.""" - - serial: str | None - strict_protocol_version: int - _device: Any | None = field(init=False, default=None, repr=False) - _settings: RadarSweepModel | None = field(init=False, default=None, repr=False) - _libre_vna_device: type[Any] = field(init=False, repr=False) - _vna_sweep_settings: type[Any] = field(init=False, repr=False) - - def __post_init__(self) -> None: - """Import native driver classes lazily and initialize backend state.""" - from python_app.hardware_full.librevna_driver import LibreVNADevice, VNASweepSettings - - self._libre_vna_device = LibreVNADevice - self._vna_sweep_settings = VNASweepSettings - - def open(self) -> None: - """Open device connection if it is not already open.""" - if self._device is not None: - return - - device = self._device = self._libre_vna_device() - try: - device.connect(serial=self.serial, strict_protocol_version=self.strict_protocol_version, timeout_s=2.0) - except Exception: - with suppress(Exception): - self.close() - raise - - @property - def is_open(self) -> bool: - """Return `True` when native device connection is open.""" - return self._device is not None - - def close(self) -> None: - """Close device connection when open.""" - if self._device is None: - return - device, self._device = self._device, None - device.disconnect() - - def configure(self, sweep: RadarSweepModel) -> None: - """Store and apply sweep settings to connected hardware.""" - self._settings = sweep - if self._device is None: - return - - settings = self._vna_sweep_settings( - f_start_hz=sweep.start_hz, - f_stop_hz=sweep.stop_hz, - points=sweep.points, - if_bandwidth_hz=sweep.if_bandwidth_hz, - power_start_dbm=sweep.power_dbm, - power_stop_dbm=sweep.power_dbm, - excited_ports=(1, 2), - ) - self._device.vna.configure(settings) - - def read_device_limits(self) -> dict[str, float | int]: - """Read frequency/IFBW/power/points limits from connected device.""" - if self._device is None: - raise RuntimeError("Failed to connect to LibreVNA device") - - device_info = self._device.get_device_info() - limits = device_info.limits - return { - "min_frequency_hz": float(limits.min_frequency_hz), - "max_frequency_hz": float(limits.max_frequency_hz), - "min_ifbw_hz": float(limits.min_ifbw_hz), - "max_ifbw_hz": float(limits.max_ifbw_hz), - "max_points": int(limits.max_points), - "min_power_dbm": float(limits.min_power_dbm), - "max_power_dbm": float(limits.max_power_dbm), - } - - def acquire(self) -> SweepResult: - """Acquire one sweep from hardware.""" - if self._settings is None: - raise RuntimeError("Radar service is not configured") - if self._device is None: - raise RuntimeError("Device not found") - - result = self._device.vna.acquire(expected_points=self._settings.points, timeout_s=20.0) - return SweepResult( - x=np.asarray(result.x, dtype=np.float32), - traces={ - "s11": np.asarray(result.trace("s11"), dtype=np.complex64), - "s21": np.asarray(result.trace("s21"), dtype=np.complex64), - }, - ) diff --git a/python_app/hardware_full/librevna_driver/__init__.py b/python_app/hardware_full/librevna_driver/__init__.py index d4fabca..cf7852f 100644 --- a/python_app/hardware_full/librevna_driver/__init__.py +++ b/python_app/hardware_full/librevna_driver/__init__.py @@ -5,7 +5,6 @@ import logging from .enums import ( HardwareFamily, PacketType, - SParameter, SweepKind, SweepScale, SyncMode, @@ -53,7 +52,6 @@ __all__ = [ "PacketType", "ParseError", "ProtocolVersionMismatch", - "SParameter", "StreamHandle", "SweepKind", "SweepResult", diff --git a/python_app/hardware_full/librevna_driver/api/__init__.py b/python_app/hardware_full/librevna_driver/api/__init__.py index 550f82a..e69de29 100644 --- a/python_app/hardware_full/librevna_driver/api/__init__.py +++ b/python_app/hardware_full/librevna_driver/api/__init__.py @@ -1,11 +0,0 @@ -"""Public controller classes.""" - -from .config import ConfigController -from .generator import GeneratorController -from .vna import VNAController - -__all__ = [ - "ConfigController", - "GeneratorController", - "VNAController", -] diff --git a/python_app/hardware_full/librevna_driver/api/vna.py b/python_app/hardware_full/librevna_driver/api/vna.py index 5cff8b8..00aa1e7 100644 --- a/python_app/hardware_full/librevna_driver/api/vna.py +++ b/python_app/hardware_full/librevna_driver/api/vna.py @@ -4,13 +4,12 @@ from __future__ import annotations from dataclasses import replace import logging -from typing import Callable -from ..enums import PacketType +from ..enums import PacketType, SyncMode from ..exceptions import ParseError -from ..models import Packet, StreamHandle, SweepResult, VNADatapointPacket, VNARawPoint, VNASweepSettings +from ..models import Packet, SweepResult, VNADatapointPacket, VNASweepSettings from ..session import LibreVNASession -from ..sweep.assembler import assemble_vna_sweep, datapoint_to_raw_point +from ..sweep.assembler import assemble_vna_sweep logger = logging.getLogger(__name__) @@ -27,22 +26,21 @@ class VNAController: """Send `SweepSettings` packet to configure VNA operation.""" effective = settings if effective.sync_mode is None: - effective = replace(effective, sync_mode=self._session.default_sync_mode) + effective = replace(effective, sync_mode=SyncMode.DISABLED) self._settings = effective self._session.clear_queue(PacketType.VNA_DATAPOINT) self._session.send(Packet(PacketType.SWEEP_SETTINGS, effective), require_ack=True) logger.info( - "VNA configured: kind=%s start=%.3fHz stop=%.3fHz points=%d ifbw=%.3fHz ports=%s", + "VNA configured: kind=%s start=%.3fHz stop=%.3fHz points=%d ifbw=%.3fHz", effective.kind.value, effective.f_start_hz, effective.f_stop_hz, effective.points, effective.if_bandwidth_hz, - effective.excited_ports, ) - def acquire(self, *, expected_points: int | None = None, timeout_s: float = 10.0) -> SweepResult: + def acquire(self, *, expected_points: int | None = None, timeout_s: float | None = None) -> SweepResult: """Acquire one complete sweep and return assembled complex traces.""" if self._settings is None: raise ParseError("VNA is not configured. Call vna.configure() before acquire().") @@ -52,6 +50,13 @@ class VNAController: if points_target <= 0: raise ValueError("expected_points must be > 0") + # Firmware integrates for 1/IFBW after dwell/PLL settling at every point. + # Allow a discarded free-running tail and USB/host overhead, but detect + # a stalled instrument independently of this whole-sweep budget. + point_seconds = 1.0 / settings.if_bandwidth_hz + settings.dwell_s + 0.001 + if timeout_s is None: + timeout_s = max(10.0, 2.0 * points_target * point_seconds + 2.0) + logger.info( "VNA acquire start: expected_points=%d timeout=%.2fs standby=%s", points_target, @@ -66,6 +71,7 @@ class VNAController: packet_type=PacketType.VNA_DATAPOINT, expected_points=points_target, timeout_s=timeout_s, + progress_timeout_s=max(1.5, 3.0 * point_seconds), payload_type=VNADatapointPacket, payload_error="Expected decoded VNADatapointPacket payload", ) @@ -79,21 +85,3 @@ class VNAController: ) logger.info("VNA acquire complete: received_points=%d", len(result.x)) return result - - def stream(self, callback: Callable[[VNARawPoint], None]) -> StreamHandle: - """Subscribe callback for every incoming VNA datapoint packet.""" - if self._settings is None: - raise ParseError("VNA is not configured. Call vna.configure() before stream().") - - settings = self._settings - num_ports = self._session.get_device_info().num_ports - - def _on_packet(packet: Packet) -> None: - """Decode datapoint packet and forward mapped raw point to callback.""" - payload = packet.payload - if not isinstance(payload, VNADatapointPacket): - raise ParseError("Expected decoded VNADatapointPacket payload") - callback(datapoint_to_raw_point(payload, settings, num_ports=num_ports)) - - logger.info("VNA stream subscription started") - return self._session.subscribe(PacketType.VNA_DATAPOINT, _on_packet) diff --git a/python_app/hardware_full/librevna_driver/enums.py b/python_app/hardware_full/librevna_driver/enums.py index e336428..7ca535f 100644 --- a/python_app/hardware_full/librevna_driver/enums.py +++ b/python_app/hardware_full/librevna_driver/enums.py @@ -76,12 +76,3 @@ class SweepScale(str, Enum): LIN = "lin" LOG = "log" - - -class SParameter(str, Enum): - """Canonical 2-port S-parameters.""" - - S11 = "S11" - S12 = "S12" - S21 = "S21" - S22 = "S22" diff --git a/python_app/hardware_full/librevna_driver/models.py b/python_app/hardware_full/librevna_driver/models.py index d59dfcc..d42d560 100644 --- a/python_app/hardware_full/librevna_driver/models.py +++ b/python_app/hardware_full/librevna_driver/models.py @@ -89,7 +89,6 @@ class VNASweepSettings: if_bandwidth_hz: float = 1_000.0 power_start_dbm: float = -10.0 power_stop_dbm: float = -10.0 - excited_ports: tuple[int, ...] = (1, 2) sweep_scale: SweepScale = SweepScale.LIN dwell_s: float = 0.0 suppress_invalid_peaks: bool = True @@ -106,12 +105,6 @@ class VNASweepSettings: raise ValueError("if_bandwidth_hz must be > 0") if self.dwell_s < 0: raise ValueError("dwell_s must be >= 0") - if not self.excited_ports: - raise ValueError("excited_ports must not be empty") - if len(set(self.excited_ports)) != len(self.excited_ports): - raise ValueError("excited_ports must not contain duplicates") - if any(port <= 0 for port in self.excited_ports): - raise ValueError("excited_ports must use 1-based positive port numbers") if self.f_stop_hz < self.f_start_hz: raise ValueError("f_stop_hz must be >= f_start_hz") if self.kind == SweepKind.POWER and self.power_stop_dbm < self.power_start_dbm: @@ -217,60 +210,14 @@ class SweepResult: normalized[key.strip().lower()] = arr self.traces = normalized - @property - def s11(self) -> np.ndarray | None: - """Return `S11` trace when available.""" - return self.traces.get("s11") - @property def s21(self) -> np.ndarray | None: """Return `S21` trace when available.""" return self.traces.get("s21") - @property - def s12(self) -> np.ndarray | None: - """Return `S12` trace when available.""" - return self.traces.get("s12") - - @property - def s22(self) -> np.ndarray | None: - """Return `S22` trace when available.""" - return self.traces.get("s22") - def trace(self, parameter: str) -> np.ndarray: """Return complex trace by parameter name.""" key = parameter.strip().lower() if key not in self.traces: raise KeyError(f"Trace '{parameter}' is not available") return self.traces[key] - - def real(self, parameter: str) -> np.ndarray: - """Return real part of selected trace.""" - return self.trace(parameter).real - - def imag(self, parameter: str) -> np.ndarray: - """Return imaginary part of selected trace.""" - return self.trace(parameter).imag - - def to_npz(self, path: str) -> None: - """Save the axis and all traces to a NumPy `.npz` archive at ``path``.""" - data: dict[str, np.ndarray] = {self.x_label: self.x} - data.update(self.traces) - np.savez(path, **data) - logger.debug("Saved SweepResult to NPZ: %s (traces=%d)", path, len(self.traces)) - - def to_csv(self, path: str) -> None: - """Save the axis and traces to CSV at ``path``, with `_real`/`_imag` columns.""" - columns: list[np.ndarray] = [self.x] - headers: list[str] = [self.x_label] - for name, values in sorted(self.traces.items()): - columns.append(values.real) - columns.append(values.imag) - headers.append(f"{name}_real") - headers.append(f"{name}_imag") - matrix = np.column_stack(columns) - np.savetxt(path, matrix, delimiter=",", header=",".join(headers), comments="") - logger.debug("Saved SweepResult to CSV: %s (traces=%d)", path, len(self.traces)) - - -PacketPayload = Any diff --git a/python_app/hardware_full/librevna_driver/protocol/codec.py b/python_app/hardware_full/librevna_driver/protocol/codec.py index 25eb5d5..d29b978 100644 --- a/python_app/hardware_full/librevna_driver/protocol/codec.py +++ b/python_app/hardware_full/librevna_driver/protocol/codec.py @@ -239,12 +239,8 @@ def decode_packet_payload(packet: Packet) -> object: def encode_sweep_settings_payload(settings: VNASweepSettings) -> bytes: """Encode `SweepSettings` payload.""" - if len(settings.excited_ports) > 4: - raise ValueError("Protocol supports at most four excited ports") sync_mode = settings.sync_mode if settings.sync_mode is not None else 0 - stage_by_port = {port: stage for stage, port in enumerate(settings.excited_ports)} - flags1 = 0 flags1 |= int(bool(settings.standby)) << 0 flags1 |= int(bool(settings.sync_master)) << 1 @@ -253,13 +249,9 @@ def encode_sweep_settings_payload(settings: VNASweepSettings) -> bytes: flags1 |= int(settings.sweep_scale.value == "log") << 4 flags1 |= (int(sync_mode) & 0x03) << 5 - flags2 = 0 - stages = len(settings.excited_ports) - 1 - flags2 |= stages & 0x07 - flags2 |= (stage_by_port.get(1, 0) & 0x07) << 3 - flags2 |= (stage_by_port.get(2, 0) & 0x07) << 6 - flags2 |= (stage_by_port.get(3, 0) & 0x07) << 9 - flags2 |= (stage_by_port.get(4, 0) & 0x07) << 12 + # One stage, stimulus on port 1; stage 7 disables port 2. + # port3Stage must stay zero: our firmware uses it for substage width. + flags2 = 7 << 6 dwell_us = int(round(settings.dwell_s * 1_000_000.0)) dwell_us = max(0, min(0xFFFF, dwell_us)) diff --git a/python_app/hardware_full/librevna_driver/session.py b/python_app/hardware_full/librevna_driver/session.py index 809c459..1a87d02 100644 --- a/python_app/hardware_full/librevna_driver/session.py +++ b/python_app/hardware_full/librevna_driver/session.py @@ -8,7 +8,7 @@ import threading import time from typing import Callable, Protocol, TypeVar -from .enums import HardwareFamily, PacketType, SyncMode +from .enums import HardwareFamily, PacketType from .exceptions import ( DeviceDisconnectedError, NackError, @@ -69,9 +69,6 @@ class LibreVNASession: self._device_status: DeviceStatus | None = None self._hardware_family: HardwareFamily = HardwareFamily.UNKNOWN - self._default_sync_mode = SyncMode.DISABLED - self._default_sync_master = False - @property def is_connected(self) -> bool: """Return `True` when USB connection is active.""" @@ -87,21 +84,6 @@ class LibreVNASession: """Hardware family inferred from `DeviceInfo` packet.""" return self._hardware_family - @property - def default_sync_mode(self) -> SyncMode: - """Default sync mode used by controllers when settings do not override.""" - return self._default_sync_mode - - @property - def default_sync_master(self) -> bool: - """Default sync master flag used by controllers.""" - return self._default_sync_master - - def set_default_sync(self, mode: SyncMode, master: bool) -> None: - """Set session-level default synchronization settings.""" - self._default_sync_mode = mode - self._default_sync_master = master - @staticmethod def list_devices() -> list[USBDeviceDescriptor]: """Enumerate available direct-USB LibreVNA devices.""" @@ -241,6 +223,7 @@ class LibreVNASession: timeout_s: float, payload_type: type[TPacketPayload], payload_error: str, + progress_timeout_s: float | None = None, ) -> list[TPacketPayload]: """Collect one sweep from point zero, returning points in ascending order. @@ -251,11 +234,14 @@ class LibreVNASession: raise ValueError("expected_points must be > 0") deadline = time.monotonic() + timeout_s + progress_timeout_s = timeout_s if progress_timeout_s is None else progress_timeout_s + progress_deadline = time.monotonic() + progress_timeout_s + previous_point: int | None = None collected: list[TPacketPayload | None] = [None] * expected_points received = 0 while received < expected_points: - remaining = deadline - time.monotonic() + remaining = min(deadline, progress_deadline) - time.monotonic() if remaining <= 0: break @@ -270,6 +256,10 @@ class LibreVNASession: f"Received out-of-range point index {point_number}, expected 0..{expected_points - 1}" ) + if point_number != previous_point: + progress_deadline = time.monotonic() + progress_timeout_s + previous_point = point_number + if point_number == 0: collected = [None] * expected_points received = 0 diff --git a/python_app/hardware_full/librevna_driver/sweep/__init__.py b/python_app/hardware_full/librevna_driver/sweep/__init__.py index 710a70b..e69de29 100644 --- a/python_app/hardware_full/librevna_driver/sweep/__init__.py +++ b/python_app/hardware_full/librevna_driver/sweep/__init__.py @@ -1,10 +0,0 @@ -"""Sweep assembly and result helpers.""" - -from .assembler import assemble_vna_sweep, datapoint_to_raw_point -from ..models import SweepResult - -__all__ = [ - "SweepResult", - "assemble_vna_sweep", - "datapoint_to_raw_point", -] diff --git a/python_app/hardware_full/librevna_driver/sweep/assembler.py b/python_app/hardware_full/librevna_driver/sweep/assembler.py index f83e9a7..9864a8c 100644 --- a/python_app/hardware_full/librevna_driver/sweep/assembler.py +++ b/python_app/hardware_full/librevna_driver/sweep/assembler.py @@ -32,12 +32,12 @@ def _find_vna_value( ) -> complex: """Find one complex receiver value for stage/port/ref tuple.""" source_mask = 1 << port_index - if reference: - source_mask |= _REFERENCE_FLAG for idx, flags in enumerate(datapoint.flags): if _extract_stage(int(flags)) != stage: continue + if bool(int(flags) & _REFERENCE_FLAG) != reference: + continue if (int(flags) & source_mask) != source_mask: continue return complex(float(datapoint.real[idx]), float(datapoint.imag[idx])) @@ -48,6 +48,14 @@ def _find_vna_value( ) +def _s21(datapoint: VNADatapointPacket) -> complex: + reference = _find_vna_value(datapoint, stage=0, port_index=0, reference=True) + receiver = _find_vna_value(datapoint, stage=0, port_index=1, reference=False) + if not np.isfinite(reference) or abs(reference) <= 1e-12 or not np.isfinite(receiver): + raise ParseError(f"Invalid receiver/reference at point {datapoint.point_number}") + return receiver / reference + + def datapoint_to_raw_point( datapoint: VNADatapointPacket, settings: VNASweepSettings, @@ -55,19 +63,9 @@ def datapoint_to_raw_point( num_ports: int, ) -> VNARawPoint: """Convert low-level VNADatapoint packet into callback-friendly object.""" - measurements: dict[str, complex] = {} - stage_by_excited_port = {port: stage for stage, port in enumerate(settings.excited_ports)} - - for excited_port, stage in stage_by_excited_port.items(): - ref = _find_vna_value(datapoint, stage=stage, port_index=excited_port - 1, reference=True) - for receiver_port in range(1, num_ports + 1): - measured = _find_vna_value( - datapoint, - stage=stage, - port_index=receiver_port - 1, - reference=False, - ) - measurements[f"S{receiver_port}{excited_port}"] = measured / ref + if num_ports < 2: + raise ParseError("S21 acquisition requires two receiver ports") + measurements = {"S21": _s21(datapoint)} zero_span = settings.f_start_hz == settings.f_stop_hz and math.isclose( settings.power_start_dbm, @@ -108,17 +106,9 @@ def assemble_vna_sweep( x = np.empty(expected_points, dtype=np.float64) seen = np.zeros(expected_points, dtype=bool) - stage_by_excited_port = {port: stage for stage, port in enumerate(settings.excited_ports)} - receiver_limit = min(num_ports, 2) - trace_names = [ - f"s{receiver_port}{excited_port}" - for excited_port in settings.excited_ports - for receiver_port in range(1, receiver_limit + 1) - if receiver_port <= 2 and excited_port <= 2 - ] - traces: dict[str, np.ndarray] = { - name: np.empty(expected_points, dtype=np.complex128) for name in trace_names - } + if num_ports < 2: + raise ParseError("S21 acquisition requires two receiver ports") + s21 = np.empty(expected_points, dtype=np.complex128) zero_span = settings.f_start_hz == settings.f_stop_hz and math.isclose( settings.power_start_dbm, @@ -130,7 +120,7 @@ def assemble_vna_sweep( for datapoint in datapoints: idx = datapoint.point_number - if idx >= expected_points: + if not 0 <= idx < expected_points: raise ParseError(f"Received out-of-range point index {idx}, expected < {expected_points}") if zero_span: @@ -140,22 +130,7 @@ def assemble_vna_sweep( else: x[idx] = float(datapoint.frequency_or_time) - for excited_port, stage in stage_by_excited_port.items(): - ref = _find_vna_value(datapoint, stage=stage, port_index=excited_port - 1, reference=True) - for receiver_port in range(1, num_ports + 1): - if receiver_port > 2 or excited_port > 2: - # Public SweepResult intentionally exposes 2-port canonical traces. - continue - measured = _find_vna_value( - datapoint, - stage=stage, - port_index=receiver_port - 1, - reference=False, - ) - trace_name = f"s{receiver_port}{excited_port}" - trace = traces.get(trace_name) - if trace is not None: - trace[idx] = measured / ref + s21[idx] = _s21(datapoint) seen[idx] = True @@ -171,7 +146,7 @@ def assemble_vna_sweep( elif power_sweep: x_label = "power_dbm" - result = SweepResult(x=x, traces=traces, x_label=x_label) + result = SweepResult(x=x, traces={"s21": s21}, x_label=x_label) logger.debug( "Assembled VNA sweep: points=%d x_label=%s traces=%s", len(result.x), diff --git a/python_app/hardware_full/librevna_driver/transport/usb.py b/python_app/hardware_full/librevna_driver/transport/usb.py index 41f87a1..d757def 100644 --- a/python_app/hardware_full/librevna_driver/transport/usb.py +++ b/python_app/hardware_full/librevna_driver/transport/usb.py @@ -6,7 +6,7 @@ from contextlib import suppress import logging import threading import time -from typing import Callable +from typing import Callable, cast from ..exceptions import DeviceDisconnectedError, TimeoutError from ..models import USBDeviceDescriptor @@ -40,8 +40,9 @@ class USBTransport: self._on_disconnect = on_disconnect self._read_chunk_size = read_chunk_size - self._ctx: usb1.USBContext | None = None # type: ignore[valid-type] - self._handle: usb1.USBDeviceHandle | None = None # type: ignore[valid-type] + self._ctx: usb1.USBContext | None = None + # libusb1 binds close() to a zero-argument finalizer on each handle. + self._handle: usb1.USBDeviceHandle | None = None self._rx_thread: threading.Thread | None = None self._stop_event = threading.Event() self._tx_lock = threading.Lock() @@ -78,7 +79,7 @@ class USBTransport: finally: if handle is not None: with suppress(usb1.USBError): - handle.close() + cast(Callable[[], None], handle.close)() devices.append(USBDeviceDescriptor(serial=serial, vendor_id=vid, product_id=pid)) @@ -114,7 +115,7 @@ class USBTransport: finally: if handle is not None and handle is not selected_handle: with suppress(usb1.USBError): - handle.close() + cast(Callable[[], None], handle.close)() if selected_handle is None: serial_msg = f" with serial '{serial}'" if serial else "" @@ -149,7 +150,7 @@ class USBTransport: self.disconnect() raise - def _claim_interface_with_busy_recovery(self, handle: "usb1.USBDeviceHandle") -> None: # type: ignore[name-defined] + def _claim_interface_with_busy_recovery(self, handle: "usb1.USBDeviceHandle") -> None: """Claim the data interface, clearing a stale BUSY claim once if needed. When a previous session's RX thread wedged inside libusb, ``disconnect()`` @@ -163,7 +164,9 @@ class USBTransport: try: handle.claimInterface(self.INTERFACE) return - except usb1.USBErrorBusy as exc: + except usb1.USBError as exc: + if exc.value != -6: # LIBUSB_ERROR_BUSY + raise logger.warning( "USB interface %d busy on claim; resetting device to clear a stale claim: %s", self.INTERFACE, @@ -202,7 +205,7 @@ class USBTransport: with suppress(usb1.USBError): self._handle.releaseInterface(self.INTERFACE) with suppress(usb1.USBError): - self._handle.close() + cast(Callable[[], None], self._handle.close)() self._handle = None if self._ctx is not None: @@ -223,11 +226,9 @@ class USBTransport: with self._tx_lock: try: written = handle.bulkWrite(self.DATA_EP_OUT, data, timeout=timeout_ms) - except usb1.USBErrorTimeout as exc: - raise TimeoutError("Timed out writing USB bulk packet") from exc - except usb1.USBErrorNoDevice as exc: - raise DeviceDisconnectedError("USB device disconnected during write") from exc except usb1.USBError as exc: + if exc.value == -7: # LIBUSB_ERROR_TIMEOUT + raise TimeoutError("Timed out writing USB bulk packet") from exc raise DeviceDisconnectedError(f"USB write failed: {exc}") from exc if written != len(data): @@ -248,16 +249,9 @@ class USBTransport: while not self._stop_event.is_set(): try: data = handle.bulkRead(self.DATA_EP_IN, self._read_chunk_size, timeout=timeout_ms) - except usb1.USBErrorTimeout: - continue - except usb1.USBErrorInterrupted: - continue - except usb1.USBErrorNoDevice as exc: - if self._on_disconnect is not None: - self._on_disconnect(DeviceDisconnectedError(f"USB device disconnected: {exc}")) - logger.warning("USB RX stopped: device disconnected: %s", exc) - return except usb1.USBError as exc: + if exc.value in (-7, -10): # LIBUSB_ERROR_TIMEOUT / INTERRUPTED + continue if self._stop_event.is_set(): return if self._on_disconnect is not None: @@ -294,7 +288,7 @@ class USBTransport: @classmethod def _iter_matching_devices( cls, - ctx: "usb1.USBContext", # type: ignore[name-defined] + ctx: "usb1.USBContext", ): """Yield USB devices matching supported LibreVNA VID/PID pairs.""" for device in ctx.getDeviceList(skip_on_error=True): diff --git a/python_app/hardware_full/librevna_multi_device_driver/__init__.py b/python_app/hardware_full/librevna_multi_device_driver/__init__.py index a9ffd9d..e69de29 100644 --- a/python_app/hardware_full/librevna_multi_device_driver/__init__.py +++ b/python_app/hardware_full/librevna_multi_device_driver/__init__.py @@ -1,23 +0,0 @@ -"""Synchronized multi-device LibreVNA driver.""" - -from python_app.hardware_full.librevna_multi_device_driver.models import SweepConfiguration, SweepMeasurementResult - -__all__ = [ - "MultiDeviceVnaController", - "SweepConfiguration", - "SweepMeasurementResult", - "list_connected_device_serial_numbers", -] - - -def __getattr__(name: str): - """Lazily import native USB pieces only when they are actually needed.""" - if name == "MultiDeviceVnaController": - from python_app.hardware_full.librevna_multi_device_driver.controller import MultiDeviceVnaController - - return MultiDeviceVnaController - if name == "list_connected_device_serial_numbers": - from python_app.hardware_full.librevna_multi_device_driver.transport import list_connected_device_serial_numbers - - return list_connected_device_serial_numbers - raise AttributeError(name) diff --git a/python_app/hardware_full/librevna_multi_device_driver/controller.py b/python_app/hardware_full/librevna_multi_device_driver/controller.py index 879a96b..09dc54c 100644 --- a/python_app/hardware_full/librevna_multi_device_driver/controller.py +++ b/python_app/hardware_full/librevna_multi_device_driver/controller.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Iterator, Sequence +from collections.abc import Sequence from contextlib import suppress from dataclasses import replace from typing import Optional @@ -11,7 +11,7 @@ import threading import time from python_app.hardware_full.librevna_multi_device_driver.cycle_collection import ( - collect_complete_running_sweep_cycles, + collect_complete_running_sweep_cycle, ) from python_app.hardware_full.librevna_multi_device_driver.models import ( SweepConfiguration, @@ -109,13 +109,6 @@ class MultiDeviceVnaController: device_connection.close() logger.debug("Multi-device controller closed") - def stop_continuous_sweep(self) -> None: - """Stop the currently running sweep without closing device transports.""" - if self._is_closed: - return - logger.info("Stopping continuous sweep") - self._send_idle_to_all_devices() - def configure_continuous_sweep( self, sweep_configuration: SweepConfiguration, @@ -179,13 +172,12 @@ class MultiDeviceVnaController: master_stimulus_ports=stimulus_ports, ) - def collect_running_sweep_cycles( + def collect_running_sweep_cycle( self, - cycle_count: int = 1, *, datapoint_timeout_seconds: float | None = None, ) -> SweepMeasurementResult: - """Collect complete cycles from the already-running synchronized sweep.""" + """Collect one complete cycle from the already-running synchronized sweep.""" if self._is_closed: raise RuntimeError("Controller is already closed") if ( @@ -197,11 +189,10 @@ class MultiDeviceVnaController: raise RuntimeError("No running sweep is configured. Call configure_continuous_sweep() first.") try: - return collect_complete_running_sweep_cycles( + return collect_complete_running_sweep_cycle( master_device_connection=self._master_device, slave_device_connections=self._slave_devices, active_sweep_configuration=self._last_applied_sweep_configuration, - cycle_count=cycle_count, master_stimulus_ports=self._last_master_stimulus_ports, switch_substage_count=self._switch_substage_count, datapoint_timeout_seconds=datapoint_timeout_seconds, @@ -211,11 +202,6 @@ class MultiDeviceVnaController: self._send_idle_to_all_devices() raise - def iter_running_sweep_cycles(self) -> Iterator[SweepMeasurementResult]: - """Yield complete sweep cycles forever from the configured sweep.""" - while True: - yield self.collect_running_sweep_cycles(1) - def _send_command_and_wait_for_acknowledgement( self, device_connection: LibreVnaUsbBulkConnection, diff --git a/python_app/hardware_full/librevna_multi_device_driver/cycle_collection.py b/python_app/hardware_full/librevna_multi_device_driver/cycle_collection.py index 58d14bd..545b0e1 100644 --- a/python_app/hardware_full/librevna_multi_device_driver/cycle_collection.py +++ b/python_app/hardware_full/librevna_multi_device_driver/cycle_collection.py @@ -14,7 +14,6 @@ from python_app.hardware_full.librevna_multi_device_driver.exceptions import ( TransientCollectionError, ) from python_app.hardware_full.librevna_multi_device_driver.models import ( - OutputChannel, SweepConfiguration, SweepMeasurementResult, build_output_channels, @@ -52,7 +51,7 @@ _MIN_CYCLE_ALIGNMENT_TOLERANCE_SECONDS = 0.5e-3 # Below this magnitude a master reference value is treated as a corrupt incident # signal rather than a real measurement: dividing by it would explode the -# S-parameter into noise. Mirrors the single-device C++ driver's reference guard. +# S-parameter into noise. Both acquisition drivers reject such references. _MIN_REFERENCE_MAGNITUDE = 1e-12 # A dropped USB frame leaves a single sweep point undelivered. Instead of @@ -76,10 +75,12 @@ class _DeviceCollectionState: points are zero-filled when the S-parameters are computed. """ - __slots__ = ("last_cycle_closed",) + __slots__ = ("last_cycle_closed", "capture_start_ns", "capture_end_ns") def __init__(self) -> None: self.last_cycle_closed = False + self.capture_start_ns = 0 + self.capture_end_ns = 0 class _CrossDeviceCycleSynchronizer: @@ -168,20 +169,16 @@ def _is_transient_collection_error(error: BaseException) -> bool: return isinstance(error, (TransientCollectionError, TimeoutError)) -def collect_complete_running_sweep_cycles( +def collect_complete_running_sweep_cycle( *, master_device_connection: LibreVnaUsbBulkConnection, slave_device_connections: Sequence[LibreVnaUsbBulkConnection], active_sweep_configuration: SweepConfiguration, - cycle_count: int, master_stimulus_ports: Sequence[int], switch_substage_count: int = 1, datapoint_timeout_seconds: float | None = None, ) -> SweepMeasurementResult: - """Collect complete cycles from an already-running synchronized sweep.""" - cycle_count = int(cycle_count) - if cycle_count < 1: - raise ValueError("cycle_count must be >= 1") + """Collect one complete cycle from an already-running synchronized sweep.""" stimulus_ports = normalize_master_stimulus_ports(master_stimulus_ports) # One output per (master port, stimulus switch position). With no per-point # switching this is one output per port, exactly as before. @@ -202,18 +199,14 @@ def collect_complete_running_sweep_cycles( ) else: frequencies_hz = np.array([float(active_sweep_configuration.start_hz)], dtype=np.float64) - # Keyed by 1-based OUTPUT number, not by master port: with per-point switching two - # outputs share a stimulus port but are measured in different stages, so each needs - # its own incident reference. master_reference_measurements_by_output = { - channel.number: np.full((cycle_count, point_count), np.nan + 1j * np.nan, dtype=complex) + channel.index: np.full(point_count, np.nan + 1j * np.nan, dtype=complex) for channel in output_channels } raw_receiver_measurements_by_s_parameter = { - name: np.full((cycle_count, point_count), np.nan + 1j * np.nan, dtype=complex) - for name in build_forward_s_parameter_names( - len(slave_device_connections), output_channels - ) + (input_pos, channel.index): np.full(point_count, np.nan + 1j * np.nan, dtype=complex) + for channel in output_channels + for input_pos in range(2 * len(slave_device_connections)) } collection_errors: list[Exception] = [] @@ -315,12 +308,12 @@ def collect_complete_running_sweep_cycles( ``stop_collection_requested`` is set so the other collector threads stop too. """ datapoints_received = 0 - expected_datapoint_count = cycle_count * point_count + expected_datapoint_count = point_count loop_start_timestamp = time.monotonic() # Tracks the last time we actually accepted a datapoint into the cycle. # Crucially, *not* updated on rejected datapoints — a device that keeps # streaming valid-looking frames the handler ignores (e.g. waiting on - # point_index=0, or after cycle_count has been reached) must still hit + # point_index=0, or after the cycle has closed) must still hit # the per-device timeout and trigger recovery instead of looping forever. last_consumed_timestamp = loop_start_timestamp has_consumed_any_datapoint = False @@ -449,15 +442,14 @@ def collect_complete_running_sweep_cycles( datapoints_received += 1 def build_cycle_tracking_handler( - cycle_aware_handler: Callable[[ParsedVnaDatapoint, int], None], + cycle_aware_handler: Callable[[ParsedVnaDatapoint], None], device_state: _DeviceCollectionState, device_label: str = "device", ) -> Callable[[ParsedVnaDatapoint], bool]: """Wrap a cycle-aware handler with cross-device cycle tracking. Returns a per-datapoint handler that anchors cycle 0 on the first - ``point_index == 0`` seen, advances the cycle counter on each point-index - wrap, drops datapoints past ``cycle_count``, and reports whether each + ``point_index == 0`` seen, closes on the next point-index wrap, and reports whether each datapoint was consumed. """ # In steady state the controller leaves the hardware sweep FREE-RUNNING @@ -468,10 +460,8 @@ def collect_complete_running_sweep_cycles( # the drain can let a wrap slip between devices, that first point 0 is only # accepted once `_CrossDeviceCycleSynchronizer` confirms every device saw # point 0 of the SAME physical sweep; a mismatch rejects the collection for - # a transient retry. From the agreed anchor, each subsequent wrap advances - # the cycle counter normally. + # a transient retry. The next wrap closes the agreed cycle. cycle_tracking_state = { - "current_cycle_index": 0, "synchronized": False, # How many mid-sweep points were thrown away before the anchor was found. # Near zero means the drain landed on a sweep boundary — the case where a @@ -483,11 +473,12 @@ def collect_complete_running_sweep_cycles( def handle_datapoint(parsed_datapoint: ParsedVnaDatapoint) -> bool: """Track the cycle index for one datapoint and dispatch it to the handler. - Returns ``True`` when the datapoint was consumed (within ``cycle_count``) + Returns ``True`` when the datapoint was consumed (within the first cycle) and ``False`` when it was ignored (pre-sync straggler or past the last requested cycle). """ current_point_index = parsed_datapoint.point_index + received_ns = time.monotonic_ns() if not cycle_tracking_state["synchronized"]: if current_point_index != 0: @@ -497,10 +488,11 @@ def collect_complete_running_sweep_cycles( # observed point 0 of the SAME physical sweep; otherwise reject the # whole collection so the caller re-drains and re-collects rather # than pairing a reference and measurement from different sweeps. - if not cycle_synchronizer.confirm_aligned_cycle_zero(time.monotonic()): + if not cycle_synchronizer.confirm_aligned_cycle_zero(received_ns / 1e9): report_cycle_misalignment() return False cycle_tracking_state["synchronized"] = True + device_state.capture_start_ns = device_state.capture_end_ns = received_ns logger.debug( "timing: %s anchored cycle 0 after skipping %d mid-sweep point(s) of %d " "(t=%.1f ms)", @@ -509,53 +501,21 @@ def collect_complete_running_sweep_cycles( point_count, time.monotonic() * 1e3, ) - cycle_aware_handler(parsed_datapoint, 0) + cycle_aware_handler(parsed_datapoint) return True - # A new sweep cycle begins exactly when the point index wraps to 0. - # Keying on ==0 (rather than "the index decreased") means a corrupt - # mid-cycle index that slipped past the in-band gate cannot fabricate a - # spurious wrap and desynchronize the cycle counter. if current_point_index == 0: - cycle_tracking_state["current_cycle_index"] += 1 - logger.debug( - "timing: %s first point of NEXT sweep arrived (cycle -> %d, t=%.1f ms)", - device_label, - cycle_tracking_state["current_cycle_index"], - time.monotonic() * 1e3, - ) - current_cycle_index = cycle_tracking_state["current_cycle_index"] - if current_cycle_index >= cycle_count: - # The sweep just wrapped past the final requested cycle, closing its - # collection window. Any of its points we never received are now - # confirmed dropped frames, not points still in flight, so flag the - # device done and let the collector thread stop immediately. The - # missing points are emitted as zero when the S-parameters are - # computed; delivered points keep their real values. + # A second point 0 closes this cycle; absent points are now losses. device_state.last_cycle_closed = True return False - - cycle_aware_handler(parsed_datapoint, current_cycle_index) - if current_point_index == point_count - 1: - logger.debug( - "timing: %s last point of cycle %d arrived (index=%d, t=%.1f ms)", - device_label, - current_cycle_index, - current_point_index, - time.monotonic() * 1e3, - ) + cycle_aware_handler(parsed_datapoint) + device_state.capture_end_ns = received_ns return True return handle_datapoint - def handle_master_datapoint(parsed_datapoint: ParsedVnaDatapoint, cycle_index: int) -> None: - """Store the master device's frequency, reference, and reflection values. - - Records the sweep-point frequency and, per output channel, the reference - receiver value and the matching reflection into the cycle/point measurement - buffers. Two outputs may share a stimulus port (different switch positions); - they are told apart by the stage tag. - """ + def handle_master_datapoint(parsed_datapoint: ParsedVnaDatapoint) -> None: + """Store the frequency and incident reference for each output stage.""" point_index = parsed_datapoint.point_index frequencies_hz[point_index] = parsed_datapoint.frequency_hz @@ -575,45 +535,20 @@ def collect_complete_running_sweep_cycles( if reference_receiver_value is None: continue - master_reference_measurements_by_output[channel.number][ - cycle_index, + master_reference_measurements_by_output[channel.index][ point_index, ] = reference_receiver_value - s_parameter_name = master_reflection_s_parameter_name(channel) - port_receiver_value = find_receiver_value( - parsed_datapoint.receiver_values_by_description_mask, - stage_index=channel.stage_index, - is_reference_receiver=False, - required_port_number=channel.stimulus_port, - ) - if port_receiver_value is not None: - raw_receiver_measurements_by_s_parameter[s_parameter_name][ - cycle_index, - point_index, - ] = port_receiver_value - def build_slave_datapoint_handler( slave_index: int, device_state: _DeviceCollectionState ) -> Callable[[ParsedVnaDatapoint], bool]: - """Build a cycle-tracking datapoint handler for the given slave device. - - The slave's two receivers map to the virtual receiver ports numbered just - above the outputs (see :func:`slave_receiver_base_port`), producing one - forward S-parameter per output channel. - """ - receiver_base_port = slave_receiver_base_port(slave_index, output_channels) - - def handle_slave_datapoint(parsed_datapoint: ParsedVnaDatapoint, cycle_index: int) -> None: + """Collect both physical receivers of this slave in the configured order.""" + def handle_slave_datapoint(parsed_datapoint: ParsedVnaDatapoint) -> None: """Store this slave's forward receiver values into the measurement buffers.""" point_index = parsed_datapoint.point_index for channel in output_channels: - first_s_parameter_name = f"S{receiver_base_port}{channel.number}" - second_s_parameter_name = f"S{receiver_base_port + 1}{channel.number}" - for receiver_port_number, s_parameter_name in ( - (1, first_s_parameter_name), - (2, second_s_parameter_name), - ): + for receiver_port_number in (1, 2): + combo = (2 * slave_index + receiver_port_number - 1, channel.index) port_receiver_value = find_receiver_value( parsed_datapoint.receiver_values_by_description_mask, stage_index=channel.stage_index, @@ -621,8 +556,7 @@ def collect_complete_running_sweep_cycles( required_port_number=receiver_port_number, ) if port_receiver_value is not None: - raw_receiver_measurements_by_s_parameter[s_parameter_name][ - cycle_index, + raw_receiver_measurements_by_s_parameter[combo][ point_index, ] = port_receiver_value @@ -661,8 +595,7 @@ def collect_complete_running_sweep_cycles( ) logger.debug( - "Collecting %d sweep cycle(s) of %d points from %d device(s) (datapoint_timeout=%.1fs)", - cycle_count, + "Collecting one sweep of %d points from %d device(s) (datapoint_timeout=%.1fs)", point_count, len(all_device_connections), datapoint_timeout_seconds, @@ -737,54 +670,20 @@ def collect_complete_running_sweep_cycles( logger.debug("Sweep cycle collection complete (per-device counts: %s)", datapoint_counts_by_device_serial) return SweepMeasurementResult( frequencies_hz=frequencies_hz, - s_parameters=calculate_last_cycle_s_parameters( + s_parameters=calculate_s_parameters( raw_receiver_measurements_by_s_parameter, master_reference_measurements_by_output, ), + capture_start_ns=master_device_state.capture_start_ns, + capture_end_ns=master_device_state.capture_end_ns, ) -def slave_receiver_base_port(slave_index: int, output_channels: Sequence[OutputChannel]) -> int: - """Return the first virtual receiver port number of one slave device. - - Receivers are numbered directly above the outputs, so the whole topology reads - as one virtual VNA whose first ``N`` ports transmit and whose remaining ports - receive. Numbering them from a fixed 3 instead would make a 4-output setup - collide with itself: the reflection of output 3 and the forward term - ``receiver 3 <- output 3`` would both be called ``S33``. With two outputs this - still yields the historical ports 3..6. - """ - return len(output_channels) + 2 * slave_index + 1 - - -def master_reflection_s_parameter_name(channel: OutputChannel) -> str: - """Return the reflection trace name for one output channel (``S11``, ``S22``, ...).""" - return f"S{channel.number}{channel.number}" - - -def build_forward_s_parameter_names( - slave_device_count: int, output_channels: Sequence[OutputChannel] -) -> list[str]: - """Return S-parameter names for the configured forward multi-device topology.""" - names: list[str] = [] - for channel in output_channels: - names.append(master_reflection_s_parameter_name(channel)) - for slave_index in range(slave_device_count): - receiver_base_port = slave_receiver_base_port(slave_index, output_channels) - names.extend( - [ - f"S{receiver_base_port}{channel.number}", - f"S{receiver_base_port + 1}{channel.number}", - ] - ) - return names - - -def calculate_last_cycle_s_parameters( - raw_receiver_measurements_by_s_parameter: dict[str, np.ndarray], +def calculate_s_parameters( + raw_receiver_measurements_by_s_parameter: dict[tuple[int, int], np.ndarray], master_reference_measurements_by_output: dict[int, np.ndarray], -) -> dict[str, np.ndarray]: - """Convert raw receiver captures from the final sweep cycle into S-parameters. +) -> dict[tuple[int, int], np.ndarray]: + """Convert raw receiver captures from the sweep cycle into S-parameters. Points whose value never arrived (a dropped USB frame, still ``NaN``) or whose incident reference is unusable (missing or a corrupt ~zero magnitude) cannot be @@ -794,45 +693,40 @@ def calculate_last_cycle_s_parameters( genuine fault rather than a stray dropped frame, and the cycle is rejected as transient so the caller recovers instead of returning a mostly-zero trace. """ - s_parameters: dict[str, np.ndarray] = {} - zero_filled_points_by_trace: dict[str, int] = {} + s_parameters: dict[tuple[int, int], np.ndarray] = {} + zero_filled_points_by_trace: dict[tuple[int, int], int] = {} point_count = 0 - for s_parameter_name, raw_receiver_measurements in raw_receiver_measurements_by_s_parameter.items(): - # The trailing digit of every name is the 1-based OUTPUT number, which is what - # selects the incident reference (see build_forward_s_parameter_names). - output_number = int(s_parameter_name[-1]) - master_reference_measurements = master_reference_measurements_by_output[output_number] + for combo, raw_receiver_measurements in raw_receiver_measurements_by_s_parameter.items(): + master_reference_measurements = master_reference_measurements_by_output[combo[1]] - last_cycle_reference = master_reference_measurements[-1] - last_cycle_measurement = raw_receiver_measurements[-1] - point_count = last_cycle_measurement.size + point_count = raw_receiver_measurements.size # Without a valid incident reference no trace at that point can be # normalized; without this trace's receiver value there is nothing to # normalize. Either way the point is unmeasurable and emitted as zero. - reference_unusable_mask = np.isnan(last_cycle_reference) | ( - np.abs(last_cycle_reference) <= _MIN_REFERENCE_MAGNITUDE + reference_unusable_mask = ~np.isfinite(master_reference_measurements) | ( + np.abs(master_reference_measurements) <= _MIN_REFERENCE_MAGNITUDE ) - measurement_missing_mask = np.isnan(last_cycle_measurement) + measurement_missing_mask = ~np.isfinite(raw_receiver_measurements) zero_fill_mask = reference_unusable_mask | measurement_missing_mask zero_filled_count = int(zero_fill_mask.sum()) if point_count and zero_filled_count > _MAX_ZERO_FILLED_POINTS_FRACTION * point_count: raise TransientCollectionError( - f"Trace {s_parameter_name} is unmeasurable at {zero_filled_count}/{point_count} " + f"Trace {combo} is unmeasurable at {zero_filled_count}/{point_count} " f"points (missing reference/measurement); rejecting cycle for recovery" ) # Substitute neutral operands so the division never produces NaN/inf, then # force the unmeasurable points to zero. Delivered points are untouched. - safe_reference = np.where(reference_unusable_mask, 1.0 + 0j, last_cycle_reference) - safe_measurement = np.where(measurement_missing_mask, 0.0 + 0j, last_cycle_measurement) + safe_reference = np.where(reference_unusable_mask, 1.0 + 0j, master_reference_measurements) + safe_measurement = np.where(measurement_missing_mask, 0.0 + 0j, raw_receiver_measurements) s_parameter_values = safe_measurement / safe_reference s_parameter_values[zero_fill_mask] = 0.0 if zero_filled_count: - zero_filled_points_by_trace[s_parameter_name] = zero_filled_count - s_parameters[s_parameter_name.lower()] = s_parameter_values + zero_filled_points_by_trace[combo] = zero_filled_count + s_parameters[combo] = s_parameter_values if zero_filled_points_by_trace: # One concise line per affected cycle (a rare dropped-frame event), so the diff --git a/python_app/hardware_full/librevna_multi_device_driver/models.py b/python_app/hardware_full/librevna_multi_device_driver/models.py index 754a04c..172b6cc 100644 --- a/python_app/hardware_full/librevna_multi_device_driver/models.py +++ b/python_app/hardware_full/librevna_multi_device_driver/models.py @@ -62,10 +62,6 @@ class OutputChannel: stimulus_port: int stage_index: int - @property - def number(self) -> int: - """Return the 1-based output number used in S-parameter names.""" - return self.index + 1 def build_output_channels( @@ -129,7 +125,9 @@ class SweepConfiguration: @dataclass(slots=True) class SweepMeasurementResult: - """Measured synchronized sweep for one active master output port.""" + """S21 traces keyed by zero-based (input, output) for one synchronized sweep.""" frequencies_hz: np.ndarray - s_parameters: dict[str, np.ndarray] + s_parameters: dict[tuple[int, int], np.ndarray] + capture_start_ns: int + capture_end_ns: int diff --git a/python_app/hardware_full/librevna_service.py b/python_app/hardware_full/librevna_service.py index 5084549..b835265 100644 --- a/python_app/hardware_full/librevna_service.py +++ b/python_app/hardware_full/librevna_service.py @@ -1,46 +1,115 @@ -"""Native single-device LibreVNA service used by acquisition and device-limit queries.""" +"""Single-device S21 acquisition over the LibreVNA USB protocol.""" from __future__ import annotations from dataclasses import dataclass, field +from contextlib import suppress +import time -from python_app.hardware_full.librevna_backends import NativeLibreVnaBackend +from python_app.hardware_full.librevna_driver import LibreVNADevice, VNASweepSettings from python_app.hardware_full.librevna_driver.models import SweepResult -from python_app.models.run_config_model import RadarSweepModel +from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData +from python_app.models.run_config_schema import RadarSweepModel @dataclass(slots=True) class LibreVnaService: - """Provide the native single-device API; factories select mock acquisition.""" + """Native backend using direct USB LibreVNA Python driver.""" serial: str | None = None strict_protocol_version: int = 14 - backend_mode: str = "auto" - _backend: NativeLibreVnaBackend = field(init=False, repr=False) - - def __post_init__(self) -> None: - if self.backend_mode.strip().lower() not in {"auto", "native"}: - raise ValueError(f"Unsupported native LibreVNA backend mode: {self.backend_mode}") - self._backend = NativeLibreVnaBackend(self.serial, self.strict_protocol_version) + _device: LibreVNADevice | None = field(init=False, default=None, repr=False) + _settings: RadarSweepModel | None = field(init=False, default=None, repr=False) def open(self) -> None: - self._backend.open() + """Open device connection if it is not already open.""" + if self._device is not None: + return + + device = self._device = LibreVNADevice() + try: + device.connect(serial=self.serial, strict_protocol_version=self.strict_protocol_version, timeout_s=2.0) + except Exception: + with suppress(Exception): + self.close() + raise + + @property + def is_open(self) -> bool: + """Return `True` when native device connection is open.""" + return self._device is not None def close(self) -> None: - self._backend.close() + """Close device connection when open.""" + if self._device is None: + return + device, self._device = self._device, None + device.disconnect() def configure(self, sweep: RadarSweepModel) -> None: - self._backend.configure(sweep) + """Store and apply sweep settings to connected hardware.""" + self._settings = sweep + if self._device is None: + return + + settings = VNASweepSettings( + f_start_hz=sweep.start_hz, + f_stop_hz=sweep.stop_hz, + points=sweep.points, + if_bandwidth_hz=sweep.if_bandwidth_hz, + power_start_dbm=sweep.power_dbm, + power_stop_dbm=sweep.power_dbm, + dwell_s=sweep.dwell_us / 1_000_000, + ) + self._device.vna.configure(settings) def read_device_limits(self) -> dict[str, float | int]: - opened_here = not self._backend.is_open + """Read frequency/IFBW/power/points limits from connected device.""" + opened_here = not self.is_open try: if opened_here: self.open() - return self._backend.read_device_limits() + assert self._device is not None + limits = self._device.get_device_info().limits + return { + "min_frequency_hz": float(limits.min_frequency_hz), + "max_frequency_hz": float(limits.max_frequency_hz), + "min_ifbw_hz": float(limits.min_ifbw_hz), + "max_ifbw_hz": float(limits.max_ifbw_hz), + "max_points": int(limits.max_points), + "min_power_dbm": float(limits.min_power_dbm), + "max_power_dbm": float(limits.max_power_dbm), + } finally: if opened_here: self.close() + def prepare_combo(self, combo: ComboKey) -> None: + """Standby acquisition starts a fresh sweep after the physical switch.""" + def acquire(self) -> SweepResult: - return self._backend.acquire() + """Acquire one sweep from hardware.""" + if self._settings is None: + raise RuntimeError("Radar service is not configured") + if self._device is None: + raise RuntimeError("Device not found") + + capture_start_ns = time.monotonic_ns() + result = self._device.vna.acquire() + result.capture_start_ns = capture_start_ns + result.capture_end_ns = time.monotonic_ns() + return result + + def acquire_collection(self, collection_id: int = 1, *, combo: ComboKey | None = None) -> SweepCollection: + """Expose one physical S21 path to the shared switch collector.""" + result = self.acquire() + return SweepCollection( + collection_id=collection_id, + monotonic_ns=result.capture_end_ns, + traces=[TraceData( + ComboKey(0, 0), result.x, result.trace("s21"), + result.capture_start_ns, result.capture_end_ns, + )], + capture_start_ns=result.capture_start_ns, + capture_end_ns=result.capture_end_ns, + ) diff --git a/python_app/hardware_full/matrix_radar_service.py b/python_app/hardware_full/matrix_radar_service.py index 7066c86..2d037e4 100644 --- a/python_app/hardware_full/matrix_radar_service.py +++ b/python_app/hardware_full/matrix_radar_service.py @@ -1,11 +1,11 @@ -"""Factory for synchronized LibreVNA matrices and the shared physical mock.""" +"""LibreVNA acquisition factory for single devices, matrices, and mock scenes.""" from __future__ import annotations from typing import Protocol -from python_app.models.dataset_model import SweepCollection -from python_app.models.run_config_model import RadarSweepModel, RunConfigModel +from python_app.models.dataset_model import ComboKey, SweepCollection +from python_app.models.run_config_schema import RadarSweepModel, RunConfigModel class MatrixRadarService(Protocol): @@ -20,37 +20,46 @@ class MatrixRadarService(Protocol): def configure(self, sweep: RadarSweepModel) -> None: """Apply sweep settings to the radar.""" - def acquire_collection(self, collection_id: int = 1) -> SweepCollection: - """Acquire one complete virtual switch matrix in canonical combo order.""" + def acquire_collection(self, collection_id: int = 1, *, combo: ComboKey | None = None) -> SweepCollection: + """Acquire the matrix, or the smallest physical sweep containing combo.""" ... - def recover(self) -> None: - """Reconnect to the radar after a transient acquisition failure.""" - - -def create_acquisition_service(config: RunConfigModel) -> MatrixRadarService: - """Create a native matrix service or the shared single/multi mock scene.""" +def create_matrix_radar_service(config: RunConfigModel) -> MatrixRadarService: + """Create the configured LibreVNA collection source and its physical switches.""" if config.radar.driver_mode == "mock" and config.radar.model in { RunConfigModel.LIBREVNA_MODEL, RunConfigModel.LIBREVNA_MULTI_MODEL, }: from python_app.hardware_full.point_target_mock_service import PointTargetMockService return PointTargetMockService(config) + if config.radar.model == RunConfigModel.LIBREVNA_MODEL: + from python_app.hardware_full.librevna_service import LibreVnaService + from python_app.hardware_full.switch_service import SwitchService + from python_app.hardware_full.switched_matrix_radar_service import SwitchedMatrixRadarService + + return SwitchedMatrixRadarService( + inner=LibreVnaService(serial=config.radar.serial or None), + output_switch=SwitchService.from_model(config.output_switch), + input_switch=SwitchService.from_model(config.input_switch), + inner_output_positions=1, + inner_input_positions=1, + settling_ms=config.runtime.settling_ms, + combos=[ComboKey(c.input, c.output) for c in (config.combos or config.build_runtime_combos())], + ) if config.radar.model != RunConfigModel.LIBREVNA_MULTI_MODEL: raise RuntimeError("matrix radar service factory requires a matrix-mode radar.model") from python_app.hardware_full.multi_device_service import MultiDeviceLibreVnaService + out_physical = config.matrix_output_switch_positions + in_physical = config.matrix_input_switch_positions inner = MultiDeviceLibreVnaService( master_serial=config.radar.serial, slave_serials=list(config.radar.multi_device.slave_serials), force_external_reference=config.radar.multi_device.force_external_reference, recovery_attempts=config.radar.multi_device.recovery_attempts, - backend_mode=config.radar.driver_mode, switch_substage_count=config.matrix_stimulus_switch_positions, ) - out_physical = config.matrix_output_switch_positions - in_physical = config.matrix_input_switch_positions if out_physical <= 1 and in_physical <= 1: return inner @@ -67,6 +76,3 @@ def create_acquisition_service(config: RunConfigModel) -> MatrixRadarService: inner_input_positions=RunConfigModel.MULTI_DEVICE_INPUT_POSITIONS, settling_ms=config.runtime.settling_ms, ) - - -create_matrix_radar_service = create_acquisition_service diff --git a/python_app/hardware_full/multi_device_service.py b/python_app/hardware_full/multi_device_service.py index 9bf9247..1750315 100644 --- a/python_app/hardware_full/multi_device_service.py +++ b/python_app/hardware_full/multi_device_service.py @@ -17,7 +17,7 @@ from python_app.hardware_full.librevna_multi_device_driver.models import ( normalize_switch_substage_count, ) from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData -from python_app.models.run_config_model import RadarSweepModel, RunConfigModel +from python_app.models.run_config_schema import RadarSweepModel, RunConfigModel if TYPE_CHECKING: from python_app.hardware_full.librevna_multi_device_driver.controller import MultiDeviceVnaController @@ -45,34 +45,6 @@ _MAX_RECOVERY_WALL_SECONDS: float = 10.0 # previously caused the periodic freeze. _MAX_TRANSIENT_COLLECTION_RETRIES: int = 2 -def _build_s_parameter_layout( - output_count: int, slave_count: int -) -> tuple[dict[int, tuple[str, ...]], dict[int, str]]: - """Return the (forward, reflection) trace names for each output position. - - Mirrors the naming in ``cycle_collection``: outputs are virtual ports 1..N and - the slaves' receivers are numbered directly above them, so a 2-output setup - keeps the historical ``s31``/``s41``/``s51``/``s61`` layout while a 4-output one - (per-point stimulus switching) shifts the receivers to ports 5..8 to stay - collision-free. - """ - receiver_ports = [ - output_count + 2 * slave_index + port_offset - for slave_index in range(slave_count) - for port_offset in (1, 2) - ] - forward_by_output = { - output_index: tuple( - f"s{receiver_port}{output_index + 1}" for receiver_port in receiver_ports - ) - for output_index in range(output_count) - } - reflection_by_output = { - output_index: f"s{output_index + 1}{output_index + 1}" - for output_index in range(output_count) - } - return forward_by_output, reflection_by_output - @dataclass(slots=True) class MultiDeviceLibreVnaService: """Acquire a virtual 2x4 switch matrix from synchronized LibreVNA devices.""" @@ -81,7 +53,6 @@ class MultiDeviceLibreVnaService: slave_serials: list[str] force_external_reference: bool = True recovery_attempts: int = 3 - backend_mode: str = "auto" # Stimulus switch positions stepped through WITHIN one frequency point. 1 keeps the # stock 2-output matrix; 2 doubles the outputs (4) without a second sweep, at the # cost of requiring the modified master bitstream and the trigger-loop controller. @@ -97,10 +68,6 @@ class MultiDeviceLibreVnaService: raise ValueError("LibreVNA multi-device mode requires exactly two slave serial numbers") self.recovery_attempts = max(0, int(self.recovery_attempts)) - mode = self.backend_mode.strip().lower() - if mode not in {"auto", "native"}: - raise ValueError(f"Unsupported multi-device backend mode: {self.backend_mode}") - self.backend_mode = mode self.switch_substage_count = normalize_switch_substage_count(self.switch_substage_count) @property @@ -126,7 +93,7 @@ class MultiDeviceLibreVnaService: self.master_serial, self.slave_serials, ) except Exception as exc: - logger.warning("Multi-device open failed (backend_mode=%s): %s", self.backend_mode, exc) + logger.warning("Multi-device open failed: %s", exc) raise def close(self) -> None: @@ -234,6 +201,7 @@ class MultiDeviceLibreVnaService: self, collection_id: int = 1, *, + combo: ComboKey | None = None, stop_event: threading.Event | None = None, ) -> SweepCollection: """Acquire one complete virtual 2x4 matrix in canonical combo order. @@ -244,17 +212,11 @@ class MultiDeviceLibreVnaService: """ if self._sweep_configuration is None: raise RuntimeError("Multi-device service is not configured") - capture_start_ns = time.monotonic_ns() - collection = self._acquire_native_collection_with_recovery( - collection_id, capture_start_ns, stop_event=stop_event - ) - collection.capture_end_ns = time.monotonic_ns() - return collection + return self._acquire_native_collection_with_recovery(collection_id, stop_event=stop_event) def _acquire_native_collection_with_recovery( self, collection_id: int, - capture_start_ns: int, *, stop_event: threading.Event | None = None, ) -> SweepCollection: @@ -265,7 +227,7 @@ class MultiDeviceLibreVnaService: for attempt_index in range(self.recovery_attempts + 1): try: return self._acquire_native_collection_with_transient_retries( - collection_id, capture_start_ns, stop_event=stop_event + collection_id, stop_event=stop_event ) except Exception as exc: # noqa: BLE001 last_error = exc @@ -311,7 +273,6 @@ class MultiDeviceLibreVnaService: def _acquire_native_collection_with_transient_retries( self, collection_id: int, - capture_start_ns: int, *, stop_event: threading.Event | None = None, ) -> SweepCollection: @@ -328,7 +289,7 @@ class MultiDeviceLibreVnaService: transient_error: TransientCollectionError | None = None for transient_attempt in range(_MAX_TRANSIENT_COLLECTION_RETRIES + 1): try: - return self._acquire_native_collection(collection_id, capture_start_ns) + return self._acquire_native_collection(collection_id) except TransientCollectionError as exc: transient_error = exc if transient_attempt >= _MAX_TRANSIENT_COLLECTION_RETRIES: @@ -345,62 +306,32 @@ class MultiDeviceLibreVnaService: assert transient_error is not None # loop either returns or raises raise transient_error - def _acquire_native_collection(self, collection_id: int, capture_start_ns: int) -> SweepCollection: + def _acquire_native_collection(self, collection_id: int) -> SweepCollection: if self._controller is None: raise RuntimeError("Multi-device controller is not open") assert self._sweep_configuration is not None self._controller.configure_continuous_sweep(self._sweep_configuration) - # Bound the sweep itself rather than reusing `capture_start_ns`: the latter - # is taken before any retry/recovery, so it would overstate how long the - # traces below took to measure. - sweep_start_ns = time.monotonic_ns() - result = self._controller.collect_running_sweep_cycles( - 1, + result = self._controller.collect_running_sweep_cycle( datapoint_timeout_seconds=LIBREVNA_NATIVE_SWEEP_TIMEOUT_SECONDS, ) - sweep_end_ns = time.monotonic_ns() - normalized_s_parameters = { - str(name).lower(): np.asarray(values, dtype=np.complex64) - for name, values in result.s_parameters.items() - } frequencies = np.asarray(result.frequencies_hz, dtype=np.float32) - - forward_by_output, reflection_by_output = _build_s_parameter_layout( - self.output_position_count, len(self.slave_serials) - ) - traces: list[TraceData] = [] - for output_pos in range(self.output_position_count): - reflection = self._required_s_parameter( - normalized_s_parameters, - reflection_by_output[output_pos], + traces = [ + TraceData( + combo=ComboKey(input_pos, output_pos), + frequency_hz=frequencies, + s21=np.asarray(result.s_parameters[input_pos, output_pos], dtype=np.complex64), + capture_start_ns=result.capture_start_ns, + capture_end_ns=result.capture_end_ns, ) - - for input_pos, s_parameter_name in enumerate(forward_by_output[output_pos]): - traces.append( - TraceData( - combo=ComboKey(input=input_pos, output=output_pos), - frequency_hz=frequencies, - s11=reflection, - s21=self._required_s_parameter(normalized_s_parameters, s_parameter_name), - # Every combo comes out of the same synchronized cycle, so - # they all share one window — no combo was measured earlier - # or later than another here. - capture_start_ns=sweep_start_ns, - capture_end_ns=sweep_end_ns, - ) - ) + for output_pos in range(self.output_position_count) + for input_pos in range(2 * len(self.slave_serials)) + ] return SweepCollection( collection_id=int(collection_id), monotonic_ns=time.monotonic_ns(), traces=traces, - capture_start_ns=capture_start_ns, + capture_start_ns=result.capture_start_ns, + capture_end_ns=result.capture_end_ns, ) - - @staticmethod - def _required_s_parameter(s_parameters: dict[str, np.ndarray], name: str) -> np.ndarray: - values = s_parameters.get(name) - if values is None: - raise RuntimeError(f"Multi-device sweep is missing required {name.upper()} trace") - return values diff --git a/python_app/hardware_full/point_target_mock_service.py b/python_app/hardware_full/point_target_mock_service.py index 4b5bab0..be812a8 100644 --- a/python_app/hardware_full/point_target_mock_service.py +++ b/python_app/hardware_full/point_target_mock_service.py @@ -9,7 +9,7 @@ import numpy as np from python_app.hardware_full.librevna_driver.models import SweepResult from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData -from python_app.models.run_config_model import RadarSweepModel, RunConfigModel +from python_app.models.run_config_schema import RadarSweepModel, RunConfigModel _WAVE_SPEED_M_S = 3e8 _FRAME_PERIOD_S = 0.05 @@ -46,9 +46,6 @@ class PointTargetMockService: def close(self) -> None: """Mock acquisition owns no hardware resources.""" - def recover(self) -> None: - """Mock acquisition has no hardware connection to recover.""" - def read_device_limits(self) -> dict[str, float | int]: raise RuntimeError("Mock radar has no hardware device limits") @@ -56,7 +53,7 @@ class PointTargetMockService: self._frequency_hz = np.linspace(sweep.start_hz, sweep.stop_hz, sweep.points, dtype=np.float32) self._phase_per_m = -2j * np.pi * self._frequency_hz.astype(np.float64) / _WAVE_SPEED_M_S - def set_active_combo(self, combo: ComboKey) -> None: + def prepare_combo(self, combo: ComboKey) -> None: if not (0 <= combo.input < self.config.input_switch.positions and 0 <= combo.output < self.config.output_switch.positions): raise ValueError(f"Mock combo is outside the antenna matrix: {combo}") @@ -66,18 +63,14 @@ class PointTargetMockService: collection = self._capture((self._active_combo,), 1) trace = collection.traces[0] return SweepResult( - x=trace.frequency_hz, traces={"s11": trace.s11, "s21": trace.s21}, + x=trace.frequency_hz, traces={"s21": trace.s21}, capture_start_ns=trace.capture_start_ns, capture_end_ns=trace.capture_end_ns, ) - def acquire_collection(self, collection_id: int = 1) -> SweepCollection: - return self._capture(self._combos, collection_id) - - def acquire_combo_collection( - self, *, input_pos: int, output_pos: int, collection_id: int = 1, - ) -> SweepCollection: - self.set_active_combo(ComboKey(input_pos, output_pos)) - return self._capture((self._active_combo,), collection_id) + def acquire_collection(self, collection_id: int = 1, *, combo: ComboKey | None = None) -> SweepCollection: + if combo is not None: + self.prepare_combo(combo) + return self._capture(self._combos if combo is None else (combo,), collection_id) def _target_position( self, sample_ns: int, @@ -107,7 +100,6 @@ class PointTargetMockService: r_rx = float(np.linalg.norm(target - rx.get(combo.input, (0.0, 0.0, 0.0)))) traces.append(TraceData( combo=combo, frequency_hz=self._frequency_hz, - s11=np.asarray(0.5 / max(1.0, r_tx * r_tx) * np.exp(self._phase_per_m * 2 * r_tx), dtype=np.complex64), s21=np.asarray(0.5 / max(1.0, r_tx * r_rx) * np.exp(self._phase_per_m * (r_tx + r_rx)), dtype=np.complex64), capture_start_ns=capture_start_ns, )) diff --git a/python_app/hardware_full/single_radar_service.py b/python_app/hardware_full/single_radar_service.py index 9902851..5d688f0 100644 --- a/python_app/hardware_full/single_radar_service.py +++ b/python_app/hardware_full/single_radar_service.py @@ -8,7 +8,8 @@ from typing import Protocol from python_app.hardware_full.kamil_adc import KamilAdcService from python_app.hardware_full.librevna_driver.models import SweepResult from python_app.hardware_full.librevna_service import LibreVnaService -from python_app.models.run_config_model import RadarSweepModel, RunConfigModel +from python_app.models.dataset_model import ComboKey +from python_app.models.run_config_schema import RadarSweepModel, RunConfigModel logger = logging.getLogger(__name__) @@ -29,6 +30,9 @@ class SingleRadarService(Protocol): """Read device capability limits.""" ... + def prepare_combo(self, combo: ComboKey) -> None: + """Select the simulated path or discard buffered data after switching.""" + def acquire(self) -> SweepResult: """Acquire one sweep.""" ... @@ -36,7 +40,7 @@ class SingleRadarService(Protocol): def create_single_radar_service(config: RunConfigModel) -> SingleRadarService: """Create the Python service for a non-matrix-radar config.""" - if config.is_matrix_radar: + if config.is_multi_device: raise RuntimeError( "single-radar service factory does not support matrix radars; " "use create_matrix_radar_service instead" @@ -49,7 +53,7 @@ def create_single_radar_service(config: RunConfigModel) -> SingleRadarService: from python_app.hardware_full.point_target_mock_service import PointTargetMockService return PointTargetMockService(config) - return LibreVnaService(serial=config.radar.serial or None, backend_mode=config.radar.driver_mode) + return LibreVnaService(serial=config.radar.serial or None) if model == RunConfigModel.KAMIL_ADC_MODEL: if config.radar.driver_mode != "native": diff --git a/python_app/hardware_full/switch_drivers/__init__.py b/python_app/hardware_full/switch_drivers/__init__.py index a5804be..651b8da 100644 --- a/python_app/hardware_full/switch_drivers/__init__.py +++ b/python_app/hardware_full/switch_drivers/__init__.py @@ -1,13 +1,11 @@ """Switch driver implementations for native GPIO and mock operation.""" from python_app.hardware_full.switch_drivers.h7992_driver import H7992Driver -from python_app.hardware_full.switch_drivers.hmc349a_driver import HMC349ADriver from python_app.hardware_full.switch_drivers.interface import SwitchDriverProtocol from python_app.hardware_full.switch_drivers.mock_driver import MockSwitchDriver __all__ = [ "H7992Driver", - "HMC349ADriver", "MockSwitchDriver", "SwitchDriverProtocol", ] diff --git a/python_app/hardware_full/switch_drivers/hmc349a_driver.py b/python_app/hardware_full/switch_drivers/hmc349a_driver.py deleted file mode 100644 index 9b254b6..0000000 --- a/python_app/hardware_full/switch_drivers/hmc349a_driver.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Native GPIO driver for HMC349A switch.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from contextlib import suppress - -from python_app.hardware_full.switch_drivers.gpio_uapi import GpioOutputLines - - -@dataclass(slots=True) -class HMC349ADriver: - """Drive HMC349A using one control GPIO line.""" - - name: str - positions: int = 2 - default_position: int = 0 - gpio_chip: str = "/dev/gpiochip0" - pin_a: int = 17 - invert_logic: bool = False - _lines: GpioOutputLines | None = field(init=False, default=None, repr=False) - _is_open: bool = field(init=False, default=False, repr=False) - _current_position: int = field(init=False, default=0, repr=False) - - def open(self) -> None: - """Open GPIO lines and switch to default position.""" - if self._is_open: - return - - self._validate() - - offsets = [self.pin_a] - self._lines = GpioOutputLines( - chip=self.gpio_chip, - offsets=offsets, - consumer=f"radar_{self.name}", - ) - try: - self._lines.open() - self._is_open = True - self.switch_to(self.default_position) - except Exception: - with suppress(Exception): - self.close() - raise - - def close(self) -> None: - """Close GPIO lines.""" - lines, self._lines = self._lines, None - self._is_open = False - if lines is not None: - lines.close() - - def position_count(self) -> int: - """Return number of supported positions.""" - return self.positions - - def switch_to(self, position: int) -> None: - """Switch hardware to selected position.""" - if not self._is_open or self._lines is None: - raise RuntimeError(f"Switch driver is not open for {self.name}") - if position < 0 or position >= self.positions: - raise ValueError(f"Position out of range for {self.name}: {position}") - - control = int(position & 0x01) - if self.invert_logic: - control ^= 0x01 - - self._lines.set_values([control]) - self._current_position = int(position) - - @property - def current_position(self) -> int: - """Return current position.""" - return self._current_position - - def _validate(self) -> None: - """Validate HMC349A driver parameters.""" - if self.positions <= 0 or self.positions > 2: - raise ValueError(f"HMC349A positions must be in range [1,2] for {self.name}") - if self.default_position < 0 or self.default_position >= self.positions: - raise ValueError(f"default_position out of range for {self.name}") - if self.pin_a < 0: - raise ValueError(f"pin_a is invalid for {self.name}") diff --git a/python_app/hardware_full/switch_drivers/interface.py b/python_app/hardware_full/switch_drivers/interface.py index 50c14f5..18cff1c 100644 --- a/python_app/hardware_full/switch_drivers/interface.py +++ b/python_app/hardware_full/switch_drivers/interface.py @@ -16,6 +16,7 @@ class SwitchDriverProtocol(Protocol): def position_count(self) -> int: """Return total number of supported positions.""" + ... def switch_to(self, position: int) -> None: """Switch to requested zero-based position.""" @@ -23,3 +24,4 @@ class SwitchDriverProtocol(Protocol): @property def current_position(self) -> int: """Return currently active position.""" + ... diff --git a/python_app/hardware_full/switch_service.py b/python_app/hardware_full/switch_service.py index ce2c39b..c99068f 100644 --- a/python_app/hardware_full/switch_service.py +++ b/python_app/hardware_full/switch_service.py @@ -3,11 +3,12 @@ from __future__ import annotations from dataclasses import dataclass, field +from collections.abc import Callable +import time from typing import TYPE_CHECKING from python_app.hardware_full.switch_drivers import ( H7992Driver, - HMC349ADriver, MockSwitchDriver, SwitchDriverProtocol, ) @@ -23,11 +24,9 @@ class SwitchService: name: str positions: int mode: str = "mock" - driver: str = "h7992" gpio_chip: str = "/dev/gpiochip0" pin_a: int = 17 pin_b: int = 27 - invert_logic: bool = False default_position: int = 0 _driver: SwitchDriverProtocol = field(init=False, repr=False) @@ -46,11 +45,9 @@ class SwitchService: name=model.name, positions=model.positions, mode=model.driver_mode, - driver=model.driver, gpio_chip=model.gpio_chip, pin_a=model.pin_a, pin_b=model.pin_b, - invert_logic=model.invert_logic, default_position=model.default_position, ) @@ -78,7 +75,6 @@ class SwitchService: def _build_driver(self) -> SwitchDriverProtocol: """Instantiate concrete driver according to configured mode and driver kind.""" mode = self.mode.strip().lower() - driver_kind = self.driver.strip().lower() if mode == "mock": return MockSwitchDriver( @@ -90,23 +86,23 @@ class SwitchService: if mode != "native": raise RuntimeError(f"Unsupported switch mode: {self.mode}") - if driver_kind == "h7992": - return H7992Driver( - name=self.name, - positions=self.positions, - default_position=self.default_position, - gpio_chip=self.gpio_chip, - pin_a=self.pin_a, - pin_b=self.pin_b, - ) - if driver_kind == "hmc349a": - return HMC349ADriver( - name=self.name, - positions=self.positions, - default_position=self.default_position, - gpio_chip=self.gpio_chip, - pin_a=self.pin_a, - invert_logic=self.invert_logic, - ) + return H7992Driver( + name=self.name, + positions=self.positions, + default_position=self.default_position, + gpio_chip=self.gpio_chip, + pin_a=self.pin_a, + pin_b=self.pin_b, + ) - raise RuntimeError(f"Unsupported switch driver: {self.driver}") + +def select_and_settle( + output_switch: SwitchDriverProtocol | None, input_switch: SwitchDriverProtocol | None, + output: int, input: int, settling_ms: int, *, wait: Callable[[float], object] = time.sleep, +) -> bool: + """Drive both switches and settle; return False when an interruptible wait stops.""" + if output_switch is not None: + output_switch.switch_to(output) + if input_switch is not None: + input_switch.switch_to(input) + return not (settling_ms > 0 and wait(settling_ms / 1000.0)) diff --git a/python_app/hardware_full/switched_matrix_radar_service.py b/python_app/hardware_full/switched_matrix_radar_service.py index fdbc546..af9aaff 100644 --- a/python_app/hardware_full/switched_matrix_radar_service.py +++ b/python_app/hardware_full/switched_matrix_radar_service.py @@ -7,9 +7,10 @@ import logging import time from python_app.hardware_full.matrix_radar_service import MatrixRadarService -from python_app.hardware_full.switch_service import SwitchService +from python_app.hardware_full.switch_service import SwitchService, select_and_settle +from python_app.hardware_full.switch_drivers.interface import SwitchDriverProtocol from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData -from python_app.models.run_config_model import RadarSweepModel, SwitchModel +from python_app.models.run_config_schema import RadarSweepModel, SwitchModel logger = logging.getLogger(__name__) @@ -20,17 +21,17 @@ class SwitchedMatrixRadarService: Implements the ``MatrixRadarService`` protocol, so the producer and the capture workflows treat it as an ordinary matrix radar that simply reports - more positions. The hardware sweep is never stopped: switches are only ever - driven BETWEEN ``acquire_collection`` calls, and the inner service's - free-running collection discards any partially swept cycle. + more positions. Switches settle before each inner acquisition. A free-running + matrix discards the partial cycle; a single device starts one standby sweep. """ inner: MatrixRadarService - output_switch: SwitchService | None - input_switch: SwitchService | None + output_switch: SwitchDriverProtocol | None + input_switch: SwitchDriverProtocol | None inner_output_positions: int inner_input_positions: int settling_ms: int = 0 + combos: list[ComboKey] | None = None # Monotonic end of the previous inner collection, so the DEBUG timing trace can # report how long the gap between "sweep collected" and "switch driven" really is # — that gap is where a stale in-flight point 0 can still slip past the drain. @@ -61,85 +62,50 @@ class SwitchedMatrixRadarService: """Apply sweep settings to the inner radar.""" self.inner.configure(sweep) - def recover(self) -> None: - """Reconnect the inner radar; switches are not on the USB transport.""" - self.inner.recover() - - def acquire_collection(self, collection_id: int = 1) -> SweepCollection: - """Acquire the full widened matrix, one inner collection per switch step. + def acquire_collection(self, collection_id: int = 1, *, combo: ComboKey | None = None) -> SweepCollection: + """Acquire the matrix, or only the physical switch step containing combo. A partial failure raises instead of returning a short collection: the preprocessor requires every runtime combo to be present, so half a matrix is worse than a dropped frame. """ capture_start_ns = time.monotonic_ns() - out_steps = self.output_switch.position_count() if self.output_switch is not None else 1 - in_steps = self.input_switch.position_count() if self.input_switch is not None else 1 - total_inputs = in_steps * self.inner_input_positions - total_outputs = out_steps * self.inner_output_positions + total_inputs, total_outputs = self._shape() - # Place each trace at its canonical index rather than appending. The GPR stage - # rejects a collection whose trace order differs from run.combos, and run.combos - # is built output-major (`build_full_combos`) while these loops run switch-major. - # Appending happens to agree for an output switch and to disagree for an input one. - slots: list[TraceData | None] = [None] * (total_inputs * total_outputs) - - for out_k in range(out_steps): - for in_k in range(in_steps): - for trace in self._acquire_step_traces(out_k, in_k, collection_id): - slots[trace.combo.output * total_inputs + trace.combo.input] = trace - - if any(trace is None for trace in slots): - missing = sum(1 for trace in slots if trace is None) - raise RuntimeError( - f"Switched matrix collection is incomplete: {missing} of {len(slots)} combos missing" - ) + combos = [combo] if combo is not None else self.combos + if combos is None: + combos = [ + ComboKey(input_pos, output_pos) + for output_pos in range(total_outputs) + for input_pos in range(total_inputs) + ] + if any(not (0 <= c.input < total_inputs and 0 <= c.output < total_outputs) for c in combos): + raise ValueError("Requested combo is outside the switched matrix") + steps = dict.fromkeys( + (c.output // self.inner_output_positions, c.input // self.inner_input_positions) + for c in combos + ) + traces = { + trace.combo: trace + for out_k, in_k in steps + for trace in self._acquire_step_traces(out_k, in_k, collection_id) + } + missing = set(combos) - traces.keys() + if missing: + raise RuntimeError(f"Switched matrix collection is incomplete: {len(missing)} combos missing") return SweepCollection( collection_id=int(collection_id), monotonic_ns=time.monotonic_ns(), - traces=[trace for trace in slots if trace is not None], + traces=list(traces.values()) if combo is not None else [traces[key] for key in combos], capture_start_ns=capture_start_ns, capture_end_ns=time.monotonic_ns(), ) - def acquire_combo_collection( - self, - *, - input_pos: int, - output_pos: int, - collection_id: int = 1, - ) -> SweepCollection: - """Acquire only the physical switch step that carries one widened combo. - - The per-combo capture workflows need a single trace at a time; sweeping - every switch position for that (a full ``acquire_collection``) multiplies - the capture time by the number of physical steps and freezes the caller - for the whole sweep. One widened combo lives entirely inside one - (out_k, in_k) step, so acquiring just that step is sufficient. The result - contains that step's traces with widened combo keys, including the - requested combo. - """ - out_steps = self.output_switch.position_count() if self.output_switch is not None else 1 - in_steps = self.input_switch.position_count() if self.input_switch is not None else 1 - total_inputs = in_steps * self.inner_input_positions - total_outputs = out_steps * self.inner_output_positions - if not (0 <= int(input_pos) < total_inputs and 0 <= int(output_pos) < total_outputs): - raise ValueError( - f"Widened combo out of range: input={input_pos} (of {total_inputs}), " - f"output={output_pos} (of {total_outputs})" - ) - - capture_start_ns = time.monotonic_ns() - out_k = int(output_pos) // self.inner_output_positions - in_k = int(input_pos) // self.inner_input_positions - traces = self._acquire_step_traces(out_k, in_k, collection_id) - return SweepCollection( - collection_id=int(collection_id), - monotonic_ns=time.monotonic_ns(), - traces=traces, - capture_start_ns=capture_start_ns, - capture_end_ns=time.monotonic_ns(), + def _shape(self) -> tuple[int, int]: + return ( + self.inner_input_positions * (self.input_switch.position_count() if self.input_switch else 1), + self.inner_output_positions * (self.output_switch.position_count() if self.output_switch else 1), ) def _acquire_step_traces(self, out_k: int, in_k: int, collection_id: int) -> list[TraceData]: @@ -151,22 +117,14 @@ class SwitchedMatrixRadarService: ended. The switch drive and settling are deliberately outside the window. """ step_start_ns = time.monotonic_ns() - if self.output_switch is not None: - self.output_switch.switch_to(out_k) - if self.input_switch is not None: - self.input_switch.switch_to(in_k) - switched_ns = time.monotonic_ns() - # Settle AFTER the last switch change and BEFORE collecting, so the - # cycle we anchor on starts with the RF path already stable. - if self.settling_ms > 0: - time.sleep(self.settling_ms / 1000.0) + select_and_settle(self.output_switch, self.input_switch, out_k, in_k, self.settling_ms) settled_ns = time.monotonic_ns() sub = self.inner.acquire_collection(collection_id) inner_end_ns = time.monotonic_ns() logger.debug( "timing: collection %d step out=%d in=%d | gap_prev_collect_to_switch=%s ms, " - "switch=%.3f ms, settle=%.2f ms, inner_collect=%.2f ms", + "switch_and_settle=%.2f ms, inner_collect=%.2f ms", collection_id, out_k, in_k, @@ -175,8 +133,7 @@ class SwitchedMatrixRadarService: if self._last_inner_end_ns else "n/a" ), - (switched_ns - step_start_ns) / 1e6, - (settled_ns - switched_ns) / 1e6, + (settled_ns - step_start_ns) / 1e6, (inner_end_ns - settled_ns) / 1e6, ) self._last_inner_end_ns = inner_end_ns diff --git a/python_app/models/collection_identity.py b/python_app/models/collection_identity.py new file mode 100644 index 0000000..05055fb --- /dev/null +++ b/python_app/models/collection_identity.py @@ -0,0 +1,35 @@ +"""Acquisition identity shared unchanged by every pipeline stage.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from typing import Protocol, TypeVar + + +class CollectionIdentity(Protocol): + @property + def collection_id(self) -> int: ... + + @property + def capture_start_ns(self) -> int: ... + + +TCollection = TypeVar("TCollection", bound=CollectionIdentity) + + +def collection_key(collection: CollectionIdentity) -> tuple[int, int]: + return collection.collection_id, collection.capture_start_ns + + +def index_collections(collections: Iterable[TCollection]) -> dict[tuple[int, int], TCollection]: + return {collection_key(collection): collection for collection in collections} + + +def history_collection_removed( + collection: CollectionIdentity, clear_before_ns: int, removed: Sequence[Mapping[str, int]], +) -> bool: + """Apply a capture-time cutoff and exact acquisition deletions to late results.""" + return (bool(clear_before_ns) and collection.capture_start_ns <= clear_before_ns) or any( + collection_key(collection) == (target["collection_id"], target["capture_start_ns"]) + for target in removed + ) diff --git a/python_app/models/dataset_model.py b/python_app/models/dataset_model.py index b3a593e..d8759df 100644 --- a/python_app/models/dataset_model.py +++ b/python_app/models/dataset_model.py @@ -44,7 +44,6 @@ class TraceData: combo: ComboKey frequency_hz: np.ndarray - s11: np.ndarray s21: np.ndarray capture_start_ns: int = 0 capture_end_ns: int = 0 @@ -93,5 +92,9 @@ class ResultCollection: # Wall-clock nanoseconds spent by the data_processor on `process_collection` # for this collection. Zero means the producer did not report a measurement. processing_duration_ns: int = 0 + settings_revision: int = 0 + capture_start_ns: int = 0 + capture_end_ns: int = 0 + speed_m_s: float = 0.0 collection_payloads: list[ResultPayload] = field(default_factory=list) blocks: list[ResultBlock] = field(default_factory=list) diff --git a/python_app/models/gui_profile_codec.py b/python_app/models/gui_profile_codec.py index ae67abb..8d1288e 100644 --- a/python_app/models/gui_profile_codec.py +++ b/python_app/models/gui_profile_codec.py @@ -1,585 +1,78 @@ -"""Encoding and decoding logic for :mod:`python_app.models.gui_profile_schema`.""" +"""Strict profile decoding driven by the typed field declarations.""" from __future__ import annotations -import dataclasses -import logging +from dataclasses import asdict, fields, is_dataclass from typing import Any -from python_app.models.json_fields import finite_number, json_object as _as_dict -from python_app.models.gui_profile_schema import ( - GuiBscanStateModel, - GuiDataActionsStateModel, - GuiGprStateModel, - GuiLegacyGprStateModel, - GuiPassThroughStateModel, - GuiPreprocessDialogStateModel, - GuiProcessingStateModel, - GuiProfileModel, - GuiSar3dStateModel, - GuiStateModel, - GuiSwitchStateModel, -) -from python_app.models.run_config_model import RunConfigModel - -logger = logging.getLogger(__name__) +from python_app.models.gui_profile_schema import GuiProfileModel, GuiStateModel, Setting +from python_app.models.json_fields import finite_number, json_object +from python_app.models.run_config_schema import RunConfigModel -def _optional_string(object_payload: dict[str, Any], key: str, fallback: str, context: str) -> str: - """Return optional string field with type validation.""" - raw_value = object_payload.get(key, fallback) - if not isinstance(raw_value, str): - raise ValueError(f"{context}.{key} must be a JSON string") - return raw_value - - -def _optional_bool(object_payload: dict[str, Any], key: str, fallback: bool, context: str) -> bool: - """Return optional boolean field with type validation.""" - raw_value = object_payload.get(key, fallback) - if not isinstance(raw_value, bool): - raise ValueError(f"{context}.{key} must be a JSON bool") - return raw_value - - -def _optional_int(object_payload: dict[str, Any], key: str, fallback: int, context: str) -> int: - """Return optional integer field with type validation.""" - raw_value = object_payload.get(key, fallback) - if isinstance(raw_value, bool) or not isinstance(raw_value, int): - raise ValueError(f"{context}.{key} must be a JSON integer") - return int(raw_value) - - -def _optional_float(object_payload: dict[str, Any], key: str, fallback: float, context: str) -> float: - """Return optional numeric field with type validation.""" - return finite_number(object_payload.get(key, fallback), f"{context}.{key}") - - -def _legacy_gpr_mode_from_algorithm(value: str) -> str | None: - """Translate the short-lived nested GPR algorithm field into legacy mode.""" - if value == "legacy_point": - return "point" - if value == "legacy_extended": - return "extended" - return None +def _decode_state(model: object, payload: dict[str, Any], context: str, *, check_constraints: bool = True) -> None: + """Read known fields recursively; unknown fields do not survive re-encoding.""" + if not is_dataclass(model) or isinstance(model, type): + raise TypeError("GUI state must be a dataclass instance") + for member in fields(model): + default = getattr(model, member.name) + value = payload.get(member.name, default) + name = f"{context}.{member.name}" + if is_dataclass(default): + _decode_state(default, json_object(payload.get(member.name), name), name, check_constraints=check_constraints) + continue + if isinstance(default, float): + value = finite_number(value, name) + elif type(value) is not type(default): + raise ValueError(f"{name} must be a JSON {type(default).__name__}") + spec = member.metadata.get("setting") + if check_constraints and isinstance(spec, Setting): + if spec.choices and value not in spec.choices: + raise ValueError(f"{name} must be one of: {', '.join(spec.choices)}") + if spec.minimum is not None: + if value < spec.minimum or (spec.exclusive_minimum and value == spec.minimum): + operator = ">" if spec.exclusive_minimum else ">=" + raise ValueError(f"{name} must be {operator} {spec.minimum:g}") + if spec.maximum is not None and value > spec.maximum: + raise ValueError(f"{name} must be <= {spec.maximum:g}") + setattr(model, member.name, value) def gui_profile_from_dict(payload: dict[str, Any]) -> GuiProfileModel: - """Decode JSON-like payload into :class:`GuiProfileModel`.""" - profile = GuiProfileModel(run_config=RunConfigModel.from_dict(payload), gui=None) - gui_payload = payload.get("gui") - if gui_payload is None: - logger.debug("Decoded GUI profile without a 'gui' section; UI state left unset") + """Read the pipeline config and optional GUI state, preserving old GPR profiles.""" + profile = GuiProfileModel(run_config=RunConfigModel.from_dict(payload)) + if payload.get("gui") is None: return profile - - gui_object = _as_dict(gui_payload, "gui") + gui_object = json_object(payload["gui"], "gui") + processing = json_object(gui_object.get("processing"), "gui.processing") + old_gpr = json_object(processing.get("gpr"), "gui.processing.gpr") gui = GuiStateModel() - gui.version = _optional_int(gui_object, "version", gui.version, "gui") + + # Older profiles kept both algorithms under gpr. Root mode and the short-lived + # algorithm selector choose the legacy page; explicit legacy values win. + root_gpr = payload.get("gpr") + root_mode = root_gpr.get("mode") if isinstance(root_gpr, dict) else None + algorithm_mode = {"legacy_point": "point", "legacy_extended": "extended"}.get( + str(old_gpr.get("algorithm", ""))) + if root_mode in ("point", "extended"): + gui.processing.legacy_gpr.mode = root_mode + if algorithm_mode is not None: + gui.processing.legacy_gpr.mode = algorithm_mode + _decode_state(gui.processing.legacy_gpr, old_gpr, "gui.processing.gpr", check_constraints=False) + _decode_state(gui, gui_object, "gui") if gui.version != 1: raise ValueError(f"Unsupported gui.version: {gui.version}") - - switches_object = _as_dict(gui_object.get("switches"), "gui.switches") - gui.switches = GuiSwitchStateModel( - combo_mode=_optional_string(switches_object, "combo_mode", gui.switches.combo_mode, "gui.switches"), - combos_text=_optional_string(switches_object, "combos_text", gui.switches.combos_text, "gui.switches"), - single_input=_optional_string(switches_object, "single_input", gui.switches.single_input, "gui.switches"), - single_output=_optional_string( - switches_object, - "single_output", - gui.switches.single_output, - "gui.switches", - ), - ) - if gui.switches.combo_mode not in {"text", "single"}: - raise ValueError("gui.switches.combo_mode must be either 'text' or 'single'") - - processing_object = _as_dict(gui_object.get("processing"), "gui.processing") - pass_through_object = _as_dict(processing_object.get("pass_through"), "gui.processing.pass_through") - bscan_object = _as_dict(processing_object.get("bscan"), "gui.processing.bscan") - gpr_object = _as_dict(processing_object.get("gpr"), "gui.processing.gpr") - legacy_gpr_object = _as_dict(processing_object.get("legacy_gpr"), "gui.processing.legacy_gpr") - sar3d_object = _as_dict(processing_object.get("sar3d"), "gui.processing.sar3d") - root_gpr_object = payload.get("gpr") - - selected_mode = _optional_string( - processing_object, - "selected_mode", - gui.processing.selected_mode, - "gui.processing", - ) - legacy_mode_default = gui.processing.legacy_gpr.mode - if isinstance(root_gpr_object, dict): - if root_gpr_object.get("mode") == "point": - legacy_mode_default = "point" - elif root_gpr_object.get("mode") == "extended": - legacy_mode_default = "extended" - legacy_algorithm_mode = _legacy_gpr_mode_from_algorithm(str(gpr_object.get("algorithm", ""))) - if legacy_algorithm_mode is not None: - legacy_mode_default = legacy_algorithm_mode - has_legacy_root_gpr_mode = ( - isinstance(root_gpr_object, dict) - and root_gpr_object.get("mode") in {"point", "extended"} - ) - if selected_mode == "gpr" and (legacy_algorithm_mode is not None or has_legacy_root_gpr_mode): - logger.debug("Migrating legacy GPR profile: rewriting selected_mode 'gpr' -> 'legacy_gpr'") - selected_mode = "legacy_gpr" - - gpr_context = "gui.processing.gpr" - legacy_gpr_context = "gui.processing.legacy_gpr" - - def legacy_string(key: str, fallback: str) -> str: - """Read a legacy field, falling back to the old nested gpr object.""" - return _optional_string( - legacy_gpr_object, - key, - _optional_string(gpr_object, key, fallback, gpr_context), - legacy_gpr_context, - ) - - def legacy_bool(key: str, fallback: bool) -> bool: - """Read a legacy bool, falling back to the old nested gpr object.""" - return _optional_bool( - legacy_gpr_object, - key, - _optional_bool(gpr_object, key, fallback, gpr_context), - legacy_gpr_context, - ) - - def legacy_int(key: str, fallback: int) -> int: - """Read a legacy integer, falling back to the old nested gpr object.""" - return _optional_int( - legacy_gpr_object, - key, - _optional_int(gpr_object, key, fallback, gpr_context), - legacy_gpr_context, - ) - - def legacy_float(key: str, fallback: float) -> float: - """Read a legacy float, falling back to the old nested gpr object.""" - return _optional_float( - legacy_gpr_object, - key, - _optional_float(gpr_object, key, fallback, gpr_context), - legacy_gpr_context, - ) - - sar3d_context = "gui.processing.sar3d" - sar3d_defaults = GuiSar3dStateModel() - - def sar3d_string(key: str) -> str: - return _optional_string(sar3d_object, key, getattr(sar3d_defaults, key), sar3d_context) - - def sar3d_bool(key: str) -> bool: - return _optional_bool(sar3d_object, key, getattr(sar3d_defaults, key), sar3d_context) - - def sar3d_int(key: str) -> int: - return _optional_int(sar3d_object, key, getattr(sar3d_defaults, key), sar3d_context) - - def sar3d_float(key: str) -> float: - return _optional_float(sar3d_object, key, getattr(sar3d_defaults, key), sar3d_context) - - gui.processing = GuiProcessingStateModel( - selected_mode=selected_mode, - pass_through=GuiPassThroughStateModel( - show_magnitude=_optional_bool( - pass_through_object, - "show_magnitude", - gui.processing.pass_through.show_magnitude, - "gui.processing.pass_through", - ), - show_phase=_optional_bool( - pass_through_object, - "show_phase", - gui.processing.pass_through.show_phase, - "gui.processing.pass_through", - ), - unwrap_phase=_optional_bool( - pass_through_object, - "unwrap_phase", - gui.processing.pass_through.unwrap_phase, - "gui.processing.pass_through", - ), - combo_filter=_optional_string( - pass_through_object, - "combo_filter", - gui.processing.pass_through.combo_filter, - "gui.processing.pass_through", - ), - fixed_y_enabled=_optional_bool( - pass_through_object, - "fixed_y_enabled", - gui.processing.pass_through.fixed_y_enabled, - "gui.processing.pass_through", - ), - y_min_db=_optional_float( - pass_through_object, - "y_min_db", - gui.processing.pass_through.y_min_db, - "gui.processing.pass_through", - ), - y_max_db=_optional_float( - pass_through_object, - "y_max_db", - gui.processing.pass_through.y_max_db, - "gui.processing.pass_through", - ), - ), - bscan=GuiBscanStateModel( - axis=_optional_string(bscan_object, "axis", gui.processing.bscan.axis, "gui.processing.bscan"), - cut_m=_optional_float(bscan_object, "cut_m", gui.processing.bscan.cut_m, "gui.processing.bscan"), - max_depth_m=_optional_float( - bscan_object, - "max_depth_m", - gui.processing.bscan.max_depth_m, - "gui.processing.bscan", - ), - gain=_optional_float(bscan_object, "gain", gui.processing.bscan.gain, "gui.processing.bscan"), - start_freq_mhz=_optional_float( - bscan_object, - "start_freq_mhz", - gui.processing.bscan.start_freq_mhz, - "gui.processing.bscan", - ), - stop_freq_mhz=_optional_float( - bscan_object, - "stop_freq_mhz", - gui.processing.bscan.stop_freq_mhz, - "gui.processing.bscan", - ), - subtract_mean_ascan=_optional_bool( - bscan_object, - "subtract_mean_ascan", - gui.processing.bscan.subtract_mean_ascan, - "gui.processing.bscan", - ), - history_window_scans=_optional_int( - bscan_object, - "history_window_scans", - gui.processing.bscan.history_window_scans, - "gui.processing.bscan", - ), - ), - gpr=GuiGprStateModel( - input_positions=_optional_string( - gpr_object, - "input_positions", - gui.processing.gpr.input_positions, - "gui.processing.gpr", - ), - output_positions=_optional_string( - gpr_object, - "output_positions", - gui.processing.gpr.output_positions, - "gui.processing.gpr", - ), - min_depth_m=_optional_float( - gpr_object, - "min_depth_m", - gui.processing.gpr.min_depth_m, - "gui.processing.gpr", - ), - max_depth_m=_optional_float( - gpr_object, - "max_depth_m", - gui.processing.gpr.max_depth_m, - "gui.processing.gpr", - ), - range_comp_power=_optional_float( - gpr_object, - "range_comp_power", - gui.processing.gpr.range_comp_power, - "gui.processing.gpr", - ), - object_min_frac=_optional_float( - gpr_object, - "object_min_frac", - gui.processing.gpr.object_min_frac, - "gui.processing.gpr", - ), - score_mode=_optional_string( - gpr_object, - "score_mode", - gui.processing.gpr.score_mode, - "gui.processing.gpr", - ), - motion_mode=_optional_string( - gpr_object, - "motion_mode", - gui.processing.gpr.motion_mode, - "gui.processing.gpr", - ), - look_angle_deg=_optional_float( - gpr_object, - "look_angle_deg", - gui.processing.gpr.look_angle_deg, - "gui.processing.gpr", - ), - direction_sign=_optional_float( - gpr_object, - "direction_sign", - gui.processing.gpr.direction_sign, - "gui.processing.gpr", - ), - speed_m_s=_optional_float( - gpr_object, - "speed_m_s", - gui.processing.gpr.speed_m_s, - "gui.processing.gpr", - ), - ignore_socket_speed_enabled=_optional_bool( - gpr_object, - "ignore_socket_speed_enabled", - gui.processing.gpr.ignore_socket_speed_enabled, - "gui.processing.gpr", - ), - max_detected_objects_to_draw=_optional_int( - gpr_object, - "max_detected_objects_to_draw", - gui.processing.gpr.max_detected_objects_to_draw, - "gui.processing.gpr", - ), - draw_top_m_objects=_optional_int( - gpr_object, - "draw_top_m_objects", - gui.processing.gpr.draw_top_m_objects, - "gui.processing.gpr", - ), - object_approach_min_frames=_optional_int( - gpr_object, - "object_approach_min_frames", - gui.processing.gpr.object_approach_min_frames, - "gui.processing.gpr", - ), - start_freq_mhz=_optional_float( - gpr_object, - "start_freq_mhz", - gui.processing.gpr.start_freq_mhz, - "gui.processing.gpr", - ), - stop_freq_mhz=_optional_float( - gpr_object, - "stop_freq_mhz", - gui.processing.gpr.stop_freq_mhz, - "gui.processing.gpr", - ), - background_subtract_enabled=_optional_bool( - gpr_object, - "background_subtract_enabled", - gui.processing.gpr.background_subtract_enabled, - "gui.processing.gpr", - ), - background_mean_count=_optional_int( - gpr_object, - "background_mean_count", - gui.processing.gpr.background_mean_count, - "gui.processing.gpr", - ), - remove_sidelobe_objects_enabled=_optional_bool( - gpr_object, - "remove_sidelobe_objects_enabled", - gui.processing.gpr.remove_sidelobe_objects_enabled, - "gui.processing.gpr", - ), - imaging_plane_y_m=_optional_float( - gpr_object, - "imaging_plane_y_m", - gui.processing.gpr.imaging_plane_y_m, - "gui.processing.gpr", - ), - render_mode=_optional_string( - gpr_object, - "render_mode", - gui.processing.gpr.render_mode, - "gui.processing.gpr", - ), - visible_x_min_m=_optional_float( - gpr_object, - "visible_x_min_m", - gui.processing.gpr.visible_x_min_m, - "gui.processing.gpr", - ), - visible_x_max_m=_optional_float( - gpr_object, - "visible_x_max_m", - gui.processing.gpr.visible_x_max_m, - "gui.processing.gpr", - ), - visible_z_min_m=_optional_float( - gpr_object, - "visible_z_min_m", - gui.processing.gpr.visible_z_min_m, - "gui.processing.gpr", - ), - visible_z_max_m=_optional_float( - gpr_object, - "visible_z_max_m", - gui.processing.gpr.visible_z_max_m, - "gui.processing.gpr", - ), - ), - legacy_gpr=GuiLegacyGprStateModel( - mode=legacy_string("mode", legacy_mode_default), - input_positions=legacy_string("input_positions", gui.processing.legacy_gpr.input_positions), - output_positions=legacy_string("output_positions", gui.processing.legacy_gpr.output_positions), - min_depth_m=legacy_float("min_depth_m", gui.processing.legacy_gpr.min_depth_m), - max_depth_m=legacy_float("max_depth_m", gui.processing.legacy_gpr.max_depth_m), - comp_power=legacy_float("comp_power", gui.processing.legacy_gpr.comp_power), - start_freq_mhz=legacy_float("start_freq_mhz", gui.processing.legacy_gpr.start_freq_mhz), - stop_freq_mhz=legacy_float("stop_freq_mhz", gui.processing.legacy_gpr.stop_freq_mhz), - speed_m_s=legacy_float("speed_m_s", gui.processing.legacy_gpr.speed_m_s), - ignore_socket_speed_enabled=legacy_bool( - "ignore_socket_speed_enabled", - gui.processing.legacy_gpr.ignore_socket_speed_enabled, - ), - look_angle_deg=legacy_float("look_angle_deg", gui.processing.legacy_gpr.look_angle_deg), - apply_freq_phase_correction=legacy_bool( - "apply_freq_phase_correction", - gui.processing.legacy_gpr.apply_freq_phase_correction, - ), - reference_mode=legacy_string( - "reference_mode", - gui.processing.legacy_gpr.reference_mode, - ), - snr_thresh=legacy_float("snr_thresh", gui.processing.legacy_gpr.snr_thresh), - snr_comp_max=legacy_float("snr_comp_max", gui.processing.legacy_gpr.snr_comp_max), - background_subtract_enabled=legacy_bool( - "background_subtract_enabled", - gui.processing.legacy_gpr.background_subtract_enabled, - ), - background_mean_count=legacy_int( - "background_mean_count", - gui.processing.legacy_gpr.background_mean_count, - ), - render_mode=legacy_string("render_mode", gui.processing.legacy_gpr.render_mode), - min_visible_pair_count=legacy_int( - "min_visible_pair_count", - gui.processing.legacy_gpr.min_visible_pair_count, - ), - visible_x_min_m=legacy_float("visible_x_min_m", gui.processing.legacy_gpr.visible_x_min_m), - visible_x_max_m=legacy_float("visible_x_max_m", gui.processing.legacy_gpr.visible_x_max_m), - visible_z_min_m=legacy_float("visible_z_min_m", gui.processing.legacy_gpr.visible_z_min_m), - visible_z_max_m=legacy_float("visible_z_max_m", gui.processing.legacy_gpr.visible_z_max_m), - ), - sar3d=GuiSar3dStateModel( - input_positions=sar3d_string("input_positions"), - output_positions=sar3d_string("output_positions"), - min_depth_m=sar3d_float("min_depth_m"), - max_depth_m=sar3d_float("max_depth_m"), - start_freq_mhz=sar3d_float("start_freq_mhz"), - stop_freq_mhz=sar3d_float("stop_freq_mhz"), - speed_m_s=sar3d_float("speed_m_s"), - ignore_socket_speed_enabled=sar3d_bool("ignore_socket_speed_enabled"), - look_angle_deg=sar3d_float("look_angle_deg"), - frame_radius=sar3d_int("frame_radius"), - voxel_m=sar3d_float("voxel_m"), - bp_ground_depth_m=sar3d_float("bp_ground_depth_m"), - comp_range_power=sar3d_float("comp_range_power"), - use_compensation=sar3d_bool("use_compensation"), - normalize_each_frame=sar3d_bool("normalize_each_frame"), - ), - ) - if gui.processing.selected_mode not in {"pass_through", "bscan", "gpr", "legacy_gpr", "sar3d"}: - raise ValueError( - "gui.processing.selected_mode must be one of: pass_through, bscan, gpr, legacy_gpr, sar3d" - ) - if gui.processing.bscan.axis not in {"abs", "real", "phase"}: - raise ValueError("gui.processing.bscan.axis must be one of: abs, real, phase") - if gui.processing.gpr.render_mode not in {"heatmap", "objects_only"}: - raise ValueError("gui.processing.gpr.render_mode must be one of: heatmap, objects_only") - if gui.processing.gpr.score_mode not in {"peak", "combined"}: - raise ValueError("gui.processing.gpr.score_mode must be one of: peak, combined") - if gui.processing.gpr.motion_mode not in {"int_minus", "int_focus"}: - raise ValueError("gui.processing.gpr.motion_mode must be one of: int_minus, int_focus") - if gui.processing.legacy_gpr.mode not in {"point", "extended"}: - raise ValueError("gui.processing.legacy_gpr.mode must be one of: point, extended") - if gui.processing.legacy_gpr.render_mode not in {"heatmap", "objects_only"}: - raise ValueError("gui.processing.legacy_gpr.render_mode must be one of: heatmap, objects_only") - if gui.processing.legacy_gpr.reference_mode not in {"frame_center", "first_tx_event"}: - raise ValueError( - "gui.processing.legacy_gpr.reference_mode must be one of: frame_center, first_tx_event" - ) - if gui.processing.gpr.range_comp_power < 0.0: - raise ValueError("gui.processing.gpr.range_comp_power must be >= 0") - if not 0.0 <= gui.processing.gpr.object_min_frac <= 1.0: - raise ValueError("gui.processing.gpr.object_min_frac must be within [0, 1]") - if gui.processing.gpr.max_detected_objects_to_draw < 0: - raise ValueError("gui.processing.gpr.max_detected_objects_to_draw must be >= 0") - if gui.processing.gpr.draw_top_m_objects < 0: - raise ValueError("gui.processing.gpr.draw_top_m_objects must be >= 0") - if gui.processing.gpr.object_approach_min_frames < 1: - raise ValueError("gui.processing.gpr.object_approach_min_frames must be >= 1") - if gui.processing.legacy_gpr.comp_power < 0.0: - raise ValueError("gui.processing.legacy_gpr.comp_power must be >= 0") - if gui.processing.legacy_gpr.snr_thresh < 0.0: - raise ValueError("gui.processing.legacy_gpr.snr_thresh must be >= 0") - if gui.processing.legacy_gpr.snr_comp_max < 0.0: - raise ValueError("gui.processing.legacy_gpr.snr_comp_max must be >= 0") - if gui.processing.legacy_gpr.min_visible_pair_count < 1: - raise ValueError("gui.processing.legacy_gpr.min_visible_pair_count must be >= 1") - if gui.processing.sar3d.frame_radius < 0: - raise ValueError("gui.processing.sar3d.frame_radius must be >= 0") - if not gui.processing.sar3d.voxel_m > 0.0: - raise ValueError("gui.processing.sar3d.voxel_m must be > 0") - - data_actions_object = _as_dict(gui_object.get("data_actions"), "gui.data_actions") - gui.data_actions = GuiDataActionsStateModel( - save_path=_optional_string( - data_actions_object, - "save_path", - gui.data_actions.save_path, - "gui.data_actions", - ), - save_name=_optional_string( - data_actions_object, - "save_name", - gui.data_actions.save_name, - "gui.data_actions", - ), - record_count=_optional_int( - data_actions_object, - "record_count", - gui.data_actions.record_count, - "gui.data_actions", - ), - ) - - preprocess_dialog_object = _as_dict(gui_object.get("preprocess_dialog"), "gui.preprocess_dialog") - gui.preprocess_dialog = GuiPreprocessDialogStateModel( - set_name=_optional_string( - preprocess_dialog_object, - "set_name", - gui.preprocess_dialog.set_name, - "gui.preprocess_dialog", - ), - radar_config_dir=_optional_string( - preprocess_dialog_object, - "radar_config_dir", - gui.preprocess_dialog.radar_config_dir, - "gui.preprocess_dialog", - ), - use_all_radar_configs=_optional_bool( - preprocess_dialog_object, - "use_all_radar_configs", - gui.preprocess_dialog.use_all_radar_configs, - "gui.preprocess_dialog", - ), - median_sweep_count=max( - 1, - _optional_int( - preprocess_dialog_object, - "median_sweep_count", - gui.preprocess_dialog.median_sweep_count, - "gui.preprocess_dialog", - ), - ), - ) - + if gui.processing.selected_mode == "gpr" and ( + algorithm_mode is not None or root_mode in ("point", "extended") + ): + gui.processing.selected_mode = "legacy_gpr" + gui.preprocess_dialog.median_sweep_count = max(1, gui.preprocess_dialog.median_sweep_count) profile.gui = gui - logger.debug("Decoded GUI profile: selected_mode=%s", gui.processing.selected_mode) return profile def gui_profile_to_dict(model: GuiProfileModel) -> dict[str, Any]: - """Encode :class:`GuiProfileModel` into JSON-serializable dictionary.""" + """Encode the canonical run config and every declared GUI field.""" payload = model.run_config.to_dict() - payload["gui"] = dataclasses.asdict(model.gui if model.gui is not None else GuiStateModel()) + payload["gui"] = asdict(model.gui if model.gui is not None else GuiStateModel()) return payload diff --git a/python_app/models/gui_profile_model.py b/python_app/models/gui_profile_model.py deleted file mode 100644 index d0f824f..0000000 --- a/python_app/models/gui_profile_model.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Facade module for GUI profile schema and codec helpers.""" - -from python_app.models.gui_profile_codec import gui_profile_from_dict, gui_profile_to_dict -from python_app.models.gui_profile_schema import ( - GuiBscanStateModel, - GuiDataActionsStateModel, - GuiGprStateModel, - GuiLegacyGprStateModel, - GuiPassThroughStateModel, - GuiPreprocessDialogStateModel, - GuiProcessingStateModel, - GuiProfileModel, - GuiSar3dStateModel, - GuiStateModel, - GuiSwitchStateModel, -) - -__all__ = [ - "GuiBscanStateModel", - "GuiDataActionsStateModel", - "GuiGprStateModel", - "GuiLegacyGprStateModel", - "GuiPassThroughStateModel", - "GuiPreprocessDialogStateModel", - "GuiProcessingStateModel", - "GuiProfileModel", - "GuiSar3dStateModel", - "GuiStateModel", - "GuiSwitchStateModel", - "gui_profile_from_dict", - "gui_profile_to_dict", -] diff --git a/python_app/models/gui_profile_schema.py b/python_app/models/gui_profile_schema.py index 1477daa..888430f 100644 --- a/python_app/models/gui_profile_schema.py +++ b/python_app/models/gui_profile_schema.py @@ -7,120 +7,163 @@ from dataclasses import dataclass, field import json import logging from pathlib import Path -from typing import Any +from typing import Protocol +from typing import Any, TypeVar -from python_app.models.run_config_model import RunConfigModel +from python_app.models.run_config_schema import RunConfigModel logger = logging.getLogger(__name__) +Scalar = str | int | float | bool +_ScalarT = TypeVar("_ScalarT", str, int, float, bool) + + +@dataclass(frozen=True, slots=True) +class Setting: + """Qt-free binding and validation metadata for one persistent GUI field.""" + + widget: str = "" + live: str = "" + web: str = "" + group: str = "" + choices: tuple[str, ...] = () + minimum: float | None = None + maximum: float | None = None + exclusive_minimum: bool = False + + +def setting( + default: _ScalarT, widget: str = "", *, live: str = "", web: str = "", + group: str = "", choices: tuple[str, ...] = (), minimum: float | None = None, + maximum: float | None = None, exclusive_minimum: bool = False, +) -> _ScalarT: + """Declare a typed dataclass field and its single profile/live/web binding.""" + return field(default=default, metadata={"setting": Setting( + widget, live, web or live, group, choices, minimum, maximum, exclusive_minimum, + )}) + + @dataclass(slots=True) class GuiSwitchStateModel: """UI-only state for switch selection controls.""" - combo_mode: str = "text" - combos_text: str = "" - single_input: str = "0" - single_output: str = "0" + combo_mode: str = setting("text", choices=('text', 'single')) + combos_text: str = setting("", '_combos_text') + single_input: str = setting("0", '_single_combo_input') + single_output: str = setting("0", '_single_combo_output') @dataclass(slots=True) class GuiPassThroughStateModel: """UI-only defaults for pass-through rendering.""" - show_magnitude: bool = True - show_phase: bool = True - unwrap_phase: bool = False - combo_filter: str = "" - fixed_y_enabled: bool = False - y_min_db: float = -100.0 - y_max_db: float = 0.0 + show_magnitude: bool = setting(True, '_show_magnitude_checkbox') + show_phase: bool = setting(True, '_show_phase_checkbox') + unwrap_phase: bool = setting(False, '_unwrap_phase_checkbox') + combo_filter: str = setting("", '_pass_through_combo_filter_input', web='combo_filter', group='Display') + fixed_y_enabled: bool = setting(False, '_pass_through_fixed_y_enabled', web='pass_through_fixed_y_enabled', group='Pass-through') + y_min_db: float = setting(-100.0, '_pass_through_y_min_db', web='pass_through_y_min_db', group='Pass-through') + y_max_db: float = setting(0.0, '_pass_through_y_max_db', web='pass_through_y_max_db', group='Pass-through') @dataclass(slots=True) class GuiBscanStateModel: """UI-only defaults for B-scan live settings.""" - axis: str = "abs" - cut_m: float = 0.0 - max_depth_m: float = 8.0 - gain: float = 1.0 - start_freq_mhz: float = 100.0 - stop_freq_mhz: float = 8800.0 - subtract_mean_ascan: bool = False + axis: str = setting("abs", '_bscan_axis', live='bscan_axis', group='B-scan', choices=('abs', 'real', 'phase')) + cut_m: float = setting(0.824, '_bscan_cut_m', live='bscan_cut_m', group='B-scan') + max_depth_m: float = setting(1.0, '_bscan_max_depth_m', live='bscan_max_depth_m', group='B-scan') + gain: float = setting(1.0, '_bscan_gain', live='bscan_gain', group='B-scan') + start_freq_mhz: float = setting(100.0, '_bscan_start_freq_mhz', live='bscan_start_freq_mhz', group='B-scan') + stop_freq_mhz: float = setting(8800.0, '_bscan_stop_freq_mhz', live='bscan_stop_freq_mhz', group='B-scan') + subtract_mean_ascan: bool = setting(False, '_bscan_subtract_mean_ascan', web='subtract_mean_ascan', group='Display') # How many past sweeps the B-scan heatmap renders once acquisition is stopped. # While running the window stays at the C++ replay window (see # `_cpp_bscan_replay_window_for_config`); this only widens the stopped-mode view. - history_window_scans: int = 50 + history_window_scans: int = setting(50, '_bscan_history_window') @dataclass(slots=True) class GuiGprStateModel: """UI-only defaults for coherent backprojection GPR live settings.""" - input_positions: str = "" - output_positions: str = "" - min_depth_m: float = 2.0 - max_depth_m: float = 14.0 - range_comp_power: float = 0.1 - # BP object-detection stop level, fraction of the global peak (Horns_motion_3libre.py - # BP_OBJECT_MIN_FRAC). Angle compensation and permittivity are fixed for coherent BP - # (Python 0.3 block), so they are not exposed here. - object_min_frac: float = 0.7 - score_mode: str = "combined" - motion_mode: str = "int_minus" + input_positions: str = setting("", '_gpr_input_positions_input', live='gpr_input_positions', group='Geometry & depth') + output_positions: str = setting("", '_gpr_output_positions_input', live='gpr_output_positions', group='Geometry & depth') + min_depth_m: float = setting(2.0, '_gpr_min_depth_m', live='gpr_min_depth_m', group='Geometry & depth') + max_depth_m: float = setting(14.0, '_gpr_max_depth_m', live='gpr_max_depth_m', group='Geometry & depth') + range_comp_power: float = setting(0.1, '_gpr_range_comp_power', live='gpr_range_comp_power', group='Imaging', minimum=0) + # BP object-detection stop level, fraction of the global peak. Angle compensation and + # permittivity are fixed for coherent BP, so they are not exposed here. + object_min_frac: float = setting(0.7, '_gpr_object_min_frac', live='gpr_object_min_frac', group='Detection', minimum=0, maximum=1) + score_mode: str = setting("combined", '_gpr_score_mode', live='gpr_score_mode', group='Imaging', choices=('peak', 'combined')) + motion_mode: str = setting("int_minus", '_gpr_motion_mode', live='gpr_motion_mode', group='Motion', choices=('int_minus', 'int_focus')) # Intra-sweep motion-correction inputs. Sweep time is derived from acquisition # metadata; speed comes from the socket unless `ignore_socket_speed_enabled`, # in which case `speed_m_s` from here is used. - look_angle_deg: float = 0.0 - direction_sign: float = 1.0 - speed_m_s: float = 0.0 - ignore_socket_speed_enabled: bool = False - max_detected_objects_to_draw: int = 5 - draw_top_m_objects: int = 2 + look_angle_deg: float = setting(0.0, '_gpr_look_angle_deg', live='gpr_look_angle_deg', group='Motion') + direction_sign: float = setting(1.0, '_gpr_direction_sign', live='gpr_direction_sign', group='Motion') + speed_m_s: float = setting(0.0, '_gpr_speed_m_s', live='gpr_speed_m_s', group='Motion') + ignore_socket_speed_enabled: bool = setting(False, '_gpr_ignore_socket_speed_enabled', live='ignore_socket_speed', group='Motion') + max_detected_objects_to_draw: int = setting( + 5, '_gpr_max_detected_objects_to_draw', live='gpr_max_detected_objects_to_draw', group='Detection', minimum=0, + ) + draw_top_m_objects: int = setting(2, '_gpr_draw_top_m_objects', live='gpr_draw_top_m_objects', group='Detection', minimum=0) # Cross-frame approach filter: show an object only after it persists as a # motion-consistent track this many consecutive frames (<= 1 disables it). - object_approach_min_frames: int = 3 - start_freq_mhz: float = 3000.0 - stop_freq_mhz: float = 6000.0 - background_subtract_enabled: bool = True - background_mean_count: int = 10 - remove_sidelobe_objects_enabled: bool = True - imaging_plane_y_m: float = 0.0 - render_mode: str = "heatmap" - visible_x_min_m: float = -2.0 - visible_x_max_m: float = 2.0 - visible_z_min_m: float = 0.0 - visible_z_max_m: float = 14.0 + object_approach_min_frames: int = setting( + 3, '_gpr_object_approach_min_frames', live='gpr_object_approach_min_frames', group='Detection', minimum=1, + ) + start_freq_mhz: float = setting(3000.0, '_gpr_start_freq_mhz', live='gpr_start_freq_mhz', group='Geometry & depth') + stop_freq_mhz: float = setting(6000.0, '_gpr_stop_freq_mhz', live='gpr_stop_freq_mhz', group='Geometry & depth') + background_subtract_enabled: bool = setting(True, '_gpr_background_subtract_enabled', live='gpr_background_subtract_enabled', group='Imaging') + background_mean_count: int = setting(10, '_gpr_background_mean_count', live='gpr_background_mean_count', group='Imaging') + remove_sidelobe_objects_enabled: bool = setting( + True, '_gpr_remove_sidelobe_objects_enabled', live='gpr_remove_sidelobe_objects_enabled', group='Imaging', + ) + imaging_plane_y_m: float = setting(0.0, '_gpr_imaging_plane_y_m', live='gpr_imaging_plane_y_m', group='Geometry & depth') + render_mode: str = setting("heatmap", '_gpr_render_mode', web='render_mode', group='Display', choices=('heatmap', 'objects_only')) + visible_x_min_m: float = setting(-2.0, '_gpr_visible_x_min_m', live='gpr_visible_x_min_m', group='Visible window') + visible_x_max_m: float = setting(2.0, '_gpr_visible_x_max_m', live='gpr_visible_x_max_m', group='Visible window') + visible_z_min_m: float = setting(0.0, '_gpr_visible_z_min_m', live='gpr_visible_z_min_m', group='Visible window') + visible_z_max_m: float = setting(14.0, '_gpr_visible_z_max_m', live='gpr_visible_z_max_m', group='Visible window') @dataclass(slots=True) class GuiLegacyGprStateModel: """UI-only defaults for legacy point/extended GPR live settings.""" - mode: str = "point" - input_positions: str = "" - output_positions: str = "" - min_depth_m: float = 2.0 - max_depth_m: float = 14.0 - comp_power: float = 0.2 - start_freq_mhz: float = 3000.0 - stop_freq_mhz: float = 6000.0 - speed_m_s: float = 0.0 - ignore_socket_speed_enabled: bool = False - look_angle_deg: float = 0.0 - apply_freq_phase_correction: bool = True - reference_mode: str = "frame_center" - snr_thresh: float = 4.5 - snr_comp_max: float = 25.0 - background_subtract_enabled: bool = True - background_mean_count: int = 10 - render_mode: str = "heatmap" - min_visible_pair_count: int = 1 - visible_x_min_m: float = -2.0 - visible_x_max_m: float = 2.0 - visible_z_min_m: float = 0.0 - visible_z_max_m: float = 14.0 + mode: str = setting("point", '_legacy_gpr_config_mode', live='legacy_gpr_mode', group='Mode', choices=('point', 'extended')) + input_positions: str = setting("", '_legacy_gpr_input_positions_input', live='gpr_input_positions', group='Geometry & depth') + output_positions: str = setting("", '_legacy_gpr_output_positions_input', live='gpr_output_positions', group='Geometry & depth') + min_depth_m: float = setting(2.0, '_legacy_gpr_min_depth_m', live='gpr_min_depth_m', group='Geometry & depth') + max_depth_m: float = setting(14.0, '_legacy_gpr_max_depth_m', live='gpr_max_depth_m', group='Geometry & depth') + comp_power: float = setting(0.2, '_legacy_gpr_comp_power', live='gpr_comp_power', group='Detection', minimum=0) + start_freq_mhz: float = setting(3000.0, '_legacy_gpr_start_freq_mhz', live='gpr_start_freq_mhz', group='Geometry & depth') + stop_freq_mhz: float = setting(6000.0, '_legacy_gpr_stop_freq_mhz', live='gpr_stop_freq_mhz', group='Geometry & depth') + speed_m_s: float = setting(0.0, '_legacy_gpr_speed_m_s', live='gpr_speed_m_s', group='Motion') + ignore_socket_speed_enabled: bool = setting(False, '_legacy_gpr_ignore_socket_speed_enabled', live='ignore_socket_speed', group='Motion') + look_angle_deg: float = setting(0.0, '_legacy_gpr_look_angle_deg', live='gpr_look_angle_deg', group='Motion') + apply_freq_phase_correction: bool = setting( + True, '_legacy_gpr_apply_freq_phase_correction', live='gpr_apply_freq_phase_correction', group='Motion', + ) + reference_mode: str = setting( + "frame_center", '_legacy_gpr_reference_mode', live='gpr_reference_mode', group='Motion', choices=('frame_center', 'first_tx_event'), + ) + snr_thresh: float = setting(4.5, '_legacy_gpr_snr_thresh', live='gpr_snr_thresh', group='Detection', minimum=0) + snr_comp_max: float = setting(25.0, '_legacy_gpr_snr_comp_max', live='gpr_snr_comp_max', group='Detection', minimum=0) + background_subtract_enabled: bool = setting( + True, '_legacy_gpr_background_subtract_enabled', live='gpr_background_subtract_enabled', group='Imaging', + ) + background_mean_count: int = setting(10, '_legacy_gpr_background_mean_count', live='gpr_background_mean_count', group='Imaging') + render_mode: str = setting("heatmap", '_legacy_gpr_render_mode', web='render_mode', group='Display', choices=('heatmap', 'objects_only')) + min_visible_pair_count: int = setting( + 1, '_legacy_gpr_min_visible_pair_count', live='legacy_gpr_min_visible_pair_count', group='Detection', minimum=1, + ) + visible_x_min_m: float = setting(-2.0, '_legacy_gpr_visible_x_min_m', live='gpr_visible_x_min_m', group='Visible window') + visible_x_max_m: float = setting(2.0, '_legacy_gpr_visible_x_max_m', live='gpr_visible_x_max_m', group='Visible window') + visible_z_min_m: float = setting(0.0, '_legacy_gpr_visible_z_min_m', live='gpr_visible_z_min_m', group='Visible window') + visible_z_max_m: float = setting(14.0, '_legacy_gpr_visible_z_max_m', live='gpr_visible_z_max_m', group='Visible window') @dataclass(slots=True) @@ -131,28 +174,30 @@ class GuiSar3dStateModel: names the GUI shows: INPUT_IDX, F_START, MIN_DEPTH, LOOK_ANGLE_DEG, ... """ - input_positions: str = "" - output_positions: str = "" - min_depth_m: float = 2.0 - max_depth_m: float = 8.0 - start_freq_mhz: float = 1500.0 - stop_freq_mhz: float = 6000.0 - speed_m_s: float = 0.0 - ignore_socket_speed_enabled: bool = False - look_angle_deg: float = 19.5 - frame_radius: int = 0 - voxel_m: float = 0.02 - bp_ground_depth_m: float = 1.25 - comp_range_power: float = 0.3 - use_compensation: bool = True - normalize_each_frame: bool = True + input_positions: str = setting("", '_sar3d_input_positions_input', live='gpr_input_positions', group='Geometry & depth') + output_positions: str = setting("", '_sar3d_output_positions_input', live='gpr_output_positions', group='Geometry & depth') + min_depth_m: float = setting(2.0, '_sar3d_min_depth_m', live='gpr_min_depth_m', group='Geometry & depth') + max_depth_m: float = setting(8.0, '_sar3d_max_depth_m', live='gpr_max_depth_m', group='Geometry & depth') + start_freq_mhz: float = setting(1500.0, '_sar3d_start_freq_mhz', live='gpr_start_freq_mhz', group='Geometry & depth') + stop_freq_mhz: float = setting(6000.0, '_sar3d_stop_freq_mhz', live='gpr_stop_freq_mhz', group='Geometry & depth') + speed_m_s: float = setting(0.0, '_sar3d_speed_m_s', live='gpr_speed_m_s', group='Motion') + ignore_socket_speed_enabled: bool = setting(False, '_sar3d_ignore_socket_speed_enabled', live='ignore_socket_speed', group='Motion') + look_angle_deg: float = setting(19.5, '_sar3d_look_angle_deg', live='gpr_look_angle_deg', group='Motion') + frame_radius: int = setting(0, '_sar3d_frame_radius', live='sar3d_frame_radius', group='3D SAR', minimum=0) + voxel_m: float = setting(0.02, '_sar3d_voxel_m', live='sar3d_voxel_m', group='3D SAR', minimum=0, exclusive_minimum=True) + bp_ground_depth_m: float = setting(1.25, '_sar3d_bp_ground_depth_m', live='sar3d_bp_ground_depth_m', group='3D SAR') + comp_range_power: float = setting(0.3, '_sar3d_comp_range_power', live='sar3d_comp_range_power', group='3D SAR') + use_compensation: bool = setting(True, '_sar3d_use_compensation', live='sar3d_use_compensation', group='3D SAR') + normalize_each_frame: bool = setting(True, '_sar3d_normalize_each_frame', live='sar3d_normalize_each_frame', group='3D SAR') @dataclass(slots=True) class GuiProcessingStateModel: """UI-only processing-section defaults.""" - selected_mode: str = "pass_through" + selected_mode: str = setting( + "pass_through", '_processing_mode', live='processor_mode', group='Processor', choices=('pass_through', 'bscan', 'gpr', 'legacy_gpr', 'sar3d'), + ) pass_through: GuiPassThroughStateModel = field(default_factory=GuiPassThroughStateModel) bscan: GuiBscanStateModel = field(default_factory=GuiBscanStateModel) gpr: GuiGprStateModel = field(default_factory=GuiGprStateModel) @@ -164,21 +209,21 @@ class GuiProcessingStateModel: class GuiDataActionsStateModel: """UI-only defaults for snapshot/export controls.""" - save_path: str = "" - save_name: str = "snapshot_manual" + save_path: str = setting("", '_save_path_input') + save_name: str = setting("snapshot_manual", '_save_name_input') # How many freshly acquired measurements the "Start + Record" action writes to # disk before it stops recording (acquisition keeps running). - record_count: int = 100 + record_count: int = setting(100, '_record_count') @dataclass(slots=True) class GuiPreprocessDialogStateModel: """UI-only defaults for preprocessing dialog controls.""" - set_name: str = "set_001" - radar_config_dir: str = "" - use_all_radar_configs: bool = False - median_sweep_count: int = 5 + set_name: str = setting("set_001") + radar_config_dir: str = setting("") + use_all_radar_configs: bool = setting(False) + median_sweep_count: int = setting(5) @dataclass(slots=True) @@ -227,3 +272,10 @@ class GuiProfileModel: from python_app.models.gui_profile_codec import gui_profile_to_dict return gui_profile_to_dict(self) + + +class ProfileWriter(Protocol): + """Publish the current profile, optionally using the acquisition provenance.""" + + def __call__(self, path: Path, /, *, allow_overwrite: bool = True, + dataset: bool = False) -> GuiProfileModel: ... diff --git a/python_app/models/run_config_codec.py b/python_app/models/run_config_codec.py index 9d41637..03be625 100644 --- a/python_app/models/run_config_codec.py +++ b/python_app/models/run_config_codec.py @@ -79,7 +79,7 @@ def _load_string_dict(payload: dict[str, Any], key: str, context: str) -> dict[s return values -def run_config_from_dict(payload: dict[str, Any]) -> RunConfigModel: +def run_config_from_dict(payload: object) -> RunConfigModel: """Decode JSON-like payload into :class:`RunConfigModel`.""" if not isinstance(payload, dict): raise ValueError("Run config root must be a JSON object") @@ -331,24 +331,6 @@ def run_config_from_dict(payload: dict[str, Any]) -> RunConfigModel: model.preprocess.s21.object_calibration, ) - s11_preprocess_payload = _as_dict(preprocess_payload.get("s11"), "preprocess.s11") - s11_calibration_payload = _as_dict(s11_preprocess_payload.get("calibration"), "preprocess.s11.calibration") - _load_preprocess_asset( - _as_dict(s11_calibration_payload.get("open"), "preprocess.s11.calibration.open"), - model.preprocess.s11.calibration.open, - ) - _load_preprocess_asset( - _as_dict(s11_calibration_payload.get("short"), "preprocess.s11.calibration.short"), - model.preprocess.s11.calibration.short, - ) - _load_preprocess_asset( - _as_dict(s11_calibration_payload.get("load"), "preprocess.s11.calibration.load"), - model.preprocess.s11.calibration.load, - ) - _load_preprocess_asset( - _as_dict(s11_preprocess_payload.get("reference"), "preprocess.s11.reference"), - model.preprocess.s11.reference, - ) notch_payload = _as_dict(preprocess_payload.get("notch"), "preprocess.notch") model.preprocess.notch = PreprocessNotchModel( enabled=_read_bool(notch_payload, "enabled", model.preprocess.notch.enabled), @@ -521,26 +503,22 @@ def run_config_to_dict(model: RunConfigModel) -> dict[str, Any]: "port1": { "name": model.output_switch.name, "driver_mode": model.output_switch.driver_mode, - "driver": model.output_switch.driver, "radar_port": model.output_switch.radar_port, "positions": model.output_switch.positions, "default_position": model.output_switch.default_position, "gpio_chip": model.output_switch.gpio_chip, "pin_a": model.output_switch.pin_a, "pin_b": model.output_switch.pin_b, - "invert_logic": model.output_switch.invert_logic, }, "port2": { "name": model.input_switch.name, "driver_mode": model.input_switch.driver_mode, - "driver": model.input_switch.driver, "radar_port": model.input_switch.radar_port, "positions": model.input_switch.positions, "default_position": model.input_switch.default_position, "gpio_chip": model.input_switch.gpio_chip, "pin_a": model.input_switch.pin_a, "pin_b": model.input_switch.pin_b, - "invert_logic": model.input_switch.invert_logic, }, }, "control_button": { @@ -586,26 +564,6 @@ def run_config_to_dict(model: RunConfigModel) -> dict[str, Any]: "bundle_path": model.preprocess.s21.object_calibration.bundle_path, }, }, - "s11": { - "calibration": { - "open": { - "set_name": model.preprocess.s11.calibration.open.set_name, - "bundle_path": model.preprocess.s11.calibration.open.bundle_path, - }, - "short": { - "set_name": model.preprocess.s11.calibration.short.set_name, - "bundle_path": model.preprocess.s11.calibration.short.bundle_path, - }, - "load": { - "set_name": model.preprocess.s11.calibration.load.set_name, - "bundle_path": model.preprocess.s11.calibration.load.bundle_path, - }, - }, - "reference": { - "set_name": model.preprocess.s11.reference.set_name, - "bundle_path": model.preprocess.s11.reference.bundle_path, - }, - }, "notch": { "enabled": model.preprocess.notch.enabled, "bands_hz": [[low_hz, high_hz] for low_hz, high_hz in model.preprocess.notch.bands_hz], diff --git a/python_app/models/run_config_model.py b/python_app/models/run_config_model.py deleted file mode 100644 index 3b4791a..0000000 --- a/python_app/models/run_config_model.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Facade module for run configuration schema, codec, and validation helpers.""" - -from python_app.models.run_config_codec import run_config_from_dict, run_config_to_dict -from python_app.models.run_config_schema import ( - ComboModel, - GprModel, - GprRxGeometryModel, - GprTxGeometryModel, - KamilAdcBandModel, - KamilAdcModel, - KamilAdcPhaseCalibrationModel, - LaserControlModel, - LaserManualModeModel, - LaserVariationModeModel, - LocatorServerRuntimeModel, - PreprocessAssetModel, - PreprocessNotchModel, - PreprocessModel, - RadarMultiDeviceModel, - RadarModel, - RadarSweepModel, - RingEndpointModel, - RingsModel, - RunConfigModel, - RuntimeModel, - S11CalibrationModel, - S11PreprocessModel, - S21PreprocessModel, - SwitchModel, -) -from python_app.models.run_config_validation import ( - load_ring_payload, - load_switch_payload, - parse_combos_from_text, -) - -__all__ = [ - "ComboModel", - "GprModel", - "GprRxGeometryModel", - "GprTxGeometryModel", - "KamilAdcBandModel", - "KamilAdcModel", - "KamilAdcPhaseCalibrationModel", - "LaserControlModel", - "LaserManualModeModel", - "LaserVariationModeModel", - "LocatorServerRuntimeModel", - "PreprocessAssetModel", - "PreprocessNotchModel", - "PreprocessModel", - "RadarMultiDeviceModel", - "RadarModel", - "RadarSweepModel", - "RingEndpointModel", - "RingsModel", - "RunConfigModel", - "RuntimeModel", - "S11CalibrationModel", - "S11PreprocessModel", - "S21PreprocessModel", - "SwitchModel", - "load_ring_payload", - "load_switch_payload", - "parse_combos_from_text", - "run_config_from_dict", - "run_config_to_dict", -] diff --git a/python_app/models/run_config_schema.py b/python_app/models/run_config_schema.py index 00ccf5a..160da42 100644 --- a/python_app/models/run_config_schema.py +++ b/python_app/models/run_config_schema.py @@ -9,6 +9,8 @@ import logging from pathlib import Path from typing import Any +from python_app.file_io import sanitize_path_component + logger = logging.getLogger(__name__) MAX_COMBOS = 4096 @@ -186,14 +188,12 @@ class SwitchModel: name: str driver_mode: str = "mock" - driver: str = "" radar_port: int = 0 positions: int = 1 default_position: int = 0 gpio_chip: str = "" pin_a: int = -1 pin_b: int = -1 - invert_logic: bool = False @dataclass(slots=True) @@ -278,23 +278,6 @@ class S21PreprocessModel: object_calibration: PreprocessAssetModel = field(default_factory=PreprocessAssetModel) -@dataclass(slots=True) -class S11CalibrationModel: - """One-port S11 OSL calibration assets.""" - - open: PreprocessAssetModel = field(default_factory=PreprocessAssetModel) - short: PreprocessAssetModel = field(default_factory=PreprocessAssetModel) - load: PreprocessAssetModel = field(default_factory=PreprocessAssetModel) - - -@dataclass(slots=True) -class S11PreprocessModel: - """One-port S11 preprocessing assets.""" - - calibration: S11CalibrationModel = field(default_factory=S11CalibrationModel) - reference: PreprocessAssetModel = field(default_factory=PreprocessAssetModel) - - @dataclass(slots=True) class PreprocessNotchModel: """Optional frequency-domain notch filter applied after calibration and reference subtraction.""" @@ -310,7 +293,6 @@ class PreprocessModel: """Selected preprocessing artifacts for live acquisition.""" s21: S21PreprocessModel = field(default_factory=S21PreprocessModel) - s11: S11PreprocessModel = field(default_factory=S11PreprocessModel) notch: PreprocessNotchModel = field(default_factory=PreprocessNotchModel) @@ -402,11 +384,6 @@ class RunConfigModel: """Return whether this config targets synchronized multi-device acquisition.""" return self.radar.model == self.LIBREVNA_MULTI_MODEL - @property - def is_matrix_radar(self) -> bool: - """Return whether this config acquires the full virtual switch matrix per sweep.""" - return self.is_multi_device - @property def matrix_output_switch_positions(self) -> int: """Physical positions of the real switch on the master stimulus path.""" @@ -449,7 +426,21 @@ class RunConfigModel: """Return whether this config targets the external Kamil ADC acquisition path.""" return self.radar.model == self.KAMIL_ADC_MODEL - def radar_key_extra_parts(self) -> list[str]: + def radar_key(self) -> str: + """Stable hardware/sweep identity used to find saved preprocess sets.""" + radar, sweep = self.radar, self.radar.sweep + serials = [radar.serial or "no_serial"] + serials.extend(value.strip() or "no_serial" for value in self._radar_key_extra_parts()) + serial = "_".join(sanitize_path_component(value) for value in serials) + points = "adc" if self.is_kamil_adc else str(int(sweep.points)) + return ( + f"{radar.model}_{serial}" + f"_st{_format_float_for_key(sweep.start_hz)}_sp{_format_float_for_key(sweep.stop_hz)}" + f"_p{points}_if{_format_float_for_key(sweep.if_bandwidth_hz)}" + f"_pw{_format_float_for_key(sweep.power_dbm)}" + ) + + def _radar_key_extra_parts(self) -> list[str]: """Return model-specific identity parts that affect captured data.""" if self.is_multi_device: return list(self.radar.multi_device.slave_serials) @@ -497,7 +488,7 @@ class RunConfigModel: def apply_device_model_constraints(self) -> None: """Apply only required wire-format constraints for the selected device model.""" - if not self.is_matrix_radar: + if not self.is_multi_device: return self._apply_matrix_virtual_switches() self.combos = self.build_runtime_combos() @@ -509,21 +500,19 @@ class RunConfigModel: self.output_switch.name = self.output_switch.name or "virtual_output" self.output_switch.driver_mode = "mock" - self.output_switch.driver = self.output_switch.driver or "h7992" self.output_switch.radar_port = 1 self.output_switch.positions = out_physical * self.matrix_virtual_output_positions self.output_switch.default_position = 0 self.input_switch.name = self.input_switch.name or "virtual_input" self.input_switch.driver_mode = "mock" - self.input_switch.driver = self.input_switch.driver or "h7992" self.input_switch.radar_port = 2 self.input_switch.positions = in_physical * self.MULTI_DEVICE_INPUT_POSITIONS self.input_switch.default_position = 0 def ensure_combos(self) -> None: """Populate combos with full matrix when no explicit run combos are set.""" - if self.is_matrix_radar: + if self.is_multi_device: self.apply_device_model_constraints() return if self.combos: @@ -531,7 +520,7 @@ class RunConfigModel: self.combos = self.build_full_combos(self.input_switch.positions, self.output_switch.positions) @classmethod - def from_dict(cls, payload: dict[str, Any]) -> RunConfigModel: + def from_dict(cls, payload: object) -> RunConfigModel: """Build model from JSON-like payload using codec layer.""" from python_app.models.run_config_codec import run_config_from_dict @@ -558,3 +547,11 @@ class RunConfigModel: from python_app.models.run_config_codec import run_config_to_dict return run_config_to_dict(self) + + +def _format_float_for_key(value: float) -> str: + """Format float compactly for deterministic key serialization.""" + integer = int(round(value)) + if abs(value - float(integer)) < 1e-6: + return str(integer) + return f"{value:.6f}".rstrip("0").rstrip(".") diff --git a/python_app/models/run_config_validation.py b/python_app/models/run_config_validation.py index 003f924..55f0b35 100644 --- a/python_app/models/run_config_validation.py +++ b/python_app/models/run_config_validation.py @@ -2,11 +2,15 @@ from __future__ import annotations +from collections.abc import Iterable +from dataclasses import asdict import logging import math +import re from typing import Any from python_app.models.json_fields import ( + finite_number, read_bool as _require_bool, read_int as _require_int, read_str as _require_str, ) from python_app.models.run_config_schema import ( @@ -14,12 +18,24 @@ from python_app.models.run_config_schema import ( ComboModel, ControlButtonModel, GprModel, + GprRxGeometryModel, + GprTxGeometryModel, RadarSweepModel, RingEndpointModel, RunConfigModel, SwitchModel, ) +POSITION_CSV_PATTERN = r"\s*(?:[0-9]+\s*(?:,\s*[0-9]+\s*)*,?\s*)?" + + +def parse_position_csv(value: str) -> list[int]: + """Non-negative positions; empty means all, a trailing comma permits editing.""" + if re.fullmatch(POSITION_CSV_PATTERN, value) is None: + raise ValueError("Positions must be non-negative integers separated by commas") + return [int(token) for token in value.split(",") if token.strip()] + + logger = logging.getLogger(__name__) # Wire-format bounds shared with the C++ pipeline. The ring header stores the @@ -37,14 +53,16 @@ def load_switch_payload( # Scalar reads reject array/object JSON types as ValueError (not TypeError). target.name = _require_str(payload, "name", target.name) target.driver_mode = _require_str(payload, "driver_mode", target.driver_mode) - target.driver = _require_str(payload, "driver", target.driver) + if "driver" in payload and payload["driver"] != "h7992": + raise ValueError(f"switches.{target.name}.driver: only H7992 is supported; check the switch and its two control pins") + if "invert_logic" in payload and payload["invert_logic"] is not False: + raise ValueError(f"switches.{target.name}.invert_logic is unsupported by H7992") target.radar_port = _require_int(payload, "radar_port", target.radar_port) target.positions = _require_int(payload, "positions", target.positions) target.default_position = _require_int(payload, "default_position", target.default_position) target.gpio_chip = _require_str(payload, "gpio_chip", target.gpio_chip) target.pin_a = _require_int(payload, "pin_a", target.pin_a) target.pin_b = _require_int(payload, "pin_b", target.pin_b) - target.invert_logic = _require_bool(payload, "invert_logic", target.invert_logic) def load_control_button_payload( @@ -130,27 +148,22 @@ def validate_acquisition_config(config: RunConfigModel) -> None: context = f"switches.{switch.name or switch.radar_port}" if switch.driver_mode not in ("mock", "native"): raise ValueError(f"{context}.driver_mode must be mock or native") - if switch.driver not in ("h7992", "hmc349a"): - raise ValueError(f"{context}.driver must be h7992 or hmc349a") if switch.radar_port not in (1, 2): raise ValueError(f"{context}.radar_port must be 1 or 2") if not 0 <= switch.default_position < switch.positions: raise ValueError(f"{context}.default_position must be within switch positions") positions, switch_mode = switch.positions, switch.driver_mode - if config.is_matrix_radar and physical_positions > 1: + if config.is_multi_device and physical_positions > 1: positions, switch_mode = physical_positions, mode if switch_mode == "native": - limit = 4 if switch.driver == "h7992" else 2 - if not 1 <= positions <= limit: - raise ValueError(f"{context}: native {switch.driver} supports 1..{limit} positions") - if not switch.gpio_chip or switch.pin_a < 0 or ( - switch.driver == "h7992" and (switch.pin_b < 0 or switch.pin_b == switch.pin_a) - ): + if not 1 <= positions <= 4: + raise ValueError(f"{context}: native H7992 supports 1..4 positions") + if not switch.gpio_chip or switch.pin_a < 0 or switch.pin_b < 0 or switch.pin_b == switch.pin_a: raise ValueError(f"{context}: GPIO chip and distinct nonnegative control pins are required") points = config.radar.kamil_adc.band.points if config.is_kamil_adc else config.radar.sweep.points - # RAW/PPR header, each combo's frequency/S11/S21 arrays and capture windows. - minimum = 44 + len(config.combos) * (28 + 20 * points) + # RAW3/PRP3 header, frequency/S21 arrays and capture windows. + minimum = 44 + len(config.combos) * (28 + 12 * points) for name in ("raw", "raw_tap", "preprocessed", "preprocessed_tap"): ring = getattr(config.rings, name) if name.endswith("_tap") and not ring.name: @@ -185,6 +198,34 @@ def validate_sweep_model(sweep: RadarSweepModel) -> None: raise ValueError("radar.sweep.stop_hz must be > radar.sweep.start_hz") +def normalize_antenna_geometry( + entries: Iterable[GprTxGeometryModel | GprRxGeometryModel | dict], + position_field: str, + position_count: int, +) -> list[dict]: + """Validate model/web geometry and supply zero for omitted coordinates.""" + context = "gpr.tx_geometry" if position_field == "output_pos" else "gpr.rx_geometry" + records, positions = [], set() + for entry in entries: + record = asdict(entry) if isinstance(entry, (GprTxGeometryModel, GprRxGeometryModel)) else entry + if not isinstance(record, dict): + raise ValueError(f"{context} entries must be records") + position = record.get(position_field) + if isinstance(position, bool) or not isinstance(position, int): + raise ValueError(f"{context} {position_field} must be an integer") + if not 0 <= position < position_count: + raise ValueError(f"{context} {position_field} is out of range [0, {position_count})") + if position in positions: + raise ValueError(f"{context} contains duplicate {position_field}: {position}") + positions.add(position) + coordinates = {key: finite_number(record.get(key, 0.0), f"{context}.{key}") + for key in ("x_m", "y_m", "z_m")} + if any(abs(value) > 3.4e38 for value in coordinates.values()): + raise ValueError(f"{context} coordinates must fit finite float32 numbers") + records.append({position_field: position, **coordinates}) + return records + + def validate_gpr_model( gpr: GprModel, *, @@ -206,31 +247,8 @@ def validate_gpr_model( if not math.isfinite(gpr.relative_permittivity) or float(gpr.relative_permittivity) <= 0.0: raise ValueError("gpr.relative_permittivity must be > 0") - seen_output_positions: set[int] = set() - for entry in gpr.tx_geometry: - if any(not math.isfinite(value) or abs(value) > 3.4e38 for value in (entry.x_m, entry.y_m, entry.z_m)): - raise ValueError("gpr.tx_geometry coordinates must be finite numbers") - output_pos = entry.output_pos - if isinstance(output_pos, bool) or not isinstance(output_pos, int): - raise ValueError("gpr.tx_geometry output_pos must be an integer") - if output_pos < 0 or output_pos >= int(output_switch_positions): - raise ValueError("gpr.tx_geometry output_pos is out of range") - if output_pos in seen_output_positions: - raise ValueError("gpr.tx_geometry contains duplicate output_pos") - seen_output_positions.add(output_pos) - - seen_input_positions: set[int] = set() - for entry in gpr.rx_geometry: - if any(not math.isfinite(value) or abs(value) > 3.4e38 for value in (entry.x_m, entry.y_m, entry.z_m)): - raise ValueError("gpr.rx_geometry coordinates must be finite numbers") - input_pos = entry.input_pos - if isinstance(input_pos, bool) or not isinstance(input_pos, int): - raise ValueError("gpr.rx_geometry input_pos must be an integer") - if input_pos < 0 or input_pos >= int(input_switch_positions): - raise ValueError("gpr.rx_geometry input_pos is out of range") - if input_pos in seen_input_positions: - raise ValueError("gpr.rx_geometry contains duplicate input_pos") - seen_input_positions.add(input_pos) + normalize_antenna_geometry(gpr.tx_geometry, "output_pos", output_switch_positions) + normalize_antenna_geometry(gpr.rx_geometry, "input_pos", input_switch_positions) def validate_combos( diff --git a/python_app/models/setting_fields.py b/python_app/models/setting_fields.py new file mode 100644 index 0000000..93e99cf --- /dev/null +++ b/python_app/models/setting_fields.py @@ -0,0 +1,63 @@ +"""Resolved profile bindings, derived from the GUI dataclass field declarations.""" + +from __future__ import annotations + +from dataclasses import dataclass, fields, is_dataclass +from typing import Iterator + +from python_app.models.gui_profile_schema import GuiStateModel, Scalar, Setting + + +@dataclass(frozen=True, slots=True) +class BoundSetting: + """A declaration resolved to its persistent field and processing page.""" + + path: tuple[str, ...] + spec: Setting + mode: str | None + + def read(self, gui: GuiStateModel) -> Scalar: + value: object = gui + for name in self.path: + value = getattr(value, name) + if not isinstance(value, (str, int, float)): + raise TypeError(f"Setting {'.'.join(self.path)} must be a scalar") + return value + + def write(self, gui: GuiStateModel, value: Scalar) -> None: + owner = gui + for name in self.path[:-1]: + owner = getattr(owner, name) + setattr(owner, self.path[-1], value) + + +def _settings(model: object, path: tuple[str, ...] = ()) -> Iterator[BoundSetting]: + if not is_dataclass(model) or isinstance(model, type): + return + for member in fields(model): + member_path = (*path, member.name) + spec = member.metadata.get("setting") + if isinstance(spec, Setting): + mode = member_path[1] if member_path[0] == "processing" and len(member_path) == 3 else None + yield BoundSetting(member_path, spec, mode) + else: + yield from _settings(getattr(model, member.name), member_path) + + +GUI_FIELDS = tuple(_settings(GuiStateModel())) + + +def active_live_fields(mode: str) -> tuple[BoundSetting, ...]: + """Use the selected page for shared fields; retain other fields' own values.""" + selected: dict[str, BoundSetting] = {} + for binding in GUI_FIELDS: + name = binding.spec.live + if name and (name not in selected or binding.mode == mode): + selected[name] = binding + return tuple(selected.values()) + + +def active_web_fields(mode: str) -> tuple[BoundSetting, ...]: + """Expose only settings belonging to the current processing page.""" + return tuple(binding for binding in GUI_FIELDS + if binding.spec.web and binding.mode in (None, mode)) diff --git a/python_app/orchestration/config_writer.py b/python_app/orchestration/config_writer.py index 9ac05bf..75e0b5d 100644 --- a/python_app/orchestration/config_writer.py +++ b/python_app/orchestration/config_writer.py @@ -6,14 +6,13 @@ import logging from pathlib import Path from python_app.file_io import atomic_write_json -from python_app.models.run_config_model import RunConfigModel, parse_combos_from_text +from python_app.models.run_config_schema import RunConfigModel from python_app.orchestration.preprocess_assets import ( - PREPROCESS_ASSET_KEYS, - PREPROCESS_ASSET_SPECS, + PREPROCESS_ASSET_NAMES, preprocess_asset_model, runtime_preprocess_asset_keys, ) -from python_app.storage.npz_store import NpzStore +from python_app.storage.npz.store import NpzStore logger = logging.getLogger(__name__) @@ -33,18 +32,14 @@ class ConfigWriter: config: RunConfigModel, ) -> None: """Export selected preprocess sets into runtime bundles and update config paths.""" - for key in PREPROCESS_ASSET_KEYS: + for key in PREPROCESS_ASSET_NAMES: preprocess_asset_model(config, key).bundle_path = "" for key in runtime_preprocess_asset_keys(config): - spec = PREPROCESS_ASSET_SPECS[key] asset = preprocess_asset_model(config, key) - bundle_path = self._runtime_dir / spec.runtime_filename + bundle_path = self._runtime_dir / f"{key}_bundle.bin" logger.debug("Exporting preprocess bundle %s (set=%s) -> %s", key, asset.set_name, bundle_path) - if key == "s21_object_calibration": - store.export_object_calibration_bundle(radar_key, asset.set_name, bundle_path, config) - else: - store.export_set_bundle(spec.set_kind, radar_key, asset.set_name, bundle_path) + store.export_set_bundle(key, radar_key, asset.set_name, bundle_path) asset.bundle_path = str(bundle_path) def write(self, config: RunConfigModel, output_path: Path) -> Path: @@ -52,6 +47,3 @@ class ConfigWriter: atomic_write_json(output_path, config.to_dict(), durable=True) logger.debug("Wrote run config to %s", output_path) return output_path - - -__all__ = ["ConfigWriter", "parse_combos_from_text"] diff --git a/python_app/orchestration/live_processing_config.py b/python_app/orchestration/live_processing_config.py index 286f39c..59972f3 100644 --- a/python_app/orchestration/live_processing_config.py +++ b/python_app/orchestration/live_processing_config.py @@ -2,7 +2,7 @@ from __future__ import annotations -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, field import logging from pathlib import Path @@ -15,14 +15,9 @@ class ProcessingLiveConfig: """Runtime-adjustable processing parameters shared with data processor.""" processor_mode: str = "pass_through" - pass_through_channel: str = "s21" - pass_through_fixed_y_enabled: bool = False - pass_through_y_min_db: float = -100.0 - pass_through_y_max_db: float = 0.0 bscan_axis: str = "abs" - bscan_channel: str = "s21" - bscan_cut_m: float = 0.0 - bscan_max_depth_m: float = 8.0 + bscan_cut_m: float = 0.824 + bscan_max_depth_m: float = 1.0 bscan_gain: float = 1.0 bscan_start_freq_mhz: float = 100.0 bscan_stop_freq_mhz: float = 8800.0 @@ -39,8 +34,7 @@ class ProcessingLiveConfig: gpr_object_min_frac: float = 0.7 gpr_score_mode: str = "combined" # Backprojection intra-sweep speed-correction mode: "int_minus" (full - # correction) or "int_focus" (focusing residual only). Mirrors Python - # Horns_motion_3libre.py MOTION_CORRECTION_MODE. + # correction) or "int_focus" (focusing residual only). gpr_motion_mode: str = "int_minus" gpr_max_detected_objects_to_draw: int = 5 gpr_draw_top_m_objects: int = 2 @@ -89,8 +83,8 @@ class ProcessingLiveConfig: # and keeps using `gpr_speed_m_s` from this file. ignore_socket_speed: bool = False reprocess_current_result: bool = True - history_command_seq: int = 0 - history_command: str = "none" + history_clear_before_ns: int = 0 + history_removed: list[dict[str, int]] = field(default_factory=list) settings_revision: int = 0 def __post_init__(self) -> None: diff --git a/python_app/orchestration/pipeline_metrics.py b/python_app/orchestration/pipeline_metrics.py index b5e8ff2..a88f36b 100644 --- a/python_app/orchestration/pipeline_metrics.py +++ b/python_app/orchestration/pipeline_metrics.py @@ -69,10 +69,6 @@ class PipelineMetrics: self._buffers: dict[str, deque[int]] = {} logger.debug("PipelineMetrics init: report_every=%d", self._report_every) - def set_log_sink(self, log_sink: Callable[[str], None] | None) -> None: - """Reassign the log sink (used when the GUI log appears after init).""" - self._log_sink = log_sink - def record(self, name: str, duration_ns: int) -> MetricReport | None: """Append one sample. Return a flushed report if the buffer is full.""" if duration_ns <= 0: diff --git a/python_app/orchestration/preprocess_assets.py b/python_app/orchestration/preprocess_assets.py index 77decad..e49e5fb 100644 --- a/python_app/orchestration/preprocess_assets.py +++ b/python_app/orchestration/preprocess_assets.py @@ -2,38 +2,14 @@ from __future__ import annotations -from dataclasses import dataclass +from python_app.models.run_config_schema import PreprocessAssetModel, RunConfigModel -from python_app.models.run_config_model import PreprocessAssetModel, RunConfigModel - - -@dataclass(frozen=True, slots=True) -class PreprocessAssetSpec: - """Stable description of one preprocess asset.""" - - display_name: str - set_kind: str - runtime_filename: str - channel: str - - -PREPROCESS_ASSET_SPECS = { - "s21_calibration": PreprocessAssetSpec("S21 Calibration", "s21_calibration", "s21_calibration_bundle.bin", "s21"), - "s21_reference": PreprocessAssetSpec("S21 Reference", "s21_reference", "s21_reference_bundle.bin", "s21"), - "s21_object_calibration": PreprocessAssetSpec("Object Calibration", "s21_object_calibration", "s21_object_calibration_bundle.bin", "s21"), - "s11_open": PreprocessAssetSpec("S11 Open", "s11_open", "s11_open_calibration_bundle.bin", "s11"), - "s11_short": PreprocessAssetSpec("S11 Short", "s11_short", "s11_short_calibration_bundle.bin", "s11"), - "s11_load": PreprocessAssetSpec("S11 Load", "s11_load", "s11_load_calibration_bundle.bin", "s11"), - "s11_reference": PreprocessAssetSpec("S11 Reference", "s11_reference", "s11_reference_bundle.bin", "s11"), +PREPROCESS_ASSET_NAMES = { + "s21_calibration": "S21 Calibration", + "s21_reference": "S21 Reference", + "s21_object_calibration": "Object Calibration", } - -PREPROCESS_ASSET_KEYS = tuple(PREPROCESS_ASSET_SPECS.keys()) -S21_PREPROCESS_ASSET_KEYS = ("s21_calibration", "s21_reference") -S11_PREPROCESS_ASSET_KEYS = ("s11_open", "s11_short", "s11_load", "s11_reference") -S11_PREPROCESS_CALIBRATION_KEYS = ("s11_open", "s11_short", "s11_load") -VISIBLE_PREPROCESS_ASSET_KEYS = (*S21_PREPROCESS_ASSET_KEYS, "s21_object_calibration") -CAPTURABLE_PREPROCESS_ASSET_KEYS = S21_PREPROCESS_ASSET_KEYS -REQUIRED_PREPROCESS_ASSET_KEYS = S21_PREPROCESS_ASSET_KEYS +REQUIRED_PREPROCESS_ASSET_KEYS = ("s21_calibration", "s21_reference") def preprocess_asset_model(config: RunConfigModel, key: str) -> PreprocessAssetModel: @@ -44,46 +20,16 @@ def preprocess_asset_model(config: RunConfigModel, key: str) -> PreprocessAssetM return config.preprocess.s21.reference if key == "s21_object_calibration": return config.preprocess.s21.object_calibration - if key == "s11_open": - return config.preprocess.s11.calibration.open - if key == "s11_short": - return config.preprocess.s11.calibration.short - if key == "s11_load": - return config.preprocess.s11.calibration.load - if key == "s11_reference": - return config.preprocess.s11.reference raise KeyError(f"Unknown preprocess asset key: {key}") def preprocess_asset_display_name(key: str) -> str: """Return human-readable label for preprocess asset.""" - return PREPROCESS_ASSET_SPECS[key].display_name - - -def preprocess_asset_channel(key: str) -> str: - """Return associated trace channel for preprocess asset.""" - return PREPROCESS_ASSET_SPECS[key].channel - - -def preprocess_asset_set_name(config: RunConfigModel, key: str) -> str: - """Return normalized selected set name for one preprocess asset.""" - return str(preprocess_asset_model(config, key).set_name).strip() + return PREPROCESS_ASSET_NAMES[key] def runtime_preprocess_asset_keys(config: RunConfigModel) -> tuple[str, ...]: - """Return preprocess assets that should be exported and validated for runtime.""" - enabled_keys: list[str] = list(REQUIRED_PREPROCESS_ASSET_KEYS) - if preprocess_asset_set_name(config, "s21_object_calibration"): - enabled_keys.append("s21_object_calibration") - - has_full_s11_calibration = all( - preprocess_asset_set_name(config, key) - for key in S11_PREPROCESS_CALIBRATION_KEYS - ) - if not has_full_s11_calibration: - return tuple(enabled_keys) - - enabled_keys.extend(S11_PREPROCESS_CALIBRATION_KEYS) - if preprocess_asset_set_name(config, "s11_reference"): - enabled_keys.append("s11_reference") - return tuple(enabled_keys) + """The through/reference pair plus the selected optional object correction.""" + return (*REQUIRED_PREPROCESS_ASSET_KEYS, "s21_object_calibration") if ( + config.preprocess.s21.object_calibration.set_name.strip() + ) else REQUIRED_PREPROCESS_ASSET_KEYS diff --git a/python_app/orchestration/process_supervisor.py b/python_app/orchestration/process_supervisor.py index 8ad403e..54ffe9c 100644 --- a/python_app/orchestration/process_supervisor.py +++ b/python_app/orchestration/process_supervisor.py @@ -1,4 +1,4 @@ -"""Process supervisor for lifecycle management of C++ pipeline binaries.""" +"""Process supervisor for lifecycle management of pipeline subprocesses.""" from __future__ import annotations @@ -133,13 +133,13 @@ class ProcessSupervisor: def is_running(self) -> bool: """Return whether acquisition-side processes are alive.""" - return self._is_alive("data_preprocessor") or self._is_alive("sweep_orchestrator") + return self._is_alive("data_preprocessor") or self._is_alive("acquisition") def is_processor_running(self) -> bool: """Return whether data processor process is alive.""" return self._is_alive("data_processor") - def start(self, config_path: Path, *, allow_clean_orchestrator_exit: bool = False) -> None: + def start(self, config_path: Path, *, allow_clean_acquisition_exit: bool = False) -> None: """Start required pipeline binaries and wait until they are ready.""" if self.is_running(): raise RuntimeError("Acquisition processes are already running") @@ -148,9 +148,9 @@ class ProcessSupervisor: self._kill_kamil_collectors() logger.info( - "Starting pipeline from config %s (allow_clean_orchestrator_exit=%s)", + "Starting pipeline from config %s (allow_clean_acquisition_exit=%s)", config_path, - allow_clean_orchestrator_exit, + allow_clean_acquisition_exit, ) acquisition_command = self._acquisition_command(config_path) command_specs = { @@ -164,12 +164,12 @@ class ProcessSupervisor: "--config", str(config_path), ], - "sweep_orchestrator": acquisition_command, + "acquisition": acquisition_command, } processor_was_running = self.is_processor_running() required_processes: list[str] = ["data_preprocessor"] - if not allow_clean_orchestrator_exit: - required_processes.append("sweep_orchestrator") + if not allow_clean_acquisition_exit: + required_processes.append("acquisition") if not processor_was_running: required_processes.insert(0, "data_processor") @@ -179,9 +179,9 @@ class ProcessSupervisor: self._spawn("data_preprocessor", command_specs["data_preprocessor"], allow_clean_exit=False) self._spawn( - "sweep_orchestrator", - command_specs["sweep_orchestrator"], - allow_clean_exit=allow_clean_orchestrator_exit, + "acquisition", + command_specs["acquisition"], + allow_clean_exit=allow_clean_acquisition_exit, ) self._wait_until_ready(required_processes) except Exception: @@ -196,12 +196,12 @@ class ProcessSupervisor: def stop(self) -> None: """Stop acquisition-side processes, keep processor process intact.""" - self._stop_processes(["sweep_orchestrator", "data_preprocessor"]) + self._stop_processes(["acquisition", "data_preprocessor"]) self._kill_kamil_collectors() - def stop_orchestrator(self) -> None: - """Stop orchestrator process only.""" - self._stop_processes(["sweep_orchestrator"]) + def stop_acquisition(self) -> None: + """Stop acquisition process only.""" + self._stop_processes(["acquisition"]) def stop_preprocessor(self) -> None: """Stop preprocessor process only.""" @@ -209,7 +209,7 @@ class ProcessSupervisor: def stop_all(self) -> None: """Stop all managed processes.""" - self._stop_processes(["sweep_orchestrator", "data_preprocessor", "data_processor"]) + self._stop_processes(["acquisition", "data_preprocessor", "data_processor"]) self._kill_kamil_collectors() def _spawn(self, name: str, command: list[str], *, allow_clean_exit: bool) -> None: @@ -276,7 +276,7 @@ class ProcessSupervisor: def _acquisition_command(self, config_path: Path) -> list[str]: """Select the same acquisition producer as the shell launcher.""" - return acquisition_command(self._project_root, config_path) + return acquisition_command(config_path) def _stop_processes(self, names: Iterable[str]) -> None: """Gracefully terminate processes, then force-kill on per-process timeout.""" @@ -552,7 +552,7 @@ class ProcessSupervisor: Reads `/proc//cmdline` so a recycled PID owned by an unrelated process is never killed. """ - native_names = {"data_processor", "data_preprocessor", "sweep_orchestrator"} + native_names = {"data_processor", "data_preprocessor"} producer_modules = { "python_app.scripts.matrix_raw_producer", "python_app.scripts.kamil_adc_raw_producer", diff --git a/python_app/orchestration/producer_selection.py b/python_app/orchestration/producer_selection.py index ba1b1c8..c554f64 100644 --- a/python_app/orchestration/producer_selection.py +++ b/python_app/orchestration/producer_selection.py @@ -7,29 +7,25 @@ from pathlib import Path import sys from python_app.models.run_config_schema import RadarModel +from python_app.models.json_fields import json_object, read_str -def acquisition_command(project_root: Path, config_path: Path) -> list[str]: +def acquisition_command(config_path: Path) -> list[str]: """Select replay first, then the configured model's acquisition producer.""" - payload = json.loads(config_path.read_text(encoding="utf-8")) - radar = payload.get("radar", {}) if isinstance(payload, dict) else {} - radar = radar if isinstance(radar, dict) else {} - replay = radar.get("replay", {}) - dataset = replay.get("dataset_dir") if isinstance(replay, dict) else None - defaults = RadarModel() - model = radar.get("model") - if model is None and isinstance(payload, dict): - model = defaults.model - driver_mode = radar.get("driver_mode") - if driver_mode is None: - driver_mode = defaults.driver_mode - if model in ("librevna", "librevna_multi") and driver_mode == "mock": - model = "librevna_multi" - module = "replay_raw_producer" if isinstance(dataset, str) and dataset.strip() else { + payload = json_object(json.loads(config_path.read_text(encoding="utf-8")), "run config") + radar = json_object(payload.get("radar"), "radar") + replay = json_object(radar.get("replay"), "radar.replay") + dataset = read_str(replay, "dataset_dir", "") + model = read_str(radar, "model", RadarModel().model) + producers = { + "librevna": "matrix_raw_producer", "librevna_multi": "matrix_raw_producer", "kamil_adc": "kamil_adc_raw_producer", - }.get(model if isinstance(model, str) else "") - command = [sys.executable, "-m", f"python_app.scripts.{module}"] if module else [ - str(project_root / "build/bin/sweep_orchestrator"), - ] - return [*command, "--config", str(config_path)] + } + if dataset.strip(): + module = "replay_raw_producer" + else: + if model not in producers: + raise ValueError(f"Unsupported radar model: {model}") + module = producers[model] + return [sys.executable, "-m", f"python_app.scripts.{module}", "--config", str(config_path)] diff --git a/python_app/orchestration/shm/__init__.py b/python_app/orchestration/shm/__init__.py index bb34e31..5f642e6 100644 --- a/python_app/orchestration/shm/__init__.py +++ b/python_app/orchestration/shm/__init__.py @@ -1,23 +1 @@ -"""Shared-memory ring readers and payload decoders.""" - -from python_app.orchestration.shm.binary_cursor import ByteCursor -from python_app.orchestration.shm.decoder import ( - PREPROC_MAGIC, - RAW_MAGIC, - RESULT_MAGIC, - decode_result_collection, - decode_trace_collection, -) -from python_app.orchestration.shm.ring_reader import ShmRingReader -from python_app.orchestration.shm.ring_writer import ShmRingWriter - -__all__ = [ - "ByteCursor", - "PREPROC_MAGIC", - "RAW_MAGIC", - "RESULT_MAGIC", - "ShmRingReader", - "ShmRingWriter", - "decode_result_collection", - "decode_trace_collection", -] +"""Shared-memory rings and payload decoding.""" diff --git a/python_app/orchestration/shm/decoder.py b/python_app/orchestration/shm/decoder.py index 137bfa9..b747080 100644 --- a/python_app/orchestration/shm/decoder.py +++ b/python_app/orchestration/shm/decoder.py @@ -14,9 +14,9 @@ from python_app.models.dataset_model import ( ) from python_app.orchestration.shm.binary_cursor import ByteCursor -RAW_MAGIC = 0x32574152 -PREPROC_MAGIC = 0x32525050 -RESULT_MAGIC = 0x324C5352 # RSL2: adds processing_duration_ns after monotonic_ns +RAW_MAGIC = 0x33574152 # RAW3 +PREPROC_MAGIC = 0x33505250 # PRP3 +RESULT_MAGIC = 0x334C5352 # RSL3: typed acquisition and processing metadata def decode_trace_collection(payload: bytes, expected_magic: int) -> SweepCollection: @@ -39,40 +39,23 @@ def decode_trace_collection(payload: bytes, expected_magic: int) -> SweepCollect freq_bytes = point_count * 4 freq = np.frombuffer(cursor.read_bytes(freq_bytes), dtype=" ResultCollection: if kind == 1: point_count = cursor.read_u32() freq = np.frombuffer(cursor.read_bytes(point_count * 4), dtype=" ResultCollection: collection_id = cursor.read_u64() monotonic_ns = cursor.read_u64() processing_duration_ns = cursor.read_u64() + settings_revision = cursor.read_u64() + capture_start_ns = cursor.read_u64() + capture_end_ns = cursor.read_u64() + speed_m_s = cursor.read_f32() collection_payload_count = cursor.read_u32() block_count = cursor.read_u32() @@ -174,10 +160,17 @@ def decode_result_collection(payload: bytes) -> ResultCollection: ) ) + if cursor.remaining_bytes() != 0: + raise ValueError("Unexpected trailing bytes in result collection") + return ResultCollection( collection_id=collection_id, monotonic_ns=monotonic_ns, processing_duration_ns=processing_duration_ns, + settings_revision=settings_revision, + capture_start_ns=capture_start_ns, + capture_end_ns=capture_end_ns, + speed_m_s=speed_m_s, collection_payloads=collection_payloads, blocks=blocks, ) diff --git a/python_app/orchestration/shm/ring_layout.py b/python_app/orchestration/shm/ring_layout.py new file mode 100644 index 0000000..05910b0 --- /dev/null +++ b/python_app/orchestration/shm/ring_layout.py @@ -0,0 +1,15 @@ +"""Shared ring layout; keep in sync with common_cpp/ipc/src/shm_ring.cpp.""" + +from pathlib import Path +import struct + +MAGIC = b"RDRRING2" +VERSION = 1 +HEADER = struct.Struct("<8sIIIIQQQ16x") +SLOT_HEADER_SIZE = 16 + + +def ring_path(name: str) -> Path: + if not name.startswith("/") or len(name) < 2 or "/" in name[1:] or "\x00" in name: + raise ValueError("ring_name must be '/' followed by a single nonempty filename") + return Path("/dev/shm") / name[1:] diff --git a/python_app/orchestration/shm/ring_reader.py b/python_app/orchestration/shm/ring_reader.py index 6a60cc1..d91a059 100644 --- a/python_app/orchestration/shm/ring_reader.py +++ b/python_app/orchestration/shm/ring_reader.py @@ -4,9 +4,7 @@ from __future__ import annotations import logging import mmap -from pathlib import Path import time -from typing import Final from python_app.models.dataset_model import ResultCollection, SweepCollection from python_app.orchestration.shm import atomics @@ -16,31 +14,23 @@ from python_app.orchestration.shm.decoder import ( decode_result_collection, decode_trace_collection, ) +from python_app.orchestration.shm.ring_layout import HEADER, MAGIC, SLOT_HEADER_SIZE, VERSION, ring_path logger = logging.getLogger(__name__) -_HEADER_SIZE: Final[int] = 64 -_SLOT_HEADER_SIZE: Final[int] = 16 -_MAGIC: Final[bytes] = b"RDRRING2" -_VERSION: Final[int] = 1 - class ShmRingReader: """Read binary payloads from a lock-free ring in `/dev/shm`.""" def __init__(self, ring_name: str, open_timeout_s: float = 2.0, open_poll_s: float = 0.01) -> None: """Open and validate ring by name, for example `/radar_results`.""" - if not ring_name.startswith("/") or len(ring_name) < 2 or "/" in ring_name[1:] or "\x00" in ring_name: - raise ValueError("ring_name must be '/' followed by a single nonempty filename") - - self._ring_name = ring_name - self._path = Path("/dev/shm") / ring_name[1:] + self.name = ring_name + self._path = ring_path(ring_name) self._wait_for_ring_file(timeout_s=open_timeout_s, poll_s=open_poll_s) - self._file = self._path.open("r+b", buffering=0) - self._mmap: mmap.mmap | None = None + with self._path.open("r+b", buffering=0) as file: + self._mmap = mmap.mmap(file.fileno(), 0) try: - self._mmap = mmap.mmap(self._file.fileno(), 0) self._validate_header_with_wait(timeout_s=1.0, poll_s=0.002) except BaseException: # A fail-fast open (absent/incompatible ring) must not leak the fd/mapping. @@ -48,18 +38,15 @@ class ShmRingReader: raise logger.debug( "Opened SHM ring reader %s (capacity=%d, slot_size=%d bytes)", - self._ring_name, + self.name, self.capacity, self.slot_size_bytes, ) def close(self) -> None: - """Close mmap and file handle.""" - if self._mmap is not None: - self._mmap.close() - self._mmap = None - self._file.close() - logger.debug("Closed SHM ring reader %s", self._ring_name) + """Close the mapping and its owned file descriptor; safe to repeat.""" + self._mmap.close() + logger.debug("Closed SHM ring reader %s", self.name) def pop_payload(self) -> bytes | None: """Read next payload from ring, or `None` if no unread payload exists.""" @@ -69,7 +56,7 @@ class ShmRingReader: return None index = read_seq % self.capacity - slot_offset = _HEADER_SIZE + index * (_SLOT_HEADER_SIZE + self.slot_size_bytes) + slot_offset = HEADER.size + index * (SLOT_HEADER_SIZE + self.slot_size_bytes) # Seqlock read mirroring the C++ pop: a slot is valid for read_seq R only if # its sequence equals R+1 and is unchanged across the payload copy (i.e. the @@ -78,7 +65,7 @@ class ShmRingReader: sequence = self._read_u64(slot_offset + 8) if sequence != read_seq + 1: # Producer overwrote this slot before we read it. Resync to latest. - logger.debug("Ring %s: slot lapped before read, resyncing to write_seq=%d", self._ring_name, write_seq) + logger.debug("Ring %s: slot lapped before read, resyncing to write_seq=%d", self.name, write_seq) atomics.advance(self._mmap, 32, write_seq) return None @@ -88,21 +75,21 @@ class ShmRingReader: if payload_size > self.slot_size_bytes: logger.debug( "Ring %s: payload_size %d exceeds slot %d, resyncing", - self._ring_name, + self.name, payload_size, self.slot_size_bytes, ) atomics.advance(self._mmap, 32, write_seq) return None - payload_offset = slot_offset + _SLOT_HEADER_SIZE + payload_offset = slot_offset + SLOT_HEADER_SIZE payload = bytes(self._mmap[payload_offset : payload_offset + payload_size]) atomics.fence() # Re-read the slot sequence after the copy; if it changed, the producer # overwrote this slot mid-copy and the payload is torn — discard and resync. if self._read_u64(slot_offset + 8) != read_seq + 1: - logger.debug("Ring %s: slot overwritten mid-copy, discarding torn payload", self._ring_name) + logger.debug("Ring %s: slot overwritten mid-copy, discarding torn payload", self.name) atomics.advance(self._mmap, 32, write_seq) return None @@ -130,39 +117,6 @@ class ShmRingReader: return None return decode_result_collection(payload) - def peek_latest_payload(self) -> bytes | None: - """Return the most recently published payload WITHOUT consuming it. - - Reads the newest slot through the seqlock and never advances `read_seq`, so a - viewer can peek the freshest frame while the ring's real consumer keeps its own - cursor — the two coexist without stealing each other's payloads. Inherently - latest-wins: always the freshest published frame, or `None` when nothing has - been published yet or the slot is being overwritten at this instant. - """ - write_seq = self._read_u64(24) - if write_seq == 0: - return None - - latest_seq = write_seq - 1 - index = latest_seq % self.capacity - slot_offset = _HEADER_SIZE + index * (_SLOT_HEADER_SIZE + self.slot_size_bytes) - - # Seqlock read with NO cursor advance: accept the slot only if its sequence - # equals the published value both before and after the copy (i.e. the producer - # did not lap this slot mid-read). Never touch read_seq, so the real consumer - # is undisturbed. - if self._read_u64(slot_offset + 8) != latest_seq + 1: - return None - payload_size = self._read_u32(slot_offset) - if payload_size > self.slot_size_bytes: - return None - payload_offset = slot_offset + _SLOT_HEADER_SIZE - payload = bytes(self._mmap[payload_offset : payload_offset + payload_size]) - atomics.fence() - if self._read_u64(slot_offset + 8) != latest_seq + 1: - return None - return payload - def drop_all(self) -> int: """Mark all unread slots as consumed and return number of dropped payloads.""" write_seq = self._read_u64(24) @@ -171,7 +125,7 @@ class ShmRingReader: return 0 dropped = int(write_seq - read_seq) atomics.advance(self._mmap, 32, write_seq) - logger.debug("Ring %s: dropped %d unread payload(s)", self._ring_name, dropped) + logger.debug("Ring %s: dropped %d unread payload(s)", self.name, dropped) return dropped @property @@ -190,27 +144,27 @@ class ShmRingReader: while True: magic = self._mmap[:8] version = self._read_u32(8) - if magic == _MAGIC and version == _VERSION: + if magic == MAGIC and version == VERSION: capacity = self.capacity slot_size = self.slot_size_bytes - expected_size = _HEADER_SIZE + capacity * (_SLOT_HEADER_SIZE + slot_size) + expected_size = HEADER.size + capacity * (SLOT_HEADER_SIZE + slot_size) if capacity > 0 and slot_size > 0 and slot_size % 8 == 0 and expected_size == len(self._mmap): return if time.monotonic() >= deadline: - if magic != _MAGIC: + if magic != MAGIC: got_magic = bytes(magic).hex() - expected_magic = _MAGIC.hex() + expected_magic = MAGIC.hex() raise RuntimeError( - f"Shared memory ring magic mismatch for {self._ring_name}: " + f"Shared memory ring magic mismatch for {self.name}: " f"got=0x{got_magic}, expected=0x{expected_magic}" ) - if version != _VERSION: + if version != VERSION: raise RuntimeError( - f"Shared memory ring version mismatch for {self._ring_name}: " - f"got={version}, expected={_VERSION}" + f"Shared memory ring version mismatch for {self.name}: " + f"got={version}, expected={VERSION}" ) - raise RuntimeError(f"Shared memory ring geometry mismatch for {self._ring_name}") + raise RuntimeError(f"Shared memory ring geometry mismatch for {self.name}") time.sleep(poll_s) @@ -219,12 +173,12 @@ class ShmRingReader: deadline = time.monotonic() + timeout_s while True: try: - if self._path.stat().st_size >= _HEADER_SIZE: + if self._path.stat().st_size >= HEADER.size: return except FileNotFoundError: pass if time.monotonic() >= deadline: - raise FileNotFoundError(f"Shared memory ring does not exist: {self._ring_name}") + raise FileNotFoundError(f"Shared memory ring does not exist: {self.name}") time.sleep(poll_s) def _read_u32(self, offset: int) -> int: diff --git a/python_app/orchestration/shm/ring_writer.py b/python_app/orchestration/shm/ring_writer.py index a4746e0..306f159 100644 --- a/python_app/orchestration/shm/ring_writer.py +++ b/python_app/orchestration/shm/ring_writer.py @@ -5,117 +5,74 @@ from __future__ import annotations import logging import mmap import os -from pathlib import Path -from typing import Final from python_app.models.run_config_schema import RingEndpointModel from python_app.models.run_config_validation import validate_ring_endpoint from python_app.orchestration.shm import atomics -logger = logging.getLogger(__name__) +from python_app.orchestration.shm.ring_layout import HEADER, MAGIC, SLOT_HEADER_SIZE, VERSION, ring_path -_HEADER_SIZE: Final[int] = 64 -_SLOT_HEADER_SIZE: Final[int] = 16 -_MAGIC: Final[bytes] = b"RDRRING2" -_VERSION: Final[int] = 1 +logger = logging.getLogger(__name__) class ShmRingWriter: - """Write binary payloads into the shared-memory ring used by C++ workers. + """The sole producer owns the ring and replaces incompatible stale segments. - The writer is the sole *owner* of the rings it opens: there is exactly one - producer per ring (the acquisition producer for the raw/raw_tap rings). On a - geometry mismatch with a pre-existing segment (e.g. a stale ring left by a prior - run with a different sweep config), the owner unlinks and recreates the segment - from scratch rather than truncating in place or diverging silently — mirroring - the clean-shm-on-restart contract on the C++/deploy side (#13). A non-owner must - never recreate a ring; readers and C++ consumers only ever attach to an existing - one. + Replacement uses unlink plus a new inode, preserving existing readers' mappings. + Compatible segments retain their unread payloads and sequence counters. """ def __init__(self, ring_name: str, capacity: int, slot_size_bytes: int) -> None: - """Open or create a POSIX SHM ring by name (as the ring owner).""" - if not ring_name.startswith("/") or len(ring_name) < 2 or "/" in ring_name[1:] or "\x00" in ring_name: - raise ValueError("ring_name must be '/' followed by a single nonempty filename") + self._path = ring_path(ring_name) validate_ring_endpoint(RingEndpointModel(ring_name, capacity, slot_size_bytes)) - self._ring_name = ring_name - self._capacity = int(capacity) - self._slot_size_bytes = int(slot_size_bytes) - self._mapped_size = _HEADER_SIZE + self._capacity * (_SLOT_HEADER_SIZE + self._slot_size_bytes) - self._path = Path("/dev/shm") / ring_name[1:] - self._file = self._mmap = None + self._capacity = capacity + self._slot_size_bytes = slot_size_bytes + self._mapped_size = HEADER.size + capacity * (SLOT_HEADER_SIZE + slot_size_bytes) + self._mmap = self._open_owned() + + def _open_owned(self) -> mmap.mmap: + """Reuse a compatible segment, otherwise replace it without truncating it.""" try: - self._open_owned() - except BaseException: - self.close() - raise - - def _open_owned(self) -> None: - """Open the ring, recreating it from scratch on a geometry/header mismatch. - - As the single owner of this ring we may safely discard a stale segment: a - size or header mismatch means the existing segment belongs to an earlier, - incompatible run, so we unlink it and create a fresh one instead of mapping - an inconsistent layout. - """ - created = not self._path.exists() - fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o660) - self._file = os.fdopen(fd, "r+b", buffering=0) - if created or self._path.stat().st_size != self._mapped_size: - # Wrong-sized stale segment: drop it entirely and recreate, so the file - # and any future mapping agree on geometry instead of being truncated - # under a producer/consumer that still expects the old layout. - if not created: - logger.info("Ring %s: stale segment with wrong size, recreating", self._ring_name) - self._file.close() - self._unlink_if_present() - fd = os.open(self._path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o660) - self._file = os.fdopen(fd, "r+b", buffering=0) - self._file.truncate(self._mapped_size) - created = True - - self._mmap = mmap.mmap(self._file.fileno(), self._mapped_size) - if created: - self._initialize_header() - logger.debug( - "Created SHM ring writer %s (capacity=%d, slot_size=%d bytes)", - self._ring_name, - self._capacity, - self._slot_size_bytes, - ) - return - - # Size matched but the header geometry/magic does not: the owner recreates - # rather than diverge. Unlink and reopen as a brand-new ring. - if not self._header_matches(): - logger.warning("Ring %s: header/geometry mismatch on existing segment, recreating", self._ring_name) - self._mmap.close() - self._file.close() - self._unlink_if_present() - fd = os.open(self._path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o660) - self._file = os.fdopen(fd, "r+b", buffering=0) - self._file.truncate(self._mapped_size) - self._mmap = mmap.mmap(self._file.fileno(), self._mapped_size) - self._initialize_header() - else: - logger.debug("Reusing existing SHM ring writer %s", self._ring_name) - - def _unlink_if_present(self) -> None: - """Remove the backing /dev/shm file if it exists (owner-only operation).""" - try: - self._path.unlink() + with self._path.open("r+b", buffering=0) as file: + if os.fstat(file.fileno()).st_size == self._mapped_size: + mapping = mmap.mmap(file.fileno(), self._mapped_size) + try: + if self._header_matches(mapping): + return mapping + except BaseException: + mapping.close() + raise + mapping.close() + logger.warning("Ring %s: stale segment, recreating", self._ring_name) except FileNotFoundError: pass + self._path.unlink(missing_ok=True) + fd = os.open(self._path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o660) + with os.fdopen(fd, "r+b", buffering=0) as file: + file.truncate(self._mapped_size) + mapping = mmap.mmap(file.fileno(), self._mapped_size) + try: + # New slots are already zero-filled. Publish the version last so readers + # cannot accept a partially initialized header. + mapping[:HEADER.size] = HEADER.pack(MAGIC, 0, self._capacity, self._slot_size_bytes, 0, 0, 0, 0) + atomics.store(mapping, 8, VERSION, 4) + except BaseException: + mapping.close() + raise + return mapping + + def _header_matches(self, mapping: mmap.mmap) -> bool: + return ( + atomics.load(mapping, 8, 4) == VERSION + and mapping[:8] == MAGIC + and atomics.load(mapping, 12, 4) == self._capacity + and atomics.load(mapping, 16, 4) == self._slot_size_bytes + ) + def close(self) -> None: - """Close mmap and file handle.""" - if self._mmap is not None: - self._mmap.close() - self._mmap = None - if self._file is not None: - self._file.close() - self._file = None - logger.debug("Closed SHM ring writer %s", self._ring_name) + """Close the mapping and its owned file descriptor; safe to repeat.""" + self._mmap.close() def push(self, payload: bytes) -> bool: """Push one payload with overwrite-oldest semantics on overflow.""" @@ -137,11 +94,11 @@ class ShmRingWriter: self._write_u64(40, self._read_u64(40) + 1) index = write_seq % self._capacity - slot_offset = _HEADER_SIZE + index * (_SLOT_HEADER_SIZE + self._slot_size_bytes) - payload_offset = slot_offset + _SLOT_HEADER_SIZE + slot_offset = HEADER.size + index * (SLOT_HEADER_SIZE + self._slot_size_bytes) + payload_offset = slot_offset + SLOT_HEADER_SIZE - # Invalidate the previous publication before changing its bytes. A peek or - # a lapped consumer must not accept the old sequence during this overwrite. + # Invalidate the previous publication before changing its bytes. A lapped + # consumer must not accept the old sequence during this overwrite. self._write_u64(slot_offset + 8, 0) atomics.fence() self._write_u32(slot_offset, len(payload)) @@ -151,50 +108,16 @@ class ShmRingWriter: self._write_u64(24, write_seq + 1) return True - @property - def ring_name(self) -> str: - """Return the POSIX SHM ring name.""" - return self._ring_name - @property def slot_size_bytes(self) -> int: """Return maximum payload size per slot.""" return self._slot_size_bytes - def _initialize_header(self) -> None: - # A new ftruncate-extended segment is already zero-filled by the kernel; - # avoid allocating and copying another complete ring just to initialize it. - self._mmap[:_HEADER_SIZE] = b"\x00" * _HEADER_SIZE - self._mmap[:8] = _MAGIC - self._write_u32(8, _VERSION) - self._write_u32(12, self._capacity) - self._write_u32(16, self._slot_size_bytes) - self._write_u32(20, 0) - self._write_u64(24, 0) - self._write_u64(32, 0) - self._write_u64(40, 0) - - def _header_matches(self) -> bool: - """Return whether the existing segment's header matches this ring's geometry. - - Used by the owner to decide whether a same-sized pre-existing segment can be - reused as-is or must be unlinked and recreated. - """ - return ( - self._mmap[:8] == _MAGIC - and self._read_u32(8) == _VERSION - and self._read_u32(12) == self._capacity - and self._read_u32(16) == self._slot_size_bytes - ) - - def _read_u32(self, offset: int) -> int: - return atomics.load(self._mmap, offset, 4) - def _read_u64(self, offset: int) -> int: return atomics.load(self._mmap, offset, 8) def _write_u32(self, offset: int, value: int) -> None: - atomics.store(self._mmap, offset, int(value), 4) + atomics.store(self._mmap, offset, value, 4) def _write_u64(self, offset: int, value: int) -> None: - atomics.store(self._mmap, offset, int(value), 8) + atomics.store(self._mmap, offset, value, 8) diff --git a/python_app/orchestration/shm_reader.py b/python_app/orchestration/shm_reader.py deleted file mode 100644 index 845978f..0000000 --- a/python_app/orchestration/shm_reader.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Facade exports for shared-memory ring reader and decoders.""" - -from python_app.orchestration.shm.binary_cursor import ByteCursor -from python_app.orchestration.shm.decoder import ( - PREPROC_MAGIC, - RAW_MAGIC, - RESULT_MAGIC, - decode_result_collection, - decode_trace_collection, -) -from python_app.orchestration.shm.ring_reader import ShmRingReader -from python_app.orchestration.shm.ring_writer import ShmRingWriter - -__all__ = [ - "ByteCursor", - "PREPROC_MAGIC", - "RAW_MAGIC", - "RESULT_MAGIC", - "ShmRingReader", - "ShmRingWriter", - "decode_result_collection", - "decode_trace_collection", -] diff --git a/python_app/scripts/check_snapshot_numpy.py b/python_app/scripts/check_snapshot_numpy.py deleted file mode 100644 index c5fec86..0000000 --- a/python_app/scripts/check_snapshot_numpy.py +++ /dev/null @@ -1,301 +0,0 @@ -"""Validate and visualize numpy-directory runtime snapshots.""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path -from typing import Any - -import numpy as np - - -def _load_json(path: Path) -> dict[str, Any]: - """Load JSON object from file path.""" - payload = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(payload, dict): - raise ValueError(f"JSON root must be object: {path}") - return payload - - -def _collection_dirs(stage_dir: Path) -> list[Path]: - """Return sorted list of collection directories for one stage.""" - if not stage_dir.exists(): - return [] - return sorted([path for path in stage_dir.iterdir() if path.is_dir()], key=lambda path: path.name) - - -def _first_trace_from_raw_or_pre(collection_dir: Path) -> tuple[np.ndarray, np.ndarray, str] | None: - """Return first trace payload from raw/preprocessed collection.""" - traces = _all_traces_from_raw_or_pre(collection_dir) - if not traces: - return None - return traces[0] - - -def _all_traces_from_raw_or_pre(collection_dir: Path) -> list[tuple[np.ndarray, np.ndarray, str]]: - """Load all traces from raw/preprocessed collection directory.""" - meta = _load_json(collection_dir / "meta.json") - traces = meta.get("traces", []) - if not isinstance(traces, list): - raise ValueError(f"Invalid traces in {collection_dir / 'meta.json'}") - - all_traces: list[tuple[np.ndarray, np.ndarray, str]] = [] - for trace_meta in traces: - if not isinstance(trace_meta, dict): - raise ValueError(f"Invalid trace record in {collection_dir / 'meta.json'}") - - freq_file = str(trace_meta.get("freq_file", "")) - s11_file = str(trace_meta.get("s11_file", "")) - s21_file = str(trace_meta.get("s21_file", "")) - freq = np.load(collection_dir / freq_file) - s11 = np.load(collection_dir / s11_file) - s21 = np.load(collection_dir / s21_file) - if freq.shape != s11.shape: - raise ValueError(f"Shape mismatch freq/s11 in {collection_dir}") - if freq.shape != s21.shape: - raise ValueError(f"Shape mismatch freq/s21 in {collection_dir}") - if not ( - np.isfinite(freq).all() - and np.isfinite(np.real(s11)).all() - and np.isfinite(np.imag(s11)).all() - and np.isfinite(np.real(s21)).all() - and np.isfinite(np.imag(s21)).all() - ): - raise ValueError(f"Non-finite values in {collection_dir}") - - label = f"i{int(trace_meta.get('input', 0))}_o{int(trace_meta.get('output', 0))}" - all_traces.append((np.asarray(freq, dtype=np.float64), np.asarray(s21, dtype=np.complex128), label)) - - return all_traces - - -def _first_trace_from_results(collection_dir: Path) -> tuple[np.ndarray, np.ndarray, str] | None: - """Return first trace payload from results collection.""" - traces = _all_traces_from_results(collection_dir) - if not traces: - return None - return traces[0] - - -def _all_traces_from_results(collection_dir: Path) -> list[tuple[np.ndarray, np.ndarray, str]]: - """Load all trace-like payloads from results collection directory.""" - meta = _load_json(collection_dir / "meta.json") - blocks = meta.get("blocks", []) - if not isinstance(blocks, list): - raise ValueError(f"Invalid blocks in {collection_dir / 'meta.json'}") - - all_traces: list[tuple[np.ndarray, np.ndarray, str]] = [] - for block in blocks: - if not isinstance(block, dict): - continue - block_dir_name = str(block.get("dir", "")) - block_dir = collection_dir / block_dir_name - payloads = block.get("payloads", []) - if not isinstance(payloads, list): - continue - for payload in payloads: - if not isinstance(payload, dict): - continue - kind = int(payload.get("kind", 0)) - if kind != 1: - continue - freq_file = str(payload.get("freq_file", "")) - trace_file = str(payload.get("trace_file", "")) - freq = np.load(block_dir / freq_file) - trace = np.load(block_dir / trace_file) - if freq.shape != trace.shape: - raise ValueError(f"Shape mismatch freq/trace in {block_dir}") - if not ( - np.isfinite(freq).all() - and np.isfinite(np.real(trace)).all() - and np.isfinite(np.imag(trace)).all() - ): - raise ValueError(f"Non-finite values in {block_dir}") - - label = ( - f"i{int(block.get('input', 0))}_o{int(block.get('output', 0))}_" - f"{str(payload.get('name', 'processor'))}" - ) - all_traces.append((np.asarray(freq, dtype=np.float64), np.asarray(trace, dtype=np.complex128), label)) - - return all_traces - - -def _validate_stage(stage_dir: Path, stage: str) -> tuple[list[int], list[tuple[np.ndarray, np.ndarray, str]]]: - """Validate one stage directory and collect representative traces.""" - collection_ids: list[int] = [] - traces: list[tuple[np.ndarray, np.ndarray, str]] = [] - for collection_dir in _collection_dirs(stage_dir): - meta = _load_json(collection_dir / "meta.json") - collection_ids.append(int(meta.get("collection_id", -1))) - if stage in {"raw", "preprocessed"}: - trace = _first_trace_from_raw_or_pre(collection_dir) - else: - trace = _first_trace_from_results(collection_dir) - if trace is not None: - traces.append(trace) - return collection_ids, traces - - -def _compare_two( - name: str, - first: tuple[np.ndarray, np.ndarray, str], - second: tuple[np.ndarray, np.ndarray, str], -) -> None: - """Print numerical difference metrics for two traces.""" - freq_a, data_a, label_a = first - freq_b, data_b, label_b = second - same_shape = freq_a.shape == freq_b.shape == data_a.shape == data_b.shape - if not same_shape: - print(f"[{name}] different shapes: {freq_a.shape}/{freq_b.shape} {data_a.shape}/{data_b.shape}") - return - - are_equal = np.array_equal(data_a, data_b) - diff = data_a - data_b - max_abs_diff = float(np.max(np.abs(diff))) - l2_diff = float(np.linalg.norm(diff)) - print( - f"[{name}] compare first two traces: {label_a} vs {label_b}, " - f"equal={are_equal}, max_abs_diff={max_abs_diff:.6g}, l2_diff={l2_diff:.6g}" - ) - - -def _plot_two( - stage: str, - first: tuple[np.ndarray, np.ndarray, str], - second: tuple[np.ndarray, np.ndarray, str], - output_dir: Path, -) -> None: - """Plot magnitude comparison for two traces.""" - try: - import matplotlib.pyplot as plt - except Exception as exc: # noqa: BLE001 - print(f"[{stage}] matplotlib is not available, plot skipped: {exc}") - return - - output_dir.mkdir(parents=True, exist_ok=True) - freq_a, data_a, label_a = first - freq_b, data_b, label_b = second - if freq_a.shape != freq_b.shape or data_a.shape != data_b.shape: - print(f"[{stage}] shapes differ, plot skipped") - return - - y_a = 20.0 * np.log10(np.maximum(np.abs(data_a), 1e-12)) - y_b = 20.0 * np.log10(np.maximum(np.abs(data_b), 1e-12)) - - fig, ax = plt.subplots(figsize=(10, 4)) - ax.plot(freq_a, y_a, linewidth=1.4, label=f"collection#1 {label_a}") - ax.plot(freq_b, y_b, linewidth=1.4, label=f"collection#2 {label_b}") - ax.set_title(f"{stage}: first two collections") - ax.set_xlabel("X axis") - ax.set_ylabel("Magnitude dB") - ax.grid(True, alpha=0.3) - ax.legend() - fig.tight_layout() - - png_path = output_dir / f"{stage}_first_two.png" - fig.savefig(png_path, dpi=150) - plt.close(fig) - print(f"[{stage}] plot saved: {png_path}") - - -def _plot_all_states_for_one_collection(stage: str, collection_dir: Path, output_dir: Path) -> None: - """Plot all switch-state traces available in one collection.""" - try: - import matplotlib.pyplot as plt - except Exception as exc: # noqa: BLE001 - print(f"[{stage}] matplotlib is not available, all-states plot skipped: {exc}") - return - - if stage in {"raw", "preprocessed"}: - traces = _all_traces_from_raw_or_pre(collection_dir) - else: - traces = _all_traces_from_results(collection_dir) - - if not traces: - print(f"[{stage}] no trace payloads found in {collection_dir.name}, all-states plot skipped") - return - - output_dir.mkdir(parents=True, exist_ok=True) - meta = _load_json(collection_dir / "meta.json") - collection_id = int(meta.get("collection_id", -1)) - - fig, ax = plt.subplots(figsize=(11, 5)) - for freq, data, label in traces: - y = 20.0 * np.log10(np.maximum(np.abs(data), 1e-12)) - ax.plot(freq, y, linewidth=1.2, label=label) - ax.set_title(f"{stage}: all switch states in one collection (id={collection_id})") - ax.set_xlabel("X axis") - ax.set_ylabel("Magnitude dB") - ax.grid(True, alpha=0.3) - ax.legend(fontsize=8, ncol=2) - fig.tight_layout() - - png_path = output_dir / f"{stage}_all_states_one_collection.png" - fig.savefig(png_path, dpi=150) - plt.close(fig) - print(f"[{stage}] all-states plot saved: {png_path} (traces={len(traces)})") - - -def _run(snapshot_dir: Path, output_dir: Path) -> None: - """Execute snapshot validation and plotting workflow.""" - config_profile_path = snapshot_dir / "config_profile.json" - if config_profile_path.exists(): - print(f"Snapshot: {snapshot_dir}\nconfig_profile={config_profile_path.name}") - else: - print(f"Snapshot: {snapshot_dir}") - - stages = ("raw", "preprocessed", "results") - for stage in stages: - stage_dir = snapshot_dir / stage - collection_dirs = _collection_dirs(stage_dir) - collection_ids, traces = _validate_stage(stage_dir, stage) - duplicate_id_count = len(collection_ids) - len(set(collection_ids)) - print( - f"[{stage}] collections={len(collection_ids)}, " - f"duplicate_ids={duplicate_id_count}, " - f"trace_samples={len(traces)}" - ) - if len(traces) >= 2: - _compare_two(stage, traces[0], traces[1]) - _plot_two(stage, traces[0], traces[1], output_dir) - else: - print(f"[{stage}] not enough trace-like collections to compare/plot (need >= 2)") - - if collection_dirs: - _plot_all_states_for_one_collection(stage, collection_dirs[-1], output_dir) - else: - print(f"[{stage}] no collections for all-states plot") - - -def main() -> None: - """CLI entrypoint.""" - parser = argparse.ArgumentParser(description="Validate and visualize runtime numpy snapshot collections.") - parser.add_argument( - "snapshot_dir", - type=Path, - help="Path to snapshot directory (contains raw/preprocessed/results).", - ) - parser.add_argument( - "--output-dir", - type=Path, - default=None, - help="Directory for output plots (default: /inspection_plots).", - ) - args = parser.parse_args() - - snapshot_dir = args.snapshot_dir.expanduser().resolve() - if not snapshot_dir.exists(): - raise FileNotFoundError(f"Snapshot directory not found: {snapshot_dir}") - - output_dir = ( - args.output_dir.expanduser().resolve() - if args.output_dir is not None - else snapshot_dir / "inspection_plots" - ) - _run(snapshot_dir, output_dir) - - -if __name__ == "__main__": - main() diff --git a/python_app/scripts/convert_legacy_preprocess_sets.py b/python_app/scripts/convert_legacy_preprocess_sets.py deleted file mode 100644 index 173a468..0000000 --- a/python_app/scripts/convert_legacy_preprocess_sets.py +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env python3 -"""Convert legacy preprocess set storage into the current two-channel format.""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path -import sys -from typing import Any - -import numpy as np - -PROJECT_ROOT = Path(__file__).resolve().parents[2] -if str(PROJECT_ROOT) not in sys.path: - sys.path.insert(0, str(PROJECT_ROOT)) - -from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData -from python_app.storage.npz_store import NpzStore - - -LEGACY_KIND_MAP: dict[str, str] = { - "calibration": "s21_calibration", - "reference": "s21_reference", - "s21_calibration": "s21_calibration", - "s21_reference": "s21_reference", - "s11_open": "s11_open", - "s11_short": "s11_short", - "s11_load": "s11_load", - "s11_reference": "s11_reference", -} - -S21_ONLY_TARGET_KINDS = {"s21_calibration", "s21_reference"} - - -def _build_parser() -> argparse.ArgumentParser: - """Return the argument parser for the legacy preprocess-set conversion CLI.""" - parser = argparse.ArgumentParser( - description=( - "Convert old preprocess-set storage from a legacy python_app/data tree into the " - "current format required by the new GUI/runtime." - ) - ) - parser.add_argument( - "source_data_dir", - type=Path, - help="Path to legacy python_app/data directory from the old project copy.", - ) - parser.add_argument( - "output_data_dir", - type=Path, - help="Destination directory where converted sets will be written in the new format.", - ) - parser.add_argument( - "--overwrite", - action="store_true", - help="Allow overwriting already converted destination sets.", - ) - return parser - - -def _load_json(path: Path) -> dict[str, Any]: - """Read a JSON file and return its top-level object, rejecting non-object roots.""" - payload = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(payload, dict): - raise ValueError(f"JSON root must be object: {path}") - return payload - - -def _read_combo_position(combo_payload: dict[str, Any], *, primary_key: str, alias_key: str) -> int: - """Return a switch position from a combo record, accepting the primary or alias key.""" - if primary_key in combo_payload: - return int(combo_payload[primary_key]) - if alias_key in combo_payload: - return int(combo_payload[alias_key]) - raise KeyError(f"Missing combo position field: {primary_key}/{alias_key}") - - -def _combo_suffix(input_pos: int, output_pos: int) -> str: - """Return the per-combo array-name suffix (e.g. ``i0_o1``) used in legacy NPZ keys.""" - return f"i{input_pos}_o{output_pos}" - - -def _load_array(arrays: Any, key: str, *, dtype: np.dtype[Any], label: str) -> np.ndarray: - """Return a flattened array of the given dtype from an NPZ mapping, by key.""" - if key not in arrays: - raise KeyError(f"Missing {label} array '{key}' in NPZ archive") - return np.asarray(arrays[key], dtype=dtype).reshape(-1) - - -def _load_legacy_collection(meta_path: Path, npz_path: Path, *, target_kind: str) -> SweepCollection: - """Build a SweepCollection from a legacy meta/NPZ pair for the given target kind. - - Reads per-combo frequency, S21 and (when present) S11 arrays. For S21-only target - kinds a missing S11 is filled with zeros; for any other kind a missing S11 is an error. - """ - meta = _load_json(meta_path) - combos_payload = meta.get("combos") - if not isinstance(combos_payload, list): - raise ValueError(f"Expected 'combos' list in {meta_path}") - - with np.load(npz_path) as arrays: - traces: list[TraceData] = [] - for combo_payload in combos_payload: - if not isinstance(combo_payload, dict): - raise ValueError(f"Expected combo object in {meta_path}") - - input_pos = _read_combo_position(combo_payload, primary_key="input", alias_key="input_pos") - output_pos = _read_combo_position(combo_payload, primary_key="output", alias_key="output_pos") - suffix = _combo_suffix(input_pos, output_pos) - - freq_key = str(combo_payload.get("freq_key") or f"freq_{suffix}") - s21_key = str(combo_payload.get("s21_key") or f"s21_{suffix}") - s11_key = str(combo_payload.get("s11_key") or f"s11_{suffix}") - - frequency_hz = _load_array(arrays, freq_key, dtype=np.float32, label="frequency") - s21 = _load_array(arrays, s21_key, dtype=np.complex64, label="S21") - - if frequency_hz.shape != s21.shape: - raise ValueError(f"Frequency/S21 shape mismatch in {npz_path}: {frequency_hz.shape} vs {s21.shape}") - - if s11_key in arrays: - s11 = _load_array(arrays, s11_key, dtype=np.complex64, label="S11") - elif target_kind in S21_ONLY_TARGET_KINDS: - s11 = np.zeros_like(s21, dtype=np.complex64) - else: - raise KeyError( - f"Missing S11 array '{s11_key}' in {npz_path}; " - f"cannot convert target kind '{target_kind}' without real S11 data" - ) - - if frequency_hz.shape != s11.shape: - raise ValueError(f"Frequency/S11 shape mismatch in {npz_path}: {frequency_hz.shape} vs {s11.shape}") - - traces.append( - TraceData( - combo=ComboKey(input=input_pos, output=output_pos), - frequency_hz=frequency_hz, - s11=s11, - s21=s21, - ) - ) - - return SweepCollection( - collection_id=int(meta.get("collection_id", 0)), - monotonic_ns=int(meta.get("monotonic_ns", 0)), - traces=traces, - ) - - -def _convert_one_set( - store: NpzStore, - *, - output_root: Path, - source_kind: str, - target_kind: str, - radar_key: str, - meta_path: Path, - overwrite: bool, -) -> None: - """Convert a single legacy set (meta + NPZ) and save it under the target kind. - - Raises if the companion NPZ is missing, or if the destination already exists and - ``overwrite`` is False. - """ - set_name = meta_path.stem - npz_path = meta_path.with_suffix(".npz") - if not npz_path.exists(): - raise FileNotFoundError(f"Missing NPZ archive for set '{set_name}': {npz_path}") - - target_dir = output_root / target_kind / radar_key - target_json = target_dir / f"{set_name}.json" - target_npz = target_dir / f"{set_name}.npz" - if not overwrite and (target_json.exists() or target_npz.exists()): - raise FileExistsError(f"Destination set already exists: {target_dir / set_name}") - - collection = _load_legacy_collection(meta_path, npz_path, target_kind=target_kind) - store.save_set(target_kind, radar_key, set_name, collection) - print( - f"[converted] {source_kind}/{radar_key}/{set_name} -> " - f"{target_kind}/{radar_key}/{set_name} (traces={len(collection.traces)})" - ) - - -def main() -> int: - """Walk the legacy data tree, convert every recognized set, and print a summary. - - Returns 0 on full success, 1 if any set failed to convert, or 2 if no convertible - sets were found. - """ - parser = _build_parser() - args = parser.parse_args() - - source_root = args.source_data_dir.expanduser().resolve() - output_root = args.output_data_dir.expanduser().resolve() - - if not source_root.exists(): - raise FileNotFoundError(f"Source data directory does not exist: {source_root}") - if source_root == output_root: - raise ValueError("Source and output directories must be different") - - store = NpzStore(output_root) - converted_count = 0 - skipped_kind_count = 0 - error_messages: list[str] = [] - - for source_kind_dir in sorted(path for path in source_root.iterdir() if path.is_dir()): - source_kind = source_kind_dir.name - target_kind = LEGACY_KIND_MAP.get(source_kind) - if target_kind is None: - skipped_kind_count += 1 - print(f"[skip-kind] {source_kind_dir}") - continue - - for radar_key_dir in sorted(path for path in source_kind_dir.iterdir() if path.is_dir()): - radar_key = radar_key_dir.name - for meta_path in sorted(radar_key_dir.glob("*.json")): - try: - _convert_one_set( - store, - output_root=output_root, - source_kind=source_kind, - target_kind=target_kind, - radar_key=radar_key, - meta_path=meta_path, - overwrite=bool(args.overwrite), - ) - converted_count += 1 - except Exception as exc: # noqa: BLE001 - error_messages.append(f"{meta_path}: {exc}") - print(f"[error] {meta_path}: {exc}") - - print( - "\nConversion summary:\n" - f" source root: {source_root}\n" - f" output root: {output_root}\n" - f" converted sets: {converted_count}\n" - f" skipped kinds: {skipped_kind_count}\n" - f" errors: {len(error_messages)}" - ) - - if error_messages: - return 1 - if converted_count == 0: - print("No convertible preprocess sets were found.") - return 2 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_app/scripts/convert_prog_libre_manual_to_vna_history.py b/python_app/scripts/convert_prog_libre_manual_to_vna_history.py deleted file mode 100644 index 65f796e..0000000 --- a/python_app/scripts/convert_prog_libre_manual_to_vna_history.py +++ /dev/null @@ -1,409 +0,0 @@ -"""Convert manual LibreVNA S11/S21 CSV captures in prog_libre to vna history JSON.""" - -from __future__ import annotations - -import argparse -import csv -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -import numpy as np - -from python_app.file_io import atomic_write_json - - -def _real_imag_keys(trace_prefix: str) -> tuple[str, str]: - """Return the LibreVNA CSV column names for a trace's real and imaginary parts.""" - return f"{trace_prefix}_Real", f"{trace_prefix}_Imaginary" - - -def _load_complex_trace(csv_path: Path, trace_prefix: str) -> tuple[np.ndarray, np.ndarray]: - """Load one LibreVNA CSV and return its (frequency_hz, complex trace) arrays.""" - real_key, imag_key = _real_imag_keys(trace_prefix) - frequencies: list[float] = [] - values: list[complex] = [] - - with csv_path.open(encoding="utf-8", newline="") as handle: - reader = csv.DictReader(handle) - for row in reader: - frequencies.append(float(row["Frequency"])) - values.append(complex(float(row[real_key]), float(row[imag_key]))) - - frequency_hz = np.asarray(frequencies, dtype=np.float64) - trace = np.asarray(values, dtype=np.complex128) - if frequency_hz.size == 0 or trace.size == 0: - raise ValueError(f"CSV has no points: {csv_path}") - if frequency_hz.shape != trace.shape: - raise ValueError(f"Frequency/trace size mismatch: {csv_path}") - return frequency_hz, trace - - -def _solve_one_port_osl( - open_trace: np.ndarray, - short_trace: np.ndarray, - load_trace: np.ndarray, -) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - """Solve ideal OSL one-port calibration coefficients.""" - directivity = load_trace - open_delta = open_trace - directivity - short_delta = short_trace - directivity - denom = open_delta - short_delta - - source_match = np.zeros_like(directivity) - reflection_tracking = np.ones_like(directivity) - - stable_mask = np.abs(denom) > 1e-18 - source_match[stable_mask] = (open_delta[stable_mask] + short_delta[stable_mask]) / denom[stable_mask] - reflection_tracking[stable_mask] = open_delta[stable_mask] * (1.0 - source_match[stable_mask]) - return directivity, source_match, reflection_tracking - - -def _apply_one_port_osl( - measured_trace: np.ndarray, - directivity: np.ndarray, - source_match: np.ndarray, - reflection_tracking: np.ndarray, -) -> np.ndarray: - """Apply OSL one-port error correction to a measured S11 trace and return it.""" - numerator = measured_trace - directivity - denominator = reflection_tracking + (source_match * numerator) - - corrected = np.array(numerator, copy=True) - stable_mask = np.abs(denominator) > 1e-18 - corrected[stable_mask] = numerator[stable_mask] / denominator[stable_mask] - return corrected - - -def _apply_through_calibration(measured_trace: np.ndarray, through_trace: np.ndarray) -> np.ndarray: - """Normalize a measured S21 trace by the through reference and return it.""" - corrected = np.array(measured_trace, copy=True) - stable_mask = np.abs(through_trace) > 1e-18 - corrected[stable_mask] = measured_trace[stable_mask] / through_trace[stable_mask] - return corrected - - -def _complex_to_points(values: np.ndarray) -> list[list[float]]: - """Return complex samples as ``[real, imag]`` pairs for JSON serialization.""" - return [[float(value.real), float(value.imag)] for value in values] - - -def _scan_file_sort_key(csv_path: Path) -> tuple[int, str]: - """Return a sort key ordering numeric scan filenames first, by integer value.""" - stem = csv_path.stem - return (int(stem), stem) if stem.isdigit() else (10**9, stem) - - -def _load_scan_series(folder: Path, trace_prefix: str) -> tuple[np.ndarray, list[tuple[str, np.ndarray]]]: - """Load every numbered scan CSV in a folder and return (frequency_hz, named traces). - - All scans must share a common frequency axis; a mismatch raises ValueError. - """ - scan_paths = [ - path - for path in sorted(folder.glob("*.csv"), key=_scan_file_sort_key) - if path.stem.isdigit() - ] - if not scan_paths: - raise FileNotFoundError(f"No numbered scan CSV files found in {folder}") - - base_frequency_hz: np.ndarray | None = None - scans: list[tuple[str, np.ndarray]] = [] - for path in scan_paths: - frequency_hz, trace = _load_complex_trace(path, trace_prefix) - if base_frequency_hz is None: - base_frequency_hz = frequency_hz - elif not np.allclose(base_frequency_hz, frequency_hz, rtol=0.0, atol=1e-6): - raise ValueError(f"Frequency axis mismatch in {path}") - scans.append((path.name, trace)) - - assert base_frequency_hz is not None - return base_frequency_hz, scans - - -def _require_matching_frequency_axis(label: str, left: np.ndarray, right: np.ndarray) -> None: - """Raise ValueError (tagged with ``label``) unless two frequency axes match within tolerance.""" - if not np.allclose(left, right, rtol=0.0, atol=1e-6): - raise ValueError(f"{label} frequency axes do not match") - - -def _build_history_payload( - *, - source_dir: Path, - mode: str, - frequency_hz: np.ndarray, - sweep_scans: list[tuple[str, np.ndarray]], - calibrated_scans: list[tuple[str, np.ndarray]], - reference_trace: np.ndarray, - primary_stage: str, - raw_record_count: int, - preprocessed_record_count: int, -) -> dict[str, Any]: - """Assemble the vna-system history payload from paired sweep and calibrated scans. - - Pairs each sweep scan with its calibrated counterpart by order (names must match), - attaching the shared reference trace and sweep config to every history entry. - """ - if len(sweep_scans) != len(calibrated_scans): - raise ValueError("Sweep/calibrated scan counts do not match") - - sweep_history: list[dict[str, Any]] = [] - reference_points = _complex_to_points(reference_trace) - for index, ((scan_name, sweep_trace), (cal_name, calibrated_trace)) in enumerate( - zip(sweep_scans, calibrated_scans, strict=True) - ): - if scan_name != cal_name: - raise ValueError(f"Scan ordering mismatch: {scan_name} vs {cal_name}") - - sweep_history.append( - { - "timestamp": float(index), - "sweep_points": _complex_to_points(sweep_trace), - "calibrated_points": _complex_to_points(calibrated_trace), - "reference_points": reference_points, - "vna_config": { - "mode": mode, - "start_freq": float(frequency_hz[0]), - "stop_freq": float(frequency_hz[-1]), - "points": int(frequency_hz.size), - }, - } - ) - - return { - "format": "vna-system-history-v1", - "converter": "python_app/scripts/convert_prog_libre_manual_to_vna_history.py", - "converted_at_utc": datetime.now(timezone.utc).isoformat(), - "source_snapshot_dir": str(source_dir.resolve()), - "input_index": 0, - "output_index": 0, - "primary_stage": primary_stage, - "raw_record_count": int(raw_record_count), - "preprocessed_record_count": int(preprocessed_record_count), - "sweep_history": sweep_history, - } - - -def _write_payload(output_path: Path, payload: dict[str, Any]) -> None: - """Write a payload as indented UTF-8 JSON, creating parent directories as needed.""" - atomic_write_json(output_path, payload, durable=True) - - -def _rmse(left: np.ndarray, right: np.ndarray) -> float: - """Return the root-mean-square magnitude difference between two complex traces.""" - return float(np.sqrt(np.mean(np.abs(left - right) ** 2))) - - -def _build_parser() -> argparse.ArgumentParser: - """Return the argument parser for the prog_libre-to-vna-history conversion CLI.""" - parser = argparse.ArgumentParser( - description="Convert manual LibreVNA S11/S21 CSV captures in prog_libre to vna history JSON.", - ) - parser.add_argument( - "--calibration-dir", - type=Path, - default=Path("prog_libre/calibration"), - help="Directory with calibration CSV files.", - ) - parser.add_argument( - "--raw-dir", - type=Path, - default=Path("prog_libre/1-6000mhz_no-calibrated_libre"), - help="Directory with uncalibrated scan CSV files and ref.csv.", - ) - parser.add_argument( - "--calibrated-dir", - type=Path, - default=Path("prog_libre/1-6000mhz_calibrated_libre"), - help="Directory with already calibrated scan CSV files and ref.csv.", - ) - parser.add_argument( - "--raw-output", - "--s11-raw-output", - dest="s11_raw_output", - type=Path, - default=Path("prog_libre/1-6000mhz_no-calibrated_libre_s11_osl_p1_vna_bscan_history.json"), - help="Output JSON for uncalibrated S11 scans after applying OSL calibration.", - ) - parser.add_argument( - "--calibrated-output", - "--s11-calibrated-output", - dest="s11_calibrated_output", - type=Path, - default=Path("prog_libre/1-6000mhz_calibrated_libre_s11_passthrough_vna_bscan_history.json"), - help="Output JSON for already calibrated S11 scans without extra calibration.", - ) - parser.add_argument( - "--s21-raw-output", - dest="s21_raw_output", - type=Path, - default=Path("prog_libre/1-6000mhz_no-calibrated_libre_s21_through_vna_bscan_history.json"), - help="Output JSON for uncalibrated S21 scans after applying through calibration.", - ) - parser.add_argument( - "--s21-calibrated-output", - dest="s21_calibrated_output", - type=Path, - default=Path("prog_libre/1-6000mhz_calibrated_libre_s21_passthrough_vna_bscan_history.json"), - help="Output JSON for already calibrated S21 scans without extra calibration.", - ) - return parser - - -def main() -> None: - """Calibrate and convert the prog_libre S11/S21 CSV captures into history JSON files. - - Builds OSL coefficients for S11 and a through reference for S21, applies them to the - uncalibrated scans, writes raw and pre-calibrated history files for both channels, and - prints RMSE diagnostics against the supplied calibrated folder. - """ - args = _build_parser().parse_args() - - calibration_dir = args.calibration_dir.expanduser().resolve() - raw_dir = args.raw_dir.expanduser().resolve() - calibrated_dir = args.calibrated_dir.expanduser().resolve() - - s11_raw_output = args.s11_raw_output.expanduser().resolve() - s11_calibrated_output = args.s11_calibrated_output.expanduser().resolve() - s21_raw_output = args.s21_raw_output.expanduser().resolve() - s21_calibrated_output = args.s21_calibrated_output.expanduser().resolve() - - s11_cal_frequency_hz, open_trace = _load_complex_trace(calibration_dir / "open_rfc18_p1.csv", "S11") - short_frequency_hz, short_trace = _load_complex_trace(calibration_dir / "short_rfc18_p1.csv", "S11") - load_frequency_hz, load_trace = _load_complex_trace(calibration_dir / "load_rfc18_p1.csv", "S11") - _require_matching_frequency_axis("S11 calibration", s11_cal_frequency_hz, short_frequency_hz) - _require_matching_frequency_axis("S11 calibration", s11_cal_frequency_hz, load_frequency_hz) - - directivity, source_match, reflection_tracking = _solve_one_port_osl(open_trace, short_trace, load_trace) - - s11_raw_frequency_hz, s11_raw_scans = _load_scan_series(raw_dir, "S11") - s11_calibrated_frequency_hz, s11_passthrough_scans = _load_scan_series(calibrated_dir, "S11") - _require_matching_frequency_axis("S11 raw vs calibration", s11_raw_frequency_hz, s11_cal_frequency_hz) - _require_matching_frequency_axis("S11 calibrated vs calibration", s11_calibrated_frequency_hz, s11_cal_frequency_hz) - - s11_raw_reference_frequency_hz, s11_raw_reference = _load_complex_trace(raw_dir / "ref.csv", "S11") - s11_calibrated_reference_frequency_hz, s11_calibrated_reference = _load_complex_trace(calibrated_dir / "ref.csv", "S11") - _require_matching_frequency_axis("S11 raw reference vs calibration", s11_raw_reference_frequency_hz, s11_cal_frequency_hz) - _require_matching_frequency_axis( - "S11 calibrated reference vs calibration", - s11_calibrated_reference_frequency_hz, - s11_cal_frequency_hz, - ) - - s11_corrected_scans = [ - ( - scan_name, - _apply_one_port_osl(trace, directivity, source_match, reflection_tracking), - ) - for scan_name, trace in s11_raw_scans - ] - s11_corrected_reference = _apply_one_port_osl(s11_raw_reference, directivity, source_match, reflection_tracking) - - s11_raw_payload = _build_history_payload( - source_dir=raw_dir, - mode="s11", - frequency_hz=s11_raw_frequency_hz, - sweep_scans=s11_raw_scans, - calibrated_scans=s11_corrected_scans, - reference_trace=s11_corrected_reference, - primary_stage="raw", - raw_record_count=len(s11_raw_scans), - preprocessed_record_count=len(s11_corrected_scans), - ) - s11_calibrated_payload = _build_history_payload( - source_dir=calibrated_dir, - mode="s11", - frequency_hz=s11_calibrated_frequency_hz, - sweep_scans=s11_passthrough_scans, - calibrated_scans=s11_passthrough_scans, - reference_trace=s11_calibrated_reference, - primary_stage="preprocessed", - raw_record_count=0, - preprocessed_record_count=len(s11_passthrough_scans), - ) - - s21_cal_frequency_hz, through_trace = _load_complex_trace(calibration_dir / "through21_rfc18.csv", "S21") - - s21_raw_frequency_hz, s21_raw_scans = _load_scan_series(raw_dir, "S21") - s21_calibrated_frequency_hz, s21_passthrough_scans = _load_scan_series(calibrated_dir, "S21") - _require_matching_frequency_axis("S21 raw vs calibration", s21_raw_frequency_hz, s21_cal_frequency_hz) - _require_matching_frequency_axis("S21 calibrated vs calibration", s21_calibrated_frequency_hz, s21_cal_frequency_hz) - - s21_raw_reference_frequency_hz, s21_raw_reference = _load_complex_trace(raw_dir / "ref.csv", "S21") - s21_calibrated_reference_frequency_hz, s21_calibrated_reference = _load_complex_trace(calibrated_dir / "ref.csv", "S21") - _require_matching_frequency_axis("S21 raw reference vs calibration", s21_raw_reference_frequency_hz, s21_cal_frequency_hz) - _require_matching_frequency_axis( - "S21 calibrated reference vs calibration", - s21_calibrated_reference_frequency_hz, - s21_cal_frequency_hz, - ) - - s21_corrected_scans = [ - ( - scan_name, - _apply_through_calibration(trace, through_trace), - ) - for scan_name, trace in s21_raw_scans - ] - s21_corrected_reference = _apply_through_calibration(s21_raw_reference, through_trace) - - s21_raw_payload = _build_history_payload( - source_dir=raw_dir, - mode="s21", - frequency_hz=s21_raw_frequency_hz, - sweep_scans=s21_raw_scans, - calibrated_scans=s21_corrected_scans, - reference_trace=s21_corrected_reference, - primary_stage="raw", - raw_record_count=len(s21_raw_scans), - preprocessed_record_count=len(s21_corrected_scans), - ) - s21_calibrated_payload = _build_history_payload( - source_dir=calibrated_dir, - mode="s21", - frequency_hz=s21_calibrated_frequency_hz, - sweep_scans=s21_passthrough_scans, - calibrated_scans=s21_passthrough_scans, - reference_trace=s21_calibrated_reference, - primary_stage="preprocessed", - raw_record_count=0, - preprocessed_record_count=len(s21_passthrough_scans), - ) - - _write_payload(s11_raw_output, s11_raw_payload) - _write_payload(s11_calibrated_output, s11_calibrated_payload) - _write_payload(s21_raw_output, s21_raw_payload) - _write_payload(s21_calibrated_output, s21_calibrated_payload) - - s11_rmse_values = [ - _rmse(corrected_trace, passthrough_trace) - for (_, corrected_trace), (_, passthrough_trace) in zip(s11_corrected_scans, s11_passthrough_scans, strict=True) - ] - s21_rmse_values = [ - _rmse(corrected_trace, passthrough_trace) - for (_, corrected_trace), (_, passthrough_trace) in zip(s21_corrected_scans, s21_passthrough_scans, strict=True) - ] - s11_reference_rmse = _rmse(s11_corrected_reference, s11_calibrated_reference) - s21_reference_rmse = _rmse(s21_corrected_reference, s21_calibrated_reference) - - print( - "Converted manual prog_libre captures to vna history JSON:\n" - f" S11 raw output: {s11_raw_output}\n" - f" S11 calibrated output: {s11_calibrated_output}\n" - f" S21 raw output: {s21_raw_output}\n" - f" S21 calibrated output: {s21_calibrated_output}\n" - f" sweep count: {len(s11_raw_scans)}\n" - f" points per sweep: {s11_raw_frequency_hz.size}\n" - f" S11 calibration: p1 ideal OSL\n" - f" S11 mean sweep RMSE vs provided calibrated folder: {float(np.mean(s11_rmse_values)):.6f}\n" - f" S11 max sweep RMSE vs provided calibrated folder: {float(np.max(s11_rmse_values)):.6f}\n" - f" S11 reference RMSE vs provided calibrated ref: {s11_reference_rmse:.6f}\n" - f" S21 calibration: through21 complex division\n" - f" S21 mean sweep RMSE vs provided calibrated folder: {float(np.mean(s21_rmse_values)):.6f}\n" - f" S21 max sweep RMSE vs provided calibrated folder: {float(np.max(s21_rmse_values)):.6f}\n" - f" S21 reference RMSE vs provided calibrated ref: {s21_reference_rmse:.6f}" - ) - - -if __name__ == "__main__": - main() diff --git a/python_app/scripts/convert_snapshot_to_vna_history.py b/python_app/scripts/convert_snapshot_to_vna_history.py deleted file mode 100644 index 17d548d..0000000 --- a/python_app/scripts/convert_snapshot_to_vna_history.py +++ /dev/null @@ -1,310 +0,0 @@ -"""Convert radar_system runtime snapshots into vna_system sweep-history JSON.""" - -from __future__ import annotations - -import argparse -import json -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -import numpy as np - -from python_app.file_io import atomic_write_json - - -from python_app.storage.npz.vna_history_json import TraceRecord, _build_sweep_history, _normalize_channel - - -@dataclass(frozen=True) -class CollectionRef: - """Minimal collection identity descriptor for stage-alignment diagnostics.""" - - stage_index: int - collection_id: int - monotonic_ns: int - - -def _load_json(path: Path) -> dict[str, Any]: - """Read a JSON file and return its top-level object, rejecting non-object roots.""" - payload = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(payload, dict): - raise ValueError(f"JSON root must be object: {path}") - return payload - - -def _collection_dirs(stage_dir: Path) -> list[Path]: - """Return collection subdirectories in numeric recording order (empty if absent).""" - if not stage_dir.exists(): - return [] - return sorted( - (path for path in stage_dir.iterdir() if path.is_dir()), - key=lambda path: (_parse_stage_index(path.name, 0), path.name), - ) - - -def _parse_stage_index(name: str, fallback: int) -> int: - """Return the leading numeric prefix of a collection dir name, or ``fallback``.""" - prefix = name.split("_", 1)[0] - return int(prefix) if prefix.isdigit() else fallback - - -def _pick_trace_meta(meta: dict[str, Any], input_index: int, output_index: int) -> dict[str, Any] | None: - """Return the trace record matching the given input/output switch indices, or None.""" - traces = meta.get("traces", []) - if not isinstance(traces, list): - return None - for trace in traces: - if not isinstance(trace, dict): - continue - if int(trace.get("input", -1)) == input_index and int(trace.get("output", -1)) == output_index: - return trace - return None - - -def _load_stage_records( - snapshot_dir: Path, - stage: str, - *, - input_index: int, - output_index: int, - channel: str, -) -> list[TraceRecord]: - """Load TraceRecords for one stage and one switch combo, for the given channel. - - Skips collections without a matching trace or array files; raises on shape mismatch - or non-finite samples. - """ - stage_dir = snapshot_dir / stage - records: list[TraceRecord] = [] - - for fallback_idx, collection_dir in enumerate(_collection_dirs(stage_dir)): - meta_path = collection_dir / "meta.json" - if not meta_path.exists(): - continue - - meta = _load_json(meta_path) - trace_meta = _pick_trace_meta(meta, input_index, output_index) - if trace_meta is None: - continue - - freq_file = str(trace_meta.get("freq_file", "")) - samples_file = str(trace_meta.get(f"{channel}_file", "")) - if not freq_file or not samples_file: - continue - - frequency_hz = np.asarray(np.load(collection_dir / freq_file), dtype=np.float64).reshape(-1) - samples = np.asarray(np.load(collection_dir / samples_file), dtype=np.complex128).reshape(-1) - if frequency_hz.shape != samples.shape: - raise ValueError(f"Shape mismatch in {collection_dir}: freq{frequency_hz.shape} vs {channel}{samples.shape}") - if frequency_hz.size == 0: - continue - if not ( - np.isfinite(frequency_hz).all() - and np.isfinite(np.real(samples)).all() - and np.isfinite(np.imag(samples)).all() - ): - raise ValueError(f"Non-finite values in {collection_dir}") - - records.append( - TraceRecord( - stage=stage, - collection_id=int(meta.get("collection_id", -1)), - monotonic_ns=int(meta.get("monotonic_ns", 0)), - stage_index=_parse_stage_index(collection_dir.name, fallback_idx), - frequency_hz=frequency_hz, - samples=samples, - capture_end_ns=int(trace_meta.get("capture_end_ns", 0)), - capture_start_ns=int(trace_meta.get("capture_start_ns", 0)), - ) - ) - - return records - - -def _load_stage_refs(snapshot_dir: Path, stage: str) -> list[CollectionRef]: - """Load `(index, collection_id, monotonic_ns)` for one snapshot stage.""" - stage_dir = snapshot_dir / stage - refs: list[CollectionRef] = [] - - for fallback_idx, collection_dir in enumerate(_collection_dirs(stage_dir)): - meta_path = collection_dir / "meta.json" - if not meta_path.exists(): - continue - meta = _load_json(meta_path) - refs.append( - CollectionRef( - stage_index=_parse_stage_index(collection_dir.name, fallback_idx), - collection_id=int(meta.get("collection_id", -1)), - monotonic_ns=int(meta.get("monotonic_ns", 0)), - ) - ) - return refs - - -def _stage_alignment_warning(pre_refs: list[CollectionRef], result_refs: list[CollectionRef]) -> str | None: - """Return warning text when preprocessed/results stages are not identity-aligned.""" - if not pre_refs or not result_refs: - return None - - pre_by_index = {ref.stage_index: ref for ref in pre_refs} - result_by_index = {ref.stage_index: ref for ref in result_refs} - common_indices = sorted(set(pre_by_index) & set(result_by_index)) - if not common_indices: - return None - - mismatches = 0 - first_mismatch: tuple[int, CollectionRef, CollectionRef] | None = None - for index in common_indices: - pre = pre_by_index[index] - result = result_by_index[index] - if pre.collection_id != result.collection_id or pre.monotonic_ns != result.monotonic_ns: - mismatches += 1 - if first_mismatch is None: - first_mismatch = (index, pre, result) - - if mismatches == 0: - return None - - assert first_mismatch is not None - idx, pre, result = first_mismatch - return ( - "WARNING: snapshot stages are not fully aligned (preprocessed vs results). " - f"Mismatches={mismatches}/{len(common_indices)}. " - f"First mismatch at index={idx}: " - f"pre=(id={pre.collection_id},ns={pre.monotonic_ns}) vs " - f"results=(id={result.collection_id},ns={result.monotonic_ns}). " - "Export uses preprocessed traces; loaded view in vna_system may differ from " - "radar_system on-screen replayed results." - ) - - -def _build_parser() -> argparse.ArgumentParser: - """Return the argument parser for the snapshot-to-vna-history conversion CLI.""" - parser = argparse.ArgumentParser( - description=( - "Convert radar_system snapshot (numpy-directory-v1) to a vna_system-compatible " - "history JSON file with `sweep_history`." - ) - ) - parser.add_argument("snapshot_dir", type=Path, help="Path to snapshot directory containing raw/preprocessed/results.") - parser.add_argument( - "-o", - "--output", - type=Path, - default=None, - help="Output JSON path (default: /vna_bscan_history.json).", - ) - parser.add_argument("--input", dest="input_index", type=int, default=0, help="Input switch index to export.") - parser.add_argument("--output-index", dest="output_index", type=int, default=0, help="Output switch index to export.") - parser.add_argument( - "--channel", - choices=("s21", "s11"), - default="s21", - help="Trace channel to export into sweep/calibrated points.", - ) - parser.add_argument( - "--primary-stage", - choices=("preprocessed", "raw"), - default="preprocessed", - help="Stage order to drive collection selection/alignment.", - ) - parser.add_argument( - "--last-n", - type=int, - default=0, - help="Keep only the last N sweeps in output (0 means all available).", - ) - return parser - - -def main() -> None: - """Convert one snapshot's chosen channel/combo into a vna-system history JSON file. - - Loads raw and preprocessed traces, builds the sweep history (optionally trimmed to the - last N sweeps), writes the output JSON, and prints a summary plus any stage-alignment - warning. - """ - parser = _build_parser() - args = parser.parse_args() - channel = _normalize_channel(args.channel) - - snapshot_dir = args.snapshot_dir.expanduser().resolve() - if not snapshot_dir.exists(): - raise FileNotFoundError(f"Snapshot directory not found: {snapshot_dir}") - - output_path = ( - args.output.expanduser().resolve() - if args.output is not None - else snapshot_dir / "vna_bscan_history.json" - ) - - raw_records = _load_stage_records( - snapshot_dir, - "raw", - input_index=args.input_index, - output_index=args.output_index, - channel=channel, - ) - preprocessed_records = _load_stage_records( - snapshot_dir, - "preprocessed", - input_index=args.input_index, - output_index=args.output_index, - channel=channel, - ) - if not raw_records and not preprocessed_records: - raise ValueError( - "No matching raw/preprocessed traces were found in snapshot " - f"for input={args.input_index}, output={args.output_index}." - ) - - sweep_history = _build_sweep_history( - raw_records, - preprocessed_records, - channel=channel, - primary_stage=args.primary_stage, - ) - if args.last_n > 0: - sweep_history = sweep_history[-args.last_n :] - if not sweep_history: - raise ValueError("Conversion produced empty sweep_history.") - - pre_refs = _load_stage_refs(snapshot_dir, "preprocessed") - result_refs = _load_stage_refs(snapshot_dir, "results") - alignment_warning = _stage_alignment_warning(pre_refs, result_refs) - - payload: dict[str, Any] = { - "format": "vna-system-history-v1", - "converter": "python_app/scripts/convert_snapshot_to_vna_history.py", - "converted_at_utc": datetime.now(timezone.utc).isoformat(), - "source_snapshot_dir": str(snapshot_dir), - "input_index": int(args.input_index), - "output_index": int(args.output_index), - "channel": channel, - "primary_stage": args.primary_stage, - "raw_record_count": len(raw_records), - "preprocessed_record_count": len(preprocessed_records), - "sweep_history": sweep_history, - } - if alignment_warning is not None: - payload["alignment_warning"] = alignment_warning - - atomic_write_json(output_path, payload, durable=True) - - print( - "Converted snapshot to vna_system history JSON:\n" - f" input snapshot: {snapshot_dir}\n" - f" output file: {output_path}\n" - f" channel: {channel}\n" - f" sweeps written: {len(sweep_history)}\n" - f" raw records: {len(raw_records)}\n" - f" pre records: {len(preprocessed_records)}" - ) - if alignment_warning is not None: - print(f"[convert-warning] {alignment_warning}") - - -if __name__ == "__main__": - main() diff --git a/python_app/scripts/create_neutral_preprocess_sets.py b/python_app/scripts/create_neutral_preprocess_sets.py deleted file mode 100644 index 2684cce..0000000 --- a/python_app/scripts/create_neutral_preprocess_sets.py +++ /dev/null @@ -1,94 +0,0 @@ -"""CLI to create neutral S21 preprocess sets for a profile (headless). - -Mirrors the GUI's "Create neutral sets" button: builds calibration = 1+0j and -reference = 0 collections for every combo of the profile and saves them under -the profile's radar key, so the data_preprocessor becomes a bit-exact -pass-through. Needed for replay profiles whose recorded data is already -preprocessed (see run_config_examples/run_config_replay_vna.example.json). - -Usage: - python -m python_app.scripts.create_neutral_preprocess_sets \ - --config run_config_examples/run_config_replay_vna.example.json \ - [--set-name neutral_replay] [--force] -""" - -from __future__ import annotations - -import argparse -import logging -from pathlib import Path - -from python_app.models.run_config_model import RunConfigModel -from python_app.storage.npz_store import NpzStore, radar_key_from_config -from python_app.workflows.kamil_adc_neutral_preprocess import ( - build_neutral_s21_sets, - supports_neutral_preprocess_sets, -) - -logger = logging.getLogger(__name__) - -PROJECT_ROOT = Path(__file__).resolve().parents[2] - - -def main() -> int: - """Create and save the neutral S21 calibration/reference pair.""" - parser = argparse.ArgumentParser(description="Create neutral S21 preprocess sets") - parser.add_argument("--config", required=True, type=Path, help="Path to a run_config profile") - parser.add_argument("--set-name", default="neutral_replay", help="Name for both saved sets") - parser.add_argument( - "--force", action="store_true", help="Overwrite the sets if they already exist" - ) - args = parser.parse_args() - - logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") - - config = RunConfigModel.load_from_path(args.config) - config.ensure_combos() - if not supports_neutral_preprocess_sets(config): - raise RuntimeError("Neutral S21 sets are not supported for this radar configuration") - - radar_key = radar_key_from_config( - model_name=config.radar.model, - serial=config.radar.serial, - sweep_start_hz=config.radar.sweep.start_hz, - sweep_stop_hz=config.radar.sweep.stop_hz, - sweep_points=config.radar.sweep.points, - ifbw_hz=config.radar.sweep.if_bandwidth_hz, - power_dbm=config.radar.sweep.power_dbm, - extra_serials=config.radar_key_extra_parts() or None, - ) - store = NpzStore(PROJECT_ROOT / "python_app/data") - - kinds = ("s21_calibration", "s21_reference") - existing = [ - kind - for kind in kinds - if args.set_name in store.list_sets(kind, radar_key) - ] - if existing and not args.force: - if len(existing) != len(kinds): - missing = ", ".join(kind for kind in kinds if kind not in existing) - raise RuntimeError( - f"Neutral set pair '{args.set_name}' is incomplete: missing {missing}; " - "use --force to replace both sets." - ) - logger.info( - "Sets '%s' already exist for %s (radar_key=%s); nothing to do (use --force to overwrite).", - args.set_name, ", ".join(existing), radar_key, - ) - return 0 - - calibration, reference = build_neutral_s21_sets(config) - store.save_sets(radar_key, args.set_name, dict(zip(kinds, (calibration, reference)))) - logger.info( - "Neutral S21 sets saved: set=%s, radar_key=%s, combos=%d, points=%d", - args.set_name, - radar_key, - len(calibration.traces), - int(calibration.traces[0].frequency_hz.size), - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_app/scripts/hardware_raw_orchestrator_test.py b/python_app/scripts/hardware_raw_orchestrator_test.py deleted file mode 100644 index 8a47d67..0000000 --- a/python_app/scripts/hardware_raw_orchestrator_test.py +++ /dev/null @@ -1,349 +0,0 @@ -"""Standalone GUI utility for inspecting raw orchestrator ring output.""" - -from __future__ import annotations - -import argparse -import ctypes -import ctypes.util -import json -import os -from pathlib import Path -import signal -import subprocess -import sys -import time - -import numpy as np -from PyQt6.QtCore import QTimer -from PyQt6.QtWidgets import QApplication, QLabel, QMainWindow, QVBoxLayout, QWidget -import pyqtgraph as pg - -PROJECT_ROOT = Path(__file__).resolve().parents[2] -if str(PROJECT_ROOT) not in sys.path: - sys.path.insert(0, str(PROJECT_ROOT)) - -from python_app.hardware_full.single_radar_service import create_single_radar_service -from python_app.models.run_config_model import RunConfigModel -from python_app.orchestration.shm_reader import ShmRingReader - - -def _shm_unlink(name: str) -> None: - """Best-effort unlink for POSIX shared-memory object.""" - libc_name = ctypes.util.find_library("c") - if libc_name is None: - return - - libc = ctypes.CDLL(libc_name, use_errno=True) - libc.shm_unlink.argtypes = [ctypes.c_char_p] - libc.shm_unlink.restype = ctypes.c_int - - result = libc.shm_unlink(name.encode("utf-8")) - if result == 0: - return - - err = ctypes.get_errno() - if err != 2: # ENOENT - raise OSError(err, f"shm_unlink failed for {name}") - - -def _read_raw_ring_name(config_path: Path) -> str: - """Extract raw ring name from run config.""" - config = json.loads(config_path.read_text(encoding="utf-8")) - ring_name = config["rings"]["raw"]["name"] - if not isinstance(ring_name, str) or not ring_name.startswith("/"): - raise RuntimeError("Config rings.raw.name must be a POSIX shm name starting with '/'") - return ring_name - - -def _read_native_summary(config_path: Path) -> str: - """Build short summary of radar/switch driver modes.""" - config = json.loads(config_path.read_text(encoding="utf-8")) - radar_mode = config["radar"]["driver_mode"] - switches = config["switches"] - if "port1" in switches and "port2" in switches: - port1_mode = switches["port1"]["driver_mode"] - port2_mode = switches["port2"]["driver_mode"] - else: - port1_mode = switches["output"]["driver_mode"] - port2_mode = switches["input"]["driver_mode"] - return f"radar={radar_mode}, port1={port1_mode}, port2={port2_mode}" - - -def _prepare_radar_if_needed(config_path: Path, *, strict: bool) -> str | None: - """Preconfigure native radar through Python service when requested.""" - config_payload = json.loads(config_path.read_text(encoding="utf-8")) - config = RunConfigModel.from_dict(config_payload) - if config.radar.driver_mode != "native": - return "Radar pre-configuration skipped (mock mode)." - if config.is_matrix_radar: - return "Radar pre-configuration skipped (matrix radar producer config)." - - radar_service = None - try: - radar_service = create_single_radar_service(config) - radar_service.open() - radar_service.configure(config.radar.sweep) - return "Radar pre-configuration completed." - except Exception as exc: - message = f"Radar pre-configuration failed ({exc})" - if strict: - raise - return f"{message}. Continuing with native C++ configuration." - finally: - if radar_service is not None: - radar_service.close() - - -class RawOrchestratorViewer(QMainWindow): - """Qt window that runs sweep_orchestrator and plots raw collections.""" - - def __init__(self, config_path: Path, reset_ring: bool, prepare_radar: bool, strict_prepare: bool) -> None: - """Initialize viewer, optionally prepare radar, and start polling.""" - super().__init__() - - self._config_path = config_path - self._raw_ring_name = _read_raw_ring_name(config_path) - self._orchestrator_process: subprocess.Popen[str] | None = None - self._raw_reader: ShmRingReader | None = None - - if reset_ring: - _shm_unlink(self._raw_ring_name) - - self._build_ui() - if prepare_radar: - self._status.setText("Preparing radar...") - prepare_status = _prepare_radar_if_needed(self._config_path, strict=strict_prepare) - if prepare_status is not None: - self._status.setText(prepare_status) - self._start_orchestrator() - self._open_reader_or_fail() - - self._timer = QTimer(self) - self._timer.setInterval(60) - self._timer.timeout.connect(self._poll) - self._timer.start() - - def _build_ui(self) -> None: - """Build viewer widgets and raw trace plot.""" - self.setWindowTitle("Raw Sweep Viewer (orchestrator)") - root = QWidget(self) - self.setCentralWidget(root) - - layout = QVBoxLayout(root) - - mode_summary = _read_native_summary(self._config_path) - self._status = QLabel(f"Starting... ({mode_summary})") - layout.addWidget(self._status) - - self._plot = pg.PlotWidget(background="#101418") - self._plot.showGrid(x=True, y=True, alpha=0.2) - self._plot.setLabel("bottom", "Frequency", units="Hz") - self._plot.setLabel("left", "Magnitude", units="dB") - layout.addWidget(self._plot) - - self.resize(1400, 900) - - def _start_orchestrator(self) -> None: - """Start sweep orchestrator subprocess.""" - binary = PROJECT_ROOT / "build/bin/sweep_orchestrator" - if not binary.exists(): - raise RuntimeError(f"Missing binary: {binary}. Build first with 'make -j4'") - - command = [str(binary), "--config", str(self._config_path)] - self._orchestrator_process = subprocess.Popen( - command, - cwd=PROJECT_ROOT, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - - def _open_reader_or_fail(self) -> None: - """Wait for raw ring readiness and open `ShmRingReader`.""" - if self._orchestrator_process is None: - raise RuntimeError("Orchestrator process is not started") - - shm_path = Path("/dev/shm") / self._raw_ring_name[1:] - deadline = time.monotonic() + 5.0 - last_reader_error: str | None = None - - while time.monotonic() < deadline: - return_code = self._orchestrator_process.poll() - if return_code is not None: - details = self._read_process_output(self._orchestrator_process) - raise RuntimeError(f"sweep_orchestrator exited with code {return_code}: {details}") - - if shm_path.exists(): - try: - self._raw_reader = ShmRingReader(self._raw_ring_name) - self._status.setText(f"Running: ring={self._raw_ring_name}") - return - except RuntimeError as exc: - last_reader_error = str(exc) - - time.sleep(0.05) - - if last_reader_error is not None: - raise RuntimeError( - f"Timed out waiting for raw ring header readiness: {self._raw_ring_name}; " - f"last error: {last_reader_error}" - ) - raise RuntimeError(f"Timed out waiting for raw ring file: {shm_path}") - - def _poll(self) -> None: - """Poll subprocess state and draw latest available raw collection.""" - if self._orchestrator_process is None: - return - - return_code = self._orchestrator_process.poll() - if return_code is not None: - details = self._read_process_output(self._orchestrator_process) - self._status.setText(f"Error: sweep_orchestrator exited ({return_code})") - raise RuntimeError(f"sweep_orchestrator exited with code {return_code}: {details}") - - if self._raw_reader is None: - return - - latest = None - for _ in range(16): - collection = self._raw_reader.pop_raw_collection() - if collection is None: - break - latest = collection - - if latest is not None: - self._draw_collection(latest) - - def _draw_collection(self, collection) -> None: - """Render all traces from one raw collection.""" - self._plot.clear() - palette = [ - "#4cc9f0", - "#f72585", - "#b8f2e6", - "#ffd166", - "#90be6d", - "#ff595e", - "#6a4c93", - "#1982c4", - "#ff9f1c", - "#2ec4b6", - "#e71d36", - "#a0c4ff", - ] - - for idx, trace in enumerate(collection.traces): - magnitude_db = 20.0 * np.log10(np.maximum(np.abs(trace.s21), 1e-12)) - label = f"input={trace.combo.input}, output={trace.combo.output}" - self._plot.plot( - trace.frequency_hz, - magnitude_db, - pen=pg.mkPen(palette[idx % len(palette)], width=1.6), - name=label, - ) - - self._status.setText( - f"Running: collection_id={collection.collection_id}, " - f"traces={len(collection.traces)}, ring={self._raw_ring_name}" - ) - - @staticmethod - def _read_process_output(process: subprocess.Popen[str]) -> str: - """Collect process stdout/stderr text for diagnostics.""" - stdout = "" - stderr = "" - if process.stdout is not None: - stdout = process.stdout.read().strip() - if process.stderr is not None: - stderr = process.stderr.read().strip() - - if stderr and stdout: - return f"{stderr}\nstdout:\n{stdout}" - if stderr: - return stderr - if stdout: - return f"stdout:\n{stdout}" - return "no output" - - def closeEvent(self, event) -> None: # noqa: N802 - """Stop subprocess and close reader before window destruction.""" - self._shutdown() - super().closeEvent(event) - - def _shutdown(self) -> None: - """Close reader and terminate subprocess.""" - if self._raw_reader is not None: - self._raw_reader.close() - self._raw_reader = None - - process = self._orchestrator_process - self._orchestrator_process = None - if process is None: - return - - if process.poll() is None: - process.terminate() - try: - process.wait(timeout=2.0) - except subprocess.TimeoutExpired: - process.kill() - process.wait(timeout=1.0) - - -def main() -> int: - """CLI entrypoint for raw orchestrator viewer.""" - parser = argparse.ArgumentParser(description="Run sweep_orchestrator and plot raw sweep collections") - parser.add_argument( - "--config", - type=Path, - default=PROJECT_ROOT / "run_config.json", - help="Path to run config JSON", - ) - parser.add_argument( - "--no-reset-ring", - action="store_true", - help="Do not unlink existing raw ring name before starting", - ) - parser.add_argument( - "--skip-radar-prepare", - action="store_true", - help="Skip Python pre-configuration of native radar before orchestrator start", - ) - parser.add_argument( - "--strict-radar-prepare", - action="store_true", - help="Fail immediately if Python pre-configuration cannot run", - ) - args = parser.parse_args() - - config_path = args.config.resolve() - if not config_path.exists(): - raise FileNotFoundError(f"Config file not found: {config_path}") - - if os.geteuid() == 0 and os.environ.get("SUDO_USER"): - print( - "Warning: running GUI test via sudo can break Qt DBus/session integration. " - "Prefer regular user with USB/GPIO permissions." - ) - - app = QApplication(sys.argv) - viewer = RawOrchestratorViewer( - config_path=config_path, - reset_ring=not args.no_reset_ring, - prepare_radar=not args.skip_radar_prepare, - strict_prepare=args.strict_radar_prepare, - ) - viewer.show() - - def _sig_handler(_signum, _frame): - """Close viewer gracefully on process signals.""" - viewer.close() - - signal.signal(signal.SIGINT, _sig_handler) - signal.signal(signal.SIGTERM, _sig_handler) - - return app.exec() - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_app/scripts/kamil_adc_calibrate.py b/python_app/scripts/kamil_adc_calibrate.py deleted file mode 100644 index 1257054..0000000 --- a/python_app/scripts/kamil_adc_calibrate.py +++ /dev/null @@ -1,649 +0,0 @@ -"""Calibrate the Kamil ADC reference-phase frequency law from live sweeps. - -Captures many raw sweeps from the collector and reports — and optionally writes -back — the calibration the runtime uses: - -* ``phase0_rad`` / ``phase1_rad`` — the unwrapped reference phase at the sweep - start and stop, taken as the MEDIAN over all captured sweeps. ``freq0_hz`` / - ``freq1_hz`` (the known sweep endpoints) come from config and are kept as-is. -* ``band.points`` — recommended as the median number of usable points landing - inside ``[band.start_hz, band.stop_hz]``, so the fixed output grid matches the - native density rather than inflating it. - -It also reports how reliably the configured band is covered (sweeps that do not -span it are rejected at runtime) and how many points the crop discards. - -The collector is launched with the ``do8_freq_ref`` arguments regardless of what -the on-disk config says, so the reference channel is always present. With -``--apply`` the config is migrated to that collector/args and the calibrated -anchors + recommended point count are written back. - -Run on the Pi, e.g.:: - - .venv/bin/python -m python_app.scripts.kamil_adc_calibrate \ - --config run_config_kamil_adc.pi.json --sweeps 200 --apply -""" - -from __future__ import annotations - -import argparse -from contextlib import suppress -import json -import logging -from pathlib import Path -import shutil -import statistics -import subprocess -import time - -import numpy as np - -from python_app.hardware_full.kamil_adc import KamilAdcService, apply_kamil_adc_laser_control -from python_app.hardware_full.kamil_adc.processing import ( - KamilAdcProcessingParams, - KamilAdcSweepProcessor, -) -from python_app.models.run_config_model import RunConfigModel - -logger = logging.getLogger("kamil_adc_calibrate") - -# The collector arguments that enable the DI8 reference overlay (mirrors the -# kamil example config / run_do8_freq_ref.sh). Forced on so calibration always -# sees the reference channel even if the on-disk config predates it. -DO8_FREQ_REF_ARGS = [ - "profile:phase", - "clock:internal", - "internal_ref_hz:2000000", - "start:di_syn2_rise", - "stop:di_syn2_fall", - "sample_clock_hz:max", - "range:2", - "duration_ms:100", - "packet_limit:0", - "do1_toggle_per_frame", - "do1_pair_subtract_avg", - "do8_freq_ref", - "do8_cycle_period:8", -] -COLLECTOR_PATH = "build/bin/kamil_adc_collector" - -# Hardware the daemon / a prior collector may be holding, and how to free it. -# Mirrors what start.sh does before an interactive launch (stop the daemon, kill -# an orphaned collector) so this tool can grab the L-Card E-502 + lasers too. -RADAR_SERVICE_NAME = "radar.service" -COLLECTOR_PROCESS_PATTERN = "kamil_adc_collector" -# The E-502 is not reacquirable the instant it is released; wait before opening. -_DEVICE_SETTLE_SECONDS = 8.0 - - -def _release_radar_hardware(*, settle_seconds: float = _DEVICE_SETTLE_SECONDS) -> None: - """Free the L-Card E-502 + lasers before we open our own collector. - - Mirrors ``start.sh`` on an interactive launch: stop the headless - ``radar.service`` daemon (if active) so it releases the hardware, then kill any - orphaned ``kamil_adc_collector`` a prior (e.g. SSH-killed) run left holding the - device. Both steps are best-effort and non-fatal — the collector's own - open-retry covers the residual settle time — this just removes the usual reason - it never frees up. Passwordless ``sudo systemctl stop radar.service`` is - provisioned in ``/etc/sudoers.d/radar``. - """ - freed = False - - systemctl = shutil.which("systemctl") - if systemctl is not None: - is_active = subprocess.run( - [systemctl, "is-active", "--quiet", RADAR_SERVICE_NAME], - check=False, - ).returncode == 0 - if is_active: - logger.info("Stopping %s so it releases the radar hardware...", RADAR_SERVICE_NAME) - stopped = subprocess.run( - ["sudo", systemctl, "stop", RADAR_SERVICE_NAME], check=False - ).returncode == 0 - if stopped: - freed = True - else: - logger.warning( - "Could not stop %s (need passwordless sudo?); continuing anyway", - RADAR_SERVICE_NAME, - ) - - pkill = shutil.which("pkill") - if pkill is not None: - # pkill returns 0 when it matched & signalled at least one process. - if subprocess.run( - [pkill, "-9", "-f", COLLECTOR_PROCESS_PATTERN], check=False - ).returncode == 0: - logger.info("Killed orphaned %s process(es) holding the device", COLLECTOR_PROCESS_PATTERN) - freed = True - - if freed: - logger.info("Waiting %.0fs for the L-Card E-502 to settle...", settle_seconds) - time.sleep(settle_seconds) - - -def _open_with_retry(service: KamilAdcService, *, attempts: int = 4, delay_s: float = 8.0) -> None: - """Open the collector, retrying the transient E-502 device-busy after a close. - - The L-Card device is not always reacquirable immediately after a previous - collector released it; a short backoff lets it settle before the next try. - """ - for attempt in range(1, attempts + 1): - try: - service.open() - return - except Exception as exc: # noqa: BLE001 — device-busy is expected and retried - logger.warning("Collector open failed (attempt %d/%d): %s", attempt, attempts, exc) - with suppress(Exception): - service.close() - if attempt < attempts: - time.sleep(delay_s) - raise RuntimeError(f"Collector did not open after {attempts} attempts") - - -def _capture_reference_phases(service: KamilAdcService, *, warmup: int, sweeps: int) -> list[np.ndarray]: - """Capture `sweeps` reference-phase arrays after discarding `warmup` sweeps.""" - for index in range(warmup): - service.read_raw_sweep() - if index == 0: - logger.info("Warming up (%d sweeps) while the sweep settles...", warmup) - - phases: list[np.ndarray] = [] - for index in range(sweeps): - raw = service.read_raw_sweep() - if raw.reference.size >= 2: - phases.append(np.unwrap(np.angle(raw.reference.astype(np.complex128)))) - if (index + 1) % 50 == 0: - logger.info("Captured %d/%d sweeps", index + 1, sweeps) - return phases - - -def _summarize(values: np.ndarray) -> str: - """Compact min / median / max summary for a 1-D array.""" - return f"min={np.min(values):.6g} median={np.median(values):.6g} max={np.max(values):.6g}" - - -def _analyze(phases: list[np.ndarray], config: RunConfigModel) -> dict: - """Derive the calibration and band diagnostics from captured phase arrays.""" - kamil = config.radar.kamil_adc - freq0, freq1 = kamil.phase_calibration.freq0_hz, kamil.phase_calibration.freq1_hz - band_start, band_stop = kamil.band.start_hz, kamil.band.stop_hz - - phase0 = float(statistics.median(float(phase[0]) for phase in phases)) - phase1 = float(statistics.median(float(phase[-1]) for phase in phases)) - if phase1 == phase0: - raise RuntimeError("Degenerate calibration: median start and stop phases are equal") - slope = (freq1 - freq0) / (phase1 - phase0) - - total_points = np.array([phase.size for phase in phases], dtype=np.float64) - f_starts, f_stops, in_band_counts, covers = [], [], [], [] - for phase in phases: - freqs = freq0 + (phase - phase0) * slope - lo, hi = float(np.min(freqs)), float(np.max(freqs)) - f_starts.append(lo) - f_stops.append(hi) - in_band_counts.append(int(np.count_nonzero((freqs >= band_start) & (freqs <= band_stop)))) - covers.append(lo <= band_start and hi >= band_stop) - - in_band = np.array(in_band_counts, dtype=np.float64) - f_start = np.array(f_starts) - f_stop = np.array(f_stops) - recommended_points = int(round(float(np.median(in_band)))) - # A band that ~95% of sweeps satisfy on each edge: start at the 95th percentile - # of per-sweep start frequencies, stop at the 5th percentile of stop frequencies. - rec_band_start = float(np.quantile(f_start, 0.95)) - rec_band_stop = float(np.quantile(f_stop, 0.05)) - rec_coverage = float(np.mean((f_start <= rec_band_start) & (f_stop >= rec_band_stop))) - return { - "sweeps": len(phases), - "phase0_rad": phase0, - "phase1_rad": phase1, - "phase_span_rad": phase1 - phase0, - "phase0_mad_rad": float(np.median(np.abs([float(p[0]) - phase0 for p in phases]))), - "phase1_mad_rad": float(np.median(np.abs([float(p[-1]) - phase1 for p in phases]))), - "freq0_hz": freq0, - "freq1_hz": freq1, - "band_start_hz": band_start, - "band_stop_hz": band_stop, - "total_points_median": float(np.median(total_points)), - "in_band_points_median": float(np.median(in_band)), - "recommended_points": recommended_points, - "cropped_fraction": 1.0 - float(np.median(in_band)) / float(np.median(total_points)), - "coverage_fraction": float(np.mean(covers)), - "lower_ok_fraction": float(np.mean(f_start <= band_start)), - "upper_ok_fraction": float(np.mean(f_stop >= band_stop)), - "rec_band_start_hz": rec_band_start, - "rec_band_stop_hz": rec_band_stop, - "rec_coverage_fraction": rec_coverage, - "f_start": f_start, - "f_stop": f_stop, - } - - -def _report(result: dict) -> None: - """Print a human-readable calibration report.""" - print("\n" + "=" * 72) - print(f"Kamil ADC calibration over {result['sweeps']} sweeps") - print("=" * 72) - print( - f"phase0_rad = {result['phase0_rad']:.6f} (median start phase, MAD " - f"{result['phase0_mad_rad']:.4f}) -> {result['freq0_hz'] / 1e9:.4f} GHz" - ) - print( - f"phase1_rad = {result['phase1_rad']:.6f} (median stop phase, MAD " - f"{result['phase1_mad_rad']:.4f}) -> {result['freq1_hz'] / 1e9:.4f} GHz" - ) - print(f"phase span = {result['phase_span_rad']:.4f} rad") - print( - f"points/sweep: total median={result['total_points_median']:.0f}, " - f"in-band median={result['in_band_points_median']:.0f}" - ) - print(f"recommended band.points = {result['recommended_points']}") - print( - f"band [{result['band_start_hz'] / 1e9:.3f}, {result['band_stop_hz'] / 1e9:.3f}] GHz: " - f"covered by {result['coverage_fraction'] * 100:.1f}% of sweeps " - f"(start<=lo: {result['lower_ok_fraction'] * 100:.1f}%, stop>=hi: {result['upper_ok_fraction'] * 100:.1f}%), " - f"{result['cropped_fraction'] * 100:.1f}% of points cropped" - ) - print(f"per-sweep start freq (GHz): {_summarize(result['f_start'] / 1e9)}") - print(f"per-sweep stop freq (GHz): {_summarize(result['f_stop'] / 1e9)}") - print( - f"suggested band for ~95%/edge: [{result['rec_band_start_hz'] / 1e9:.3f}, " - f"{result['rec_band_stop_hz'] / 1e9:.3f}] GHz -> covers " - f"{result['rec_coverage_fraction'] * 100:.1f}% of sweeps" - ) - if result["coverage_fraction"] < 0.95: - print( - "WARNING: many sweeps do not cover the configured band and would be rejected; " - "consider the suggested band above (or longer laser settling if sweeps are partial)." - ) - print("=" * 72 + "\n") - - -_TWO_PI = 2.0 * np.pi - - -def _capture_raw_sweeps( - service: KamilAdcService, *, warmup: int, sweeps: int -) -> list[tuple[np.ndarray, np.ndarray, np.ndarray]]: - """Capture full ``(steps, main, reference)`` arrays after discarding `warmup`. - - Unlike :func:`_capture_reference_phases`, this keeps the complete complex main - and reference samples (and their step indices) so the diagnosis can study the - raw phase, amplitude, and the pass-through behaviour — not just the endpoints. - """ - for index in range(warmup): - service.read_raw_sweep() - if index == 0: - logger.info("Warming up (%d sweeps) while the sweep settles...", warmup) - - captured: list[tuple[np.ndarray, np.ndarray, np.ndarray]] = [] - for index in range(sweeps): - raw = service.read_raw_sweep() - if raw.reference.size >= 2 and raw.main.size == raw.reference.size: - captured.append( - ( - np.asarray(raw.steps), - raw.main.astype(np.complex128), - raw.reference.astype(np.complex128), - ) - ) - if (index + 1) % 50 == 0: - logger.info("Captured %d/%d sweeps", index + 1, sweeps) - return captured - - -def _diagnose( - sweeps: list[tuple[np.ndarray, np.ndarray, np.ndarray]], config: RunConfigModel -) -> dict: - """Characterise how the reference phase / frequency axis moves across sweeps. - - The reference arm has a fixed electrical delay, so each sweep's unwrapped phase - is a straight ramp; the sweep start frequency wandering shifts that ramp - vertically (the *physical float*). When ``np.angle(ref[0])`` crosses the +/-pi - cut, ``np.unwrap`` re-anchors the whole ramp one 2*pi turn away (the *spurious - branch wrap*) — a ~``2*pi*slope`` jump of the entire frequency axis that wrecks - the band mapping for that sweep. - - This separates the two: it reconstructs the anchor phase per sweep, unwraps it - *across* sweeps (the candidate fix), and reports how that removes discrete 2*pi - steps while leaving the slow physical float intact. It also runs the live - pass-through (``KamilAdcSweepProcessor``) to flag which sweeps actually come out - distorted, and correlates those with the detected branch wraps. - """ - kamil = config.radar.kamil_adc - cal = kamil.phase_calibration - phase0, phase1 = float(cal.phase0_rad), float(cal.phase1_rad) - freq0, freq1 = float(cal.freq0_hz), float(cal.freq1_hz) - slope = (freq1 - freq0) / (phase1 - phase0) # Hz per rad - band_start, band_stop = float(kamil.band.start_hz), float(kamil.band.stop_hz) - - n = len(sweeps) - anchor_phase = np.empty(n) # unwrap(angle(ref))[0] == angle(ref[0]) in (-pi, pi] - stop_phase = np.empty(n) # unwrap(angle(ref))[-1] - span_rad = np.empty(n) # stop_phase - anchor_phase (fixed-delay invariant) - n_points = np.empty(n, dtype=np.int64) - f_lo = np.empty(n) # min reconstructed frequency (config calibration) - f_hi = np.empty(n) # max reconstructed frequency - ref_amp_med = np.empty(n) - - for i, (_steps, _main, reference) in enumerate(sweeps): - phase = np.unwrap(np.angle(reference)) - anchor_phase[i] = phase[0] - stop_phase[i] = phase[-1] - span_rad[i] = phase[-1] - phase[0] - n_points[i] = reference.size - freqs = freq0 + (phase - phase0) * slope - f_lo[i] = float(np.min(freqs)) - f_hi[i] = float(np.max(freqs)) - ref_amp_med[i] = float(np.median(np.abs(reference))) - - # --- Candidate fix: unwrap the anchor phase ACROSS the sweep-time axis ------ - # The physical float moves the anchor smoothly; a branch wrap injects a +/-2*pi - # step. Unwrapping along sweep index removes the discrete steps and keeps the - # slow float. branch_turns is the integer turns each sweep was wrapped by. - anchor_unwrapped = np.unwrap(anchor_phase) - branch_turns = np.round((anchor_unwrapped - anchor_phase) / _TWO_PI).astype(np.int64) - axis_shift = branch_turns * _TWO_PI * slope # Hz the whole axis was displaced - f_lo_corr = f_lo + axis_shift - f_hi_corr = f_hi + axis_shift - - # Branch wraps: consecutive anchor steps above pi are spurious 2*pi jumps. - anchor_step = np.diff(anchor_phase) - wrap_indices = np.nonzero(np.abs(anchor_step) > np.pi)[0] + 1 # sweep index after wrap - - covers_raw = (f_lo <= band_start) & (f_hi >= band_stop) - covers_corr = (f_lo_corr <= band_start) & (f_hi_corr >= band_stop) - - passthrough = _passthrough_flatness(sweeps, kamil) - - return { - "sweeps": n, - "slope_hz_per_rad": slope, - "axis_2pi_shift_hz": _TWO_PI * slope, - "band_start_hz": band_start, - "band_stop_hz": band_stop, - "anchor_phase": anchor_phase, - "anchor_unwrapped": anchor_unwrapped, - "branch_turns": branch_turns, - "n_branch_wraps": int(wrap_indices.size), - "wrap_indices": wrap_indices, - "span_rad": span_rad, - "n_points": n_points, - "f_lo": f_lo, - "f_hi": f_hi, - "f_lo_corr": f_lo_corr, - "f_hi_corr": f_hi_corr, - "axis_shift_hz": axis_shift, - "ref_amp_med": ref_amp_med, - "coverage_raw": float(np.mean(covers_raw)), - "coverage_corr": float(np.mean(covers_corr)), - # Largest sweep-to-sweep step of the *unwrapped* anchor: the fix is only - # valid if this stays below pi (otherwise the cross-sweep unwrap is itself - # ambiguous). Reported so we can confirm the float really is slow. - "max_unwrapped_anchor_step_rad": float(np.max(np.abs(np.diff(anchor_unwrapped)))) - if n > 1 - else 0.0, - **passthrough, - } - - -def _passthrough_flatness( - sweeps: list[tuple[np.ndarray, np.ndarray, np.ndarray]], kamil -) -> dict: - """Run the live processor and measure how flat each pass-through trace is. - - Builds the same :class:`KamilAdcSweepProcessor` the runtime uses, processes - every sweep onto the fixed grid, then measures each trace's deviation from a - robust (median) reference trace. A branch-wrapped sweep lands on a frequency - axis offset by ~one 2*pi turn, so after resampling it beats against the - reference — showing up as a large phase-residual std. This is the observable - symptom ("constant offset + oscillations") tied back to the raw-phase analysis. - """ - processor = KamilAdcSweepProcessor(KamilAdcProcessingParams.from_kamil_model(kamil)) - traces = [processor.process(main, reference) for _steps, main, reference in sweeps] - passed = np.array([trace is not None for trace in traces]) - n = len(traces) - phase_residual_std = np.full(n, np.nan) - amp_residual_std = np.full(n, np.nan) - - stack = np.array([trace for trace in traces if trace is not None], dtype=np.complex128) - if stack.shape[0] >= 3: - # Robust per-grid-point reference: median of real/imag over passing sweeps. - reference_trace = np.median(stack.real, axis=0) + 1j * np.median(stack.imag, axis=0) - safe_reference = np.where(np.abs(reference_trace) < 1e-12, 1.0 + 0j, reference_trace) - passed_index = 0 - for i, trace in enumerate(traces): - if trace is None: - continue - ratio = trace.astype(np.complex128) / safe_reference - phase_residual_std[i] = float(np.std(np.angle(ratio))) - amp_residual_std[i] = float(np.std(np.abs(ratio))) - passed_index += 1 - - return { - "passthrough_passed_fraction": float(np.mean(passed)), - "passthrough_phase_residual_std": phase_residual_std, - "passthrough_amp_residual_std": amp_residual_std, - } - - -def _report_diagnosis(result: dict) -> None: - """Print the detailed reference-phase / frequency-axis diagnosis.""" - n = result["sweeps"] - print("\n" + "=" * 72) - print(f"Kamil ADC reference-phase DIAGNOSIS over {n} sweeps") - print("=" * 72) - print( - f"calibration slope = {result['slope_hz_per_rad'] / 1e6:.4f} MHz/rad\n" - f"one 2*pi branch wrap = {result['axis_2pi_shift_hz'] / 1e6:.2f} MHz axis shift" - ) - print("-" * 72) - print( - f"anchor phase angle(ref[0]): {_summarize(result['anchor_phase'])} rad\n" - f"phase span (stop - start) : {_summarize(result['span_rad'])} rad " - f"(std {np.std(result['span_rad']):.4f} -> fixed-delay {'OK' if np.std(result['span_rad']) < 1.0 else 'SUSPECT'})\n" - f"points per sweep : {_summarize(result['n_points'].astype(float))}" - ) - print("-" * 72) - print( - f"branch wraps detected : {result['n_branch_wraps']} " - f"(at sweep indices {result['wrap_indices'].tolist()})" - ) - print(f"branch turns range : {_summarize(result['branch_turns'].astype(float))} turns") - print( - f"max sweep-to-sweep step of UNWRAPPED anchor = " - f"{result['max_unwrapped_anchor_step_rad']:.4f} rad " - f"({'< pi: cross-sweep unwrap is unambiguous' if result['max_unwrapped_anchor_step_rad'] < np.pi else '>= pi: AMBIGUOUS, fix may misstep'})" - ) - print("-" * 72) - print("reconstructed band edges (GHz), config calibration:") - print(f" raw start {_summarize(result['f_lo'] / 1e9)} stop {_summarize(result['f_hi'] / 1e9)}") - print(f" corrected start {_summarize(result['f_lo_corr'] / 1e9)} stop {_summarize(result['f_hi_corr'] / 1e9)}") - print( - f"band [{result['band_start_hz'] / 1e9:.3f}, {result['band_stop_hz'] / 1e9:.3f}] GHz covered: " - f"raw {result['coverage_raw'] * 100:.1f}% -> corrected {result['coverage_corr'] * 100:.1f}%" - ) - print("-" * 72) - phase_res = result["passthrough_phase_residual_std"] - finite = phase_res[np.isfinite(phase_res)] - print(f"pass-through: {result['passthrough_passed_fraction'] * 100:.1f}% of sweeps covered the band") - if finite.size: - print(f" phase-residual std vs median trace: {_summarize(finite)} rad") - # Sweeps whose pass-through trace is far from flat (the visible glitches). - bad = np.nonzero(phase_res > (np.median(finite) + 5.0 * (np.std(finite) + 1e-9)))[0] - print(f" distorted sweeps (>5 sigma residual): {bad.tolist()}") - print(f" do they coincide with branch wraps? wraps at {result['wrap_indices'].tolist()}") - print("-" * 72) - print( - "HOW TO READ THIS:\n" - " * phase span std small -> fixed reference delay confirmed (model holds).\n" - " * 'branch wraps' are the suspected glitches: angle(ref[0]) crossing +/-pi.\n" - " * If the UNWRAPPED-anchor max step stays < pi, the sweep-to-sweep float is\n" - " slow enough that a CROSS-SWEEP unwrap can separate the real drift from the\n" - " spurious 2*pi wrap. That is the candidate fix.\n" - " * Expect: 'corrected' band edges move CONTINUOUSLY (no ~157 MHz steps) while\n" - " 'raw' jumps by one wrap at each detected wrap index, and the distorted\n" - " pass-through sweeps line up with those wrap indices." - ) - print("=" * 72 + "\n") - - -def _dump_diagnosis( - path: Path, - result: dict, - sweeps: list[tuple[np.ndarray, np.ndarray, np.ndarray]], -) -> None: - """Save per-sweep metrics and the full raw sweeps to an ``.npz`` for offline study.""" - arrays = {key: value for key, value in result.items() if isinstance(value, np.ndarray)} - arrays["raw_steps"] = np.array([steps for steps, _m, _r in sweeps], dtype=object) - arrays["raw_main"] = np.array([main for _s, main, _r in sweeps], dtype=object) - arrays["raw_reference"] = np.array([reference for _s, _m, reference in sweeps], dtype=object) - np.savez(path, **arrays) - print(f"Saved diagnosis data ({len(sweeps)} sweeps) to {path}") - - -def _plot_diagnosis(path: Path, result: dict) -> None: - """Render the key diagnostic plots to a PNG (no-op if matplotlib is missing).""" - try: - import matplotlib - - matplotlib.use("Agg") - import matplotlib.pyplot as plt - except Exception as exc: # noqa: BLE001 — plotting is optional - logger.warning("Plotting skipped (matplotlib unavailable: %s)", exc) - return - - sweep_index = np.arange(result["sweeps"]) - fig, axes = plt.subplots(3, 1, figsize=(11, 12), sharex=True) - - axes[0].plot(sweep_index, result["anchor_phase"], ".", label="angle(ref[0]) raw (wrapped)") - axes[0].plot(sweep_index, result["anchor_unwrapped"], "-", label="unwrapped across sweeps (fix)") - for wrap in result["wrap_indices"]: - axes[0].axvline(wrap, color="r", alpha=0.3) - axes[0].set_ylabel("anchor phase [rad]") - axes[0].legend(loc="best") - axes[0].set_title("Reference anchor phase: raw wraps vs cross-sweep unwrap") - - axes[1].plot(sweep_index, result["f_lo"] / 1e9, ".", label="start raw") - axes[1].plot(sweep_index, result["f_lo_corr"] / 1e9, "-", label="start corrected") - axes[1].plot(sweep_index, result["f_hi"] / 1e9, ".", label="stop raw") - axes[1].plot(sweep_index, result["f_hi_corr"] / 1e9, "-", label="stop corrected") - axes[1].axhline(result["band_start_hz"] / 1e9, color="k", ls="--", alpha=0.5) - axes[1].axhline(result["band_stop_hz"] / 1e9, color="k", ls="--", alpha=0.5) - axes[1].set_ylabel("reconstructed band edge [GHz]") - axes[1].legend(loc="best") - - axes[2].plot(sweep_index, result["passthrough_phase_residual_std"], ".", label="phase residual std") - for wrap in result["wrap_indices"]: - axes[2].axvline(wrap, color="r", alpha=0.3, label="_branch wrap") - axes[2].set_ylabel("pass-through residual [rad]") - axes[2].set_xlabel("sweep index") - axes[2].legend(loc="best") - - fig.tight_layout() - fig.savefig(path, dpi=110) - plt.close(fig) - print(f"Saved diagnosis plot to {path}") - - -def _apply(config_path: Path, result: dict) -> None: - """Write the migrated collector args + calibrated anchors + points to the config.""" - payload = json.loads(config_path.read_text(encoding="utf-8")) - kamil = payload.setdefault("radar", {}).setdefault("kamil_adc", {}) - kamil["executable_path"] = COLLECTOR_PATH - kamil["args"] = list(DO8_FREQ_REF_ARGS) - kamil["phase_calibration"] = { - "phase0_rad": result["phase0_rad"], - "freq0_hz": result["freq0_hz"], - "phase1_rad": result["phase1_rad"], - "freq1_hz": result["freq1_hz"], - } - kamil["band"] = { - "start_hz": result["band_start_hz"], - "stop_hz": result["band_stop_hz"], - "points": result["recommended_points"], - } - config_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - print(f"Applied calibration to {config_path}") - - -def main() -> int: - parser = argparse.ArgumentParser(description="Calibrate Kamil ADC reference phase -> frequency") - parser.add_argument("--config", required=True, type=Path, help="Path to the kamil_adc run config") - parser.add_argument("--sweeps", type=int, default=200, help="Sweeps to average (default 200)") - parser.add_argument("--warmup", type=int, default=10, help="Sweeps to discard first (default 10)") - parser.add_argument("--apply", action="store_true", help="Write the calibration back to --config") - parser.add_argument("--no-laser", action="store_true", help="Skip laser setup (already running)") - parser.add_argument( - "--no-release", - action="store_true", - help="Do not stop radar.service / kill orphaned collectors before opening", - ) - parser.add_argument( - "--diagnose", - action="store_true", - help="Investigate reference-phase branch wraps instead of calibrating", - ) - parser.add_argument( - "--dump", type=Path, default=None, help="(--diagnose) save raw sweeps + metrics to this .npz" - ) - parser.add_argument( - "--plot", type=Path, default=None, help="(--diagnose) render diagnostic plots to this .png" - ) - args = parser.parse_args() - - logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") - config = RunConfigModel.load_from_path(args.config) - if not config.is_kamil_adc: - raise SystemExit("Config is not a kamil_adc profile") - - # Force the reference-producing collector regardless of the on-disk config. - config.radar.kamil_adc.executable_path = COLLECTOR_PATH - config.radar.kamil_adc.args = list(DO8_FREQ_REF_ARGS) - - # Free the device the headless daemon / an orphaned collector may be holding, - # so opening our own collector does not fail with E-502 device-busy. - if not args.no_release: - _release_radar_hardware() - - if not args.no_laser: - logger.info("Applying laser control...") - apply_kamil_adc_laser_control(config) - - service = KamilAdcService(config) - logger.info("Opening collector: %s", " ".join(service.command)) - _open_with_retry(service) - - if args.diagnose: - try: - sweeps = _capture_raw_sweeps(service, warmup=args.warmup, sweeps=args.sweeps) - finally: - service.close() - if len(sweeps) < max(2, args.sweeps // 2): - raise SystemExit(f"Only {len(sweeps)} usable sweeps captured; check the reference signal") - result = _diagnose(sweeps, config) - _report_diagnosis(result) - if args.dump is not None: - _dump_diagnosis(args.dump, result, sweeps) - if args.plot is not None: - _plot_diagnosis(args.plot, result) - return 0 - - try: - phases = _capture_reference_phases(service, warmup=args.warmup, sweeps=args.sweeps) - finally: - service.close() - - if len(phases) < max(2, args.sweeps // 2): - raise SystemExit(f"Only {len(phases)} usable sweeps captured; check the reference signal") - - result = _analyze(phases, config) - _report(result) - if args.apply: - _apply(args.config, result) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_app/scripts/kamil_adc_raw_producer.py b/python_app/scripts/kamil_adc_raw_producer.py index 1e94747..249e746 100644 --- a/python_app/scripts/kamil_adc_raw_producer.py +++ b/python_app/scripts/kamil_adc_raw_producer.py @@ -2,36 +2,21 @@ from __future__ import annotations -import argparse -from contextlib import ExitStack, suppress +from contextlib import ExitStack import logging -from pathlib import Path -import signal import threading import time import numpy as np from python_app.hardware_full.kamil_adc import KamilAdcService -from python_app.hardware_full.switch_service import SwitchService +from python_app.hardware_full.switch_service import SwitchService, select_and_settle from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData -from python_app.models.run_config_model import RunConfigModel -from python_app.orchestration.shm import ShmRingWriter -from python_app.storage.npz.serialize import RAW_MAGIC, serialize_trace_collection +from python_app.models.run_config_schema import RunConfigModel +from python_app.scripts.producer_common import RawPublisher, close_devices, load_producer_config, retry_open logger = logging.getLogger(__name__) -# The producer waits for the Kamil ADC collector forever: a device/collector that -# is absent at boot or disappears mid-run must never kill the producer, only make -# it wait. Reconnect uses a capped exponential backoff so a long absence does not -# busy-spin, and every wait is interruptible by SIGINT/SIGTERM (stop_requested) for -# a prompt clean exit. Mirrors matrix_raw_producer._open_radar_with_retry. -_OPEN_RETRY_MIN_S = 1.0 -_OPEN_RETRY_MAX_S = 10.0 -# Throttle open-failure logging during a long wait so a permanently absent device -# does not flood the process log: log the first failure, then every Nth attempt. -_OPEN_RETRY_LOG_EVERY = 30 - def _open_radar_with_retry( config: RunConfigModel, @@ -40,33 +25,10 @@ def _open_radar_with_retry( output_switch: SwitchService | None, stop_requested: threading.Event, ) -> bool: - """Open+configure the radar and both switches, retrying forever until stop. + """Reopen the ADC and Python-owned switches, cleaning partial opens before retry.""" + close_devices(input_switch, output_switch, radar) - Used for both the initial open and every in-loop reconnect, so a collector - that is absent at boot or disappears mid-run never kills the producer — it just - waits. Any partially-opened components are closed before each retry so a - relaunched collector starts clean. Returns ``True`` once everything is open, or - ``False`` if a stop was requested before the device became available. Backoff is - capped and every wait is interruptible by SIGTERM. - - ``input_switch``/``output_switch`` are ``None`` in switch-aware mode, where the - collector owns the GPIO lines and the producer must not open them. - """ - # Tear down any prior open first: open()/switch.open() are idempotent no-ops - # while still "open", so a mid-run reconnect must close them to force a fresh - # collector relaunch and TTY re-attach. - if input_switch is not None: - with suppress(Exception): - input_switch.close() - if output_switch is not None: - with suppress(Exception): - output_switch.close() - with suppress(Exception): - radar.close() - - attempt = 0 - delay = _OPEN_RETRY_MIN_S - while not stop_requested.is_set(): + def attempt() -> KamilAdcService: try: radar.open(stop_event=stop_requested) radar.configure(config.radar.sweep) @@ -74,82 +36,28 @@ def _open_radar_with_retry( output_switch.open() if input_switch is not None: input_switch.open() - except Exception as exc: # noqa: BLE001 — waiting for the device is the point - # Drop any partial open (collector process, TTY reader, switches) - # before the next attempt so the relaunch starts from a clean state. - if input_switch is not None: - with suppress(Exception): - input_switch.close() - if output_switch is not None: - with suppress(Exception): - output_switch.close() - with suppress(Exception): - radar.close() - attempt += 1 - if attempt == 1 or attempt % _OPEN_RETRY_LOG_EVERY == 0: - logger.warning( - "Kamil ADC not available (attempt %d); retrying every up to %.0fs " - "until the device is present: %s", - attempt, - _OPEN_RETRY_MAX_S, - exc, - ) - if stop_requested.wait(delay): - return False - delay = min(delay * 2.0, _OPEN_RETRY_MAX_S) - continue + except Exception: + close_devices(input_switch, output_switch, radar) + raise + return radar - if attempt > 0: - logger.info("Kamil ADC opened after %d attempt(s).", attempt + 1) - return True - - logger.debug("Stop requested before the Kamil ADC became available.") - return False + return retry_open(attempt, stop_requested, "Kamil ADC") is not None def main() -> int: """Run producer process until config or signal requests exit.""" - parser = argparse.ArgumentParser(description="Publish Kamil ADC raw sweeps to SHM rings") - parser.add_argument("--config", required=True, type=Path, help="Path to run_config.json") - args = parser.parse_args() - - logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") - stop_requested = threading.Event() - - def request_stop(_signum: int, _frame: object) -> None: - stop_requested.set() - - signal.signal(signal.SIGINT, request_stop) - signal.signal(signal.SIGTERM, request_stop) - - config = RunConfigModel.load_from_path(args.config) + config_path, config, stop_requested = load_producer_config("Publish Kamil ADC raw sweeps to SHM rings") if not config.is_kamil_adc: raise RuntimeError("kamil_adc_raw_producer requires radar.model='kamil_adc'") - config.ensure_combos() logger.info( "Kamil ADC raw producer starting: config=%s, combos=%d, continuous=%s", - args.config, len(config.combos), config.runtime.continuous, + config_path, len(config.combos), config.runtime.continuous, ) radar = input_switch = output_switch = None cleanup = ExitStack() try: - raw_writer = ShmRingWriter( - config.rings.raw.name, - config.rings.raw.capacity, - config.rings.raw.slot_size_bytes, - ) - cleanup.callback(raw_writer.close) - raw_tap_writer = ShmRingWriter( - config.rings.raw_tap.name, - config.rings.raw_tap.capacity, - config.rings.raw_tap.slot_size_bytes, - ) - cleanup.callback(raw_tap_writer.close) - logger.debug( - "Opened SHM ring writers: raw=%s, raw_tap=%s", - config.rings.raw.name, config.rings.raw_tap.name, - ) + publisher = RawPublisher(config.rings, cleanup) # Switch-aware mode: with native switches the collector drives the RF switches # itself, in the hardware gap between sweeps, and tags each sweep with its combo # — no sweep lost at a switch boundary. The producer then only reads tagged @@ -162,7 +70,7 @@ def main() -> int: ) radar = KamilAdcService( config, - switch_config_path=str(args.config) if collector_driven else None, + switch_config_path=str(config_path) if collector_driven else None, ) input_switch = None if collector_driven else SwitchService.from_model(config.input_switch) output_switch = None if collector_driven else SwitchService.from_model(config.output_switch) @@ -175,7 +83,6 @@ def main() -> int: return 0 # asked to stop before a device became available collection_id = 1 - publish_failures = 0 while not stop_requested.is_set(): collection_start = time.monotonic() capture_start_ns = time.monotonic_ns() @@ -193,9 +100,8 @@ def main() -> int: else: if input_switch is None or output_switch is None: raise RuntimeError("Producer-driven switches are not initialized") - output_switch.switch_to(combo.output) - input_switch.switch_to(combo.input) - if config.runtime.settling_ms > 0 and stop_requested.wait(config.runtime.settling_ms / 1000.0): + if not select_and_settle(output_switch, input_switch, combo.output, combo.input, + config.runtime.settling_ms, wait=stop_requested.wait): break # Stamped after switching and settling so the window covers # the sweep alone, not the dead time before it. @@ -205,7 +111,6 @@ def main() -> int: TraceData( combo=ComboKey(input=combo.input, output=combo.output), frequency_hz=np.asarray(sweep.x, dtype=np.float32), - s11=np.asarray(sweep.trace("s11"), dtype=np.complex64), s21=np.asarray(sweep.trace("s21"), dtype=np.complex64), capture_start_ns=sweep_start_ns, capture_end_ns=time.monotonic_ns(), @@ -244,18 +149,7 @@ def main() -> int: capture_start_ns=capture_start_ns, capture_end_ns=time.monotonic_ns(), ) - payload = serialize_trace_collection(collection, RAW_MAGIC) - if raw_writer.push(payload): - raw_tap_writer.push(payload) # best-effort GUI tap; never fatal - else: - # Oversized payload vs the ring slot is a persistent config error, not - # a device fault: log (throttled) and skip rather than killing the producer. - publish_failures += 1 - if publish_failures == 1 or publish_failures % 100 == 0: - logger.error( - "Raw payload %d B exceeds ring slot %d B; dropping collection %d (drops=%d)", - len(payload), raw_writer.slot_size_bytes, collection_id, publish_failures, - ) + publisher.publish(collection) if not config.runtime.continuous: break @@ -264,15 +158,7 @@ def main() -> int: logger.info("Kamil ADC collection %d acquired in %.3f s", collection_id, collection_duration_s) collection_id += 1 finally: - if output_switch is not None: - with suppress(Exception): - output_switch.close() - if input_switch is not None: - with suppress(Exception): - input_switch.close() - if radar is not None: - with suppress(Exception): - radar.close() + close_devices(output_switch, input_switch, radar) cleanup.close() logger.info("Kamil ADC raw producer stopped") diff --git a/python_app/scripts/laser_temp_monitor.py b/python_app/scripts/laser_temp_monitor.py index 86a8d67..ed6d242 100644 --- a/python_app/scripts/laser_temp_monitor.py +++ b/python_app/scripts/laser_temp_monitor.py @@ -36,7 +36,7 @@ from python_app.hardware_full.laser_control.monitoring import ( ReadingWriter, resolve_period_s, ) -from python_app.models.run_config_model import RunConfigModel +from python_app.models.run_config_schema import RunConfigModel logger = logging.getLogger("laser_temp_monitor") diff --git a/python_app/scripts/librevna_generator_sweep.py b/python_app/scripts/librevna_generator_sweep.py deleted file mode 100644 index 3db35e8..0000000 --- a/python_app/scripts/librevna_generator_sweep.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 -"""Standalone LibreVNA generator sweep driven by a local Python config.""" - -from __future__ import annotations - -import logging -import sys - -from pathlib import Path - -PROJECT_ROOT = Path(__file__).resolve().parents[2] -if str(PROJECT_ROOT) not in sys.path: - sys.path.insert(0, str(PROJECT_ROOT)) - -from python_app.generator_sweep.config import resolve_generator_sweep_config -from python_app.scripts.librevna_generator_sweep_config import CONFIG - - -def main() -> int: - """Load local generator config and execute the sweep.""" - resolved = resolve_generator_sweep_config(CONFIG) - logging.basicConfig( - level=resolved.log_level, - format="%(asctime)s %(levelname)s %(name)s: %(message)s", - ) - - from python_app.generator_sweep.runner import GeneratorSweepRunner - - runner = GeneratorSweepRunner(CONFIG) - runner.run() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_app/scripts/librevna_generator_sweep_config.py b/python_app/scripts/librevna_generator_sweep_config.py deleted file mode 100644 index dbfb4a8..0000000 --- a/python_app/scripts/librevna_generator_sweep_config.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Local editable config for ``librevna_generator_sweep.py``.""" - -from __future__ import annotations - -from python_app.generator_sweep.config import GeneratorSweepConfig - - -CONFIG = GeneratorSweepConfig( - log_level="INFO", # DEBUG, INFO, WARNING, ERROR, or CRITICAL. - serial=None, # None or exact serial string, for example "206930A15532". - strict_protocol_version=14, # Exact value for the connected device: 14. - start_hz=100_000_000.0, # Script constraint: > 0. Device reports frequency range 0.0 .. 6_000_000_000.0 Hz. - stop_hz=6_000_000_000.0, # Script constraint: >= start_hz. Device reports frequency range 0.0 .. 6_000_000_000.0 Hz. - step_hz=0, # > 0 to use step mode, or 0 to disable and use points mode. - points=501, # 0 to disable, or >= 2 to use points mode; if start_hz == stop_hz then only 1 is allowed. - hold_time_ms=50.0, # >= 0 ms. Float values like 0.5 are allowed. - loop=True, # True or False. - port=1, # Only 1 or 2. - power_dbm=-10.0, # Device reports source power range -40.0 .. 0.0 dBm. - amplitude_correction=False, # True = apply source amplitude calibration, False = use raw generator level. - settle_timeout_ms=1_000, # > 0 ms. - status_poll_ms=10.0, # > 0 ms. Float values like 0.2 are allowed. - post_lock_delay_us=0, # >= 0 us. - gpio_chip="/dev/gpiochip0", # Non-empty Linux GPIO chip path. - sweep_start_pin=5, # BCM GPIO, >= 0, must differ from curr_step_pin and pwm_pin. - curr_step_pin=6, # BCM GPIO, >= 0, must differ from sweep_start_pin and pwm_pin. - curr_step_initial_level=0, # Only 0 or 1. - pwm_pin=12, # Only 12, 13, 18, or 19 on Raspberry Pi 5. - pwm_frequency_hz=2_000_000, # > 0 Hz. - pwm_duty_cycle=0.5, # Range: 0.0 < value <= 1.0. -) diff --git a/python_app/scripts/manual_smoke_run.py b/python_app/scripts/manual_smoke_run.py deleted file mode 100644 index f4488dd..0000000 --- a/python_app/scripts/manual_smoke_run.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Manual smoke scenario for end-to-end pipeline check without GUI.""" - -from __future__ import annotations - -import argparse -import ctypes -import ctypes.util -import os -from pathlib import Path -import sys -import time - -import numpy as np - -PROJECT_ROOT = Path(__file__).resolve().parents[2] -if str(PROJECT_ROOT) not in sys.path: - sys.path.insert(0, str(PROJECT_ROOT)) - -from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData -from python_app.models.run_config_model import RunConfigModel -from python_app.orchestration.config_writer import ConfigWriter -from python_app.orchestration.process_supervisor import ProcessSupervisor -from python_app.orchestration.shm_reader import ShmRingReader -from python_app.storage.npz_store import NpzStore, radar_key_from_config - - -def _make_unique_ring_name(prefix: str) -> str: - """Build unique POSIX SHM ring name.""" - stamp = time.monotonic_ns() - return f"/{prefix}_{os.getpid()}_{stamp}" - - -def _shm_unlink(name: str) -> None: - """Best-effort unlink for POSIX shared-memory object.""" - libc_name = ctypes.util.find_library("c") - if libc_name is None: - return - - libc = ctypes.CDLL(libc_name, use_errno=True) - libc.shm_unlink.argtypes = [ctypes.c_char_p] - libc.shm_unlink.restype = ctypes.c_int - - result = libc.shm_unlink(name.encode("utf-8")) - if result == 0: - return - - err = ctypes.get_errno() - if err != 2: # ENOENT - raise OSError(err, f"shm_unlink failed for {name}") - - -def build_synthetic_collection( - config: RunConfigModel, - value_scale: float, - *, - s11_scale: float, - s11_phase_offset: float, -) -> SweepCollection: - """Build synthetic sweep collection for all configured switch combos.""" - traces: list[TraceData] = [] - combos = RunConfigModel.build_full_combos(config.input_switch.positions, config.output_switch.positions) - - for combo in combos: - frequency_hz = np.linspace( - config.radar.sweep.start_hz, - config.radar.sweep.stop_hz, - config.radar.sweep.points, - dtype=np.float32, - ) - phase = np.linspace(0.0, np.pi * 2.0, config.radar.sweep.points, dtype=np.float32) - reflected_phase = (phase * 0.6) + s11_phase_offset - s11 = (s11_scale * (np.cos(reflected_phase) + 1j * np.sin(reflected_phase))).astype(np.complex64) - s21 = value_scale * (np.cos(phase) + 1j * np.sin(phase)).astype(np.complex64) - - traces.append( - TraceData( - combo=ComboKey(input=combo.input, output=combo.output), - frequency_hz=frequency_hz, - s11=s11, - s21=s21, - ) - ) - - return SweepCollection(collection_id=1, monotonic_ns=time.monotonic_ns(), traces=traces) - - -def main() -> int: - """Run manual smoke-test pipeline scenario.""" - parser = argparse.ArgumentParser() - parser.add_argument("--duration", type=float, default=3.0) - args = parser.parse_args() - - project_root = PROJECT_ROOT - store = NpzStore(project_root / "python_app/data") - config_writer = ConfigWriter(project_root / "python_app/runtime") - supervisor = ProcessSupervisor(project_root) - - config = RunConfigModel.load_from_path(project_root / "run_config.json") - config.radar.driver_mode = "mock" - config.input_switch.driver_mode = "mock" - config.output_switch.driver_mode = "mock" - config.combos = RunConfigModel.build_full_combos(config.input_switch.positions, config.output_switch.positions) - config.rings.raw.name = _make_unique_ring_name("radar_raw_smoke") - config.rings.raw_tap.name = _make_unique_ring_name("radar_raw_tap_smoke") - config.rings.preprocessed.name = _make_unique_ring_name("radar_preprocessed_smoke") - config.rings.preprocessed_tap.name = _make_unique_ring_name("radar_preprocessed_tap_smoke") - config.rings.results.name = _make_unique_ring_name("radar_results_smoke") - - radar_key = radar_key_from_config( - model_name=config.radar.model, - serial=config.radar.serial, - sweep_start_hz=config.radar.sweep.start_hz, - sweep_stop_hz=config.radar.sweep.stop_hz, - sweep_points=config.radar.sweep.points, - ifbw_hz=config.radar.sweep.if_bandwidth_hz, - power_dbm=config.radar.sweep.power_dbm, - extra_serials=config.radar_key_extra_parts() or None, - ) - - s21_calibration_set = build_synthetic_collection(config, value_scale=1.0, s11_scale=0.15, s11_phase_offset=0.4) - s21_reference_set = build_synthetic_collection(config, value_scale=0.3, s11_scale=0.08, s11_phase_offset=0.9) - s11_open_set = build_synthetic_collection(config, value_scale=0.85, s11_scale=0.95, s11_phase_offset=0.1) - s11_short_set = build_synthetic_collection(config, value_scale=0.85, s11_scale=0.95, s11_phase_offset=3.2) - s11_load_set = build_synthetic_collection(config, value_scale=0.85, s11_scale=0.05, s11_phase_offset=1.4) - s11_reference_set = build_synthetic_collection(config, value_scale=0.3, s11_scale=0.18, s11_phase_offset=1.1) - - store.save_set("s21_calibration", radar_key, "smoke_cal", s21_calibration_set) - store.save_set("s21_reference", radar_key, "smoke_ref", s21_reference_set) - store.save_set("s11_open", radar_key, "smoke_open", s11_open_set) - store.save_set("s11_short", radar_key, "smoke_short", s11_short_set) - store.save_set("s11_load", radar_key, "smoke_load", s11_load_set) - store.save_set("s11_reference", radar_key, "smoke_s11_ref", s11_reference_set) - - config.preprocess.s21.calibration.set_name = "smoke_cal" - config.preprocess.s21.reference.set_name = "smoke_ref" - config.preprocess.s11.calibration.open.set_name = "smoke_open" - config.preprocess.s11.calibration.short.set_name = "smoke_short" - config.preprocess.s11.calibration.load.set_name = "smoke_load" - config.preprocess.s11.reference.set_name = "smoke_s11_ref" - config_writer.prepare_preprocess_bundles(store, radar_key, config) - - config_path = config_writer.write(config, project_root / "python_app/runtime/run_config_smoke.json") - - result_reader: ShmRingReader | None = None - try: - supervisor.start(config_path) - result_reader = ShmRingReader(config.rings.results.name) - deadline = time.monotonic() + args.duration - received = 0 - - while time.monotonic() < deadline: - result = result_reader.pop_result_collection() if result_reader is not None else None - if result is not None: - received += 1 - time.sleep(0.02) - - print(f"Received result collections: {received}") - finally: - supervisor.stop_all() - if result_reader is not None: - result_reader.close() - _shm_unlink(config.rings.raw.name) - _shm_unlink(config.rings.raw_tap.name) - _shm_unlink(config.rings.preprocessed.name) - _shm_unlink(config.rings.preprocessed_tap.name) - _shm_unlink(config.rings.results.name) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_app/scripts/matrix_raw_producer.py b/python_app/scripts/matrix_raw_producer.py index 86426e4..6bda705 100644 --- a/python_app/scripts/matrix_raw_producer.py +++ b/python_app/scripts/matrix_raw_producer.py @@ -1,142 +1,64 @@ -"""Raw producer for native matrix radars and the shared single/multi mock scene.""" +"""Publish LibreVNA single-device, matrix, or simulated collections to SHM.""" from __future__ import annotations -import argparse import logging -from contextlib import ExitStack, suppress -from pathlib import Path -import signal +from contextlib import ExitStack import threading import time -from python_app.hardware_full.matrix_radar_service import MatrixRadarService, create_acquisition_service -from python_app.logging_setup import coerce_level -from python_app.models.run_config_model import RunConfigModel -from python_app.orchestration.shm import ShmRingWriter -from python_app.storage.npz.serialize import RAW_MAGIC, serialize_trace_collection +from python_app.hardware_full.matrix_radar_service import MatrixRadarService, create_matrix_radar_service +from python_app.models.run_config_schema import RunConfigModel +from python_app.scripts.producer_common import RawPublisher, close_devices, load_producer_config, retry_open logger = logging.getLogger(__name__) -# The producer waits for the matrix radar forever: a device that is absent at boot -# or disappears mid-run must never kill the producer, only make it wait. Reconnect -# uses a capped exponential backoff so a long absence does not busy-spin, and every -# wait is interruptible by SIGINT/SIGTERM (stop_requested) for a prompt clean exit. -_OPEN_RETRY_MIN_S = 1.0 -_OPEN_RETRY_MAX_S = 10.0 -# Throttle open-failure logging during a long wait so a permanently absent device -# does not flood the process log: log the first failure, then every Nth attempt. -_OPEN_RETRY_LOG_EVERY = 30 - def _open_radar_with_retry( config: RunConfigModel, previous: MatrixRadarService | None, stop_requested: threading.Event, ) -> MatrixRadarService | None: - """Open+configure the matrix radar, retrying forever until success or stop. + """Recreate a failed LibreVNA service and wait interruptibly for its return.""" + close_devices(previous) - Used for both the initial open and every in-loop reconnect, so a device that is - absent at boot or disappears mid-run never kills the producer — it just waits. - Returns the opened service, or ``None`` if a stop was requested before any device - became available. Backoff is capped and every wait is interruptible by SIGTERM. - """ - if previous is not None: - with suppress(Exception): - previous.close() - - attempt = 0 - delay = _OPEN_RETRY_MIN_S - while not stop_requested.is_set(): - radar = create_acquisition_service(config) + def attempt() -> MatrixRadarService: + radar = create_matrix_radar_service(config) try: radar.open() radar.configure(config.radar.sweep) - except Exception as exc: # noqa: BLE001 — waiting for the device is the point - with suppress(Exception): - radar.close() # drop any partial open before the next attempt - attempt += 1 - if attempt == 1 or attempt % _OPEN_RETRY_LOG_EVERY == 0: - logger.warning( - "Matrix radar not available (attempt %d); retrying every up to %.0fs " - "until the device is present: %s", - attempt, - _OPEN_RETRY_MAX_S, - exc, - ) - if stop_requested.wait(delay): - with suppress(Exception): - radar.close() - return None - delay = min(delay * 2.0, _OPEN_RETRY_MAX_S) - continue - - if attempt > 0: - logger.info("Matrix radar opened after %d attempt(s).", attempt + 1) + except Exception: + close_devices(radar) + raise return radar - logger.debug("Stop requested before any matrix radar became available.") - return None + return retry_open(attempt, stop_requested, "LibreVNA radar") def main() -> int: """Run producer process until config or signal requests exit.""" - parser = argparse.ArgumentParser(description="Publish matrix-radar raw sweeps to SHM rings") - parser.add_argument("--config", required=True, type=Path, help="Path to run_config.json") - args = parser.parse_args() - - logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") - stop_requested = threading.Event() - - def request_stop(_signum: int, _frame: object) -> None: - stop_requested.set() - - signal.signal(signal.SIGINT, request_stop) - signal.signal(signal.SIGTERM, request_stop) - - config = RunConfigModel.load_from_path(args.config) - config.apply_device_model_constraints() - # Honor the configured verbosity so the DEBUG switch/sweep timing trace can be - # turned on from the profile instead of requiring a code edit. basicConfig above - # only installed the handler; the package logger owns the level. - logging.getLogger("python_app").setLevel(coerce_level(config.logging.level)) + config_path, config, stop_requested = load_producer_config("Publish LibreVNA raw sweeps to SHM rings") logger.info( - "Matrix radar raw producer starting: config=%s, model=%s, continuous=%s", - args.config, config.radar.model, config.runtime.continuous, + "LibreVNA radar raw producer starting: config=%s, model=%s, continuous=%s", + config_path, config.radar.model, config.runtime.continuous, ) radar: MatrixRadarService | None = None cleanup = ExitStack() try: - raw_writer = ShmRingWriter( - config.rings.raw.name, - config.rings.raw.capacity, - config.rings.raw.slot_size_bytes, - ) - cleanup.callback(raw_writer.close) - raw_tap_writer = ShmRingWriter( - config.rings.raw_tap.name, - config.rings.raw_tap.capacity, - config.rings.raw_tap.slot_size_bytes, - ) - cleanup.callback(raw_tap_writer.close) - logger.debug( - "Opened SHM ring writers: raw=%s, raw_tap=%s", - config.rings.raw.name, config.rings.raw_tap.name, - ) + publisher = RawPublisher(config.rings, cleanup) radar = _open_radar_with_retry(config, previous=None, stop_requested=stop_requested) if radar is None: return 0 # asked to stop before a device became available collection_id = 1 - publish_failures = 0 while not stop_requested.is_set(): collection_start = time.monotonic() try: collection = radar.acquire_collection(collection_id=collection_id) except Exception as exc: # noqa: BLE001 — reconnect forever, never give up logger.warning( - "Matrix radar acquisition failed; reconnecting and waiting for the device: %s", + "LibreVNA radar acquisition failed; reconnecting and waiting for the device: %s", exc, exc_info=True, ) @@ -145,36 +67,22 @@ def main() -> int: break # stop requested while waiting to reconnect continue - payload = serialize_trace_collection(collection, RAW_MAGIC) - if raw_writer.push(payload): - raw_tap_writer.push(payload) # best-effort GUI tap; never fatal - else: - # An oversized payload vs the ring slot is a persistent config error, - # not a device fault: log it (throttled) and skip the collection rather - # than letting a RuntimeError escape the loop and kill the producer. - publish_failures += 1 - if publish_failures == 1 or publish_failures % 100 == 0: - logger.error( - "Raw payload %d B exceeds ring slot %d B; dropping collection %d (drops=%d)", - len(payload), raw_writer.slot_size_bytes, collection_id, publish_failures, - ) + publisher.publish(collection) if not config.runtime.continuous: break collection_duration_s = time.monotonic() - collection_start if collection_id == 1 or collection_id % 20 == 0 or collection_duration_s > 2.0: logger.info( - "matrix radar collection %d acquired in %.3f s", + "LibreVNA radar collection %d acquired in %.3f s", collection_id, collection_duration_s, ) collection_id += 1 finally: - if radar is not None: - with suppress(Exception): - radar.close() + close_devices(radar) cleanup.close() - logger.info("matrix radar raw producer stopped") + logger.info("LibreVNA radar raw producer stopped") return 0 diff --git a/python_app/scripts/per_point_switching_check.py b/python_app/scripts/per_point_switching_check.py index f56ec04..1207710 100644 --- a/python_app/scripts/per_point_switching_check.py +++ b/python_app/scripts/per_point_switching_check.py @@ -41,7 +41,7 @@ from python_app.hardware_full.librevna_multi_device_driver.models import ( ) from python_app.hardware_full.matrix_radar_service import create_matrix_radar_service from python_app.models.dataset_model import SweepCollection -from python_app.models.run_config_model import RunConfigModel +from python_app.models.run_config_schema import RunConfigModel logger = logging.getLogger(__name__) @@ -117,7 +117,7 @@ def main() -> int: config = RunConfigModel.load_from_path(args.config) config.apply_device_model_constraints() - if not config.is_matrix_radar: + if not config.is_multi_device: raise SystemExit("нужен матричный radar.model (librevna_multi)") substages = config.matrix_stimulus_switch_positions outputs = config.output_switch.positions diff --git a/python_app/scripts/producer_common.py b/python_app/scripts/producer_common.py new file mode 100644 index 0000000..2281e0f --- /dev/null +++ b/python_app/scripts/producer_common.py @@ -0,0 +1,105 @@ +"""Shared CLI, interruptible device retry, and raw-ring publication.""" + +from __future__ import annotations + +import argparse +from collections.abc import Callable +from contextlib import ExitStack, suppress +import logging +from pathlib import Path +import signal +import threading +from typing import Protocol, TypeVar + +from python_app.logging_setup import coerce_level +from python_app.models.dataset_model import SweepCollection +from python_app.models.run_config_schema import RingEndpointModel, RingsModel, RunConfigModel +from python_app.models.run_config_validation import validate_acquisition_config +from python_app.orchestration.shm.decoder import RAW_MAGIC +from python_app.orchestration.shm.ring_writer import ShmRingWriter +from python_app.storage.npz.serialize import serialize_trace_collection + +logger = logging.getLogger(__name__) +T = TypeVar("T") + + +def load_producer_config(description: str) -> tuple[Path, RunConfigModel, threading.Event]: + parser = argparse.ArgumentParser(description=description) + parser.add_argument("--config", required=True, type=Path, help="Path to run_config.json") + path = parser.parse_args().config + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") + stop_requested = threading.Event() + + def request_stop(_signum: int, _frame: object) -> None: + stop_requested.set() + + signal.signal(signal.SIGINT, request_stop) + signal.signal(signal.SIGTERM, request_stop) + config = RunConfigModel.load_from_path(path) + config.ensure_combos() + validate_acquisition_config(config) + logging.getLogger("python_app").setLevel(coerce_level(config.logging.level)) + return path, config, stop_requested + + +class _Closable(Protocol): + def close(self) -> None: ... + + +def close_devices(*devices: _Closable | None) -> None: + """Attempt every hardware cleanup even if one disconnected device raises.""" + for device in devices: + if device is not None: + with suppress(Exception): + device.close() + + +def retry_open( + attempt: Callable[[], T], stop_requested: threading.Event, device_name: str, +) -> T | None: + """Retry an attempt that cleans up on failure, with capped, interruptible backoff.""" + failures = 0 + delay = 1.0 + while not stop_requested.is_set(): + try: + device = attempt() + except Exception as exc: + failures += 1 + if failures == 1 or failures % 30 == 0: + logger.warning("%s unavailable (attempt %d); retrying until present: %s", device_name, failures, exc) + if stop_requested.wait(delay): + break + delay = min(delay * 2, 10.0) + else: + if failures: + logger.info("%s opened after %d attempt(s)", device_name, failures + 1) + return device + return None + + +class RawPublisher: + """Publish one validated frame to raw and, when enabled, its independent GUI tap.""" + + def __init__(self, rings: RingsModel, cleanup: ExitStack) -> None: + def open_writer(endpoint: RingEndpointModel) -> ShmRingWriter: + writer = ShmRingWriter(endpoint.name, endpoint.capacity, endpoint.slot_size_bytes) + cleanup.callback(writer.close) + return writer + + self._raw = open_writer(rings.raw) + self._tap = open_writer(rings.raw_tap) if rings.raw_tap.name else None + self._dropped = 0 + + def publish(self, collection: SweepCollection) -> bool: + payload = serialize_trace_collection(collection, RAW_MAGIC, require_capture_window=True) + if not self._raw.push(payload): + self._dropped += 1 + if self._dropped == 1 or self._dropped % 100 == 0: + logger.error( + "Raw payload %d B exceeds ring slot %d B; dropping collection %d (drops=%d)", + len(payload), self._raw.slot_size_bytes, collection.collection_id, self._dropped, + ) + return False + if self._tap is not None: + self._tap.push(payload) # independent latest-wins tap; never backpressure raw + return True diff --git a/python_app/scripts/replay_raw_producer.py b/python_app/scripts/replay_raw_producer.py index b16c6c2..c8266e8 100644 --- a/python_app/scripts/replay_raw_producer.py +++ b/python_app/scripts/replay_raw_producer.py @@ -19,24 +19,20 @@ never see time jump backwards. ``collection_id`` increases forever. from __future__ import annotations -import argparse from contextlib import ExitStack import json import logging from pathlib import Path import re -import signal import statistics -import threading import time import numpy as np -from python_app.logging_setup import coerce_level from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData -from python_app.models.run_config_model import RunConfigModel -from python_app.orchestration.shm import ShmRingWriter -from python_app.storage.npz.serialize import RAW_MAGIC, serialize_trace_collection +from python_app.orchestration.shm.decoder import RAW_MAGIC +from python_app.scripts.producer_common import RawPublisher, load_producer_config +from python_app.storage.npz.serialize import serialize_trace_collection logger = logging.getLogger(__name__) @@ -105,10 +101,8 @@ def _load_dataset(dataset_dir: Path, combos: list[ComboKey], expected_points: in ) s21 = np.load(s21_path, allow_pickle=False).astype(np.complex64, copy=False) freq = np.load(freq_path, allow_pickle=False).astype(np.float32, copy=False) - s11_path = folder / f"i{combo.input}_o{combo.output}_s11.npy" - s11 = np.load(s11_path, allow_pickle=False).astype(np.complex64, copy=False) if s11_path.is_file() else np.zeros_like(s21) - if freq.ndim != 1 or s21.shape != freq.shape or s11.shape != freq.shape: - raise RuntimeError(f"Snapshot {name}: frequency, S11 and S21 must be equally sized 1D arrays") + if freq.ndim != 1 or s21.shape != freq.shape: + raise RuntimeError(f"Snapshot {name}: frequency and S21 must be equally sized 1D arrays") if expected_points and s21.size != expected_points: raise RuntimeError( f"{s21_path}: {s21.size} points, but radar.sweep.points = " @@ -119,7 +113,6 @@ def _load_dataset(dataset_dir: Path, combos: list[ComboKey], expected_points: in TraceData( combo=combo, frequency_hz=freq, - s11=s11, s21=s21, capture_start_ns=window[0], capture_end_ns=window[1], @@ -157,7 +150,6 @@ def _build_collection( TraceData( combo=source.combo, frequency_hz=source.frequency_hz, - s11=source.s11, s21=source.s21, capture_start_ns=capture[0], capture_end_ns=capture[1], @@ -174,22 +166,7 @@ def _build_collection( def main() -> int: """Replay the configured dataset into the raw SHM rings until stopped.""" - parser = argparse.ArgumentParser(description="Replay a recorded dataset to raw SHM rings") - parser.add_argument("--config", required=True, type=Path, help="Path to run_config.json") - args = parser.parse_args() - - logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") - stop_requested = threading.Event() - - def request_stop(_signum: int, _frame: object) -> None: - stop_requested.set() - - signal.signal(signal.SIGINT, request_stop) - signal.signal(signal.SIGTERM, request_stop) - - config = RunConfigModel.load_from_path(args.config) - config.apply_device_model_constraints() - logging.getLogger("python_app").setLevel(coerce_level(config.logging.level)) + _, config, stop_requested = load_producer_config("Replay a recorded dataset to raw SHM rings") replay = config.radar.replay if not replay.dataset_dir: @@ -211,23 +188,15 @@ def main() -> int: cleanup = ExitStack() try: - raw_writer = ShmRingWriter( - config.rings.raw.name, - config.rings.raw.capacity, - config.rings.raw.slot_size_bytes, - ) - cleanup.callback(raw_writer.close) - raw_tap_writer = ShmRingWriter( - config.rings.raw_tap.name, - config.rings.raw_tap.capacity, - config.rings.raw_tap.slot_size_bytes, - ) - cleanup.callback(raw_tap_writer.close) + publisher = RawPublisher(config.rings, cleanup) payload_size = len(serialize_trace_collection( - _build_collection(dataset, 0, 1, time.monotonic_ns()), RAW_MAGIC + _build_collection(dataset, 0, 1, time.monotonic_ns()), RAW_MAGIC, require_capture_window=True, )) - for ring in (config.rings.raw, config.rings.raw_tap): + for name in ("raw", "raw_tap", "preprocessed", "preprocessed_tap"): + ring = getattr(config.rings, name) + if name.endswith("_tap") and not ring.name: + continue if ring.slot_size_bytes < payload_size: raise ValueError(f"Replay requires {payload_size} bytes per slot; ring {ring.name} has {ring.slot_size_bytes}") @@ -257,10 +226,14 @@ def main() -> int: break collection = _build_collection(dataset, index, collection_id, frame_ns) - payload = serialize_trace_collection(collection, RAW_MAGIC) - if not raw_writer.push(payload): + # Acquisition identity uses real monotonic time, even at accelerated + # replay rates. Keep the recorded duration for legacy GPR and the + # virtual per-trace timeline for SAR aperture/motion reconstruction. + duration_ns = collection.capture_end_ns - collection.capture_start_ns + collection.capture_start_ns = time.monotonic_ns() + collection.capture_end_ns = collection.capture_start_ns + duration_ns + if not publisher.publish(collection): raise RuntimeError(f"Replay raw ring rejected collection {collection_id}") - raw_tap_writer.push(payload) if collection_id == 1 or collection_id % 50 == 0: logger.info( "replay collection %d published (snapshot %d/%d, cycle %d)", diff --git a/python_app/scripts/vna_only_raw_test.py b/python_app/scripts/vna_only_raw_test.py deleted file mode 100644 index 8059789..0000000 --- a/python_app/scripts/vna_only_raw_test.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Minimal GUI tool for direct LibreVNA raw acquisition checks.""" - -from __future__ import annotations - -import argparse -from pathlib import Path -import signal -import sys - -PROJECT_ROOT = Path(__file__).resolve().parents[2] -if str(PROJECT_ROOT) not in sys.path: - sys.path.insert(0, str(PROJECT_ROOT)) - -from PyQt6.QtWidgets import QApplication - -from python_app.scripts.hardware_raw_orchestrator_test import RawOrchestratorViewer - - -def main() -> int: - """Run standalone raw-viewer GUI against a selected run config.""" - parser = argparse.ArgumentParser( - description="Intermediate test: native VNA acquisition with mock switch drivers" - ) - parser.add_argument( - "--config", - type=Path, - default=PROJECT_ROOT / "run_config.json", - help="Path to run config JSON", - ) - parser.add_argument( - "--no-reset-ring", - action="store_true", - help="Do not unlink existing raw ring name before starting", - ) - parser.add_argument( - "--skip-radar-prepare", - action="store_true", - help="Skip Python pre-configuration of native radar before orchestrator start", - ) - parser.add_argument( - "--strict-radar-prepare", - action="store_true", - help="Fail immediately if Python pre-configuration cannot run", - ) - args = parser.parse_args() - - config_path = args.config.resolve() - if not config_path.exists(): - raise FileNotFoundError(f"Config file not found: {config_path}") - - app = QApplication(sys.argv) - viewer = RawOrchestratorViewer( - config_path=config_path, - reset_ring=not args.no_reset_ring, - prepare_radar=not args.skip_radar_prepare, - strict_prepare=args.strict_radar_prepare, - ) - viewer.setWindowTitle("Raw Sweep Viewer (VNA native + mock switches)") - viewer.show() - - def _sig_handler(_signum, _frame): - """Close viewer gracefully on process signals.""" - viewer.close() - - signal.signal(signal.SIGINT, _sig_handler) - signal.signal(signal.SIGTERM, _sig_handler) - - return app.exec() - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_app/storage/npz/__init__.py b/python_app/storage/npz/__init__.py index f03262d..a30d836 100644 --- a/python_app/storage/npz/__init__.py +++ b/python_app/storage/npz/__init__.py @@ -1,13 +1 @@ -"""NPZ-based dataset storage implementation and serialization helpers.""" - -from python_app.storage.npz.paths import radar_key_from_config -from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, RESULT_MAGIC -from python_app.storage.npz.store import NpzStore - -__all__ = [ - "NpzStore", - "PREPROC_MAGIC", - "RAW_MAGIC", - "RESULT_MAGIC", - "radar_key_from_config", -] +"""NPZ-based dataset storage.""" diff --git a/python_app/storage/npz/paths.py b/python_app/storage/npz/paths.py index 633f62c..4959c11 100644 --- a/python_app/storage/npz/paths.py +++ b/python_app/storage/npz/paths.py @@ -2,35 +2,7 @@ from __future__ import annotations -import re -from collections.abc import Sequence - - -def radar_key_from_config( - model_name: str, - serial: str, - sweep_start_hz: float, - sweep_stop_hz: float, - sweep_points: int, - ifbw_hz: float, - power_dbm: float, - extra_serials: Sequence[str] | None = None, -) -> str: - """Build deterministic key for calibration/reference set lookup.""" - serial_parts = [serial or "no_serial"] - if extra_serials: - serial_parts.extend(str(value).strip() or "no_serial" for value in extra_serials) - serial_part = "_".join(sanitize_path_component(value) for value in serial_parts) - start_token = format_float_for_key(sweep_start_hz) - stop_token = format_float_for_key(sweep_stop_hz) - points_token = "adc" if model_name.strip().lower() == "kamil_adc" else str(int(sweep_points)) - ifbw_token = format_float_for_key(ifbw_hz) - power_token = format_float_for_key(power_dbm) - return ( - f"{model_name}_{serial_part}" - f"_st{start_token}_sp{stop_token}" - f"_p{points_token}_if{ifbw_token}_pw{power_token}" - ) +from python_app.file_io import sanitize_path_component def radar_config_filename_prefix( @@ -73,18 +45,3 @@ def _frequency_unit(hz: float) -> tuple[str, float]: def collection_dir_name(index: int, collection_id: int, monotonic_ns: int) -> str: """Build canonical collection directory name for snapshot stages.""" return f"{index:04d}_id{int(collection_id)}_ns{int(monotonic_ns)}" - - -def sanitize_path_component(name: str) -> str: - """Convert arbitrary string into filesystem-safe component.""" - cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", name.strip()) - cleaned = cleaned.strip("._") - return cleaned or "snapshot" - - -def format_float_for_key(value: float) -> str: - """Format float compactly for deterministic key serialization.""" - integer = int(round(value)) - if abs(value - float(integer)) < 1e-6: - return str(integer) - return f"{value:.6f}".rstrip("0").rstrip(".") diff --git a/python_app/storage/npz/serialize.py b/python_app/storage/npz/serialize.py index 2dfac1d..a8f8fd6 100644 --- a/python_app/storage/npz/serialize.py +++ b/python_app/storage/npz/serialize.py @@ -1,4 +1,4 @@ -"""Binary payload serialization for sweep/result collections.""" +"""Binary payload serialization for raw/preprocessed collections.""" from __future__ import annotations @@ -7,25 +7,17 @@ from numbers import Integral import numpy as np -from python_app.models.dataset_model import ResultCollection, SweepCollection - -RAW_MAGIC = 0x32574152 -PREPROC_MAGIC = 0x32525050 -RESULT_MAGIC = 0x324C5352 # RSL2: adds processing_duration_ns after monotonic_ns +from python_app.models.dataset_model import SweepCollection -def _write_interleaved_complex(buffer: bytearray, values: np.ndarray) -> None: - """Append complex64 array as interleaved float32 real/imag pairs.""" - interleaved = np.empty(values.size * 2, dtype=" None: +def validate_trace_collection( + collection: SweepCollection, *, require_finite: bool = False, require_capture_window: bool = False, +) -> None: """Reject ambiguous combinations and malformed traces before saving any files.""" for field in ("collection_id", "monotonic_ns", "capture_start_ns", "capture_end_ns"): _validate_u64(getattr(collection, field), field) + if require_capture_window and not 0 < collection.capture_start_ns <= collection.capture_end_ns: + raise ValueError("Raw collection requires a positive, ordered capture window") if require_finite and not collection.traces: raise ValueError("Preprocess set must contain at least one trace") seen: set[tuple[int, int]] = set() @@ -43,11 +35,10 @@ def validate_trace_collection(collection: SweepCollection, *, require_finite: bo seen.add(combo) arrays = ( np.asarray(trace.frequency_hz, dtype=np.float32), - np.asarray(trace.s11, dtype=np.complex64), np.asarray(trace.s21, dtype=np.complex64), ) if any(array.ndim != 1 for array in arrays) or len({array.shape for array in arrays}) != 1: - raise ValueError(f"Trace {combo} frequency, S11 and S21 must be equally sized 1D arrays") + raise ValueError(f"Trace {combo} frequency and S21 must be equally sized 1D arrays") if require_finite and (not arrays[0].size or any(not np.isfinite(array).all() for array in arrays)): raise ValueError(f"Preprocess trace {combo} must contain finite, nonempty arrays") @@ -57,28 +48,28 @@ def _validate_u64(value: int, field: str) -> None: raise ValueError(f"{field} must be an unsigned 64-bit integer") -def serialize_trace_collection(collection: SweepCollection, magic: int) -> bytes: +def serialize_trace_collection( + collection: SweepCollection, magic: int, *, require_capture_window: bool = False, +) -> bytes: """Serialize one raw/preprocessed trace collection into ring-compatible binary format. - The trailer is appended after the trace blocks so older readers, which stop at - the last block, still decode the traces: first the collection capture window, - then a per-trace window table (one ``(start_ns, end_ns)`` pair per trace, in - trace order). See :func:`python_app.orchestration.shm.decoder.decode_trace_collection` + Each trace stores its frequency array followed by its complex S21 array, using + 12 bytes per point. The trailer contains the collection capture window and a + per-trace window table (one ``(start_ns, end_ns)`` pair per trace, in trace + order). See :func:`python_app.orchestration.shm.decoder.decode_trace_collection` and ``read_trace_collection`` in ``common_cpp/ipc/src/shared_types.cpp``. """ - validate_trace_collection(collection) + validate_trace_collection(collection, require_capture_window=require_capture_window) buffer = bytearray() buffer.extend(struct.pack(" bytes struct.pack(" bytes: - """Serialize one processed collection with result blocks/payloads.""" - - def serialize_payload(buffer: bytearray, payload) -> None: - """Append one payload in ring-compatible result format.""" - name_bytes = payload.processing_name.encode("utf-8") - if len(name_bytes) > 0xFFFF: - raise ValueError("processing_name is too long") - - buffer.extend(struct.pack(" Iterator[Path]: destination.parent.mkdir(parents=True, exist_ok=True) if destination.exists(): raise FileExistsError(f"Snapshot directory already exists: {destination}") - staging = Path(tempfile.mkdtemp(prefix=f".{destination.name}.", dir=destination.parent)) + staging = destination.with_name(f".{destination.name}.{uuid4().hex}") + staging.mkdir() try: yield staging - if destination.exists(): - raise FileExistsError(f"Snapshot directory already exists: {destination}") - staging.rename(destination) + rename_no_replace(staging, destination) finally: if staging.exists(): shutil.rmtree(staging) -def _matching_collection( - history: list[SweepCollection], anchor: SweepCollection | ResultCollection, - used: set[int], *, same_timestamp: bool, -) -> SweepCollection | None: - """Match preprocessing identities exactly and raw sweeps by their capture window. - - Producers stamp raw/preprocessed collections at different instants. Their capture - window is copied unchanged; results keep the preprocessed timestamp. Old files - without timing fall back to the latest preceding collection with the same id. - """ - candidates = [ - (index, item) for index, item in enumerate(history) - if index not in used and int(item.collection_id) == int(anchor.collection_id) - ] - timestamp = int(anchor.monotonic_ns) - start_ns = int(getattr(anchor, "capture_start_ns", 0)) - end_ns = int(getattr(anchor, "capture_end_ns", 0)) - if same_timestamp and timestamp: - candidates = [(index, item) for index, item in candidates if int(item.monotonic_ns) == timestamp] - elif start_ns or end_ns: - candidates = [ - (index, item) for index, item in candidates - if (int(item.capture_start_ns), int(item.capture_end_ns)) == (start_ns, end_ns) - ] - elif timestamp: - candidates = [(index, item) for index, item in candidates if int(item.monotonic_ns) <= timestamp] - if not candidates: - return None - index, item = max(candidates, key=lambda pair: (int(pair[1].monotonic_ns), pair[0])) - used.add(index) - return item - - -def select_aligned_histories( - raw_history: list[SweepCollection], - preprocessed_history: list[SweepCollection], - result_history: list[ResultCollection], - last_n: int, - *, - include_unaligned: bool = False, -) -> tuple[list[SweepCollection], list[SweepCollection], list[ResultCollection], dict[str, Any]]: - """Select aligned tails, or retain every stage independently for complete exports.""" - if last_n <= 0: - raise ValueError("last_n must be > 0") - if include_unaligned: - return ( - raw_history[-last_n:], preprocessed_history[-last_n:], result_history[-last_n:], - {"selection_mode": "independent_tail", "anchor_stage": "none", - "selected_collection_ids": [int(item.collection_id) for item in result_history[-last_n:]], - "aligned_key_count": 0}, - ) - - selected_raw: list[SweepCollection] = [] - selected_pre: list[SweepCollection] = [] - selected_results = result_history[-last_n:] - used_raw: set[int] = set() - used_pre: set[int] = set() - if selected_results: - anchors = selected_results - anchor_stage = "results" - for result in reversed(selected_results): - pre = _matching_collection(preprocessed_history, result, used_pre, same_timestamp=True) - raw = _matching_collection(raw_history, pre or result, used_raw, same_timestamp=False) - if pre is not None: - selected_pre.append(pre) - if raw is not None: - selected_raw.append(raw) - selected_pre.reverse() - selected_raw.reverse() - elif preprocessed_history: - selected_pre = preprocessed_history[-last_n:] - anchors = selected_pre - anchor_stage = "preprocessed" - for pre in reversed(selected_pre): - raw = _matching_collection(raw_history, pre, used_raw, same_timestamp=False) - if raw is not None: - selected_raw.append(raw) - selected_raw.reverse() - else: - selected_raw = raw_history[-last_n:] - anchors = selected_raw - anchor_stage = "raw" if anchors else "none" - - return selected_raw, selected_pre, selected_results, { - "selection_mode": f"{anchor_stage}_tail_with_optional_alignment", - "anchor_stage": anchor_stage, - "selected_collection_ids": [int(item.collection_id) for item in anchors], - "aligned_key_count": len(anchors), - "raw_missing_count": len(anchors) - len(selected_raw), - "preprocessed_missing_count": len(anchors) - len(selected_pre), - "result_missing_count": len(anchors) - len(selected_results), - } - - -def save_trace_history_binary(stage_dir: Path, history: list[SweepCollection], magic: int) -> None: - """Write binary trace history with lightweight metadata sidecars.""" - stage_dir.mkdir(parents=True, exist_ok=True) - logger.debug("Writing %d binary trace collection(s) to %s", len(history), stage_dir) - for index, collection in enumerate(history): - binary_path = stage_dir / f"{index:04d}.bin" - metadata_path = stage_dir / f"{index:04d}.json" - binary_path.write_bytes(serialize_trace_collection(collection, magic)) - metadata_path.write_text( - json.dumps( - { - "collection_id": collection.collection_id, - "monotonic_ns": collection.monotonic_ns, - "capture_start_ns": int(collection.capture_start_ns), - "capture_end_ns": int(collection.capture_end_ns), - "trace_count": len(collection.traces), - # Also in the .bin trailer; repeated here so per-combo timing is - # readable without decoding the binary payload. - "traces": [ - { - "input": int(trace.combo.input), - "output": int(trace.combo.output), - "capture_start_ns": int(trace.capture_start_ns), - "capture_end_ns": int(trace.capture_end_ns), - } - for trace in collection.traces - ], - }, - indent=2, - ), - encoding="utf-8", - ) - - -def save_result_history_binary(stage_dir: Path, history: list[ResultCollection]) -> None: - """Write binary processed-result history with metadata sidecars.""" - stage_dir.mkdir(parents=True, exist_ok=True) - logger.debug("Writing %d binary result collection(s) to %s", len(history), stage_dir) - for index, collection in enumerate(history): - binary_path = stage_dir / f"{index:04d}.bin" - metadata_path = stage_dir / f"{index:04d}.json" - binary_path.write_bytes(serialize_result_collection(collection)) - metadata_path.write_text( - json.dumps( - { - "collection_id": collection.collection_id, - "monotonic_ns": collection.monotonic_ns, - "processing_duration_ns": int(collection.processing_duration_ns), - "collection_payload_count": len(collection.collection_payloads), - "block_count": len(collection.blocks), - }, - indent=2, - ), - encoding="utf-8", - ) +def save_processing_settings(directory: Path, settings: dict[int, dict] | None) -> None: + """Merge immutable revision settings before publishing results that refer to them.""" + if not settings: + return + path = directory / "processing_settings.json" + saved = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} + if not isinstance(saved, dict): + raise ValueError("Saved processing settings must be a JSON object") + updated = dict(saved) + for revision, value in settings.items(): + if isinstance(revision, bool) or not isinstance(revision, Integral) or not 0 <= revision < 1 << 64 or not isinstance(value, dict): + raise ValueError("Processing settings require uint64 revision keys and JSON objects") + key = str(revision) + if key in saved and saved[key] != value: + raise ValueError(f"Processing revision {revision} already has different settings") + updated[key] = value + if updated != saved: + atomic_write_json(path, updated, durable=True) def save_trace_history_numpy( @@ -202,47 +73,43 @@ def save_trace_history_numpy( logger.debug("Writing %d NumPy trace collection(s) to %s", len(history), stage_dir) for index, collection in enumerate(history, start=index_offset): collection_dir = stage_dir / collection_dir_name(index, collection.collection_id, collection.monotonic_ns) - collection_dir.mkdir(parents=True, exist_ok=False) + with atomic_snapshot_directory(collection_dir) as collection_dir: + traces_meta: list[dict[str, int | str]] = [] + for trace in collection.traces: + tag = f"i{trace.combo.input}_o{trace.combo.output}" + freq = np.asarray(trace.frequency_hz, dtype=np.float32) + s21 = np.asarray(trace.s21, dtype=np.complex64) + np.save(collection_dir / f"{tag}_freq.npy", freq) + np.save(collection_dir / f"{tag}_s21.npy", s21) + traces_meta.append( + { + "input": int(trace.combo.input), + "output": int(trace.combo.output), + "points": int(freq.size), + # When each combo was measured, which in a switched matrix is + # spread across the collection window rather than aligned with it. + "capture_start_ns": int(trace.capture_start_ns), + "capture_end_ns": int(trace.capture_end_ns), + "freq_file": f"{tag}_freq.npy", + "s21_file": f"{tag}_s21.npy", + } + ) - traces_meta: list[dict[str, int | str]] = [] - for trace in collection.traces: - tag = f"i{trace.combo.input}_o{trace.combo.output}" - freq = np.asarray(trace.frequency_hz, dtype=np.float32) - s11 = np.asarray(trace.s11, dtype=np.complex64) - s21 = np.asarray(trace.s21, dtype=np.complex64) - np.save(collection_dir / f"{tag}_freq.npy", freq) - np.save(collection_dir / f"{tag}_s11.npy", s11) - np.save(collection_dir / f"{tag}_s21.npy", s21) - traces_meta.append( - { - "input": int(trace.combo.input), - "output": int(trace.combo.output), - "points": int(freq.size), - # When each combo was measured, which in a switched matrix is - # spread across the collection window rather than aligned with it. - "capture_start_ns": int(trace.capture_start_ns), - "capture_end_ns": int(trace.capture_end_ns), - "freq_file": f"{tag}_freq.npy", - "s11_file": f"{tag}_s11.npy", - "s21_file": f"{tag}_s21.npy", - } + (collection_dir / "meta.json").write_text( + json.dumps( + { + "collection_id": int(collection.collection_id), + "monotonic_ns": int(collection.monotonic_ns), + "capture_start_ns": int(collection.capture_start_ns), + "capture_end_ns": int(collection.capture_end_ns), + "trace_count": len(collection.traces), + "traces": traces_meta, + }, + indent=2, + ), + encoding="utf-8", ) - (collection_dir / "meta.json").write_text( - json.dumps( - { - "collection_id": int(collection.collection_id), - "monotonic_ns": int(collection.monotonic_ns), - "capture_start_ns": int(collection.capture_start_ns), - "capture_end_ns": int(collection.capture_end_ns), - "trace_count": len(collection.traces), - "traces": traces_meta, - }, - indent=2, - ), - encoding="utf-8", - ) - def _save_result_payload(directory: Path, base_name: str, payload: ResultPayload) -> dict[str, Any]: """Persist every supported payload kind identically at collection and combo scope.""" @@ -293,35 +160,39 @@ def save_result_history_numpy( logger.debug("Writing %d NumPy result collection(s) to %s", len(history), stage_dir) for index, collection in enumerate(history, start=index_offset): collection_dir = stage_dir / collection_dir_name(index, collection.collection_id, collection.monotonic_ns) - collection_dir.mkdir(parents=True, exist_ok=False) - collection_payload_meta = [] - for payload_index, payload in enumerate(collection.collection_payloads): - name = sanitize_path_component(payload.processing_name or "processor") - base_name = f"collection_{payload_index:03d}_{name}_kind{payload.kind}" - collection_payload_meta.append(_save_result_payload(collection_dir, base_name, payload)) - - blocks_meta = [] - for block_index, block in enumerate(collection.blocks): - block_dir = collection_dir / f"block_{block_index:03d}_i{block.combo.input}_o{block.combo.output}" - block_dir.mkdir(parents=True, exist_ok=False) - payload_meta = [] - for payload_index, payload in enumerate(block.payloads): + with atomic_snapshot_directory(collection_dir) as collection_dir: + collection_payload_meta = [] + for payload_index, payload in enumerate(collection.collection_payloads): name = sanitize_path_component(payload.processing_name or "processor") - base_name = f"{payload_index:03d}_{name}_kind{payload.kind}" - payload_meta.append(_save_result_payload(block_dir, base_name, payload)) - blocks_meta.append({ - "input": int(block.combo.input), "output": int(block.combo.output), - "payload_count": len(block.payloads), "dir": block_dir.name, "payloads": payload_meta, - }) + base_name = f"collection_{payload_index:03d}_{name}_kind{payload.kind}" + collection_payload_meta.append(_save_result_payload(collection_dir, base_name, payload)) - (collection_dir / "meta.json").write_text( - json.dumps({ - "collection_id": int(collection.collection_id), - "monotonic_ns": int(collection.monotonic_ns), - "processing_duration_ns": int(collection.processing_duration_ns), - "collection_payload_count": len(collection.collection_payloads), - "collection_payloads": collection_payload_meta, - "block_count": len(collection.blocks), "blocks": blocks_meta, - }, indent=2), - encoding="utf-8", - ) + blocks_meta = [] + for block_index, block in enumerate(collection.blocks): + block_dir = collection_dir / f"block_{block_index:03d}_i{block.combo.input}_o{block.combo.output}" + block_dir.mkdir(parents=True, exist_ok=False) + payload_meta = [] + for payload_index, payload in enumerate(block.payloads): + name = sanitize_path_component(payload.processing_name or "processor") + base_name = f"{payload_index:03d}_{name}_kind{payload.kind}" + payload_meta.append(_save_result_payload(block_dir, base_name, payload)) + blocks_meta.append({ + "input": int(block.combo.input), "output": int(block.combo.output), + "payload_count": len(block.payloads), "dir": block_dir.name, "payloads": payload_meta, + }) + + (collection_dir / "meta.json").write_text( + json.dumps({ + "collection_id": int(collection.collection_id), + "monotonic_ns": int(collection.monotonic_ns), + "processing_duration_ns": int(collection.processing_duration_ns), + "settings_revision": int(collection.settings_revision), + "capture_start_ns": int(collection.capture_start_ns), + "capture_end_ns": int(collection.capture_end_ns), + "speed_m_s": float(collection.speed_m_s), + "collection_payload_count": len(collection.collection_payloads), + "collection_payloads": collection_payload_meta, + "block_count": len(collection.blocks), "blocks": blocks_meta, + }, indent=2), + encoding="utf-8", + ) diff --git a/python_app/storage/npz/store.py b/python_app/storage/npz/store.py index baa912f..62fb862 100644 --- a/python_app/storage/npz/store.py +++ b/python_app/storage/npz/store.py @@ -1,4 +1,4 @@ -"""Concrete :class:`StoreApi` implementation backed by NPZ files.""" +"""Preprocess sets and datasets backed by NPZ files.""" from __future__ import annotations @@ -9,26 +9,24 @@ import logging import os from pathlib import Path import shutil -import tempfile from typing import Any +from uuid import uuid4 from zipfile import BadZipFile import numpy as np +from python_app.file_io import make_directory, rename_no_replace, sync_directory, temporary_file from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection, TraceData -from python_app.models.run_config_model import RunConfigModel -from python_app.storage.npz.paths import radar_key_from_config, sanitize_path_component -from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, serialize_trace_collection, validate_trace_collection +from python_app.file_io import sanitize_path_component +from python_app.orchestration.shm.decoder import RAW_MAGIC +from python_app.storage.npz.serialize import serialize_trace_collection, validate_trace_collection from python_app.storage.npz.snapshot_numpy import ( - save_result_history_binary, save_result_history_numpy, - save_trace_history_binary, save_trace_history_numpy, - select_aligned_histories, + save_processing_settings, atomic_snapshot_directory, ) from python_app.storage.npz.vna_history_json import build_vna_history_payload -from python_app.storage.store_api import StoreApi logger = logging.getLogger(__name__) _SET_METADATA_KEY = "metadata_json" @@ -44,22 +42,11 @@ def _compose_snapshot_stem(name: str, name_prefix: str) -> str: class SnapshotStreamWriter: - """Append received raw/preprocessed/result collections to a snapshot dir in chunks. - - A streaming alternative to :meth:`NpzStore.save_runtime_snapshot_numpy` for long - recordings: the caller flushes small chunks as measurements arrive and frees them, - so memory stays flat instead of buffering every measurement. The on-disk layout is - identical (``raw``/``preprocessed``/``results`` subtrees), and per-collection - directory indices continue across chunks via a running offset per stage. - """ - - __slots__ = ("_dir", "_raw_written", "_preprocessed_written", "_result_written") + """Publish complete collections incrementally, retaining them if a later write fails.""" def __init__(self, snapshot_dir: Path) -> None: self._dir = snapshot_dir - self._raw_written = 0 - self._preprocessed_written = 0 - self._result_written = 0 + self._counts = {"raw": 0, "preprocessed": 0, "results": 0} @property def directory(self) -> Path: @@ -67,66 +54,50 @@ class SnapshotStreamWriter: @property def result_count(self) -> int: - """Number of result collections written to disk so far.""" - return self._result_written + return self._counts["results"] def append( self, raw: list[SweepCollection], preprocessed: list[SweepCollection], results: list[ResultCollection], + *, processing_settings: dict[int, dict] | None = None, ) -> None: - """Write a chunk without requiring equal stage lengths, continuing each index.""" - # Stage a whole chunk before publishing complete collection directories. A - # failed write leaves earlier chunks intact and the next retry uses the same indices. - chunk_dir = Path(tempfile.mkdtemp(prefix=".chunk-", dir=self._dir)) - published: list[Path] = [] - try: - save_trace_history_numpy(chunk_dir / "raw", raw, index_offset=self._raw_written) - save_trace_history_numpy( - chunk_dir / "preprocessed", preprocessed, index_offset=self._preprocessed_written - ) - save_result_history_numpy(chunk_dir / "results", results, index_offset=self._result_written) - for stage in ("raw", "preprocessed", "results"): - stage_dir = self._dir / stage - stage_dir.mkdir(exist_ok=True) - for source in sorted((chunk_dir / stage).iterdir()): - destination = stage_dir / source.name - if destination.exists(): - raise FileExistsError(f"Collection directory already exists: {destination}") - source.rename(destination) - published.append(destination) - except BaseException: - for destination in published: - shutil.rmtree(destination) - raise - finally: - shutil.rmtree(chunk_dir) - self._raw_written += len(raw) - self._preprocessed_written += len(preprocessed) - self._result_written += len(results) + """Append a chunk; after failure, only the unwritten collections may be retried.""" + save_processing_settings(self._dir, processing_settings) + for stage, history in (("raw", raw), ("preprocessed", preprocessed)): + stage_dir = self._dir / stage + stage_dir.mkdir(exist_ok=True) + for collection in history: + save_trace_history_numpy(stage_dir, [collection], index_offset=self._counts[stage]) + self._counts[stage] += 1 + stage_dir = self._dir / "results" + stage_dir.mkdir(exist_ok=True) + for result in results: + save_result_history_numpy(stage_dir, [result], index_offset=self.result_count) + self._counts["results"] += 1 -class NpzStore(StoreApi): +class NpzStore: """Persist preprocess sets and runtime snapshots using NumPy files.""" def __init__(self, root_dir: Path) -> None: """Create store rooted at `root_dir`.""" self._root_dir = root_dir - self._root_dir.mkdir(parents=True, exist_ok=True) + make_directory(self._root_dir, durable=True) logger.debug("NpzStore rooted at %s", self._root_dir) - @staticmethod - def _vna_json_output_dir(output_root_dir: Path, output_stem: str) -> Path: - """Return per-export directory for VNA history JSON files.""" - return output_root_dir / output_stem - - def save_set(self, kind: str, radar_key: str, set_name: str, collection: SweepCollection) -> None: - """Persist named preprocess set as NPZ and metadata JSON.""" + def save_set( + self, kind: str, radar_key: str, set_name: str, collection: SweepCollection, + *, metadata: dict[str, Any] | None = None, + ) -> None: + """Persist a named set with optional JSON metadata embedded in the NPZ.""" self._validate_set_name(set_name) validate_trace_collection(collection, require_finite=True) + if metadata is not None and not isinstance(metadata, dict): + raise ValueError("Set user metadata must be a JSON object") set_dir = self._set_dir(kind, radar_key) - set_dir.mkdir(parents=True, exist_ok=True) + make_directory(set_dir, durable=True) npz_path = set_dir / f"{set_name}.npz" meta_path = set_dir / f"{set_name}.json" @@ -137,10 +108,8 @@ class NpzStore(StoreApi): for trace in collection.traces: suffix = f"i{trace.combo.input}_o{trace.combo.output}" freq_key = f"freq_{suffix}" - s11_key = f"s11_{suffix}" s21_key = f"s21_{suffix}" payload[freq_key] = np.asarray(trace.frequency_hz, dtype=np.float32) - payload[s11_key] = np.asarray(trace.s11, dtype=np.complex64) payload[s21_key] = np.asarray(trace.s21, dtype=np.complex64) combo_records.append( { @@ -149,7 +118,6 @@ class NpzStore(StoreApi): "capture_start_ns": int(trace.capture_start_ns), "capture_end_ns": int(trace.capture_end_ns), "freq_key": freq_key, - "s11_key": s11_key, "s21_key": s21_key, } ) @@ -160,24 +128,30 @@ class NpzStore(StoreApi): "capture_start_ns": int(collection.capture_start_ns), "capture_end_ns": int(collection.capture_end_ns), "combos": combo_records, + "metadata": {} if metadata is None else metadata, } # Metadata in the NPZ makes a single atomic replacement authoritative. The # JSON remains a readable sidecar and legacy NPZ/JSON pairs remain loadable. - metadata_json = json.dumps(meta, indent=2) + metadata_json = json.dumps(meta, indent=2, allow_nan=False) payload[_SET_METADATA_KEY] = np.asarray(metadata_json) npz_tmp = meta_tmp = backup_path = None archive_replaced = False try: - npz_tmp = self._temporary_file(npz_path) - meta_tmp = self._temporary_file(meta_path) - backup_path = self._temporary_file(npz_path) if npz_path.exists() else None + npz_tmp = temporary_file(npz_path) + meta_tmp = temporary_file(meta_path) + backup_path = temporary_file(npz_path) if npz_path.exists() else None if backup_path is not None: backup_path.unlink() os.link(npz_path, backup_path) with npz_tmp.open("wb") as npz_file: np.savez(npz_file, **payload) - meta_tmp.write_text(metadata_json, encoding="utf-8") + npz_file.flush() + os.fsync(npz_file.fileno()) + with meta_tmp.open("w", encoding="utf-8") as meta_file: + meta_file.write(metadata_json) + meta_file.flush() + os.fsync(meta_file.fileno()) npz_tmp.replace(npz_path) archive_replaced = True meta_tmp.replace(meta_path) @@ -193,6 +167,7 @@ class NpzStore(StoreApi): logger.exception("Set rollback failed; previous archive retained at %s", backup_path) backup_path = None # retain the recovery copy rather than deleting it below raise + sync_directory(set_dir) raise finally: for tmp_path in (npz_tmp, meta_tmp, backup_path): @@ -200,6 +175,7 @@ class NpzStore(StoreApi): continue with suppress(OSError): tmp_path.unlink(missing_ok=True) + sync_directory(set_dir) logger.info( "Saved set %s/%s/%s (%d traces) to %s", kind, radar_key, set_name, len(combo_records), npz_path ) @@ -209,7 +185,8 @@ class NpzStore(StoreApi): self._validate_set_name(set_name) for collection in collections.values(): validate_trace_collection(collection, require_finite=True) - backup_dir = Path(tempfile.mkdtemp(prefix=".sets-", dir=self._root_dir)) + backup_dir = self._root_dir / f".sets-{uuid4().hex}" + backup_dir.mkdir() backups: dict[Path, Path | None] = {} retain_backups = False try: @@ -236,6 +213,12 @@ class NpzStore(StoreApi): except OSError: rollback_failed = True logger.exception("Failed to restore previous set file %s", destination) + for directory in {path.parent for path in backups if path.parent.exists()}: + try: + sync_directory(directory) + except OSError: + rollback_failed = True + logger.exception("Failed to sync restored set directory %s", directory) retain_backups = rollback_failed if retain_backups: raise RuntimeError( @@ -250,6 +233,12 @@ class NpzStore(StoreApi): def load_set(self, kind: str, radar_key: str, set_name: str) -> SweepCollection: """Load named preprocess set from NPZ representation.""" + return self.load_set_with_metadata(kind, radar_key, set_name)[0] + + def load_set_with_metadata( + self, kind: str, radar_key: str, set_name: str, + ) -> tuple[SweepCollection, dict[str, Any]]: + """Load a named set and its user metadata; older sets have empty metadata.""" self._validate_set_name(set_name) set_dir = self._set_dir(kind, radar_key) npz_path = set_dir / f"{set_name}.npz" @@ -269,13 +258,16 @@ class NpzStore(StoreApi): ) if not isinstance(meta, dict): raise ValueError("Set metadata must be a JSON object") + metadata = meta.get("metadata", {}) + if not isinstance(metadata, dict): + raise ValueError("Set user metadata must be a JSON object") + json.dumps(metadata, allow_nan=False) traces: list[TraceData] = [] for combo in meta["combos"]: traces.append( TraceData( combo=ComboKey(input=combo["input"], output=combo["output"]), frequency_hz=np.asarray(arrays[combo["freq_key"]], dtype=np.float32), - s11=np.asarray(arrays[combo["s11_key"]], dtype=np.complex64), s21=np.asarray(arrays[combo["s21_key"]], dtype=np.complex64), capture_start_ns=combo.get("capture_start_ns", 0), capture_end_ns=combo.get("capture_end_ns", 0), @@ -291,7 +283,7 @@ class NpzStore(StoreApi): ) validate_trace_collection(collection, require_finite=True) logger.debug("Loaded set %s (%d traces)", set_label, len(traces)) - return collection + return collection, metadata except (OSError, BadZipFile, EOFError, KeyError, TypeError, ValueError) as exc: logger.exception("Corrupted preprocess set %s", set_label) raise RuntimeError(f"Corrupted preprocess set {set_label}: {exc}") from exc @@ -301,34 +293,10 @@ class NpzStore(StoreApi): set_dir = self._set_dir(kind, radar_key) if not set_dir.exists(): return [] - if kind == "s21_object_calibration": - return sorted(path.name for path in set_dir.iterdir() - if not path.name.startswith(".") and (path / "cal_meta.json").is_file()) return sorted(path.stem for path in set_dir.glob("*.npz")) - def save_object_calibration(self, radar_key: str, set_name: str, calibration: dict[str, Any]) -> None: - """Save an object calibration in its shared archive format without overwriting a set.""" - from python_app.workflows.mimo_calibration import save_calibration - - self._validate_set_name(set_name) - destination = self._set_dir("s21_object_calibration", radar_key) / set_name - if destination.exists(): - raise FileExistsError(f"Object calibration already exists: {set_name}") - save_calibration(calibration, destination) - - def load_object_calibration(self, radar_key: str, set_name: str) -> dict[str, Any]: - """Load and validate a saved object calibration, retaining all estimation metadata.""" - from python_app.workflows.mimo_calibration import load_calibration - - self._validate_set_name(set_name) - return load_calibration(self._set_dir("s21_object_calibration", radar_key) / set_name) - def has_combo_coverage(self, kind: str, radar_key: str, set_name: str, combos: list[ComboKey]) -> bool: """Validate that named set covers all required switch combinations.""" - if kind == "s21_object_calibration": - calibration = self.load_object_calibration(radar_key, set_name) - return all(combo.input in calibration["rx_input_pos"] and combo.output in calibration["tx_output_pos"] - for combo in combos) collection = self.load_set(kind, radar_key, set_name) existing = {(trace.combo.input, trace.combo.output) for trace in collection.traces} required = {(combo.input, combo.output) for combo in combos} @@ -339,36 +307,11 @@ class NpzStore(StoreApi): collection = self.load_set(kind, radar_key, set_name) return self._export_collection_bundle(collection, output_path) - def export_object_calibration_bundle( - self, radar_key: str, set_name: str, output_path: Path, config: RunConfigModel, - ) -> Path: - """Resample pair multipliers on the through-calibration grids used by the acquisition.""" - from python_app.workflows.mimo_calibration import calibration_on_grid, validate_for_config - - calibration = self.load_object_calibration(radar_key, set_name) - validate_for_config(calibration, config.to_dict()) - tx_indices = {int(port): index for index, port in enumerate(calibration["tx_output_pos"])} - rx_indices = {int(port): index for index, port in enumerate(calibration["rx_input_pos"])} - source = self.load_set("s21_calibration", radar_key, config.preprocess.s21.calibration.set_name) - by_combo = {trace.combo: trace for trace in source.traces} - grids: dict[bytes, np.ndarray] = {} - traces: list[TraceData] = [] - for combo in config.combos: - key = ComboKey(input=combo.input, output=combo.output) - trace = by_combo[key] - frequency = trace.frequency_hz - grid_key = frequency.tobytes() - if grid_key not in grids: - grids[grid_key] = calibration_on_grid(calibration, frequency) - correction = grids[grid_key][tx_indices[key.output], rx_indices[key.input]] - traces.append(TraceData(key, frequency, np.zeros(frequency.size, np.complex64), correction)) - return self._export_collection_bundle(SweepCollection(0, 0, traces), output_path) - def _export_collection_bundle(self, collection: SweepCollection, output_path: Path) -> Path: """Publish a complete binary trace bundle using one atomic replacement.""" validate_trace_collection(collection, require_finite=True) output_path.parent.mkdir(parents=True, exist_ok=True) - temporary = self._temporary_file(output_path) + temporary = temporary_file(output_path) try: temporary.write_bytes(serialize_trace_collection(collection, RAW_MAGIC)) temporary.replace(output_path) @@ -377,28 +320,6 @@ class NpzStore(StoreApi): logger.info("Exported %d traces to bundle %s", len(collection.traces), output_path) return output_path - def save_runtime_snapshot( - self, - output_dir: Path, - raw_history: list[SweepCollection], - preprocessed_history: list[SweepCollection], - result_history: list[ResultCollection], - last_n: int, - ) -> Path: - """Save historical runtime collections using binary on-disk format.""" - if last_n <= 0: - raise ValueError("last_n must be > 0") - - output_dir.mkdir(parents=True, exist_ok=True) - timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") - snapshot_dir = output_dir / f"snapshot_{timestamp}" - with atomic_snapshot_directory(snapshot_dir) as staging: - save_trace_history_binary(staging / "raw", raw_history[-last_n:], RAW_MAGIC) - save_trace_history_binary(staging / "preprocessed", preprocessed_history[-last_n:], PREPROC_MAGIC) - save_result_history_binary(staging / "results", result_history[-last_n:]) - logger.info("Saved binary runtime snapshot (last_n=%d) to %s", last_n, snapshot_dir) - return snapshot_dir - def snapshot_directory( self, output_root_dir: Path, snapshot_name: str, *, name_prefix: str = "" ) -> Path: @@ -435,126 +356,19 @@ class NpzStore(StoreApi): raw_history: list[SweepCollection], preprocessed_history: list[SweepCollection], result_history: list[ResultCollection], - last_n: int, *, name_prefix: str = "", - include_unaligned: bool = False, - ) -> tuple[Path, dict[str, Any]]: - """Save historical runtime collections in NumPy tree format.""" - if last_n <= 0: - raise ValueError("last_n must be > 0") - - snapshot_stem = _compose_snapshot_stem(snapshot_name, name_prefix) - - output_root_dir.mkdir(parents=True, exist_ok=True) - snapshot_dir = output_root_dir / snapshot_stem - if snapshot_dir.exists(): - raise FileExistsError(f"Snapshot directory already exists: {snapshot_dir}") - - selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories( - raw_history, - preprocessed_history, - result_history, - last_n, - include_unaligned=include_unaligned, - ) - - if selection_summary.get("anchor_stage") == "results": - expected = min(int(last_n), len(result_history)) - if len(selected_results) != expected: - raise RuntimeError( - "Snapshot results selection invariant failed: " - f"expected={expected}, selected={len(selected_results)}" - ) - + processing_settings: dict[int, dict] | None = None, + ) -> Path: + """Save exactly the supplied collections and their processing revisions.""" + snapshot_dir = self.snapshot_directory(output_root_dir, snapshot_name, name_prefix=name_prefix) with atomic_snapshot_directory(snapshot_dir) as staging: - save_trace_history_numpy(staging / "raw", selected_raw) - save_trace_history_numpy(staging / "preprocessed", selected_preprocessed) - save_result_history_numpy(staging / "results", selected_results) - - selection_summary["raw_count"] = len(selected_raw) - selection_summary["preprocessed_count"] = len(selected_preprocessed) - selection_summary["result_count"] = len(selected_results) - selection_summary["snapshot_stem"] = snapshot_stem - selection_summary["snapshot_dir"] = str(snapshot_dir) - logger.info( - "Saved NumPy runtime snapshot to %s (raw=%d preprocessed=%d results=%d, anchor=%s)", - snapshot_dir, - len(selected_raw), - len(selected_preprocessed), - len(selected_results), - selection_summary.get("anchor_stage"), - ) - return snapshot_dir, selection_summary - - def save_runtime_vna_history_json( - self, - output_root_dir: Path, - output_name: str, - raw_history: list[SweepCollection], - preprocessed_history: list[SweepCollection], - result_history: list[ResultCollection], - last_n: int, - *, - input_index: int = 0, - output_index: int = 0, - channel: str = "s21", - primary_stage: str = "preprocessed", - include_unaligned: bool = False, - ) -> tuple[Path, dict[str, Any]]: - """Save runtime history as vna_system-compatible JSON file.""" - if last_n <= 0: - raise ValueError("last_n must be > 0") - - output_stem = output_name.strip() or datetime.now(timezone.utc).strftime("snapshot_%Y%m%d_%H%M%S") - output_stem = sanitize_path_component(output_stem) - - output_root_dir.mkdir(parents=True, exist_ok=True) - output_dir = self._vna_json_output_dir(output_root_dir, output_stem) - output_path = output_dir / f"{output_stem}_vna_bscan_history.json" - - selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories( - raw_history, - preprocessed_history, - result_history, - last_n, - include_unaligned=include_unaligned, - ) - payload = build_vna_history_payload( - selected_raw, - selected_preprocessed, - selected_results, - input_index=input_index, - output_index=output_index, - channel=channel, - primary_stage=primary_stage, - ) - with atomic_snapshot_directory(output_dir) as staging: - (staging / output_path.name).write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") - - summary: dict[str, Any] = dict(selection_summary) - summary["raw_count"] = len(selected_raw) - summary["preprocessed_count"] = len(selected_preprocessed) - summary["result_count"] = len(selected_results) - summary["raw_record_count"] = int(payload.get("raw_record_count", 0)) - summary["preprocessed_record_count"] = int(payload.get("preprocessed_record_count", 0)) - summary["sweep_count"] = len(payload.get("sweep_history", [])) - summary["input_index"] = int(input_index) - summary["output_index"] = int(output_index) - summary["channel"] = str(channel) - summary["primary_stage"] = str(primary_stage) - summary["output_stem"] = output_stem - summary["output_dir"] = str(output_dir) - summary["output_path"] = str(output_path) - logger.info( - "Saved VNA history JSON to %s (input=%d output=%d channel=%s sweeps=%d)", - output_path, - int(input_index), - int(output_index), - channel, - summary["sweep_count"], - ) - return output_path, summary + save_processing_settings(staging, processing_settings) + save_trace_history_numpy(staging / "raw", raw_history) + save_trace_history_numpy(staging / "preprocessed", preprocessed_history) + save_result_history_numpy(staging / "results", result_history) + logger.info("Saved NumPy runtime snapshot to %s", snapshot_dir) + return snapshot_dir def save_runtime_vna_history_json_batch( self, @@ -563,33 +377,19 @@ class NpzStore(StoreApi): raw_history: list[SweepCollection], preprocessed_history: list[SweepCollection], result_history: list[ResultCollection], - last_n: int, *, - channel: str = "s21", primary_stage: str = "preprocessed", name_prefix: str = "", - include_unaligned: bool = False, - ) -> tuple[list[Path], dict[str, Any]]: + ) -> list[Path]: """Save one runtime VNA-history JSON per available combo.""" - if last_n <= 0: - raise ValueError("last_n must be > 0") - output_stem = _compose_snapshot_stem(output_name, name_prefix) output_root_dir.mkdir(parents=True, exist_ok=True) - output_dir = self._vna_json_output_dir(output_root_dir, output_stem) - - selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories( - raw_history, - preprocessed_history, - result_history, - last_n, - include_unaligned=include_unaligned, - ) + output_dir = output_root_dir / output_stem combos = sorted( { (int(trace.combo.input), int(trace.combo.output)) - for collection in [*selected_raw, *selected_preprocessed] + for collection in [*raw_history, *preprocessed_history] for trace in collection.traces } ) @@ -601,45 +401,51 @@ class NpzStore(StoreApi): payloads: list[dict[str, Any]] = [] for input_index, output_index in combos: output_path = output_dir / ( - f"{output_stem}_i{input_index}_o{output_index}_{channel}_vna_bscan_history.json" + f"{output_stem}_i{input_index}_o{output_index}_s21_vna_bscan_history.json" ) if output_path.exists(): raise FileExistsError(f"Output JSON file already exists: {output_path}") payloads.append( build_vna_history_payload( - selected_raw, - selected_preprocessed, - selected_results, + raw_history, + preprocessed_history, + result_history, input_index=input_index, output_index=output_index, - channel=channel, primary_stage=primary_stage, ) ) output_paths.append(output_path) - with atomic_snapshot_directory(output_dir) as staging: + output_existed = output_dir.exists() + output_dir.mkdir(exist_ok=True) + published: list[Path] = [] + try: + # Publish complete files without overwriting a concurrent export. for output_path, payload in zip(output_paths, payloads, strict=True): - (staging / output_path.name).write_text( - json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8" - ) + temporary = temporary_file(output_path) + try: + with temporary.open("w", encoding="utf-8") as handle: + json.dump(payload, handle, ensure_ascii=False, indent=2, allow_nan=False) + handle.flush() + os.fsync(handle.fileno()) + rename_no_replace(temporary, output_path) + published.append(output_path) + finally: + temporary.unlink(missing_ok=True) + sync_directory(output_dir) + except BaseException: + for path in published: + path.unlink() + if not output_existed: + with suppress(OSError): + output_dir.rmdir() + raise - summary: dict[str, Any] = dict(selection_summary) - summary["raw_count"] = len(selected_raw) - summary["preprocessed_count"] = len(selected_preprocessed) - summary["result_count"] = len(selected_results) - summary["preprocessed_record_count"] = int(sum(int(payload.get("preprocessed_record_count", 0)) for payload in payloads)) - summary["combo_count"] = len(combos) - summary["combos"] = [list(combo) for combo in combos] - summary["channel"] = str(channel) - summary["primary_stage"] = str(primary_stage) - summary["output_stem"] = output_stem - summary["output_dir"] = str(output_dir) - summary["output_paths"] = [str(path) for path in output_paths] logger.info( - "Saved %d VNA history JSON file(s) to %s (channel=%s)", len(output_paths), output_dir, channel + "Saved %d S21 VNA history JSON file(s) to %s", len(output_paths), output_dir ) - return output_paths, summary + return output_paths def _set_dir(self, kind: str, radar_key: str) -> Path: """Return directory for set kind and radar key.""" @@ -653,15 +459,6 @@ class NpzStore(StoreApi): if not value or value in {".", ".."} or Path(value).name != value or "\\" in value: raise ValueError(f"Invalid set path component: {value!r}") - @staticmethod - def _temporary_file(destination: Path) -> Path: - """Create a unique temporary file beside its atomic-replacement destination.""" - with tempfile.NamedTemporaryFile( - dir=destination.parent, prefix=f".{destination.name}.", suffix=".tmp", delete=False - ) as handle: - path = Path(handle.name) - return path - def preview_png_dir(self, kind: str, set_name: str, radar_key: str) -> Path: """Return directory for preview PNGs of one set/radar variant. @@ -671,6 +468,3 @@ class NpzStore(StoreApi): filesystem-safe by construction. """ return self._root_dir / "preview_png" / kind / sanitize_path_component(set_name) / radar_key - - -__all__ = ["NpzStore", "radar_key_from_config"] diff --git a/python_app/storage/npz/vna_history_json.py b/python_app/storage/npz/vna_history_json.py index edae0e2..ab78d55 100644 --- a/python_app/storage/npz/vna_history_json.py +++ b/python_app/storage/npz/vna_history_json.py @@ -5,11 +5,14 @@ from __future__ import annotations from dataclasses import dataclass from datetime import datetime, timezone import logging +import json +from pathlib import Path from typing import Any import numpy as np -from python_app.models.dataset_model import ResultCollection, SweepCollection, TraceData +from python_app.models.collection_identity import index_collections +from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection, TraceData logger = logging.getLogger(__name__) @@ -18,30 +21,15 @@ logger = logging.getLogger(__name__) class TraceRecord: """One raw/preprocessed trace selected for vna_history export.""" - stage: str collection_id: int monotonic_ns: int stage_index: int frequency_hz: np.ndarray samples: np.ndarray - # End of this trace's own sweep, or 0 when the producer reported no per-trace - # timing. Preferred over the collection timestamp for the exported sweep time: - # in a switched matrix each combo is measured at a different instant. + # Collection window identifies the acquisition across pipeline stages. capture_end_ns: int = 0 capture_start_ns: int = 0 - - -def _normalize_channel(channel: str) -> str: - normalized = str(channel).strip().lower() - if normalized not in {"s21", "s11"}: - raise ValueError("channel must be either 's21' or 's11'") - return normalized - - -def _select_trace_samples(trace: TraceData, channel: str) -> np.ndarray: - if channel == "s11": - return np.asarray(trace.s11, dtype=np.complex128).reshape(-1) - return np.asarray(trace.s21, dtype=np.complex128).reshape(-1) + trace_capture_end_ns: int = 0 def _pick_trace(collection: SweepCollection, input_index: int, output_index: int) -> TraceData | None: @@ -57,7 +45,6 @@ def _build_stage_records( *, input_index: int, output_index: int, - channel: str, ) -> list[TraceRecord]: records: list[TraceRecord] = [] for stage_index, collection in enumerate(history): @@ -66,11 +53,11 @@ def _build_stage_records( continue frequency_hz = np.asarray(trace.frequency_hz, dtype=np.float64).reshape(-1) - samples = _select_trace_samples(trace, channel) + samples = np.asarray(trace.s21, dtype=np.complex128).reshape(-1) if frequency_hz.shape != samples.shape: raise ValueError( f"Shape mismatch in {stage} stage for collection_id={collection.collection_id}: " - f"freq{frequency_hz.shape} vs {channel}{samples.shape}" + f"freq{frequency_hz.shape} vs S21{samples.shape}" ) if frequency_hz.size == 0: continue @@ -83,14 +70,14 @@ def _build_stage_records( records.append( TraceRecord( - stage=stage, collection_id=int(collection.collection_id), monotonic_ns=int(collection.monotonic_ns), stage_index=int(stage_index), frequency_hz=frequency_hz, samples=samples, - capture_end_ns=int(trace.capture_end_ns), - capture_start_ns=int(trace.capture_start_ns), + capture_end_ns=int(collection.capture_end_ns), + capture_start_ns=int(collection.capture_start_ns), + trace_capture_end_ns=int(trace.capture_end_ns), ) ) return records @@ -104,69 +91,59 @@ def _build_sweep_history( raw_records: list[TraceRecord], preprocessed_records: list[TraceRecord], *, - channel: str, primary_stage: str, ) -> list[dict[str, Any]]: + # Runtime identities are exact. Only archives with no collection window need + # the old monotonic/index fallback; never merge their repeated IDs in a dict. + raw_index = index_collections(record for record in raw_records if record.capture_start_ns) + pre_index = index_collections(record for record in preprocessed_records if record.capture_start_ns) + pairs = [(raw_index.get(key), pre_index.get(key)) for key in raw_index.keys() | pre_index.keys()] primary = preprocessed_records if primary_stage == "preprocessed" else raw_records secondary = raw_records if primary_stage == "preprocessed" else preprocessed_records - used: set[int] = set() - pairs: list[tuple[TraceRecord, TraceRecord | None]] = [] + secondary = [record for record in secondary if not record.capture_start_ns] for base in primary: - candidates = [ - (index, record) for index, record in enumerate(secondary) - if index not in used and record.collection_id == base.collection_id - ] - if base.capture_start_ns or base.capture_end_ns: - candidates = [ - (index, record) for index, record in candidates - if (record.capture_start_ns, record.capture_end_ns) == (base.capture_start_ns, base.capture_end_ns) - ] - elif base.monotonic_ns: - # Raw is produced before preprocessing; never pair it to a newer run. - candidates = [ - (index, record) for index, record in candidates - if (record.monotonic_ns <= base.monotonic_ns if primary_stage == "preprocessed" - else record.monotonic_ns >= base.monotonic_ns) - ] - match = None - if candidates: - index, match = min(candidates, key=lambda pair: abs(pair[1].monotonic_ns - base.monotonic_ns)) - used.add(index) - pairs.append((base, match)) - # A tap can deliver a sweep while another stage drops it or is still processing. - # Preserve those records as well instead of silently discarding the extra stage. - pairs.extend((record, None) for index, record in enumerate(secondary) if index not in used) - pairs.sort(key=lambda pair: (pair[0].monotonic_ns, pair[0].stage_index)) - - history: list[dict[str, Any]] = [] - for fallback_index, (base, match) in enumerate(pairs): - raw = base if base.stage == "raw" else match - pre = base if base.stage == "preprocessed" else match - sweep_source = raw or pre - calibrated_source = pre or raw - if sweep_source is None or calibrated_source is None: + if base.capture_start_ns: continue + candidates = [record for record in secondary if record.collection_id == base.collection_id and ( + not base.monotonic_ns or + (record.monotonic_ns <= base.monotonic_ns if primary_stage == "preprocessed" + else record.monotonic_ns >= base.monotonic_ns) + )] + match = min(candidates, key=lambda record: ( + abs(record.monotonic_ns - base.monotonic_ns), abs(record.stage_index - base.stage_index), + )) if candidates else None + if match is not None: + secondary = [record for record in secondary if record is not match] + pairs.append((match, base) if primary_stage == "preprocessed" else (base, match)) + pairs.extend((record, None) if primary_stage == "preprocessed" else (None, record) for record in secondary) + def source(pair: tuple[TraceRecord | None, TraceRecord | None]) -> TraceRecord: + raw, pre = pair + base = (pre or raw) if primary_stage == "preprocessed" else (raw or pre) + assert base is not None + return base + + pairs.sort(key=lambda pair: (source(pair).capture_start_ns or source(pair).monotonic_ns, + source(pair).stage_index)) + history: list[dict[str, Any]] = [] + for fallback_index, (raw, pre) in enumerate(pairs): + base = source((raw, pre)) start_freq_hz = float(base.frequency_hz[0]) stop_freq_hz = float(base.frequency_hz[-1]) # Prefer the exported trace's own sweep time: with a switching matrix the # combos of one collection are measured milliseconds apart, so the # collection timestamp misplaces every combo but the last. - if base.capture_end_ns > 0: - timestamp_sec = float(base.capture_end_ns) / 1_000_000_000.0 - elif base.monotonic_ns > 0: - timestamp_sec = float(base.monotonic_ns) / 1_000_000_000.0 - else: - timestamp_sec = float(fallback_index) + timestamp_ns = base.trace_capture_end_ns or base.capture_end_ns or base.monotonic_ns + timestamp_sec = timestamp_ns / 1_000_000_000.0 if timestamp_ns else float(fallback_index) history.append( { "timestamp": timestamp_sec, - "sweep_points": _complex_to_points(sweep_source.samples), - "calibrated_points": _complex_to_points(calibrated_source.samples), + "sweep_points": _complex_to_points(raw.samples) if raw is not None else [], + "calibrated_points": _complex_to_points(pre.samples) if pre is not None else [], "reference_points": [], "vna_config": { - "mode": channel, + "mode": "s21", "start_freq": start_freq_hz, "stop_freq": stop_freq_hz, "points": int(base.frequency_hz.size), @@ -180,30 +157,18 @@ def _stage_alignment_warning(pre_history: list[SweepCollection], result_history: if not pre_history or not result_history: return None - common_size = min(len(pre_history), len(result_history)) - if common_size <= 0: + def keys(history: list[SweepCollection] | list[ResultCollection]) -> set[tuple[int, int, int]]: + return {(item.collection_id, item.capture_start_ns or item.monotonic_ns, + -1 if item.capture_start_ns else index) for index, item in enumerate(history)} + + pre_keys, result_keys = keys(pre_history), keys(result_history) + missing_results = len(pre_keys - result_keys) + missing_pre = len(result_keys - pre_keys) + if not missing_results and not missing_pre: return None - - mismatches = 0 - first_mismatch: tuple[int, SweepCollection, ResultCollection] | None = None - for index in range(common_size): - pre = pre_history[index] - result = result_history[index] - if int(pre.collection_id) != int(result.collection_id) or int(pre.monotonic_ns) != int(result.monotonic_ns): - mismatches += 1 - if first_mismatch is None: - first_mismatch = (index, pre, result) - - if mismatches == 0 or first_mismatch is None: - return None - - index, pre, result = first_mismatch return ( "WARNING: snapshot stages are not fully aligned (preprocessed vs results). " - f"Mismatches={mismatches}/{common_size}. " - f"First mismatch at index={index}: " - f"pre=(id={int(pre.collection_id)},ns={int(pre.monotonic_ns)}) vs " - f"results=(id={int(result.collection_id)},ns={int(result.monotonic_ns)}). " + f"Sweeps without results={missing_results}; results without preprocessed sweeps={missing_pre}. " "Export uses preprocessed traces; loaded view in vna_system may differ from " "radar_system on-screen replayed results." ) @@ -216,11 +181,9 @@ def build_vna_history_payload( *, input_index: int, output_index: int, - channel: str = "s21", primary_stage: str = "preprocessed", ) -> dict[str, Any]: """Build vna_system-compatible history JSON payload from runtime histories.""" - channel = _normalize_channel(channel) if primary_stage not in {"preprocessed", "raw"}: raise ValueError("primary_stage must be either 'preprocessed' or 'raw'") @@ -229,14 +192,12 @@ def build_vna_history_payload( raw_history, input_index=input_index, output_index=output_index, - channel=channel, ) preprocessed_records = _build_stage_records( "preprocessed", preprocessed_history, input_index=input_index, output_index=output_index, - channel=channel, ) if not raw_records and not preprocessed_records: raise ValueError( @@ -247,7 +208,6 @@ def build_vna_history_payload( sweep_history = _build_sweep_history( raw_records, preprocessed_records, - channel=channel, primary_stage=primary_stage, ) if not sweep_history: @@ -255,12 +215,12 @@ def build_vna_history_payload( payload: dict[str, Any] = { "format": "vna-system-history-v1", - "converter": "python_app/scripts/convert_snapshot_to_vna_history.py", + "converter": "python_app/storage/npz/vna_history_json.py", "converted_at_utc": datetime.now(timezone.utc).isoformat(), "source_snapshot_dir": "", "input_index": int(input_index), "output_index": int(output_index), - "channel": channel, + "channel": "s21", "primary_stage": str(primary_stage), "raw_record_count": len(raw_records), "preprocessed_record_count": len(preprocessed_records), @@ -285,3 +245,55 @@ def build_vna_history_payload( len(sweep_history), ) return payload + + +def collection_directories(stage_dir: Path) -> list[Path]: + """Order a snapshot stage numerically, including indices beyond four digits.""" + def order(path: Path) -> tuple[int, str]: + prefix = path.name.split("_", 1)[0] + return (int(prefix) if prefix.isdigit() else 0, path.name) + + return sorted((path for path in stage_dir.iterdir() if path.is_dir()), key=order) if stage_dir.exists() else [] + + +def load_vna_history_snapshot( + snapshot_dir: Path, *, input_index: int, output_index: int, primary_stage: str = "preprocessed", +) -> dict[str, Any]: + """Load snapshot identities and selected S21 arrays, then use the runtime exporter.""" + if not snapshot_dir.is_dir(): + raise FileNotFoundError(f"Snapshot directory not found: {snapshot_dir}") + histories: dict[str, list[SweepCollection]] = {"raw": [], "preprocessed": []} + results: list[ResultCollection] = [] + for stage in ("raw", "preprocessed", "results"): + for directory in collection_directories(snapshot_dir / stage): + meta_path = directory / "meta.json" + if not meta_path.exists(): + continue + meta = json.loads(meta_path.read_text(encoding="utf-8")) + if not isinstance(meta, dict): + raise ValueError(f"JSON root must be object: {meta_path}") + cid, ns = int(meta.get("collection_id", -1)), int(meta.get("monotonic_ns", 0)) + start, end = int(meta.get("capture_start_ns", 0)), int(meta.get("capture_end_ns", 0)) + if stage == "results": + results.append(ResultCollection(cid, ns, capture_start_ns=start, capture_end_ns=end)) + continue + collection = SweepCollection(cid, ns, capture_start_ns=start, capture_end_ns=end) + for trace in meta.get("traces", []): + if (int(trace.get("input", -1)), int(trace.get("output", -1))) != (input_index, output_index): + continue + if not trace.get("freq_file") or not trace.get("s21_file"): + continue + collection.traces.append(TraceData( + ComboKey(input_index, output_index), np.load(directory / trace["freq_file"]), + np.load(directory / trace["s21_file"]), + capture_start_ns=int(trace.get("capture_start_ns", 0)), + capture_end_ns=int(trace.get("capture_end_ns", 0)), + )) + break + histories[stage].append(collection) + payload = build_vna_history_payload( + histories["raw"], histories["preprocessed"], results, + input_index=input_index, output_index=output_index, primary_stage=primary_stage, + ) + payload["source_snapshot_dir"] = str(snapshot_dir) + return payload diff --git a/python_app/storage/npz_store.py b/python_app/storage/npz_store.py deleted file mode 100644 index d649ebe..0000000 --- a/python_app/storage/npz_store.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Facade exports for NPZ-based storage implementation.""" - -from python_app.storage.npz.paths import ( - collection_dir_name as _collection_dir_name, - format_float_for_key as _format_float_for_key, - radar_key_from_config, - sanitize_path_component as _sanitize_path_component, -) -from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, RESULT_MAGIC -from python_app.storage.npz.store import NpzStore - -__all__ = [ - "NpzStore", - "PREPROC_MAGIC", - "RAW_MAGIC", - "RESULT_MAGIC", - "_collection_dir_name", - "_format_float_for_key", - "_sanitize_path_component", - "radar_key_from_config", -] diff --git a/python_app/storage/store_api.py b/python_app/storage/store_api.py deleted file mode 100644 index 4fbd374..0000000 --- a/python_app/storage/store_api.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Abstract storage API for preprocess sets.""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from pathlib import Path - -from python_app.models.dataset_model import ComboKey, SweepCollection - - -class StoreApi(ABC): - """Storage contract used by workflows and GUI code.""" - - @abstractmethod - def save_set(self, kind: str, radar_key: str, set_name: str, collection: SweepCollection) -> None: - """Persist a named set.""" - raise NotImplementedError - - @abstractmethod - def load_set(self, kind: str, radar_key: str, set_name: str) -> SweepCollection: - """Load a named set.""" - raise NotImplementedError - - @abstractmethod - def list_sets(self, kind: str, radar_key: str) -> list[str]: - """List set names for kind/radar key.""" - raise NotImplementedError - - @abstractmethod - def has_combo_coverage(self, kind: str, radar_key: str, set_name: str, combos: list[ComboKey]) -> bool: - """Check whether stored set covers requested switch combinations.""" - raise NotImplementedError - - @abstractmethod - def export_set_bundle(self, kind: str, radar_key: str, set_name: str, output_path: Path) -> Path: - """Export set into binary bundle file for C++ preprocessing.""" - raise NotImplementedError diff --git a/python_app/tests/gui_test_support.py b/python_app/tests/gui_test_support.py index 2bfb5b5..b40cbad 100644 --- a/python_app/tests/gui_test_support.py +++ b/python_app/tests/gui_test_support.py @@ -1,12 +1,20 @@ """Construct desktop fixtures without touching a running application's resources.""" from pathlib import Path +from typing import TypeVar +from PyQt6.QtWidgets import QWidget +from python_app.gui.controllers.sections.views import SectionView + +_Widget = TypeVar("_Widget", bound=QWidget) import json import os import tempfile from unittest import mock +from PyQt6.QtCore import QCoreApplication, QEvent + from python_app.gui.app_window import AppWindow +from python_app.gui.controllers.web_controller import WebBridge def isolated_window(add_cleanup) -> AppWindow: @@ -24,6 +32,21 @@ def isolated_window(add_cleanup) -> AppWindow: with mock.patch.dict(os.environ, { "RADAR_SYSTEM_PROFILE": str(profile), "RADAR_SYSTEM_AUTO_START": "0", "RADAR_SYSTEM_AUTO_APPLY_RADAR": "0", - }), mock.patch.object(AppWindow, "_init_web_ui"): + }), mock.patch.object(WebBridge, "start_server"): window = AppWindow(root) + + def destroy_window() -> None: + window.close() + # Close stops services; Qt must destroy the widget tree before Python GC + # can collect the controller callbacks and their graphics objects. + window.deleteLater() + QCoreApplication.sendPostedEvents(window, QEvent.Type.DeferredDelete) + + add_cleanup(destroy_window) return window + + +def widget(view: SectionView, name: str, kind: type[_Widget]) -> _Widget: + value = view.fields[name] + assert isinstance(value, kind), name + return value diff --git a/python_app/tests/test_antenna_geometry.py b/python_app/tests/test_antenna_geometry.py index 9f98afa..6632ce7 100644 --- a/python_app/tests/test_antenna_geometry.py +++ b/python_app/tests/test_antenna_geometry.py @@ -8,17 +8,16 @@ from pathlib import Path import unittest from unittest import mock -import numpy as np os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") from PyQt6.QtCore import QSignalBlocker -from PyQt6.QtWidgets import QApplication +from PyQt6.QtWidgets import QApplication, QComboBox, QDoubleSpinBox from python_app.gui.antenna_geometry_editor import AntennaGeometryEditor -from python_app.tests.gui_test_support import isolated_window -from python_app.models.dataset_model import ResultCollection, ResultPayload -from python_app.models.gui_profile_model import GuiProfileModel -from python_app.models.run_config_model import GprTxGeometryModel, GprRxGeometryModel +from python_app.tests.gui_test_support import isolated_window, widget +from python_app.models.dataset_model import ResultCollection +from python_app.models.gui_profile_schema import GuiProfileModel +from python_app.models.run_config_schema import GprTxGeometryModel, GprRxGeometryModel class AntennaGeometryTest(unittest.TestCase): @@ -38,7 +37,9 @@ class AntennaGeometryTest(unittest.TestCase): changed = mock.Mock() editor.changed.connect(changed) with mock.patch("python_app.gui.antenna_geometry_editor.QMessageBox.warning") as warning: - editor._table.item(0, 1).setText("nan") + item = editor._table.item(0, 1) + assert item is not None + item.setText("nan") warning.assert_called_once() changed.assert_not_called() self.assertEqual(editor.records()[0]["x_m"], 1.0) @@ -52,13 +53,24 @@ class AntennaGeometryTest(unittest.TestCase): editor._remove_button.click() self.assertEqual(editor.records(), []) self.assertTrue(editor._add_button.isEnabled()) + changed = mock.Mock() + editor.changed.connect(changed) editor._add_button.click() - self.assertEqual(editor.records()[0]["input_pos"], 0) + self.assertEqual(editor.records(), []) + changed.assert_not_called() + item = editor._table.item(0, 1) + assert item is not None + item.setText("1.25") + self.assertEqual(editor.records()[0], {"input_pos": 0, "x_m": 1.25, "y_m": 0.0, "z_m": 0.0}) + changed.assert_called_once() - def test_web_geometry_rejects_duplicates_and_out_of_range_before_mutating(self) -> None: + def test_invalid_web_geometry_is_rejected_before_mutating(self) -> None: editor = AntennaGeometryEditor("output_pos", [], position_count=2) for records in ([{"output_pos": 2}], [{"output_pos": 0}, {"output_pos": 0}], - [{"output_pos": True}], [{"output_pos": 0, "x_m": True}]): + [{"output_pos": True}], [{"output_pos": 0.5}], [{"output_pos": "0"}], + [{}], [None], [{"output_pos": 0, "x_m": None}], + [{"output_pos": 0, "x_m": True}], [{"output_pos": 0, "y_m": float("nan")}], + [{"output_pos": 0, "z_m": float("inf")}], [{"output_pos": 0, "x_m": 10**400}]): with self.subTest(records=records), self.assertRaises(ValueError): editor.set_records(records) self.assertEqual(editor.records(), []) @@ -69,63 +81,75 @@ class WindowGeometryTest(unittest.TestCase): def setUpClass(cls) -> None: cls.app = QApplication.instance() or QApplication([]) cls.window = isolated_window(cls.addClassCleanup) - cls.window._timer.stop() - cls.window._shutdown_web_ui() - cls.initial_config = cls.window._build_config() - - @classmethod - def tearDownClass(cls) -> None: - cls.window.close() + cls.window._pipeline.timer.stop() + cls.initial_config = cls.window._profile.build_config() def setUp(self) -> None: self.w = self.window - self.w._apply_loaded_profile(GuiProfileModel(run_config=self.initial_config.clone()), Path("run_config.json")) + self.mode = self.w._view.processing.mode + self.gain = widget(self.w._view.processing, "_bscan_gain", QDoubleSpinBox) + self.w._profile.apply(GuiProfileModel(run_config=self.initial_config.clone()), Path("run_config.json")) def test_profile_and_live_config_preserve_all_coordinates(self) -> None: config = self.initial_config.clone() config.gpr.tx_geometry = [GprTxGeometryModel(output_pos=0, x_m=1.25, y_m=-2.0, z_m=3.0)] config.gpr.rx_geometry = [GprRxGeometryModel(input_pos=0, x_m=-1.25, y_m=4.0, z_m=-5.0)] - self.w._apply_loaded_profile(GuiProfileModel(run_config=config), Path("run_config.json")) - rebuilt = self.w._build_config() + self.w._profile.apply(GuiProfileModel(run_config=config), Path("run_config.json")) + rebuilt = self.w._profile.build_config() self.assertEqual(rebuilt.gpr, config.gpr) - live = self.w._live_processing_config().to_dict() + live = self.w._live.build().to_dict() self.assertEqual(live["tx_geometry"], [asdict(entry) for entry in config.gpr.tx_geometry]) self.assertEqual(live["rx_geometry"], [asdict(entry) for entry in config.gpr.rx_geometry]) - schema = self.w._web_settings_schema() + schema = self.w._config.web_schema() self.assertFalse(any(entry.get("applies_on_start") for entry in schema)) def test_malformed_web_batch_restores_every_changed_widget(self) -> None: - original_gain = self.w._bscan_gain.value() - original_mode = self.w._processing_mode.currentText() - with mock.patch.object(self.w, "_show_exception") as error, mock.patch.object(self.w._live_config_writer, "write") as write: - self.w._apply_web_live_settings({ + original_gain = self.gain.value() + original_mode = self.mode.currentText() + with mock.patch.object(self.w._log, "show_exception") as error, mock.patch.object(self.w._live.writer, "write") as write: + self.w._live.apply_web({ "processor_mode": "bscan", "bscan_gain": original_gain + 1, "tx_geometry": [{"output_pos": 0}, {"output_pos": 0}], }) error.assert_called_once() write.assert_not_called() - self.assertEqual(self.w._bscan_gain.value(), original_gain) - self.assertEqual(self.w._processing_mode.currentText(), original_mode) + self.assertEqual(self.gain.value(), original_gain) + self.assertEqual(self.mode.currentText(), original_mode) def test_stale_results_cannot_draw_over_new_geometry(self) -> None: revision = (1 << 63) + 123456789 - self.w._required_processing_revision = revision - with QSignalBlocker(self.w._processing_mode): - self.w._processing_mode.setCurrentText("gpr") - words = [(revision >> shift) & 0xffff for shift in range(0, 64, 16)] - fresh = ResultCollection(collection_id=1, monotonic_ns=1, collection_payloads=[ResultPayload( - processing_name="processing_revision", kind=4, table=np.asarray([words], dtype=np.float32), - )]) - with mock.patch.object(self.w, "_draw_gpr_map", return_value=True) as draw: - self.assertFalse(self.w._draw_results(ResultCollection(collection_id=1, monotonic_ns=1))) - self.assertTrue(self.w._draw_results(fresh)) - draw.assert_called_once_with(fresh) - self.assertFalse(self.w._result_collection_has_trace(fresh)) + self.w._plots.required_revision = revision + with QSignalBlocker(self.mode): + self.mode.setCurrentText("gpr") + fresh = ResultCollection(collection_id=1, monotonic_ns=1, settings_revision=revision) + with mock.patch.object(self.w._plots.gpr, "draw", return_value=True) as draw: + self.assertFalse(self.w._plots.draw(ResultCollection(collection_id=1, monotonic_ns=1))) + self.assertTrue(self.w._plots.draw(fresh)) + draw.assert_called_once() + self.assertIs(draw.call_args.args[0], fresh) + self.assertFalse(self.w._plots.has_output(fresh)) + + def test_desktop_and_web_reject_empty_active_geometry(self) -> None: + with QSignalBlocker(self.mode): + self.mode.setCurrentText("gpr") + editor = self.w._view.processing.geometries[0] + original = editor.records() + editor._table.selectAll() + with mock.patch("python_app.gui.antenna_geometry_editor.QMessageBox.warning") as warning, \ + mock.patch.object(self.w._live.writer, "write") as write: + editor._remove_rows() + warning.assert_called_once() + write.assert_not_called() + self.assertEqual(editor.records(), original) + with mock.patch.object(self.w._log, "show_exception") as error: + self.w._live.apply_web({"tx_geometry": []}) + error.assert_called_once() + self.assertEqual(editor.records(), original) def test_invalid_or_mixed_history_commands_do_not_clear_data(self) -> None: - with mock.patch.object(self.w, "_show_exception") as error, mock.patch.object(self.w, "_clear_all_runtime_history") as clear: - self.w._apply_web_live_settings({"history_command": "bogus"}) - self.w._apply_web_live_settings({"history_command": "clear_all", "bscan_gain": 2}) + with mock.patch.object(self.w._log, "show_exception") as error, mock.patch.object(self.w._snapshots, "delete_history") as clear: + self.w._live.apply_web({"history_command": "bogus"}) + self.w._live.apply_web({"history_command": "clear_all", "bscan_gain": 2}) self.assertEqual(error.call_count, 2) clear.assert_not_called() diff --git a/python_app/tests/test_bscan_history_window.py b/python_app/tests/test_bscan_history_window.py index 85a3696..72a5820 100644 --- a/python_app/tests/test_bscan_history_window.py +++ b/python_app/tests/test_bscan_history_window.py @@ -1,15 +1,4 @@ -"""Configurable B-scan display window. - -The B-scan used to be pinned to the C++ replay window (~50 sweeps) by two separate -mechanisms: the render-side history limit and a monotonically rising -`collection_id` floor. Widening only the first would have changed nothing, because -the floor kept filtering older collections out for good. - -The floor is gone: selection is positional, which is the only criterion that holds -when ids are sparse (the results ring drops) or restart from 1 (a new C++ run). -These tests pin the observable consequences — how many columns end up on screen — -rather than any internal counter. -""" +"""Stopped B-scan windows and exact result identities through real controllers.""" from __future__ import annotations @@ -19,11 +8,13 @@ import unittest import numpy as np os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") -from PyQt6.QtWidgets import QApplication # noqa: E402 +from PyQt6.QtWidgets import QApplication, QComboBox, QLineEdit, QSpinBox # noqa: E402 from python_app.gui.app_window import AppWindow # noqa: E402 -from python_app.tests.gui_test_support import isolated_window +from python_app.tests.gui_test_support import isolated_window, widget from collections import deque # noqa: E402 +from unittest import mock +from PyQt6.QtCore import QSignalBlocker from python_app.gui.runtime.history import record_result_history # noqa: E402 from python_app.models.dataset_model import ( # noqa: E402 @@ -31,6 +22,7 @@ from python_app.models.dataset_model import ( # noqa: E402 ResultBlock, ResultCollection, ResultPayload, + SweepCollection, ) _app: QApplication | None = None @@ -39,15 +31,11 @@ _window: AppWindow | None = None def setUpModule() -> None: global _app, _window - _app = QApplication.instance() or QApplication([]) + instance = QApplication.instance() + _app = instance if isinstance(instance, QApplication) else QApplication([]) _window = isolated_window(unittest.addModuleCleanup) -def tearDownModule() -> None: - if _window is not None: - _window.close() - - def _bscan_result(collection_id: int) -> ResultCollection: payload = ResultPayload( processing_name="bscan", @@ -56,117 +44,131 @@ def _bscan_result(collection_id: int) -> ResultCollection: trace=np.array([collection_id + 0j, collection_id + 0j], dtype=np.complex64), ) block = ResultBlock(combo=ComboKey(input=0, output=0), payloads=[payload]) - return ResultCollection(collection_id=collection_id, monotonic_ns=collection_id, blocks=[block]) + return ResultCollection(collection_id=collection_id, monotonic_ns=collection_id, + capture_start_ns=collection_id, capture_end_ns=collection_id + 1, blocks=[block]) class BscanDisplayWindowTest(unittest.TestCase): def setUp(self) -> None: + assert _window is not None self.w = _window - self._original_is_running = self.w._supervisor.is_running - self.w._result_history.clear() - self.w._bscan_render_signature = None + self.mode = self.w._view.processing.mode + self.history_window = widget(self.w._view.processing, "_bscan_history_window", QSpinBox) + with QSignalBlocker(self.mode): + self.mode.setCurrentText("bscan") + self.w._plots.set_mode("bscan") + self._original_is_running = self.w._pipeline.supervisor.is_running + self.w._history.results.clear() + self.w._plots.bscan.signature = None def tearDown(self) -> None: - self.w._supervisor.is_running = self._original_is_running - self.w._result_history.clear() - self.w._bscan_render_signature = None + self.w._pipeline.supervisor.is_running = self._original_is_running + self.w._history.results.clear() + self.w._plots.bscan.signature = None def _set_running(self, running: bool) -> None: - self.w._supervisor.is_running = lambda: running + self.w._pipeline.supervisor.is_running = lambda: running def _fill_history(self, count: int, *, id_step: int = 1) -> None: for index in range(count): - self.w._result_history.append(_bscan_result(1 + index * id_step)) + self.w._history.results.append(_bscan_result(1 + index * id_step)) def test_running_acquisition_ignores_the_user_window(self) -> None: - # A live rebuild runs on every incoming result, so the live path stays pinned - # to the replay window no matter what the operator typed for stopped review. self._set_running(True) - self.w._bscan_history_window.setValue(300) - self.assertEqual(self.w._bscan_display_window_scans(), self.w._bscan_cpp_replay_window) + self.history_window.setValue(300) + self.assertEqual(self.w._plots.window_scans(), self.w._plots.live_window) def test_stopped_acquisition_uses_the_user_window(self) -> None: self._set_running(False) - self.w._bscan_history_window.setValue(300) - self.assertEqual(self.w._bscan_display_window_scans(), 300) + self.history_window.setValue(300) + self.assertEqual(self.w._plots.window_scans(), 300) def test_widening_the_window_brings_older_frames_back(self) -> None: - # The regression this whole change exists for: a live run renders 50 columns, - # and widening the window after Stop must reach back over the retained history - # rather than stay pinned to whatever the live path last drew. self._fill_history(300) self._set_running(True) - self.w._sync_bscan_history_from_results() + self.w._plots.sync_bscan() self.assertEqual( - len(self.w._bscan_history_by_combo[(0, 0)]), self.w._bscan_cpp_replay_window + len(self.w._plots.bscan.history_by_combo[(0, 0)]), self.w._plots.live_window ) self._set_running(False) - self.w._bscan_history_window.setValue(300) - self.w._sync_bscan_history_from_results() - self.assertEqual(len(self.w._bscan_history_by_combo[(0, 0)]), 300) + self.history_window.setValue(300) + self.w._plots.sync_bscan() + self.assertEqual(len(self.w._plots.bscan.history_by_combo[(0, 0)]), 300) def test_restarted_collection_ids_do_not_hide_the_fresh_run(self) -> None: - # Regression: Start after a stopped review made the image melt to 0 columns and - # then snap back to 50. A new C++ run numbers from 1, so entries that are newest - # by position carry the smallest ids; any id-based cut-off derived from the old - # run rejected exactly them. self._fill_history(300) self._set_running(False) - self.w._bscan_history_window.setValue(300) - self.w._sync_bscan_history_from_results() + self.history_window.setValue(300) + self.w._plots.sync_bscan() self._set_running(True) - window = self.w._bscan_cpp_replay_window + window = self.w._plots.live_window for fresh in range(1, window + 1): - self.w._result_history.append(_bscan_result(fresh)) - self.w._sync_bscan_history_from_results() - self.assertEqual(len(self.w._bscan_history_by_combo[(0, 0)]), window) + self.w._history.results.append(_bscan_result(fresh)) + self.w._plots.sync_bscan() + self.assertEqual(len(self.w._plots.bscan.history_by_combo[(0, 0)]), window) def test_window_counts_entries_not_collection_ids(self) -> None: - # The results ring overwrites unread slots when the producer outruns the GUI - # poll loop, so retained collection ids are sparse. Counting in ids rather than - # in entries is exactly the case where asking for 150 sweeps rendered only 87. self._fill_history(300, id_step=3) self._set_running(False) - self.w._bscan_history_window.setValue(150) - self.w._sync_bscan_history_from_results() - self.assertEqual(len(self.w._bscan_history_by_combo[(0, 0)]), 150) + self.history_window.setValue(150) + self.w._plots.sync_bscan() + self.assertEqual(len(self.w._plots.bscan.history_by_combo[(0, 0)]), 150) def test_reprocessed_results_replace_rather_than_duplicate_columns(self) -> None: - # The processor's recompute is triggered by feeding sweeps back through it, and - # it republishes them under their ORIGINAL ids. Two independent mechanisms keep - # that from doubling every column, and neither may be confused with the removed - # `collection_id` floor: a threshold cannot tell a duplicate from its original, - # since they share the id. Deduplication is by key equality. self._set_running(False) - self.w._bscan_history_window.setValue(300) + self.history_window.setValue(300) self._fill_history(200) - self.w._sync_bscan_history_from_results() - self.assertEqual(len(self.w._bscan_history_by_combo[(0, 0)]), 200) + self.w._plots.sync_bscan() + self.assertEqual(len(self.w._plots.bscan.history_by_combo[(0, 0)]), 200) - # Intake: a recomputed collection replaces the entry holding the same key. - for collection in list(self.w._result_history): - record_result_history(self.w._result_history, _bscan_result(collection.collection_id)) - self.assertEqual(len(self.w._result_history), 200) + for collection in list(self.w._history.results): + record_result_history(self.w._history.results, _bscan_result(collection.collection_id)) + self.assertEqual(len(self.w._history.results), 200) - # Render: a duplicate that reached the deque by another path is still collapsed, - # keeping the newer of the two. - self.w._result_history.append(_bscan_result(200)) - self.w._bscan_render_signature = None - self.w._sync_bscan_history_from_results() - self.assertEqual(len(self.w._bscan_history_by_combo[(0, 0)]), 200) + self.w._history.results.append(_bscan_result(200)) + self.w._plots.bscan.signature = None + self.w._plots.sync_bscan() + self.assertEqual(len(self.w._plots.bscan.history_by_combo[(0, 0)]), 200) def test_rebuild_renders_the_full_widened_window(self) -> None: self._fill_history(300) self._set_running(False) - self.w._bscan_history_window.setValue(300) - self.w._sync_bscan_history_from_results() - self.assertEqual(len(self.w._bscan_history_by_combo[(0, 0)]), 300) + self.history_window.setValue(300) + self.w._plots.sync_bscan() + self.assertEqual(len(self.w._plots.bscan.history_by_combo[(0, 0)]), 300) - self.w._bscan_history_window.setValue(50) - self.w._sync_bscan_history_from_results() - self.assertEqual(len(self.w._bscan_history_by_combo[(0, 0)]), 50) + self.history_window.setValue(50) + self.w._plots.sync_bscan() + self.assertEqual(len(self.w._plots.bscan.history_by_combo[(0, 0)]), 50) + + def test_replay_requires_stopped_acquisition_and_live_processor(self) -> None: + self.w._history.preprocessed.append(SweepCollection(collection_id=1, monotonic_ns=1, traces=[])) + self.addCleanup(self.w._history.preprocessed.clear) + with mock.patch.object(self.w._pipeline, "_result_reader", mock.Mock()), \ + mock.patch.object(self.w._pipeline.supervisor, "is_processor_running", return_value=True): + with QSignalBlocker(self.mode): + self.mode.setCurrentText("bscan") + self._set_running(False) + self.assertTrue(self.w._pipeline.can_reprocess_history()) + self.assertFalse(self.w._live.build(reprocess_current_result=False).reprocess_current_result) + self._set_running(True) + self.assertFalse(self.w._pipeline.can_reprocess_history()) + self._set_running(False) + with mock.patch.object(self.w._pipeline.supervisor, "is_processor_running", return_value=False): + self.assertFalse(self.w._pipeline.can_reprocess_history()) + with QSignalBlocker(self.mode): + self.mode.setCurrentText("pass_through") + self.w._plots.set_mode("pass_through") + self.assertFalse(self.w._pipeline.can_reprocess_history()) + self.assertFalse(self.w._pipeline.can_reprocess_history()) + + def test_acquisition_combo_editor_does_not_reprocess_display(self) -> None: + with mock.patch.object(self.w._live.writer, "write") as write: + self.w._view.switches.single_output.setText("1") + write.assert_not_called() + self.assertFalse(self.w._pipeline.replay.timer.isActive()) class ResultHistoryOrderTest(unittest.TestCase): @@ -174,55 +176,33 @@ class ResultHistoryOrderTest(unittest.TestCase): The processor reprocessing its retained sweeps also delivers sweeps whose first result the GUI missed (the results ring overwrites unread slots). Appended, such a - sweep showed up as the newest B-scan column, and was recorded to disk once more. + sweep showed up as the newest B-scan column. Its first arrival must still be + recorded to disk; replacing an already retained result must not duplicate it. """ @staticmethod def _ids(history: deque[ResultCollection]) -> list[int]: return [int(collection.collection_id) for collection in history] - def test_a_new_sweep_is_appended(self) -> None: - history = deque(_bscan_result(cid) for cid in (1, 2)) - self.assertTrue(record_result_history(history, _bscan_result(3))) - self.assertEqual(self._ids(history), [1, 2, 3]) - def test_a_late_result_takes_its_place_in_acquisition_order(self) -> None: - history = deque(_bscan_result(cid) for cid in (1, 2, 4, 5)) - self.assertFalse(record_result_history(history, _bscan_result(3))) - self.assertEqual(self._ids(history), [1, 2, 3, 4, 5]) - def test_a_reprocessed_result_replaces_its_entry(self) -> None: - history = deque(_bscan_result(cid) for cid in (1, 2, 3)) - reprocessed = _bscan_result(2) - self.assertFalse(record_result_history(history, reprocessed)) - self.assertEqual(self._ids(history), [1, 2, 3]) - self.assertIs(history[1], reprocessed) - def test_a_full_history_drops_its_oldest_entry(self) -> None: - history = deque((_bscan_result(cid) for cid in (1, 2, 4, 5)), maxlen=4) - record_result_history(history, _bscan_result(3)) - self.assertEqual(self._ids(history), [2, 3, 4, 5]) - record_result_history(history, _bscan_result(1)) # older than all it keeps - self.assertEqual(self._ids(history), [2, 3, 4, 5]) - record_result_history(history, _bscan_result(6)) - self.assertEqual(self._ids(history), [3, 4, 5, 6]) def test_only_new_sweeps_are_recorded_to_disk(self) -> None: window = _window + assert window is not None arrivals = [_bscan_result(cid) for cid in (2, 3, 2, 1)] # 2 reprocessed, 1 late - reader = type("Reader", (), {"pop_result_collection": lambda _self: arrivals.pop(0) if arrivals else None})() + reader = mock.Mock(pop_result_collection=mock.Mock(side_effect=lambda: arrivals.pop(0) if arrivals else None)) recorded: list[int] = [] - window._result_reader = reader - self.addCleanup(setattr, window, "_result_reader", None) - window._record_collection = lambda stage, collection: recorded.append(int(collection.collection_id)) - self.addCleanup(delattr, window, "_record_collection") - window._result_history.clear() - self.addCleanup(window._result_history.clear) + window._history.results.clear() + self.addCleanup(window._history.results.clear) - window._read_all_results() + with mock.patch.object(window._pipeline, "_result_reader", reader), \ + mock.patch.object(window._recording, "record", side_effect=lambda stage, collection: recorded.append(collection.collection_id)): + window._pipeline.read_results() - self.assertEqual(recorded, [2, 3]) - self.assertEqual(self._ids(window._result_history), [1, 2, 3]) + self.assertEqual(recorded, [2, 3, 1]) + self.assertEqual(self._ids(window._history.results), [1, 2, 3]) if __name__ == "__main__": diff --git a/python_app/tests/test_bscan_replay.py b/python_app/tests/test_bscan_replay.py index aaf006d..9605aaa 100644 --- a/python_app/tests/test_bscan_replay.py +++ b/python_app/tests/test_bscan_replay.py @@ -1,366 +1,246 @@ -"""Re-feeding retained sweeps to the processor to recompute the whole B-scan. - -The processor keeps only ~50 preprocessed sweeps of its own, so a live settings edit -used to refresh just the newest 50 columns. The GUI holds up to 1000 of them and hands -them back through the processor's input ring, which only works if the bytes the GUI -writes are exactly the ones the C++ side expects — that wire-format contract is what -the first test pins, without needing the pipeline running. - -The guard tests pin the other half: the GUI must never write into that ring while -`data_preprocessor` owns it. -""" +"""Full-history replay uses exact acquisition keys and owns its ring lifecycle.""" from __future__ import annotations -from contextlib import suppress from dataclasses import replace -import json import os +import struct from pathlib import Path import unittest +from unittest.mock import Mock, patch import numpy as np os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") -from PyQt6.QtCore import QSignalBlocker # noqa: E402 -from PyQt6.QtWidgets import QApplication # noqa: E402 +from PyQt6.QtWidgets import QApplication -from python_app.gui.app_window import AppWindow # noqa: E402 -from python_app.tests.gui_test_support import isolated_window -from python_app.gui.runtime.history import collection_key # noqa: E402 -from python_app.models.dataset_model import ( # noqa: E402 - ComboKey, - ResultBlock, - ResultCollection, - ResultPayload, - SweepCollection, - TraceData, -) -from python_app.orchestration.shm import ShmRingWriter # noqa: E402 -from python_app.orchestration.shm.ring_reader import ShmRingReader # noqa: E402 -from python_app.storage.npz.serialize import PREPROC_MAGIC, serialize_trace_collection # noqa: E402 - -# Mirrors kPreprocessedCollectionMagic in common_cpp/ipc/src/shared_types.cpp. If this -# ever drifts, the processor silently rejects everything the GUI re-feeds. -_CPP_PREPROCESSED_MAGIC = 0x32525050 +from python_app.gui.controllers.bscan_replay import BscanReplay +from python_app.gui.controllers.history_controller import HistoryController +from python_app.models.collection_identity import collection_key +from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection, TraceData +from python_app.models.run_config_schema import RingEndpointModel +from python_app.orchestration.shm.decoder import PREPROC_MAGIC, RESULT_MAGIC, decode_trace_collection +from python_app.orchestration.shm.ring_reader import ShmRingReader +from python_app.orchestration.shm.ring_writer import ShmRingWriter +from python_app.storage.npz.serialize import serialize_trace_collection _app: QApplication | None = None -_window: AppWindow | None = None def setUpModule() -> None: - global _app, _window - _app = QApplication.instance() or QApplication([]) - _window = isolated_window(unittest.addModuleCleanup) - - -def tearDownModule() -> None: - if _window is not None: - _window.close() + global _app + instance = QApplication.instance() + _app = instance if isinstance(instance, QApplication) else QApplication([]) def _sweep(collection_id: int, *, combos: int = 3, points: int = 64) -> SweepCollection: - traces = [ - TraceData( - combo=ComboKey(input=index, output=index + 1), - frequency_hz=np.linspace(1e9, 8e9, points, dtype=np.float32), - s11=(np.arange(points) + index).astype(np.complex64) + 0.5j, - s21=(np.arange(points) * 2 + index).astype(np.complex64) - 0.25j, - ) - for index in range(combos) - ] - return SweepCollection( - collection_id=collection_id, - monotonic_ns=collection_id * 1_000_000 + 7, - traces=traces, - capture_start_ns=collection_id * 10, - capture_end_ns=collection_id * 10 + 5, - ) - - -def _bscan_result(collection_id: int, *, points: int = 32) -> ResultCollection: - payload = ResultPayload( - processing_name="bscan", - kind=1, - frequency_hz=np.linspace(0.0, 1.0, points, dtype=np.float32), - trace=(np.arange(points) + collection_id).astype(np.complex64), - ) - return ResultCollection( - collection_id=collection_id, - monotonic_ns=collection_id, - blocks=[ResultBlock(combo=ComboKey(input=0, output=0), payloads=[payload])], - ) - - -class PreprocessedWireFormatTest(unittest.TestCase): - """The bytes the GUI re-feeds must be the ones the C++ processor decodes.""" - - def setUp(self) -> None: - self.ring_name = f"/radar_test_{self._testMethodName}" - with suppress(OSError): - (Path("/dev/shm") / self.ring_name[1:]).unlink() - self.writer = ShmRingWriter(self.ring_name, 8, 1 << 20) - self.reader = ShmRingReader(self.ring_name) - self.addCleanup(self._cleanup) - - def _cleanup(self) -> None: - with suppress(Exception): - self.reader.close() - with suppress(Exception): - self.writer.close() - with suppress(OSError): - (Path("/dev/shm") / self.ring_name[1:]).unlink() - - def _assert_same(self, original: SweepCollection, decoded: SweepCollection) -> None: - self.assertEqual(decoded.collection_id, original.collection_id) - self.assertEqual(decoded.monotonic_ns, original.monotonic_ns) - self.assertEqual(decoded.capture_start_ns, original.capture_start_ns) - self.assertEqual(decoded.capture_end_ns, original.capture_end_ns) - self.assertEqual(len(decoded.traces), len(original.traces)) - for left, right in zip(original.traces, decoded.traces, strict=True): - self.assertEqual((left.combo.input, left.combo.output), (right.combo.input, right.combo.output)) - np.testing.assert_array_equal(left.frequency_hz, right.frequency_hz) - np.testing.assert_array_equal(left.s11, right.s11) - np.testing.assert_array_equal(left.s21, right.s21) - - def test_magic_matches_the_cpp_decoder(self) -> None: - payload = serialize_trace_collection(_sweep(1), PREPROC_MAGIC) - self.assertEqual(int.from_bytes(payload[:4], "little"), _CPP_PREPROCESSED_MAGIC) - - def test_round_trip_preserves_every_field(self) -> None: - original = _sweep(42) - self.assertTrue(self.writer.push(serialize_trace_collection(original, PREPROC_MAGIC))) - - decoded = self.reader.pop_preprocessed_collection() - self.assertIsNotNone(decoded) - assert decoded is not None - self._assert_same(original, decoded) - - def test_batch_keeps_order(self) -> None: - originals = [_sweep(cid) for cid in range(100, 105)] - for collection in originals: - self.assertTrue(self.writer.push(serialize_trace_collection(collection, PREPROC_MAGIC))) - - decoded = [] - while (collection := self.reader.pop_preprocessed_collection()) is not None: - decoded.append(collection) - - self.assertEqual([c.collection_id for c in decoded], [c.collection_id for c in originals]) - for left, right in zip(originals, decoded, strict=True): - self._assert_same(left, right) - - def test_overflow_drops_oldest(self) -> None: - # Why the replay pushes in chunks instead of one burst: an undrained ring - # silently overwrites, which would punch holes into the very image we are - # trying to make coherent. - capacity = 8 - for cid in range(200, 200 + capacity + 4): - self.writer.push(serialize_trace_collection(_sweep(cid), PREPROC_MAGIC)) - - survived = [] - while (collection := self.reader.pop_preprocessed_collection()) is not None: - survived.append(collection.collection_id) - - self.assertEqual(len(survived), capacity) - self.assertEqual(survived[-1], 200 + capacity + 3) + traces = [TraceData( + combo=ComboKey(input=index, output=index + 1), + frequency_hz=np.linspace(1e9, 8e9, points, dtype=np.float32), + s21=(np.arange(points) * 2 + index).astype(np.complex64) - 0.25j, + capture_start_ns=collection_id * 10 + index, + capture_end_ns=collection_id * 10 + index + 1, + ) for index in range(combos)] + return SweepCollection(collection_id, collection_id * 1_000_000 + 7, traces, + capture_start_ns=collection_id * 10, capture_end_ns=collection_id * 10 + 5) def _result_of(sweep: SweepCollection) -> ResultCollection: - """The processor's result of a sweep: it keeps the sweep's id and timestamp.""" - return replace(_bscan_result(sweep.collection_id), monotonic_ns=sweep.monotonic_ns) + return ResultCollection(sweep.collection_id, sweep.monotonic_ns + 100, + capture_start_ns=sweep.capture_start_ns, capture_end_ns=sweep.capture_end_ns, settings_revision=7) -class _FakeProcessor: - """Both ends of the processor during a feed-back. - Each pushed sweep comes back as a result, after whatever the results ring already - held when the feed-back started. - """ - slot_size_bytes = 1 << 20 +class _ProcessorReader(ShmRingReader): + """Boundary stand-in: decode pushed wire bytes and queue processor results.""" - def __init__(self, sweeps: list[SweepCollection], already_published: list[ResultCollection]) -> None: - self._unprocessed = list(sweeps) - self._published = list(already_published) + def __init__(self) -> None: + self.published: list[ResultCollection] = [] + self.received: list[SweepCollection] = [] + self.wait_once = False - def push(self, _payload: bytes) -> bool: - self._published.append(_result_of(self._unprocessed.pop(0))) + @property + def capacity(self) -> int: + return 50 + + def push(self, payload: bytes) -> bool: + sweep = decode_trace_collection(payload, PREPROC_MAGIC) + self.received.append(sweep) + self.published.append(_result_of(sweep)) return True def pop_result_collection(self) -> ResultCollection | None: - return self._published.pop(0) if self._published else None - - def close(self) -> None: - return None + if self.wait_once: + self.wait_once = False + return None + return self.published.pop(0) if self.published else None -class _IdleResultReader: - """Stands in for a connected results reader that simply has nothing to hand out.""" - - def pop_result_collection(self): - return None - - def close(self) -> None: - return None - - -class ReplayGuardTest(unittest.TestCase): - """The GUI must refuse to write into a ring `data_preprocessor` still owns.""" - +class ReplayTest(unittest.TestCase): def setUp(self) -> None: - self.w = _window - # The 50 ms ring poll and the debounce would both run against this half-faked - # window while the test drives it by hand; park them for the duration. - self.w._timer.stop() - self.w._bscan_reprocess_timer.stop() - self.addCleanup(self.w._timer.start) - self.addCleanup(self.w._bscan_reprocess_timer.stop) - - self._original_is_running = self.w._supervisor.is_running - self._original_is_processor_running = self.w._supervisor.is_processor_running - self._original_mode = self.w._processing_mode.currentText() - # Changing the mode fires the live-settings handler, which would kick off the - # very replay these tests are inspecting. - with QSignalBlocker(self.w._processing_mode): - self.w._processing_mode.setCurrentText("bscan") - self.w._supervisor.is_running = lambda: False - self.w._supervisor.is_processor_running = lambda: True - self.w._result_reader = _IdleResultReader() - - def tearDown(self) -> None: - self.w._supervisor.is_running = self._original_is_running - self.w._supervisor.is_processor_running = self._original_is_processor_running - with QSignalBlocker(self.w._processing_mode): - self.w._processing_mode.setCurrentText(self._original_mode) - self.w._result_reader = None - - def test_refuses_while_acquisition_runs(self) -> None: - self.w._supervisor.is_running = lambda: True - self.assertFalse(self.w._can_reprocess_history()) - - def test_refuses_when_processor_is_down(self) -> None: - self.w._supervisor.is_processor_running = lambda: False - self.assertFalse(self.w._can_reprocess_history()) - - def test_refuses_outside_bscan_mode(self) -> None: - self.w._processing_mode.setCurrentText("pass_through") - self.assertFalse(self.w._can_reprocess_history()) - - def test_refuses_without_a_results_reader(self) -> None: - self.w._result_reader = None - self.assertFalse(self.w._can_reprocess_history()) - - def test_allows_when_stopped_with_a_live_processor(self) -> None: - self.assertTrue(self.w._can_reprocess_history()) - - def test_replayed_results_are_rendered_instead_of_runtime_history(self) -> None: - """Regression: 300 re-sent, 300 recovered, only 145 drawn. - - `_pre_history` and `_result_history` come from two rings that drop - independently, so the re-sent sweeps only partly overlap the recorded results. - The non-overlapping ones get appended to the deque under old ids, leaving its - newest `window` entries a mix of freshly and stale-processed frames. Rendering - from the replayed set instead sidesteps the whole problem. - """ - window = 30 - replayed = [_bscan_result(cid) for cid in range(9000, 9000 + window)] - - self.w._result_history.clear() - # Stand-in for a runtime history whose ids barely overlap the replayed ones. - self.w._result_history.extend(_bscan_result(cid) for cid in range(1, 200)) - self.addCleanup(self.w._result_history.clear) - - self.w._bscan_replay_results = replayed - self.addCleanup(self.w._discard_bscan_replay_results) - self.w._bscan_history_window.setValue(window) - self.w._bscan_render_signature = None - self.w._sync_bscan_history_from_results() - - self.assertEqual(len(self.w._bscan_history_by_combo[(0, 0)]), window) - - def test_feed_back_renders_its_sweeps_in_acquisition_order(self) -> None: - """Regression: after switching the displayed combo the columns came out rotated. - - The processor was still replaying its own sweeps when the GUI fed its history - back, and the GUI took the first results to arrive for its own, in arrival order. - """ - sweeps = [_sweep(cid) for cid in range(1, 11)] - self.w._pre_history.clear() - self.w._pre_history.extend(sweeps) - self.addCleanup(self.w._pre_history.clear) - self.w._result_history.clear() - self.addCleanup(self.w._result_history.clear) - with QSignalBlocker(self.w._bscan_history_window): - self.w._bscan_history_window.setValue(len(sweeps)) - # Already in the results ring: the tail of the processor's own replay and a - # result of a sweep that is not being fed back. - processor = _FakeProcessor(sweeps, [_result_of(sweep) for sweep in sweeps[6:]] + [_bscan_result(999)]) - self.w._replay_ring_writer = processor - self.addCleanup(setattr, self.w, "_replay_ring_writer", None) - self.w._result_reader = processor - self.addCleanup(self.w._discard_bscan_replay_results) - - self.assertTrue(self.w._reprocess_history_through_processor()) - - self.assertEqual( - [collection_key(collection) for collection in self.w._bscan_replay_results], - [collection_key(sweep) for sweep in sweeps], + self.history = HistoryController() + self.warnings: list[str] = [] + self.replay = BscanReplay(self.history, lambda: None, self.warning, lambda _text: None) + self.addCleanup(self.replay.close) + self.ring = RingEndpointModel("/unused_test_replay", 50, 1 << 20) + self.processor = _ProcessorReader() + self.writer = Mock(spec=ShmRingWriter, push=self.processor.push, slot_size_bytes=1 << 20) + writer_patch = patch( + "python_app.gui.controllers.bscan_replay.ShmRingWriter", return_value=self.writer, ) - self.assertIn(999, [collection.collection_id for collection in self.w._result_history]) + self.writer_factory = writer_patch.start() + self.addCleanup(writer_patch.stop) - def test_changing_the_displayed_combo_recomputes_nothing(self) -> None: - # Every combo's B-scan is cached already; recomputing raced the processor (above). - writes: list[dict] = [] - self.w._write_live_processing_config = lambda **kwargs: writes.append(kwargs) - self.addCleanup(delattr, self.w, "_write_live_processing_config") + def warning(self, text: str, *, details: str | None = None, once_key: str | None = None) -> None: + self.warnings.append(text) - self.w._single_combo_output.editingFinished.emit() + def run_replay(self) -> bool: + return self.replay.run(window=300, revision=7, ring=self.ring, reader=self.processor, + continue_replay=lambda: True, pump_events=lambda _pause: None) - self.assertEqual(writes, []) - self.assertFalse(self.w._bscan_reprocess_timer.isActive()) + def test_result_ring_capacity_bounds_each_replay_burst(self) -> None: + name = f"/radar_replay_small_result_{os.getpid()}" + self.addCleanup((Path("/dev/shm") / name[1:]).unlink, missing_ok=True) + results = ShmRingWriter(name, 2, 256) + self.addCleanup(results.close) + reader = ShmRingReader(name) + self.addCleanup(reader.close) - def test_processor_reprocesses_only_while_the_gui_cannot(self) -> None: - # Its own replay would duplicate the feed-back and race it for the results ring. - def written_flag() -> bool: - self.w._write_live_processing_config() - config = json.loads(self.w._live_config_writer.path.read_text(encoding="utf-8")) - return bool(config["reprocess_current_result"]) + def process(payload: bytes) -> bool: + sweep = decode_trace_collection(payload, PREPROC_MAGIC) + return results.push(struct.pack(" None: - self.w._bscan_replay_results = [_bscan_result(1)] - self.w._discard_bscan_replay_results() - self.assertEqual(self.w._bscan_replay_results, []) - self.assertIsNone(self.w._bscan_render_signature) + def test_replays_full_history_in_capture_order_despite_independent_ring_drops(self) -> None: + sweeps = [_sweep(cid) for cid in range(1, 301)] + self.history.preprocessed.extend(sweeps) + self.history.results.extend(_result_of(_sweep(cid)) for cid in range(300, 450)) + # Old processor replay arrives before the GUI's own results, including a + # restarted collection id whose capture key differs from the requested one. + unrelated = _result_of(replace(_sweep(1), capture_start_ns=9000, capture_end_ns=9005)) + stale_tail = replace(_result_of(sweeps[-1]), settings_revision=6) + self.processor.published = [stale_tail, *[_result_of(sweep) for sweep in sweeps[6:10]], unrelated] + self.assertTrue(self.run_replay()) + source = self.history.bscan_source() + self.assertEqual([collection_key(item) for item in source], [collection_key(item) for item in sweeps]) + self.assertEqual(len(self.processor.received), 300) + self.assertTrue(all(item.settings_revision == 7 for item in source)) + self.assertIsNot(source[-1], stale_tail) + self.assertIn(unrelated, self.history.results) + self.writer_factory.assert_called_once_with(self.ring.name, self.ring.capacity, self.ring.slot_size_bytes) - def test_replay_refuses_to_re_enter_itself(self) -> None: - # Pumping the event loop mid-replay can fire the debounce again; a nested burst - # would share the ring and corrupt the result accounting. - self.w._pre_history.clear() - self.w._pre_history.extend(_sweep(cid) for cid in range(1, 6)) - self.addCleanup(self.w._pre_history.clear) + def test_refuses_without_reader_or_active_ring_geometry(self) -> None: + self.history.preprocessed.append(_sweep(1)) + self.assertFalse(self.replay.run(window=1, revision=7, ring=self.ring, reader=None, + continue_replay=lambda: True, pump_events=lambda _pause: None)) + self.assertFalse(self.replay.run(window=1, revision=7, ring=None, reader=self.processor, + continue_replay=lambda: True, pump_events=lambda _pause: None)) + self.writer_factory.assert_not_called() + self.assertIn("geometry is unknown", self.warnings[-1]) - self.w._bscan_replay_active = True - self.addCleanup(setattr, self.w, "_bscan_replay_active", False) + def test_nested_replay_is_refused(self) -> None: + self.history.preprocessed.append(_sweep(1)) + self.processor.wait_once = True + nested: list[bool] = [] - self.assertFalse(self.w._reprocess_history_through_processor()) + def pump(_pause: float) -> None: + nested.append(self.run_replay()) - def test_replay_refuses_without_an_active_run_config(self) -> None: - # A GUI restarted against a still-running processor does not know the ring - # geometry, and ShmRingWriter would recreate the segment underneath it. - self.w._pre_history.clear() - self.w._pre_history.extend(_sweep(cid) for cid in range(1, 6)) - self.addCleanup(self.w._pre_history.clear) + self.assertTrue(self.replay.run(window=1, revision=7, ring=self.ring, reader=self.processor, + continue_replay=lambda: True, pump_events=pump)) + self.assertEqual(nested, [False]) + self.assertFalse(self.replay.active) - previous = getattr(self.w, "_active_run_config", None) - self.w._active_run_config = None - self.addCleanup(setattr, self.w, "_active_run_config", previous) + def test_close_discard_or_settings_edit_during_pump_cancels_current_replay(self) -> None: + self.history.preprocessed.extend(_sweep(cid) for cid in range(1, 25)) + for action in ("close", "discard", "schedule"): + with self.subTest(action=action): + self.processor.published.clear() + self.processor.received.clear() + self.processor.wait_once = True - self.assertIsNone(self.w._ensure_replay_ring_writer()) - self.assertFalse(self.w._reprocess_history_through_processor()) + def pump(_pause: float) -> None: + if action == "schedule": + self.replay.schedule(True) + elif action == "close": + self.replay.close() + else: + self.replay.discard() + self.assertFalse(self.run_replay()) + + self.assertFalse(self.replay.run( + window=24, revision=7, ring=self.ring, reader=self.processor, + continue_replay=lambda: True, pump_events=pump, + )) + self.assertLess(len(self.processor.received), 24) + self.assertEqual(list(self.history.results), []) + self.assertEqual(self.history.replay_results, []) + self.assertFalse(self.replay.active) + self.assertEqual(self.replay.timer.isActive(), action == "schedule") + + def test_runtime_guard_is_rechecked_before_io_and_after_event_pumping(self) -> None: + self.history.preprocessed.extend(_sweep(cid) for cid in range(1, 25)) + for stop_at in ("push", "pop", "pump"): + with self.subTest(stop_at=stop_at): + self.processor.published.clear() + self.processor.received.clear() + self.processor.wait_once = stop_at == "pump" + running = True + + def guard() -> bool: + count = len(self.processor.received) + return running and not (stop_at == "push" and count >= 1 + or stop_at == "pop" and count >= 12) + + def pump(_pause: float) -> None: + nonlocal running + running = False + + self.assertFalse(self.replay.run( + window=24, revision=7, ring=self.ring, reader=self.processor, + continue_replay=guard, pump_events=pump, + )) + self.assertEqual(len(self.processor.received), 1 if stop_at == "push" else 12) + self.assertEqual(list(self.history.results), []) + self.assertEqual(self.history.replay_results, []) + self.assertFalse(self.replay.active) + + def test_small_ring_is_drained_before_another_burst(self) -> None: + self.ring.capacity = 2 + self.history.preprocessed.extend(_sweep(cid) for cid in range(1, 16)) + + def push(payload: bytes) -> bool: + self.assertLess(len(self.processor.published), self.ring.capacity) + return self.processor.push(payload) + + self.writer.push = push + self.assertTrue(self.run_replay()) + self.assertEqual(len(self.history.replay_results), 15) + + def test_discard_and_close_release_replay_source_timer_and_writer(self) -> None: + self.history.preprocessed.append(_sweep(1)) + self.assertTrue(self.run_replay()) + self.replay.discard() + self.assertEqual(self.history.bscan_source(), list(self.history.results)) + self.replay.schedule(False) + self.assertFalse(self.replay.timer.isActive()) + self.replay.schedule(True) + self.assertTrue(self.replay.timer.isActive()) + self.replay.close() + self.replay.close() + self.assertFalse(self.replay.timer.isActive()) + self.writer.close.assert_called_once() if __name__ == "__main__": diff --git a/python_app/tests/test_collection_identity.py b/python_app/tests/test_collection_identity.py new file mode 100644 index 0000000..f3a76a9 --- /dev/null +++ b/python_app/tests/test_collection_identity.py @@ -0,0 +1,31 @@ +"""Exact acquisition identity survives stage timestamps and reused IDs.""" + +import unittest + +from python_app.models.collection_identity import collection_key, history_collection_removed, index_collections +from python_app.models.dataset_model import ResultCollection, SweepCollection + + +class CollectionIdentityTest(unittest.TestCase): + def test_stage_lookup_uses_only_id_and_capture_start(self) -> None: + old = SweepCollection(1, 500, capture_start_ns=10, capture_end_ns=20) + fresh = SweepCollection(1, 100, capture_start_ns=30, capture_end_ns=40) + index = index_collections([old, fresh]) + result = ResultCollection(1, 900, capture_start_ns=30, capture_end_ns=40) + self.assertIs(index.get(collection_key(result)), fresh) + self.assertNotIn((1, 31), index) + self.assertNotIn((2, 30), index) + + def test_clear_and_remove_ignore_stage_timestamp_and_capture_end(self) -> None: + deleted = [{"collection_id": 1, "capture_start_ns": 30}] + for timestamp, end in ((1, 30), (900, 800)): + result = ResultCollection(1, timestamp, capture_start_ns=30, capture_end_ns=end) + self.assertTrue(history_collection_removed(result, 30, [])) + self.assertTrue(history_collection_removed(result, 0, deleted)) + self.assertFalse(history_collection_removed(result, 29, [])) + fresh = ResultCollection(1, 1, capture_start_ns=31, capture_end_ns=32) + self.assertFalse(history_collection_removed(fresh, 30, deleted)) + + +if __name__ == "__main__": + unittest.main() diff --git a/python_app/tests/test_combo_controls.py b/python_app/tests/test_combo_controls.py index a07dba9..1a418d6 100644 --- a/python_app/tests/test_combo_controls.py +++ b/python_app/tests/test_combo_controls.py @@ -3,8 +3,6 @@ from __future__ import annotations import os -from pathlib import Path -from types import SimpleNamespace import unittest from unittest import mock @@ -12,12 +10,11 @@ import numpy as np os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") from PyQt6.QtCore import QSignalBlocker # noqa: E402 -from PyQt6.QtWidgets import QApplication # noqa: E402 +from PyQt6.QtWidgets import QApplication, QComboBox, QLineEdit, QPushButton # noqa: E402 -from python_app.tests.gui_test_support import isolated_window -from python_app.gui.controllers.app_window_snapshot_mixin import AppWindowSnapshotMixin # noqa: E402 -from python_app.models.dataset_model import ComboKey, ResultBlock, ResultCollection, ResultPayload # noqa: E402 -from python_app.models.run_config_model import RunConfigModel # noqa: E402 +from python_app.tests.gui_test_support import isolated_window, widget +from python_app.models.dataset_model import ComboKey, ResultBlock, ResultCollection, ResultPayload, SweepCollection # noqa: E402 +from python_app.models.run_config_schema import RunConfigModel # noqa: E402 class ComboNavigationTest(unittest.TestCase): @@ -25,106 +22,87 @@ class ComboNavigationTest(unittest.TestCase): def setUpClass(cls) -> None: cls.app = QApplication.instance() or QApplication([]) cls.window = isolated_window(cls.addClassCleanup) - cls.window._timer.stop() - cls.window._bscan_reprocess_timer.stop() - - @classmethod - def tearDownClass(cls) -> None: - cls.window.close() + cls.window._pipeline.timer.stop() + cls.window._pipeline.replay.timer.stop() def setUp(self) -> None: self.w = self.window - config = RunConfigModel() - config.radar.model = "librevna_multi" - config.apply_device_model_constraints() - self.w._defaults_config = config - self.w._set_combo_selection_mode("text") - with QSignalBlocker(self.w._processing_mode): - self.w._processing_mode.setCurrentText("bscan") - self.w._result_history.clear() - self.w._bscan_replay_results = [] - self.w._bscan_render_signature = None - self.w._single_combo_input.setText("0") - self.w._single_combo_output.setText("0") + self.root = self.w._profile.state.path.parent + self.mode = self.w._view.processing.mode + self.single_input = self.w._view.switches.single_input + self.single_output = self.w._view.switches.single_output + self.next_button = self.w._view.switches.next_combo + self.w._profile.state.config = RunConfigModel.load_from_path(self.w._profile.state.path) + self.w._radar.set_combo_mode("text") + with QSignalBlocker(self.mode): + self.mode.setCurrentText("bscan") + self.w._pipeline.reset_history() + self.w._plots.bscan.clear() + self.single_input.setText("0") + self.single_output.setText("0") - def test_next_button_draws_the_next_cached_combo_without_reprocessing(self) -> None: - self.w._result_history.append(ResultCollection( - collection_id=1, - monotonic_ns=1, - blocks=[ - ResultBlock(combo=ComboKey(input=0, output=output), payloads=[ResultPayload( - processing_name="bscan", kind=1, - frequency_hz=np.array([0.0, 1.0], dtype=np.float32), - trace=np.array([output + 1, output + 2], dtype=np.complex64), - )]) - for output in (0, 1) - ], - )) - with mock.patch.object(self.w, "_write_live_processing_config") as write_config: - self.w._next_combo_button.click() + def _seed_bscan(self, pairs: tuple[tuple[int, int], ...] | list[tuple[int, int]] = ((0, 0), (0, 1))) -> None: + result = ResultCollection(1, 1, capture_start_ns=1, blocks=[ + ResultBlock(ComboKey(*key), [ResultPayload( + processing_name="bscan", kind=1, + frequency_hz=np.array([0.0, 1.0], dtype=np.float32), + trace=np.array([key[1] + 1, key[1] + 2], dtype=np.complex64), + )]) for key in pairs + ]) + self.w._history.results.append(result) + self.w._plots.draw(result) - self.assertEqual(self.w._single_combo_input.text(), "0") - self.assertEqual(self.w._single_combo_output.text(), "1") - self.assertIn("in0/out1", self.w._bscan_plot.getPlotItem().titleLabel.text) - write_config.assert_not_called() - self.assertFalse(self.w._bscan_reprocess_timer.isActive()) + def test_next_button_draws_cached_combo_without_changing_acquisition(self) -> None: + for model in ("librevna", "librevna_multi", "kamil_adc"): + with self.subTest(model=model): + self.w._profile.state.config.radar.model = model + self._seed_bscan() + self.w._plots.select_combo(0, 0) + with mock.patch.object(self.w._live, "publish") as write_config: + self.next_button.click() + self.assertEqual(self.w._plots.display_key, (0, 1)) + self.assertEqual(self.single_input.text(), "0") + self.assertEqual(self.single_output.text(), "0") + plot = self.w._plots.bscan.widget.getPlotItem() + assert plot is not None + self.assertIn("in0/out1", plot.titleLabel.text) + write_config.assert_not_called() + self.assertFalse(self.w._pipeline.replay.timer.isActive()) - def test_navigation_wraps_with_output_changing_first(self) -> None: - keys = self.w._available_display_combo_keys() - visited = [(0, 0)] + def test_navigation_wraps_only_through_recorded_pairs(self) -> None: + keys = [(0, 0), (0, 3), (2, 1)] + self._seed_bscan(keys) + self.w._plots.select_combo(*keys[0]) + visited = [self.w._plots.display_key] for _ in keys: - self.w._next_combo_button.click() - visited.append((int(self.w._single_combo_input.text()), int(self.w._single_combo_output.text()))) + self.next_button.click() + visited.append(self.w._plots.display_key) self.assertEqual(visited, keys + [keys[0]]) - - def test_invalid_current_pair_recovers_to_the_first_pair(self) -> None: - self.w._single_combo_input.setText("invalid") - self.w._single_combo_output.setText("99") - self.w._next_combo_button.click() - self.assertEqual(self.w._single_combo_input.text(), "0") - self.assertEqual(self.w._single_combo_output.text(), "0") - - def test_unknown_selection_keeps_the_previous_fields(self) -> None: with self.assertRaisesRegex(ValueError, "Unknown channel combination"): - self.w._select_display_combo(999, 999) - self.assertEqual(self.w._single_combo_input.text(), "0") - self.assertEqual(self.w._single_combo_output.text(), "0") + self.w._plots.select_combo(999, 999) + self.assertEqual(self.w._plots.display_key, keys[0]) + + def _seed_unaligned_stages(self) -> tuple[list[SweepCollection], list[SweepCollection], list[ResultCollection]]: + raw = [SweepCollection(i, i, capture_start_ns=i) for i in range(1, 1003)] + pre = [SweepCollection(i, i, capture_start_ns=i) for i in range(11, 1012)] + results = [ResultCollection(i, i, capture_start_ns=i) for i in range(21, 1021)] + self.w._pipeline.reset_history() + self.w._history.raw.extend(raw) + self.w._history.preprocessed.extend(pre) + self.w._history.results.extend(results) + return raw[-1000:], pre[-1000:], results[-1000:] -class ManualSaveLimitTest(unittest.TestCase): - def setUp(self) -> None: - self.w = AppWindowSnapshotMixin() - self.w._raw_history = ["raw"] - self.w._pre_history = ["pre"] - self.w._result_history = ["result"] - self.w._drain_runtime_rings_for_snapshot = mock.Mock() - self.w._show_exception = mock.Mock() + def test_save_cancels_when_drain_restarts_acquisition_with_another_profile(self) -> None: + self._seed_unaligned_stages() + active = self.w._pipeline.active_config + self.addCleanup(setattr, self.w._pipeline, "active_config", active) + with mock.patch.object(self.w._pipeline, "drain_snapshot", side_effect=lambda: setattr( + self.w._pipeline, "active_config", RunConfigModel(), + )), mock.patch.object(self.w._snapshots.store, "save_runtime_snapshot_numpy") as save: + self.w._snapshots.save() + save.assert_not_called() - def test_snapshot_always_saves_1000_and_preserves_unaligned_stages(self) -> None: - self.w._write_snapshot_dataset = mock.Mock() - self.w._save_snapshot() - self.w._write_snapshot_dataset.assert_called_once_with( - ["raw"], ["pre"], ["result"], 1000, include_unaligned=True, - ) - - def test_json_export_uses_the_same_fixed_limit(self) -> None: - self.w._save_path_input = SimpleNamespace(text=lambda: "/tmp/measurements") - self.w._save_name_input = SimpleNamespace(text=lambda: "scene") - self.w._radar_config_name_prefix = lambda: "radar" - save_json = mock.Mock(return_value=([Path("/tmp/measurements/scene/trace.json")], { - "output_stem": "scene", "output_dir": "/tmp/measurements/scene", - })) - self.w._store = SimpleNamespace(save_runtime_vna_history_json_batch=save_json) - self.w._write_gui_profile_to_path = mock.Mock() - self.w._log = mock.Mock() - - self.w._save_vna_history_json() - - self.w._show_exception.assert_not_called() - save_json.assert_called_once_with( - Path("/tmp/measurements"), "scene", ["raw"], ["pre"], ["result"], 1000, - channel="s21", primary_stage="preprocessed", name_prefix="radar", include_unaligned=True, - ) if __name__ == "__main__": diff --git a/python_app/tests/test_control_button_controller.py b/python_app/tests/test_control_button_controller.py new file mode 100644 index 0000000..d38b4b7 --- /dev/null +++ b/python_app/tests/test_control_button_controller.py @@ -0,0 +1,84 @@ +"""GPIO startup failures are optional; queued presses cannot survive shutdown.""" + +import threading +import os +import unittest +from unittest import mock + +from PyQt6.QtCore import QObject, pyqtSignal +from PyQt6.QtWidgets import QApplication + +from python_app.gui.controllers.control_button_controller import ControlButtonController +from python_app.models.run_config_schema import ControlButtonModel +from python_app.tests.gui_test_support import isolated_window + + +class _Watcher(QObject): + pressed = pyqtSignal() + failed = pyqtSignal(str) + + def __init__(self, *, parent: QObject, **_kwargs: object) -> None: + super().__init__(parent) + self.stopped = False + + def start(self) -> None: + pass + + def stop(self) -> None: + self.stopped = True + self.pressed.emit() + + +class ControlButtonControllerTest(unittest.TestCase): + def setUp(self) -> None: + self.app = QApplication.instance() or QApplication([]) + self.config = ControlButtonModel(enabled=True, pin=17) + self.capture = mock.Mock() + self.parent = QObject() + self.controller = ControlButtonController(self.parent, lambda: self.config, self.capture, mock.MagicMock()) + self.addCleanup(self.controller.close) + + def test_close_rejects_already_queued_and_stop_time_presses(self) -> None: + with mock.patch("python_app.gui.controllers.control_button_controller.ControlButtonWatcher", _Watcher): + self.controller.start() + watcher = self.controller._watcher + assert isinstance(watcher, _Watcher) + watcher.pressed.emit() + self.capture.assert_called_once() + worker = threading.Thread(target=watcher.pressed.emit) + worker.start() + worker.join(timeout=1) + self.controller.close() + self.app.processEvents() + self.assertTrue(watcher.stopped) + self.capture.assert_called_once() + self.assertIsNone(self.controller._watcher) + + def test_disabled_invalid_and_unavailable_gpio_do_not_start_capture(self) -> None: + with mock.patch("python_app.gui.controllers.control_button_controller.ControlButtonWatcher") as create: + self.config.enabled = False + self.controller.start() + self.config.enabled, self.config.pin = True, -1 + self.controller.start() + create.assert_not_called() + self.config.pin = 17 + create.side_effect = OSError("GPIO unavailable") + self.controller.start() + self.assertIsNone(self.controller._watcher) + self.capture.assert_not_called() + + def test_gpio_capture_failure_is_logged_without_a_modal_dialog(self) -> None: + window = isolated_window(self.addCleanup) + window._profile.state.config.control_button.enabled = True + window._profile.state.config.control_button.pin = 17 + window._control_button._capture_reference = lambda: window._log.show_error("GPIO capture failed") + with mock.patch("python_app.gui.controllers.control_button_controller.ControlButtonWatcher", _Watcher), \ + mock.patch.dict(os.environ, {"RADAR_SYSTEM_HEADLESS": "0"}), \ + mock.patch("python_app.gui.controllers.log_controller.QMessageBox") as dialog, \ + self.assertLogs("python_app.gui", level="ERROR") as logs: + window._control_button.start() + watcher = window._control_button._watcher + assert isinstance(watcher, _Watcher) + watcher.pressed.emit() + dialog.assert_not_called() + self.assertTrue(any("GPIO capture failed" in line for line in logs.output)) diff --git a/python_app/tests/test_file_io.py b/python_app/tests/test_file_io.py index 7ab2d05..e019ac4 100644 --- a/python_app/tests/test_file_io.py +++ b/python_app/tests/test_file_io.py @@ -1,13 +1,17 @@ """Settings commits retain valid files on failure and concurrent writes.""" from concurrent.futures import ThreadPoolExecutor +import ctypes +import errno import json +import os from pathlib import Path +import stat import tempfile import unittest from unittest import mock -from python_app.file_io import atomic_write_json +from python_app.file_io import atomic_write_json, rename_no_replace from python_app.orchestration.gui_session_state import GuiSessionStateStore @@ -32,6 +36,51 @@ class AtomicJsonTest(unittest.TestCase): atomic_write_json(self.path, {'bad': float('nan')}) self.assertEqual(json.loads(self.path.read_text()), {'old': True}) + def test_replacement_preserves_existing_permissions(self) -> None: + self.path.chmod(0o664) + atomic_write_json(self.path, {"new": True}, durable=True) + self.assertEqual(stat.S_IMODE(self.path.stat().st_mode), 0o664) + + def test_new_file_uses_process_umask(self) -> None: + path = self.path.with_name("new.json") + previous = os.umask(0o027) + try: + atomic_write_json(path, {"new": True}) + finally: + os.umask(previous) + self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o640) + + def test_publish_cannot_replace_existing_file_or_empty_directory(self) -> None: + source = self.path.with_name('source') + source.write_text('new', encoding='utf-8') + before = self.path.read_bytes() + with self.assertRaises(FileExistsError): + rename_no_replace(source, self.path) + self.assertEqual(self.path.read_bytes(), before) + self.assertEqual(source.read_text(), 'new') + source.unlink() + source.mkdir() + destination = self.path.with_name('destination') + destination.mkdir() + with self.assertRaises(FileExistsError): + rename_no_replace(source, destination) + self.assertTrue(source.is_dir()) + self.assertTrue(destination.is_dir()) + + def test_publish_propagates_filesystem_permission_error_without_touching_source(self) -> None: + destination = self.path.with_name('denied.json') + + def deny(*_args): + ctypes.set_errno(errno.EACCES) + return -1 + + before = self.path.read_bytes() + with mock.patch('python_app.file_io._renameat2', side_effect=deny): + with self.assertRaises(PermissionError): + rename_no_replace(self.path, destination) + self.assertEqual(self.path.read_bytes(), before) + self.assertFalse(destination.exists()) + def test_concurrent_writers_publish_complete_json_and_leave_no_temp_files(self) -> None: payloads = [{'index': i, 'values': [i] * 1000} for i in range(32)] with ThreadPoolExecutor(max_workers=8) as executor: diff --git a/python_app/tests/test_generator_sweep.py b/python_app/tests/test_generator_sweep.py deleted file mode 100644 index 3b88ee1..0000000 --- a/python_app/tests/test_generator_sweep.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Standalone generator configuration and independent resource cleanup.""" - -from dataclasses import replace -import unittest -from unittest import mock - -from python_app.generator_sweep.config import GeneratorSweepConfig, resolve_generator_sweep_config -from python_app.generator_sweep.runner import GeneratorSweepRunner - - -class GeneratorSweepTest(unittest.TestCase): - def test_frequency_grid_includes_stop_with_and_without_an_exact_step(self) -> None: - for step, expected in ((2, (10, 12, 14, 15)), (5, (10, 15))): - config = GeneratorSweepConfig(start_hz=10, stop_hz=15, step_hz=step) - self.assertEqual(resolve_generator_sweep_config(config).frequencies_hz, expected) - config = GeneratorSweepConfig(start_hz=10, stop_hz=15, points=3) - self.assertEqual(resolve_generator_sweep_config(config).frequencies_hz, (10, 12, 15)) - - def test_malformed_types_nonfinite_values_and_excessive_grids_fail_before_open(self) -> None: - valid = GeneratorSweepConfig(points=3) - for changes in ({'points': 3.5}, {'port': True}, {'sweep_start_pin': 5.5}, - {'power_dbm': float('inf')}, {'hold_time_ms': float('nan')}, - {'points': 1_000_001}, {'points': 0, 'step_hz': 1, 'stop_hz': 10 ** 10}): - with self.subTest(changes=changes), self.assertRaises(ValueError): - resolve_generator_sweep_config(replace(valid, **changes)) - - def test_each_cleanup_failure_still_releases_the_other_resources(self) -> None: - for failure in ('pwm_stop', 'device_idle', 'device_close', 'marker_reset', 'marker_close'): - with self.subTest(failure=failure): - runner = GeneratorSweepRunner(GeneratorSweepConfig(points=3)) - runner._pwm = mock.Mock() - runner._markers = mock.Mock() - device = runner._device = mock.Mock(is_connected=True) - actions = { - 'pwm_stop': runner._pwm.disable, 'device_idle': device.generator.set_idle, - 'device_close': device.disconnect, 'marker_reset': runner._markers.reset_to_idle, - 'marker_close': runner._markers.close, - } - actions[failure].side_effect = OSError('cleanup failure') - with self.assertLogs('python_app.generator_sweep.runner', level='WARNING'): - runner._close() - for action in actions.values(): - action.assert_called_once() - runner._pwm.close.assert_called_once() - self.assertIsNone(runner._device) diff --git a/python_app/tests/test_gui_action_lifecycle.py b/python_app/tests/test_gui_action_lifecycle.py index fb0a16f..a9f63bd 100644 --- a/python_app/tests/test_gui_action_lifecycle.py +++ b/python_app/tests/test_gui_action_lifecycle.py @@ -2,8 +2,11 @@ from __future__ import annotations +from contextlib import ExitStack + from collections import deque import itertools +import json import os from pathlib import Path import tempfile @@ -12,21 +15,24 @@ import unittest from unittest import mock os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") -from PyQt6.QtWidgets import QApplication # noqa: E402 -from PyQt6.QtCore import QTimer # noqa: E402 +from PyQt6.QtWidgets import QApplication, QCheckBox, QComboBox, QDoubleSpinBox, QLineEdit, QSpinBox # noqa: E402 +from PyQt6.QtCore import QSignalBlocker, QTimer # noqa: E402 from python_app.gui.app_window import AppWindow # noqa: E402 -from python_app.gui.controllers.app_window_config.profile_io_mixin import AppWindowConfigProfileIOMixin # noqa: E402 -from python_app.gui.controllers.app_window_config.live_processing_mixin import AppWindowLiveProcessingMixin # noqa: E402 -from python_app.gui.controllers.app_window_pipeline_mixin import AppWindowPipelineMixin # noqa: E402 -from python_app.gui.controllers.app_window_preprocess_mixin import AppWindowPreprocessMixin # noqa: E402 -from python_app.gui.controllers.app_window_snapshot_mixin import AppWindowSnapshotMixin # noqa: E402 -from python_app.gui.controllers.app_window_recording_mixin import AppWindowRecordingMixin # noqa: E402 +from python_app.gui.controllers.pipeline_controller import PipelineController +from python_app.orchestration.process_supervisor import ProcessSupervisor +from functools import partial +from python_app.gui.runtime.history import build_processor_run_signature, build_run_history_signature +from python_app.tests.gui_test_support import isolated_window, widget +from python_app.tests.test_reader_recovery import pipeline_fixture +from python_app.gui.runtime.pipeline_pause import PipelinePause from python_app.models.dataset_model import ResultCollection, SweepCollection # noqa: E402 -from python_app.models.gui_profile_model import GuiProfileModel, GuiStateModel # noqa: E402 -from python_app.models.run_config_model import ComboModel, GprRxGeometryModel, GprTxGeometryModel, RunConfigModel # noqa: E402 -from python_app.storage.npz_store import NpzStore # noqa: E402 -from python_app.workflows.kamil_adc_neutral_preprocess import build_neutral_s21_sets # noqa: E402 +from python_app.models.gui_profile_schema import GuiPreprocessDialogStateModel, GuiProfileModel, GuiStateModel # noqa: E402 +from python_app.models.run_config_schema import ComboModel, GprRxGeometryModel, GprTxGeometryModel, RunConfigModel # noqa: E402 +from python_app.orchestration.live_processing_config import ProcessingLiveConfig, ProcessingLiveConfigWriter +from python_app.models.setting_fields import GUI_FIELDS +from python_app.workflows.neutral_preprocess import build_neutral_s21_sets # noqa: E402 +from python_app.workflows.sequential_capture_workflow import SequentialCaptureState def setUpModule() -> None: @@ -34,242 +40,520 @@ def setUpModule() -> None: _app = QApplication.instance() or QApplication([]) -def _reader(method: str, collections: list): +def _mock(value: object) -> mock.Mock: + assert isinstance(value, mock.Mock) + return value + + +def _reader(method: str, collections: list) -> mock.Mock: pending = deque(collections) - return SimpleNamespace(**{method: lambda: pending.popleft() if pending else None, "close": mock.Mock()}) + return mock.Mock(**{method: mock.Mock(side_effect=lambda: pending.popleft() if pending else None)}) -class _Runtime(AppWindowPipelineMixin, AppWindowSnapshotMixin, AppWindowPreprocessMixin): - def __init__(self) -> None: - self._raw_reader = self._pre_reader = self._result_reader = None - self._raw_history = deque(maxlen=2) - self._pre_history = deque(maxlen=2) - self._result_history = deque(maxlen=2) - self._max_pop_per_poll = self._max_pop_per_snapshot_drain = 1 - self._single_capture_active = True - self._single_capture_start_ns = 10 - self._single_capture_seen_raw = False - self._single_capture_target_collection_id = None - self._capture_session = None - self._preprocess_capture_busy = self._closing = False - self._resume_pipeline_after_capture = True - self._stop_request_seq = 0 - self._pipeline_metrics = mock.Mock() - self._record_collection = mock.Mock() - self._supervisor = mock.Mock() - self._supervisor.is_running.return_value = False - self._supervisor.is_processor_running.return_value = True - self._status_label = mock.Mock() - for name in ("_log", "_log_debug", "_log_exception", "_show_error", "_show_exception", - "_update_history_indicator", "_finalize_recording_on_stop", "_web_update_snapshot", - "_build_config", "_update_capture_dialog_state", "close"): - setattr(self, name, mock.Mock()) - self._pump_events_during_drain = mock.Mock() +def _window(case: unittest.TestCase) -> AppWindow: + supervisor = mock.Mock(spec=ProcessSupervisor) + supervisor.is_running.return_value = supervisor.is_processor_running.return_value = False + supervisor.collect_exit_reports.return_value, supervisor.pids.return_value = [], {} + with mock.patch("python_app.gui.controllers.pipeline_controller.ProcessSupervisor", return_value=supervisor): + window = isolated_window(case.addCleanup) + window._pipeline.timer.stop() + return window + + +def _pipeline(case: unittest.TestCase) -> PipelineController: + pipeline, supervisor = pipeline_fixture(case) + supervisor.is_processor_running.return_value = True + pipeline.max_pop_per_poll = pipeline.max_pop_per_snapshot_drain = 1 + pipeline.recording.record = mock.Mock() + pipeline.recording.finish = mock.Mock() + pipeline.pump_events = mock.Mock() + pipeline._build_config = mock.Mock(return_value=RunConfigModel()) + return pipeline + + +class PersistentHistoryDeletionTest(unittest.TestCase): + def setUp(self) -> None: + cleanup = ExitStack() + self.addCleanup(cleanup.close) + folder = cleanup.enter_context(tempfile.TemporaryDirectory()) + self.path = Path(folder) / "processing_live.json" + self.writer = _window(self) + self.writer._live.writer = ProcessingLiveConfigWriter(self.path) + self.first = dict(collection_id=1, capture_start_ns=100) + self.second = dict(collection_id=2, capture_start_ns=200) + self.remove(self.first) + + def remove(self, target: dict[str, int]) -> None: + self.writer._history.results.append(ResultCollection(target["collection_id"], 0, capture_start_ns=target["capture_start_ns"])) + self.writer._history.delete(remove_last=True) + self.writer._live.publish() + + def test_delete_publishes_before_plot_cleanup_and_keeps_tombstone_on_failure(self) -> None: + for boundary in ("live", "plot"): + with self.subTest(boundary=boundary): + w = self.writer + removed = ResultCollection(2, 210, capture_start_ns=200) + w._history.results.append(removed) + release = mock.Mock() + pause = PipelinePause(lambda: True, release) + events = [] + + def action(name: str) -> None: + events.append(name) + if name == boundary: + raise OSError(name) + + with mock.patch.object(w._plots, "clear", side_effect=lambda: action("plot")), \ + mock.patch.object(w._pipeline, "pause", return_value=pause), \ + mock.patch.object(w._pipeline, "drop_pending", return_value=(0, 0, 0)), \ + mock.patch.object(w._pipeline, "publish_live", side_effect=lambda: action("live")), \ + mock.patch.object(w._pipeline, "settle_deleted_results"), \ + mock.patch.object(w._snapshots, "_error") as error: + w._snapshots.delete_history(remove_last=True) + self.assertEqual(events, ["live", "plot"]) + self.assertFalse(w._history.results) + self.assertFalse(w._history.record_result(removed)) + error.assert_called_once() + release.assert_called_once_with(True) + + def test_ordinary_update_and_second_removal_preserve_all_pending_targets(self) -> None: + self.writer._live.publish() + self.assertEqual(json.loads(self.path.read_text())["history_removed"], [self.first]) + self.remove(self.second) + saved = json.loads(self.path.read_text()) + self.assertEqual(saved["history_removed"], [self.first, self.second]) + self.assertEqual(saved["history_clear_before_ns"], 0) + self.assertEqual(self.writer._live.processing_settings[saved["settings_revision"]], saved) + + def test_clear_replaces_removals_with_a_cutoff_that_survives_later_updates(self) -> None: + self.remove(self.second) + with mock.patch("python_app.gui.controllers.history_controller.time.monotonic_ns", return_value=500): + self.writer._history.delete(remove_last=False) + self.writer._live.publish() + self.writer._live.publish() + saved = json.loads(self.path.read_text()) + self.assertEqual(saved["history_removed"], []) + self.assertEqual(saved["history_clear_before_ns"], 500) + + def test_failed_write_retains_deletion_state_for_the_next_ordinary_update(self) -> None: + for command in ("remove_last", "clear_all"): + with self.subTest(command=command): + before = self.path.read_bytes() + revisions = dict(self.writer._live.processing_settings) + with mock.patch("python_app.file_io.os.replace", side_effect=OSError("disk full")), mock.patch( + "python_app.gui.controllers.history_controller.time.monotonic_ns", return_value=500, + ), self.assertRaisesRegex(OSError, "disk full"): + self.writer._history.results.append(ResultCollection(2, 0, capture_start_ns=200)) + self.writer._history.delete(remove_last=command == "remove_last") + self.writer._live.publish() + self.assertEqual(self.path.read_bytes(), before) + self.assertEqual(self.writer._live.processing_settings, revisions) + expected_targets = [self.first, self.second] if command == "remove_last" else [] + expected_cutoff = 0 if command == "remove_last" else 500 + self.assertEqual(self.writer._history.removed, expected_targets) + self.assertEqual(self.writer._history.clear_before_ns, expected_cutoff) + self.writer._live.publish() + saved = json.loads(self.path.read_text()) + self.assertEqual(saved["history_removed"], expected_targets) + self.assertEqual(saved["history_clear_before_ns"], expected_cutoff) + + def test_late_deleted_results_never_reach_history_or_recording_but_next_acquisition_does(self) -> None: + def result(ns: int, start: int, end: int) -> ResultCollection: + return ResultCollection(1, ns, capture_start_ns=start, capture_end_ns=end) + + for cutoff, target, late, fresh in ( + (0, dict(collection_id=1, capture_start_ns=100), result(600, 100, 200), result(700, 201, 300)), + (200, None, result(600, 100, 200), result(700, 201, 300)), + ): + with self.subTest(cutoff=cutoff, target=target): + pipeline = _pipeline(self) + pipeline.history.clear_before_ns = cutoff + pipeline.history.removed[:] = [target] if target is not None else [] + pipeline._result_reader = _reader("pop_result_collection", [late, fresh]) + self.assertIsNone(pipeline.read_results()) + self.assertEqual(list(pipeline.history.results), []) + _mock(pipeline.recording.record).assert_not_called() + self.assertIs(pipeline.read_results(), fresh) + self.assertEqual(list(pipeline.history.results), [fresh]) + _mock(pipeline.recording.record).assert_called_once_with("results", fresh) class NeutralSetCreationTest(unittest.TestCase): def test_failed_second_set_preserves_gui_selection_and_previous_data(self) -> None: - w = _Runtime() + w = _window(self) config = RunConfigModel.from_dict({"radar": { "model": "librevna", "driver_mode": "mock", "sweep": {"points": 7}, }}) - w._build_config.return_value = config - w._ensure_preprocess_dialog = mock.Mock(return_value=SimpleNamespace(set_name=lambda: "new_neutral")) - w._radar_key = mock.Mock(return_value="radar") - w._selected_preprocess_sets = {"s21_calibration": "old_cal", "s21_reference": "old_ref"} - w._selected_preprocess_radar_key = "previous_radar" - w._processor_run_signature = "processor_signature" - w._history_run_signature = "history_signature" - w._refresh_sets = mock.Mock() - w._reset_runtime_history = mock.Mock() - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - w._store = NpzStore(root) - calibration, reference = build_neutral_s21_sets(config) - w._store.save_set("s21_calibration", "radar", "old_cal", calibration) - w._store.save_set("s21_reference", "radar", "old_ref", reference) - before = {path: path.read_bytes() for path in root.rglob("*") if path.is_file()} - save = w._store.save_set + w._preprocess._build_config = lambda: config + w._preprocess.ensure_dialog = mock.Mock(return_value=SimpleNamespace(set_name=lambda: "new_neutral")) + w._preprocess.selected_sets = {"s21_calibration": "old_cal", "s21_reference": "old_ref"} + w._preprocess.radar_key = "previous_radar" + w._pipeline._processor_signature = ("processor_signature",) + w._history.run_signature = ("history_signature",) + w._preprocess.refresh_sets = mock.Mock() + w._pipeline.reset_history = mock.Mock() + root = w._profile._project_root / "python_app/data" + calibration, reference = build_neutral_s21_sets(config) + w._snapshots.store.save_set("s21_calibration", config.radar_key(), "old_cal", calibration) + w._snapshots.store.save_set("s21_reference", config.radar_key(), "old_ref", reference) + before = {path: path.read_bytes() for path in root.rglob("*") if path.is_file()} + save = w._snapshots.store.save_set - def save_set(kind, *args): - if kind == "s21_reference": - raise OSError("reference write failed") - return save(kind, *args) + def save_set(kind, *args): + if kind == "s21_reference": + raise OSError("reference write failed") + return save(kind, *args) - with mock.patch.object(w._store, "save_set", side_effect=save_set): - w._create_neutral_sets() - self.assertEqual({path: path.read_bytes() for path in root.rglob("*") if path.is_file()}, before) - self.assertEqual(w._selected_preprocess_sets, {"s21_calibration": "old_cal", "s21_reference": "old_ref"}) - self.assertEqual(w._selected_preprocess_radar_key, "previous_radar") - self.assertEqual((w._processor_run_signature, w._history_run_signature), ("processor_signature", "history_signature")) - w._show_exception.assert_called_once() - w._refresh_sets.assert_not_called() - w._reset_runtime_history.assert_not_called() + with mock.patch.object(w._snapshots.store, "save_set", side_effect=save_set), w._log.capture_errors() as errors: + w._preprocess.create_neutral_sets() + self.assertIn("reference write failed", errors[0]) + self.assertEqual({path: path.read_bytes() for path in root.rglob("*") if path.is_file()}, before) + self.assertEqual(w._preprocess.selected_sets, {"s21_calibration": "old_cal", "s21_reference": "old_ref"}) + self.assertEqual(w._preprocess.radar_key, "previous_radar") + self.assertEqual((w._pipeline._processor_signature, w._history.run_signature), (("processor_signature",), ("history_signature",))) + _mock(w._preprocess.refresh_sets).assert_not_called() + _mock(w._pipeline.reset_history).assert_not_called() class SnapshotDrainTest(unittest.TestCase): + def test_single_capture_without_raw_tap_uses_first_fresh_result(self) -> None: + pipeline = _pipeline(self) + pipeline.single_capture_active, pipeline._single_capture_start_ns = True, 10 + stale = ResultCollection(1, 20, capture_start_ns=9, capture_end_ns=9) + fresh = ResultCollection(1, 30, capture_start_ns=11, capture_end_ns=18) + later = ResultCollection(2, 40, capture_start_ns=21, capture_end_ns=28) + pipeline._result_reader = _reader("pop_result_collection", [stale, fresh, later]) + with mock.patch.object(pipeline.plots, "has_output", return_value=True), \ + mock.patch.object(pipeline.plots, "draw") as draw, \ + mock.patch("python_app.gui.controllers.pipeline_controller.time.monotonic_ns", return_value=30): + pipeline.poll() + self.assertTrue(pipeline.single_capture_active) + draw.assert_not_called() + pipeline.poll() + self.assertFalse(pipeline.single_capture_active) + draw.assert_called_once_with(fresh) + def test_save_drain_preserves_recording_and_single_capture_tracking(self) -> None: - w = _Runtime() - raw = SweepCollection(collection_id=5, monotonic_ns=20, traces=[], capture_start_ns=1, capture_end_ns=8) - preprocessed = SweepCollection(collection_id=5, monotonic_ns=20, traces=[]) - result = ResultCollection(collection_id=5, monotonic_ns=20, blocks=[]) - w._raw_reader = _reader("pop_raw_collection", [raw]) - w._pre_reader = _reader("pop_preprocessed_collection", [preprocessed]) - w._result_reader = _reader("pop_result_collection", [result, result]) + pipeline = _pipeline(self) + pipeline.single_capture_active, pipeline._single_capture_start_ns = True, 10 + raw = SweepCollection(collection_id=5, monotonic_ns=20, traces=[], capture_start_ns=11, capture_end_ns=18) + preprocessed = SweepCollection(collection_id=5, monotonic_ns=21, traces=[], capture_start_ns=11, capture_end_ns=18) + result = ResultCollection(collection_id=5, monotonic_ns=22, blocks=[], capture_start_ns=11, capture_end_ns=18) + pipeline._raw_reader = _reader("pop_raw_collection", [raw]) + pipeline._pre_reader = _reader("pop_preprocessed_collection", [preprocessed]) + pipeline._result_reader = _reader("pop_result_collection", [result, result]) - w._drain_runtime_rings_for_snapshot() + with self.assertNoLogs("python_app.gui.controllers.pipeline_controller", level="WARNING"): + pipeline.drain_snapshot() - self.assertEqual(w._record_collection.call_args_list, [ + self.assertEqual(_mock(pipeline.recording.record).call_args_list, [ mock.call("raw", raw), mock.call("preprocessed", preprocessed), mock.call("results", result), ]) - self.assertTrue(w._single_capture_seen_raw) - self.assertEqual(w._single_capture_target_collection_id, 5) - w._pipeline_metrics.record.assert_any_call("acquisition", 7) - self.assertEqual(list(w._result_history), [result]) - w._log_exception.assert_not_called() + self.assertTrue(pipeline.single_capture_active) + self.assertEqual(list(pipeline.history.results), [result]) def test_continuous_producer_cannot_prevent_snapshot_timeout(self) -> None: - w = _Runtime() - collection = SweepCollection(collection_id=1, monotonic_ns=20, traces=[]) + pipeline = _pipeline(self) + collection = SweepCollection(collection_id=1, monotonic_ns=20, traces=[], capture_start_ns=11, capture_end_ns=18) pop = mock.Mock(return_value=collection) - w._raw_reader = SimpleNamespace(pop_raw_collection=pop) + pipeline._raw_reader = mock.Mock(pop_raw_collection=pop) ticks = itertools.count(0.0, 0.1) - with mock.patch("python_app.gui.controllers.app_window_snapshot_mixin.time.monotonic", side_effect=lambda: next(ticks)): - w._drain_runtime_rings_for_snapshot() + with mock.patch("python_app.gui.controllers.pipeline_controller.time.monotonic", side_effect=lambda: next(ticks)), \ + self.assertNoLogs("python_app.gui.controllers.pipeline_controller", level="WARNING"): + pipeline.drain_snapshot() self.assertGreater(pop.call_count, 1) self.assertLess(pop.call_count, 15) - w._log_exception.assert_not_called() def test_stop_drain_reads_the_whole_tail_when_history_is_full(self) -> None: - w = _Runtime() - w._raw_history = deque([SweepCollection(collection_id=0, monotonic_ns=0, traces=[])], maxlen=1) - tail = [SweepCollection(collection_id=i, monotonic_ns=i, traces=[]) for i in range(1, 11)] - w._raw_reader = _reader("pop_raw_collection", tail) + pipeline = _pipeline(self) + pipeline.history.raw = deque([SweepCollection(collection_id=0, monotonic_ns=2, traces=[], capture_start_ns=1, capture_end_ns=2)], maxlen=1) + tail = [SweepCollection(collection_id=i, monotonic_ns=i + 20, traces=[], capture_start_ns=i + 10, capture_end_ns=i + 11) for i in range(1, 11)] + pipeline._raw_reader = _reader("pop_raw_collection", tail) clock = [0.0] - w._pump_events_during_drain.side_effect = lambda pause: clock.__setitem__(0, clock[0] + pause) - with mock.patch("python_app.gui.controllers.app_window_pipeline_mixin.time.monotonic", side_effect=lambda: clock[0]): - w._drain_rings_until_quiet(timeout_s=1.0, poll_s=0.01) - self.assertEqual([call.args[1].collection_id for call in w._record_collection.call_args_list], list(range(1, 11))) - self.assertEqual(w._raw_history[-1].collection_id, 10) + _mock(pipeline.pump_events).side_effect = lambda pause: clock.__setitem__(0, clock[0] + pause) + with mock.patch("python_app.gui.controllers.pipeline_controller.time.monotonic", side_effect=lambda: clock[0]): + pipeline.drain_rings(timeout_s=1.0, poll_s=0.01) + self.assertEqual([call.args[1].collection_id for call in _mock(pipeline.recording.record).call_args_list], list(range(1, 11))) + self.assertEqual(pipeline.history.raw[-1].collection_id, 10) class ShutdownReentryTest(unittest.TestCase): def test_reader_failure_on_stop_is_reported_and_cancels_automatic_resume(self) -> None: - w = _Runtime() - w._supervisor.is_running.return_value = True - w._drain_rings_until_quiet = mock.Mock(side_effect=ValueError("corrupt ring payload")) - raw = w._raw_reader = SimpleNamespace(close=mock.Mock()) - preprocessed = w._pre_reader = SimpleNamespace(close=mock.Mock()) - result = w._result_reader = SimpleNamespace(close=mock.Mock(), _ring_name="/results") - expected_stop_seq = w._stop_request_seq + 1 - with mock.patch("python_app.gui.controllers.app_window_pipeline_mixin.ShmRingReader") as reopen: - w._stop_run() - w._supervisor.stop.assert_called_once() - w._supervisor.stop_all.assert_not_called() - w._finalize_recording_on_stop.assert_called_once() + pipeline = _pipeline(self) + _mock(pipeline.supervisor.is_running).return_value = True + pipeline.drain_rings = mock.Mock(side_effect=ValueError("corrupt ring payload")) + raw = pipeline._raw_reader = mock.Mock(close=mock.Mock()) + preprocessed = pipeline._pre_reader = mock.Mock(close=mock.Mock()) + result = pipeline._result_reader = mock.Mock(close=mock.Mock()) + result.configure_mock(name="/results") + expected_stop_seq = pipeline.stop_generation + 1 + with mock.patch("python_app.gui.controllers.pipeline_controller.ShmRingReader") as reopen: + pipeline.stop() + _mock(pipeline.supervisor.stop).assert_called_once() + _mock(pipeline.supervisor.stop_all).assert_not_called() + _mock(pipeline.recording.finish).assert_called_once() for reader in (raw, preprocessed, result): reader.close.assert_called_once() - self.assertIsNone(w._raw_reader) - self.assertIsNone(w._pre_reader) - self.assertIs(w._result_reader, reopen.return_value) + self.assertEqual(pipeline.readers, (None, None, reopen.return_value)) reopen.assert_called_once_with("/results") - w._show_exception.assert_called_once() - self.assertIn("corrupt ring", str(w._show_exception.call_args.args[1])) - self.assertFalse(w._stopping_run) - self.assertFalse(w._single_capture_active) - self.assertEqual(w._status_label.setText.call_args.args[0], "Status: error") - self.assertFalse(w._pipeline_resume_allowed(expected_stop_seq)) + _mock(pipeline._show_exception).assert_called_once() + self.assertIn("corrupt ring", str(_mock(pipeline._show_exception).call_args.args[1])) + self.assertFalse(pipeline.stopping) + self.assertFalse(pipeline.single_capture_active) + _mock(pipeline._status).assert_called_with("Status: error") + self.assertFalse(pipeline.can_resume(expected_stop_seq)) def test_unsupported_kamil_adc_mock_fails_before_starting_any_process(self) -> None: - w = _Runtime() + pipeline = _pipeline(self) config = RunConfigModel() config.radar.model = "kamil_adc" config.apply_device_model_constraints() - w._build_config.return_value = config - w._stop_all_processes = mock.Mock() - w._start_run(single_capture=True) - w._supervisor.start.assert_not_called() - w._show_exception.assert_called_once() - self.assertIn("native", str(w._show_exception.call_args.args[1])) + _mock(pipeline._build_config).return_value = config + pipeline.stop_all = mock.Mock() + pipeline.start(single_capture=True) + _mock(pipeline.supervisor.start).assert_not_called() + _mock(pipeline._show_exception).assert_called_once() + self.assertIn("native", str(_mock(pipeline._show_exception).call_args.args[1])) def test_stop_blocks_nested_start_poll_and_duplicate_stop(self) -> None: - w = _Runtime() - w._supervisor.is_running.return_value = True + pipeline = _pipeline(self) + _mock(pipeline.supervisor.is_running).return_value = True def reenter(**_kwargs): - w._start_run() - w._poll_rings() - w._stop_run() + pipeline.start() + pipeline.poll() + pipeline.stop() - w._drain_rings_until_quiet = mock.Mock(side_effect=reenter) - w._stop_run() - w._supervisor.stop_orchestrator.assert_called_once() - w._supervisor.stop_preprocessor.assert_called_once() - w._finalize_recording_on_stop.assert_called_once() - w._build_config.assert_not_called() - w._web_update_snapshot.assert_not_called() - self.assertFalse(w._stopping_run) - self.assertFalse(w._resume_pipeline_after_capture) + pipeline.drain_rings = mock.Mock(side_effect=reenter) + pipeline.stop() + _mock(pipeline.supervisor.stop_acquisition).assert_called_once() + _mock(pipeline.supervisor.stop_preprocessor).assert_called_once() + _mock(pipeline.recording.finish).assert_called_once() + _mock(pipeline._build_config).assert_not_called() + _mock(pipeline._poll_ui).assert_not_called() + self.assertFalse(pipeline.stopping) + self.assertFalse(pipeline.should_run) def test_close_during_stop_waits_until_tail_is_finalized(self) -> None: - w = _Runtime() - w._supervisor.is_running.return_value = True - event = mock.Mock() + pipeline = _pipeline(self) + _mock(pipeline.supervisor.is_running).return_value = True ordering = [] def close_during_drain(**_kwargs): - AppWindow.closeEvent(w, event) + self.assertFalse(pipeline.begin_close()) - w._drain_rings_until_quiet = mock.Mock(side_effect=close_during_drain) - w._finalize_recording_on_stop.side_effect = lambda: ordering.append("finalize") - w.close.side_effect = lambda: ordering.append("close") - w._stop_run() - event.ignore.assert_called() - w._supervisor.stop_all.assert_called_once() + pipeline.drain_rings = mock.Mock(side_effect=close_during_drain) + _mock(pipeline.recording.finish).side_effect = lambda **_kwargs: ordering.append("finalize") + _mock(pipeline._deferred_close).side_effect = lambda: ordering.append("close") + pipeline.stop() + _mock(pipeline.supervisor.stop_all).assert_called_once() self.assertEqual(ordering, ["finalize", "close"]) - self.assertFalse(w._close_after_stop) def test_explicit_stop_during_apply_pause_cancels_apply_and_restart(self) -> None: - w = _Runtime() - w._supervisor.is_running.return_value = True - w._start_run = mock.Mock() - w._drain_rings_until_quiet = mock.Mock(side_effect=lambda **_kwargs: w._stop_run()) - w._apply_radar_settings() - w._build_config.assert_not_called() - w._start_run.assert_not_called() + pipeline = _pipeline(self) + _mock(pipeline.supervisor.is_running).return_value = True + pipeline.start = mock.Mock() + pipeline.drain_rings = mock.Mock(side_effect=lambda **_kwargs: pipeline.stop()) + pipeline.apply_radar() + _mock(pipeline._build_config).assert_called_once() + _mock(pipeline.start).assert_not_called() class CaptureBatchControlTest(unittest.TestCase): def test_stop_between_combos_keeps_captures_and_allows_later_resume(self) -> None: - w = _Runtime() + w = _window(self) captured = [] - session = SimpleNamespace( + session = mock.Mock( kind="s21_reference", is_complete=lambda: len(captured) == 3, state=lambda: SimpleNamespace(total_count=3, captured_count=len(captured)), capture_current_combo=lambda: captured.append(len(captured)), ) - w._capture_session = session + w._preprocess.capture_session = session dialog = mock.Mock() - w._ensure_preprocess_dialog = lambda: dialog - w._record_preprocess_capture = mock.Mock() - w._pump_events_during_drain = lambda pause: AppWindowPipelineMixin._pump_events_during_drain(w, pause) - QTimer.singleShot(0, w._stop_run) - with mock.patch("python_app.gui.controllers.app_window_preprocess_mixin.QTimer.singleShot", side_effect=lambda _delay, fn: fn()): - w._capture_all_remaining() + w._preprocess.ensure_dialog = lambda: dialog + w._preprocess._record_capture = mock.Mock() + w._pipeline.pump_events = mock.Mock(side_effect=lambda pause: PipelineController.pump_events(w._pipeline, pause)) + QTimer.singleShot(0, w._pipeline.stop) + with mock.patch("python_app.gui.controllers.preprocess_controller.QTimer.singleShot", side_effect=lambda _delay, fn: fn()): + w._preprocess.capture_all() self.assertEqual(captured, [0]) - self.assertIs(w._capture_session, session) - self.assertFalse(w._preprocess_capture_busy) + self.assertIs(w._preprocess.capture_session, session) + self.assertFalse(w._preprocess.capture_busy) self.assertIn("paused", dialog.set_status.call_args.args[0]) - w._capture_all_remaining() + w._preprocess.capture_all() self.assertEqual(captured, [0, 1, 2]) - self.assertEqual(w._record_preprocess_capture.call_count, 3) + self.assertEqual(_mock(w._preprocess._record_capture).call_count, 3) + + +class PipelinePauseTest(unittest.TestCase): + def runtime(self): + pipeline = _pipeline(self) + pipeline.start = mock.Mock() + pipeline.drain_rings = mock.Mock() + return pipeline + + def test_backoff_intent_is_suspended_until_operation_finishes(self) -> None: + pipeline = self.runtime() + pipeline.should_run = True + pause = pipeline.pause() + self.assertFalse(pipeline.should_run) + pipeline.poll() + _mock(pipeline.supervisor.start).assert_not_called() + _mock(pipeline.start).assert_not_called() + pause.finish() + pause.finish() + self.assertFalse(pipeline.paused) + _mock(pipeline.start).assert_called_once_with() + + def test_reentrant_pause_is_refused_before_hardware_shutdown_repeats(self) -> None: + pipeline = self.runtime() + pipeline.should_run = True + _mock(pipeline.supervisor.is_running).return_value = True + failures = [] + + def reenter(**_kwargs): + with self.assertRaisesRegex(RuntimeError, "current pipeline operation"): + pipeline.pause() + failures.append(True) + + _mock(pipeline.drain_rings).side_effect = reenter + with pipeline.pause(): + self.assertTrue(failures) + _mock(pipeline.start).assert_not_called() + _mock(pipeline.supervisor.stop_acquisition).assert_called_once() + _mock(pipeline.supervisor.stop_preprocessor).assert_called_once() + _mock(pipeline.start).assert_called_once() + + def test_explicit_stop_during_pause_cancels_resumption(self) -> None: + pipeline = self.runtime() + pipeline.should_run = True + with pipeline.pause() as pause: + pipeline.stop() + self.assertFalse(pause.valid) + self.assertFalse(pipeline.paused) + _mock(pipeline.start).assert_not_called() + + def test_qt_stop_or_close_during_pause_drain_cancels_before_operation(self) -> None: + for close in (False, True): + with self.subTest(close=close): + w = _window(self) + pipeline = w._pipeline + _mock(pipeline.supervisor.is_running).return_value = True + pipeline.pump_events = lambda pause_s: PipelineController.pump_events(pipeline, pause_s) + pipeline._deferred_close = w.close + dispatched = [] + + def interrupt() -> None: + dispatched.append(pipeline.stopping) + (w.close if close else pipeline.stop)() + + QTimer.singleShot(0, interrupt) + with self.assertRaises(InterruptedError): + pipeline.pause() + self.assertEqual(dispatched, [True]) + self.assertFalse(pipeline.paused) + self.assertFalse(pipeline.should_run) + _mock(pipeline.supervisor.start).assert_not_called() + if close: + self.assertTrue(w._closed) + + def test_rejected_sequence_cannot_release_another_operations_pause(self) -> None: + w = _window(self) + w._pipeline.should_run = True + with mock.patch.object(w._pipeline, "start") as start: + pause = w._preprocess._capture_pause = w._pipeline.pause() + with w._log.capture_errors() as errors: + w._preprocess.start_sequence("s21_reference") + self.assertEqual(len(errors), 1) + self.assertTrue(pause.valid) + start.assert_not_called() + w._preprocess._release_pause() + start.assert_called_once() + + def test_invalid_apply_preserves_the_current_pipeline(self) -> None: + pipeline = self.runtime() + _mock(pipeline.supervisor.is_running).return_value = True + _mock(pipeline._build_config).side_effect = ValueError("incomplete sweep") + pipeline.apply_radar() + _mock(pipeline.supervisor.stop_acquisition).assert_not_called() + _mock(pipeline.supervisor.stop_all).assert_not_called() + _mock(pipeline.start).assert_not_called() + _mock(pipeline._show_exception).assert_called_once() + + def test_multiclick_capture_owns_pause_until_abort_even_during_backoff(self) -> None: + w = _window(self) + w._pipeline.should_run = True + dialog = w._preprocess.ensure_dialog() + dialog.set_set_name("reference") + session = mock.Mock(kind="s21_reference") + session.state.return_value = SequentialCaptureState("s21_reference", "reference", 0, 1, + ComboModel(0, 0), False, False) + with mock.patch("python_app.gui.controllers.preprocess_controller.SequentialCaptureSession", return_value=session), \ + mock.patch.object(w._pipeline, "start") as start, w._log.capture_errors() as errors: + w._preprocess.start_sequence("s21_reference") + self.assertEqual(errors, []) + session.open.assert_called_once() + self.assertTrue(w._pipeline.paused) + self.assertFalse(w._pipeline.should_run) + start.assert_not_called() + w._preprocess.abort() + session.close.assert_called_once() + self.assertFalse(w._pipeline.paused) + start.assert_called_once() class GuiProfileWriteTest(unittest.TestCase): + def test_rejected_web_batch_restores_empty_combo_and_original_error(self) -> None: + window = _window(self) + self.assertEqual(window._view.switches.display_combo.currentIndex(), -1) + mode, gain = window._view.processing.mode.currentText(), widget(window._view.processing, "_bscan_gain", QDoubleSpinBox).value() + error = ValueError("Incompatible processing settings") + with self.assertRaises(ValueError) as raised: + window._config.apply_web_fields( + {"processor_mode": "bscan", "bscan_gain": gain + 1}, + mock.Mock(side_effect=error), + ) + self.assertIs(raised.exception, error) + self.assertEqual(window._view.processing.mode.currentText(), mode) + self.assertEqual(widget(window._view.processing, "_bscan_gain", QDoubleSpinBox).value(), gain) + self.assertEqual(window._view.switches.display_combo.count(), 0) + self.assertEqual(window._view.switches.display_combo.currentIndex(), -1) + + def test_all_processing_pages_and_lazy_dialog_round_trip_nondefault_values(self) -> None: + window = _window(self) + profile = window._profile.build_profile() + expected = profile.gui + assert expected is not None + for binding in GUI_FIELDS: + if not binding.spec.widget or binding.path == ("switches", "combos_text"): + continue + widget = window._config.fields[binding.spec.widget] + if isinstance(widget, QCheckBox): + value = not widget.isChecked() + elif isinstance(widget, QComboBox): + value = widget.itemText((widget.currentIndex() + 1) % widget.count()) + elif isinstance(widget, (QSpinBox, QDoubleSpinBox)): + value = widget.value() + widget.singleStep() + if value > widget.maximum(): + value = widget.value() - widget.singleStep() + value = int(value) if isinstance(widget, QSpinBox) else round(value, widget.decimals()) + elif isinstance(widget, QLineEdit): + value = "0:0" if binding.spec.web == "combo_filter" else "0" if "positions" in binding.path[-1] else "1" + else: + self.fail(f"Uncovered settings widget: {binding.path}") + binding.write(expected, value) + expected.processing.selected_mode = "pass_through" + expected.preprocess_dialog = GuiPreprocessDialogStateModel("roundtrip", str(window._profile._project_root), True, 7) + window._profile.apply(profile, window._profile.state.path) + saved = window._profile.write(window._profile._project_root / "roundtrip.json") + self.assertEqual(saved.gui, expected) + self.assertEqual(GuiProfileModel.load_from_path(window._profile._project_root / "roundtrip.json").gui, expected) + def test_dataset_profile_uses_actual_preprocessing_and_preserves_current_gui(self) -> None: for applied, selected in (("", "next_run"), ("applied", "")): with self.subTest(applied=applied), tempfile.TemporaryDirectory() as folder: pending = GuiProfileModel(gui=GuiStateModel()) + assert pending.gui is not None pending.gui.processing.bscan.max_depth_m = 7.5 pending.run_config.gpr.tx_geometry = [GprTxGeometryModel(output_pos=0, x_m=.4, y_m=.2, z_m=.1)] pending.run_config.preprocess.s21.object_calibration.set_name = selected @@ -281,6 +565,7 @@ class GuiProfileWriteTest(unittest.TestCase): active.output_switch.positions = 2 active.combos = [ComboModel(input=1, output=1)] pending.run_config.radar.serial = "pending_B" + pending.run_config.radar.model = "librevna_multi" pending.run_config.radar.sweep.points = 257 active.preprocess.s21.object_calibration.set_name = applied active.preprocess.s21.object_calibration.bundle_path = "/bundles/object" if applied else "" @@ -288,17 +573,14 @@ class GuiProfileWriteTest(unittest.TestCase): active.preprocess.s21.calibration.bundle_path = "/bundles/through" active.preprocess.s21.reference.set_name = "actual_reference" active.preprocess.s21.reference.bundle_path = "/bundles/reference" - active.preprocess.s11.calibration.open.set_name = "actual_open" - active.preprocess.s11.calibration.open.bundle_path = "/bundles/open" - active.preprocess.s11.reference.set_name = "actual_s11_reference" active.preprocess.notch.enabled = True active.preprocess.notch.bands_hz = [(2e9, 2.1e9)] - writer = AppWindowConfigProfileIOMixin() - writer._build_gui_profile = pending.clone - writer._active_run_config = active + writer = _window(self) + writer._profile.build_profile = pending.clone + writer._pipeline.active_config = active dataset_path, settings_path = Path(folder) / "dataset.json", Path(folder) / "settings.json" - saved = writer._write_gui_profile_to_path(dataset_path, dataset=True) - writer._write_gui_profile_to_path(settings_path) + saved = writer._profile.write(dataset_path, dataset=True) + writer._profile.write(settings_path) dataset = GuiProfileModel.load_from_path(dataset_path).to_dict() expected = pending.to_dict() acquired = active.to_dict() @@ -311,28 +593,6 @@ class GuiProfileWriteTest(unittest.TestCase): saved.run_config.preprocess.s21.calibration.set_name = "mutated_return_value" self.assertEqual(active.preprocess.s21.calibration.set_name, "actual_through") - def test_dataset_profile_retains_original_device_and_geometry_after_loading_another_profile(self) -> None: - acquired = GuiProfileModel() - acquired.run_config.ensure_combos() - acquired.run_config.radar.serial = "device_A" - acquired.run_config.gpr.tx_geometry = [GprTxGeometryModel(output_pos=0, x_m=-.4)] - acquired.run_config.gpr.rx_geometry = [GprRxGeometryModel(input_pos=0, x_m=.4)] - pending = acquired.clone() - pending.run_config.radar.model = "librevna_multi" - pending.run_config.radar.serial = "device_B" - pending.run_config.gpr.tx_geometry[0].x_m = 2. - pending.run_config.gpr.rx_geometry[0].x_m = -2. - writer = AppWindowConfigProfileIOMixin() - writer._build_gui_profile = pending.clone - writer._active_run_config = acquired.run_config - with tempfile.TemporaryDirectory() as folder: - path = Path(folder) / "dataset.json" - writer._write_gui_profile_to_path(path, dataset=True) - saved = GuiProfileModel.load_from_path(path) - self.assertEqual(saved.run_config.radar, acquired.run_config.radar) - self.assertEqual(saved.run_config.gpr, acquired.run_config.gpr) - self.assertEqual(saved.run_config.combos, acquired.run_config.combos) - def test_applied_geometry_tracks_successful_live_writes_only_while_acquiring(self) -> None: for running, failed_write in ((True, False), (False, False), (True, True)): with self.subTest(running=running, failed_write=failed_write): @@ -340,154 +600,117 @@ class GuiProfileWriteTest(unittest.TestCase): active.gpr.tx_geometry = [GprTxGeometryModel(output_pos=0, x_m=-.4)] active.gpr.rx_geometry = [GprRxGeometryModel(input_pos=0, x_m=.4)] original = active.clone() - writer = SimpleNamespace( - _active_run_config=active, - _supervisor=SimpleNamespace(is_running=lambda: running), - _live_config_writer=SimpleNamespace(write=mock.Mock(side_effect=OSError("disk full") if failed_write else None)), - _live_processing_config=mock.Mock(return_value=object()), - _can_reprocess_history=lambda: False, - _gpr_tx_geometry_input=SimpleNamespace(records=lambda: [dict(output_pos=0, x_m=1., y_m=2., z_m=3.)]), - _gpr_rx_geometry_input=SimpleNamespace(records=lambda: [dict(input_pos=0, x_m=-1., y_m=-2., z_m=-3.)]), - ) - if failed_write: - with self.assertRaises(OSError): - AppWindowLiveProcessingMixin._write_live_processing_config(writer) - else: - AppWindowLiveProcessingMixin._write_live_processing_config(writer) + writer = _window(self) + writer._pipeline.active_config = active + _mock(writer._pipeline.supervisor.is_running).return_value = running + with QSignalBlocker(writer._view.processing.geometries[0]), QSignalBlocker(writer._view.processing.geometries[1]): + writer._view.processing.geometries[0].set_records([dict(output_pos=0, x_m=1., y_m=2., z_m=3.)]) + writer._view.processing.geometries[1].set_records([dict(input_pos=0, x_m=-1., y_m=-2., z_m=-3.)]) + writer._live.processing_settings.clear() + with mock.patch.object(writer._live.writer, "write", side_effect=OSError("disk full") if failed_write else None) as write: + if failed_write: + with self.assertRaises(OSError): + writer._live.publish() + else: + writer._live.publish() if running and not failed_write: self.assertEqual(active.gpr.tx_geometry, [GprTxGeometryModel(0, 1., 2., 3.)]) self.assertEqual(active.gpr.rx_geometry, [GprRxGeometryModel(0, -1., -2., -3.)]) else: self.assertEqual(active.gpr, original.gpr) + if failed_write: + self.assertEqual(writer._live.processing_settings, {}) + else: + self.assertEqual(list(writer._live.processing_settings.values()), [write.call_args.args[0].to_dict()]) def test_dataset_profile_without_active_run_preserves_ui_selection(self) -> None: - writer = AppWindowConfigProfileIOMixin() + writer = _window(self) pending = GuiProfileModel() pending.run_config.preprocess.s21.object_calibration.set_name = "selected" - writer._build_gui_profile = pending.clone - writer._active_run_config = None + writer._profile.build_profile = pending.clone + writer._pipeline.active_config = None with tempfile.TemporaryDirectory() as folder: path = Path(folder) / "dataset.json" - writer._write_gui_profile_to_path(path, dataset=True) + writer._profile.write(path, dataset=True) self.assertEqual(GuiProfileModel.load_from_path(path).to_dict(), pending.to_dict()) def test_failed_profile_replace_preserves_existing_profile_and_cleans_temporary(self) -> None: - writer = AppWindowConfigProfileIOMixin() - writer._build_gui_profile = lambda: GuiProfileModel() + writer = _window(self) + writer._profile.build_profile = lambda: GuiProfileModel() with tempfile.TemporaryDirectory() as folder: path = Path(folder) / "profile.json" previous = b'{"previous": "profile"}' path.write_bytes(previous) with mock.patch("python_app.file_io.os.replace", side_effect=OSError("disk failure")): with self.assertRaisesRegex(OSError, "disk failure"): - writer._write_gui_profile_to_path(path) + writer._profile.write(path) self.assertEqual(path.read_bytes(), previous) self.assertEqual(list(path.parent.iterdir()), [path]) -class HistoryCommandRestoreTest(unittest.TestCase): - def test_malformed_or_exhausted_counter_cannot_prevent_gui_start(self) -> None: - with tempfile.TemporaryDirectory() as folder: - path = Path(folder) / "live.json" - for content in ('[]', 'null', 'invalid', '{"history_command_seq": NaN}', - '{"history_command_seq": Infinity}', '{"history_command_seq": true}', - '{"history_command_seq": 1.5}', '{"history_command_seq": -1}', - '{"history_command_seq": 18446744073709551615}'): - path.write_text(content) - with self.subTest(content=content): - self.assertEqual(AppWindow._load_history_command_seq(path), 0) - path.write_text('{"history_command_seq": 9876543210}') - self.assertEqual(AppWindow._load_history_command_seq(path), 9876543210) - - class PreprocessHistoryRestartTest(unittest.TestCase): - def _runtime(self): - w = _Runtime() + def runtime(self): + pipeline = _pipeline(self) config = RunConfigModel.from_dict({"radar": {"model": "librevna", "driver_mode": "mock"}}) - config.input_switch.driver = config.output_switch.driver = "h7992" config.input_switch.radar_port, config.output_switch.radar_port = 1, 2 config.preprocess.s21.calibration.set_name = "through" config.preprocess.s21.reference.set_name = "empty" - w._build_config.return_value = config - w._processor_run_signature = w._build_processor_run_signature(config) - w._history_run_signature = w._build_run_history_signature(config) - w._project_root = Path("/tmp/object-calibration-test") - w._store = mock.Mock() - w._store.has_combo_coverage.return_value = True - w._config_writer = mock.Mock() - w._config_writer.write.return_value = w._project_root / "runtime.json" - w._radar_key = mock.Mock(return_value="radar") - w._supervisor.pids.return_value = {} - w._bscan_reprocess_timer = mock.Mock() - for name in ("_validate_processing_mode_constraints", "_prepare_radar_for_native_acquisition", - "_close_replay_ring_writer", "_discard_bscan_replay_results", "_close_readers", - "_drop_pending_ring_payloads", "_reset_runtime_history", "_stop_all_processes", - "_write_live_processing_config"): - setattr(w, name, mock.Mock()) - return w, config + _mock(pipeline._build_config).return_value = config + pipeline._processor_signature = build_processor_run_signature(config) + pipeline.history.run_signature = build_run_history_signature(config) + pipeline._config_writer = mock.Mock() + _mock(pipeline._config_writer.write).return_value = Path("/tmp/runtime-test.json") + pipeline._validate_assets = mock.Mock() + return pipeline, config - def test_changed_preprocess_clears_native_history_before_both_start_modes(self) -> None: - for single_capture in (False, True): - with self.subTest(single_capture=single_capture): - w, config = self._runtime() - config.preprocess.s21.object_calibration.set_name = "sphere" + def test_preprocess_change_controls_history_for_both_start_modes(self) -> None: + for changed, single_capture in itertools.product((False, True), repeat=2): + with self.subTest(changed=changed, single_capture=single_capture): + pipeline, config = self.runtime() + retained = ResultCollection(1, 20, capture_start_ns=10) + pipeline.history.results.append(retained) + if changed: + config.preprocess.s21.object_calibration.set_name = "sphere" events = [] - w._write_live_processing_config.side_effect = lambda **_kwargs: events.append("clear") - w._supervisor.start.side_effect = lambda *_args, **_kwargs: events.append("start") - with mock.patch("python_app.gui.controllers.app_window_pipeline_mixin.ShmRingReader"): - w._start_run(single_capture=single_capture) - w._show_exception.assert_not_called() - w._write_live_processing_config.assert_called_once_with( - history_command="clear_all", bump_history_seq=True) - self.assertEqual(events, ["clear", "start"]) - w._reset_runtime_history.assert_called_once() - self.assertEqual(w._history_run_signature, w._build_run_history_signature(config)) - - def test_unchanged_preprocess_retains_history_and_existing_processor(self) -> None: - w, config = self._runtime() - with mock.patch("python_app.gui.controllers.app_window_pipeline_mixin.ShmRingReader"): - w._start_run(single_capture=True) - w._show_exception.assert_not_called() - w._write_live_processing_config.assert_not_called() - w._reset_runtime_history.assert_not_called() - w._stop_all_processes.assert_not_called() + _mock(pipeline.publish_live).side_effect = lambda: events.append("clear") + _mock(pipeline.supervisor.start).side_effect = lambda *_args, **_kwargs: events.append("start") + with mock.patch("python_app.gui.controllers.pipeline_controller.ShmRingReader"): + pipeline.start(single_capture=single_capture) + _mock(pipeline._show_exception).assert_not_called() + self.assertEqual(events, ["clear", "start"] if changed else ["start"]) + self.assertEqual(list(pipeline.history.results), [] if changed else [retained]) + self.assertEqual(pipeline.history.clear_before_ns > 0, changed) + _mock(pipeline.supervisor.stop_all).assert_not_called() def test_capture_override_is_cloned_and_skips_display_mode_constraints(self) -> None: - w, config = self._runtime() - w._object_capture_job = object() - with mock.patch("python_app.gui.controllers.app_window_pipeline_mixin.ShmRingReader"): - w._start_run(config_override=config) - w._show_exception.assert_not_called() - w._build_config.assert_not_called() - w._validate_processing_mode_constraints.assert_not_called() - exported = w._config_writer.prepare_preprocess_bundles.call_args.args[2] + pipeline, config = self.runtime() + pause = pipeline.pause() + with mock.patch("python_app.gui.controllers.pipeline_controller.ShmRingReader"), \ + mock.patch("python_app.gui.controllers.pipeline_controller.validate_processing_mode_constraints") as validate: + pipeline.start(config_override=config, pause=pause) + _mock(pipeline._show_exception).assert_not_called() + _mock(pipeline._build_config).assert_not_called() + validate.assert_not_called() + exported = _mock(pipeline._config_writer.write).call_args.args[0] self.assertIsNot(exported, config) self.assertFalse(config.runtime.continuous) self.assertTrue(exported.runtime.continuous) + pause.finish(resume=False) - def test_regular_start_and_other_hardware_actions_wait_for_object_capture(self) -> None: - for action, args in (("_start_run", ()), ("_capture_tmp_reference", ()), - ("_create_neutral_sets", ()), ("_start_capture_sequence", ("s21_reference",)), - ("_apply_radar_settings", ()), ("_save_snapshot", ()), - ("_save_vna_history_json", ()), ("_remove_last_runtime_history", ()), - ("_clear_all_runtime_history", ())): - with self.subTest(action=action): - w, _ = self._runtime() - w._object_capture_job = object() - getattr(w, action)(*args) - w._build_config.assert_not_called() - w._show_error.assert_called_once() - - def test_profile_load_and_recording_wait_for_object_capture_even_before_pipeline_start(self) -> None: - for action in (AppWindowConfigProfileIOMixin._ensure_ready_to_load_config, - AppWindowRecordingMixin._start_run_with_recording): - with self.subTest(action=action.__name__): - w, _ = self._runtime() - w._object_capture_job = object() - self.assertFalse(action(w)) - w._build_config.assert_not_called() - w._supervisor.is_running.assert_not_called() - w._show_error.assert_called_once() + def test_hardware_actions_wait_for_object_capture_before_pipeline_start(self) -> None: + w = _window(self) + w._preprocess.object_calibration.capture = mock.Mock() + actions = (w._pipeline.start, w._preprocess.capture_tmp_reference, + w._preprocess.create_neutral_sets, lambda: w._preprocess.start_sequence("s21_reference"), + w._pipeline.apply_radar, w._snapshots.save, partial(w._snapshots.save, json_export=True), + partial(w._snapshots.delete_history, remove_last=True), partial(w._snapshots.delete_history, remove_last=False), + w._profile._ready_to_load, partial(w._snapshots.start_recording, 1)) + for action in actions: + with self.subTest(action=action), w._log.capture_errors() as errors: + action() + self.assertEqual(len(errors), 1) + self.assertIn("object", errors[0].lower()) + _mock(w._pipeline.supervisor.start).assert_not_called() if __name__ == "__main__": diff --git a/python_app/tests/test_gui_profile_codec.py b/python_app/tests/test_gui_profile_codec.py index b6a84ad..6df1678 100644 --- a/python_app/tests/test_gui_profile_codec.py +++ b/python_app/tests/test_gui_profile_codec.py @@ -4,7 +4,7 @@ from __future__ import annotations import unittest -from python_app.models.gui_profile_model import ( +from python_app.models.gui_profile_schema import ( GuiPassThroughStateModel, GuiProcessingStateModel, GuiProfileModel, diff --git a/python_app/tests/test_hardware_lifecycle.py b/python_app/tests/test_hardware_lifecycle.py index 8d538e2..ee6b204 100644 --- a/python_app/tests/test_hardware_lifecycle.py +++ b/python_app/tests/test_hardware_lifecycle.py @@ -11,24 +11,26 @@ import threading import tty import unittest from dataclasses import dataclass +from contextlib import ExitStack from unittest.mock import Mock, patch import numpy as np from python_app.hardware_full.kamil_adc.tty_reader import KamilAdcTtyReader from python_app.hardware_full.laser_control.protocol import Protocol -from python_app.hardware_full.librevna_backends import NativeLibreVnaBackend +from python_app.hardware_full.librevna_service import LibreVnaService +from python_app.hardware_full.librevna_driver.api.vna import VNAController from python_app.hardware_full.librevna_driver.enums import PacketType -from python_app.hardware_full.librevna_driver.exceptions import DeviceDisconnectedError -from python_app.hardware_full.librevna_driver.models import Packet +from python_app.hardware_full.librevna_driver.exceptions import DeviceDisconnectedError, TimeoutError as DeviceTimeout +from python_app.hardware_full.librevna_driver.models import Packet, VNADatapointPacket from python_app.hardware_full.librevna_driver.session import LibreVNASession from python_app.hardware_full.librevna_driver.transport.usb import USBTransport from python_app.hardware_full.switch_drivers.h7992_driver import H7992Driver -from python_app.hardware_full.switch_drivers.hmc349a_driver import HMC349ADriver from python_app.hardware_full.switched_matrix_radar_service import SwitchedMatrixRadarService -from python_app.models.run_config_model import RunConfigModel -from python_app.models.dataset_model import ComboKey, TraceData -from python_app.scripts import kamil_adc_raw_producer, matrix_raw_producer, replay_raw_producer +from python_app.models.run_config_schema import RadarSweepModel, RunConfigModel +from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData +from python_app.orchestration.shm.decoder import RAW_MAGIC, decode_trace_collection +from python_app.scripts import kamil_adc_raw_producer, matrix_raw_producer, replay_raw_producer, producer_common from python_app.workflows.radar_config_variants import scan_radar_config_variants @@ -52,11 +54,13 @@ class HardwareCleanupTest(unittest.TestCase): resource.close.assert_called_once() def test_native_open_failure_allows_fresh_retry(self) -> None: - backend = NativeLibreVnaBackend(None, 14) + cleanup = ExitStack() + self.addCleanup(cleanup.close) + backend = LibreVnaService() failed, working = Mock(), Mock() failed.connect.side_effect = OSError("USB unavailable") failed.disconnect.side_effect = OSError("disconnect failed") - backend._libre_vna_device = Mock(side_effect=[failed, working]) + cleanup.enter_context(patch("python_app.hardware_full.librevna_service.LibreVNADevice", side_effect=[failed, working])) with self.assertRaisesRegex(OSError, "USB unavailable"): backend.open() self.assertFalse(backend.is_open) @@ -68,16 +72,14 @@ class HardwareCleanupTest(unittest.TestCase): self.assertFalse(backend.is_open) def test_gpio_initialization_failure_releases_lines(self) -> None: - for driver_type, module in ((H7992Driver, "h7992_driver"), (HMC349ADriver, "hmc349a_driver")): - with self.subTest(driver=driver_type.__name__): - lines = Mock(set_values=Mock(side_effect=OSError("GPIO write failed"))) - driver = driver_type("test") - with patch(f"python_app.hardware_full.switch_drivers.{module}.GpioOutputLines", return_value=lines): - with self.assertRaisesRegex(OSError, "GPIO write failed"): - driver.open() - lines.close.assert_called_once() - self.assertFalse(driver._is_open) - self.assertIsNone(driver._lines) + lines = Mock(set_values=Mock(side_effect=OSError("GPIO write failed"))) + driver = H7992Driver("test") + with patch("python_app.hardware_full.switch_drivers.h7992_driver.GpioOutputLines", return_value=lines): + with self.assertRaisesRegex(OSError, "GPIO write failed"): + driver.open() + lines.close.assert_called_once() + self.assertFalse(driver._is_open) + self.assertIsNone(driver._lines) def test_serial_connect_is_idempotent_and_failed_close_clears_state(self) -> None: connection = Mock(is_open=True) @@ -93,6 +95,37 @@ class HardwareCleanupTest(unittest.TestCase): class SessionLifecycleTest(unittest.TestCase): + def test_slow_sweep_completes_but_lost_progress_times_out_promptly(self) -> None: + for ifbw, dwell_us in ((10, 0), (1000, 100_000)): + with self.subTest(ifbw=ifbw, dwell_us=dwell_us): + session = LibreVNASession() + backend = LibreVnaService() + backend._device = Mock(vna=VNAController(session)) + elapsed, index, stalled = 0.0, 0, False + + def receive(_packet_type, *, timeout_s): + nonlocal elapsed, index + if stalled or timeout_s < 0.1: + elapsed += timeout_s + raise DeviceTimeout("No datapoint") + elapsed += 0.1 + point = VNADatapointPacket( + 1_000_000 + index, 0, index, + np.array([2.0, 1.0]), np.zeros(2), np.array([0x13, 0x02]), + ) + index += 1 + return Packet(PacketType.VNA_DATAPOINT, point) + + with patch.object(session, "send"), patch.object(session, "get_device_info", return_value=Mock(num_ports=2)), patch.object(session, "wait_for_packet", side_effect=receive), patch("python_app.hardware_full.librevna_driver.session.time.monotonic", side_effect=lambda: elapsed): + backend.configure(RadarSweepModel(points=301, if_bandwidth_hz=ifbw, dwell_us=dwell_us)) + result = backend.acquire() + self.assertEqual(len(result.x), 301) + np.testing.assert_array_equal(result.trace("s21"), np.full(301, 0.5)) + elapsed, stalled = 0.0, True + with self.assertRaises(DeviceTimeout): + backend.acquire() + self.assertLess(elapsed, 5.0) + def test_acquisition_discards_tail_before_first_point_zero(self) -> None: self._assert_complete_cycle([(1, "old"), (2, "old"), (0, "new"), (1, "new"), (2, "new")]) @@ -269,13 +302,14 @@ class ProducerCleanupTest(unittest.TestCase): config = RunConfigModel() config.radar.replay.dataset_dir = "/test/data" config.rings.raw.slot_size_bytes = 1 + config.rings.raw_tap.name = "/test_raw_tap" trace = TraceData( ComboKey(0, 0), np.array([1, 2], dtype=np.float32), - np.array([0, 0], dtype=np.complex64), np.array([1, 2], dtype=np.complex64), + np.array([1, 2], dtype=np.complex64), ) dataset = module.ReplayDataset([[trace]], [20]) raw, tap = Mock(), Mock() - with patch.object(module.RunConfigModel, "load_from_path", return_value=config), patch.object(module.signal, "signal"), patch.object(module, "_load_dataset", return_value=dataset), patch.object(module, "ShmRingWriter", side_effect=[raw, tap]), patch("sys.argv", [module.__name__, "--config", "/test/config.json"]): + with patch.object(producer_common.RunConfigModel, "load_from_path", return_value=config), patch.object(producer_common.signal, "signal"), patch.object(module, "_load_dataset", return_value=dataset), patch.object(producer_common, "ShmRingWriter", side_effect=[raw, tap]), patch("sys.argv", [module.__name__, "--config", "/test/config.json"]): with self.assertRaisesRegex(ValueError, "Replay requires .* bytes per slot"): module.main() raw.push.assert_not_called() @@ -289,8 +323,9 @@ class ProducerCleanupTest(unittest.TestCase): config = RunConfigModel() config.radar.model = model config.radar.replay.dataset_dir = "/test/data" + config.rings.raw_tap.name = "/test_raw_tap" writer = Mock() - with patch.object(module.RunConfigModel, "load_from_path", return_value=config), patch.object(module.signal, "signal"), patch.object(module, "ShmRingWriter", side_effect=[writer, OSError("second ring failed")]), patch("sys.argv", [module.__name__, "--config", "/test/config.json"]): + with patch.object(producer_common.RunConfigModel, "load_from_path", return_value=config), patch.object(producer_common.signal, "signal"), patch.object(producer_common, "ShmRingWriter", side_effect=[writer, OSError("second ring failed")]), patch("sys.argv", [module.__name__, "--config", "/test/config.json"]): if module is replay_raw_producer: with patch.object(module, "_load_dataset", return_value=Mock(median_gap_ns=1, __len__=Mock(return_value=1))): with self.assertRaisesRegex(OSError, "second ring failed"): @@ -300,6 +335,34 @@ class ProducerCleanupTest(unittest.TestCase): module.main() writer.close.assert_called_once() + def test_disabled_tap_opens_and_publishes_only_the_required_raw_ring(self) -> None: + config = RunConfigModel() + config.rings.raw_tap.name = "" + collection = SweepCollection(1, 50, capture_start_ns=10, capture_end_ns=20) + raw = Mock() + with patch.object(producer_common, "ShmRingWriter", return_value=raw) as factory, ExitStack() as cleanup: + publisher = producer_common.RawPublisher(config.rings, cleanup) + self.assertTrue(publisher.publish(collection)) + factory.assert_called_once_with(config.rings.raw.name, config.rings.raw.capacity, config.rings.raw.slot_size_bytes) + raw.close.assert_called_once() + decoded = decode_trace_collection(raw.push.call_args.args[0], RAW_MAGIC) + self.assertEqual((decoded.collection_id, decoded.capture_start_ns, decoded.capture_end_ns), (1, 10, 20)) + + def test_rejected_raw_is_not_mirrored_but_tap_rejection_does_not_reject_raw(self) -> None: + raw, tap = Mock(push=Mock(return_value=False), slot_size_bytes=64), Mock(push=Mock(return_value=False)) + config = RunConfigModel() + config.rings.raw_tap.name = "/test_raw_tap" + collection = SweepCollection(1, 50, capture_start_ns=10, capture_end_ns=20) + with patch.object(producer_common, "ShmRingWriter", side_effect=[raw, tap]), ExitStack() as cleanup: + publisher = producer_common.RawPublisher(config.rings, cleanup) + self.assertFalse(publisher.publish(collection)) + tap.push.assert_not_called() + raw.push.return_value = True + self.assertTrue(publisher.publish(collection)) + self.assertEqual(raw.push.call_args, tap.push.call_args) + raw.close.assert_called_once() + tap.close.assert_called_once() + class RadarVariantValidationTest(unittest.TestCase): def test_scan_rejects_malformed_sweeps_and_keeps_valid_overlay(self) -> None: diff --git a/python_app/tests/test_kamil_adc_processing.py b/python_app/tests/test_kamil_adc_processing.py index 6059cc1..7485fe8 100644 --- a/python_app/tests/test_kamil_adc_processing.py +++ b/python_app/tests/test_kamil_adc_processing.py @@ -9,6 +9,7 @@ accumulating forward) and lets us reason precisely about the frequency mapping. from __future__ import annotations +from dataclasses import replace import unittest import numpy as np @@ -26,7 +27,7 @@ def _reference(phase: np.ndarray, amplitude: float = 1000.0) -> np.ndarray: # Calibration shared by the processor tests: phase 0 rad -> 2 GHz, phase 100 rad # -> 4 GHz (slope 2e7 Hz/rad). Band [2.5, 3.5] GHz corresponds to phase [25, 75]. -_CALIBRATION = dict( +_CALIBRATION = KamilAdcProcessingParams( phase0_rad=0.0, freq0_hz=2_000_000_000.0, phase1_rad=100.0, @@ -39,7 +40,7 @@ _CALIBRATION = dict( class ProcessingParamsTest(unittest.TestCase): def _params(self, **overrides: float) -> KamilAdcProcessingParams: - return KamilAdcProcessingParams(**{**_CALIBRATION, **overrides}) # type: ignore[arg-type] + return replace(_CALIBRATION, **overrides) def test_slope_is_constant_from_anchors(self) -> None: self.assertAlmostEqual(self._params().hz_per_rad, 2.0e7) @@ -63,7 +64,7 @@ class ProcessingParamsTest(unittest.TestCase): class SweepProcessorTest(unittest.TestCase): def _processor(self, **overrides: float) -> KamilAdcSweepProcessor: - return KamilAdcSweepProcessor(KamilAdcProcessingParams(**{**_CALIBRATION, **overrides})) # type: ignore[arg-type] + return KamilAdcSweepProcessor(replace(_CALIBRATION, **overrides)) def test_grid_is_fixed_and_identical_across_calls(self) -> None: processor = self._processor() @@ -91,8 +92,8 @@ class SweepProcessorTest(unittest.TestCase): line = lambda ref: ((processor.reference_frequency_axis(ref) - 3.0e9) / 1.0e9) * np.abs(ref) s_short = processor.process(line(ref_short), ref_short) s_long = processor.process(line(ref_long), ref_long) - self.assertIsNotNone(s_short) - self.assertIsNotNone(s_long) + assert s_short is not None + assert s_long is not None expected = (processor.grid_hz.astype(np.float64) - 3.0e9) / 1.0e9 # Endpoint normalization would compress the longer sweep's axis and break this. np.testing.assert_allclose(s_short.real, expected, atol=1e-3) @@ -105,7 +106,7 @@ class SweepProcessorTest(unittest.TestCase): # |main| = 8 everywhere -> |S| = 8 / 4 = 2. main = 8.0 * np.exp(1j * np.angle(ref)) result = processor.process(main, ref) - self.assertIsNotNone(result) + assert result is not None np.testing.assert_allclose(np.abs(result), 2.0, atol=1e-3) def test_rejects_sweep_that_does_not_cover_band(self) -> None: @@ -118,7 +119,7 @@ class SweepProcessorTest(unittest.TestCase): processor = self._processor() ref = _reference(np.linspace(0.0, 100.0, 401)) # freq [2.0, 4.0] GHz result = processor.process(np.abs(ref).astype(np.complex128), ref) - self.assertIsNotNone(result) + assert result is not None self.assertEqual(result.shape, (11,)) self.assertEqual(result.dtype, np.complex64) np.testing.assert_allclose(np.abs(result), 1.0, atol=1e-3) # main=|ref| -> |S|=1 @@ -130,7 +131,7 @@ class SweepProcessorTest(unittest.TestCase): # S(f) = (f - 2.5 GHz) / 1 GHz -> must resample to that same line on the grid. main = ((freqs - 2.5e9) / 1.0e9) * np.abs(ref) result = processor.process(main, ref) - self.assertIsNotNone(result) + assert result is not None expected = (processor.grid_hz.astype(np.float64) - 2.5e9) / 1.0e9 np.testing.assert_allclose(result.real, expected, atol=1e-3) np.testing.assert_allclose(result.imag, 0.0, atol=1e-3) @@ -141,7 +142,7 @@ class SweepProcessorTest(unittest.TestCase): processor = self._processor(phase1_rad=-100.0) ref = _reference(np.linspace(0.0, -100.0, 401)) result = processor.process(np.abs(ref).astype(np.complex128), ref) - self.assertIsNotNone(result) + assert result is not None self.assertEqual(result.shape, (11,)) np.testing.assert_allclose(np.abs(result), 1.0, atol=1e-3) @@ -163,7 +164,7 @@ class SweepProcessorTest(unittest.TestCase): ref[10] = 0.0 # vanished reference samples must be ignored, not crash ref[200] = 0.0 result = processor.process(main, ref) - self.assertIsNotNone(result) + assert result is not None self.assertTrue(np.all(np.isfinite(result.real))) self.assertTrue(np.all(np.isfinite(result.imag))) @@ -195,7 +196,7 @@ class SweepProcessorTest(unittest.TestCase): covering = _reference(np.linspace(0.0, 100.0, 401)) self.assertIsNotNone(processor.process(np.abs(covering).astype(np.complex128), covering)) anchor_after_accept = processor._previous_anchor_rad - self.assertIsNotNone(anchor_after_accept) + assert anchor_after_accept is not None short = _reference(np.linspace(0.0, 40.0, 201)) # does not span the band self.assertIsNone(processor.process(np.ones(201, dtype=np.complex128), short)) self.assertEqual(processor._previous_anchor_rad, anchor_after_accept) @@ -210,10 +211,12 @@ class SweepProcessorTest(unittest.TestCase): ref_home = _reference(ramp_home) ref_drift = _reference(ramp_drift) self.assertIsNotNone(processor.process(np.abs(ref_home).astype(np.complex128), ref_home)) + assert processor._previous_anchor_rad is not None self.assertAlmostEqual(processor._previous_anchor_rad, 3.0, places=2) self.assertIsNotNone(processor.process(np.abs(ref_drift).astype(np.complex128), ref_drift)) # Without correction this would be ~-2.98 (one turn below); corrected it # continues smoothly from 3.0 to ~3.3. + assert processor._previous_anchor_rad is not None self.assertAlmostEqual(processor._previous_anchor_rad, 3.3, places=2) diff --git a/python_app/tests/test_kamil_adc_service.py b/python_app/tests/test_kamil_adc_service.py index 2a17899..c9e0ebe 100644 --- a/python_app/tests/test_kamil_adc_service.py +++ b/python_app/tests/test_kamil_adc_service.py @@ -21,7 +21,7 @@ from python_app.hardware_full.kamil_adc.protocol import ( MAIN_MARKER, REFERENCE_MARKER, ) -from python_app.models.run_config_model import RunConfigModel +from python_app.models.run_config_schema import RunConfigModel from python_app.orchestration.process_supervisor import ProcessSupervisor @@ -235,24 +235,16 @@ class KamilAdcConfigTest(unittest.TestCase): ) service = KamilAdcService(config) - class _UnreapableProcess: - pid = 2_000_000_000 # implausible; killpg is patched out below anyway - - def poll(self) -> None: - return None # always "alive" - - def wait(self, timeout: float | None = None) -> int: - raise subprocess.TimeoutExpired(cmd="kamil_adc_collector", timeout=timeout) - - service._process = _UnreapableProcess() # type: ignore[assignment] + process = mock.Mock(spec=subprocess.Popen, pid=2_000_000_000) + process.poll.return_value = None # always "alive" + process.wait.side_effect = subprocess.TimeoutExpired("kamil_adc_collector", timeout=3.0) + service._process = process with mock.patch("python_app.hardware_full.kamil_adc.service.os.killpg"): service.close() # must not raise def test_drain_after_switch_waits_for_fresh_sweeps(self) -> None: """After a switch change, drain must skip the configured number of freshly published sweeps before returning, so the next capture is post-switch.""" - import types - from python_app.hardware_full.kamil_adc import service as service_module with tempfile.TemporaryDirectory() as tmp_dir: @@ -272,18 +264,19 @@ class KamilAdcConfigTest(unittest.TestCase): } ) service = KamilAdcService(config) - service._reader = types.SimpleNamespace(published_count=10) # type: ignore[assignment] - service._process = types.SimpleNamespace(poll=lambda: None) # type: ignore[assignment] + reader = mock.Mock(spec=KamilAdcTtyReader, published_count=10) + service._reader = reader + service._process = mock.Mock(poll=mock.Mock(return_value=None)) # Each poll-sleep advances the published count, as the reader thread would. def _advance(_seconds: float) -> None: - service._reader.published_count += 1 + reader.published_count += 1 with mock.patch.object(service_module.time, "sleep", _advance): service.drain_after_switch(sweeps=3) # Started at 10, must have waited for at least 3 more sweeps. - self.assertGreaterEqual(service._reader.published_count, 13) + self.assertGreaterEqual(reader.published_count, 13) def test_drain_after_switch_is_noop_when_not_open(self) -> None: with tempfile.TemporaryDirectory() as tmp_dir: diff --git a/python_app/tests/test_laser_control_protocol.py b/python_app/tests/test_laser_control_protocol.py index 1a079be..0ab1b39 100644 --- a/python_app/tests/test_laser_control_protocol.py +++ b/python_app/tests/test_laser_control_protocol.py @@ -2,14 +2,16 @@ from __future__ import annotations +from contextlib import ExitStack + import unittest -from unittest.mock import patch +from unittest.mock import Mock, patch from python_app.hardware_full.laser_control.controller import DEVICE_MAIN_MESSAGE_ID, LaserController from python_app.hardware_full.laser_control.models import VariationType from python_app.hardware_full.laser_control.protocol import Protocol, TaskType from python_app.hardware_full.kamil_adc import apply_kamil_adc_laser_control -from python_app.models.run_config_model import RunConfigModel +from python_app.models.run_config_schema import RunConfigModel DEVICE_MAIN_MANUAL_HEX = "1111ff37ffa518ab000000000000000a8000000a8000ff003d2acc2c163f" @@ -35,34 +37,10 @@ class _FakeProtocol: return b"\x00\x00" if length == 2 else b"" -class _FakeLaserController: - instances: list["_FakeLaserController"] = [] - - def __init__(self, **kwargs: object) -> None: - self.kwargs = kwargs - self.calls: list[tuple[str, object]] = [] - _FakeLaserController.instances.append(self) - - def connect(self) -> bool: - self.calls.append(("connect", None)) - return True - - def reset(self) -> None: - self.calls.append(("reset", None)) - - def set_manual_mode(self, **kwargs: object) -> None: - self.calls.append(("set_manual_mode", kwargs)) - - def start_variation(self, **kwargs: object) -> None: - self.calls.append(("start_variation", kwargs)) - - def disconnect(self) -> None: - self.calls.append(("disconnect", None)) - - class LaserControlProtocolCompatibilityTest(unittest.TestCase): def setUp(self) -> None: - _FakeLaserController.instances.clear() + self.controller = Mock(spec=LaserController) + self.controller.connect.return_value = True @staticmethod def _kamil_config(laser_payload: dict[str, object]) -> RunConfigModel: @@ -133,19 +111,21 @@ class LaserControlProtocolCompatibilityTest(unittest.TestCase): def test_ld1_and_ld2_encode_current_step_identically(self) -> None: # Regression guard for the LD2 step-scale bug: both current-variation channels # must encode the step with the same scale (current_ma_to_n), like min/max. - params = dict(static_temp1=28.0, static_temp2=28.9, static_current1=33.0, static_current2=35.0, - min_value=33.0, max_value=35.0, step=0.05, time_step=50, delay_time=10, - message_id=DEVICE_MAIN_MESSAGE_ID, pi_coeff1_p=2560, pi_coeff1_i=128, - pi_coeff2_p=2560, pi_coeff2_i=128) - ld1 = Protocol.encode_task_enable(task_type=TaskType.CHANGE_CURRENT_LD1, **params) - ld2 = Protocol.encode_task_enable(task_type=TaskType.CHANGE_CURRENT_LD2, **params) + ld1, ld2 = [Protocol.encode_task_enable( + task_type=task_type, static_temp1=28.0, static_temp2=28.9, + static_current1=33.0, static_current2=35.0, min_value=33.0, max_value=35.0, + step=0.05, time_step=50, delay_time=10, message_id=DEVICE_MAIN_MESSAGE_ID, + pi_coeff1_p=2560, pi_coeff1_i=128, pi_coeff2_p=2560, pi_coeff2_i=128, + ) for task_type in (TaskType.CHANGE_CURRENT_LD1, TaskType.CHANGE_CURRENT_LD2)] # Word 5 (step) is at bytes 10:12 in both frames (sync, header, task, min, max, step). self.assertEqual(ld1[10:12], ld2[10:12]) def test_start_sequence_matches_device_main_order(self) -> None: + cleanup = ExitStack() + self.addCleanup(cleanup.close) fake_protocol = _FakeProtocol() controller = LaserController(pi_coeff1_p=2560, pi_coeff1_i=128, pi_coeff2_p=2560, pi_coeff2_i=128) - controller._protocol = fake_protocol + cleanup.enter_context(patch.object(controller, "_protocol", fake_protocol)) with patch("python_app.hardware_full.laser_control.controller.time.sleep", return_value=None): controller.reset() @@ -181,9 +161,11 @@ class LaserControlProtocolCompatibilityTest(unittest.TestCase): ) def test_stop_sequence_restores_device_main_manual_bytes(self) -> None: + cleanup = ExitStack() + self.addCleanup(cleanup.close) fake_protocol = _FakeProtocol() controller = LaserController(pi_coeff1_p=2560, pi_coeff1_i=128, pi_coeff2_p=2560, pi_coeff2_i=128) - controller._protocol = fake_protocol + cleanup.enter_context(patch.object(controller, "_protocol", fake_protocol)) with patch("python_app.hardware_full.laser_control.controller.time.sleep", return_value=None): controller.set_manual_mode( @@ -229,16 +211,16 @@ class LaserControlProtocolCompatibilityTest(unittest.TestCase): } ) - with patch("python_app.hardware_full.laser_control.controller.LaserController", _FakeLaserController): + with patch("python_app.hardware_full.laser_control.controller.LaserController", return_value=self.controller): applied = apply_kamil_adc_laser_control(config) self.assertTrue(applied) - controller = _FakeLaserController.instances[0] + controller = self.controller self.assertEqual( - [name for name, _payload in controller.calls], + [call[0] for call in controller.method_calls], ["connect", "reset", "set_manual_mode", "start_variation", "disconnect"], ) - manual_payload = controller.calls[2][1] + manual_payload = controller.set_manual_mode.call_args.kwargs self.assertEqual( manual_payload, { @@ -250,7 +232,7 @@ class LaserControlProtocolCompatibilityTest(unittest.TestCase): }, ) # The variation handoff itself (type + params), not just call order, must be right. - variation_payload = controller.calls[3][1] + variation_payload = controller.start_variation.call_args.kwargs self.assertEqual(variation_payload["variation_type"], VariationType.CHANGE_CURRENT_LD1) self.assertEqual(variation_payload["params"]["step"], 0.05) self.assertEqual(variation_payload["params"]["min_value"], 33.0) @@ -265,30 +247,30 @@ class LaserControlProtocolCompatibilityTest(unittest.TestCase): "manual": {"temp1": 26.0, "temp2": 27.0, "current1": 30.0, "current2": 31.0}, } ) - with patch("python_app.hardware_full.laser_control.controller.LaserController", _FakeLaserController): + with patch("python_app.hardware_full.laser_control.controller.LaserController", return_value=self.controller): applied = apply_kamil_adc_laser_control(config) self.assertTrue(applied) - controller = _FakeLaserController.instances[0] - self.assertEqual([name for name, _ in controller.calls], ["connect", "reset", "set_manual_mode", "disconnect"]) - self.assertEqual(controller.calls[2][1]["current1"], 30.0) + controller = self.controller + self.assertEqual([call[0] for call in controller.method_calls], ["connect", "reset", "set_manual_mode", "disconnect"]) + self.assertEqual(controller.set_manual_mode.call_args.kwargs["current1"], 30.0) def test_apply_radar_unknown_variation_type_raises(self) -> None: config = self._kamil_config( {"enabled": True, "port": "/dev/ttyUSB0", "mode": "variation", "variation": {"variation_type": "NOT_A_REAL_TYPE"}} ) - with patch("python_app.hardware_full.laser_control.controller.LaserController", _FakeLaserController): + with patch("python_app.hardware_full.laser_control.controller.LaserController", return_value=self.controller): with self.assertRaisesRegex(ValueError, "variation_type"): apply_kamil_adc_laser_control(config) def test_apply_radar_skips_disabled_laser_control(self) -> None: config = self._kamil_config({"enabled": False}) - with patch("python_app.hardware_full.laser_control.controller.LaserController", _FakeLaserController): + with patch("python_app.hardware_full.laser_control.controller.LaserController", return_value=self.controller) as factory: applied = apply_kamil_adc_laser_control(config) self.assertFalse(applied) - self.assertEqual(_FakeLaserController.instances, []) + factory.assert_not_called() if __name__ == "__main__": diff --git a/python_app/tests/test_laser_temp_monitoring.py b/python_app/tests/test_laser_temp_monitoring.py index 54a2c0e..6d6ac5e 100644 --- a/python_app/tests/test_laser_temp_monitoring.py +++ b/python_app/tests/test_laser_temp_monitoring.py @@ -5,6 +5,7 @@ from __future__ import annotations import tempfile import threading import unittest +from unittest import mock from dataclasses import dataclass from pathlib import Path from typing import List, Optional @@ -134,6 +135,7 @@ class MonitorTest(unittest.TestCase): assert reading is not None self.assertEqual(reading.seq, 7) self.assertAlmostEqual(reading.temp1, 28.01) + assert reading.temp_ext1 is not None and reading.current2 is not None self.assertAlmostEqual(reading.temp_ext1, 22.0) self.assertAlmostEqual(reading.current2, 35.0) @@ -144,26 +146,19 @@ class MonitorTest(unittest.TestCase): ]) stop = threading.Event() - class _OneShotWriter: - def __init__(self) -> None: - self.written: List[TemperatureReading] = [] - - def write(self, reading: TemperatureReading) -> None: - self.written.append(reading) - stop.set() # stop after the first publish - - writer = _OneShotWriter() + writer = mock.Mock(spec=ReadingWriter) + writer.write.side_effect = lambda _reading: stop.set() monitor = LaserTemperatureMonitor(controller, writer, period_s=0.0) monitor.run(stop) - self.assertEqual(len(writer.written), 1) - self.assertEqual(writer.written[0].seq, 0) + writer.write.assert_called_once() + self.assertEqual(writer.write.call_args.args[0].seq, 0) class CheckerTest(unittest.TestCase): - def _checker(self, **kwargs: object) -> LaserTemperatureChecker: + def _checker(self, reminder_every: int = 0) -> LaserTemperatureChecker: return LaserTemperatureChecker(target_temp1=28.0, target_temp2=28.9, - tolerance_c=0.03, **kwargs) + tolerance_c=0.03, reminder_every=reminder_every) def _reading(self, t1: float, t2: float, seq: int = 0) -> TemperatureReading: return TemperatureReading(seq=seq, mono_ns=seq, temp1=t1, temp2=t2) diff --git a/python_app/tests/test_librevna_multidevice.py b/python_app/tests/test_librevna_multidevice.py index e134e97..9b347f3 100644 --- a/python_app/tests/test_librevna_multidevice.py +++ b/python_app/tests/test_librevna_multidevice.py @@ -11,6 +11,10 @@ from __future__ import annotations import struct import threading import unittest +from unittest.mock import Mock, patch +from collections.abc import Callable + +from python_app.models.dataset_model import SweepCollection from python_app.hardware_full.librevna_driver.enums import PacketType as NativePacketType from python_app.hardware_full.librevna_driver.models import Packet @@ -21,8 +25,8 @@ from python_app.hardware_full.librevna_multi_device_driver.cycle_collection impo _CrossDeviceCycleSynchronizer, _expected_sweep_cycle_seconds, _is_transient_collection_error, - build_forward_s_parameter_names, - calculate_last_cycle_s_parameters, + calculate_s_parameters, + collect_complete_running_sweep_cycle, ) from python_app.hardware_full.librevna_multi_device_driver.exceptions import TransientCollectionError from python_app.hardware_full.librevna_multi_device_driver.models import ( @@ -38,9 +42,8 @@ from python_app.hardware_full.librevna_multi_device_driver.protocol import ( from python_app.hardware_full.multi_device_service import ( _MAX_TRANSIENT_COLLECTION_RETRIES, MultiDeviceLibreVnaService, - _build_s_parameter_layout, ) -from python_app.models.run_config_model import RadarSweepModel +from python_app.models.run_config_schema import RadarSweepModel def _control_frame(packet_type: NativePacketType, payload: bytes) -> bytes: @@ -121,6 +124,42 @@ class FrameScannerResyncTest(unittest.TestCase): self.assertGreater(scanner.discarded_byte_count, 0) +class CaptureWindowTest(unittest.TestCase): + def test_native_capture_window_excludes_discarded_tail_and_queue_drain(self) -> None: + for first_point in (1, 3): + with self.subTest(first_point=first_point): + clock = threading.local() + configuration = SweepConfiguration(1_000_000, 2_000_000, 5, 1000, -10.0) + + def connection(serial): + sequence = iter(enumerate([*range(first_point, 5), *range(5)])) + + def receive(*, timeout_seconds): + offset, point = next(sequence) + clock.ns = 1_000_000_000 + offset * 20_000_000 + values = [(0x13, 2 + 0j), (0x01, 1 + 0j), (0x02, 1 + 0j), + (0x33, 2 + 0j), (0x21, 1 + 0j), (0x22, 1 + 0j)] + return 27, _datapoint_frame(1_000_000 + point * 250_000, point, values)[4:-4] + + return Mock(serial_number=serial, receive_packet=receive) + + master, slave1, slave2 = [connection(serial) for serial in ("M", "A", "B")] + service = MultiDeviceLibreVnaService("M", ["A", "B"]) + service.configure(RadarSweepModel(start_hz=1_000_000, stop_hz=2_000_000, points=5, if_bandwidth_hz=1000)) + service._controller = Mock(collect_running_sweep_cycle=lambda *_args, **_kwargs: collect_complete_running_sweep_cycle( + master_device_connection=master, slave_device_connections=[slave1, slave2], + active_sweep_configuration=configuration, master_stimulus_ports=(1, 2), + )) + with patch("python_app.hardware_full.multi_device_service.time.monotonic_ns", side_effect=lambda: getattr(clock, "ns", 1)): + collection = service.acquire_collection(7) + start = 1_000_000_000 + (5 - first_point) * 20_000_000 + self.assertEqual((collection.capture_start_ns, collection.capture_end_ns), (start, start + 80_000_000)) + self.assertEqual(len(collection.traces), 8) + for trace in collection.traces: + self.assertEqual((trace.capture_start_ns, trace.capture_end_ns), (start, start + 80_000_000)) + np.testing.assert_array_equal(trace.s21, np.full(5, 0.5)) + + class CrossDeviceCycleSynchronizerTest(unittest.TestCase): """N1: cycle-0 anchors are accepted only when all devices saw the same wrap.""" @@ -193,6 +232,7 @@ class DatapointParseHardeningTest(unittest.TestCase): def test_valid_payload_parses(self) -> None: parsed = parse_vna_datapoint_payload(self._payload(7, [(0x00, 1 + 2j), (0x10, 3 + 4j)])) self.assertIsNotNone(parsed) + assert parsed is not None self.assertEqual(parsed.point_index, 7) self.assertEqual(parsed.frequency_hz, 3_000_000_000) self.assertEqual(set(parsed.receiver_values_by_description_mask), {0x00, 0x10}) @@ -235,83 +275,26 @@ class CycleAlignmentToleranceCalibrationTest(unittest.TestCase): self.assertAlmostEqual(dual, 2.0 * single, places=6) -class LastCycleSParameterTest(unittest.TestCase): - """Reference/measurement division, gap zero-filling, and the over-loss guard. - - A handful of dropped USB frames (points that never arrived, left as ``NaN``) - are emitted as zero so the cycle is kept instead of stalling on recovery, while - delivered points keep their true normalized value. Losing more than - ``_MAX_ZERO_FILLED_POINTS_FRACTION`` of a trace is treated as a genuine fault - and rejected as transient. The tiny-trace cases below cross that fraction with a - single missing point, so they still raise. - """ +class SParameterTest(unittest.TestCase): + """Normalize delivered samples, zero isolated losses, reject excessive loss.""" def test_divides_measurement_by_reference(self) -> None: - reference = {1: np.array([[2 + 0j, 0 + 2j, 4 + 0j]], dtype=complex)} - measurement = {"S31": np.array([[4 + 0j, 0 + 4j, 2 + 0j]], dtype=complex)} - result = calculate_last_cycle_s_parameters(measurement, reference) - np.testing.assert_allclose(result["s31"], np.array([2 + 0j, 2 + 0j, 0.5 + 0j])) + result = calculate_s_parameters( + {(0, 0): np.array([4, 4j, 2])}, {0: np.array([2, 2j, 4])}) + np.testing.assert_allclose(result[(0, 0)], [2, 2, .5]) - def test_single_dropped_measurement_is_zero_filled(self) -> None: - # One missing point in a realistically sized trace (< the loss fraction) is - # emitted as zero; every delivered point keeps its real value, no extras. - point_count = 200 - reference = {1: np.full((1, point_count), 2 + 0j, dtype=complex)} - measurement = {"S31": np.full((1, point_count), 4 + 0j, dtype=complex)} - measurement["S31"][0, 100] = np.nan + 1j * np.nan - result = calculate_last_cycle_s_parameters(measurement, reference) - self.assertEqual(result["s31"][100], 0) - np.testing.assert_allclose(np.delete(result["s31"], 100), 2 + 0j) - - def test_single_dropped_reference_is_zero_filled(self) -> None: - # A missing master reference cannot normalize that point for any trace, so - # the point is zeroed; the rest of the trace is untouched. - point_count = 200 - reference = {1: np.full((1, point_count), 2 + 0j, dtype=complex)} - reference[1][0, 50] = np.nan + 1j * np.nan - measurement = {"S31": np.full((1, point_count), 4 + 0j, dtype=complex)} - result = calculate_last_cycle_s_parameters(measurement, reference) - self.assertEqual(result["s31"][50], 0) - np.testing.assert_allclose(np.delete(result["s31"], 50), 2 + 0j) - - def test_single_near_zero_reference_is_zero_filled(self) -> None: - # A ~zero incident reference would explode the division into noise; one such - # point is zeroed rather than aborting the whole cycle, and stays finite. - point_count = 200 - reference = {1: np.full((1, point_count), 2 + 0j, dtype=complex)} - reference[1][0, 75] = 0 + 0j - measurement = {"S31": np.full((1, point_count), 4 + 0j, dtype=complex)} - result = calculate_last_cycle_s_parameters(measurement, reference) - self.assertEqual(result["s31"][75], 0) - self.assertTrue(np.isfinite(result["s31"]).all()) - - def test_excessive_loss_is_transient(self) -> None: - # Losing more than the tolerated fraction is a real fault, not a stray - # dropped frame: reject as transient so the caller recovers properly. - point_count = 200 - reference = {1: np.full((1, point_count), 2 + 0j, dtype=complex)} - measurement = {"S31": np.full((1, point_count), 4 + 0j, dtype=complex)} - measurement["S31"][0, :40] = np.nan + 1j * np.nan # 20% > 5% tolerance - with self.assertRaises(TransientCollectionError): - calculate_last_cycle_s_parameters(measurement, reference) - - def test_nan_reference_is_transient(self) -> None: - reference = {1: np.array([[2 + 0j, np.nan + 1j * np.nan, 4 + 0j]], dtype=complex)} - measurement = {"S31": np.array([[4 + 0j, 4 + 0j, 2 + 0j]], dtype=complex)} - with self.assertRaises(TransientCollectionError): - calculate_last_cycle_s_parameters(measurement, reference) - - def test_nan_measurement_is_transient(self) -> None: - reference = {1: np.array([[2 + 0j, 2 + 0j, 4 + 0j]], dtype=complex)} - measurement = {"S31": np.array([[4 + 0j, np.nan + 1j * np.nan, 2 + 0j]], dtype=complex)} - with self.assertRaises(TransientCollectionError): - calculate_last_cycle_s_parameters(measurement, reference) - - def test_near_zero_reference_is_transient(self) -> None: - reference = {1: np.array([[2 + 0j, 0 + 0j, 4 + 0j]], dtype=complex)} - measurement = {"S31": np.array([[4 + 0j, 4 + 0j, 2 + 0j]], dtype=complex)} - with self.assertRaises(TransientCollectionError): - calculate_last_cycle_s_parameters(measurement, reference) + def test_lost_measurement_or_unusable_reference(self) -> None: + for source, value in (("measurement", np.nan), ("reference", np.nan), ("reference", 0)): + for lost in (1, 40): + with self.subTest(source=source, value=value, lost=lost): + reference, measurement = np.full(200, 2 + 0j), np.full(200, 4 + 0j) + (measurement if source == "measurement" else reference)[:lost] = value + if lost == 40: + with self.assertRaises(TransientCollectionError): + calculate_s_parameters({(0, 0): measurement}, {0: reference}) + else: + result = calculate_s_parameters({(0, 0): measurement}, {0: reference})[(0, 0)] + np.testing.assert_array_equal(result, np.r_[0, np.full(199, 2)]) class TransientErrorClassificationTest(unittest.TestCase): @@ -333,11 +316,15 @@ class TransientErrorClassificationTest(unittest.TestCase): class _StubService(MultiDeviceLibreVnaService): """Non-slots subclass so a test can inject collection behaviour + count reopens.""" - def _acquire_native_collection(self, collection_id: int, capture_start_ns: int): # type: ignore[override] - self.collection_calls += 1 - return self.behavior(collection_id, capture_start_ns) + behavior: Callable[[int], SweepCollection] + collection_calls: int = 0 + reopen_calls: int = 0 - def recover(self, **_kwargs) -> None: # type: ignore[override] + def _acquire_native_collection(self, collection_id: int) -> SweepCollection: + self.collection_calls += 1 + return self.behavior(collection_id) + + def recover(self, **_kwargs) -> None: self.reopen_calls += 1 @@ -345,41 +332,43 @@ class TransientRetryTierTest(unittest.TestCase): """F1/F3: transient failures retry in place (no USB reopen); fatals escalate.""" @staticmethod - def _stub(behavior) -> _StubService: - service = _StubService(master_serial="M", slave_serials=["A", "B"], backend_mode="native") + def _stub(behavior: Callable[[int], SweepCollection]) -> _StubService: + service = _StubService(master_serial="M", slave_serials=["A", "B"]) service.behavior = behavior service.collection_calls = 0 service.reopen_calls = 0 return service def test_succeeds_after_transient_failures_without_reopen(self) -> None: - def flaky(_collection_id: int, _capture_start_ns: int) -> str: + collection = SweepCollection(1, 1, []) + + def flaky(_collection_id: int) -> SweepCollection: if service.collection_calls <= _MAX_TRANSIENT_COLLECTION_RETRIES: raise TransientCollectionError("dropped datapoint") - return "collection" + return collection service = self._stub(flaky) - result = service._acquire_native_collection_with_transient_retries(1, 0) - self.assertEqual(result, "collection") + result = service._acquire_native_collection_with_transient_retries(1) + self.assertIs(result, collection) self.assertEqual(service.collection_calls, _MAX_TRANSIENT_COLLECTION_RETRIES + 1) self.assertEqual(service.reopen_calls, 0) # never reopened USB for transient errors def test_exhausted_transient_retries_raise_transient(self) -> None: - def always_transient(_collection_id: int, _capture_start_ns: int) -> str: + def always_transient(_collection_id: int) -> SweepCollection: raise TransientCollectionError("persistent drop") service = self._stub(always_transient) with self.assertRaises(TransientCollectionError): - service._acquire_native_collection_with_transient_retries(1, 0) + service._acquire_native_collection_with_transient_retries(1) self.assertEqual(service.collection_calls, _MAX_TRANSIENT_COLLECTION_RETRIES + 1) def test_fatal_error_is_not_retried_in_place(self) -> None: - def fatal(_collection_id: int, _capture_start_ns: int) -> str: + def fatal(_collection_id: int) -> SweepCollection: raise RuntimeError("USB transport failed") service = self._stub(fatal) with self.assertRaises(RuntimeError): - service._acquire_native_collection_with_transient_retries(1, 0) + service._acquire_native_collection_with_transient_retries(1) self.assertEqual(service.collection_calls, 1) # fatal escalates immediately, no in-place retry @@ -435,35 +424,6 @@ class PerPointStimulusSwitchingTest(unittest.TestCase): with self.assertRaises(ValueError): build_output_channels((1, 2), bad) - def test_trace_names_do_not_collide_and_stay_stable(self) -> None: - two_output_names = build_forward_s_parameter_names(2, build_output_channels((1, 2), 1)) - self.assertEqual( - two_output_names, - ["S11", "S31", "S41", "S51", "S61", "S22", "S32", "S42", "S52", "S62"], - ) - four_output_names = build_forward_s_parameter_names(2, build_output_channels((1, 2), 2)) - # Receivers move to ports 5..8 so no forward term can be named like a - # reflection (a fixed base of 3 would make receiver 3 <- output 3 collide - # with the reflection of output 3). - self.assertEqual(len(four_output_names), len(set(four_output_names))) - self.assertEqual(len(four_output_names), 4 * 5) - for reflection in ("S11", "S22", "S33", "S44"): - self.assertIn(reflection, four_output_names) - - def test_service_and_collector_agree_on_trace_names(self) -> None: - # The service looks these up by name in whatever the collector produced, so a - # silent divergence would surface only as a missing-trace failure on hardware. - for substage_count in (1, 2): - channels = build_output_channels((1, 2), substage_count) - produced = {name.lower() for name in build_forward_s_parameter_names(2, channels)} - forward_by_output, reflection_by_output = _build_s_parameter_layout( - len(channels), slave_count=2 - ) - requested = set(reflection_by_output.values()) - for names in forward_by_output.values(): - requested.update(names) - self.assertEqual(requested, produced, msg=f"substages={substage_count}") - def test_sweep_settings_carry_stages_and_substage_width(self) -> None: configuration = SweepConfiguration( start_hz=1_000_000, stop_hz=2_000_000, points=101, if_bandwidth=1000, power_dbm=-10.0 @@ -517,7 +477,7 @@ class PerPointStimulusSwitchingTest(unittest.TestCase): def test_service_reports_four_outputs_when_switching(self) -> None: service = MultiDeviceLibreVnaService( - master_serial="M", slave_serials=["A", "B"], backend_mode="native", switch_substage_count=2 + master_serial="M", slave_serials=["A", "B"], switch_substage_count=2 ) self.assertEqual(service.output_position_count, 4) diff --git a/python_app/tests/test_live_settings_regressions.py b/python_app/tests/test_live_settings_regressions.py new file mode 100644 index 0000000..a8c04f5 --- /dev/null +++ b/python_app/tests/test_live_settings_regressions.py @@ -0,0 +1,94 @@ +"""Settings reach the processor atomically and preserve the saved frequency band.""" + +from copy import deepcopy +import unittest +from unittest import mock + +from PyQt6.QtWidgets import QApplication, QComboBox, QDoubleSpinBox, QStackedWidget + +from python_app.gui.antenna_geometry_editor import AntennaGeometryEditor +from python_app.gui.controllers.sections.processing_section import build_processing_group +from python_app.models.gui_profile_schema import GuiStateModel +from python_app.models.dataset_model import SweepCollection +from python_app.models.run_config_schema import RunConfigModel +from python_app.tests.gui_test_support import isolated_window, widget + + +class LiveSettingsRegressionTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.app = QApplication.instance() or QApplication([]) + + def test_initial_processing_band_uses_sweep_limits_before_profile_values(self) -> None: + config, gui = RunConfigModel(), GuiStateModel() + config.radar.sweep.start_hz, config.radar.sweep.stop_hz = 1e6, 9.5e9 + for settings in (gui.processing.bscan, gui.processing.gpr, + gui.processing.legacy_gpr, gui.processing.sar3d): + settings.start_freq_mhz, settings.stop_freq_mhz = 20, 9200 + section = build_processing_group(config, gui) + self.addCleanup(section.group.deleteLater) + for mode in ("bscan", "gpr", "legacy_gpr", "sar3d"): + for bound, expected in (("start", 20), ("stop", 9200)): + with self.subTest(mode=mode, bound=bound): + editor = widget(section, f"_{mode}_{bound}_freq_mhz", QDoubleSpinBox) + self.assertEqual((editor.minimum(), editor.maximum(), editor.value()), (1, 9500, expected)) + + def test_failed_live_write_restores_web_batch_and_desktop_changes(self) -> None: + window = isolated_window(self.addCleanup) + window._pipeline.timer.stop() + mode = window._view.processing.mode + gain = widget(window._view.processing, "_bscan_gain", QDoubleSpinBox) + pages = window._view.processing.pages + live = window._live + before = (window._profile.build_gui_state(), window._config.geometry(), + live.active_mode, pages.currentIndex(), window._plots.stack.currentWidget(), + live.revision, deepcopy(live.processing_settings), window._plots.required_revision, + live.writer.path.read_bytes()) + for source in ("web", "web_geometry", "desktop_mode", "desktop_value", "desktop_geometry"): + with self.subTest(source=source), \ + mock.patch.object(live.writer, "write", side_effect=OSError("disk full")), \ + mock.patch.object(window._preprocess, "invalidate_object_calibration_after_geometry_change") as invalidate, \ + mock.patch.object(window._log, "show_exception") as error: + if source == "web": + live.apply_web({"processor_mode": "bscan", "bscan_gain": gain.value() + .1}) + elif source in {"web_geometry", "desktop_geometry"}: + editor = window._view.processing.geometries[0] + records = editor.records() + records[0]["x_m"] += .125 + if source == "web_geometry": + live.apply_web({"tx_geometry": records}) + else: + editor.set_records(records) + elif source == "desktop_mode": + mode.setCurrentText("bscan") + else: + widget(window._view.processing, "_gpr_range_comp_power", QDoubleSpinBox).setValue(.4) + error.assert_called_once() + invalidate.assert_not_called() + self.assertEqual((window._profile.build_gui_state(), window._config.geometry(), + live.active_mode, pages.currentIndex(), window._plots.stack.currentWidget(), + live.revision, live.processing_settings, window._plots.required_revision, + live.writer.path.read_bytes()), before) + with mock.patch.object(window._plots, "draw_bscan", side_effect=ValueError("invalid plot")), \ + mock.patch.object(window._log, "show_exception") as error: + live.apply_web({"processor_mode": "bscan", "bscan_gain": gain.value() + .1}) + error.assert_called_once() + self.assertEqual((mode.currentText(), live.active_mode, pages.currentIndex()), ("bscan", "bscan", 1)) + self.assertEqual(live.processing_settings[live.revision]["processor_mode"], "bscan") + self.assertNotEqual(live.writer.path.read_bytes(), before[-1]) + + def test_mode_publication_selects_replay_from_new_mode_before_changing_the_plot(self) -> None: + window = isolated_window(self.addCleanup) + window._pipeline.timer.stop() + window._history.preprocessed.append(SweepCollection(1, 1, [], capture_start_ns=1, capture_end_ns=2)) + with mock.patch.object(window._pipeline.supervisor, "is_processor_running", return_value=True), \ + mock.patch.object(window._pipeline, "_result_reader"), \ + mock.patch.object(window._pipeline, "drain_results", return_value=None), \ + mock.patch.object(window._pipeline, "schedule_replay"), \ + mock.patch.object(window._plots, "draw_bscan"): + for mode, reprocess in (("bscan", False), ("pass_through", True)): + with self.subTest(mode=mode): + window._live.apply_web({"processor_mode": mode}) + published = window._live.processing_settings[window._live.revision] + self.assertEqual(published["processor_mode"], mode) + self.assertEqual(published["reprocess_current_result"], reprocess) diff --git a/python_app/tests/test_live_settings_schema.py b/python_app/tests/test_live_settings_schema.py index 62b42b9..c5beb31 100644 --- a/python_app/tests/test_live_settings_schema.py +++ b/python_app/tests/test_live_settings_schema.py @@ -1,60 +1,33 @@ -"""Guard tests keeping the live-settings schema and the config builder in lock-step. +"""Every native live setting has one declaration or an explicit runtime source.""" -`_live_processing_config` builds ProcessingLiveConfig by reading every widget through -`_WEB_LIVE_SCHEMA` (the same map the embedded web form renders from), plus a small set -of fields set explicitly (`_NON_WIDGET_LIVE_FIELDS`). These tests fail the moment a new -config field is added without being placed in one of those two sets — which is exactly -the drift that previously hid GPR settings from the web form. -""" - -from __future__ import annotations - -import dataclasses as dc +from dataclasses import fields import unittest -from python_app.gui.controllers.app_window_config.live_processing_mixin import ( - _NON_WIDGET_LIVE_FIELDS, - _WEB_DISPLAY_SCHEMA, - _WEB_LIVE_GETTERS, - web_apply_field_names, -) +from python_app.models.setting_fields import GUI_FIELDS, active_live_fields, active_web_fields from python_app.orchestration.live_processing_config import ProcessingLiveConfig class LiveSettingsSchemaCoverageTest(unittest.TestCase): - @staticmethod - def _config_fields() -> set[str]: - return {field.name for field in dc.fields(ProcessingLiveConfig)} - - def test_schema_plus_non_widget_covers_every_config_field(self) -> None: - schema_fields = set(_WEB_LIVE_GETTERS) - covered = schema_fields | _NON_WIDGET_LIVE_FIELDS - config_fields = self._config_fields() - self.assertEqual( - covered, - config_fields, - msg=( - f"uncovered config fields: {sorted(config_fields - covered)}; " - f"stray names: {sorted(covered - config_fields)}" - ), - ) - - def test_widget_and_non_widget_sets_are_disjoint(self) -> None: - # A field is either widget-backed (in the schema) or explicitly non-widget — never both. - self.assertEqual(set(_WEB_LIVE_GETTERS) & _NON_WIDGET_LIVE_FIELDS, set()) - - def test_display_fields_are_not_live_config(self) -> None: - extra = {f for f, *_ in _WEB_DISPLAY_SCHEMA} - self.assertEqual(extra & self._config_fields(), set()) - self.assertEqual(extra & set(_WEB_LIVE_GETTERS), set()) - - def test_web_apply_names_cover_the_whole_form(self) -> None: - form_fields = ( - set(_WEB_LIVE_GETTERS) - | {f for f, *_ in _WEB_DISPLAY_SCHEMA} - ) - # The web may apply every form field plus the history command, and nothing else. - self.assertEqual(web_apply_field_names(), form_fields | {"history_command"}) + def test_bindings_cover_live_config_and_select_each_page_once(self) -> None: + config_names = {field.name for field in fields(ProcessingLiveConfig)} + runtime = {"reprocess_current_result", "history_clear_before_ns", "history_removed", "settings_revision"} + geometry = {"relative_permittivity", "tx_geometry", "rx_geometry"} + modes = next(binding.spec.choices for binding in GUI_FIELDS if binding.spec.live == "processor_mode") + widget_names = [binding.spec.widget for binding in GUI_FIELDS if binding.spec.widget] + self.assertEqual(len(widget_names), len(set(widget_names))) + for mode in modes: + with self.subTest(mode=mode): + live = active_live_fields(mode) + live_names = {binding.spec.live for binding in live} + self.assertEqual(len(live), len(live_names)) + self.assertEqual(live_names | geometry | runtime, config_names) + self.assertFalse(live_names & (geometry | runtime)) + web = active_web_fields(mode) + web_names = {binding.spec.web for binding in web} + self.assertEqual(len(web), len(web_names)) + self.assertIn("processor_mode", web_names) + self.assertTrue(all(binding.mode in (None, mode) for binding in web)) + self.assertFalse({binding.spec.web for binding in web if not binding.spec.live} & config_names) if __name__ == "__main__": diff --git a/python_app/tests/test_locator_client.py b/python_app/tests/test_locator_client.py deleted file mode 100644 index c534eed..0000000 --- a/python_app/tests/test_locator_client.py +++ /dev/null @@ -1,36 +0,0 @@ -"""The locator test client's framing survives fragmented timeout polls.""" - -import socket -import threading -import unittest -from unittest.mock import Mock - -from locator_test_client import encode_packet, read_packet, recv_exactly - - -class LocatorClientTest(unittest.TestCase): - def test_partial_header_and_payload_survive_timeouts(self) -> None: - payload = {"vlc": 5.5} - packet = encode_packet(payload, 7) - sock = Mock() - sock.recv.side_effect = [packet[:3], socket.timeout(), packet[3:8], - packet[8:10], socket.timeout(), packet[10:]] - self.assertEqual(read_packet(sock), (7, payload)) - - def test_timeout_wait_can_be_stopped_without_losing_thread(self) -> None: - stop = threading.Event() - sock = Mock() - - def receive(_size): - stop.set() - raise socket.timeout() - - sock.recv.side_effect = receive - with self.assertRaisesRegex(ConnectionError, "stopped"): - recv_exactly(sock, 8, stop) - - def test_eof_in_the_middle_of_packet_is_reported(self) -> None: - sock = Mock() - sock.recv.side_effect = [b"123", b""] - with self.assertRaisesRegex(ConnectionError, "closed"): - read_packet(sock) diff --git a/python_app/tests/test_log_panel_buffer.py b/python_app/tests/test_log_panel_buffer.py index 86d887d..5b6c730 100644 --- a/python_app/tests/test_log_panel_buffer.py +++ b/python_app/tests/test_log_panel_buffer.py @@ -9,7 +9,7 @@ from __future__ import annotations import unittest -from python_app.gui.app_window import _PanelLogBuffer +from python_app.gui.controllers.log_controller import _PanelLogBuffer class PanelLogBufferTest(unittest.TestCase): diff --git a/python_app/tests/test_mimo_calibration.py b/python_app/tests/test_mimo_calibration.py index 5501063..cc5044c 100644 --- a/python_app/tests/test_mimo_calibration.py +++ b/python_app/tests/test_mimo_calibration.py @@ -1,19 +1,19 @@ -"""Physical MIMO estimation, legacy archives and failure-preserving writes.""" +"""Notebook mathematics and acquisition provenance of live object corrections.""" from __future__ import annotations import copy -import json from pathlib import Path import tempfile import unittest from unittest import mock -from typing import Any import numpy as np from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData +from python_app.storage.npz.store import NpzStore from python_app.workflows import mimo_calibration as mc +from python_app.workflows import object_calibration as oc class MimoCalibrationTest(unittest.TestCase): @@ -35,6 +35,7 @@ class MimoCalibrationTest(unittest.TestCase): 'run': {'combos': [dict(input=input_, output=output) for output in (1, 5) for input_ in (2, 7)]}, } + def signal(self, target=None, *, gains=False): target = self.target if target is None else target pitch = np.radians(self.settings.pitch_deg) @@ -55,68 +56,176 @@ class MimoCalibrationTest(unittest.TestCase): signal *= np.array([1., 2.])[:, None, None] * np.array([1., .5])[None, :, None] return signal, paths + def collection(self, samples=None, *, freq=None, identity=1): samples = self.signal()[0] if samples is None else samples freq = self.freq if freq is None else freq - traces = [TraceData(ComboKey(input_, output), freq.copy(), np.zeros(freq.size, np.complex64), samples[t, r].copy()) + traces = [TraceData(ComboKey(input_, output), freq.copy(), samples[t, r].copy()) for t, output in enumerate((1, 5)) for r, input_ in enumerate((2, 7))] return SweepCollection(identity, identity, traces) - def calibration(self): - return mc.compute_from_collections([{'collections': [self.collection()], 'target_scene_m': self.target}], - self.config, settings=self.settings) - def archive(self, root, collection, *, name='0000_id1_ns1'): - snapshot = root / 'preprocessed' / name - snapshot.mkdir(parents=True) - (root / 'config_profile.json').write_text(json.dumps(self.config)) - (snapshot / 'meta.json').write_text('{}') - for trace in collection.traces: - suffix = f'i{trace.combo.input}_o{trace.combo.output}' - np.save(snapshot / f'{suffix}_s21.npy', trace.s21) - np.save(snapshot / f'{suffix}_freq.npy', trace.frequency_hz) - np.save(snapshot / f'{suffix}_s11.npy', trace.s11) - return snapshot + def calibration(self, samples=None): + through = self.collection(np.ones((2, 2, self.freq.size), complex)) + provenance = oc.capture_provenance(self.config, 'captured-radar', through) + return mc.compute_from_collections( + [{'collections': [self.collection(samples)], 'target_scene_m': self.target}], + self.config, settings=self.settings, provenance=provenance, + ) + + def object_set(self): + collection, metadata = self.calibration() + return collection, metadata, self.collection(np.ones((2, 2, self.freq.size), complex)) + + def test_partial_geometry_calibrates_subset_and_preserves_full_capture(self): + from PyQt6.QtWidgets import QApplication, QDoubleSpinBox + from python_app.gui.object_calibration_dialog import ObjectCalibrationDialog + + app = QApplication.instance() or QApplication([]) + self.assertIsNotNone(app) + capture = self.collection() + known = {trace.combo for trace in capture.traces} + full = [ComboKey(i, o) for o in (1, 5, 6) for i in (2, 7, 8)] + capture.traces.extend(TraceData(combo, self.freq.copy(), np.ones(self.freq.size, complex)) + for combo in full if combo not in known) + self.config['run']['combos'] = [dict(input=combo.input, output=combo.output) for combo in full] + through = copy.deepcopy(capture) + for trace in through.traces: + trace.s21.fill(1) + provenance = oc.capture_provenance(self.config, 'captured-radar', through) + dialog = ObjectCalibrationDialog('partial') + self.addCleanup(dialog.close) + dialog.add_capture([capture], self.config, provenance) + for column, value in enumerate(self.target, 2): + item = dialog._table.item(0, column) + assert item is not None + item.setText(str(value)) + for name in ('sphere_radius_m', 'antenna_height_m', 'coarse_range_search_m'): + field = dialog._settings[name] + assert isinstance(field, QDoubleSpinBox) + field.setValue(getattr(self.settings, name)) + for field in dialog._refine_axes.values(): + field.setChecked(False) + groups, config, settings, provenance = dialog.calculation_inputs() + correction, metadata = mc.compute_from_collections( + groups, config, settings=mc.Settings(**settings), provenance=provenance, + ) + self.assertEqual(metadata['rx_input_pos'], [2, 7]) + self.assertEqual(metadata['tx_output_pos'], [1, 5]) + self.assertEqual(len(groups[0]['collections'][0].traces), 9) + self.assertEqual({trace.combo for trace in correction.traces}, set(full)) + self.assertEqual(metadata['unchanged_combos'], + [dict(input=combo.input, output=combo.output) for combo in full if combo not in known]) + for trace in correction.traces: + if trace.combo not in known: + np.testing.assert_array_equal(trace.s21, np.ones(self.freq.size)) + self.assertGreater(metadata['diagnostics'][0]['coherence_after'], .999) + oc.validate_object_set(correction, metadata, config, 'captured-radar', through) def test_point_and_sphere_recover_relative_antenna_phases(self): for radius in (0., .16): with self.subTest(radius=radius): self.settings.sphere_radius_m = radius signal, paths = self.signal() - cal = self.calibration() - corrected = mc.apply_calibration(signal, self.freq, cal) - contributions = corrected * np.exp(2j * np.pi * self.freq * paths[:, :, None] / mc.C_LIGHT) - phases = np.angle(contributions.mean(axis=2)) - np.testing.assert_allclose(phases, np.full((2, 2), 1.1), atol=.004) - np.testing.assert_allclose(np.abs(cal['pair']), 1., atol=2e-7) - self.assertGreater(cal['meta']['quality']['coherence_final'][0], .9999) - self.assertLess(cal['meta']['quality']['coherence_before'][0], .98) - diagnostic = cal['meta']['diagnostics'][0] - self.assertEqual(np.shape(diagnostic['phase_before_deg']), (2, 2)) - np.testing.assert_allclose(diagnostic['phase_after_deg'], np.full((2, 2), np.degrees(1.1)), atol=.25) - self.assertIsNone(cal['meta']['quality']['coherence_leave_one_out'][0]) + collection, metadata = self.calibration() + pair = np.array([trace.s21 for trace in collection.traces]).reshape(signal.shape) + contributions = signal * pair * np.exp(2j * np.pi * self.freq * paths[:, :, None] / mc.C_LIGHT) + np.testing.assert_allclose(np.angle(contributions.mean(axis=2)), np.full((2, 2), 1.1), atol=.004) + np.testing.assert_allclose(np.abs(pair), 1., atol=2e-7) + diagnostic = metadata['diagnostics'][0] + self.assertGreater(diagnostic['coherence_after'], .9999) + self.assertLess(diagnostic['coherence_before'], .98) + np.testing.assert_allclose(diagnostic['phase_after_deg'], np.zeros((2, 2)), atol=.25) - def test_amplitude_equalization_is_explicit(self): + def test_phase_maps_match_notebook_cell5_with_separate_sum_references(self): + before = np.array([[1j, 3], [-1j, 1]]) + after = np.array([[-1, -2j], [-1j, -3]]) + with mock.patch.object(mc._Estimator, 'broadband', side_effect=[before, after]): + diagnostic = self.calibration()[1]['diagnostics'][0] + np.testing.assert_allclose(diagnostic['phase_before_deg'], [[90, 0], [-90, 0]], atol=1e-10) + np.testing.assert_allclose(diagnostic['phase_after_deg'], + [[-36.869897645844, 53.130102354156], + [53.130102354156, -36.869897645844]], atol=1e-10) + self.assertAlmostEqual(diagnostic['phase_std_before_deg'], np.sqrt(4050)) + self.assertAlmostEqual(diagnostic['phase_std_after_deg'], 45) + + + def test_overflow_in_inverted_pair_correction_is_rejected(self): self.settings.apply_amplitude = True - signal, paths = self.signal(gains=True) - cal = mc.compute_from_collections([{'collections': [self.collection(signal)], 'target_scene_m': self.target}], - self.config, settings=self.settings) - corrected = mc.apply_calibration(signal, self.freq, cal) - # Compare amplitudes after removing the physical propagation phase. - levels = np.abs((corrected * np.exp(2j * np.pi * self.freq * paths[:, :, None] / mc.C_LIGHT)).mean(axis=2)) - self.assertLess(levels.max() / levels.min(), 1.01) - self.assertGreater(np.max(np.abs(cal['pair'])), 1.5) + with mock.patch.object(mc._Estimator, 'to_full_grid', return_value=np.full(self.freq.size, 1e-25)), \ + np.errstate(over='ignore', invalid='ignore'), \ + self.assertRaisesRegex(ValueError, 'pair corrections.*non-finite'): + self.calibration() + + + def test_through_hash_tracks_only_canonical_pair_frequency_and_s21_content(self): + through = self.collection() + fingerprint = oc.through_fingerprint(through) + reordered = copy.deepcopy(through) + reordered.traces.reverse() + reordered.collection_id += 1 + reordered.monotonic_ns += 1000 + for trace in reordered.traces: + trace.frequency_hz = trace.frequency_hz.astype('>f4') + trace.s21 = trace.s21.astype('>c8') + trace.capture_start_ns = 99 + self.assertEqual(oc.through_fingerprint(reordered), fingerprint) + for field in ('frequency_hz', 's21'): + changed = copy.deepcopy(through) + getattr(changed.traces[0], field)[0] += 1024 if field == 'frequency_hz' else .1j + with self.subTest(field=field): + self.assertNotEqual(oc.through_fingerprint(changed), fingerprint) + changed = copy.deepcopy(through) + changed.traces[0].combo = ComboKey(999, 1) + self.assertNotEqual(oc.through_fingerprint(changed), fingerprint) + + + def test_object_set_accepts_combo_subset_and_reordered_geometry_with_new_reference(self): + collection, metadata, through = self.object_set() + config = copy.deepcopy(self.config) + config['run']['combos'] = config['run']['combos'][1:2] + config['gpr']['tx_geometry'].reverse() + config['gpr']['rx_geometry'].reverse() + config['preprocess']['s21']['reference']['set_name'] = 'new empty scene' + oc.validate_object_set(collection, metadata, config, 'captured-radar', through) + + + def test_object_set_rejects_changed_capture_dependencies_and_missing_provenance(self): + collection, metadata, through = self.object_set() + for dependency in ('radar', 'geometry', 'through_name', 'through_content', 'provenance'): + config = copy.deepcopy(self.config) + active_through = copy.deepcopy(through) + active_metadata = copy.deepcopy(metadata) + key = 'captured-radar' + if dependency == 'radar': + key = 'another-radar' + elif dependency == 'geometry': + config['gpr']['tx_geometry'][0]['z_m'] += .01 + elif dependency == 'through_name': + config['preprocess']['s21']['calibration']['set_name'] = 'another-through' + elif dependency == 'through_content': + active_through.traces[0].s21[0] += .1 + else: + active_metadata.clear() + with self.subTest(dependency=dependency), self.assertRaisesRegex(ValueError, 'Object calibration'): + oc.validate_object_set(collection, active_metadata, config, key, active_through) + + + def test_object_set_rejects_missing_pairs_shifted_grids_and_invalid_factors(self): + collection, metadata, through = self.object_set() + for defect in ('missing', 'duplicate', 'grid', 'zero', 'nan'): + corrupted = copy.deepcopy(collection) + if defect == 'missing': + corrupted.traces.pop() + elif defect == 'duplicate': + corrupted.traces.append(corrupted.traces[0]) + elif defect == 'grid': + corrupted.traces[0].frequency_hz = corrupted.traces[0].frequency_hz + 1024 + else: + corrupted.traces[0].s21[0] = 0 if defect == 'zero' else np.nan + with self.subTest(defect=defect), self.assertRaises(ValueError): + oc.validate_object_set(corrupted, metadata, self.config, 'captured-radar', through) - def test_archive_and_live_estimation_share_the_same_math(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - signal = self.signal()[0] - self.archive(root, self.collection(signal), name='0000_id1_ns1') - self.archive(root, self.collection(signal * 2), name='0001_id2_ns2') - archived = mc.compute_calibration([{'path': root / 'preprocessed', 'target_scene_m': self.target}], settings=self.settings) - live = mc.compute_from_collections([{'collections': [self.collection(signal), self.collection(signal * 2)], - 'target_scene_m': self.target}], self.config, settings=self.settings) - np.testing.assert_array_equal(archived['pair'], live['pair']) def test_frequency_geometry_channel_and_signal_failures_are_explicit(self): cases = [] @@ -129,38 +238,10 @@ class MimoCalibrationTest(unittest.TestCase): for index, collection in enumerate(cases): with self.subTest(collection=index): with self.assertRaises(ValueError): - mc.compute_from_collections([{'collections': [collection], 'target_scene_m': self.target}], self.config, settings=self.settings) + mc.compute_from_collections([{'collections': [collection], 'target_scene_m': self.target}], self.config, settings=self.settings, provenance={}) with self.assertRaisesRegex(ValueError, 'at least one'): - mc.compute_from_collections([], self.config, settings=self.settings) + mc.compute_from_collections([], self.config, settings=self.settings, provenance={}) - def test_config_checks_phase_base_and_geometry_but_allows_new_reference(self): - cal = self.calibration() - updated = copy.deepcopy(self.config) - updated['preprocess']['s21']['reference']['set_name'] = 'new_background' - mc.validate_for_config(cal, updated) - for edit in ('geometry', 'through', 'combo', 'serial'): - updated = copy.deepcopy(self.config) - if edit == 'geometry': - updated['gpr']['tx_geometry'][0]['x_m'] += .01 - elif edit == 'through': - updated['preprocess']['s21']['calibration']['set_name'] = 'new_through' - elif edit == 'serial': - updated['radar']['serial'] = 'another_device' - else: - updated['run']['combos'].append({'input': 999, 'output': 1}) - with self.subTest(edit=edit), self.assertRaises(ValueError): - mc.validate_for_config(cal, updated) - - def test_object_corrected_sources_are_rejected_for_live_and_archives(self): - for field in ('set_name', 'bundle_path'): - self.config['preprocess']['s21']['object_calibration'] = {field: 'active'} - with self.subTest(field=field), self.assertRaisesRegex(ValueError, 'already active'): - self.calibration() - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.archive(root, self.collection()) - with self.assertRaisesRegex(ValueError, 'already active'): - mc.compute_calibration([{'path': root / 'preprocessed', 'target_scene_m': self.target}], settings=self.settings) def test_cancel_is_checked_inside_refinement(self): self.settings.refine_axes = ('z',) @@ -176,209 +257,42 @@ class MimoCalibrationTest(unittest.TestCase): with self.assertRaises(InterruptedError): mc.compute_from_collections([{'collections': [self.collection()], 'target_scene_m': self.target}], - self.config, settings=self.settings, progress=progress, cancelled=cancelled) + self.config, settings=self.settings, provenance={}, progress=progress, cancelled=cancelled) self.assertEqual(state['checks'], 6) - def test_apply_rejects_broadcasting_and_corrupted_archives(self): - cal = self.calibration() - with self.assertRaisesRegex(ValueError, 'invalid shape'): - mc.apply_calibration(np.ones(self.freq.size), self.freq, cal) - cal['pair'][0, 0, 0] *= 2 - with self.assertRaisesRegex(ValueError, 'disagree'): - mc.validate_calibration(cal) - - def test_interpolation_is_continuous_at_nonlinear_archive_edges(self): - self.settings.phase_model = 'raw' - cal = self.calibration() - # Keep a consistent separable archive with a deliberately nonlinear phase. - phase = .4 * ((self.freq - self.freq.mean()) / 1e9) ** 3 - cal['tx'][1] *= np.exp(1j * phase) - cal['pair'] = cal['tx'][:, None] * cal['rx'][None] - edge = mc.calibration_on_grid(cal, [self.freq[-1], self.freq[-1] + 1]) - self.assertLess(abs(np.angle(edge[1, 0, 1] / edge[1, 0, 0])), 1e-6) - shifted = mc.calibration_on_grid(cal, self.freq + 10000) - self.assertIsNot(shifted, cal['pair']) - - def test_save_roundtrip_and_write_failure_preserve_the_previous_archive(self): - cal = self.calibration() - with tempfile.TemporaryDirectory() as directory: - destination = Path(directory) / 'cal' - destination.mkdir() - (destination / 'notes.txt').write_text('preserve user notes') - mc.save_calibration(cal, destination) - np.testing.assert_array_equal(mc.load_calibration(destination)['pair'], cal['pair']) - original = {path.name: path.read_bytes() for path in destination.iterdir()} - save = np.save - - def fail_second(path, *args, **kwargs): - if Path(path).name == 'cal_pair.npy': - raise OSError('disk full') - return save(path, *args, **kwargs) - - with mock.patch.object(mc.np, 'save', side_effect=fail_second), self.assertRaises(OSError): - mc.save_calibration(cal, destination, overwrite=True) - self.assertEqual(original, {path.name: path.read_bytes() for path in destination.iterdir()}) - self.assertEqual(list(Path(directory).glob('.cal.*')), []) - with self.assertRaises(FileExistsError): - mc.save_calibration(cal, destination) - - def test_non_overwriting_save_preserves_archive_published_during_staging(self): - with tempfile.TemporaryDirectory() as directory: - destination = Path(directory) / 'cal' - with self.assertRaises(FileExistsError): - with mc._staged_directory(destination, False) as stage: - (stage / 'cal_pair.npy').write_bytes(b'new correction') - destination.mkdir() - (destination / 'cal_pair.npy').write_bytes(b'concurrent correction') - (destination / 'cal_meta.json').write_bytes(b'concurrent metadata') - self.assertEqual({path.name: path.read_bytes() for path in destination.iterdir()}, { - 'cal_pair.npy': b'concurrent correction', 'cal_meta.json': b'concurrent metadata', - }) - self.assertEqual(list(Path(directory).glob('.cal.*')), []) - - def test_failed_commit_restores_old_archive_and_failed_rollback_keeps_backups(self): - cal = self.calibration() - with tempfile.TemporaryDirectory() as directory: - destination = Path(directory) / 'cal' - mc.save_calibration(cal, destination) - original = (destination / 'cal_pair.npy').read_bytes() - replace = Path.replace - - def fail_commit(path, target): - if path.name.startswith('.cal.new-'): - raise OSError('rename failed') - return replace(path, target) - - with mock.patch.object(Path, 'replace', autospec=True, side_effect=fail_commit), self.assertRaises(OSError): - mc.save_calibration(cal, destination, overwrite=True) - self.assertEqual((destination / 'cal_pair.npy').read_bytes(), original) - - def fail_rollback(path, target): - if path.name.startswith(('.cal.new-', '.cal.old-')): - raise OSError('filesystem unavailable') - return replace(path, target) - - with mock.patch.object(Path, 'replace', autospec=True, side_effect=fail_rollback), self.assertRaisesRegex(RuntimeError, 'originals retained'): - mc.save_calibration(cal, destination, overwrite=True) - backup = next(Path(directory).glob('.cal.old-*')) - self.assertEqual((backup / 'cal_pair.npy').read_bytes(), original) - - def test_offline_failure_preserves_previous_postprocessed_and_original_data(self): - cal = self.calibration() - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - snapshot = self.archive(root, self.collection()) - post = root / 'postprocessed'; post.mkdir() - (post / 'calibration_applied.json').write_text('{}') - (post / 'old_data.bin').write_bytes(b'old data') - original = {path.name: path.read_bytes() for path in snapshot.iterdir()} - with mock.patch.object(mc.np, 'save', side_effect=OSError('disk full')), self.assertRaises(OSError): - mc.apply_to_measurement(root, cal, overwrite=True) - self.assertEqual((post / 'old_data.bin').read_bytes(), b'old data') - self.assertEqual(original, {path.name: path.read_bytes() for path in snapshot.iterdir()}) - self.assertEqual(list(root.glob('.postprocessed.*')), []) - - def test_object_corrected_archive_is_rejected_even_when_source_profile_has_object_off(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - snapshot = self.archive(root, self.collection()) - post, *_ = mc.apply_to_measurement(root, self.calibration()) - for prefixes in (None, [snapshot.name[:4]]): - with self.subTest(prefixes=prefixes), self.assertRaisesRegex(ValueError, 'already applied'): - mc.compute_calibration([{'path': post, 'snapshots': prefixes, 'target_scene_m': self.target}], - settings=self.settings) - # A corrected sibling does not change the original preprocessing data. - original = mc.compute_calibration([{'path': root / 'preprocessed', 'target_scene_m': self.target}], - settings=self.settings) - np.testing.assert_array_equal(original['pair'], self.calibration()['pair']) - - def test_snapshot_local_correction_marker_is_checked_for_target_and_empty_scene(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - snapshot = self.archive(root, self.collection()) - (snapshot / 'calibration_applied.json').write_text('{}') - targets = [{'path': root / 'preprocessed', 'snapshots': [snapshot.name[:4]], - 'target_scene_m': self.target}] - with self.assertRaisesRegex(ValueError, 'already applied'): - mc.compute_calibration(targets, settings=self.settings) - untouched = root / 'untouched' - self.archive(untouched, self.collection()) - targets = [{'path': untouched / 'preprocessed', 'target_scene_m': self.target}] - with self.assertRaisesRegex(ValueError, 'already applied'): - mc.compute_calibration(targets, root / 'preprocessed', self.settings) - - def test_offline_grid_cache_uses_the_entire_axis_and_preserves_s11_and_dtype(self): - cal = self.calibration() - collection = self.collection(np.ones((2, 2, self.freq.size), np.complex64)) - alternate = self.freq.copy() - alternate[64] += 1e7 - collection.traces[2].frequency_hz = alternate - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - snapshot = self.archive(root, collection) - original_s11 = (snapshot / 'i2_o5_s11.npy').read_bytes() - post, count, fixed, _ = mc.apply_to_measurement(root, cal) - self.assertEqual((count, fixed), (1, 4)) - corrected = np.load(post / snapshot.name / 'i2_o5_s21.npy') - self.assertEqual(corrected.dtype, np.complex64) - expected = mc.calibration_on_grid(cal, alternate)[1, 0] - np.testing.assert_allclose(corrected, expected, atol=1e-7) - self.assertGreater(abs(corrected[64] - cal['pair'][1, 0, 64]), .001) - self.assertEqual((post / snapshot.name / 'i2_o5_s11.npy').read_bytes(), original_s11) - - def test_empty_scene_grids_and_geometry_are_checked_before_subtraction(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - target_root, empty_root = root / 'target', root / 'empty' - self.archive(target_root, self.collection()) - zero = np.zeros((2, 2, self.freq.size), complex) - self.archive(empty_root, self.collection(zero)) - self.archive(empty_root, self.collection(zero, freq=self.freq + 10000), name='0001_id2_ns2') - targets = [{'path': target_root / 'preprocessed', 'target_scene_m': self.target}] - with self.assertRaisesRegex(ValueError, 'frequency grids differ'): - mc.compute_calibration(targets, empty_root / 'preprocessed', self.settings) - profile = copy.deepcopy(self.config) - profile['gpr']['rx_geometry'][0]['z_m'] += .1 - (empty_root / 'config_profile.json').write_text(json.dumps(profile)) - with self.assertRaisesRegex(ValueError, 'antenna geometry differs'): - mc.compute_calibration(targets, empty_root / 'preprocessed', self.settings) - - def test_snapshot_prefixes_must_select_one_distinct_complete_snapshot(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self.archive(root, self.collection()) - self.archive(root, self.collection(), name='0001_id2_ns2') - for prefixes in (['000'], ['0000', 'missing'], ['0000', '0000']): - with self.subTest(prefixes=prefixes), self.assertRaises(ValueError): - mc.list_snapshots(root / 'preprocessed', prefixes) - - def test_multiple_snapshots_are_averaged_before_one_gated_fft(self): - collections = [self.collection(identity=index + 1) for index in range(8)] - with mock.patch.object(mc.np.fft, 'ifft', wraps=np.fft.ifft) as transform: - mc.compute_from_collections([{'collections': collections, 'target_scene_m': self.target}], - self.config, settings=self.settings) - self.assertEqual(transform.call_count, 1) def test_original_scipy_refinement_finishes_with_valid_diagnostics(self): self.settings.refine_axes = ('z',) self.settings.coarse_range_search_m = .1 - cal = self.calibration() - self.assertGreater(cal['meta']['diagnostics'][0]['coherence_after'], .999) - refined = cal['meta']['targets'][0]['refined_scene_m'] + _, metadata = self.calibration() + self.assertGreater(metadata['diagnostics'][0]['coherence_after'], .999) + refined = metadata['targets'][0]['refined_scene_m'] self.assertTrue(np.all(np.isfinite(refined))) - def test_bad_settings_and_duplicate_geometry_fail_early(self): - invalid_settings: tuple[dict[str, Any], ...] = ( - {'input_idx': (0, 0)}, {'output_idx': (True,)}, {'gate_taper_m': 0.}, - {'f_max': 0.}, {'refine_axes': ('z', 'z')}, {'phase_model': 'unknown'}, - ) - for fields in invalid_settings: - with self.subTest(fields=fields), self.assertRaises(ValueError): - mc.Settings(**fields) - self.config['gpr']['tx_geometry'].append(self.config['gpr']['tx_geometry'][0]) - with self.assertRaisesRegex(ValueError, 'duplicate antenna geometry'): - self.calibration() + def test_common_phase_invariance_and_explicit_amplitude_equalization(self): + signal, paths = self.signal(gains=True) + _, original = self.calibration(signal) + _, rotated = self.calibration(signal * np.exp(2.8j)) + for key in ('phase_before_deg', 'phase_after_deg'): + np.testing.assert_allclose(rotated['diagnostics'][0][key], original['diagnostics'][0][key], atol=1e-5) + self.settings.apply_amplitude = True + collection, _ = self.calibration(signal) + pair = np.array([trace.s21 for trace in collection.traces]).reshape(signal.shape) + levels = np.abs((signal * pair * np.exp(2j * np.pi * self.freq * paths[:, :, None] / mc.C_LIGHT)).mean(axis=2)) + self.assertLess(levels.max() / levels.min(), 1.01) + self.assertGreater(np.max(np.abs(pair)), 1.5) -if __name__ == '__main__': - unittest.main() + def test_estimated_npz_roundtrip_keeps_physical_pairs_full_grid_and_provenance(self): + collection, metadata, through = self.object_set() + with tempfile.TemporaryDirectory() as directory: + store = NpzStore(Path(directory)) + store.save_set('s21_object_calibration', 'captured-radar', 'sphere', collection, metadata=metadata) + loaded, stored = store.load_set_with_metadata('s21_object_calibration', 'captured-radar', 'sphere') + self.assertEqual(stored['provenance'], metadata['provenance']) + self.assertEqual(stored['targets'][0]['captures'][0]['monotonic_ns'], 1) + for before, after in zip(collection.traces, loaded.traces, strict=True): + self.assertEqual(before.combo, after.combo) + np.testing.assert_array_equal(after.frequency_hz, self.freq.astype(np.float32)) + np.testing.assert_array_equal(after.s21, before.s21) + oc.validate_object_set(loaded, stored, self.config, 'captured-radar', through) diff --git a/python_app/tests/test_kamil_adc_neutral_preprocess.py b/python_app/tests/test_neutral_preprocess.py similarity index 91% rename from python_app/tests/test_kamil_adc_neutral_preprocess.py rename to python_app/tests/test_neutral_preprocess.py index 8bc42b8..415c39c 100644 --- a/python_app/tests/test_kamil_adc_neutral_preprocess.py +++ b/python_app/tests/test_neutral_preprocess.py @@ -7,8 +7,8 @@ import unittest import numpy as np from python_app.hardware_full.point_target_mock_service import PointTargetMockService -from python_app.models.run_config_model import RunConfigModel -from python_app.workflows.kamil_adc_neutral_preprocess import build_neutral_s21_sets, supports_neutral_preprocess_sets +from python_app.models.run_config_schema import RunConfigModel +from python_app.workflows.neutral_preprocess import build_neutral_s21_sets, supports_neutral_preprocess_sets def _kamil_config(*, points: int = 5) -> RunConfigModel: @@ -39,7 +39,6 @@ class KamilAdcNeutralPreprocessTest(unittest.TestCase): ) for trace in calibration.traces: np.testing.assert_array_equal(trace.frequency_hz, expected_frequency) - np.testing.assert_array_equal(trace.s11, np.zeros(5, dtype=np.complex64)) np.testing.assert_array_equal(trace.s21, np.ones(5, dtype=np.complex64)) for trace in reference.traces: np.testing.assert_array_equal(trace.frequency_hz, expected_frequency) @@ -66,10 +65,9 @@ class KamilAdcNeutralPreprocessTest(unittest.TestCase): trace = calibration.traces[0] self.assertEqual(trace.frequency_hz.dtype, np.float32) self.assertEqual(trace.s21.dtype, np.complex64) - self.assertEqual(trace.s11.dtype, np.complex64) def test_calibration_s21_is_a_nonzero_divisor(self) -> None: - # The C++ through-calibrator divides measured/calibration, so calibration S21 + # The preprocessor divides measured/calibration, so calibration S21 # must never be zero — that is the whole point of the '1+0j neutral' contract. calibration, _reference = build_neutral_s21_sets(_kamil_config()) for trace in calibration.traces: diff --git a/python_app/tests/test_neutral_preprocess_cli.py b/python_app/tests/test_neutral_preprocess_cli.py deleted file mode 100644 index ad7f73f..0000000 --- a/python_app/tests/test_neutral_preprocess_cli.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Neutral pair creation must report incomplete sets and roll back failed writes.""" - -from __future__ import annotations - -from pathlib import Path -import tempfile -import unittest -from unittest import mock - -import numpy as np - -from python_app.models.run_config_model import RunConfigModel -from python_app.scripts import create_neutral_preprocess_sets as cli -from python_app.storage.npz_store import NpzStore -from python_app.workflows.kamil_adc_neutral_preprocess import build_neutral_s21_sets - - -class NeutralPreprocessCliTest(unittest.TestCase): - def setUp(self) -> None: - self.directory = self.enterContext(tempfile.TemporaryDirectory()) - self.root = Path(self.directory) - self.store = NpzStore(self.root) - self.config = RunConfigModel.from_dict({"radar": { - "model": "librevna", "driver_mode": "mock", "sweep": {"points": 7}, - }}) - calibration, reference = build_neutral_s21_sets(self.config) - calibration.traces[0].s21[:] = 3 + 1j - reference.traces[0].s21[:] = 2 - 1j - self.previous = {"s21_calibration": calibration, "s21_reference": reference} - - def _main(self, *options: str) -> int: - with ( - mock.patch.object(cli.RunConfigModel, "load_from_path", return_value=self.config), - mock.patch.object(cli, "radar_key_from_config", return_value="radar"), - mock.patch.object(cli, "NpzStore", return_value=self.store), - mock.patch("sys.argv", ["neutral", "--config", "profile.json", "--set-name", "neutral", *options]), - ): - return cli.main() - - def _files(self) -> dict[str, bytes]: - return {str(path.relative_to(self.root)): path.read_bytes() for path in self.root.rglob("*") if path.is_file()} - - def _fail_second_archive(self): - save_archive = np.savez - count = 0 - - def write(*args, **kwargs): - nonlocal count - count += 1 - if count == 2: - raise OSError("reference write failed") - return save_archive(*args, **kwargs) - - return mock.patch("python_app.storage.npz.store.np.savez", side_effect=write) - - def test_partial_existing_pair_reports_missing_kind_without_overwriting(self) -> None: - for present, missing in [("s21_calibration", "s21_reference"), ("s21_reference", "s21_calibration")]: - with self.subTest(present=present): - self.store = NpzStore(self.root / present) - self.store.save_set(present, "radar", "neutral", self.previous[present]) - before = self._files() - with self.assertRaisesRegex(RuntimeError, f"missing {missing}; use --force to replace both"): - self._main() - self.assertEqual(self._files(), before) - - def test_complete_existing_pair_is_skipped(self) -> None: - self.store.save_sets("radar", "neutral", self.previous) - before = self._files() - with mock.patch.object(self.store, "save_sets", wraps=self.store.save_sets) as save: - self.assertEqual(self._main(), 0) - save.assert_not_called() - self.assertEqual(self._files(), before) - - def test_force_replaces_both_existing_sets_with_neutral_data(self) -> None: - self.store.save_sets("radar", "neutral", self.previous) - self.assertEqual(self._main("--force"), 0) - for kind, value in [("s21_calibration", 1), ("s21_reference", 0)]: - np.testing.assert_array_equal(self.store.load_set(kind, "radar", "neutral").traces[0].s21, value) - self.assertEqual(list(self.root.glob(".sets-*")), []) - - def test_force_completes_either_partial_pair(self) -> None: - for present in self.previous: - with self.subTest(present=present): - self.store = NpzStore(self.root / present) - self.store.save_set(present, "radar", "neutral", self.previous[present]) - self.assertEqual(self._main("--force"), 0) - for kind, value in [("s21_calibration", 1), ("s21_reference", 0)]: - np.testing.assert_array_equal(self.store.load_set(kind, "radar", "neutral").traces[0].s21, value) - - def test_second_archive_failure_restores_previous_pair_byte_for_byte(self) -> None: - self.store.save_sets("radar", "neutral", self.previous) - before = self._files() - with self._fail_second_archive(), self.assertRaisesRegex(OSError, "reference write failed"): - self._main("--force") - self.assertEqual(self._files(), before) - - def test_second_archive_failure_removes_new_pair(self) -> None: - with self._fail_second_archive(), self.assertRaisesRegex(OSError, "reference write failed"): - self._main() - self.assertEqual(self._files(), {}) - - def test_failed_rollback_keeps_the_previous_archive_for_recovery(self) -> None: - self.store.save_sets("radar", "neutral", self.previous) - previous_archive = (self.root / "s21_calibration/radar/neutral.npz").read_bytes() - replace = Path.replace - - def replace_file(path: Path, destination: Path): - if path.parent.name.startswith(".sets-") and path.name == "s21_calibration.npz": - raise OSError("restore failed") - return replace(path, destination) - - with ( - self._fail_second_archive(), mock.patch.object(Path, "replace", replace_file), - self.assertRaisesRegex(RuntimeError, "rollback was incomplete; previous files are retained") as error, - ): - self._main("--force") - backups = list(self.root.glob(".sets-*/s21_calibration.npz")) - self.assertEqual(len(backups), 1) - self.assertEqual(backups[0].read_bytes(), previous_archive) - self.assertIn(str(backups[0].parent), str(error.exception)) - - def test_native_single_is_rejected_before_any_write(self) -> None: - self.config.radar.driver_mode = "native" - with self.assertRaisesRegex(RuntimeError, "not supported for this radar configuration"): - self._main() - self.assertEqual(self._files(), {}) diff --git a/python_app/tests/test_npz_store.py b/python_app/tests/test_npz_store.py new file mode 100644 index 0000000..1641030 --- /dev/null +++ b/python_app/tests/test_npz_store.py @@ -0,0 +1,81 @@ +"""Embedded preprocess metadata and compatibility with older NPZ sets.""" + +import json +from pathlib import Path +import tempfile +import unittest + +import numpy as np + +from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData +from python_app.storage.npz.store import NpzStore + + +class SetMetadataTest(unittest.TestCase): + def setUp(self) -> None: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.store = NpzStore(self.root) + self.kind = "s21_object_calibration" + self.collection = SweepCollection(7, 100, [TraceData( + ComboKey(2, 3), np.array([1e9, 2e9], dtype=np.float32), + np.array([1 + 2j, 3 - 4j], dtype=np.complex64), + capture_start_ns=80, capture_end_ns=90, + )], capture_start_ns=80, capture_end_ns=90) + self.metadata = {"radar_key": "radar", "geometry": [{"input_pos": 2, "x": 0.1}], + "through": {"name": "through", "hash": "abc"}, "collection_id": "user value"} + self.store.save_set(self.kind, "radar", "object", self.collection, metadata=self.metadata) + self.archive = self.root / self.kind / "radar/object.npz" + self.sidecar = self.archive.with_suffix(".json") + + + def test_metadata_and_collection_round_trip_through_generic_set_api(self) -> None: + loaded, metadata = self.store.load_set_with_metadata(self.kind, "radar", "object") + self.assertEqual(metadata, self.metadata) + self.assertEqual((loaded.collection_id, loaded.monotonic_ns), (7, 100)) + self.assertEqual((loaded.capture_start_ns, loaded.capture_end_ns), (80, 90)) + self.assertEqual((loaded.traces[0].capture_start_ns, loaded.traces[0].capture_end_ns), (80, 90)) + np.testing.assert_array_equal(loaded.traces[0].s21, self.collection.traces[0].s21) + np.testing.assert_array_equal(loaded.traces[0].frequency_hz, self.collection.traces[0].frequency_hz) + self.assertEqual(self.store.load_set(self.kind, "radar", "object").collection_id, 7) + self.assertEqual(self.store.list_sets(self.kind, "radar"), ["object"]) + self.assertTrue(self.store.has_combo_coverage(self.kind, "radar", "object", [ComboKey(2, 3)])) + self.assertFalse(self.store.has_combo_coverage(self.kind, "radar", "object", [ComboKey(0, 0)])) + + + def test_embedded_metadata_is_authoritative_when_sidecar_is_stale_or_missing(self) -> None: + for sidecar in ('{"metadata": {"radar_key": "wrong"}}', "broken JSON", None): + with self.subTest(sidecar=sidecar): + if sidecar is None: + self.sidecar.unlink() + else: + self.sidecar.write_text(sidecar, encoding="utf-8") + _, metadata = self.store.load_set_with_metadata(self.kind, "radar", "object") + self.assertEqual(metadata, self.metadata) + + + def test_old_sets_without_user_metadata_load_with_empty_dict(self) -> None: + with np.load(self.archive, allow_pickle=False) as stored: + arrays = {key: stored[key] for key in stored.files if key != "metadata_json"} + envelope = json.loads(self.sidecar.read_text(encoding="utf-8")) + del envelope["metadata"] + self.sidecar.write_text(json.dumps(envelope), encoding="utf-8") + for embedded in (True, False): + with self.subTest(embedded=embedded): + payload = dict(arrays) + if embedded: + payload["metadata_json"] = np.asarray(json.dumps(envelope)) + np.savez(self.archive, **payload) + collection, metadata = self.store.load_set_with_metadata(self.kind, "radar", "object") + self.assertEqual(metadata, {}) + np.testing.assert_array_equal(collection.traces[0].s21, self.collection.traces[0].s21) + + + def test_invalid_metadata_cannot_overwrite_saved_set(self) -> None: + before = (self.archive.read_bytes(), self.sidecar.read_bytes()) + for metadata in ({"value": float("nan")}, {"value": [float("inf")]}, {"value": object()}): + with self.subTest(metadata=metadata), self.assertRaises((TypeError, ValueError)): + self.store.save_set(self.kind, "radar", "object", self.collection, metadata=metadata) + self.assertEqual((self.archive.read_bytes(), self.sidecar.read_bytes()), before) + diff --git a/python_app/tests/test_object_calibration_gui.py b/python_app/tests/test_object_calibration_gui.py deleted file mode 100644 index ab0d326..0000000 --- a/python_app/tests/test_object_calibration_gui.py +++ /dev/null @@ -1,304 +0,0 @@ -"""Optional selection, live sample input and real target-phase diagnostics.""" - -from __future__ import annotations - -import copy -import os -import queue -import threading -import time -import unittest -from types import SimpleNamespace -from unittest import mock - -import numpy as np - -os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") -from PyQt6.QtCore import QTimer # noqa: E402 -from PyQt6.QtWidgets import QApplication, QPushButton, QWidget # noqa: E402 - -from python_app.gui.controllers.object_calibration_controller import ObjectCalibrationController, _calculate # noqa: E402 -from python_app.gui.object_calibration_dialog import ObjectCalibrationDialog # noqa: E402 -from python_app.gui.preprocess_dialog import PreprocessDialog # noqa: E402 -from python_app.tests import test_mimo_calibration as numeric_fixture # noqa: E402 - - -def setUpModule() -> None: - global app - app = QApplication.instance() or QApplication([]) - - -class ObjectCalibrationSelectorTest(unittest.TestCase): - def setUp(self) -> None: - self.dialog = PreprocessDialog() - self.addCleanup(self.dialog.close) - - def test_optional_off_is_distinct_from_a_saved_set_named_off(self) -> None: - self.dialog.set_available_sets({"s21_object_calibration": ["Off", "sphere"]}) - changes = mock.Mock() - self.dialog.selection_changed.connect(changes) - combo = self.dialog._set_combos["s21_object_calibration"] - self.assertEqual(combo.currentText(), "Off") - self.assertEqual(self.dialog.selection_snapshot()["s21_object_calibration"], "") - combo.setCurrentIndex(combo.findData("Off")) - self.assertEqual(self.dialog.selection_snapshot()["s21_object_calibration"], "Off") - changes.assert_called_once() - self.dialog.set_available_sets({"s21_object_calibration": ["Off", "sphere"]}) - self.assertEqual(self.dialog.selection_snapshot()["s21_object_calibration"], "Off") - self.dialog.set_selected_sets({"s21_object_calibration": ""}, emit_signal=False) - self.assertEqual(combo.currentText(), "Off") - self.assertEqual(combo.currentData(), "") - - def test_object_actions_do_not_start_a_sequential_hardware_capture(self) -> None: - capture, imported, created = mock.Mock(), mock.Mock(), mock.Mock() - self.dialog.start_sequence_requested.connect(capture) - self.dialog.import_object_calibration_requested.connect(imported) - self.dialog.create_object_calibration_requested.connect(created) - self.dialog._import_object_calibration_button.click() - self.dialog._create_object_calibration_button.click() - imported.assert_called_once() - created.assert_called_once() - capture.assert_not_called() - self.assertEqual([button.text() for button in self.dialog.findChildren(QPushButton) - if button.text().startswith("Start ")], - ["Start S21 Calibration", "Start S21 Reference"]) - - -class ObjectCalibrationDialogTest(unittest.TestCase): - def setUp(self) -> None: - self.fixture = numeric_fixture.MimoCalibrationTest() - self.fixture.setUp() - self.dialog = ObjectCalibrationDialog("sphere") - self.addCleanup(self.dialog.close) - - def _capture_with_target(self) -> None: - self.dialog.add_capture([self.fixture.collection()], self.fixture.config) - for column, coordinate in enumerate(self.fixture.target, start=2): - self.dialog._table.item(0, column).setText(str(coordinate)) - - def test_capture_is_frozen_and_xyz_must_be_entered_after_measurement(self) -> None: - collection = self.fixture.collection() - self.dialog.add_capture([collection], self.fixture.config) - expected = collection.traces[0].s21.copy() - collection.traces[0].s21.fill(0) - with self.assertRaisesRegex(ValueError, "X, Y и Z"): - self.dialog.calculation_inputs() - for column, coordinate in enumerate(self.fixture.target, start=2): - self.dialog._table.item(0, column).setText(str(coordinate)) - groups, config, settings = self.dialog.calculation_inputs() - np.testing.assert_array_equal(groups[0]["collections"][0].traces[0].s21, expected) - self.assertEqual(groups[0]["target_scene_m"], self.fixture.target) - self.assertEqual(config, self.fixture.config) - self.assertEqual(settings["input_idx"], (2, 7)) - self.assertEqual(settings["output_idx"], (1, 5)) - self.dialog._table.item(0, 2).setText("nan") - with self.assertRaisesRegex(ValueError, "конечными"): - self.dialog.calculation_inputs() - - def test_changed_geometry_is_rejected_without_appending_a_capture(self) -> None: - self._capture_with_target() - changed = copy.deepcopy(self.fixture.config) - changed["gpr"]["tx_geometry"][0]["x_m"] += 1 - with self.assertRaisesRegex(ValueError, "изменились"): - self.dialog.add_capture([self.fixture.collection()], changed) - self.assertEqual(self.dialog._table.rowCount(), 1) - - def test_actual_numeric_output_draws_both_target_phase_curves(self) -> None: - calibration = self.fixture.calibration() - self.dialog.show_calibration(calibration) - curves = self.dialog._phase_plot.listDataItems() - self.assertEqual(len(curves), 2) - diagnostic = calibration["meta"]["diagnostics"][0] - phases = np.array([diagnostic["phase_before_deg"], diagnostic["phase_after_deg"]]).reshape(2, -1) - expected = (phases - phases[0, 0] + 180) % 360 - 180 - for curve, phases in zip(curves, expected): - np.testing.assert_allclose(curve.getData()[1], phases) - self.assertLess(np.ptp(curves[1].getData()[1]), .5) - self.assertIn(f"{diagnostic['coherence_before']:.3f}", self.dialog._quality_label.text()) - self.assertIn(f"{diagnostic['coherence_after']:.3f}", self.dialog._quality_label.text()) - - def test_phase_wrap_uses_the_same_first_pair_baseline_for_both_curves(self) -> None: - calibration = self.fixture.calibration() - diagnostic = calibration["meta"]["diagnostics"][0] - diagnostic["phase_before_deg"] = [[179, -179], [178, -178]] - diagnostic["phase_after_deg"] = [[-179, -179], [-179, -179]] - self.dialog.show_calibration(calibration) - before, after = self.dialog._phase_plot.listDataItems() - np.testing.assert_allclose(before.getData()[1], [0, 2, -1, 3]) - np.testing.assert_allclose(after.getData()[1], [2, 2, 2, 2]) - - def test_capture_and_calculation_disable_edits_but_keep_cancel_available(self) -> None: - self._capture_with_target() - for setter in (self.dialog.set_capture_pending, self.dialog.set_calculating): - setter(True) - self.assertFalse(self.dialog._capture_button.isEnabled()) - self.assertFalse(self.dialog._table.isEnabled()) - self.assertTrue(self.dialog._cancel_button.isEnabled()) - setter(False) - self.assertTrue(self.dialog._capture_button.isEnabled()) - self.assertTrue(self.dialog._table.isEnabled()) - - -class ObjectCalibrationWorkerTest(unittest.TestCase): - def _controller_with_data(self): - fixture = numeric_fixture.MimoCalibrationTest() - fixture.setUp() - owner = QWidget() - self.addCleanup(owner.close) - panel = SimpleNamespace(set_name=lambda: "sphere", set_set_name=mock.Mock(), set_status=mock.Mock()) - current_config = SimpleNamespace(to_dict=lambda: copy.deepcopy(fixture.config)) - owner._ensure_preprocess_dialog = mock.Mock(return_value=panel) - owner._build_config = mock.Mock(return_value=current_config) - owner._radar_key = mock.Mock(return_value="current-config-key") - owner._store = mock.Mock() - owner._selected_preprocess_sets = {} - for name in ("_show_exception", "_show_error", "_refresh_sets", "_log"): - setattr(owner, name, mock.Mock()) - controller = ObjectCalibrationController(owner) - controller.open_creation() - self.addCleanup(controller.shutdown) - dialog = controller._dialog - dialog.add_capture([fixture.collection()], fixture.config) - for column, coordinate in enumerate(fixture.target, start=2): - dialog._table.item(0, column).setText(str(coordinate)) - for key in ("sphere_radius_m", "antenna_height_m", "pitch_deg", "coarse_range_search_m"): - dialog._settings[key].setValue(getattr(fixture.settings, key)) - for field in dialog._refine_axes.values(): - field.setChecked(False) - return owner, controller, fixture, current_config - - def _wait_for_completion(self, controller) -> None: - deadline = time.monotonic() + 5 - while controller._worker is not None and time.monotonic() < deadline: - app.processEvents() - time.sleep(.005) - self.assertIsNone(controller._worker, "Background calculation did not complete") - - def test_actual_calculation_runs_off_gui_thread_and_saves_under_current_config(self) -> None: - owner, controller, _, current_config = self._controller_with_data() - from python_app.workflows.mimo_calibration import compute_from_collections - - worker_threads, gui_ticks = [], [] - - def compute(*args, **kwargs): - worker_threads.append(threading.get_ident()) - return compute_from_collections(*args, **kwargs) - - timer = QTimer() - timer.timeout.connect(lambda: gui_ticks.append(threading.get_ident())) - timer.start(1) - self.addCleanup(timer.stop) - with mock.patch("python_app.workflows.mimo_calibration.compute_from_collections", side_effect=compute): - controller._start_calculation() - self.assertTrue(controller._dialog._calculating) - self._wait_for_completion(controller) - owner._show_exception.assert_not_called() - self.assertEqual(len(worker_threads), 1) - self.assertNotEqual(worker_threads[0], threading.get_ident()) - self.assertTrue(gui_ticks) - self.assertEqual(set(gui_ticks), {threading.get_ident()}) - owner._radar_key.assert_called_once_with(current_config) - key, name, calibration = owner._store.save_object_calibration.call_args.args - self.assertEqual((key, name), ("current-config-key", "sphere")) - self.assertEqual(owner._selected_preprocess_sets["s21_object_calibration"], name) - self.assertEqual(len(controller._dialog._phase_plot.listDataItems()), 2) - self.assertGreater(calibration["meta"]["diagnostics"][0]["coherence_after"], .999) - - def test_cancelled_background_calculation_keeps_samples_and_creates_no_saved_set(self) -> None: - owner, controller, _, _ = self._controller_with_data() - entered = threading.Event() - - def compute(*_args, cancelled, progress, **_kwargs): - entered.set() - progress("Safe cancellation boundary") - while not cancelled(): - time.sleep(.005) - raise InterruptedError - - with mock.patch("python_app.workflows.mimo_calibration.compute_from_collections", side_effect=compute): - controller._start_calculation() - self.assertTrue(entered.wait(1)) - controller.cancel() - self._wait_for_completion(controller) - owner._store.save_object_calibration.assert_not_called() - owner._show_exception.assert_not_called() - self.assertEqual(controller._dialog._table.rowCount(), 1) - self.assertTrue(controller._dialog._capture_button.isEnabled()) - self.assertIn("отменён", controller._dialog._status_label.text()) - - def test_thread_start_failure_restores_editor_without_saving(self) -> None: - owner, controller, _, _ = self._controller_with_data() - with mock.patch("python_app.gui.controllers.object_calibration_controller.threading.Thread.start", - side_effect=RuntimeError("thread limit")): - controller._start_calculation() - self.assertIsNone(controller._worker) - self.assertFalse(controller._dialog._calculating) - self.assertTrue(controller._dialog._capture_button.isEnabled()) - owner._store.save_object_calibration.assert_not_called() - owner._show_exception.assert_called_once() - self.assertIn("thread limit", controller._dialog._status_label.text()) - - def test_capture_registration_failure_is_visible_and_does_not_leave_busy_dialog(self) -> None: - owner, controller, _, _ = self._controller_with_data() - with mock.patch("python_app.gui.controllers.object_calibration_capture.capture_object_sweeps", - side_effect=RuntimeError("Capture already running")): - controller._capture_data() - self.assertIsNone(controller._capture) - self.assertTrue(controller._dialog._capture_button.isEnabled()) - self.assertIn("Capture already running", controller._dialog._status_label.text()) - owner._show_exception.assert_called_once() - - def test_import_waits_for_capture_or_calculation_before_opening_file_picker(self) -> None: - owner, controller, _, _ = self._controller_with_data() - for busy in ("capture", "calculation"): - with self.subTest(busy=busy): - owner._object_capture_job = object() if busy == "capture" else None - controller._worker = mock.Mock() if busy == "calculation" else None - with mock.patch("python_app.gui.controllers.object_calibration_controller.QFileDialog.getExistingDirectory") as picker: - controller.import_calibration() - picker.assert_not_called() - owner._show_error.assert_called() - controller._worker = None - owner._object_capture_job = None - - def test_worker_uses_only_data_and_honours_cancellation_without_subtracting_another_background(self) -> None: - fixture = numeric_fixture.MimoCalibrationTest() - fixture.setUp() - cancelled, messages, result = threading.Event(), queue.Queue(), queue.Queue() - cancelled.set() - with mock.patch("python_app.workflows.mimo_calibration.compute_from_collections", - side_effect=InterruptedError) as compute: - _calculate([{"collections": [fixture.collection()], "target_scene_m": fixture.target}], - fixture.config, {}, cancelled, messages, result) - self.assertEqual(result.get_nowait(), ("cancelled", None)) - self.assertNotIn("empty_collections", compute.call_args.kwargs) - - def test_calculate_rejects_invalid_xyz_before_starting_a_worker(self) -> None: - owner = QWidget() - self.addCleanup(owner.close) - owner._ensure_preprocess_dialog = mock.Mock(return_value=SimpleNamespace(set_name=lambda: "sphere")) - owner._show_exception = mock.Mock() - controller = ObjectCalibrationController(owner) - controller.open_creation() - self.addCleanup(controller.shutdown) - fixture = numeric_fixture.MimoCalibrationTest() - fixture.setUp() - controller._dialog.add_capture([fixture.collection()], fixture.config) - with mock.patch("python_app.gui.controllers.object_calibration_controller.threading.Thread") as thread: - controller._start_calculation() - thread.assert_not_called() - owner._show_exception.assert_called_once() - self.assertFalse(controller._dialog._calculating) - - def test_closing_cancels_capture_and_worker_without_waiting_for_a_thread(self) -> None: - owner = QWidget() - self.addCleanup(owner.close) - controller = ObjectCalibrationController(owner) - controller._dialog = ObjectCalibrationDialog("sphere", owner) - controller._capture = mock.Mock() - controller._worker = mock.Mock() - controller.shutdown() - self.assertTrue(controller._cancel.is_set()) - controller._capture.cancel.assert_called_once() - controller._worker.join.assert_not_called() - self.assertFalse(controller._poll.isActive()) diff --git a/python_app/tests/test_object_calibration_runtime.py b/python_app/tests/test_object_calibration_runtime.py index 2ebf212..e419238 100644 --- a/python_app/tests/test_object_calibration_runtime.py +++ b/python_app/tests/test_object_calibration_runtime.py @@ -1,319 +1,294 @@ -"""Object correction persistence, runtime export and capture lifecycle regressions.""" +"""Object capture owns a pause; calculation saves against captured provenance.""" from __future__ import annotations -from copy import deepcopy -from dataclasses import asdict -import os +from contextlib import ExitStack + from pathlib import Path import tempfile -from types import SimpleNamespace import unittest from unittest import mock import numpy as np +from PyQt6 import sip +from PyQt6.QtWidgets import QApplication, QWidget -os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") -from PyQt6.QtWidgets import QApplication, QDialog # noqa: E402 - -from python_app.gui.controllers.object_calibration_capture import ObjectSweepCapture # noqa: E402 -from python_app.gui.controllers.app_window_config.live_processing_mixin import AppWindowLiveProcessingMixin # noqa: E402 -from python_app.gui.runtime.history import build_processor_run_signature, build_run_history_signature # noqa: E402 -from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData # noqa: E402 -from python_app.models.run_config_model import ( # noqa: E402 - ComboModel, GprRxGeometryModel, GprTxGeometryModel, RunConfigModel, -) -from python_app.orchestration.config_writer import ConfigWriter # noqa: E402 -from python_app.orchestration.preprocess_assets import runtime_preprocess_asset_keys # noqa: E402 -from python_app.orchestration.shm.decoder import decode_trace_collection # noqa: E402 -from python_app.storage.npz.store import NpzStore # noqa: E402 -from python_app.storage.npz.serialize import RAW_MAGIC # noqa: E402 -from python_app.workflows.mimo_calibration import C_LIGHT, calibration_on_grid # noqa: E402 +from python_app.gui.controllers.history_controller import HistoryController +from python_app.gui.controllers.object_calibration_capture import ObjectSweepCapture +from python_app.gui.controllers.pipeline_controller import PipelineController +from python_app.gui.controllers.plot_controller import PlotController +from python_app.gui.controllers.preprocess_controller import PreprocessController +from python_app.gui.controllers.recording_controller import RecordingController +from python_app.gui.runtime.pipeline_pause import PipelinePause +from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData +from python_app.models.gui_profile_schema import GuiPreprocessDialogStateModel +from python_app.models.run_config_schema import ComboModel, GprRxGeometryModel, GprTxGeometryModel, RunConfigModel +from python_app.orchestration.config_writer import ConfigWriter +from python_app.orchestration.live_processing_config import ProcessingLiveConfig +from python_app.orchestration.process_supervisor import ProcessSupervisor +from python_app.storage.npz.store import NpzStore +from python_app.workflows.object_calibration import capture_provenance +from python_app.workflows.sequential_capture_workflow import CaptureBatch, SequentialCaptureSession, SequentialCaptureState def setUpModule() -> None: - global _app - _app = QApplication.instance() or QApplication([]) + global app + app = QApplication.instance() or QApplication([]) def _config() -> RunConfigModel: config = RunConfigModel() - config.input_switch.positions = 3 - config.output_switch.positions = 4 + config.input_switch.positions, config.output_switch.positions = 3, 4 config.combos = [ComboModel(0, 1), ComboModel(2, 3)] config.gpr.tx_geometry = [GprTxGeometryModel(1, -.4), GprTxGeometryModel(3, .4)] config.gpr.rx_geometry = [GprRxGeometryModel(0, -.2), GprRxGeometryModel(2, .2)] config.preprocess.s21.calibration.set_name = "through" config.preprocess.s21.reference.set_name = "empty" + config.rings.preprocessed_tap.name = "/object_capture_test_preprocessed_tap" return config -def _calibration(config: RunConfigModel) -> dict: - frequency = np.array([1e8, 2e8, 3e8]) - parameters = np.array([[.3, .02, 0], [-.8, -.01, 0], [.4, .03, 0], [-.1, -.02, 0]]) - corrections = np.exp(-1j * (parameters[:, 0, None] - 2 * np.pi * frequency * parameters[:, 1, None] / C_LIGHT)) - return dict( - freq_hz=frequency, tx=corrections[:2], rx=corrections[2:], - pair=corrections[:2, None] * corrections[None, 2:], valid=np.ones(3, bool), - tx_output_pos=np.array([1, 3], np.int32), rx_input_pos=np.array([0, 2], np.int32), - antenna_params=parameters, dir=None, - meta={"geometry": { - "tx_xyz_m": [[entry.x_m, entry.y_m, entry.z_m] for entry in config.gpr.tx_geometry], - "rx_xyz_m": [[entry.x_m, entry.y_m, entry.z_m] for entry in config.gpr.rx_geometry], - }}, - ) - - -class ObjectCalibrationStoreTest(unittest.TestCase): - def setUp(self) -> None: - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - self.root = Path(temporary.name) - self.store = NpzStore(self.root / "sets") - self.config = _config() - self.calibration = _calibration(self.config) - traces = [ - TraceData(ComboKey(0, 1), np.array([1e8, 2e8, 3e8], np.float32), np.ones(3), np.ones(3)), - TraceData(ComboKey(2, 3), np.array([1.2e8, 1.8e8, 2.6e8], np.float32), np.ones(3), np.ones(3)), - ] - for kind, name in (("s21_calibration", "through"), ("s21_reference", "empty")): - self.store.save_set(kind, "radar", name, SweepCollection(1, 10, traces)) - - def test_default_disabled_and_legacy_config_round_trip(self) -> None: - payload = self.config.to_dict() - del payload["preprocess"]["s21"]["object_calibration"] - config = RunConfigModel.from_dict(payload) - self.assertEqual(config.preprocess.s21.object_calibration.set_name, "") - self.assertNotIn("s21_object_calibration", runtime_preprocess_asset_keys(config)) - original_run = build_run_history_signature(config) - original_processor = build_processor_run_signature(config) - config.preprocess.s21.object_calibration.set_name = "object" - self.assertEqual(config.clone().preprocess.s21.object_calibration.set_name, "object") - self.assertIn("s21_object_calibration", runtime_preprocess_asset_keys(config)) - self.assertNotEqual(original_run, build_run_history_signature(config)) - self.assertEqual(original_processor, build_processor_run_signature(config)) - - def test_saved_set_retains_coefficients_and_refuses_overwrite(self) -> None: - self.store.save_object_calibration("radar", "object", self.calibration) - self.assertEqual(self.store.list_sets("s21_object_calibration", "radar"), ["object"]) - loaded = self.store.load_object_calibration("radar", "object") - np.testing.assert_array_equal(loaded["pair"], self.calibration["pair"]) - with self.assertRaises(FileExistsError): - self.store.save_object_calibration("radar", "object", self.calibration) - self.assertTrue(self.store.has_combo_coverage("s21_object_calibration", "radar", "object", [ComboKey(2, 3)])) - self.assertFalse(self.store.has_combo_coverage("s21_object_calibration", "radar", "object", [ComboKey(1, 3)])) - with self.assertRaises(ValueError): - self.store.load_object_calibration("radar", "../object") - - def test_writer_exports_physical_pair_factors_on_each_runtime_grid(self) -> None: - self.store.save_object_calibration("radar", "object", self.calibration) - self.config.preprocess.s21.object_calibration.set_name = "object" - ConfigWriter(self.root / "runtime").prepare_preprocess_bundles(self.store, "radar", self.config) - bundle = Path(self.config.preprocess.s21.object_calibration.bundle_path) - collection = decode_trace_collection(bundle.read_bytes(), RAW_MAGIC) - self.assertEqual([trace.combo for trace in collection.traces], [ComboKey(0, 1), ComboKey(2, 3)]) - for index, trace in enumerate(collection.traces): - np.testing.assert_allclose(trace.s21, calibration_on_grid(self.calibration, trace.frequency_hz)[index, index], rtol=1e-6) - np.testing.assert_array_equal(trace.s11, np.zeros(3)) - self.config.preprocess.s21.object_calibration.set_name = "" - ConfigWriter(self.root / "runtime").prepare_preprocess_bundles(self.store, "radar", self.config) - self.assertEqual(self.config.preprocess.s21.object_calibration.bundle_path, "") - - def test_invalid_geometry_or_correction_keeps_previous_export(self) -> None: - self.store.save_object_calibration("radar", "object", self.calibration) - destination = self.root / "factors.bin" - destination.write_bytes(b"previous bundle") - self.config.gpr.tx_geometry[0].z_m = .1 - with self.assertRaisesRegex(ValueError, "geometry"): - self.store.export_object_calibration_bundle("radar", "object", destination, self.config) - self.assertEqual(destination.read_bytes(), b"previous bundle") - broken = deepcopy(self.calibration) - broken["pair"][0, 0, 0] = np.nan - with self.assertRaises(ValueError): - self.store.save_object_calibration("radar", "bad", broken) - self.assertEqual(self.store.list_sets("s21_object_calibration", "radar"), ["object"]) +def _through() -> SweepCollection: + return SweepCollection(1, 10, [ + TraceData(ComboKey(i, o), np.array([1e8, 2e8, 3e8], np.float32), np.ones(3, np.complex64)) + for o in (1, 3) for i in (0, 2) + ]) class ObjectCaptureLifecycleTest(unittest.TestCase): def setUp(self) -> None: - self.dialog = mock.Mock() - self.owner = SimpleNamespace( - _stop_request_seq=0, _capture_session=None, _preprocess_capture_busy=False, - _single_capture_active=False, _recording_active=False, _closing=False, - _build_config=lambda: _config(), _pre_history=[], _SINGLE_CAPTURE_TIMEOUT_S=300, - _ensure_preprocess_dialog=lambda: SimpleNamespace(median_sweep_count=lambda: 2), - _supervisor=SimpleNamespace(is_running=lambda: self.running), + cleanup = ExitStack() + self.addCleanup(cleanup.close) + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.config = _config() + self.store = NpzStore(self.root / "sets") + self.store.save_set("s21_calibration", self.config.radar_key(), "through", _through()) + self.parent = QWidget() + self.addCleanup(self.parent.close) + self.running = False + self.supervisor = mock.create_autospec(ProcessSupervisor, instance=True) + self.supervisor.is_running.side_effect = lambda: self.running + self.supervisor.is_processor_running.return_value = False + self.supervisor.pids.side_effect = lambda: {"acquisition": 1, "data_preprocessor": 2} if self.running else {} + self.history = HistoryController() + self.plots = mock.create_autospec(PlotController, instance=True) + self.recording = RecordingController(self.store, {}, mock.Mock(), mock.Mock(), mock.Mock()) + self.error = mock.Mock() + self.pipeline = PipelineController( + self.root, self.store, ConfigWriter(self.root / "runtime"), self.history, self.plots, self.recording, + build_config=lambda: self.config.clone(), live_config=ProcessingLiveConfig, publish_live=mock.Mock(), + capture_guard=lambda: self.preprocess.busy_reason(), refresh_radar_limits=lambda: True, + status=mock.Mock(), history_changed=mock.Mock(), poll_ui=mock.Mock(), show_exception=self.error, + deferred_close=mock.Mock(), ) - self.running = False - self.owner._pipeline_resume_allowed = lambda seq: not self.owner._closing and seq == self.owner._stop_request_seq - self.owner._stop_run = mock.Mock(side_effect=self._stop) - self.owner._start_run = mock.Mock(side_effect=self._start) + self.pipeline.supervisor = self.supervisor + self.start = cleanup.enter_context(mock.patch.object(self.pipeline, "start", side_effect=self._start)) + self.stop = cleanup.enter_context(mock.patch.object(self.pipeline, "stop_pipeline", side_effect=self._stop)) + self.preprocess = PreprocessController( + self.parent, self.config, GuiPreprocessDialogStateModel(median_sweep_count=2), + self.store, self.pipeline, self.plots, build_config=lambda: self.config.clone(), + current_radar_key=lambda: self.config.radar_key(), geometry=lambda: self.config.gpr, + summary=mock.Mock(), error=self.error, + ) + self.addCleanup(self.preprocess.shutdown) + self.controller = self.preprocess.object_calibration + self.controller.open_creation("object") + assert self.controller.dialog is not None + self.dialog = self.controller.dialog + self.add_capture = cleanup.enter_context(mock.patch.object(self.dialog, "add_capture", wraps=self.dialog.add_capture)) + cleanup.enter_context(mock.patch("python_app.gui.controllers.object_calibration_capture.QTimer.singleShot")) - def _stop(self) -> None: - self.running = False - self.owner._stop_request_seq += 1 + def _stop(self, *, keep_processor: bool) -> None: + self.running, self.pipeline.should_run = False, False - def _start(self, **_kwargs) -> None: - self.running = True + def _start(self, *, config_override: RunConfigModel | None = None, pause: PipelinePause | None = None) -> bool: + if pause is not None: + if not pause.valid: + return False + elif self.preprocess.busy_reason() is not None: + raise RuntimeError("busy job was not cleared before resume") + self.running, self.pipeline.should_run = True, pause is None + return True def _capture(self) -> ObjectSweepCapture: - with mock.patch("python_app.gui.controllers.object_calibration_capture.QTimer.singleShot"): - view = QDialog() - view.add_capture = self.dialog.add_capture - view.set_status = self.dialog.set_status - view.set_capture_pending = self.dialog.set_capture_pending - self.addCleanup(view.close) - self.view = view - return ObjectSweepCapture(self.owner, view) - - def test_captures_fresh_preprocessed_matrix_without_changing_selected_object(self) -> None: - config = _config() - config.preprocess.s21.object_calibration.set_name = "existing" - self.owner._build_config = lambda: config.clone() - self.running = True - capture = self._capture() + self.controller.capture_data() + capture = self.controller.capture + assert capture is not None capture._start() - override = self.owner._start_run.call_args.kwargs["config_override"] + return capture + + def test_missing_tap_and_replay_report_error_before_pausing(self) -> None: + for invalid in ("tap", "replay"): + with self.subTest(invalid=invalid): + self.config = _config() + if invalid == "tap": + self.config.rings.preprocessed_tap.name = "" + else: + self.config.radar.replay.dataset_dir = "/recorded" + self.controller.capture_data() + self.assertIsNone(self.controller.capture) + self.assertFalse(self.dialog._capture_pending) + self.stop.assert_not_called() + self.start.assert_not_called() + self.assertEqual(self.error.call_count, 2) + + def test_preprocess_refresh_signals_report_invalid_pending_config(self) -> None: + dialog = self.preprocess.ensure_dialog() + with mock.patch.object(self.preprocess, "_current_radar_key", side_effect=ValueError("incomplete sweep")): + dialog.refresh_requested.emit() + dialog.radar_config_dir_changed.emit() + dialog.multi_radar_option_changed.emit() + self.assertEqual(self.error.call_count, 3) + self.assertEqual(self.error.call_args.args[0], "Failed to refresh preprocessing sets") + + def test_queued_capture_update_ignores_dialog_deleted_after_close(self) -> None: + dialog = self.preprocess.ensure_dialog() + session = mock.create_autospec(SequentialCaptureSession, instance=True) + session.is_complete.return_value = False + session.state.return_value = SequentialCaptureState("s21_reference", "reference", 1, 2, ComboModel(0, 1), True, False) + session.kind = "s21_reference" + session.capture_current_combo.return_value = CaptureBatch(ComboModel(0, 0), (_through().traces[0],)) + self.preprocess.capture_session = session + with mock.patch.object(self.preprocess, "_record_capture"), \ + mock.patch("python_app.gui.controllers.preprocess_controller.QTimer.singleShot") as scheduled: + self.preprocess.capture_next() + self.assertEqual(scheduled.call_count, 1) + callback = scheduled.call_args.args[1] + self.pipeline.begin_close() + self.preprocess.shutdown() + sip.delete(dialog) + try: + self.assertTrue(sip.isdeleted(dialog)) + callback() + finally: + self.preprocess.dialog = None + + def test_captures_fresh_exact_keys_and_clears_busy_job_before_resume(self) -> None: + self.config.preprocess.s21.object_calibration.set_name = "existing" + self.running = self.pipeline.should_run = True + capture = self._capture() + override: RunConfigModel = self.start.call_args.kwargs["config_override"] self.assertEqual(override.preprocess.s21.object_calibration.set_name, "") self.assertEqual(len(override.combos), 4) - self.assertEqual(config.preprocess.s21.object_calibration.set_name, "existing") - stale = SweepCollection(1, capture._start_ns - 1) - first, second = SweepCollection(2, capture._start_ns + 1), SweepCollection(3, capture._start_ns + 2) - self.owner._pre_history = [stale, first, first, second] + self.assertEqual(self.config.preprocess.s21.object_calibration.set_name, "existing") + stale, first, second = [SweepCollection(1, capture._start_ns + 100, + capture_start_ns=capture._start_ns + offset, capture_end_ns=capture._start_ns + offset + 1) + for offset in (-1, 1, 2)] + self.history.preprocessed.extend([stale, first, first, second]) capture._poll() - self.dialog.add_capture.assert_called_once() - self.assertEqual([c.collection_id for c in self.dialog.add_capture.call_args.args[0]], [2, 3]) - self.assertEqual(self.owner._start_run.call_count, 2) - self.assertEqual(self.owner._start_run.call_args.kwargs, {}) + self.add_capture.assert_called_once() + self.assertEqual(self.add_capture.call_args.args[0], [first, second]) + self.assertEqual(self.add_capture.call_args.args[2], + capture_provenance(override.to_dict(), override.radar_key(), _through())) + self.assertEqual(self.start.call_count, 2) + self.assertIsNone(self.preprocess.object_capture) + self.assertFalse(self.pipeline.paused) capture.cancel() - self.assertEqual(self.owner._stop_run.call_count, 2) + self.assertEqual(self.stop.call_count, 2) + self.assertEqual(self.pipeline.stop_generation, 0) - def test_stop_cancels_capture_and_prevents_automatic_resume(self) -> None: - self.running = True + def test_stop_cancels_restart_intent_but_capture_cancel_restores_backoff(self) -> None: + for backoff, stop in ((False, True), (True, True), (True, False)): + with self.subTest(backoff=backoff, stop=stop): + self.running, self.pipeline.should_run = not backoff, backoff + self.start.reset_mock() + capture = self._capture() + self.assertTrue(self.pipeline.paused) + self.assertFalse(self.pipeline.should_run) + capture._poll() + self.assertEqual(self.start.call_count, 1) + if stop: + self.pipeline.stop() + capture._poll() + else: + capture.cancel() + self.add_capture.assert_not_called() + self.assertEqual(self.start.call_count, 1 if stop else 2) + self.assertEqual(self.running, not stop) + self.assertEqual(self.pipeline.should_run, not stop) + self.assertFalse(self.pipeline.paused) + self.assertIsNone(self.preprocess.object_capture) + + def test_missing_producer_finishes_without_waiting_for_timeout(self) -> None: capture = self._capture() - capture._start() - self.owner._stop_run() + self.supervisor.pids.side_effect = None + self.supervisor.pids.return_value = {"data_preprocessor": 2} capture._poll() - self.dialog.add_capture.assert_not_called() - self.assertEqual(self.owner._start_run.call_count, 1) - self.assertFalse(self.running) + self.assertIsNone(self.preprocess.object_capture) + self.assertFalse(self.pipeline.paused) + self.assertIn("stopped before all sweeps", self.dialog._status_label.text()) + self.add_capture.assert_not_called() - def test_recording_is_not_interrupted(self) -> None: - self.owner._recording_active = True - capture = self._capture() - capture._start() - self.owner._start_run.assert_not_called() - self.owner._stop_run.assert_not_called() - self.dialog.add_capture.assert_not_called() - self.dialog.set_capture_pending.assert_called_once_with(False) + def test_cancel_during_start_keeps_lease_until_start_returns(self) -> None: + self.running = self.pipeline.should_run = True - def test_stop_before_queued_start_never_opens_acquisition(self) -> None: - capture = self._capture() - self.owner._stop_run() - capture._start() - self.owner._start_run.assert_not_called() - self.assertIsNone(self.owner._object_capture_job) + def start(*, config_override: RunConfigModel | None = None, pause: PipelinePause | None = None) -> bool: + if pause is not None: + capture = self.preprocess.object_capture + assert capture is not None + capture.cancel() + self.assertIs(self.preprocess.object_capture, capture) + self.assertTrue(pause.valid) + return self._start(config_override=config_override, pause=pause) - def test_stale_capture_does_not_stop_a_new_run_after_explicit_stop(self) -> None: - capture = self._capture() - capture._start() - self.owner._stop_run() - self.owner._start_run() - capture._poll() - self.assertEqual(self.owner._stop_run.call_count, 1) + self.start.side_effect = start + self._capture() + self.assertEqual(self.start.call_count, 2) + self.assertEqual(self.stop.call_count, 2) + self.assertIsNone(self.preprocess.object_capture) + self.assertFalse(self.pipeline.paused) self.assertTrue(self.running) - self.dialog.add_capture.assert_not_called() + self.add_capture.assert_not_called() - def test_second_capture_is_rejected_without_mutating_first(self) -> None: - first = self._capture() - with self.assertRaisesRegex(RuntimeError, "already in progress"): - self._capture() - self.assertIs(self.owner._object_capture_job, first) - first.cancel() + def test_stop_or_close_before_deferred_capture_start_does_not_touch_pipeline(self) -> None: + for closing in (False, True): + with self.subTest(closing=closing): + self.controller.capture_data() + capture = self.controller.capture + assert capture is not None + if closing: + self.pipeline.begin_close() + else: + self.pipeline.stop_generation += 1 + capture._start() + self.start.assert_not_called() + self.stop.assert_not_called() + self.assertIsNone(self.preprocess.object_capture) - def test_rejected_data_still_restores_original_running_pipeline(self) -> None: - self.running = True - self.dialog.add_capture.side_effect = ValueError("configuration changed") - capture = self._capture() - capture._start() - self.owner._pre_history = [SweepCollection(2, capture._start_ns + 1), SweepCollection(3, capture._start_ns + 2)] - capture._poll() - self.assertTrue(self.running) - self.assertEqual(self.owner._start_run.call_count, 2) - self.assertIsNone(self.owner._object_capture_job) - - def test_capture_lock_is_held_while_stop_drains_events(self) -> None: - capture = self._capture() - capture._start() - locks = [] - original_stop = self.owner._stop_run.side_effect - self.owner._stop_run.side_effect = lambda: (locks.append(self.owner._object_capture_job), original_stop()) - capture.cancel() - self.assertEqual(locks, [capture]) - self.assertIsNone(self.owner._object_capture_job) - - def test_old_acquisition_with_new_preprocessing_timestamp_is_not_captured(self) -> None: - capture = self._capture() - capture._start() - stale = SweepCollection(1, capture._start_ns + 100, capture_start_ns=capture._start_ns - 100) - self.owner._pre_history = [stale] - capture._poll() - self.assertEqual(capture._collections, []) - capture.cancel() - - def test_replay_is_rejected_before_acquisition(self) -> None: - config = _config() - config.radar.replay.dataset_dir = "saved_scene" - self.owner._build_config = lambda: config - capture = self._capture() - capture._start() - self.owner._start_run.assert_not_called() - - -class ObjectGeometryLifecycleTest(unittest.TestCase): - def setUp(self) -> None: - config = _config() - config.preprocess.s21.object_calibration.set_name = "object" - self.tx, self.rx = [asdict(entry) for entry in config.gpr.tx_geometry], [asdict(entry) for entry in config.gpr.rx_geometry] - self.owner = SimpleNamespace( - _selected_preprocess_sets={"s21_object_calibration": "object"}, _selected_preprocess_radar_key="radar", - _store=SimpleNamespace(load_object_calibration=mock.Mock(return_value=_calibration(config))), - _gpr_tx_geometry_input=SimpleNamespace(records=lambda: self.tx), - _gpr_rx_geometry_input=SimpleNamespace(records=lambda: self.rx), - _supervisor=SimpleNamespace(is_running=lambda: True), _preprocess_dialog=mock.Mock(), - _stop_request_seq=0, _refresh_preprocess_summary_labels=mock.Mock(), _start_run=mock.Mock(), _log_warning=mock.Mock(), - _active_run_config=config, _radar_key=lambda _config: "radar", - ) - self.owner._pipeline_resume_allowed = lambda seq: seq == self.owner._stop_request_seq - self.owner._stop_run = mock.Mock(side_effect=lambda: setattr(self.owner, "_stop_request_seq", self.owner._stop_request_seq + 1)) - - def test_same_geometry_retains_object_correction_without_restart(self) -> None: - AppWindowLiveProcessingMixin._invalidate_object_calibration_after_geometry_change(self.owner) - self.assertEqual(self.owner._selected_preprocess_sets["s21_object_calibration"], "object") - self.owner._stop_run.assert_not_called() - self.owner._start_run.assert_not_called() - - def test_changed_geometry_disables_correction_and_reloads_acquisition(self) -> None: - self.tx[0]["z_m"] = .1 - AppWindowLiveProcessingMixin._invalidate_object_calibration_after_geometry_change(self.owner) - self.assertEqual(self.owner._selected_preprocess_sets["s21_object_calibration"], "") - self.owner._stop_run.assert_called_once() - self.owner._start_run.assert_called_once() - self.owner._log_warning.assert_called_once() - - def test_explicit_stop_during_geometry_change_prevents_resume(self) -> None: - self.tx[0]["z_m"] = .1 - self.owner._stop_run.side_effect = lambda: setattr(self.owner, "_stop_request_seq", self.owner._stop_request_seq + 2) - AppWindowLiveProcessingMixin._invalidate_object_calibration_after_geometry_change(self.owner) - self.owner._start_run.assert_not_called() - - def test_pending_off_selection_still_reloads_an_incompatible_active_correction(self) -> None: - self.owner._selected_preprocess_sets["s21_object_calibration"] = "" - self.tx[0]["z_m"] = .1 - AppWindowLiveProcessingMixin._invalidate_object_calibration_after_geometry_change(self.owner) - self.owner._stop_run.assert_called_once() - self.owner._start_run.assert_called_once() - - -if __name__ == "__main__": - unittest.main() + def test_calculation_preserves_captures_and_saves_only_success_with_original_provenance(self) -> None: + captured = _config() + provenance = capture_provenance(captured.to_dict(), captured.radar_key(), _through()) + metadata = {"provenance": provenance, "targets": [], "diagnostics": [], + "tx_output_pos": [1, 3], "rx_input_pos": [0, 2]} + self.dialog.add_capture([_through()], captured.to_dict(), provenance) + for name, error in (("matching", None), ("changed", None), + ("cancelled", InterruptedError("cancelled")), ("failed", ValueError("bad input"))): + with self.subTest(outcome=name): + self.config = captured.clone() + if name == "changed": + self.config.radar.sweep.points += 1 + self.dialog._set_name_input.setText(name) + with mock.patch.object(self.dialog, "calculation_inputs", return_value=([], captured.to_dict(), {}, provenance)), \ + mock.patch("python_app.workflows.mimo_calibration.compute_from_collections", + return_value=(_through(), metadata), side_effect=error): + self.controller._start_calculation() + worker = self.controller._worker + assert worker is not None + worker.join(timeout=2) + self.assertFalse(worker.is_alive()) + self.controller._poll_calculation() + self.assertEqual(self.dialog._table.rowCount(), 1) + self.assertFalse(self.dialog._calculating) + self.assertEqual(self.preprocess.selected_sets["s21_object_calibration"], "matching") + if error is not None: + self.assertNotIn(name, self.store.list_sets("s21_object_calibration", captured.radar_key())) + else: + saved, saved_meta = self.store.load_set_with_metadata("s21_object_calibration", captured.radar_key(), name) + self.assertEqual(saved_meta["provenance"], provenance) + self.assertEqual(len(saved.traces), 4) + expected = {"matching": "Applies on the next Start", "changed": "Not selected", + "cancelled": "cancelled", "failed": "failed"} + self.assertIn(expected[name], self.dialog._status_label.text()) + self.error.assert_called_once() diff --git a/python_app/tests/test_orchestration_recovery.py b/python_app/tests/test_orchestration_recovery.py index f812949..e83c0fd 100644 --- a/python_app/tests/test_orchestration_recovery.py +++ b/python_app/tests/test_orchestration_recovery.py @@ -21,8 +21,8 @@ import unittest from pathlib import Path from unittest import mock -from python_app.models.run_config_model import RunConfigModel -from python_app.gui.controllers.app_window_pipeline_mixin import AppWindowPipelineMixin +from python_app.models.run_config_schema import RunConfigModel +from python_app.tests.test_reader_recovery import pipeline_fixture from python_app.orchestration.config_writer import ConfigWriter from python_app.orchestration.process_supervisor import ProcessExitReport, ProcessSupervisor from python_app.orchestration.restart_policy import RestartPolicy @@ -52,66 +52,48 @@ class RestartPolicyTest(unittest.TestCase): self.assertTrue(policy.should_restart_now(now_s=112.0, last_restart_s=100.0, consecutive_failures=2)) def test_first_restart_is_immediate(self) -> None: - # last_restart defaults to 0.0 in the mixin, so the very first crash restarts now. + # last_restart defaults to 0.0 in PipelineController, so the very first crash restarts now. self.assertTrue(RestartPolicy().should_restart_now(now_s=5_000.0, last_restart_s=0.0, consecutive_failures=0)) class PipelineRecoveryPollingTest(unittest.TestCase): def setUp(self) -> None: - self.window = mock.Mock(spec=AppWindowPipelineMixin) - self.window._pipeline_should_run = True - self.window._supervisor = mock.Mock() - self.window._supervisor.collect_exit_reports.return_value = [] + self.pipeline, self.supervisor = pipeline_fixture(self) + self.pipeline.should_run = True - def test_missing_child_keeps_retrying_without_fresh_exit_reports(self) -> None: - self.window._supervisor.pids.return_value = {"data_processor": 1, "data_preprocessor": 2} - for _ in range(3): - AppWindowPipelineMixin._handle_process_exit_reports(self.window) - self.assertEqual(self.window._recover_pipeline_after_crash.call_count, 3) - - def test_failed_start_with_no_children_keeps_retrying(self) -> None: - self.window._supervisor.pids.return_value = {} - for _ in range(3): - AppWindowPipelineMixin._handle_process_exit_reports(self.window) - self.assertEqual(self.window._recover_pipeline_after_crash.call_count, 3) + def test_missing_children_keep_retrying_without_fresh_exit_reports(self) -> None: + with mock.patch.object(self.pipeline, "_recover_pipeline_after_crash") as recover: + for pids in ({"data_processor": 1, "data_preprocessor": 2}, {}): + with self.subTest(pids=pids): + recover.reset_mock() + self.supervisor.pids.return_value = pids + for _ in range(3): + self.pipeline.poll() + self.assertEqual(recover.call_count, 3) def test_failed_restarts_retry_after_backoff_without_new_exit_reports(self) -> None: - self.window._supervisor.pids.return_value = {} - self.window._supervisor.start.side_effect = RuntimeError("device unavailable") - self.window._active_run_config = RunConfigModel() - self.window._active_run_config_path = Path("run_config.json") - self.window._pipeline_restart_count = 0 - self.window._last_pipeline_restart_s = 0.0 - self.window._RESTART_POLICY = RestartPolicy() - self.window._log_error = mock.Mock() - self.window._log_exception = mock.Mock() - self.window._recover_pipeline_after_crash = ( - lambda: AppWindowPipelineMixin._recover_pipeline_after_crash(self.window) - ) - - with mock.patch( - "python_app.gui.controllers.app_window_pipeline_mixin.time.monotonic", - side_effect=[100.0, 101.0, 106.0], - ): + self.supervisor.start.side_effect = RuntimeError("device unavailable") + self.pipeline.active_config = RunConfigModel() + self.pipeline.active_config_path = Path("run_config.json") + with mock.patch("python_app.gui.controllers.pipeline_controller.time.monotonic", + side_effect=[100.0, 101.0, 106.0]): for _ in range(3): - AppWindowPipelineMixin._handle_process_exit_reports(self.window) - - self.assertEqual(self.window._supervisor.start.call_count, 2) - self.assertEqual(self.window._pipeline_restart_count, 2) - self.assertTrue(self.window._pipeline_should_run) + self.pipeline.poll() + self.assertEqual(self.supervisor.start.call_count, 2) + self.assertEqual(self.pipeline._restart_count, 2) + self.assertTrue(self.pipeline.should_run) def test_healthy_pipeline_does_not_restart(self) -> None: - self.window._supervisor.pids.return_value = { - "data_processor": 1, "data_preprocessor": 2, "sweep_orchestrator": 3, - } - AppWindowPipelineMixin._handle_process_exit_reports(self.window) - self.window._recover_pipeline_after_crash.assert_not_called() + self.supervisor.pids.return_value = {"data_processor": 1, "data_preprocessor": 2, "acquisition": 3} + self.pipeline.poll() + self.supervisor.start.assert_not_called() def test_explicit_stop_does_not_restart_missing_children(self) -> None: - self.window._pipeline_should_run = False - self.window._supervisor.pids.return_value = {} - AppWindowPipelineMixin._handle_process_exit_reports(self.window) - self.window._recover_pipeline_after_crash.assert_not_called() + self.pipeline.stop() + self.pipeline.poll() + self.supervisor.start.assert_not_called() + self.assertFalse(self.pipeline.should_run) + self.assertEqual(self.pipeline.stop_generation, 1) class ConfigWriterTest(unittest.TestCase): @@ -154,10 +136,6 @@ class RadarModelCommandTest(unittest.TestCase): path.write_text(json.dumps({"radar": {"model": model, "driver_mode": driver_mode}})) return path - def test_malformed_root_falls_back_to_native_producer(self) -> None: - path = self.root / "bad.json" - path.write_text("[]") - self.assertTrue(self.supervisor._acquisition_command(path)[0].endswith("sweep_orchestrator")) def test_null_replay_directory_does_not_select_replay(self) -> None: path = self._config_with_model("librevna") @@ -180,14 +158,12 @@ class RadarModelCommandTest(unittest.TestCase): cmd = self.supervisor._acquisition_command(self._config_with_model("kamil_adc")) self.assertEqual(cmd[:3], [sys.executable, "-m", "python_app.scripts.kamil_adc_raw_producer"]) - def test_native_orchestrator_for_librevna(self) -> None: - cmd = self.supervisor._acquisition_command(self._config_with_model("librevna")) - self.assertTrue(cmd[0].endswith("build/bin/sweep_orchestrator")) - def test_single_and_multi_mock_share_the_simulator(self) -> None: + def test_single_and_multi_use_python_in_both_modes(self) -> None: for model in ("librevna", "librevna_multi"): - command = self.supervisor._acquisition_command(self._config_with_model(model, "mock")) - self.assertEqual(command[:3], [sys.executable, "-m", "python_app.scripts.matrix_raw_producer"]) + for mode in ("native", "mock"): + command = self.supervisor._acquisition_command(self._config_with_model(model, mode)) + self.assertEqual(command[:3], [sys.executable, "-m", "python_app.scripts.matrix_raw_producer"]) def test_omitted_driver_mode_uses_the_default_simulator(self) -> None: path = self._config_with_model("librevna") @@ -203,11 +179,6 @@ class RadarModelCommandTest(unittest.TestCase): self.assertEqual(self.supervisor._acquisition_command(path)[:3], [sys.executable, "-m", "python_app.scripts.matrix_raw_producer"]) - def test_omitted_model_with_native_mode_uses_native_producer(self) -> None: - path = self.root / "native.json" - path.write_text(json.dumps({"radar": {"driver_mode": "native"}})) - self.assertTrue(self.supervisor._acquisition_command(path)[0].endswith("sweep_orchestrator")) - class ProcessExitReportTest(unittest.TestCase): @staticmethod @@ -273,7 +244,7 @@ class SupervisorLifecycleTest(unittest.TestCase): self.assertEqual(reports[0].level, "ERROR") def test_clean_exit_reported_as_info(self) -> None: - self.supervisor._spawn("sweep_orchestrator", + self.supervisor._spawn("acquisition", [sys.executable, "-c", "import sys; sys.exit(0)"], allow_clean_exit=True) reports = self._await_reports() diff --git a/python_app/tests/test_persistence_integrity.py b/python_app/tests/test_persistence_integrity.py index 3880c18..1f3a52f 100644 --- a/python_app/tests/test_persistence_integrity.py +++ b/python_app/tests/test_persistence_integrity.py @@ -2,8 +2,11 @@ from __future__ import annotations import json +import os from pathlib import Path +import stat import tempfile +from typing import cast import unittest from unittest import mock @@ -13,10 +16,9 @@ from python_app.models.dataset_model import ( ComboKey, ResultBlock, ResultCollection, ResultPayload, SweepCollection, TraceData, ) from python_app.storage.npz.store import NpzStore -from python_app.storage.npz.snapshot_numpy import select_aligned_histories from python_app.storage.npz.vna_history_json import build_vna_history_payload from python_app.scripts.replay_raw_producer import _load_dataset -from python_app.scripts.convert_snapshot_to_vna_history import _collection_dirs, _load_stage_records, _build_sweep_history +from python_app.storage.npz.vna_history_json import collection_directories, load_vna_history_snapshot from python_app.workflows.sequential_capture_workflow import combine_collections_via_median, combine_traces_via_median @@ -25,7 +27,6 @@ def _sweep(cid: int = 1, ns: int = 20, *, start: int = 10, end: int = 15) -> Swe collection_id=cid, monotonic_ns=ns, capture_start_ns=start, capture_end_ns=end, traces=[TraceData( combo=ComboKey(0, 0), frequency_hz=np.array([1e9, 2e9], dtype=np.float32), - s11=np.array([1 + 2j, 3 - 4j], dtype=np.complex64), s21=np.array([5 - 6j, 7 + 8j], dtype=np.complex64), capture_start_ns=start, capture_end_ns=end, )], @@ -52,7 +53,7 @@ class PersistenceIntegrityTest(unittest.TestCase): self.output = self.root / 'output' def test_all_calibration_and_reference_kinds_preserve_arrays_and_timing(self) -> None: - for kind in ('s21_calibration', 's21_reference', 's11_open', 's11_short', 's11_load', 's11_reference'): + for kind in ('s21_calibration', 's21_reference'): with self.subTest(kind=kind): original = _sweep(7, 100) second = _sweep().traces[0] @@ -67,22 +68,10 @@ class PersistenceIntegrityTest(unittest.TestCase): for before, after in zip(original.traces, loaded.traces, strict=True): self.assertEqual(before.combo, after.combo) self.assertEqual((after.capture_start_ns, after.capture_end_ns), (10, 15)) - for field in ('frequency_hz', 's11', 's21'): + for field in ('frequency_hz', 's21'): np.testing.assert_array_equal(getattr(before, field), getattr(after, field)) - def test_new_set_remains_loadable_without_json_sidecar(self) -> None: - self.store.save_set('s21_reference', 'radar', 'full', _sweep()) - (self.root / 'sets/s21_reference/radar/full.json').unlink() - loaded = self.store.load_set('s21_reference', 'radar', 'full') - np.testing.assert_array_equal(loaded.traces[0].s21, _sweep().traces[0].s21) - def test_legacy_npz_json_pair_is_still_loadable(self) -> None: - self.store.save_set('s21_reference', 'radar', 'full', _sweep()) - archive = self.root / 'sets/s21_reference/radar/full.npz' - with np.load(archive, allow_pickle=False) as arrays: - legacy = {key: arrays[key] for key in arrays.files if key != 'metadata_json'} - np.savez(archive, **legacy) - self.assertEqual(self.store.load_set('s21_reference', 'radar', 'full').monotonic_ns, 20) def test_failed_set_commit_preserves_previous_data_and_sidecar(self) -> None: self.store.save_set('s21_reference', 'radar', 'full', _sweep(1, 20)) @@ -130,13 +119,59 @@ class PersistenceIntegrityTest(unittest.TestCase): self.assertEqual(self.store.load_set('s21_reference', 'radar', 'full').collection_id, 1) self.assertEqual(list(self.root.rglob('*.tmp')), []) + def test_set_files_are_synced_before_commit_and_directory_afterwards(self) -> None: + events = [] + fsync, replace = os.fsync, Path.replace + + def sync_file(descriptor): + events.append('directory' if stat.S_ISDIR(os.fstat(descriptor).st_mode) else 'file') + fsync(descriptor) + + def commit(path, destination): + events.append('replace') + return replace(path, destination) + + with mock.patch('os.fsync', side_effect=sync_file), mock.patch.object(Path, 'replace', commit): + self.store.save_set('s21_calibration', 'radar', 'full', _sweep()) + first_file = events.index('file') + self.assertEqual(events[:first_file], ['directory', 'directory']) + self.assertEqual(events[first_file:], ['file', 'file', 'replace', 'replace', 'directory']) + + def test_set_sync_failure_does_not_replace_previous_measurement(self) -> None: + self.store.save_set('s21_calibration', 'radar', 'full', _sweep()) + archive = self.root / 'sets/s21_calibration/radar/full.npz' + original = archive.read_bytes() + for failed_sync in (1, 2): + with self.subTest(failed_sync=failed_sync), mock.patch('python_app.storage.npz.store.logger.exception'): + with mock.patch('os.fsync', side_effect=[None] * (failed_sync - 1) + [OSError('sync failed')]): + with self.assertRaises(OSError): + self.store.save_set('s21_calibration', 'radar', 'full', _sweep(2, 30)) + self.assertEqual(archive.read_bytes(), original) + self.assertEqual(list(archive.parent.glob('*.tmp')), []) + + def test_sets_and_snapshots_preserve_permissions_and_respect_umask(self) -> None: + previous = os.umask(0o027) + try: + self.store.save_set('s21_calibration', 'radar', 'full', _sweep()) + directory = self.store.save_runtime_snapshot_numpy(self.output, 'full', [_sweep()], [], []) + finally: + os.umask(previous) + for path in (self.root / 'sets/s21_calibration/radar').iterdir(): + self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o640) + path.chmod(0o664) + for path in [directory, *directory.rglob('*')]: + self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o750 if path.is_dir() else 0o640) + self.store.save_set('s21_calibration', 'radar', 'full', _sweep(2, 30)) + for path in (self.root / 'sets/s21_calibration/radar').iterdir(): + self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o664) + def test_invalid_sets_are_rejected_before_overwriting_valid_data(self) -> None: self.store.save_set('s21_calibration', 'radar', 'full', _sweep()) invalid = [_sweep(), _sweep(), _sweep(), _sweep()] invalid[0].traces.append(invalid[0].traces[0]) invalid[1].traces[0].s21 = np.array([1], dtype=np.complex64) invalid[2].traces[0].frequency_hz = np.array([[1, 2]], dtype=np.float32) - invalid[3].traces[0].s11[0] = complex(float('nan'), 0) + invalid[3].traces[0].s21[0] = complex(float('nan'), 0) for collection in invalid: with self.subTest(collection=collection), self.assertRaises(ValueError): self.store.save_set('s21_calibration', 'radar', 'full', collection) @@ -150,7 +185,7 @@ class PersistenceIntegrityTest(unittest.TestCase): def test_invalid_channel_identifiers_are_rejected_without_truncation(self) -> None: for position in (0.5, True, -1, 2 ** 32): sweep = _sweep() - sweep.traces[0].combo = ComboKey(position, 0) + sweep.traces[0].combo = ComboKey(cast(int, position), 0) with self.subTest(position=position), self.assertRaises(ValueError): self.store.save_set('s21_reference', 'radar', 'invalid', sweep) self.assertEqual(self.store.list_sets('s21_reference', 'radar'), []) @@ -173,10 +208,9 @@ class PersistenceIntegrityTest(unittest.TestCase): raw = [_sweep(1, 10), _sweep(2, 30)] pre = [_sweep(1, 20), _sweep(3, 40)] result = [ResultCollection(1, 20)] - directory, summary = self.store.save_runtime_snapshot_numpy( - self.output, 'full', raw, pre, result, 1000, include_unaligned=True, + directory = self.store.save_runtime_snapshot_numpy( + self.output, 'full', raw, pre, result, ) - self.assertEqual((summary['raw_count'], summary['preprocessed_count'], summary['result_count']), (2, 2, 1)) for stage, ids in [('raw', [1, 2]), ('preprocessed', [1, 3]), ('results', [1])]: metadata = [json.loads(path.read_text()) for path in sorted((directory / stage).glob('*/meta.json'))] self.assertEqual([item['collection_id'] for item in metadata], ids) @@ -184,11 +218,15 @@ class PersistenceIntegrityTest(unittest.TestCase): def test_snapshot_result_payloads_preserve_every_kind_at_both_scopes(self) -> None: payloads = _payloads() result = ResultCollection(1, 20, processing_duration_ns=12345, + settings_revision=2**63 + 7, capture_start_ns=2**60 + 3, + capture_end_ns=2**60 + 9, speed_m_s=1.25, collection_payloads=payloads, blocks=[ResultBlock(ComboKey(0, 0), payloads)]) - directory, _ = self.store.save_runtime_snapshot_numpy(self.output, 'full', [], [], [result], 1000) + directory = self.store.save_runtime_snapshot_numpy(self.output, 'full', [], [], [result]) collection_dir = next((directory / 'results').iterdir()) meta = json.loads((collection_dir / 'meta.json').read_text()) self.assertEqual(meta['processing_duration_ns'], 12345) + self.assertEqual((meta['settings_revision'], meta['capture_start_ns'], meta['capture_end_ns'], meta['speed_m_s']), + (2**63 + 7, 2**60 + 3, 2**60 + 9, 1.25)) for base, entries in [(collection_dir, meta['collection_payloads']), (collection_dir / meta['blocks'][0]['dir'], meta['blocks'][0]['payloads'])]: self.assertEqual([entry['kind'] for entry in entries], [1, 2, 3, 4]) @@ -204,18 +242,18 @@ class PersistenceIntegrityTest(unittest.TestCase): for payload in (ResultPayload('unknown', 9), ResultPayload('bad_table', 4, table=np.array([1, 2]))): with self.subTest(kind=payload.kind), self.assertRaises(ValueError): self.store.save_runtime_snapshot_numpy( - self.output, 'failed', [_sweep()], [], [ResultCollection(1, 20, collection_payloads=[payload])], 1000, + self.output, 'failed', [_sweep()], [], [ResultCollection(1, 20, collection_payloads=[payload])], ) self.assertFalse((self.output / 'failed').exists()) self.assertEqual(list(self.output.iterdir()), []) def test_snapshot_write_failure_can_be_retried_with_same_name(self) -> None: with mock.patch('numpy.save', side_effect=OSError('disk full')), self.assertRaises(OSError): - self.store.save_runtime_snapshot_numpy(self.output, 'retry', [_sweep()], [], [], 1000) - directory, _ = self.store.save_runtime_snapshot_numpy(self.output, 'retry', [_sweep()], [], [], 1000) + self.store.save_runtime_snapshot_numpy(self.output, 'retry', [_sweep()], [], []) + directory = self.store.save_runtime_snapshot_numpy(self.output, 'retry', [_sweep()], [], []) self.assertTrue(directory.exists()) with self.assertRaises(FileExistsError): - self.store.save_runtime_snapshot_numpy(self.output, 'retry', [_sweep()], [], [], 1000) + self.store.save_runtime_snapshot_numpy(self.output, 'retry', [_sweep()], [], []) def test_stream_failure_keeps_prior_chunks_and_retries_without_partial_data(self) -> None: writer = self.store.create_snapshot_stream(self.output, 'stream') @@ -229,6 +267,44 @@ class PersistenceIntegrityTest(unittest.TestCase): self.assertEqual(writer.result_count, 2) self.assertEqual([path.name.split('_')[0] for path in sorted((writer.directory / 'raw').iterdir())], ['0000', '0001']) + def test_stream_failure_preserves_collections_already_published_in_current_chunk(self) -> None: + writer = self.store.create_snapshot_stream(self.output, 'partial') + good = ResultCollection(1, 20) + bad = ResultCollection(2, 40, collection_payloads=[ResultPayload('invalid', 9)]) + with self.assertRaises(ValueError): + writer.append([_sweep(1, 10), _sweep(2, 30)], [], [good, bad]) + self.assertEqual(writer.result_count, 1) + self.assertEqual(len(list((writer.directory / 'raw').iterdir())), 2) + self.assertEqual(len(list((writer.directory / 'results').iterdir())), 1) + writer.append([], [], [ResultCollection(2, 40)]) + self.assertEqual(writer.result_count, 2) + self.assertEqual([path.name.split('_')[0] for path in sorted((writer.directory / 'results').iterdir())], ['0000', '0001']) + + def test_processing_revisions_are_saved_and_cannot_be_reassigned(self) -> None: + settings = {4: {'settings_revision': 4, 'gain': 2}} + directory = self.store.save_runtime_snapshot_numpy( + self.output, 'snapshot', [], [], [], processing_settings=settings, + ) + self.assertEqual(json.loads((directory / 'processing_settings.json').read_text()), {'4': settings[4]}) + writer = self.store.create_snapshot_stream(self.output, 'stream') + writer.append([], [], [ResultCollection(1, 10)], processing_settings=settings) + later = {5: {'settings_revision': 5, 'gain': 3}} + writer.append([], [], [ResultCollection(2, 20)], processing_settings=later) + path = writer.directory / 'processing_settings.json' + self.assertEqual(json.loads(path.read_text()), {'4': settings[4], '5': later[5]}) + with self.assertRaisesRegex(ValueError, 'already has different settings'): + writer.append([], [], [ResultCollection(3, 30)], processing_settings={4: {'gain': 9}}) + self.assertEqual(writer.result_count, 2) + self.assertEqual(json.loads(path.read_text()), {'4': settings[4], '5': later[5]}) + + def test_failed_processing_settings_write_does_not_publish_results(self) -> None: + writer = self.store.create_snapshot_stream(self.output, 'stream') + with mock.patch('python_app.storage.npz.snapshot_numpy.atomic_write_json', side_effect=OSError('disk full')): + with self.assertRaises(OSError): + writer.append([], [], [ResultCollection(1, 10)], processing_settings={1: {'gain': 2}}) + self.assertEqual(writer.result_count, 0) + self.assertEqual(list(writer.directory.iterdir()), []) + def test_json_export_preserves_unmatched_raw_sweeps_and_matches_reused_ids_by_timing(self) -> None: raw_old = _sweep(1, 10, start=1, end=5) raw_new = _sweep(1, 100, start=90, end=95) @@ -244,33 +320,37 @@ class PersistenceIntegrityTest(unittest.TestCase): second = _sweep().traces[0] second.combo = ComboKey(0, 1) sweep.traces.append(second) - write_text = Path.write_text + dump = json.dump writes = 0 - def fail_second_file(path, *args, **kwargs): + def fail_second_file(*args, **kwargs): nonlocal writes writes += 1 if writes == 2: raise OSError('disk full') - return write_text(path, *args, **kwargs) + return dump(*args, **kwargs) - with mock.patch.object(Path, 'write_text', fail_second_file), self.assertRaises(OSError): - self.store.save_runtime_vna_history_json_batch(self.output, 'export', [sweep], [], [], 1000) + with mock.patch('python_app.storage.npz.store.json.dump', fail_second_file), self.assertRaises(OSError): + self.store.save_runtime_vna_history_json_batch(self.output, 'export', [sweep], [], []) self.assertEqual(list(self.output.iterdir()), []) - paths, summary = self.store.save_runtime_vna_history_json_batch(self.output, 'export', [sweep], [], [], 1000, channel='s11') - self.assertEqual(summary['combo_count'], 2) + paths = self.store.save_runtime_vna_history_json_batch(self.output, 'export', [sweep], [], []) + self.assertEqual(len(paths), 2) self.assertEqual(len(paths), 2) for path in paths: - self.assertEqual(json.loads(path.read_text())['channel'], 's11') + self.assertEqual(json.loads(path.read_text())['channel'], 's21') - def test_saved_dataset_replay_preserves_s11_and_accepts_legacy_s21_only(self) -> None: - directory, _ = self.store.save_runtime_snapshot_numpy(self.output, 'replay', [_sweep()], [], [], 1000) - dataset = _load_dataset(directory / 'raw', [ComboKey(0, 0)], 2) - np.testing.assert_array_equal(dataset.snapshots[0][0].s11, _sweep().traces[0].s11) - self.assertEqual(dataset.snapshots[0][0].capture_end_ns, 15) - next((directory / 'raw').glob('*/i0_o0_s11.npy')).unlink() - legacy = _load_dataset(directory / 'raw', [ComboKey(0, 0)], 2) - np.testing.assert_array_equal(legacy.snapshots[0][0].s11, np.zeros(2, dtype=np.complex64)) + def test_json_export_can_follow_numpy_without_overwriting_either_format(self) -> None: + directory = self.store.save_runtime_snapshot_numpy(self.output, 'both', [_sweep()], [], []) + directory.chmod(0o775) + before = {path: path.read_bytes() for path in directory.rglob('*') if path.is_file()} + paths = self.store.save_runtime_vna_history_json_batch(self.output, 'both', [_sweep()], [], []) + self.assertEqual(stat.S_IMODE(directory.stat().st_mode), 0o775) + self.assertEqual({path: path.read_bytes() for path in before}, before) + exported = paths[0].read_bytes() + with self.assertRaises(FileExistsError): + self.store.save_runtime_vna_history_json_batch(self.output, 'both', [_sweep(2, 30)], [], []) + self.assertEqual(paths[0].read_bytes(), exported) + self.assertEqual({path: path.read_bytes() for path in before}, before) def test_long_recording_replay_and_converter_use_numeric_directory_order(self) -> None: writer = self.store.create_snapshot_stream(self.output, 'long') @@ -280,49 +360,20 @@ class PersistenceIntegrityTest(unittest.TestCase): folder.rename(folder.with_name(f'{index}_{folder.name.split("_", 1)[1]}')) dataset = _load_dataset(writer.directory / 'raw', [ComboKey(0, 0)], 2) self.assertEqual(dataset.timestamps_ns, [10, 20, 30]) - self.assertEqual([int(path.name.split('_')[0]) for path in _collection_dirs(writer.directory / 'raw')], [9998, 9999, 10000]) + self.assertEqual([int(path.name.split('_')[0]) for path in collection_directories(writer.directory / 'raw')], [9998, 9999, 10000]) def test_offline_and_runtime_json_exports_share_matching_and_preserve_orphan_sweeps(self) -> None: raw = [_sweep(1, 10, start=1, end=5), _sweep(1, 100, start=90, end=95)] pre = [_sweep(1, 110, start=90, end=95)] - directory, _ = self.store.save_runtime_snapshot_numpy( - self.output, 'convert', raw, pre, [], 1000, include_unaligned=True, + directory = self.store.save_runtime_snapshot_numpy( + self.output, 'convert', raw, pre, [], ) - records = { - stage: _load_stage_records(directory, stage, input_index=0, output_index=0, channel='s21') - for stage in ('raw', 'preprocessed') - } - offline = _build_sweep_history(records['raw'], records['preprocessed'], channel='s21', primary_stage='preprocessed') + offline = load_vna_history_snapshot(directory, input_index=0, output_index=0)["sweep_history"] runtime = build_vna_history_payload(raw, pre, [], input_index=0, output_index=0) self.assertEqual(offline, runtime['sweep_history']) self.assertEqual(len(offline), 2) -class SnapshotAlignmentTest(unittest.TestCase): - def test_reused_ids_and_missing_frames_do_not_cross_align_runs(self) -> None: - raw_old = _sweep(1, 10, start=1, end=5) - pre_old = _sweep(1, 20, start=1, end=5) - pre_new = _sweep(1, 200, start=150, end=175) - selected_raw, selected_pre, results, summary = select_aligned_histories( - [raw_old], [pre_old, pre_new], [ResultCollection(1, 200)], 1, - ) - self.assertEqual(selected_raw, []) - self.assertEqual(selected_pre, [pre_new]) - self.assertEqual(summary['raw_missing_count'], 1) - self.assertEqual(results[0].monotonic_ns, 200) - - def test_result_does_not_pair_with_wrong_preprocessed_timestamp(self) -> None: - _, pre, _, summary = select_aligned_histories([], [_sweep(1, 20)], [ResultCollection(1, 200)], 1000) - self.assertEqual(pre, []) - self.assertEqual(summary['preprocessed_missing_count'], 1) - - def test_independent_stage_limit_is_exactly_requested_tail(self) -> None: - raw = [_sweep(cid=index, ns=index) for index in range(1005)] - selected_raw, _, _, _ = select_aligned_histories(raw, [], [], 1000, include_unaligned=True) - self.assertEqual(len(selected_raw), 1000) - self.assertEqual((selected_raw[0].collection_id, selected_raw[-1].collection_id), (5, 1004)) - - class CalibrationMedianIntegrityTest(unittest.TestCase): def test_rejects_frequency_axes_that_changed_between_captures(self) -> None: first, second = _sweep(), _sweep() diff --git a/python_app/tests/test_point_target_mock.py b/python_app/tests/test_point_target_mock.py index ac77aa2..eb080bb 100644 --- a/python_app/tests/test_point_target_mock.py +++ b/python_app/tests/test_point_target_mock.py @@ -2,19 +2,19 @@ from __future__ import annotations -from pathlib import Path +from contextlib import ExitStack + import unittest from unittest import mock import numpy as np -from python_app.hardware_full.matrix_radar_service import create_acquisition_service +from python_app.hardware_full.matrix_radar_service import create_matrix_radar_service from python_app.hardware_full.point_target_mock_service import PointTargetMockService from python_app.hardware_full.single_radar_service import create_single_radar_service from python_app.models.dataset_model import ComboKey -from python_app.models.run_config_model import RunConfigModel +from python_app.models.run_config_schema import RunConfigModel from python_app.models.run_config_schema import GprRxGeometryModel, GprTxGeometryModel -from python_app.workflows.multi_radar_capture_workflow import MultiRadarSequentialCaptureSession from python_app.workflows.radar_config_variants import RadarConfigVariant from python_app.workflows.sequential_capture_workflow import SequentialCaptureSession @@ -41,13 +41,15 @@ def _config() -> RunConfigModel: class PointTargetMockTest(unittest.TestCase): def setUp(self) -> None: - self.clock = self.enterContext(mock.patch(_CLOCK, return_value=_EPOCH_NS)) - self.sleep = self.enterContext(mock.patch(_SLEEP)) + cleanup = ExitStack() + self.addCleanup(cleanup.close) + self.clock = cleanup.enter_context(mock.patch(_CLOCK, return_value=_EPOCH_NS)) + self.sleep = cleanup.enter_context(mock.patch(_SLEEP)) self.config = _config() self.service = PointTargetMockService(self.config) self.service.configure(self.config.radar.sweep) - def test_all_channels_follow_bistatic_and_monostatic_distance(self) -> None: + def test_all_channels_follow_bistatic_distance(self) -> None: collection = self.service.acquire_collection(collection_id=43) target = np.array([0, .25, 3.75]) self.assertEqual(collection.collection_id, 43) @@ -59,7 +61,6 @@ class PointTargetMockTest(unittest.TestCase): r_rx = np.linalg.norm(target - [rx.x_m, rx.y_m, rx.z_m]) phase_per_m = -2j * np.pi * trace.frequency_hz.astype(np.float64) / 3e8 np.testing.assert_allclose(trace.s21, .5 / (r_tx * r_rx) * np.exp(phase_per_m * (r_tx + r_rx)), rtol=1e-6) - np.testing.assert_allclose(trace.s11, .5 / r_tx**2 * np.exp(phase_per_m * 2 * r_tx), rtol=1e-6) self.assertEqual((trace.capture_start_ns, trace.capture_end_ns), (_EPOCH_NS, _EPOCH_NS)) self.assertEqual(trace.s21.dtype, np.dtype(np.complex64)) self.assertEqual((collection.capture_start_ns, collection.capture_end_ns, collection.monotonic_ns), (_EPOCH_NS,) * 3) @@ -70,14 +71,12 @@ class PointTargetMockTest(unittest.TestCase): traces = self.service.acquire_collection().traces for trace in traces[1:]: np.testing.assert_array_equal(trace.s21, traces[0].s21) - np.testing.assert_array_equal(trace.s11, traces[0].s11) - def test_geometry_changes_receiver_path_without_changing_transmitter_echo(self) -> None: + def test_geometry_changes_receiver_path(self) -> None: before = self.service.acquire_collection().traces[0] self.config.gpr.rx_geometry[0].z_m = 1 after = self.service.acquire_collection().traces[0] self.assertFalse(np.array_equal(before.s21, after.s21)) - np.testing.assert_array_equal(before.s11, after.s11) def test_motion_uses_elapsed_time_and_returns_after_eight_seconds(self) -> None: tx = {0: (-1., 0., 0.), 1: (1., 0., 0.)} @@ -132,13 +131,16 @@ class PointTargetMockTest(unittest.TestCase): self.assertLess(max(ranges), unambiguous_range_m) def test_single_manual_capture_selects_actual_antenna_and_keeps_capture_time(self) -> None: - session = SequentialCaptureSession(self.config, "s21_calibration", "mock_geometry") + radar = mock.Mock(wraps=self.service) + with mock.patch("python_app.workflows.sequential_capture_workflow.create_single_radar_service", return_value=radar): + session = SequentialCaptureSession(self.config, "s21_calibration", "mock_geometry") session.open() try: - first = session.capture_current_combo() - second = session.capture_current_combo() + first = session.capture_current_combo().display_trace + second = session.capture_current_combo().display_trace finally: session.close() + radar.configure.assert_called_once_with(self.config.radar.sweep) self.assertEqual((first.combo, second.combo), (ComboKey(0, 0), ComboKey(1, 0))) self.assertFalse(np.array_equal(first.s21, second.s21)) self.assertEqual((second.capture_start_ns, second.capture_end_ns), (_EPOCH_NS, _EPOCH_NS)) @@ -147,13 +149,13 @@ class PointTargetMockTest(unittest.TestCase): single = create_single_radar_service(self.config) self.assertIsInstance(single, PointTargetMockService) single.configure(self.config.radar.sweep) - self.assertIsInstance(create_acquisition_service(self.config), PointTargetMockService) + self.assertIsInstance(create_matrix_radar_service(self.config), PointTargetMockService) self.config.radar.model = RunConfigModel.LIBREVNA_MULTI_MODEL self.config.radar.multi_device.stimulus_switch_positions = 2 self.config.radar.multi_device.input_switch_positions = 2 self.config.radar.multi_device.output_switch_positions = 2 self.config.apply_device_model_constraints() - matrix = create_acquisition_service(self.config) + matrix = create_matrix_radar_service(self.config) self.assertIsInstance(matrix, PointTargetMockService) matrix.open() matrix.configure(self.config.radar.sweep) @@ -166,15 +168,15 @@ class PointTargetMockTest(unittest.TestCase): def test_multi_variant_single_capture_drains_each_variant_before_sampling(self) -> None: radar = mock.Mock(wraps=self.service) events = [] - radar.drain_after_switch = mock.Mock(side_effect=lambda: events.append("drain")) + radar.prepare_combo.side_effect = lambda combo: (events.append("prepare"), self.service.prepare_combo(combo)) radar.acquire.side_effect = lambda: (events.append("acquire"), self.service.acquire())[1] - variants = [RadarConfigVariant(Path(name), name, self.config, name) for name in ("a", "b")] + variants = [RadarConfigVariant(name, self.config, name) for name in ("a", "b")] with mock.patch( - "python_app.workflows.multi_radar_capture_workflow.create_single_radar_service", + "python_app.workflows.sequential_capture_workflow.create_single_radar_service", return_value=radar, ): - session = MultiRadarSequentialCaptureSession( - base_config=self.config, kind="s21_calibration", set_name="variants", radar_variants=variants, + session = SequentialCaptureSession( + config=self.config, kind="s21_calibration", set_name="variants", radar_variants=variants, ) session.open() try: @@ -182,14 +184,14 @@ class PointTargetMockTest(unittest.TestCase): batch = session.capture_current_combo() finally: session.close() - self.assertEqual(events, ["drain", "acquire"] * 4) + self.assertEqual(events, ["prepare", "acquire"] * 4) self.assertEqual([trace.combo for trace in batch.traces], [ComboKey(1, 0)] * 2) for trace in batch.traces: self.assertEqual((trace.capture_start_ns, trace.capture_end_ns), (_EPOCH_NS, _EPOCH_NS)) def test_invalid_or_unconfigured_mock_fails_clearly(self) -> None: with self.assertRaisesRegex(ValueError, "outside the antenna matrix"): - self.service.acquire_combo_collection(input_pos=2, output_pos=0) + self.service.acquire_collection(combo=ComboKey(2, 0)) with self.assertRaisesRegex(RuntimeError, "not configured"): PointTargetMockService(self.config).acquire_collection() self.config.input_switch.positions = 0 diff --git a/python_app/tests/test_processing.py b/python_app/tests/test_processing.py index fcbb899..7035f48 100644 --- a/python_app/tests/test_processing.py +++ b/python_app/tests/test_processing.py @@ -1,41 +1,30 @@ -"""Processing-interpretation tests across processor modes. - -Pins the agreed semantics (not just current behaviour): - * GPR object filtering is mode-agnostic — keep finite rows with score >= threshold - (a normalized float for coherent GPR, a pair count for legacy GPR; same compare) - inside the visible X/Z window, then apply draw limits. When more than - max_detected_objects survive, ALL are hidden; legacy GPR passes draw_limits=None - (count rules disabled) but is otherwise filtered identically — so heatmap markers - match the objects-only view in both modes. - * B-scan level/colormap/mean-subtraction/history transforms are deterministic. - * ProcessingLiveConfig normalizes optional position lists to concrete int lists. -""" +"""GPR object filtering, B-scan transforms and live-setting normalization.""" from __future__ import annotations +import json import math import unittest from collections import deque +from collections.abc import Iterable, Sequence import numpy as np -from python_app.gui.controllers.app_window_plot.bscan_plot_mixin import ( +from python_app.gui.controllers.renderers.bscan import ( apply_mean_ascan_subtraction, bscan_levels, bscan_lookup_table, - build_lut, rebuild_bscan_history_from_results, ) -from python_app.gui.controllers.app_window_plot.gpr_plot_mixin import AppWindowGprPlotMixin +from python_app.gui.controllers.renderers.common import build_lut +from python_app.gui.controllers.renderers.gpr import normalized_display_range, display_y_min, object_label_candidates from python_app.models.dataset_model import ( ComboKey, ResultBlock, ResultCollection, ResultPayload, ) -from python_app.orchestration.gpr_locator import ( - apply_object_draw_limits, - collection_has_gpr_payloads, +from python_app.gui.result_payloads import ( collection_payload_by_name, collection_payloads_by_prefix, filter_object_rows, @@ -44,17 +33,14 @@ from python_app.orchestration.gpr_locator import ( from python_app.orchestration.live_processing_config import ProcessingLiveConfig -def _table(name: str, rows) -> ResultPayload: +def _table(name: str, rows: Sequence[Sequence[float]]) -> ResultPayload: return ResultPayload(processing_name=name, kind=4, table=np.asarray(rows, dtype=np.float32)) -def _collection(payloads) -> ResultCollection: +def _collection(payloads: Iterable[ResultPayload]) -> ResultCollection: return ResultCollection(collection_id=1, monotonic_ns=1, collection_payloads=list(payloads)) -# --------------------------------------------------------------------------- # -# Shared result-collection lookup helpers -# --------------------------------------------------------------------------- # class CollectionLookupTest(unittest.TestCase): def test_payload_by_name_matches_name_and_optional_kind(self) -> None: col = _collection([_table("gpr_points", [[0, 0, 1]]), _table("gpr_region_centers", [[1, 1, 2, 9]])]) @@ -72,15 +58,7 @@ class CollectionLookupTest(unittest.TestCase): self.assertEqual([p.processing_name for p in masks], ["gpr_region_mask_0", "gpr_region_mask_1"]) self.assertEqual(collection_payloads_by_prefix(col, "nope_"), []) - def test_has_gpr_payloads(self) -> None: - self.assertTrue(collection_has_gpr_payloads(_collection([_table("gpr_points", [[0, 0, 1]])]))) - self.assertFalse(collection_has_gpr_payloads(_collection([_table("pass_through", [[0, 0, 1]])]))) - self.assertFalse(collection_has_gpr_payloads(_collection([]))) - -# --------------------------------------------------------------------------- # -# GPR object row extraction -# --------------------------------------------------------------------------- # class GprObjectRowsTest(unittest.TestCase): def test_extracts_first_three_columns_of_points(self) -> None: rows = gpr_object_rows(_collection([_table("gpr_points", [[1, 2, 3], [4, 5, 6]])])) @@ -98,44 +76,12 @@ class GprObjectRowsTest(unittest.TestCase): self.assertEqual(gpr_object_rows(_collection([_table("gpr_points", [[1, 2]])])).shape, (0, 3)) -# --------------------------------------------------------------------------- # -# Object draw limits (hide-all-when-over, then top-M) -# --------------------------------------------------------------------------- # -class ApplyObjectDrawLimitsTest(unittest.TestCase): - @staticmethod - def _rows(n: int) -> np.ndarray: - return np.column_stack([np.arange(n), np.arange(n), np.arange(n)]).astype(np.float32) - - def test_none_limits_passes_through(self) -> None: - rows = self._rows(5) - self.assertTrue(np.array_equal(apply_object_draw_limits(rows, None), rows)) - - def test_hides_all_when_over_max(self) -> None: - self.assertEqual(apply_object_draw_limits(self._rows(6), (5, 3)).shape, (0, 3)) - - def test_keeps_top_m_when_within_max(self) -> None: - out = apply_object_draw_limits(self._rows(4), (5, 2)) - self.assertEqual(out.shape, (2, 3)) - self.assertTrue(np.array_equal(out, self._rows(4)[:2])) - - def test_max_zero_hides_any_objects(self) -> None: - # max_detected_objects == 0 means "hide all" (any object exceeds it). - self.assertEqual(apply_object_draw_limits(self._rows(1), (0, 5)).shape, (0, 3)) - - def test_empty_input_passes_through(self) -> None: - empty = np.zeros((0, 3), dtype=np.float32) - self.assertEqual(apply_object_draw_limits(empty, (5, 3)).shape, (0, 3)) - - -# --------------------------------------------------------------------------- # -# Mode-agnostic object filtering (the core both modes + the locator share) -# --------------------------------------------------------------------------- # class FilterObjectRowsTest(unittest.TestCase): BOUNDS = {"x_bounds": (-2.0, 2.0), "z_bounds": (0.0, 10.0)} - def _filter(self, rows, *, min_score, draw_limits=None): + def _filter(self, rows: Sequence[Sequence[float]], *, min_score: float) -> np.ndarray: return filter_object_rows(np.asarray(rows, dtype=np.float32), min_score=min_score, - draw_limits=draw_limits, **self.BOUNDS) + **self.BOUNDS) def test_keeps_in_window_and_at_or_above_threshold(self) -> None: out = self._filter([[0.0, 5.0, 0.5], [1.0, 1.0, 0.9]], min_score=0.5) # score==threshold kept (inclusive) @@ -159,21 +105,10 @@ class FilterObjectRowsTest(unittest.TestCase): def test_legacy_pair_count_threshold_uses_same_compare(self) -> None: # legacy GPR passes an integer pair-count threshold; the >= compare is identical. - out = self._filter([[0.0, 5.0, 3.0], [0.0, 5.0, 2.0]], min_score=3, draw_limits=None) + out = self._filter([[0.0, 5.0, 3.0], [0.0, 5.0, 2.0]], min_score=3) self.assertTrue(np.array_equal(out, np.array([[0.0, 5.0, 3.0]], dtype=np.float32))) - def test_draw_limits_hide_all_when_over(self) -> None: - rows = [[0.0, 5.0, 1.0]] * 4 - self.assertEqual(self._filter(rows, min_score=0.0, draw_limits=(3, 2)).shape[0], 0) - def test_legacy_none_limits_skips_count_rule(self) -> None: - rows = [[0.0, 5.0, 1.0]] * 4 - self.assertEqual(self._filter(rows, min_score=0.0, draw_limits=None).shape[0], 4) - - -# --------------------------------------------------------------------------- # -# B-scan transforms -# --------------------------------------------------------------------------- # class BscanTransformTest(unittest.TestCase): def test_mean_ascan_subtraction(self) -> None: history = deque([np.array([1.0, 1.0], dtype=np.float32), np.array([3.0, 3.0], dtype=np.float32)]) @@ -207,7 +142,10 @@ class BscanTransformTest(unittest.TestCase): class RebuildBscanHistoryTest(unittest.TestCase): @staticmethod - def _bscan_collection(cid: int, combo, depth, amps, *, name="bscan", kind=1) -> ResultCollection: + def _bscan_collection( + cid: int, combo: tuple[int, int], depth: Sequence[float], amps: Sequence[float], + *, name: str = "bscan", kind: int = 1, + ) -> ResultCollection: trace = np.asarray(amps, dtype=np.float32).astype(np.complex64) payload = ResultPayload(processing_name=name, kind=kind, frequency_hz=np.asarray(depth, dtype=np.float32), trace=trace) @@ -254,31 +192,25 @@ class RebuildBscanHistoryTest(unittest.TestCase): self.assertEqual([sweep[0] for sweep in by_combo[(0, 0)]], [100.0, 1.0, 2.0]) -# --------------------------------------------------------------------------- # -# GPR display-range helpers (pure static math on the mixin) -# --------------------------------------------------------------------------- # class GprDisplayHelperTest(unittest.TestCase): def test_normalized_range_orders_and_expands(self) -> None: - self.assertEqual(AppWindowGprPlotMixin._normalized_display_range(2.0, 5.0), (2.0, 5.0)) - self.assertEqual(AppWindowGprPlotMixin._normalized_display_range(5.0, 2.0), (2.0, 5.0)) # reordered - low, high = AppWindowGprPlotMixin._normalized_display_range(3.0, 3.0) # zero span expands to 0.1 + self.assertEqual(normalized_display_range(2.0, 5.0), (2.0, 5.0)) + self.assertEqual(normalized_display_range(5.0, 2.0), (2.0, 5.0)) # reordered + low, high = normalized_display_range(3.0, 3.0) # zero span expands to 0.1 self.assertAlmostEqual(high - low, 0.1) self.assertAlmostEqual(0.5 * (low + high), 3.0) def test_display_y_min_keeps_surface_margin(self) -> None: - self.assertEqual(AppWindowGprPlotMixin._gpr_display_y_min(2.0, 5.0), 2.0) # surface not visible - self.assertAlmostEqual(AppWindowGprPlotMixin._gpr_display_y_min(0.0, 10.0), -0.3) # 3% of span - self.assertAlmostEqual(AppWindowGprPlotMixin._gpr_display_y_min(-1.0, 1.0), -1.06) # min 0.06 margin + self.assertEqual(display_y_min(2.0, 5.0), 2.0) # surface not visible + self.assertAlmostEqual(display_y_min(0.0, 10.0), -0.3) # 3% of span + self.assertAlmostEqual(display_y_min(-1.0, 1.0), -1.06) # min 0.06 margin def test_object_label_candidates_are_distinct_positions(self) -> None: - candidates = AppWindowGprPlotMixin._gpr_object_label_candidates(0.0, 0.0, x_span=1.0, z_span=1.0) + candidates = object_label_candidates(0.0, 0.0, x_span=1.0, z_span=1.0) self.assertEqual(len(candidates), 10) self.assertTrue(all(math.isfinite(x) and math.isfinite(z) for x, z, _ in candidates)) -# --------------------------------------------------------------------------- # -# ProcessingLiveConfig normalization -# --------------------------------------------------------------------------- # class ProcessingLiveConfigTest(unittest.TestCase): def test_none_positions_become_empty_lists(self) -> None: cfg = ProcessingLiveConfig(gpr_input_positions=None, gpr_output_positions=None) @@ -286,7 +218,7 @@ class ProcessingLiveConfigTest(unittest.TestCase): self.assertEqual(cfg.gpr_output_positions, []) def test_positions_coerced_to_ints(self) -> None: - cfg = ProcessingLiveConfig(gpr_input_positions=[1.5, 2.9], gpr_output_positions=[0.0]) + cfg = ProcessingLiveConfig(**json.loads('{"gpr_input_positions": [1.5, 2.9], "gpr_output_positions": [0.0]}')) self.assertEqual(cfg.gpr_input_positions, [1, 2]) self.assertEqual(cfg.gpr_output_positions, [0]) diff --git a/python_app/tests/test_processor_history.py b/python_app/tests/test_processor_history.py index ac8b617..db6f79d 100644 --- a/python_app/tests/test_processor_history.py +++ b/python_app/tests/test_processor_history.py @@ -1,7 +1,7 @@ """The processor's history stays the acquisition, in order, when the GUI feeds sweeps back. To recompute the whole B-scan the GUI pushes its retained sweeps into the processor's -input ring again (AppWindowBscanReplayMixin). The processor used to append them to its +input ring again (BscanReplay). The processor used to append them to its own history as if they were new, so its next replay published every sweep two or three times and out of order, and the B-scan came out with its columns rotated. @@ -12,6 +12,7 @@ from __future__ import annotations import json import os +import signal import subprocess import tempfile import time @@ -22,9 +23,10 @@ import numpy as np from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData from python_app.orchestration.live_processing_config import ProcessingLiveConfig, ProcessingLiveConfigWriter -from python_app.orchestration.shm import ShmRingWriter +from python_app.orchestration.shm.ring_writer import ShmRingWriter from python_app.orchestration.shm.ring_reader import ShmRingReader -from python_app.storage.npz.serialize import PREPROC_MAGIC, serialize_trace_collection +from python_app.orchestration.shm.decoder import PREPROC_MAGIC +from python_app.storage.npz.serialize import serialize_trace_collection _ROOT = Path(__file__).resolve().parents[2] _PROCESSOR = _ROOT / "build/bin/data_processor" @@ -37,7 +39,6 @@ def _sweep(collection_id: int, monotonic_ns: int) -> SweepCollection: trace = TraceData( combo=ComboKey(input=0, output=0), frequency_hz=np.linspace(1e9, 6e9, points, dtype=np.float32), - s11=np.zeros(points, dtype=np.complex64), s21=np.full(points, collection_id, dtype=np.complex64), capture_start_ns=monotonic_ns, capture_end_ns=monotonic_ns + 1000, @@ -49,6 +50,7 @@ def _sweep(collection_id: int, monotonic_ns: int) -> SweepCollection: @unittest.skipUnless(_PROCESSOR.exists(), "data_processor is not built (run make)") class _ProcessorTestCase(unittest.TestCase): result_slot_size_bytes: int | None = None + idle_sleep_ms: int | None = None def setUp(self) -> None: workdir = Path(tempfile.mkdtemp(prefix="processor_history_")) @@ -57,6 +59,8 @@ class _ProcessorTestCase(unittest.TestCase): ring["name"] = f"/test_processor_history_{os.getpid()}_{name}" if self.result_slot_size_bytes is not None: config["rings"]["results"]["slot_size_bytes"] = self.result_slot_size_bytes + if self.idle_sleep_ms is not None: + config["run"]["idle_sleep_ms"] = self.idle_sleep_ms config["run"]["combos"] = [{"input": 0, "output": 0}] config["run"]["processing_live_config_path"] = str(workdir / "processing_live.json") config["run"]["locator_server"]["enabled"] = False @@ -118,18 +122,56 @@ class _ProcessorTestCase(unittest.TestCase): deadline = time.monotonic() + _TIMEOUT_S while time.monotonic() < deadline: result = self._results.pop_result_collection() - if result is not None: - payload = next((item for item in result.collection_payloads - if item.processing_name == "processing_revision"), None) - if payload is not None: - actual = sum(int(word) << (index * 16) for index, word in enumerate(payload.table[0])) - if actual == revision and (collection_id is None or result.collection_id == collection_id): - return result + if (result is not None and result.settings_revision == revision + and (collection_id is None or result.collection_id == collection_id)): + return result time.sleep(0.01) self.fail(f"No processor result for settings revision {revision}: {self._stderr_path.read_text()}") class FedBackSweepsTest(_ProcessorTestCase): + def test_remove_targets_exact_sweep_with_reused_ids_and_preserves_missing_targets(self) -> None: + start_ns = time.monotonic_ns() + sweeps = [_sweep(cid, start_ns + index * 1_000_000) + for index, cid in enumerate((7, 8, 7, 9, 8))] + self._push(sweeps) + self.assertEqual(self._published_ids(5), [7, 8, 7, 9, 8]) + targets = [ + ({"collection_id": 7, "capture_start_ns": sweeps[2].capture_start_ns}, [7, 8, 9, 8]), + ({"collection_id": 8, "capture_start_ns": sweeps[1].capture_start_ns}, [7, 9, 8]), + ({"collection_id": 7, "capture_start_ns": start_ns + 20_000_000}, [7, 9, 8]), + ] + for sequence, (target, expected) in enumerate(targets, start=1): + with self.subTest(target=target): + self._live.settings_revision = sequence + self._live.history_removed.append(target) + self._live_writer.write(self._live) + self.assertEqual(self._published_ids(len(expected)), expected) + + def test_removed_target_arriving_after_command_cannot_reenter_history(self) -> None: + now = time.monotonic_ns() + seed, target, following = [_sweep(cid, now + cid * 1_000_000) for cid in (1, 2, 3)] + self._push([seed]) + self.assertEqual(self._published_ids(1), [1]) + self._live.settings_revision = 1 + self._live.history_removed = [dict(collection_id=target.collection_id, capture_start_ns=target.capture_start_ns)] + self._live_writer.write(self._live) + self.assertEqual(self._published_ids(1), [1]) # Confirms the command was applied. + self._push([target, following]) + self.assertEqual(self._published_ids(1), [3]) + self._live.settings_revision = 2 + self._live_writer.write(self._live) + self.assertEqual(self._published_ids(2), [1, 3]) + + def test_result_preserves_capture_window_for_missing_tap_counterparts(self) -> None: + sweep = _sweep(1, time.monotonic_ns()) + sweep.capture_start_ns = (1 << 63) + 987654321 + sweep.capture_end_ns = sweep.capture_start_ns + 123456789 + self._push([sweep]) + result = self._result_for_revision(0) + self.assertEqual(result.capture_start_ns, sweep.capture_start_ns) + self.assertEqual(result.capture_end_ns, sweep.capture_end_ns) + def test_fed_back_sweeps_are_processed_but_not_retained(self) -> None: start_ns = time.monotonic_ns() sweeps = [_sweep(index, start_ns + index * 1_000_000) for index in range(1, _SWEEPS + 1)] @@ -151,35 +193,99 @@ class FedBackSweepsTest(_ProcessorTestCase): self._live.gpr_input_positions = [0] self._live.gpr_output_positions = [0] self._live.gpr_background_subtract_enabled = False + self._live.gpr_speed_m_s = 0.75 self._live.tx_geometry = [{"output_pos": 0, "x_m": 0.0, "y_m": 0.0, "z_m": 0.0}] self._live.rx_geometry = [{"input_pos": 0, "x_m": 0.0, "y_m": 0.0, "z_m": 0.0}] self._live.settings_revision = (1 << 63) + 987654321 self._live_writer.write(self._live) self._push([_sweep(1, time.monotonic_ns())]) original = self._result_for_revision(self._live.settings_revision) + self.assertEqual(original.speed_m_s, 0.75) original_image = next(payload for payload in original.collection_payloads if payload.processing_name == "gpr_accumulator") # A malformed live file retains the last accepted geometry and revision. accepted_revision = self._live.settings_revision self._live.settings_revision += 1 + self._live.gpr_speed_m_s = 1.25 self._live.tx_geometry.append(dict(self._live.tx_geometry[0])) self._live_writer.write(self._live) self._push([_sweep(2, time.monotonic_ns())]) retained = self._result_for_revision(accepted_revision, collection_id=2) + self.assertEqual(retained.speed_m_s, 0.75) retained_image = next(payload for payload in retained.collection_payloads if payload.processing_name == "gpr_accumulator") np.testing.assert_array_equal(retained_image.image_x_axis, original_image.image_x_axis) self._live.tx_geometry = self._live.tx_geometry[:1] self._live.tx_geometry[0]["x_m"] = 2.0 self._live.rx_geometry[0]["x_m"] = 2.0 + self._live.gpr_speed_m_s = 2.5 self._live.settings_revision += 1 self._live_writer.write(self._live) updated = self._result_for_revision(self._live.settings_revision) + self.assertEqual(updated.speed_m_s, 2.5) updated_image = next(payload for payload in updated.collection_payloads if payload.processing_name == "gpr_accumulator") self.assertEqual(updated.collection_id, retained.collection_id) - self.assertEqual(updated.monotonic_ns, retained.monotonic_ns) + self.assertEqual(updated.capture_start_ns, retained.capture_start_ns) np.testing.assert_allclose(updated_image.image_x_axis, original_image.image_x_axis + 2.0, atol=1e-6) self.assertIsNone(self._processor.poll()) +class PendingInputHistoryTest(_ProcessorTestCase): + idle_sleep_ms = 1000 + + def test_clear_history_discards_already_queued_input_and_accepts_future_sweeps(self) -> None: + now = time.monotonic_ns() + sweeps = [_sweep(cid, now + cid * 1_000_000) for cid in (1, 2, 3, 4)] + self._push(sweeps[:1]) + self.assertEqual(self._published_ids(1), [1]) + # The seed has completed and the processor is in its configured idle wait. + # Freeze the consumer while preparing a backlog and the Clear command. + # No further data is produced until that backlog has been consumed. + self._processor.send_signal(signal.SIGSTOP) + os.waitpid(self._processor.pid, os.WUNTRACED) + try: + self._push(sweeps[1:3]) + self._live.history_clear_before_ns = time.monotonic_ns() + self._live.settings_revision = 1 + self._live_writer.write(self._live) + finally: + self._processor.send_signal(signal.SIGCONT) + deadline = time.monotonic() + _TIMEOUT_S + while self._input._read_u64(32) != self._input._read_u64(24): + self.assertLess(time.monotonic(), deadline, self._stderr_path.read_text()) + time.sleep(.01) + delayed = _sweep(2, time.monotonic_ns()) + delayed.capture_start_ns, delayed.capture_end_ns = sweeps[1].capture_start_ns, sweeps[1].capture_end_ns + self._push([delayed, _sweep(4, time.monotonic_ns())]) + self.assertEqual(self._published_ids(1), [4]) + self._live.settings_revision = 2 + self._live_writer.write(self._live) + self.assertEqual(self._published_ids(1), [4]) + + def test_multiple_deletions_survive_coalesced_live_updates_and_queued_replays(self) -> None: + now = time.monotonic_ns() + sweeps = [_sweep(cid, now + cid * 1_000_000) for cid in (1, 2, 3, 4)] + self._push(sweeps[:3]) + self.assertEqual(self._published_ids(3), [1, 2, 3]) + self._processor.send_signal(signal.SIGSTOP) + os.waitpid(self._processor.pid, os.WUNTRACED) + try: + for sweep in reversed(sweeps[1:3]): + self._live.history_removed.append(dict(collection_id=sweep.collection_id, + capture_start_ns=sweep.capture_start_ns)) + self._live.settings_revision += 1 + self._live_writer.write(self._live) + # A normal settings write follows both clicks before the processor wakes. + self._live.bscan_gain = 2. + self._live.settings_revision += 1 + self._live_writer.write(self._live) + self._push(sweeps[1:]) + finally: + self._processor.send_signal(signal.SIGCONT) + self.assertEqual(self._published_ids(2), [1, 4]) + self._live.settings_revision += 1 + self._live_writer.write(self._live) + self.assertEqual(self._published_ids(2), [1, 4]) + + class OversizedResultsTest(_ProcessorTestCase): result_slot_size_bytes = 4096 @@ -227,7 +333,6 @@ class Sar3dRecoveryTest(_ProcessorTestCase): # df=25 MHz gives a 6 m unambiguous one-way range. The history tests' # 64-point sweep aliases before this test's 2..5 m depth gate. sweep.traces[0].frequency_hz = np.linspace(1e9, 6e9, 201, dtype=np.float32) - sweep.traces[0].s11 = np.zeros(201, dtype=np.complex64) sweep.traces[0].s21 = np.exp(-4j * np.pi * sweep.traces[0].frequency_hz * 4.0 / 3e8).astype(np.complex64) self._push([sweep]) diff --git a/python_app/tests/test_reader_recovery.py b/python_app/tests/test_reader_recovery.py new file mode 100644 index 0000000..f5e93f5 --- /dev/null +++ b/python_app/tests/test_reader_recovery.py @@ -0,0 +1,176 @@ +"""Reader recovery requires data, rather than an open mapping or an empty poll.""" + +from __future__ import annotations + +from contextlib import ExitStack + +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from PyQt6.QtWidgets import QApplication + +from python_app.gui.controllers.history_controller import HistoryController +from python_app.gui.controllers.pipeline_controller import PipelineController +from python_app.gui.controllers.plot_controller import PlotController +from python_app.gui.controllers.recording_controller import RecordingController +from python_app.models.dataset_model import ResultCollection, SweepCollection +from python_app.models.gui_profile_schema import GuiProcessingStateModel +from python_app.models.run_config_schema import RunConfigModel +from python_app.models.run_config_schema import GprModel +from python_app.orchestration.config_writer import ConfigWriter +from python_app.orchestration.live_processing_config import ProcessingLiveConfig +from python_app.orchestration.process_supervisor import ProcessSupervisor +from python_app.orchestration.shm.ring_reader import ShmRingReader +from python_app.storage.npz.store import NpzStore + +_app: QApplication | None = None + + +def pipeline_fixture(test: unittest.TestCase) -> tuple[PipelineController, mock.Mock]: + """Use real controllers, isolating only processes and external GUI callbacks.""" + cleanup = ExitStack() + test.addCleanup(cleanup.close) + global _app + instance = QApplication.instance() + _app = instance if isinstance(instance, QApplication) else QApplication([]) + root = Path(cleanup.enter_context(tempfile.TemporaryDirectory())) + store, history = NpzStore(root / "data"), HistoryController() + plots = PlotController(history, GuiProcessingStateModel, GprModel, lambda: False, mock.Mock()) + test.addCleanup(plots.stack.close) + recording = RecordingController(store, {}, mock.Mock(), mock.Mock(), mock.Mock()) + supervisor = mock.Mock(spec=ProcessSupervisor) + supervisor.collect_exit_reports.return_value = [] + supervisor.pids.return_value = {} + supervisor.is_running.return_value = False + supervisor.is_processor_running.return_value = False + with mock.patch("python_app.gui.controllers.pipeline_controller.ProcessSupervisor", return_value=supervisor): + pipeline = PipelineController(root, store, ConfigWriter(root / "runtime"), history, plots, recording, + build_config=RunConfigModel, live_config=ProcessingLiveConfig, publish_live=mock.Mock(), + capture_guard=lambda: None, refresh_radar_limits=lambda: False, status=mock.Mock(), + history_changed=mock.Mock(), poll_ui=mock.Mock(), show_exception=mock.Mock(), deferred_close=mock.Mock()) + test.addCleanup(pipeline.timer.stop) + test.addCleanup(pipeline.replay.close) + return pipeline, supervisor + + +def _reader(name: str) -> mock.Mock: + reader = mock.Mock(spec=ShmRingReader) + reader.name = name + reader.pop_raw_collection.return_value = None + reader.pop_preprocessed_collection.return_value = None + reader.pop_result_collection.return_value = None + return reader + + +class ReaderRecoveryTest(unittest.TestCase): + def setUp(self) -> None: + cleanup = ExitStack() + self.addCleanup(cleanup.close) + self.pipeline, self.supervisor = pipeline_fixture(self) + cleanup.enter_context(mock.patch.multiple(self.pipeline, _READER_ERROR_RECONNECT_AT=2, + _READER_ERROR_RELOG_EVERY=3, _READER_ERROR_STOP_AT=5)) + self.pipeline.max_pop_per_poll = 1 + self.pipeline._raw_reader = self.raw = _reader("/raw") + self.pipeline._pre_reader = self.pre = _reader("/preprocessed") + self.pipeline._result_reader = self.result = _reader("/results") + self.stop = cleanup.enter_context(mock.patch.object(self.pipeline, "stop_all")) + self.draw = cleanup.enter_context(mock.patch.object(self.pipeline.plots, "draw")) + self.error = cleanup.enter_context(mock.patch.object(self.pipeline, "_show_exception")) + self.status = cleanup.enter_context(mock.patch.object(self.pipeline, "_status")) + self.open_reader = cleanup.enter_context(mock.patch( + "python_app.gui.controllers.pipeline_controller.ShmRingReader", side_effect=_reader)) + + def fail_poll(self, error: Exception) -> None: + assert self.pipeline._raw_reader is not None + with mock.patch.object(self.pipeline._raw_reader, "pop_raw_collection", side_effect=error): + self.pipeline.poll() + + def test_successful_reconnect_and_empty_polls_preserve_the_failure_streak(self) -> None: + self.fail_poll(ValueError("corrupt frame")) + self.open_reader.assert_not_called() + self.fail_poll(ValueError("corrupt frame")) + self.assertEqual(self.open_reader.call_args_list, [mock.call("/raw"), mock.call("/preprocessed"), mock.call("/results")]) + for reader in (self.raw, self.pre, self.result): + reader.close.assert_called_once() + for _ in range(3): + self.pipeline.poll() + self.assertEqual(self.pipeline._reader_error_repeat_count, 2) + self.assertEqual(self.pipeline._last_reader_error_signature, ("ValueError", "corrupt frame")) + self.stop.assert_not_called() + + def test_alternating_error_signatures_reach_stop_despite_empty_polls(self) -> None: + for count in range(1, 6): + self.fail_poll(ValueError("bad size") if count % 2 else RuntimeError("bad magic")) + if count < 5: + self.stop.assert_not_called() + self.pipeline.poll() + self.assertEqual(self.pipeline._reader_error_repeat_count, count) + self.stop.assert_called_once() + self.assertEqual(self.open_reader.call_count, 3) + self.assertEqual(self.pipeline._reader_error_repeat_count, 0) + self.assertIsNone(self.pipeline._last_reader_error_signature) + self.status.assert_called_with("Status: error") + + def test_failed_reconnect_does_not_cancel_escalation(self) -> None: + self.open_reader.side_effect = OSError("mapping unavailable") + for _ in range(5): + self.fail_poll(RuntimeError("closed mapping")) + self.open_reader.assert_called_once_with("/raw") + self.stop.assert_called_once() + + def test_only_a_received_collection_resets_the_streak_for_each_stage(self) -> None: + stages = ((self.raw.pop_raw_collection, self.pipeline.history.raw, SweepCollection(1, 100)), + (self.pre.pop_preprocessed_collection, self.pipeline.history.preprocessed, SweepCollection(1, 110)), + (self.result.pop_result_collection, self.pipeline.history.results, ResultCollection(1, 110))) + for pop, history, collection in stages: + with self.subTest(collection=type(collection).__name__, stage=pop): + self.pipeline._reader_error_repeat_count = 4 + self.pipeline._last_reader_error_signature = ("ValueError", "corrupt frame") + pop.return_value = collection + self.pipeline.poll() + pop.return_value = None + self.assertEqual(list(history), [collection]) + self.assertEqual(self.pipeline._reader_error_repeat_count, 0) + self.assertIsNone(self.pipeline._last_reader_error_signature) + self.fail_poll(ValueError("another corrupt frame")) + self.assertEqual(self.pipeline._reader_error_repeat_count, 1) + self.stop.assert_not_called() + + def test_identical_failures_are_periodically_logged_without_resetting_the_streak(self) -> None: + with self.assertLogs("python_app.gui.controllers.pipeline_controller", level="ERROR") as logs: + for _ in range(3): + self.fail_poll(ValueError("corrupt frame")) + self.assertEqual(self.pipeline._reader_error_repeat_count, 3) + self.assertEqual([record.getMessage() for record in logs.records], [ + "Reader poll failed", "Reader poll still failing (repeat #3)"]) + + def test_read_failure_skips_display_and_uses_reader_recovery(self) -> None: + self.result.pop_result_collection.return_value = ResultCollection(1, 110) + self.fail_poll(ValueError("corrupt frame")) + self.assertEqual(self.pipeline._reader_error_repeat_count, 1) + self.draw.assert_not_called() + self.error.assert_not_called() + self.stop.assert_not_called() + + def test_preprocess_change_rejects_late_results_while_processor_stays_alive(self) -> None: + for restart in (False, True): + with self.subTest(restart_processor=restart), mock.patch.object(self.pipeline, "publish_live") as publish: + old, fresh = [ResultCollection(1, start, capture_start_ns=start) for start in (100, 201)] + self.pipeline.history.results.append(old) + self.pipeline._processor_signature = ("old processor",) + self.supervisor.is_processor_running.return_value = True + with mock.patch("python_app.gui.controllers.history_controller.time.monotonic_ns", return_value=200): + self.pipeline.preprocessing_changed(restart_processor=restart) + self.result.pop_result_collection.side_effect = [old, fresh] + self.assertIsNone(self.pipeline.read_results()) + self.assertFalse(self.pipeline.history.results) + self.assertIs(self.pipeline.read_results(), fresh) + publish.assert_called_once() + self.assertEqual(self.pipeline._processor_signature, None if restart else ("old processor",)) + self.supervisor.stop_all.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/python_app/tests/test_recording_controller.py b/python_app/tests/test_recording_controller.py new file mode 100644 index 0000000..7584425 --- /dev/null +++ b/python_app/tests/test_recording_controller.py @@ -0,0 +1,329 @@ +"""Recording lifecycle, bounded disk work, and acquisition profile provenance.""" + +from __future__ import annotations + +from collections.abc import Callable +from contextlib import nullcontext +from pathlib import Path +import tempfile +import threading +import time +import unittest +from unittest import mock + +import numpy as np + +from python_app.gui.controllers import recording_controller as rec +from python_app.gui.controllers.profile_controller import ProfileController +from python_app.gui.controllers.recording_controller import RecordingController +from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection, TraceData +from python_app.models.gui_profile_schema import GuiProfileModel +from python_app.models.run_config_schema import RunConfigModel +from python_app.storage.npz.store import NpzStore + + +def _sweep(cid: int, ns: int | None = None) -> SweepCollection: + ns = time.monotonic_ns() if ns is None else ns + return SweepCollection(cid, ns + 10, capture_start_ns=ns, capture_end_ns=ns + 1, + traces=[TraceData(ComboKey(0, 0), np.array([1e9], dtype=np.float32), + np.array([1 + 2j], dtype=np.complex64))]) + + +def _result(cid: int, revision: int = 1, ns: int | None = None) -> ResultCollection: + ns = time.monotonic_ns() if ns is None else ns + return ResultCollection(cid, ns + 10, settings_revision=revision, capture_start_ns=ns, capture_end_ns=ns + 1) + + +class _Writer: + """The disk boundary can block or fail while the real controller keeps running.""" + + def __init__(self, directory: Path) -> None: + self.directory = directory + self.chunks: list[tuple[int, int, int]] = [] + self.settings: dict[int, dict] = {} + self.block: tuple[threading.Event, threading.Event] | None = None + self.fail = False + + def append(self, raw: list[SweepCollection], preprocessed: list[SweepCollection], + results: list[ResultCollection], *, processing_settings: dict[int, dict] | None = None) -> None: + if self.block is not None: + entered, release = self.block + entered.set() + if not release.wait(timeout=2): + raise TimeoutError("test did not release disk writer") + if self.fail: + raise OSError("disk full") + self.chunks.append((len(raw), len(preprocessed), len(results))) + self.settings.update(processing_settings or {}) + + +class RecordingControllerTest(unittest.TestCase): + def setUp(self) -> None: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.store = NpzStore(self.root / "sets") + self.output = self.root / "datasets" + self.pending = GuiProfileModel() + self.active_config: RunConfigModel | None = None + self.processing_settings: dict[int, dict] = {1: {"settings_revision": 1, "gain": 2}} + self.errors: list[tuple[str, str | None]] = [] + self.logs: list[str] = [] + self.profiles: list[tuple[bool, bool]] = [] + self.fail_profile = False + self.controller = RecordingController(self.store, self.processing_settings, self._profile, + self.logs.append, self._error) + self.writer = _Writer(self.output / "unused") + self.real_create = self.store.create_snapshot_stream + + def create(root: Path, name: str, *, name_prefix: str = "") -> _Writer: + self.writer = _Writer(self.real_create(root, name, name_prefix=name_prefix).directory) + return self.writer + + patcher = mock.patch.object(self.store, "create_snapshot_stream", side_effect=create) + self.create = patcher.start() + self.addCleanup(patcher.stop) + self.addCleanup(self._finish) + + def _error(self, message: str, *, details: str | None = None) -> None: + self.errors.append((message, details)) + + def _profile(self, path: Path, *, allow_overwrite: bool = True, dataset: bool = False) -> GuiProfileModel: + self.profiles.append((allow_overwrite, dataset)) + if self.fail_profile: + raise OSError("profile write failed") + return ProfileController.write_profile(path, self.pending, self.active_config, + allow_overwrite=allow_overwrite, dataset=dataset) + + def _start(self, target: int = 1000, *, running: bool = True, name: str = "run", + callback: Callable[[], bool] = lambda: True) -> None: + self.controller.start(self.output, name, "pref", target, running=running, start_acquisition=callback) + + def _feed(self, cid: int, revision: int = 1) -> None: + self.controller.record("raw", _sweep(cid)) + self.controller.record("preprocessed", _sweep(cid)) + self.controller.record("results", _result(cid, revision)) + + def _drain(self) -> None: + deadline = time.monotonic() + 2 + while self.controller.active and time.monotonic() < deadline: + self.controller.poll() + time.sleep(0.002) + self.assertFalse(self.controller.active, "writer did not drain") + + def _finish(self) -> None: + self.controller.finish() + self._drain() + + def test_pause_reports_incomplete_recording_with_count_target_and_reason(self) -> None: + self._start(10) + self._feed(1) + self.controller.finish() + self._drain() + self.assertTrue(any("1 of 10" in message and "stopped" in message.lower() + and "pipeline" in message.lower() for message in self.logs), self.logs) + self.assertFalse(any("recording complete:" in message.lower() for message in self.logs)) + + def test_arm_guards_and_running_pipeline_is_not_restarted(self) -> None: + self._start(0) + self.assertFalse(self.controller.active) + start = mock.Mock(return_value=True) + self._start(3, callback=start) + self._start(4, callback=start) + start.assert_not_called() + self.assertEqual(self.create.call_count, 1) + self.assertEqual(len(self.errors), 2) + self.assertEqual(self.profiles, [(False, True)]) + self.assertEqual(self.controller.status(), {"active": True, "collected": 0, "target": 3}) + + def test_thread_or_initial_profile_failure_cleans_new_directory_and_allows_retry(self) -> None: + for failure in ("profile", "thread"): + with self.subTest(failure=failure): + self.fail_profile = failure == "profile" + with mock.patch.object(rec.threading.Thread, "start", side_effect=RuntimeError("no threads")): + self._start() + self.assertFalse((self.output / "pref_run").exists()) + self.assertEqual(self.controller.status(), {"active": False, "collected": 0, "target": 0}) + self.fail_profile = False + self._start() + + def test_existing_profile_is_preserved_on_destination_collision(self) -> None: + path = self.output / "pref_run/config_profile.json" + path.parent.mkdir(parents=True) + path.write_bytes(b"existing profile") + self._start() + self.create.assert_not_called() + self.assertEqual(path.read_bytes(), b"existing profile") + self.assertFalse(self.controller.active) + self.assertEqual(len(self.errors), 1) + + def test_running_profile_uses_applied_not_pending_object_calibration(self) -> None: + for index, (applied, selected) in enumerate((("", "next_run"), ("applied", ""))): + with self.subTest(applied=applied): + self.pending.run_config.preprocess.s21.object_calibration.set_name = selected + self.active_config = self.pending.run_config.clone() + self.active_config.preprocess.s21.object_calibration.set_name = applied + self.active_config.preprocess.s21.object_calibration.bundle_path = "/bundles/active" if applied else "" + self._start(name=str(index)) + recorded = GuiProfileModel.load_from_path(self.writer.directory / "config_profile.json") + self.assertEqual(recorded.run_config.preprocess, self.active_config.preprocess) + self._finish() + + def test_new_run_profile_is_refreshed_and_technical_stop_does_not_disarm(self) -> None: + self.pending.run_config.preprocess.s21.object_calibration.set_name = "next_run" + self.active_config = self.pending.run_config.clone() + self.active_config.preprocess.s21.object_calibration.set_name = "previous_run" + + def start() -> bool: + initial = GuiProfileModel.load_from_path(self.writer.directory / "config_profile.json") + self.assertEqual(initial.run_config.preprocess, self.pending.run_config.preprocess) + self.controller.finish() # technical Stop while preparing a processor restart + self.controller.poll() + self.active_config = self.pending.run_config.clone() + self.active_config.preprocess.s21.calibration.bundle_path = "/bundles/new_through" + return True + + self._start(running=False, callback=start) + recorded = GuiProfileModel.load_from_path(self.writer.directory / "config_profile.json") + assert self.active_config is not None + self.assertEqual(recorded.run_config.preprocess, self.active_config.preprocess) + self.assertEqual(self.profiles, [(False, False), (True, True)]) + self.assertEqual(self.errors, []) + + def test_failed_or_raising_start_preserves_accepted_raw_but_removes_empty_dataset(self) -> None: + self.create.side_effect = self.real_create + for accepted in (False, True): + for raises in (False, True): + name = f"{accepted}_{raises}" + + def start() -> bool: + if accepted: + self.controller.record("raw", _sweep(1)) + if raises: + raise RuntimeError("startup interrupted") + return False # includes explicit Stop/Close, even if a worker started + + self._start(running=False, name=name, callback=start) + self._drain() + destination = self.output / f"pref_{name}" + self.assertEqual(destination.exists(), accepted) + if accepted: + self.assertTrue((destination / "config_profile.json").exists()) + paths = list((destination / "raw").glob("*/i0_o0_s21.npy")) + self.assertEqual(len(paths), 1) + np.testing.assert_array_equal(np.load(paths[0]), np.array([1 + 2j], dtype=np.complex64)) + + def test_failed_profile_refresh_keeps_initial_profile_and_received_data(self) -> None: + def start() -> bool: + self.controller.record("raw", _sweep(1)) + self.fail_profile = True + return True + + self._start(running=False, callback=start) + self._drain() + self.assertEqual(self.writer.chunks, [(1, 0, 0)]) + GuiProfileModel.load_from_path(self.writer.directory / "config_profile.json") + self.assertIn("update disk recording config profile", self.errors[0][0]) + + def test_target_or_writer_error_during_start_keeps_session_until_callback_returns(self) -> None: + def start() -> bool: + self.writer.fail = fail + self._feed(1) + session = self.controller._session + assert session is not None and session.thread is not None + session.thread.join(timeout=1) + self.controller.poll() + self.assertTrue(self.controller.active) + self._start() # a second arm cannot replace this still-starting session + return True + + for index, fail in enumerate((False, True)): + with self.subTest(fail=fail), self.assertLogs(rec.logger, level="ERROR") if fail else nullcontext(): + self._start(1, running=False, name=str(fail), callback=start) + self._drain() + self.assertEqual(self.create.call_count, index + 1) + self.assertEqual(self.writer.chunks, [] if fail else [(1, 1, 1)]) + self.assertEqual(self.profiles[-2:], [(False, False), (True, True)]) + + def test_recording_gates_old_captures_chunks_stages_and_finishes_exactly_at_target(self) -> None: + with mock.patch.object(rec, "_RECORDING_CHUNK_SIZE", 2): + self._start(5) + self.controller.record("results", _result(99, ns=0)) + self.assertEqual(self.controller.status()["collected"], 0) + self.processing_settings[2] = {"settings_revision": 2, "gain": 9} + for cid in range(7): + self._feed(cid, 1 if cid < 2 else 2) + self._drain() + self.assertEqual(tuple(sum(chunk[stage] for chunk in self.writer.chunks) for stage in range(3)), (5, 5, 5)) + self.assertTrue(all(max(chunk) <= 2 for chunk in self.writer.chunks)) + self.assertEqual(self.writer.settings, self.processing_settings) + + def test_results_without_taps_and_stop_with_only_raw_stages_are_preserved(self) -> None: + self._start(1) + self.controller.record("results", _result(1)) + self._drain() + self.assertEqual(self.writer.chunks, [(0, 0, 1)]) + with mock.patch.object(rec, "_RECORDING_CHUNK_SIZE", 4), mock.patch.object(rec, "_RECORDING_QUEUE_CHUNKS", 16): + self._start(name="raw") + for cid in range(50): + self.controller.record("raw", _sweep(cid)) + self.controller.record("preprocessed", _sweep(1)) + self._finish() + self.assertEqual(tuple(sum(chunk[stage] for chunk in self.writer.chunks) for stage in range(3)), (50, 1, 0)) + self.controller.finish() # idle Stop remains a no-op + + def test_stop_returns_while_writer_is_blocked_and_flushes_partial_chunk_after_release(self) -> None: + with mock.patch.object(rec, "_RECORDING_CHUNK_SIZE", 2): + self._start() + entered, release = threading.Event(), threading.Event() + self.addCleanup(release.set) + self.writer.block = entered, release + self.controller.record("raw", _sweep(1)) + self.controller.record("raw", _sweep(2)) + self.assertTrue(entered.wait(timeout=1)) + self._feed(3) + finished = threading.Event() + stop = threading.Thread(target=lambda: (self.controller.finish(), finished.set())) + stop.start() + try: + self.assertTrue(finished.wait(timeout=0.5), "Stop waited for disk") + self.assertTrue(self.controller.active) + finally: + release.set() + stop.join(timeout=2) + self._drain() + self.assertEqual(self.writer.chunks, [(2, 0, 0), (1, 1, 1)]) + + def test_full_queue_keeps_one_terminal_chunk_and_failure_releases_queued_memory(self) -> None: + for fail in (False, True): + with self.subTest(fail=fail), mock.patch.object(rec, "_RECORDING_CHUNK_SIZE", 1), \ + mock.patch.object(rec, "_RECORDING_QUEUE_CHUNKS", 1): + self._start(name=str(fail)) + entered, release = threading.Event(), threading.Event() + self.addCleanup(release.set) + self.writer.block, self.writer.fail = (entered, release), fail + self.controller.record("raw", _sweep(1)) + self.assertTrue(entered.wait(timeout=1)) + self.controller.record("preprocessed", _sweep(1)) + returned = threading.Event() + producer = threading.Thread(target=lambda: (self.controller.record("results", _result(1)), returned.set())) + producer.start() + try: + self.assertTrue(returned.wait(timeout=0.5), "Queue overflow waited for disk") + self._feed(2) + session = self.controller._session + assert session is not None and session.thread is not None + self.assertFalse(session.thread.daemon) + finally: + producer.join(timeout=1) + with self.assertLogs(rec.logger, level="ERROR") if fail else nullcontext(): + release.set() + self._drain() + self.assertTrue(session.queue.empty()) + self.assertIsNone(session.tail) + self.assertEqual(self.writer.chunks, [] if fail else [(1, 0, 0), (0, 1, 0), (0, 0, 1)]) + self.assertEqual(len(self.errors), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/python_app/tests/test_recording_mixin.py b/python_app/tests/test_recording_mixin.py deleted file mode 100644 index 81df524..0000000 --- a/python_app/tests/test_recording_mixin.py +++ /dev/null @@ -1,493 +0,0 @@ -"""Tests for the streaming disk-recording mixin. - -The recorder preserves each received stage independently, buffers a small chunk, -and flushes it to a stream writer — so memory stays flat regardless of how many -measurements are recorded. These tests pin that behaviour (chunking, finish-at-N, -arm guards, only-new gating) against light stubs of the AppWindow collaborators — no Qt, -hardware, or disk needed. -""" - -from __future__ import annotations - -from pathlib import Path -import threading -import tempfile -import unittest -from unittest import mock - -import numpy as np - -from python_app.gui.controllers import app_window_recording_mixin as rec -from python_app.gui.controllers.app_window_recording_mixin import AppWindowRecordingMixin -from python_app.gui.controllers.app_window_config.profile_io_mixin import AppWindowConfigProfileIOMixin -from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData -from python_app.models.gui_profile_model import GuiProfileModel -from python_app.storage.npz.store import NpzStore - - -class _Collection: - def __init__(self, collection_id: int, monotonic_ns: int = 0) -> None: - self.collection_id = collection_id - self.monotonic_ns = monotonic_ns - - -class _Spin: - def __init__(self, value: int) -> None: - self._value = value - - def value(self) -> int: - return self._value - - -class _Text: - def __init__(self, value: str) -> None: - self._value = value - - def text(self) -> str: - return self._value - - -class _Supervisor: - def __init__(self, running: bool) -> None: - self._running = running - - def is_running(self) -> bool: - return self._running - - -class _FakePath: - def __init__(self, exists: bool) -> None: - self._exists = exists - - def exists(self) -> bool: - return self._exists - - def __str__(self) -> str: - return "/tmp/dataset" - - -class _FakeWriter: - def __init__(self) -> None: - self.directory = Path("/tmp/dataset") - self.appends: list[tuple[int, int, int]] = [] - self.result_count = 0 - - def append(self, raw, preprocessed, results) -> None: - self.appends.append((len(raw), len(preprocessed), len(results))) - self.result_count += len(results) - - -class _FakeStore: - def __init__(self, *, exists: bool = False) -> None: - self._exists = exists - self.created: list[tuple] = [] - self.writer: _FakeWriter | None = None - - def create_snapshot_stream(self, root, name, *, name_prefix=""): - if self._exists: - raise FileExistsError(f"Snapshot directory already exists: {root}/{name}") - self.created.append((root, name, name_prefix)) - self.writer = _FakeWriter() - return self.writer - - -class _Harness(AppWindowRecordingMixin): - def __init__(self, *, count, running, dest_exists=False, store_exists=False) -> None: - self._init_recording_state() - self._record_count = _Spin(count) - self._supervisor = _Supervisor(running) - self._store = _FakeStore(exists=store_exists) - self._save_path_input = _Text("/tmp") - self._save_name_input = _Text("run") - self._dest_exists = dest_exists - self.started = False - self.errors: list[str] = [] - self.exceptions: list[tuple] = [] - self.logs: list[str] = [] - self.profiles: list[tuple] = [] - - def _snapshot_destination_dir(self): - return _FakePath(self._dest_exists) - - def _radar_config_name_prefix(self) -> str: - return "pref" - - def _snapshot_config_profile_path(self, directory): - return Path(directory) / "config_profile.json" - - def _write_gui_profile_to_path(self, path, allow_overwrite=True, dataset=False) -> None: - self.profiles.append((path, allow_overwrite)) - - def _start_run(self) -> None: - self.started = True - self._supervisor._running = True - - def _show_error(self, message, *, details=None) -> None: - self.errors.append(message) - - def _show_exception(self, context, exc) -> None: - self.exceptions.append((context, exc)) - - def _log(self, message) -> None: - self.logs.append(message) - - # convenience for tests: stamp collections just after the arm instant so the - # "only record sweeps produced after arming" gate accepts them. - def feed(self, collection_id: int, *, ns: int | None = None) -> None: - if ns is None: - ns = self._recording_since_ns + 1 - self._record_collection("raw", _Collection(collection_id, ns)) - self._record_collection("preprocessed", _Collection(collection_id, ns)) - self._record_collection("results", _Collection(collection_id, ns)) - - -class _RecordingTestBase(unittest.TestCase): - def _make(self, **kwargs) -> _Harness: - harness = _Harness(**kwargs) - self.addCleanup(harness._shutdown_recording) # never leak the writer thread - return harness - - def _drain_and_finalize(self, harness: _Harness) -> None: - """Wait for the writer thread to flush all chunks, then finalize on the GUI side.""" - thread = harness._recording_thread - self.assertIsNotNone(thread) - thread.join(timeout=2.0) - self.assertFalse(thread.is_alive(), "writer thread did not drain") - harness._poll_recording_writer() - - -class RecordingArmTest(_RecordingTestBase): - def test_writer_thread_start_failure_cleans_new_dataset_and_allows_retry(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - h = self._make(count=5, running=True) - h._store = NpzStore(Path(temporary) / "sets") - h._save_path_input = _Text(str(Path(temporary) / "datasets")) - destination = Path(temporary) / "datasets/pref_run" - - def write_profile(path, allow_overwrite=True, dataset=False): - path.write_text("fresh profile", encoding="utf-8") - - with mock.patch.object(h, "_write_gui_profile_to_path", side_effect=write_profile): - with mock.patch.object(rec.threading.Thread, "start", side_effect=RuntimeError("cannot start new thread")): - h._start_run_with_recording() - self.assertFalse(destination.exists()) - self.assertFalse(h._recording_active) - self.assertFalse(h._recording_collecting) - self.assertIsNone(h._recording_thread) - self.assertIsNone(h._recording_writer) - self.assertIsNone(h._recording_queue) - self.assertEqual(h._recording_target, 0) - self.assertEqual(len(h.exceptions), 1) - h._start_run_with_recording() - self.assertTrue(h._recording_active) - self.assertEqual((destination / "config_profile.json").read_text(), "fresh profile") - - def test_existing_profile_is_preserved_when_recording_destination_collides(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - h = self._make(count=5, running=True) - h._store = NpzStore(Path(temporary) / "sets") - h._save_path_input = _Text(str(Path(temporary) / "datasets")) - destination = Path(temporary) / "datasets/pref_run" - destination.mkdir(parents=True) - profile = destination / "config_profile.json" - profile.write_bytes(b"existing profile") - with mock.patch.object(rec.threading.Thread, "start", side_effect=RuntimeError("cannot start new thread")) as start: - h._start_run_with_recording() - start.assert_not_called() - self.assertEqual(profile.read_bytes(), b"existing profile") - self.assertFalse(h._recording_active) - - def test_arm_starts_a_stopped_run_and_opens_a_stream(self) -> None: - h = self._make(count=3, running=False) - h._start_run_with_recording() - self.assertTrue(h.started) - self.assertTrue(h._recording_active) - self.assertEqual(h._recording_target, 3) - self.assertEqual(len(h._store.created), 1) # one stream opened - self.assertEqual(h.profiles[0][1], False) # config profile written, no overwrite - - def test_arm_does_not_restart_a_running_pipeline(self) -> None: - h = self._make(count=3, running=True) - h._start_run_with_recording() - self.assertFalse(h.started) - self.assertTrue(h._recording_active) - - def test_running_recording_profile_uses_applied_object_calibration(self) -> None: - for applied, selected in (("", "next_run"), ("applied", "")): - with self.subTest(applied=applied), tempfile.TemporaryDirectory() as temporary: - h = self._make(count=3, running=True) - h._store = NpzStore(Path(temporary) / "sets") - h._save_path_input = _Text(str(Path(temporary) / "datasets")) - pending = GuiProfileModel() - pending.run_config.preprocess.s21.object_calibration.set_name = selected - h._active_run_config = pending.run_config.clone() - h._active_run_config.preprocess.s21.object_calibration.set_name = applied - h._active_run_config.preprocess.s21.object_calibration.bundle_path = "/bundles/applied" if applied else "" - h._build_gui_profile = pending.clone - with mock.patch.object(h, "_write_gui_profile_to_path", side_effect=lambda *args, **kwargs: - AppWindowConfigProfileIOMixin._write_gui_profile_to_path(h, *args, **kwargs)): - h._start_run_with_recording() - path = Path(temporary) / "datasets/pref_run/config_profile.json" - recorded = GuiProfileModel.load_from_path(path) - self.assertEqual(recorded.run_config.preprocess, h._active_run_config.preprocess) - self.assertFalse(h.started) - self.assertEqual(h.exceptions, []) - - def test_new_recording_profile_uses_new_run_instead_of_previous_history(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - h = self._make(count=3, running=False) - h._store = NpzStore(Path(temporary) / "sets") - h._save_path_input = _Text(str(Path(temporary) / "datasets")) - pending = GuiProfileModel() - pending.run_config.preprocess.s21.object_calibration.set_name = "next_run" - h._active_run_config = pending.run_config.clone() - h._active_run_config.preprocess.s21.object_calibration.set_name = "previous_run" - h._build_gui_profile = pending.clone - path = Path(temporary) / "datasets/pref_run/config_profile.json" - - def start_new_run(): - initial = GuiProfileModel.load_from_path(path) - self.assertEqual(initial.run_config.preprocess, pending.run_config.preprocess) - h._active_run_config = pending.run_config.clone() - h._active_run_config.preprocess.s21.object_calibration.bundle_path = "/bundles/new_object" - h._active_run_config.preprocess.s21.calibration.bundle_path = "/bundles/new_through" - h._supervisor._running = True - - with mock.patch.object(h, "_start_run", side_effect=start_new_run), mock.patch.object( - h, "_write_gui_profile_to_path", side_effect=lambda *args, **kwargs: - AppWindowConfigProfileIOMixin._write_gui_profile_to_path(h, *args, **kwargs), - ): - h._start_run_with_recording() - recorded = GuiProfileModel.load_from_path(path) - self.assertEqual(recorded.run_config.preprocess, h._active_run_config.preprocess) - self.assertEqual(h.exceptions, []) - self.assertTrue(h._recording_active) - - def test_arm_refuses_when_destination_exists(self) -> None: - h = self._make(count=3, running=True, dest_exists=True) - h._start_run_with_recording() - self.assertFalse(h._recording_active) - self.assertEqual(len(h._store.created), 0) - self.assertEqual(len(h.errors), 1) - - def test_second_arm_while_recording_is_refused(self) -> None: - h = self._make(count=5, running=True) - h._start_run_with_recording() - h._start_run_with_recording() # already in progress - self.assertEqual(len(h._store.created), 1) # no second stream - self.assertEqual(len(h.errors), 1) - self.assertTrue(h._recording_active) - - def test_failed_pipeline_start_does_not_leave_an_empty_recorder_armed(self) -> None: - h = self._make(count=5, running=False) - with mock.patch.object(h, '_start_run'), mock.patch.object(rec.shutil, 'rmtree') as remove: - h._start_run_with_recording() - self.assertFalse(h._recording_active) - self.assertFalse(h._recording_collecting) - self.assertIsNone(h._recording_thread) - remove.assert_called_once_with(h._store.writer.directory) - self.assertTrue(any('acquisition did not start' in message for message in h.logs)) - - def test_failed_pipeline_start_retains_received_raw_data_and_profile(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - h = self._make(count=5, running=False) - h._store = NpzStore(Path(temporary) / 'sets') - h._save_path_input = _Text(str(Path(temporary) / 'datasets')) - samples = np.array([1 + 2j], dtype=np.complex64) - - def failed_start(): - h._record_collection('raw', SweepCollection( - collection_id=1, monotonic_ns=h._recording_since_ns + 1, - traces=[TraceData(ComboKey(0, 0), np.array([1e9], dtype=np.float32), samples, samples)], - )) - - def write_profile(path, allow_overwrite=True, dataset=False): - path.write_text('retained configuration', encoding='utf-8') - - with mock.patch.object(h, '_start_run', side_effect=failed_start), mock.patch.object( - h, '_write_gui_profile_to_path', side_effect=write_profile, - ): - h._start_run_with_recording() - destination = Path(temporary) / 'datasets/pref_run' - self.assertFalse(h._recording_active) - self.assertEqual((destination / 'config_profile.json').read_text(), 'retained configuration') - files = list((destination / 'raw').glob('*/i0_o0_s21.npy')) - self.assertEqual(len(files), 1) - np.testing.assert_array_equal(np.load(files[0]), samples) - - -class RecordingStreamTest(_RecordingTestBase): - def test_ignores_collections_from_before_arming(self) -> None: - h = self._make(count=2, running=True) - h._start_run_with_recording() - arm = h._recording_since_ns - h._record_collection("results", _Collection(1, monotonic_ns=arm - 1)) - self.assertEqual(h._recording_status()["collected"], 0) - self.assertIsNotNone(h._store.writer) - self.assertEqual(h._store.writer.result_count, 0) - - def test_streams_one_chunk_and_finishes_at_target(self) -> None: - h = self._make(count=3, running=True) - h._start_run_with_recording() - writer = h._store.writer - for cid in (1, 2, 3): - h.feed(cid) - self._drain_and_finalize(h) - self.assertEqual(writer.appends, [(3, 3, 3)]) # one flush at target - self.assertEqual(writer.result_count, 3) - self.assertFalse(h._recording_active) # disarmed after the writer drained - - def test_flushes_in_chunks_so_memory_stays_flat(self) -> None: - with mock.patch.object(rec, "_RECORDING_CHUNK_SIZE", 2): - h = self._make(count=5, running=True) - h._start_run_with_recording() - writer = h._store.writer - for cid in range(1, 6): - h.feed(cid) - self._drain_and_finalize(h) - self.assertEqual(tuple(sum(chunk[stage] for chunk in writer.appends) for stage in range(3)), (5, 5, 5)) - self.assertTrue(all(max(chunk) <= 2 for chunk in writer.appends)) - self.assertEqual(writer.result_count, 5) - self.assertFalse(h._recording_active) - - def test_does_not_record_beyond_target(self) -> None: - h = self._make(count=2, running=True) - h._start_run_with_recording() - writer = h._store.writer - for cid in (1, 2, 3, 4): # 4 results, target 2 - h.feed(cid) - self._drain_and_finalize(h) - self.assertEqual(writer.result_count, 2) - - def test_records_results_with_missing_raw_or_pre(self) -> None: - h = self._make(count=1, running=True) - h._start_run_with_recording() - writer = h._store.writer - # result with no matching raw/pre buffered (e.g. a dropped raw frame) - h._record_collection("results", _Collection(9, monotonic_ns=h._recording_since_ns + 1)) - self._drain_and_finalize(h) - self.assertEqual(writer.appends, [(0, 0, 1)]) - self.assertEqual(writer.result_count, 1) - - def test_stop_flushes_the_partial_chunk(self) -> None: - # Stop pressed before a chunk fills (and before the target): the buffered - # measurements must be written, not lost. - h = self._make(count=1000, running=True) # target large -> never reached here - h._start_run_with_recording() - writer = h._store.writer - for cid in (1, 2, 3): # 3 < chunk size and < target -> buffered, not yet flushed - h.feed(cid) - self.assertEqual(writer.appends, []) # nothing flushed yet - h._finalize_recording_on_stop() # Stop pressed - self.assertEqual(writer.appends, [(3, 3, 3)]) # partial chunk written - self.assertEqual(writer.result_count, 3) - self.assertFalse(h._recording_active) # recording ended - - def test_stop_when_idle_is_a_noop(self) -> None: - h = self._make(count=5, running=True) - h._finalize_recording_on_stop() # nothing armed - self.assertEqual(len(h._store.created), 0) - self.assertFalse(h._recording_active) - - def test_raw_stage_streams_when_processor_has_not_returned_any_results(self) -> None: - with mock.patch.object(rec, "_RECORDING_CHUNK_SIZE", 4): - h = self._make(count=1000, running=True) - h._start_run_with_recording() - for cid in range(50): # raws whose results never arrive - h._record_collection("raw", _Collection(cid, monotonic_ns=h._recording_since_ns + 1)) - self.assertLess(len(h._recording_chunk_raw), 4) - h._finalize_recording_on_stop() - self.assertEqual(sum(chunk[0] for chunk in h._store.writer.appends), 50) - self.assertEqual(h._store.writer.result_count, 0) - - def test_shutdown_flushes_data_without_results(self) -> None: - h = self._make(count=1000, running=True) - h._start_run_with_recording() - h._record_collection("raw", _Collection(1, h._recording_since_ns + 1)) - h._record_collection("preprocessed", _Collection(1, h._recording_since_ns + 2)) - h._shutdown_recording() - self.assertEqual(h._store.writer.appends, [(1, 1, 0)]) - self.assertFalse(h._recording_active) - - def test_full_queue_does_not_drop_the_finish_sentinel(self) -> None: - with mock.patch.object(rec, "_RECORDING_QUEUE_CHUNKS", 1), mock.patch.object(rec, "_RECORDING_CHUNK_SIZE", 1): - h = self._make(count=1000, running=True) - h._start_run_with_recording() - entered, release = threading.Event(), threading.Event() - self.addCleanup(release.set) - append = h._store.writer.append - - def slow_append(*args): - entered.set() - release.wait(timeout=2) - append(*args) - - h._store.writer.append = slow_append - h._record_collection("raw", _Collection(1, h._recording_since_ns + 1)) - self.assertTrue(entered.wait(timeout=1)) - h._record_collection("preprocessed", _Collection(1, h._recording_since_ns + 2)) - self.assertTrue(h._recording_queue.full()) - timer = threading.Timer(0.05, release.set) - timer.start() - h._stop_collecting() - timer.join(timeout=1) - self._drain_and_finalize(h) - self.assertEqual(h._store.writer.appends, [(1, 0, 0), (0, 1, 0)]) - - def test_stop_keeps_a_slow_writer_alive_until_accepted_queue_is_written(self) -> None: - h = self._make(count=1000, running=True) - h._start_run_with_recording() - entered, release = threading.Event(), threading.Event() - self.addCleanup(release.set) - append = h._store.writer.append - - def slow_append(*args): - entered.set() - release.wait(timeout=2) - append(*args) - - h._store.writer.append = slow_append - h.feed(1) - h._flush_recording_chunk() - self.assertTrue(entered.wait(timeout=1)) - h.feed(2) - thread = h._recording_thread - with mock.patch.object(thread, 'join'): - h._finalize_recording_on_stop() - self.assertTrue(h._recording_active) - self.assertIs(h._recording_thread, thread) - self.assertFalse(h._recording_stop.is_set()) - release.set() - self._drain_and_finalize(h) - self.assertEqual(h._store.writer.result_count, 2) - - def test_write_failure_with_full_queue_does_not_block_the_producer(self) -> None: - with mock.patch.object(rec, "_RECORDING_QUEUE_CHUNKS", 1), mock.patch.object(rec, "_RECORDING_CHUNK_SIZE", 1): - h = self._make(count=1000, running=True) - h._start_run_with_recording() - entered, release = threading.Event(), threading.Event() - self.addCleanup(release.set) - - def fail_append(*args): - entered.set() - release.wait(timeout=2) - raise OSError('disk full') - - h._store.writer.append = fail_append - h._record_collection("raw", _Collection(1, h._recording_since_ns + 1)) - self.assertTrue(entered.wait(timeout=1)) - h._record_collection("preprocessed", _Collection(1, h._recording_since_ns + 2)) - producer = threading.Thread( - target=h._record_collection, - args=("results", _Collection(1, h._recording_since_ns + 3)), daemon=True, - ) - producer.start() - release.set() - producer.join(timeout=1) - self.assertFalse(producer.is_alive()) - self.assertFalse(h._recording_active) - self.assertEqual(len(h.errors), 1) - - -if __name__ == "__main__": - unittest.main() diff --git a/python_app/tests/test_review_gui_regressions.py b/python_app/tests/test_review_gui_regressions.py new file mode 100644 index 0000000..a62f609 --- /dev/null +++ b/python_app/tests/test_review_gui_regressions.py @@ -0,0 +1,184 @@ +"""User-visible failure contracts from the final lifecycle review.""" + +from functools import partial +import unittest +from unittest import mock + +import numpy as np +from PyQt6.QtCore import QSignalBlocker +from PyQt6.QtGui import QValidator +from PyQt6.QtWidgets import QLineEdit + +from python_app.models.dataset_model import ResultCollection, ResultPayload, SweepCollection +from python_app.models.run_config_schema import RunConfigModel +from python_app.tests.gui_test_support import isolated_window, widget +from python_app.tests.test_reader_recovery import pipeline_fixture +from python_app.gui.controllers.web_controller import _WebActionCall + + +class ReviewGuiRegressions(unittest.TestCase): + def test_refused_actions_report_shutdown_sequence_and_refresh_errors(self) -> None: + pipeline_fixture(self) + window = isolated_window(self.addCleanup) + pipeline = window._pipeline + pipeline.timer.stop() + pipeline._stopping = True + actions = (window._preprocess.capture_tmp_reference, window._preprocess.create_neutral_sets, + partial(window._preprocess.start_sequence, "s21_reference"), window._snapshots.save, + partial(window._snapshots.delete_history, remove_last=True), + partial(window._snapshots.start_recording, 10), + partial(window._profile.load, window._profile.state.path)) + try: + for action in actions: + with self.subTest(action=action): + call = _WebActionCall(action) + window._web._run_action(call) + self.assertIn("shutdown", call.error or "") + finally: + pipeline._stopping = False + with pipeline.pause(), mock.patch.object(pipeline, "_capture_guard", return_value="Finish the active capture sequence first"): + call = _WebActionCall(pipeline.start) + window._web._run_action(call) + self.assertIn("capture sequence", call.error or "") + window._preprocess.ensure_dialog() + with mock.patch.object(window._preprocess, "_current_radar_key", return_value="changed"), \ + mock.patch.object(window._preprocess, "_build_config", side_effect=ValueError("invalid sweep")), \ + window._log.capture_errors() as errors: + window._preprocess.reset_selection() + self.assertEqual(len(errors), 1) + self.assertIn("invalid sweep", errors[0]) + + def test_refused_start_preserves_processor_history_and_continuous_run(self) -> None: + pipeline, supervisor = pipeline_fixture(self) + config = RunConfigModel.from_dict({"radar": {"model": "librevna", "driver_mode": "mock"}}) + config.input_switch.radar_port, config.output_switch.radar_port = 1, 2 + config.preprocess.s21.calibration.set_name = "missing" + config.preprocess.s21.reference.set_name = "missing" + supervisor.is_processor_running.return_value = True + original = ResultCollection(1, 2) + pipeline.history.results.append(original) + for running in (False, True): + with self.subTest(running=running): + supervisor.is_running.return_value = running + pipeline.should_run = running + self.assertFalse(pipeline.start(single_capture=running, config_override=config)) + supervisor.stop_all.assert_not_called() + supervisor.stop_acquisition.assert_not_called() + self.assertEqual(list(pipeline.history.results), [original]) + self.assertEqual(pipeline.should_run, running) + + def test_pause_during_single_capture_does_not_start_continuous_run(self) -> None: + pipeline, supervisor = pipeline_fixture(self) + supervisor.is_running.return_value = True + pipeline.single_capture_active = True + with mock.patch.object(pipeline, "drain_rings"), mock.patch.object(pipeline, "start") as start: + operation = pipeline.pause() + operation.finish() + start.assert_not_called() + + def test_capture_failure_releases_pause_before_presenting_error(self) -> None: + pipeline_fixture(self) + window = isolated_window(self.addCleanup) + window._pipeline.timer.stop() + paused_when_reported = [] + with mock.patch("python_app.gui.controllers.preprocess_controller.capture_reference_set", side_effect=OSError("USB lost")), \ + mock.patch.object(window._preprocess, "_error", side_effect=lambda *_: paused_when_reported.append(window._pipeline.paused)): + window._preprocess.capture_tmp_reference() + self.assertEqual(paused_when_reported, [False]) + for action in ("finalize", "abort", "drain"): + with self.subTest(action=action): + paused_when_reported.clear() + pre, pipeline = window._preprocess, window._pipeline + dialog = pre.ensure_dialog() + if action != "drain": + pre._capture_pause = pipeline.pause() + pre.capture_session = mock.Mock(kind="s21_reference", set_name="test", is_complete=lambda: True) + pre.capture_session.finalize.return_value = ("key", SweepCollection(1, 2)) + with mock.patch.object(pre, "_error", side_effect=lambda *_: paused_when_reported.append(pipeline.paused)), \ + mock.patch.object(pipeline, "_show_exception", side_effect=lambda *_: paused_when_reported.append(pipeline.paused)): + if action == "finalize": + with mock.patch.object(pre, "refresh_sets", side_effect=OSError("refresh failed")): + pre.finalize() + elif action == "abort": + with mock.patch.object(dialog, "set_status", side_effect=OSError("status failed")): + pre.abort() + else: + with mock.patch.object(pipeline.supervisor, "is_running", return_value=True), \ + mock.patch.object(pipeline, "drain_rings", side_effect=OSError("drain failed")): + pipeline.apply_radar() + self.assertEqual(paused_when_reported, [False]) + + def test_failed_resume_retains_restart_intent_until_explicit_stop(self) -> None: + pipeline, supervisor = pipeline_fixture(self) + pipeline.should_run = True + pipeline.active_config = RunConfigModel() + pipeline.active_config_path = pipeline._project_root / "runtime.json" + with mock.patch.object(pipeline, "start", return_value=False), \ + mock.patch.object(pipeline, "_open_readers"), mock.patch.object(pipeline, "drop_pending"): + pipeline.pause().finish() + self.assertTrue(pipeline.should_run) + pipeline._last_restart_s = 0 + pipeline.poll() + supervisor.start.assert_called_once_with(pipeline.active_config_path, allow_clean_acquisition_exit=False) + pipeline.stop() + supervisor.start.reset_mock() + pipeline.poll() + supervisor.start.assert_not_called() + + def test_display_failure_keeps_acquisition_and_rejects_malformed_position_input(self) -> None: + pipeline, supervisor = pipeline_fixture(self) + pipeline.should_run = True + supervisor.pids.return_value = dict.fromkeys(("acquisition", "data_preprocessor", "data_processor"), 1) + with mock.patch.object(pipeline, "read_results", return_value=ResultCollection(1, 2)), \ + mock.patch.object(pipeline.plots, "draw", side_effect=ValueError("bad display")), \ + mock.patch.object(pipeline, "stop_all") as stop, \ + self.assertLogs("python_app.gui.controllers.pipeline_controller", "ERROR") as logs: + pipeline.poll() + pipeline.poll() + stop.assert_not_called() + self.assertTrue(pipeline.should_run) + self.assertEqual(len(logs.records), 1) + for owner, method in ((pipeline.recording, "poll"), (pipeline, "_finish_single_capture_if_ready")): + with self.subTest(method=method), mock.patch.object(owner, method, side_effect=ValueError(method)), \ + mock.patch.object(pipeline, "stop_all") as stop, \ + mock.patch.object(pipeline, "_check_single_capture_deadline") as deadline: + pipeline.single_capture_active = owner is pipeline + pipeline.poll() + stop.assert_not_called() + self.assertTrue(pipeline.should_run) + if pipeline.single_capture_active: + deadline.assert_called_once() + window = isolated_window(self.addCleanup) + window._pipeline.timer.stop() + for mode in ("gpr", "legacy_gpr", "sar3d"): + field = widget(window._view.processing, f"_{mode}_input_positions_input", QLineEdit) + validator = field.validator() + self.assertIsNotNone(validator) + assert validator is not None + self.assertEqual(validator.validate("0,1;", 4)[0], QValidator.State.Invalid) + with QSignalBlocker(field): + field.setText("0, 1,") + # A trailing comma during editing is harmless to the shared CSV parser. + for table in (np.empty((0, 0)), np.array([[99.]])): + result = ResultCollection(1, 2, collection_payloads=[ResultPayload(processing_name="sar3d_status", kind=4, table=table)]) + self.assertFalse(window._plots.sar3d.draw(result)) + self.assertIn("error", window._plots.sar3d.status_label.text().lower()) + + def test_stop_or_close_cancels_paused_actions_without_error_dialog(self) -> None: + pipeline_fixture(self) + window = isolated_window(self.addCleanup) + window._pipeline.timer.stop() + actions = (window._pipeline.apply_radar, window._preprocess.capture_tmp_reference, + window._preprocess.create_neutral_sets, + partial(window._preprocess.start_sequence, "s21_reference"), + partial(window._snapshots.delete_history, remove_last=False)) + with mock.patch.object(window._pipeline, "pause", side_effect=InterruptedError("Pipeline operation cancelled")), \ + mock.patch.object(window._pipeline, "_show_exception") as pipeline_error, \ + mock.patch.object(window._preprocess, "_error") as preprocess_error, \ + mock.patch.object(window._snapshots, "_error") as snapshot_error: + for action in actions: + with self.subTest(action=action): + action() + pipeline_error.assert_not_called() + preprocess_error.assert_not_called() + snapshot_error.assert_not_called() diff --git a/python_app/tests/test_run_config_codec.py b/python_app/tests/test_run_config_codec.py index f1fa474..231cd76 100644 --- a/python_app/tests/test_run_config_codec.py +++ b/python_app/tests/test_run_config_codec.py @@ -41,6 +41,26 @@ def _valid_payload() -> dict: class StrictNumericTypingTest(unittest.TestCase): """Every config integer reads strictly; null falls back to the default.""" + def test_switch_driver_is_only_a_guard_for_old_h7992_profiles(self) -> None: + payload = _valid_payload() + for port in ("port1", "port2"): + switch = payload["switches"][port] + self.assertNotIn("driver", switch) + switch["driver"] = "h7992" + self.assertNotIn("driver", RunConfigModel.from_dict(payload).to_dict()["switches"][port]) + for unsupported in ("hmc349a", "unknown", "", None, 1, {}, []): + switch["driver"] = unsupported + with self.subTest(port=port, driver=unsupported), self.assertRaisesRegex(ValueError, "driver"): + RunConfigModel.from_dict(payload) + switch.pop("driver") + switch["invert_logic"] = False + self.assertNotIn("invert_logic", RunConfigModel.from_dict(payload).to_dict()["switches"][port]) + for unsupported in (True, None, 0, "false", {}, []): + switch["invert_logic"] = unsupported + with self.subTest(port=port, invert_logic=unsupported), self.assertRaisesRegex(ValueError, "invert_logic"): + RunConfigModel.from_dict(payload) + switch.pop("invert_logic") + def _reject_int(self, section: list[str], key: str, value: object) -> None: payload = _valid_payload() node = payload @@ -109,7 +129,6 @@ class AcquisitionValidationTest(unittest.TestCase): config.radar.sweep.points = 32 for port, switch in enumerate((config.output_switch, config.input_switch), 1): switch.radar_port = port - switch.driver = "h7992" switch.gpio_chip = "/dev/gpiochip0" switch.pin_a, switch.pin_b = 17, 27 for name in ("raw", "preprocessed", "raw_tap", "preprocessed_tap"): @@ -123,7 +142,8 @@ class AcquisitionValidationTest(unittest.TestCase): ring = getattr(config.rings, name) ring.name = "/test" original = ring.slot_size_bytes - ring.slot_size_bytes = 4096 + minimum_frame = 44 + len(config.combos) * (28 + 12 * config.radar.sweep.points) + ring.slot_size_bytes = (minimum_frame - 1) // 8 * 8 with self.subTest(ring=name), self.assertRaisesRegex(ValueError, f"rings.{name}"): validate_acquisition_config(config) ring.slot_size_bytes = original @@ -154,7 +174,7 @@ class AcquisitionValidationTest(unittest.TestCase): def test_bad_switch_settings_are_reported_before_hardware_open(self) -> None: for field, value in ( - ("driver_mode", "typo"), ("driver", "typo"), ("default_position", 1), + ("driver_mode", "typo"), ("default_position", 1), ("radar_port", 0), ("pin_a", -1), ("pin_b", 17), ("gpio_chip", ""), ): config = self.config() @@ -207,7 +227,9 @@ class ComboValidationTest(unittest.TestCase): def test_direct_model_positions_cannot_be_silently_truncated(self) -> None: for value in (0.5, True, '0'): with self.subTest(value=value), self.assertRaisesRegex(ValueError, 'integer'): - validate_combos([ComboModel(value, 0)], input_positions=2, output_positions=2) + combo = ComboModel(0, 0) + setattr(combo, 'input', value) + validate_combos([combo], input_positions=2, output_positions=2) def test_within_bounds_passes(self) -> None: validate_combos( [ComboModel(input=0, output=0), ComboModel(input=3, output=1)], @@ -266,9 +288,10 @@ class RingValidationTest(unittest.TestCase): class GprValidationTest(unittest.TestCase): @staticmethod def _gpr(**overrides: object) -> GprModel: - base = {"relative_permittivity": 4.0, "tx_geometry": [], "rx_geometry": []} - base.update(overrides) - return GprModel(**base) # type: ignore[arg-type] + model = GprModel(relative_permittivity=4.0) + for key, value in overrides.items(): + setattr(model, key, value) + return model def _validate(self, gpr: GprModel) -> None: validate_gpr_model(gpr, input_switch_positions=4, output_switch_positions=2) @@ -298,10 +321,14 @@ class GprValidationTest(unittest.TestCase): rx_geometry=[GprRxGeometryModel(input_pos=0, x_m=0.0, y_m=0.0, z_m=0.0)], )) - def test_direct_geometry_positions_cannot_be_silently_truncated(self) -> None: - for value in (0.5, True, '0'): - with self.subTest(value=value), self.assertRaisesRegex(ValueError, 'integer'): - self._validate(self._gpr(tx_geometry=[GprTxGeometryModel(output_pos=value)])) + def test_direct_geometry_values_cannot_be_silently_coerced(self) -> None: + for field, values, message in (("output_pos", (0.5, True, "0"), "integer"), + ("x_m", (True, "0", None, float("nan"), float("inf"), 3.5e38, 10**400), "number")): + for value in values: + with self.subTest(field=field, value=value), self.assertRaisesRegex(ValueError, message): + entry = GprTxGeometryModel() + setattr(entry, field, value) + self._validate(self._gpr(tx_geometry=[entry])) class ParseCombosFromTextTest(unittest.TestCase): diff --git a/python_app/tests/test_runtime_history.py b/python_app/tests/test_runtime_history.py new file mode 100644 index 0000000..4f2bbe0 --- /dev/null +++ b/python_app/tests/test_runtime_history.py @@ -0,0 +1,120 @@ +"""Deleting an acquisition and changing startup settings respect stage ownership.""" + +from __future__ import annotations + +from collections import deque +from dataclasses import fields +import unittest +from unittest import mock + +from python_app.gui.controllers.history_controller import HistoryController +from python_app.gui.runtime.history import ( + build_processor_run_signature, build_run_history_signature, record_result_history, +) +from python_app.models.dataset_model import ResultCollection, SweepCollection +from python_app.models.run_config_schema import RunConfigModel + + +def sweep(cid: int, ns: int, window: tuple[int, int] = (0, 0)) -> SweepCollection: + return SweepCollection(cid, ns, capture_start_ns=window[0], capture_end_ns=window[1]) + + +class RemoveLastHistoryTest(unittest.TestCase): + def test_newest_capture_is_removed_across_stages_despite_late_processing(self) -> None: + old, fresh = sweep(1, 900, (10, 20)), sweep(1, 100, (30, 40)) + pre_old, pre_fresh = sweep(1, 950, (10, 20)), sweep(1, 150, (30, 40)) + late_old = ResultCollection(1, 1000, capture_start_ns=10, capture_end_ns=20) + fresh_result = ResultCollection(1, 200, capture_start_ns=30, capture_end_ns=40) + history = HistoryController() + history.raw.extend([fresh, old]) + history.preprocessed.extend([pre_old, pre_fresh]) + history.results.extend([fresh_result, late_old]) + history.delete(remove_last=True) + self.assertEqual((list(history.raw), list(history.preprocessed), list(history.results)), ([old], [pre_old], [late_old])) + self.assertEqual(history.removed, [{"collection_id": 1, "capture_start_ns": 30}]) + self.assertFalse(history.record_result(fresh_result)) + + def test_missing_stages_do_not_delete_another_acquisition_with_reused_id(self) -> None: + old, fresh = sweep(1, 100, (10, 20)), sweep(1, 200, (30, 40)) + late = ResultCollection(1, 900, capture_start_ns=10, capture_end_ns=20) + for raw, pre in (([old], [fresh]), ([old, fresh], [])): + history = HistoryController() + history.raw.extend(raw) + history.preprocessed.extend(pre) + history.results.append(late) + history.delete(remove_last=True) + self.assertEqual((list(history.raw), list(history.preprocessed), list(history.results)), ([old], [], [late])) + + def test_results_only_and_empty_histories(self) -> None: + result = ResultCollection(1, 100, capture_start_ns=10, capture_end_ns=20) + history = HistoryController() + history.results.append(result) + history.replay_results.append(result) + history.delete(remove_last=True) + history.delete(remove_last=True) + self.assertFalse(history.results or history.replay_results) + self.assertEqual(history.removed, [{"collection_id": 1, "capture_start_ns": 10}]) + with mock.patch("python_app.gui.controllers.history_controller.time.monotonic_ns", return_value=50): + history.delete(remove_last=False) + self.assertEqual(history.clear_before_ns, 50) + self.assertEqual(history.removed, []) + self.assertFalse(history.record_result(result)) + + def test_results_sort_by_capture_and_reprocessed_identity_is_replaced(self) -> None: + old = ResultCollection(1, 300, capture_start_ns=10, capture_end_ns=20) + middle = ResultCollection(2, 500, capture_start_ns=30, capture_end_ns=40) + fresh = ResultCollection(1, 100, capture_start_ns=50, capture_end_ns=60) + history = deque([old, fresh], maxlen=3) + self.assertTrue(record_result_history(history, middle)) + self.assertEqual(list(history), [old, middle, fresh]) + reprocessed = ResultCollection(1, 900, capture_start_ns=10, capture_end_ns=20) + self.assertFalse(record_result_history(history, reprocessed)) + self.assertIs(history[0], reprocessed) + self.assertFalse(record_result_history(history, ResultCollection(3, 999, capture_start_ns=1))) + self.assertFalse(record_result_history(deque(maxlen=0), fresh)) + + +class RuntimeSignatureTest(unittest.TestCase): + def test_every_ring_endpoint_setting_requires_restart(self) -> None: + original = RunConfigModel() + signature = build_processor_run_signature(original) + for ring_field in fields(original.rings): + for field, value in (("name", "/other"), ("capacity", 7), ("slot_size_bytes", 8192)): + with self.subTest(ring=ring_field.name, field=field): + changed = original.clone() + setattr(getattr(changed.rings, ring_field.name), field, value) + self.assertNotEqual(build_processor_run_signature(changed), signature) + + def test_locator_and_live_reader_settings_require_restart(self) -> None: + original = RunConfigModel() + signature = build_processor_run_signature(original) + for field in fields(original.runtime.locator_server): + with self.subTest(field=field.name): + changed = original.clone() + value = getattr(changed.runtime.locator_server, field.name) + setattr(changed.runtime.locator_server, field.name, value + "x" if isinstance(value, str) else value + 1) + self.assertNotEqual(build_processor_run_signature(changed), signature) + for field, value in (("processing_live_config_path", "other.json"), ("idle_sleep_ms", 8)): + changed = original.clone() + setattr(changed.runtime, field, value) + self.assertNotEqual(build_processor_run_signature(changed), signature) + + def test_notch_and_preprocess_assets_reset_history_without_restarting_processor(self) -> None: + original = RunConfigModel() + for section, field, value in ( + ("notch", "enabled", True), ("notch", "bands_hz", [(1e8, 2e8)]), + ("notch", "taper_width_hz", 1e7), ("notch", "taper_type", "linear"), + ("s21.calibration", "set_name", "other"), ("s21.reference", "bundle_path", "other.bin"), + ): + with self.subTest(section=section, field=field): + changed = original.clone() + target = changed.preprocess + for part in section.split("."): + target = getattr(target, part) + setattr(target, field, value) + self.assertEqual(build_processor_run_signature(changed), build_processor_run_signature(original)) + self.assertNotEqual(build_run_history_signature(changed), build_run_history_signature(original)) + + +if __name__ == "__main__": + unittest.main() diff --git a/python_app/tests/test_sar3d_plot.py b/python_app/tests/test_sar3d_plot.py index 0da19bd..98fa5a9 100644 --- a/python_app/tests/test_sar3d_plot.py +++ b/python_app/tests/test_sar3d_plot.py @@ -3,22 +3,18 @@ from __future__ import annotations import os -from types import SimpleNamespace import unittest from unittest import mock import numpy as np os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") -from PyQt6.QtCore import Qt # noqa: E402 -from PyQt6.QtWidgets import QApplication, QStackedWidget # noqa: E402 +from PyQt6.QtCore import Qt, QSignalBlocker # noqa: E402 +from PyQt6.QtWidgets import QApplication, QCheckBox, QComboBox, QDoubleSpinBox # noqa: E402 -from python_app.gui.controllers.app_window_pipeline_mixin import AppWindowPipelineMixin # noqa: E402 -from python_app.gui.controllers.app_window_config.live_processing_mixin import AppWindowLiveProcessingMixin # noqa: E402 -from python_app.gui.controllers.app_window_plot_mixin import AppWindowPlotMixin # noqa: E402 -from python_app.gui.controllers.app_window_plot.sar3d_plot_mixin import SAR3D_VIEWS, sar3d_status # noqa: E402 -from python_app.gui.controllers.app_window_ui_mixin import AppWindowUiMixin # noqa: E402 -from python_app.models.dataset_model import ResultCollection, ResultPayload # noqa: E402 +from python_app.gui.controllers.renderers.sar3d import SAR3D_STATES, SAR3D_VIEWS, Sar3dRenderer # noqa: E402 +from python_app.models.dataset_model import ComboKey, ResultCollection, ResultPayload, TraceData # noqa: E402 +from python_app.tests.gui_test_support import widget def _status(code: int, message: str) -> ResultPayload: @@ -27,7 +23,8 @@ def _status(code: int, message: str) -> ResultPayload: def _collection(*payloads: ResultPayload) -> ResultCollection: - return ResultCollection(collection_id=5, monotonic_ns=1500, collection_payloads=list(payloads)) + return ResultCollection(collection_id=5, monotonic_ns=1500, collection_payloads=list(payloads), + capture_start_ns=1200, capture_end_ns=1400) def _projections() -> ResultCollection: @@ -43,114 +40,131 @@ def _projections() -> ResultCollection: table=np.array([[2, 0.5, -0.2, 2]], dtype=np.float32))) -class _View(AppWindowPlotMixin, AppWindowUiMixin, AppWindowPipelineMixin, AppWindowLiveProcessingMixin): - def __init__(self) -> None: - self._processing_mode = SimpleNamespace(currentText=lambda: "sar3d") - self._plot_stack = QStackedWidget() - self._build_sar3d_plot_page() - - class Sar3dPlotTest(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.app = QApplication.instance() or QApplication([]) def setUp(self) -> None: - self.view = _View() - self.addCleanup(self.view._plot_stack.close) + self.renderer = Sar3dRenderer() + self.addCleanup(self.renderer.widget.close) def test_all_four_projections_have_visible_images_and_correct_peak_axes(self) -> None: - self.assertTrue(self.view._draw_results(_projections())) - self.assertEqual(self.view._sar3d_status_label.text(), "SAR3D: ready — CUDA") + self.assertTrue(self.renderer.draw(_projections())) + self.assertEqual(self.renderer.status_label.text(), "SAR3D: ready — CUDA") positions = {"sar3d_front": (0.5, -0.2), "sar3d_side": (2, -0.2), "sar3d_top": (0.5, 2), "sar3d_oblique": (0.5, -0.2)} for name, position in positions.items(): - image = self.view._sar3d_images[name] + image = self.renderer.images[name] self.assertTrue(image.isVisible(), name) self.assertGreater(image.boundingRect().width(), 0) self.assertGreater(image.boundingRect().height(), 0) - xx, yy = self.view._sar3d_peaks[name].getData() + xx, yy = self.renderer.peaks[name].getData() np.testing.assert_allclose([xx[0], yy[0]], position) - self.assertTrue(all(label.isVisible() for label in self.view._sar3d_axis_labels.values())) + self.assertTrue(all(label.isVisible() for label in self.renderer.axis_labels.values())) + self.renderer.clear() + self.assertEqual(self.renderer.status_label.text(), "SAR3D: waiting for measurements") def test_unavailable_warning_clears_previous_images_and_labels_and_shows_reason(self) -> None: - self.view._draw_results(_projections()) + self.renderer.draw(_projections()) message = "CUDA unavailable: драйвер " - self.assertFalse(self.view._draw_results(_collection(_status(3, message)))) - self.assertEqual(self.view._sar3d_status_label.text(), f"SAR3D: warning — {message}") - self.assertEqual(self.view._sar3d_status_label.textFormat(), Qt.TextFormat.PlainText) - self.assertTrue(all(not image.isVisible() for image in self.view._sar3d_images.values())) - self.assertTrue(all(not label.isVisible() for label in self.view._sar3d_axis_labels.values())) + self.assertFalse(self.renderer.draw(_collection(_status(3, message)))) + self.assertEqual(self.renderer.status_label.text(), f"SAR3D: warning — {message}") + self.assertEqual(self.renderer.status_label.textFormat(), Qt.TextFormat.PlainText) + self.assertTrue(all(not image.isVisible() for image in self.renderer.images.values())) + self.assertTrue(all(not label.isVisible() for label in self.renderer.axis_labels.values())) def test_no_signal_and_missing_projections_explain_empty_panels(self) -> None: - self.assertFalse(self.view._draw_results(_collection(_status(1, "No signal in selected range")))) - self.assertIn("no signal", self.view._sar3d_status_label.text()) - self.assertFalse(self.view._draw_results(_collection())) - self.assertEqual(self.view._sar3d_status_label.text(), "SAR3D: waiting for projection data") + self.assertFalse(self.renderer.draw(_collection(_status(1, "No signal in selected range")))) + self.assertIn("no signal", self.renderer.status_label.text()) + self.assertFalse(self.renderer.draw(_collection())) + self.assertEqual(self.renderer.status_label.text(), "SAR3D: waiting for projection data") - def test_ready_cuda_status_and_clear_restores_waiting(self) -> None: - self.assertTrue(self.view._draw_results(_projections())) - self.assertEqual(self.view._sar3d_status_label.text(), "SAR3D: ready — CUDA") - self.view._clear_sar3d_plot() - self.assertEqual(self.view._sar3d_status_label.text(), "SAR3D: waiting for measurements") - def test_malformed_status_is_a_visible_error_instead_of_a_qt_exception(self) -> None: - for table in (np.empty((0, 0)), np.array([[2, np.nan]]), np.array([[2, 256]]), - np.array([[2, 0.5]]), np.array([[99]]), np.array([2])): - with self.subTest(table=table): - payload = _status(2, "failure") - payload.table = table - self.assertEqual(sar3d_status(_collection(payload)), (2, "Invalid SAR3D status payload")) - self.assertFalse(self.view._draw_results(_collection(payload))) + +class Sar3dIntegrationTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.app = QApplication.instance() or QApplication([]) + + def setUp(self) -> None: + from python_app.tests.gui_test_support import isolated_window + + self.view = isolated_window(self.addCleanup) + self.view._pipeline.timer.stop() + self.mode = self.view._view.processing.mode + with QSignalBlocker(self.mode): + self.mode.setCurrentText("sar3d") + self.view._plots.set_mode("sar3d") + self.view._plots.required_revision = None + self.renderer = self.view._plots.sar3d + + def test_capture_preview_keeps_sar_page_and_uses_saved_trace_settings(self) -> None: + self.view._plots.set_mode("sar3d") + fields = self.view._view.processing + phase = widget(fields, "_show_phase_checkbox", QCheckBox) + unwrap = widget(fields, "_unwrap_phase_checkbox", QCheckBox) + fixed = widget(fields, "_pass_through_fixed_y_enabled", QCheckBox) + lower = widget(fields, "_pass_through_y_min_db", QDoubleSpinBox) + upper = widget(fields, "_pass_through_y_max_db", QDoubleSpinBox) + revision = self.view._live.revision + phase.setChecked(False) + unwrap.setChecked(True) + fixed.setChecked(True) + lower.setValue(-70) + upper.setValue(-5) + self.assertEqual(self.view._live.revision, revision) + trace = TraceData(ComboKey(0, 0), np.array([1., 2.]), np.array([0.1 + 0j, 0.2 + 0j])) + self.view._plots.draw_trace(trace, "Captured reference") + self.assertIs(self.view._plots.stack.currentWidget(), self.renderer.widget) + self.assertFalse(self.view._plots.trace.phase.curves) + self.assertTrue(self.view._config.processing_settings().pass_through.unwrap_phase) + np.testing.assert_allclose(self.view._plots.trace.magnitude.view.viewRange()[1], [-70, -5]) def test_single_capture_finishes_reported_failure_without_losing_its_result(self) -> None: for code in (1, 2, 3): with self.subTest(code=code): result = _collection(_status(code, "processor explanation")) - self.view._result_history = [result] - self.view._single_capture_active = True - self.view._single_capture_start_ns = 1000 - self.view._single_capture_seen_raw = True - self.view._single_capture_target_collection_id = 5 - self.view._log = mock.Mock() - self.view._log_error = mock.Mock() - self.view._log_warning = mock.Mock() - self.view._status_label = mock.Mock() - self.view._stop_run = mock.Mock(side_effect=lambda: setattr(self.view, "_single_capture_active", False)) - self.assertTrue(self.view._finish_single_capture_if_ready()) - self.assertFalse(self.view._single_capture_active) - self.assertEqual(self.view._result_history, [result]) - self.assertIn("processor explanation", self.view._sar3d_status_label.text()) - state = {1: "no signal in selected range", 2: "error", 3: "warning"}[code] - self.view._status_label.setText.assert_called_once_with(f"Status: {state}") - log = self.view._log_error if code == 2 else self.view._log_warning - log.assert_called_once_with( - f"Single capture: SAR3D {state}: processor explanation") + self.view._history.results.clear() + self.view._history.record_result(result) + self.view._pipeline.single_capture_active = True + self.view._pipeline._single_capture_start_ns = 1000 + self.view._pipeline._single_capture_target_key = (5, 1200) + with self.assertLogs("python_app.gui.controllers.pipeline_controller", level="WARNING") as logged: + self.assertTrue(self.view._pipeline._finish_single_capture_if_ready()) + self.assertFalse(self.view._pipeline.single_capture_active) + self.assertEqual(list(self.view._history.results), [result]) + self.assertIn("processor explanation", self.renderer.status_label.text()) + state = SAR3D_STATES[code] + self.assertEqual(self.view._view.status.text(), f"Status: {state}") + self.assertEqual(logged.records[-1].levelname, "ERROR" if code == 2 else "WARNING") + self.assertEqual(logged.records[-1].getMessage(), + f"Single capture: SAR3D {state}: processor explanation") def test_live_edit_retains_failure_reason_then_recovers_to_ready_images(self) -> None: - self.view._result_history = [] - self.view._write_live_processing_config = mock.Mock() - self.view._show_exception = mock.Mock() + self.view._history.results.clear() + self.view._live.publish = mock.Mock() + self.view._log.show_exception = mock.Mock() for result, expected in ((_collection(_status(2, "CUDA kernel failure")), "error"), (_collection(_status(3, "Driver initialization unavailable")), "warning"), (_collection(_status(2, "fewer than two frequency points")), "error"), (_collection(_status(1, "No signal in selected depth range")), "no signal"), (_projections(), "ready — CUDA")): - self.view._drain_results_until_quiet = mock.Mock(side_effect=[None, result]) - self.view._on_processing_live_settings_changed() - self.assertIn(expected, self.view._sar3d_status_label.text()) - self.view._show_exception.assert_not_called() - self.assertTrue(all(image.isVisible() for image in self.view._sar3d_images.values())) + self.view._pipeline.drain_results = mock.Mock(return_value=result) + self.view._live.changed() + self.assertIn(expected, self.renderer.status_label.text()) + self.view._log.show_exception.assert_not_called() + self.assertTrue(all(image.isVisible() for image in self.renderer.images.values())) def test_history_redraw_preserves_error_but_stale_revision_clears_old_images(self) -> None: - self.view._result_history = [_collection(_status(2, "processor error"))] - self.view._redraw_after_history_deletion() - self.assertEqual(self.view._sar3d_status_label.text(), "SAR3D: error — processor error") - self.view._draw_results(_projections()) - self.view._required_processing_revision = 10 - self.view._redraw_after_history_deletion() - self.assertEqual(self.view._sar3d_status_label.text(), "SAR3D: waiting for measurements") - self.assertTrue(all(not image.isVisible() for image in self.view._sar3d_images.values())) + self.view._history.record_result(_collection(_status(2, "processor error"))) + self.view._plots.redraw() + self.assertEqual(self.renderer.status_label.text(), "SAR3D: error — processor error") + self.view._plots.draw(_projections()) + self.view._plots.required_revision = 10 + self.view._plots.redraw() + self.assertEqual(self.renderer.status_label.text(), "SAR3D: waiting for measurements") + self.assertTrue(all(not image.isVisible() for image in self.renderer.images.values())) if __name__ == "__main__": diff --git a/python_app/tests/test_shm_ipc.py b/python_app/tests/test_shm_ipc.py index d44a36b..d0635d3 100644 --- a/python_app/tests/test_shm_ipc.py +++ b/python_app/tests/test_shm_ipc.py @@ -6,7 +6,6 @@ Pins the agreed contract: struct.error / UnicodeDecodeError); * the ring is latest-wins: on overflow the oldest slot is overwritten and a slow reader keeps the freshest frames; - * peek_latest returns the newest frame without consuming it; * opening a missing/incompatible ring fails fast. """ @@ -18,15 +17,13 @@ import time import unittest from contextlib import suppress from pathlib import Path +from typing import cast from unittest import mock import numpy as np from python_app.models.dataset_model import ( ComboKey, - ResultBlock, - ResultCollection, - ResultPayload, SweepCollection, TraceData, ) @@ -39,18 +36,16 @@ from python_app.orchestration.shm.decoder import ( ) from python_app.orchestration.shm.ring_reader import ShmRingReader from python_app.orchestration.shm.ring_writer import ShmRingWriter -from python_app.storage.npz.serialize import serialize_result_collection, serialize_trace_collection +from python_app.storage.npz.serialize import serialize_trace_collection def _trace(in_pos: int, out_pos: int, n: int, *, capture_ns: tuple[int, int] = (0, 0)) -> TraceData: """Build a trace with float32-exact data so round-trips compare exactly.""" freq = np.arange(n, dtype=np.float32) + 1.0 - s11 = (np.arange(n, dtype=np.float32) + 0.5j * np.arange(n, dtype=np.float32)).astype(np.complex64) s21 = (-np.arange(n, dtype=np.float32) + 2.0j * np.arange(n, dtype=np.float32)).astype(np.complex64) return TraceData( combo=ComboKey(input=in_pos, output=out_pos), frequency_hz=freq, - s11=s11, s21=s21, capture_start_ns=capture_ns[0], capture_end_ns=capture_ns[1], @@ -74,7 +69,6 @@ class TraceCollectionRoundTripTest(unittest.TestCase): for original, got in zip(collection.traces, decoded.traces): self.assertEqual((got.combo.input, got.combo.output), (original.combo.input, original.combo.output)) self.assertTrue(np.array_equal(got.frequency_hz, original.frequency_hz)) - self.assertTrue(np.array_equal(got.s11, original.s11)) self.assertTrue(np.array_equal(got.s21, original.s21)) self.assertEqual( (got.capture_start_ns, got.capture_end_ns), @@ -87,28 +81,67 @@ class TraceCollectionRoundTripTest(unittest.TestCase): def test_preprocessed_round_trips(self) -> None: self._assert_round_trips(PREPROC_MAGIC) + def test_s21_only_wire_uses_literal_raw3_prp3_and_twelve_bytes_per_point(self) -> None: + collection = SweepCollection(7, 123, [_trace(1, 2, 3)], capture_start_ns=100, capture_end_ns=120) + for magic, label in ((RAW_MAGIC, b"RAW3"), (PREPROC_MAGIC, b"PRP3")): + with self.subTest(magic=label): + wire = serialize_trace_collection(collection, magic) + expected = label + struct.pack(" None: + collection = SweepCollection(1, 2, [_trace(0, 0, 2)]) + for magic, old in ((RAW_MAGIC, b"RAW2"), (PREPROC_MAGIC, b"PPR2"), (PREPROC_MAGIC, b"PRP2")): + with self.subTest(old=old), self.assertRaisesRegex(ValueError, "magic"): + decode_trace_collection(old + serialize_trace_collection(collection, magic)[4:], magic) + + def test_complex_components_are_decoded_without_arithmetic_changes(self) -> None: + words = [0x3f800000, 0x7fc01234, 0x80000000, 0xbf800000] # (1, NaN), (-0, -1) + samples = struct.pack("<4I", *words) + wire = b"RAW3" + struct.pack(" None: + for magic in (RAW_MAGIC, PREPROC_MAGIC): + for point_count in (3, 0xffffffff): + with self.subTest(magic=magic, points=point_count), self.assertRaisesRegex(ValueError, "truncated"): + wire = struct.pack(" None: collection = SweepCollection(collection_id=1, monotonic_ns=2, traces=[]) decoded = decode_trace_collection(serialize_trace_collection(collection, RAW_MAGIC), RAW_MAGIC) self.assertEqual(decoded.traces, []) - def test_payload_without_per_trace_window_table_still_decodes(self) -> None: - # A producer built before per-trace timing stops after the collection - # window; its traces must still decode, with the timestamps left at zero. - collection = SweepCollection( - collection_id=4, - monotonic_ns=5, - traces=[_trace(1, 0, 3, capture_ns=(7, 9))], - capture_start_ns=6, - capture_end_ns=10, - ) - full = serialize_trace_collection(collection, RAW_MAGIC) - legacy = full[: -(4 + 16 * len(collection.traces))] + def test_live_raw_requires_capture_window_but_calibration_bundles_do_not(self) -> None: + collection = SweepCollection(1, 100, [_trace(0, 0, 2)]) + for start, end in ((0, 0), (0, 1), (10, 9)): + collection.capture_start_ns, collection.capture_end_ns = start, end + with self.subTest(window=(start, end)), self.assertRaisesRegex(ValueError, "capture window"): + serialize_trace_collection(collection, RAW_MAGIC, require_capture_window=True) + collection.capture_start_ns = collection.capture_end_ns = 10 + serialize_trace_collection(collection, RAW_MAGIC, require_capture_window=True) + collection.capture_start_ns = collection.capture_end_ns = 0 + decoded = decode_trace_collection(serialize_trace_collection(collection, RAW_MAGIC), RAW_MAGIC) + self.assertEqual((decoded.capture_start_ns, decoded.capture_end_ns), (0, 0)) - decoded = decode_trace_collection(legacy, RAW_MAGIC) - self.assertEqual((decoded.capture_start_ns, decoded.capture_end_ns), (6, 10)) - self.assertEqual(len(decoded.traces), 1) - self.assertEqual((decoded.traces[0].capture_start_ns, decoded.traces[0].capture_end_ns), (0, 0)) + def test_timing_trailer_is_required(self) -> None: + collection = SweepCollection(4, 5, [_trace(1, 0, 3, capture_ns=(7, 9))], + capture_start_ns=6, capture_end_ns=10) + for magic in (RAW_MAGIC, PREPROC_MAGIC): + payload = serialize_trace_collection(collection, magic) + for removed_bytes in (20, 36): + with self.subTest(magic=magic, removed_bytes=removed_bytes), self.assertRaisesRegex(ValueError, "truncated"): + decode_trace_collection(payload[:-removed_bytes], magic) def test_per_trace_window_count_mismatch_is_rejected(self) -> None: collection = SweepCollection( @@ -121,66 +154,54 @@ class TraceCollectionRoundTripTest(unittest.TestCase): decode_trace_collection(corrupt, RAW_MAGIC) -class ResultCollectionRoundTripTest(unittest.TestCase): +class ResultCollectionDecodeTest(unittest.TestCase): def test_empty_tables_and_images_retain_their_declared_dimensions(self) -> None: - for shape in ((0, 3), (2, 0), (0, 0)): + for rows, columns in ((0, 3), (2, 0), (0, 0)): for kind in (3, 4): - with self.subTest(shape=shape, kind=kind): - payload = ResultPayload('empty', kind, table=np.empty(shape, dtype=np.float32), - image=np.empty(shape, dtype=np.float32), - image_x_axis=np.zeros(shape[1], dtype=np.float32), - image_y_axis=np.zeros(shape[0], dtype=np.float32)) - if kind == 4 and shape == (2, 0): + with self.subTest(shape=(rows, columns), kind=kind): + wire = struct.pack(" None: - payload = ResultPayload('bad', 1, frequency_hz=np.ones((1, 2)), trace=np.ones((1, 2))) - with self.assertRaises(ValueError): - serialize_result_collection(ResultCollection(1, 2, collection_payloads=[payload])) - - def test_all_payload_kinds_round_trip(self) -> None: - image = np.arange(6, dtype=np.float32).reshape((2, 3)) - table = np.array([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], dtype=np.float32) - collection = ResultCollection( - collection_id=9, - monotonic_ns=42, - processing_duration_ns=1000, - collection_payloads=[ - ResultPayload( - processing_name="gpr_accumulator", kind=3, - image_x_axis=np.array([0.0, 1.0, 2.0], dtype=np.float32), - image_y_axis=np.array([0.0, 1.0], dtype=np.float32), - image=image, - ), - ResultPayload(processing_name="gpr_points", kind=4, table=table), - ], - blocks=[ - ResultBlock(combo=ComboKey(input=1, output=0), payloads=[ - ResultPayload( - processing_name="bscan", kind=1, - frequency_hz=np.array([1.0, 2.0], dtype=np.float32), - trace=np.array([1 + 1j, 2 - 2j], dtype=np.complex64), - ), - ResultPayload(processing_name="snr", kind=2, scalar_value=2.5), - ]), - ], - ) - decoded = decode_result_collection(serialize_result_collection(collection)) + def test_all_payload_kinds_decode_from_wire_format(self) -> None: + # A fixed frame independent of Python encoding preserves the C++ wire contract. + wire = struct.pack(" None: - corrupt = struct.pack(" None: - corrupt = struct.pack(" ShmRingReader: + def _reader(self, name: str, **kw: float) -> ShmRingReader: reader = ShmRingReader(name, **kw) self.addCleanup(self._safe_close, reader) return reader @staticmethod - def _safe_close(obj: object) -> None: + def _safe_close(obj: ShmRingReader | ShmRingWriter) -> None: with suppress(Exception): - obj.close() # type: ignore[attr-defined] + obj.close() @staticmethod def _cleanup(name: str, writer: ShmRingWriter) -> None: @@ -265,6 +286,27 @@ class RingRoundTripTest(_RingTestCase): class RingOverflowTest(_RingTestCase): + def test_overwrite_during_read_discards_torn_payload(self) -> None: + from python_app.orchestration.shm import atomics + + name = self._ring_name() + writer = self._writer(name, capacity=1, slot_size=8) + reader = self._reader(name) + writer.push(b"old") + fence = atomics.fence + overwritten = False + + def overwrite_after_copy(): + nonlocal overwritten + fence() + if not overwritten: + overwritten = True + writer.push(b"new") + + with mock.patch.object(atomics, "fence", side_effect=overwrite_after_copy): + self.assertIsNone(reader.pop_payload()) + self.assertEqual(reader.pop_payload(), b"new") + def test_reader_cannot_rewind_cursor_after_producer_laps_during_commit(self) -> None: from python_app.orchestration.shm import atomics @@ -298,46 +340,34 @@ class RingOverflowTest(_RingTestCase): self.assertEqual(survivors, [b"f3", b"f4", b"f5", b"f6"]) -class PeekLatestTest(_RingTestCase): - def test_peek_rejects_a_slot_while_it_is_being_overwritten(self) -> None: - name = self._ring_name() - writer = self._writer(name, capacity=1, slot_size=8) - reader = self._reader(name) - writer.push(b"aaaaaaaa") - observations = [] - write_size = writer._write_u32 - - def overwrite_in_progress(offset: int, value: int) -> None: - write_size(offset, value) - if offset == 64: - writer._mmap[80:84] = b"bbbb" - observations.append(reader.peek_latest_payload()) - - with mock.patch.object(writer, "_write_u32", side_effect=overwrite_in_progress): - writer.push(b"bbbbbbbb") - - self.assertEqual(observations, [None]) - self.assertEqual(reader.peek_latest_payload(), b"bbbbbbbb") - - def test_peek_is_latest_and_non_consuming(self) -> None: - name = self._ring_name() - writer = self._writer(name, capacity=8, slot_size=64) - reader = self._reader(name) - self.assertIsNone(reader.peek_latest_payload()) # nothing published yet - for i in range(3): - writer.push(f"f{i}".encode()) - self.assertEqual(reader.peek_latest_payload(), b"f2") # newest - self.assertEqual(reader.peek_latest_payload(), b"f2") # stable, not consumed - self.assertEqual(reader.pop_payload(), b"f0") # consumer cursor untouched - writer.push(b"f3") - self.assertEqual(reader.peek_latest_payload(), b"f3") # follows the newest - - class RingOpenTest(_RingTestCase): + def test_same_size_segment_is_reused_only_with_matching_header(self) -> None: + for corrupt_header in (False, True): + with self.subTest(corrupt_header=corrupt_header): + name = self._ring_name(str(corrupt_header)) + writer = self._writer(name, capacity=1, slot_size=8) + reader = self._reader(name) + writer.push(b"old") + if corrupt_header: + writer._write_u32(8, 99) + path = Path("/dev/shm") / name[1:] + old_inode = path.stat().st_ino + replacement = ShmRingWriter(name, 1, 8) + self.addCleanup(self._safe_close, replacement) + new_reader = self._reader(name) + self.assertEqual(path.stat().st_ino == old_inode, not corrupt_header) + self.assertEqual(new_reader.pop_payload(), None if corrupt_header else b"old") + if corrupt_header: + self.assertEqual(reader.pop_payload(), b"old") + replacement.push(b"new") + self.assertEqual(new_reader.pop_payload(), b"new") + replacement.close() + replacement.close() + def test_invalid_geometry_and_names_fail_before_opening_files(self) -> None: for capacity, slot in ((True, 8), (1.5, 8), (1, 9), (2 ** 32, 8)): with self.subTest(capacity=capacity, slot=slot), self.assertRaises(ValueError): - ShmRingWriter(self._ring_name(), capacity, slot) + ShmRingWriter(self._ring_name(), cast(int, capacity), slot) for name in ("/", "/../tmp/ring", "/nested/ring", "/bad\x00ring"): with self.subTest(name=name): with self.assertRaises(ValueError): diff --git a/python_app/tests/test_single_capture.py b/python_app/tests/test_single_capture.py index 137f6b8..bcb1009 100644 --- a/python_app/tests/test_single_capture.py +++ b/python_app/tests/test_single_capture.py @@ -1,7 +1,7 @@ """Single-capture completion logic. The single capture starts a one-shot pipeline and must finish on exactly the result -that belongs to its sweep. Because collection ids restart per orchestrator run, a stale +that belongs to its sweep. Because collection ids restart per acquisition run, a stale result from a previous run can carry the SAME id; the start timestamp disambiguates them. These tests pin that boundary (fresh vs stale same-id) and the readiness state machine — the path that previously hung on the simulator when start_ns was recorded too late. @@ -11,11 +11,12 @@ from __future__ import annotations import os import unittest +from unittest import mock import numpy as np os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") -from PyQt6.QtWidgets import QApplication # noqa: E402 +from PyQt6.QtWidgets import QApplication, QComboBox # noqa: E402 from python_app.gui.app_window import AppWindow # noqa: E402 from python_app.tests.gui_test_support import isolated_window @@ -32,71 +33,71 @@ _window: AppWindow | None = None def setUpModule() -> None: global _app, _window - _app = QApplication.instance() or QApplication([]) + instance = QApplication.instance() + _app = instance if isinstance(instance, QApplication) else QApplication([]) _window = isolated_window(unittest.addModuleCleanup) -def tearDownModule() -> None: - if _window is not None: - _window.close() - - -def _result(collection_id: int, monotonic_ns: int) -> ResultCollection: +def _result(collection_id: int, capture_start_ns: int) -> ResultCollection: payload = ResultPayload( processing_name="pass_through", kind=1, frequency_hz=np.array([1.0, 2.0], dtype=np.float32), trace=np.array([1 + 1j, 2 - 2j], dtype=np.complex64), ) block = ResultBlock(combo=ComboKey(input=0, output=0), payloads=[payload]) - return ResultCollection(collection_id=collection_id, monotonic_ns=monotonic_ns, blocks=[block]) + return ResultCollection(collection_id=collection_id, monotonic_ns=capture_start_ns + 100, blocks=[block], + capture_start_ns=capture_start_ns, capture_end_ns=capture_start_ns + 1) class SingleCaptureBoundaryTest(unittest.TestCase): def setUp(self) -> None: + assert _window is not None self.w = _window - self.w._processing_mode.setCurrentText("pass_through") - self.w._single_capture_active = True - self.w._single_capture_start_ns = 1000 - self.w._single_capture_seen_raw = True - self.w._single_capture_target_collection_id = 5 - self.w._result_history.clear() + self.pipeline = self.w._pipeline + self.pipeline.timer.stop() + self.w._view.processing.mode.setCurrentText("pass_through") + self.pipeline.single_capture_active = True + self.pipeline._single_capture_start_ns = 1000 + self.pipeline._single_capture_target_key = (5, 1500) + self.w._history.preprocessed.clear() + self.w._history.results.clear() def test_picks_fresh_result_not_stale_same_id(self) -> None: - self.w._result_history.append(_result(5, 500)) # stale: same id, before start_ns - self.w._result_history.append(_result(5, 1500)) # fresh: at/after start_ns - target = self.w._find_single_capture_target_result() - self.assertIsNotNone(target) - self.assertEqual(target.monotonic_ns, 1500) + stale, fresh = _result(5, 500), _result(5, 1500) + self.w._history.results.extend((stale, fresh)) + with mock.patch.object(self.w._plots, "draw", wraps=self.w._plots.draw) as draw: + self.assertTrue(self.pipeline._finish_single_capture_if_ready()) + draw.assert_called_once_with(fresh) def test_stale_same_id_result_is_not_matched(self) -> None: - self.w._result_history.append(_result(5, 500)) # only a stale same-id result - self.assertIsNone(self.w._find_single_capture_target_result()) + self.w._history.results.append(_result(5, 500)) + self.assertFalse(self.pipeline._finish_single_capture_if_ready()) def test_wrong_collection_id_is_not_matched(self) -> None: - self.w._result_history.append(_result(9, 1500)) # fresh but a different sweep - self.assertIsNone(self.w._find_single_capture_target_result()) + self.w._history.results.append(_result(9, 1500)) + self.assertFalse(self.pipeline._finish_single_capture_if_ready()) def test_not_ready_until_a_fresh_raw_is_seen(self) -> None: # The readiness guard that the timing fix makes reachable: with no raw recognized # as fresh yet, the capture must not complete (it must keep waiting, not hang). - self.w._single_capture_seen_raw = False - self.w._result_history.append(_result(5, 1500)) - self.assertFalse(self.w._finish_single_capture_if_ready()) - self.assertTrue(self.w._single_capture_active) + self.pipeline._single_capture_target_key = None + self.w._history.results.append(_result(5, 1500)) + self.assertFalse(self.pipeline._finish_single_capture_if_ready()) + self.assertTrue(self.pipeline.single_capture_active) def test_completes_and_stops_on_the_fresh_result(self) -> None: - self.w._result_history.append(_result(5, 1500)) - self.assertTrue(self.w._finish_single_capture_if_ready()) - self.assertFalse(self.w._single_capture_active) # _stop_run cleared the flag + self.w._history.results.append(_result(5, 1500)) + self.assertTrue(self.pipeline._finish_single_capture_if_ready()) + self.assertFalse(self.pipeline.single_capture_active) def test_empty_processed_trace_completes_capture_instead_of_waiting_for_timeout(self) -> None: result = _result(5, 1500) payload = result.blocks[0].payloads[0] payload.frequency_hz = np.empty(0, dtype=np.float32) payload.trace = np.empty(0, dtype=np.complex64) - self.w._result_history.append(result) - self.assertTrue(self.w._finish_single_capture_if_ready()) - self.assertFalse(self.w._single_capture_active) + self.w._history.results.append(result) + self.assertTrue(self.pipeline._finish_single_capture_if_ready()) + self.assertFalse(self.pipeline.single_capture_active) if __name__ == "__main__": diff --git a/python_app/tests/test_socket_speed_controls.py b/python_app/tests/test_socket_speed_controls.py index 2355d6f..b387850 100644 --- a/python_app/tests/test_socket_speed_controls.py +++ b/python_app/tests/test_socket_speed_controls.py @@ -20,46 +20,23 @@ from python_app.tests.gui_test_support import isolated_window _app: QApplication | None = None _window: AppWindow | None = None -# mode -> (checkbox, speed field) attributes of its page -_PAGES = { - "gpr": ("_gpr_ignore_socket_speed_enabled", "_gpr_speed_m_s"), - "legacy_gpr": ("_legacy_gpr_ignore_socket_speed_enabled", "_legacy_gpr_speed_m_s"), - "sar3d": ("_sar3d_ignore_socket_speed_enabled", "_sar3d_speed_m_s"), -} - - def setUpModule() -> None: global _app, _window - _app = QApplication.instance() or QApplication([]) + instance = QApplication.instance() + _app = instance if isinstance(instance, QApplication) else QApplication([]) _window = isolated_window(unittest.addModuleCleanup) -def tearDownModule() -> None: - if _window is not None: - _window.close() - - class SocketSpeedControlsTest(unittest.TestCase): - def test_speed_field_is_editable_only_while_the_socket_is_ignored(self) -> None: + def test_socket_authority_controls_editability_and_published_speed(self) -> None: assert _window is not None - for mode, (checkbox_name, speed_name) in _PAGES.items(): - checkbox, speed = getattr(_window, checkbox_name), getattr(_window, speed_name) + section = _window._view.processing + for mode, (checkbox, speed) in zip(("gpr", "legacy_gpr", "sar3d"), section.speed_controls, strict=True): + section.mode.setCurrentText(mode) + speed.setValue(1.25) for ignore in (True, False): checkbox.setChecked(ignore) - self.assertEqual(speed.isEnabled(), ignore, msg=f"{mode}, ignore={ignore}") - - def test_live_config_takes_the_active_page(self) -> None: - assert _window is not None - for mode, (checkbox_name, speed_name) in _PAGES.items(): - _window._set_combo_current_text(_window._processing_mode, mode) - getattr(_window, checkbox_name).setChecked(True) - getattr(_window, speed_name).setValue(1.25) - config = _window._live_processing_config() - self.assertTrue(config.ignore_socket_speed, msg=mode) - self.assertAlmostEqual(config.gpr_speed_m_s, 1.25, msg=mode) - getattr(_window, checkbox_name).setChecked(False) - self.assertFalse(_window._live_processing_config().ignore_socket_speed, msg=mode) - - -if __name__ == "__main__": - unittest.main() + self.assertEqual(speed.isEnabled(), ignore, msg=mode) + config = _window._live.build() + self.assertEqual(config.ignore_socket_speed, ignore, msg=mode) + self.assertAlmostEqual(config.gpr_speed_m_s, 1.25, msg=mode) diff --git a/python_app/tests/test_storage_webui.py b/python_app/tests/test_storage_webui.py index 6472692..97093d2 100644 --- a/python_app/tests/test_storage_webui.py +++ b/python_app/tests/test_storage_webui.py @@ -5,40 +5,43 @@ Pins the agreed semantics: * vna_history export fails loud (ValueError) when no matching traces / bad stage; * the web bridge rejects unknown live-settings fields (HTTP 400), serves immutable snapshot copies, and forwards controls as Qt signals; - * the web fan-out is latest-wins — a full client queue drops its oldest frame. + * the web fan-out is latest-wins — each client retains the latest message per type. """ from __future__ import annotations +from contextlib import ExitStack + import asyncio import os import tempfile +import threading +import time import unittest from pathlib import Path -from types import SimpleNamespace from unittest import mock import numpy as np os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") # before any Qt import from PyQt6.QtWidgets import QApplication # noqa: E402 -from fastapi import HTTPException, WebSocketDisconnect # noqa: E402 +from PyQt6.QtCore import QTimer # noqa: E402 +from fastapi import FastAPI, HTTPException, Request # noqa: E402 -from python_app.gui.controllers.app_window_web_mixin import ( # noqa: E402 - AppWindowWebController, - AppWindowWebMixin, +from python_app.gui.controllers.web_controller import ( # noqa: E402 _WebActionCall, + web_ui_port, ) +from python_app.tests.gui_test_support import isolated_window # noqa: E402 from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData # noqa: E402 from python_app.storage.npz.paths import radar_config_filename_prefix # noqa: E402 from python_app.storage.npz.store import NpzStore # noqa: E402 from python_app.storage.npz.vna_history_json import ( # noqa: E402 _complex_to_points, - _normalize_channel, build_vna_history_payload, ) -from python_app.webui.controller import WebActionError # noqa: E402 -from python_app.webui.routes import post_live_settings, ws # noqa: E402 +from python_app.webui.controller import WebActionError, WebController # noqa: E402 +from python_app.webui.routes import post_live_settings # noqa: E402 from python_app.webui.streaming import RingBroadcaster # noqa: E402 @@ -51,45 +54,10 @@ def _trace(in_pos: int, out_pos: int) -> TraceData: return TraceData( combo=ComboKey(input=in_pos, output=out_pos), frequency_hz=np.array([1.0, 2.0], dtype=np.float32), - s11=np.array([1 + 1j, 2 + 2j], dtype=np.complex64), s21=np.array([3 + 3j, 4 - 4j], dtype=np.complex64), ) -# --------------------------------------------------------------------------- # -# NPZ set persistence -# --------------------------------------------------------------------------- # -class NpzStoreRoundTripTest(unittest.TestCase): - def setUp(self) -> None: - self._dir = tempfile.TemporaryDirectory() - self.addCleanup(self._dir.cleanup) - self.store = NpzStore(Path(self._dir.name)) - - def test_save_then_load_round_trips_traces_and_metadata(self) -> None: - col = SweepCollection(collection_id=7, monotonic_ns=11, traces=[_trace(0, 0), _trace(1, 0)], - capture_start_ns=100, capture_end_ns=200) - self.store.save_set("calibration", "radar1", "set1", col) - loaded = self.store.load_set("calibration", "radar1", "set1") - - self.assertEqual(loaded.collection_id, 7) - self.assertEqual((loaded.capture_start_ns, loaded.capture_end_ns), (100, 200)) - self.assertEqual(len(loaded.traces), 2) - by_combo = {(t.combo.input, t.combo.output): t for t in loaded.traces} - self.assertTrue(np.array_equal(by_combo[(0, 0)].s21, _trace(0, 0).s21)) - self.assertTrue(np.array_equal(by_combo[(1, 0)].frequency_hz, _trace(1, 0).frequency_hz)) - - def test_set_appears_in_listing_and_leaves_no_tmp(self) -> None: - self.store.save_set("calibration", "radar1", "set1", SweepCollection(collection_id=1, monotonic_ns=1, - traces=[_trace(0, 0)])) - self.assertIn("set1", self.store.list_sets("calibration", "radar1")) - leftover = list(Path(self._dir.name).rglob("*.tmp")) - self.assertEqual(leftover, []) - - def test_load_missing_set_raises(self) -> None: - with self.assertRaises(Exception): - self.store.load_set("calibration", "radar1", "absent") - - # --------------------------------------------------------------------------- # # Radar-config filename prefix # --------------------------------------------------------------------------- # @@ -135,12 +103,6 @@ class VnaHistoryTest(unittest.TestCase): with self.assertRaisesRegex(ValueError, "primary_stage"): build_vna_history_payload([], self._sweeps((0, 0)), [], input_index=0, output_index=0, primary_stage="x") - def test_normalize_channel(self) -> None: - self.assertEqual(_normalize_channel("S21"), "s21") - self.assertEqual(_normalize_channel(" s11 "), "s11") - with self.assertRaises(ValueError): - _normalize_channel("s99") - def test_complex_to_points(self) -> None: self.assertEqual(_complex_to_points(np.array([1 + 2j, 3 - 4j])), [[1.0, 2.0], [3.0, -4.0]]) @@ -150,159 +112,155 @@ class VnaHistoryTest(unittest.TestCase): # --------------------------------------------------------------------------- # class WebControllerTest(unittest.TestCase): def setUp(self) -> None: - self._cfg_dir = tempfile.TemporaryDirectory() - self.addCleanup(self._cfg_dir.cleanup) - self.configs_dir = Path(self._cfg_dir.name) - (self.configs_dir / "alpha.json").write_text("{}", encoding="utf-8") - (self.configs_dir / "beta.json").write_text("{}", encoding="utf-8") - self.controller = AppWindowWebController(self.configs_dir) - self.addCleanup(self.controller.deleteLater) + self.window = isolated_window(self.addCleanup) + self.controller = self.window._web + self.configs_dir = self.controller._run_configs_dir + self.configs_dir.mkdir(exist_ok=True) + for name in ("alpha", "beta"): + (self.configs_dir / f"{name}.json").write_text("{}", encoding="utf-8") - def test_rejects_unknown_field(self) -> None: + def test_valid_live_fields_return_updated_schema_and_errors_reach_caller(self) -> None: + updated = [{"name": "gpr_object_min_frac", "value": 0.5}] + with mock.patch.object(self.window._live, "apply_web") as apply, \ + mock.patch.object(self.window._config, "web_schema", return_value=updated): + self.assertEqual(self.controller.apply_live_settings({"gpr_object_min_frac": 0.5}), updated) + apply.assert_called_once_with({"gpr_object_min_frac": 0.5}) + apply.side_effect = ValueError("Duplicate antenna position: 0") + with self.assertRaisesRegex(WebActionError, "Duplicate antenna position"): + self.controller.apply_live_settings({"tx_geometry": []}) with self.assertRaisesRegex(ValueError, "Unknown live-settings"): self.controller.apply_live_settings({"definitely_not_a_field": 1}) - def test_known_field_emits_and_returns_snapshot(self) -> None: - received: list[dict] = [] - updated = [{"name": "gpr_object_min_frac", "value": 0.5}] + def test_snapshots_return_copies_and_keep_frame_until_a_new_grab(self) -> None: + with mock.patch.object(self.controller, "_grab_frame_if_due", side_effect=[{"seq": 1}, None]): + self.controller.refresh_snapshot() + original = self.controller.status() + modified = self.controller.status() + modified["running"] = not original["running"] + self.assertEqual(self.controller.status(), original) + settings = self.controller.current_live_settings() + settings.clear() + self.assertTrue(self.controller.current_live_settings()) + self.controller.refresh_snapshot() + self.assertEqual(self.controller.peek_frame(), {"seq": 1}) - def apply(fields, call): - received.append(fields) - self.controller.update_snapshot(status={}, live_settings=updated, frame=None) - call.done.set() - - self.controller.apply_settings_requested.connect(apply) - out = self.controller.apply_live_settings({"gpr_object_min_frac": 0.5}) - self.assertEqual(received, [{"gpr_object_min_frac": 0.5}]) - self.assertEqual(out, updated) - - def test_live_settings_error_is_raised_to_the_caller(self) -> None: - self.controller.apply_settings_requested.connect( - lambda fields, call: (setattr(call, "error", "Duplicate antenna position: 0"), call.done.set()) - ) - with self.assertRaisesRegex(WebActionError, "Duplicate antenna position"): - self.controller.apply_live_settings({"tx_geometry": []}) - - def test_snapshot_is_replaced_and_returned_as_copy(self) -> None: - self.controller.update_snapshot(status={"running": True}, live_settings=[{"name": "x"}], frame={"seq": 1}) - status = self.controller.status() - self.assertEqual(status, {"running": True}) - status["running"] = False # mutating the copy must not affect the controller - self.assertTrue(self.controller.status()["running"]) - self.assertEqual(self.controller.peek_frame(), {"seq": 1}) - - def test_frame_only_updates_when_present(self) -> None: - self.controller.update_snapshot(status={}, live_settings=[], frame={"seq": 1}) - self.controller.update_snapshot(status={}, live_settings=[], frame=None) # no new grab - self.assertEqual(self.controller.peek_frame(), {"seq": 1}) # keeps the last frame - - def test_controls_emit_signals(self) -> None: - # Controls are synchronous now: each emits a call the GUI slot must finalize. - # In-thread, emit() runs the slot directly, so finalizing it returns at once. - fired: list[str] = [] - - def handler(label): - def slot(call): - fired.append(label) - call.done.set() - return slot - - self.controller.start_requested.connect(handler("start")) - self.controller.stop_requested.connect(handler("stop")) - self.controller.single_capture_requested.connect(handler("single")) - self.controller.capture_requested.connect(handler("capture")) - self.controller.start() - self.controller.stop() - self.controller.single_capture() - self.controller.capture_tmp_reference() - self.assertEqual(fired, ["start", "stop", "single", "capture"]) - - def test_action_error_is_raised_to_the_caller(self) -> None: - # An error the GUI slot records on the call surfaces as WebActionError (HTTP 400). - self.controller.start_requested.connect( - lambda call: (setattr(call, "error", "destination already exists"), call.done.set()) - ) - with self.assertRaisesRegex(WebActionError, "destination already exists"): + def test_controls_call_the_shared_runtime_actions(self) -> None: + with mock.patch.object(self.window._pipeline, "start") as start, \ + mock.patch.object(self.window._pipeline, "stop") as stop, \ + mock.patch.object(self.window._preprocess, "capture_tmp_reference") as capture, \ + mock.patch.object(self.window._snapshots, "delete_history") as remove: self.controller.start() + self.controller.stop() + self.controller.single_capture() + self.controller.capture_tmp_reference() + self.controller.remove_last_measurement() + self.controller.clear_history() + self.assertEqual(start.call_args_list, [mock.call(), mock.call(single_capture=True)]) + stop.assert_called_once() + capture.assert_called_once() + self.assertEqual(remove.call_args_list, [mock.call(remove_last=True), mock.call(remove_last=False)]) - def test_start_recording_forwards_path_name_count(self) -> None: - received: list[tuple[str, str, int]] = [] - self.controller.start_recording_requested.connect( - lambda p, n, c, call: (received.append((p, n, c)), call.done.set()) - ) - self.controller.start_recording("/tmp/out", "run1", 250) - self.assertEqual(received, [("/tmp/out", "run1", 250)]) + def test_reported_errors_keep_nested_actions_separate(self) -> None: + inner = _WebActionCall(lambda: self.window._log.show_error("inner failed")) - def test_lists_configs_sorted_and_load_emits_signal(self) -> None: + def outer() -> None: + self.window._log.show_error("outer failed") + self.controller._run_action(inner) + self.window._log.show_error("second outer error") + + with mock.patch.object(self.window._pipeline, "start", side_effect=outer): + with self.assertRaisesRegex(WebActionError, "outer failed"): + self.controller.start() + self.assertEqual(inner.error, "inner failed") + self.assertIsNone(self.window._log._capture) + + def test_desktop_error_during_web_event_pump_is_not_returned_to_web(self) -> None: + call = _WebActionCall(lambda: self.window._pipeline.pump_events(0.02)) + QTimer.singleShot(0, lambda: self.window._log.show_error("Desktop action failed")) + with mock.patch.dict(os.environ, {"RADAR_SYSTEM_HEADLESS": "0"}), \ + mock.patch("python_app.gui.controllers.log_controller.QMessageBox") as dialog: + self.controller._run_action(call) + self.assertIsNone(call.error) + dialog.return_value.setText.assert_called_once_with("Desktop action failed") + dialog.return_value.exec.assert_called_once() + self.assertIsNone(self.window._log._capture) + + def test_recording_checks_count_before_changing_destination(self) -> None: + count = self.controller._record_count + count.setMaximum(100) + before = (self.controller._save_path.text(), self.controller._save_name.text(), count.value()) + with mock.patch.object(self.window._snapshots, "start_recording") as record: + call = mock.Mock(wraps=self.controller.start_recording) + for invalid in (0, -1, True, 1.5, "2"): + with self.subTest(count=invalid), self.assertRaisesRegex(ValueError, "positive integer"): + call("/tmp/new", "run", invalid) + with self.assertRaisesRegex(WebActionError, "from 1 to 100"): + self.controller.start_recording("/tmp/new", "run", 101) + record.assert_not_called() + self.assertEqual((self.controller._save_path.text(), self.controller._save_name.text(), count.value()), before) + self.controller.start_recording("/tmp/new", "run", 25) + record.assert_called_once_with(25) + self.assertEqual((self.controller._save_path.text(), self.controller._save_name.text(), count.value()), + ("/tmp/new", "run", 25)) + + def test_config_filename_is_validated_on_gui_thread(self) -> None: self.assertEqual(self.controller.list_configs(), ["alpha.json", "beta.json"]) - requested: list[str] = [] - self.controller.load_config_requested.connect( - lambda name, call: (requested.append(name), call.done.set()) - ) - self.controller.load_config("beta.json") - self.assertEqual(requested, ["beta.json"]) + with mock.patch.object(self.window._profile, "load") as load: + self.controller.load_config("beta.json") + load.assert_called_once_with(self.configs_dir / "beta.json") + for bad in ("ghost.json", "../escape.json", "sub/alpha.json", "alpha.txt"): + with self.assertRaisesRegex(WebActionError, "Unknown run config"): + self.controller.load_config(bad) + self.controller._action_requested.disconnect() + self.controller._action_requested.connect( + lambda call: ((self.configs_dir / "alpha.json").unlink(), self.controller._run_action(call))) + with self.assertRaisesRegex(WebActionError, "Unknown run config"): + self.controller.load_config("alpha.json") + self.assertEqual(load.call_count, 1) - def test_load_config_rejects_unknown_or_unsafe_name(self) -> None: - # Missing file, traversal, sub-path, and non-json are all refused (HTTP 400 upstream). - for bad in ("ghost.json", "../escape.json", "sub/alpha.json", "alpha.txt"): - with self.assertRaisesRegex(ValueError, "Unknown run config"): - self.controller.load_config(bad) + def test_save_updates_shared_destination_and_blank_path_keeps_previous(self) -> None: + with mock.patch.object(self.window._snapshots, "save") as save: + self.controller.save_dataset("/tmp/out", "run1") + self.assertEqual((self.controller._save_path.text(), self.controller._save_name.text()), ("/tmp/out", "run1")) + self.controller.save_dataset("", "") + self.assertEqual((self.controller._save_path.text(), self.controller._save_name.text()), ("/tmp/out", "")) + self.assertEqual(save.call_count, 2) - def test_save_dataset_forwards_path_and_name(self) -> None: - received: list[tuple[str, str]] = [] - self.controller.save_dataset_requested.connect( - lambda p, n, call: (received.append((p, n)), call.done.set()) - ) - self.controller.save_dataset("/tmp/out", "run1") - self.assertEqual(received, [("/tmp/out", "run1")]) - - def test_combo_controls_forward_selection_and_next_to_gui(self) -> None: - received: list = [] - self.controller.select_combo_requested.connect( - lambda inp, out, call: (received.append((inp, out)), call.done.set()) - ) - self.controller.next_combo_requested.connect( - lambda call: (received.append("next"), call.done.set()) - ) - - self.controller.select_combo(0, 1) - self.controller.next_combo() - - self.assertEqual(received, [(0, 1), "next"]) - - def test_invalid_recording_count_is_rejected_without_dispatch(self) -> None: - fired = mock.Mock() - self.controller.start_recording_requested.connect(fired) - for count in (0, -1, True, 1.5, "2"): - with self.subTest(count=count), self.assertRaisesRegex(ValueError, "positive integer"): - self.controller.start_recording("/tmp", "run", count) - fired.assert_not_called() + def test_combo_controls_forward_selection_and_next(self) -> None: + with mock.patch.object(self.window._plots, "select_combo") as select, \ + mock.patch.object(self.window._plots, "select_next_combo") as next_combo: + self.controller.select_combo(0, 1) + self.controller.next_combo() + select.assert_called_once_with(0, 1) + next_combo.assert_called_once() def test_timed_out_queued_action_cannot_run_later(self) -> None: - queued = [] - self.controller.start_requested.connect(queued.append) - with mock.patch("python_app.gui.controllers.app_window_web_mixin._WEB_ACTION_TIMEOUT_S", 0): + queued: list[_WebActionCall] = [] + self.controller._action_requested.disconnect() + self.controller._action_requested.connect(queued.append) + with mock.patch("python_app.gui.controllers.web_controller._WEB_ACTION_TIMEOUT_S", 0), \ + mock.patch.object(self.window._pipeline, "start") as action: with self.assertRaisesRegex(WebActionError, "cancelled"): self.controller.start() - w = AppWindowWebMixin() - action = mock.Mock() - w._run_web_action(queued[0], action) - action.assert_not_called() + self.controller._run_action(queued[0]) + action.assert_not_called() self.assertFalse(self.controller._pending_calls) - def test_timeout_reports_an_already_running_action_without_repeating_it(self) -> None: - started = [] - self.controller.start_requested.connect(lambda call: (call.begin(), started.append(call))) - with mock.patch("python_app.gui.controllers.app_window_web_mixin._WEB_ACTION_TIMEOUT_S", 0): + def test_timeout_reports_running_action_without_repeating_it(self) -> None: + started: list[_WebActionCall] = [] + self.controller._action_requested.disconnect() + self.controller._action_requested.connect(lambda call: (call.begin(), started.append(call))) + with mock.patch("python_app.gui.controllers.web_controller._WEB_ACTION_TIMEOUT_S", 0): with self.assertRaisesRegex(WebActionError, "still running"): self.controller.start() self.assertFalse(started[0].done.is_set()) started[0].finish() self.assertTrue(started[0].done.is_set()) - def test_shutdown_releases_waiting_requests_and_rejects_new_controls(self) -> None: - queued = [] - self.controller.start_requested.connect(lambda call: (queued.append(call), self.controller.shutdown())) + def test_shutdown_releases_waiters_and_rejects_new_controls(self) -> None: + queued: list[_WebActionCall] = [] + self.controller._action_requested.disconnect() + self.controller._action_requested.connect(lambda call: (queued.append(call), self.controller.shutdown())) with self.assertRaisesRegex(WebActionError, "closing"): self.controller.start() self.assertTrue(queued[0].done.is_set()) @@ -310,30 +268,37 @@ class WebControllerTest(unittest.TestCase): with self.assertRaisesRegex(WebActionError, "closing"): self.controller.next_combo() - -class WebActionLifecycleTest(unittest.TestCase): - def test_closed_window_does_not_run_queued_control(self) -> None: - w = AppWindowWebMixin() - w._closing = True - call = _WebActionCall() + def test_closing_pipeline_does_not_run_queued_control(self) -> None: action = mock.Mock() - w._run_web_action(call, action) + call = _WebActionCall(action) + self.window._pipeline.begin_close() + self.controller._run_action(call) action.assert_not_called() self.assertTrue(call.done.is_set()) self.assertEqual(call.error, "The application is closing") - def test_out_of_range_recording_count_does_not_change_destination_or_arm(self) -> None: - w = AppWindowWebMixin() - w._record_count = SimpleNamespace(minimum=lambda: 1, maximum=lambda: 100, setValue=mock.Mock()) - w._save_path_input = SimpleNamespace(setText=mock.Mock()) - w._save_name_input = SimpleNamespace(setText=mock.Mock()) - w._start_run_with_recording = mock.Mock() - with self.assertRaisesRegex(ValueError, "from 1 to 100"): - w._start_web_recording("/tmp/new", "run", 101) - w._save_path_input.setText.assert_not_called() - w._save_name_input.setText.assert_not_called() - w._record_count.setValue.assert_not_called() - w._start_run_with_recording.assert_not_called() + def test_worker_thread_action_runs_on_gui_thread(self) -> None: + threads: list[int] = [] + errors: list[Exception] = [] + + def request() -> None: + try: + self.controller.start() + except Exception as exc: + errors.append(exc) + + with mock.patch.object(self.window._pipeline, "start", side_effect=lambda: threads.append(threading.get_ident())): + worker = threading.Thread(target=request) + worker.start() + deadline = time.monotonic() + 2 + while worker.is_alive() and time.monotonic() < deadline: + _app.processEvents() + time.sleep(0.001) + self.controller.shutdown() + worker.join(timeout=1) + self.assertFalse(worker.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(threads, [threading.get_ident()]) class WebPortTest(unittest.TestCase): @@ -342,7 +307,7 @@ class WebPortTest(unittest.TestCase): with mock.patch.dict(os.environ, env, clear=False): if value is None: os.environ.pop("RADAR_SYSTEM_WEBUI_PORT", None) - return AppWindowWebMixin._web_ui_port() + return web_ui_port() def test_default_when_unset_or_invalid(self) -> None: self.assertEqual(self._port_with_env(None), 8080) @@ -356,9 +321,13 @@ class WebPortTest(unittest.TestCase): class WebLiveSettingsRouteTest(unittest.TestCase): def setUp(self) -> None: + cleanup = ExitStack() + self.addCleanup(cleanup.close) self.controller = mock.Mock() - self.request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(controller=self.controller))) - self.enterContext(mock.patch( + app = FastAPI() + app.state.controller = self.controller + self.request = Request({"type": "http", "app": app}) + cleanup.enter_context(mock.patch( "python_app.webui.routes.asyncio.to_thread", new=mock.AsyncMock(side_effect=lambda func, *args: func(*args)), )) @@ -384,51 +353,54 @@ class WebLiveSettingsRouteTest(unittest.TestCase): # --------------------------------------------------------------------------- # # Latest-wins fan-out # --------------------------------------------------------------------------- # -class WebSocketInitialFrameTest(unittest.TestCase): - def test_new_client_receives_cached_plot_even_when_no_new_frame_is_published(self) -> None: - self.enterContext(mock.patch( - "python_app.webui.routes.asyncio.to_thread", - new=mock.AsyncMock(side_effect=lambda func, *args: func(*args)), - )) - for frame in ({"type": "frame", "seq": 7, "png_b64": "cached"}, None): - with self.subTest(frame=frame): - controller = SimpleNamespace(peek_frame=lambda: frame) - broadcaster = mock.Mock() - status = {"type": "status", "running": False} - broadcaster.register.side_effect = lambda queue: queue.put_nowait(status) - socket = SimpleNamespace( - app=SimpleNamespace(state=SimpleNamespace(controller=controller, broadcaster=broadcaster)), - accept=mock.AsyncMock(), close=mock.AsyncMock(), - send_json=mock.AsyncMock(side_effect=WebSocketDisconnect), - ) - asyncio.run(ws(socket)) - socket.send_json.assert_awaited_once_with(frame if frame is not None else status) - broadcaster.unregister.assert_called_once() - socket.close.assert_awaited_once() - - class RingBroadcasterFanOutTest(unittest.TestCase): - def test_full_client_queue_drops_oldest(self) -> None: + def test_capture_enabled_only_between_first_connect_and_last_disconnect(self) -> None: async def scenario() -> None: - broadcaster = RingBroadcaster(controller=object()) - queue: asyncio.Queue = asyncio.Queue(maxsize=1) - broadcaster.register(queue) - broadcaster._publish({"type": "frame", "seq": 1}) - broadcaster._publish({"type": "frame", "seq": 2}) # evicts seq 1 - self.assertEqual(queue.qsize(), 1) - self.assertEqual(queue.get_nowait(), {"type": "frame", "seq": 2}) # newest survives + controller = mock.Mock(spec=WebController) + broadcaster = RingBroadcaster(controller=controller) + controller.peek_frame.return_value = None + controller.status.return_value = {} + controller.current_live_settings.return_value = [] + queue = broadcaster.register() + second = broadcaster.register() + controller.set_frame_streaming.assert_called_once_with(True) + broadcaster.unregister(queue) + broadcaster.unregister(queue) # idempotent + controller.set_frame_streaming.assert_called_once_with(True) + broadcaster.unregister(second) + self.assertEqual(controller.set_frame_streaming.call_args_list, [mock.call(True), mock.call(False)]) asyncio.run(scenario()) - def test_unregister_stops_delivery(self) -> None: + def test_idle_skips_capture_and_new_frame_survives_status_heartbeat(self) -> None: async def scenario() -> None: - broadcaster = RingBroadcaster(controller=object()) - queue: asyncio.Queue = asyncio.Queue(maxsize=4) - broadcaster.register(queue) - broadcaster.unregister(queue) - broadcaster.unregister(queue) # idempotent - broadcaster._publish({"type": "frame", "seq": 1}) - self.assertEqual(queue.qsize(), 0) + controller = mock.Mock(spec=WebController) + controller.status.return_value = {"running": False} + controller.current_live_settings.return_value = [] + frame = {"type": "frame", "seq": 1, "png_b64": "plot"} + controller.peek_frame.return_value = frame + broadcaster = RingBroadcaster(controller) + published = asyncio.Event() + publish = broadcaster._publish + + def record(message: dict) -> None: + publish(message) + if message["type"] == "frame": + published.set() + + broadcaster._publish = record + broadcaster.start() + try: + await asyncio.sleep(0) # first polling iteration, with no clients + controller.peek_frame.assert_not_called() + queue = broadcaster.register() + await asyncio.wait_for(published.wait(), timeout=1) + messages = [await queue.get() for _ in range(3)] + self.assertIn(frame, messages) + self.assertEqual({message["type"] for message in messages}, {"frame", "status", "settings"}) + finally: + await broadcaster.stop() + controller.set_frame_streaming.assert_called_with(False) asyncio.run(scenario()) diff --git a/python_app/tests/test_switched_matrix_capture.py b/python_app/tests/test_switched_matrix_capture.py index 7a84bab..3e108f9 100644 --- a/python_app/tests/test_switched_matrix_capture.py +++ b/python_app/tests/test_switched_matrix_capture.py @@ -17,7 +17,7 @@ from python_app.hardware_full.matrix_radar_service import create_matrix_radar_se from python_app.hardware_full.multi_device_service import MultiDeviceLibreVnaService from python_app.hardware_full.switched_matrix_radar_service import SwitchedMatrixRadarService from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData -from python_app.models.run_config_model import RunConfigModel +from python_app.models.run_config_schema import RunConfigModel from python_app.workflows.sequential_capture_workflow import SequentialCaptureSession _INNER_INPUTS = 4 @@ -40,17 +40,13 @@ class _FakeInnerMatrixRadar: def configure(self, sweep) -> None: pass - def recover(self) -> None: - pass - - def acquire_collection(self, collection_id: int = 1) -> SweepCollection: + def acquire_collection(self, collection_id: int = 1, *, combo: ComboKey | None = None) -> SweepCollection: self.acquire_count += 1 frequency_hz = np.linspace(1e6, 2e6, _POINTS, dtype=np.float32) traces = [ TraceData( combo=ComboKey(input=input_pos, output=output_pos), frequency_hz=frequency_hz, - s11=np.full(_POINTS, complex(self.acquire_count, 0), dtype=np.complex64), s21=np.full(_POINTS, complex(input_pos, output_pos), dtype=np.complex64), ) for output_pos in range(_INNER_OUTPUTS) @@ -79,6 +75,10 @@ class _FakeSwitch: def position_count(self) -> int: return self.positions + @property + def current_position(self) -> int: + return self.switched_to[-1] if self.switched_to else 0 + def switch_to(self, position: int) -> None: self.switched_to.append(int(position)) @@ -98,13 +98,13 @@ def _switched_service(input_steps: int = 3) -> tuple[SwitchedMatrixRadarService, class SwitchedMatrixComboAcquisitionTest(unittest.TestCase): - """acquire_combo_collection must acquire exactly one physical switch step.""" + """Targeted acquire_collection must acquire exactly one physical switch step.""" def test_acquires_only_the_step_containing_the_combo(self) -> None: service, inner, input_switch = _switched_service(input_steps=3) # Widened input 9 lives in physical step 9 // 4 = 2. - collection = service.acquire_combo_collection(input_pos=9, output_pos=1) + collection = service.acquire_collection(combo=ComboKey(9, 1)) self.assertEqual(inner.acquire_count, 1) self.assertEqual(input_switch.switched_to, [2]) @@ -120,9 +120,9 @@ class SwitchedMatrixComboAcquisitionTest(unittest.TestCase): def test_rejects_out_of_range_combo(self) -> None: service, _inner, _input_switch = _switched_service(input_steps=3) with self.assertRaises(ValueError): - service.acquire_combo_collection(input_pos=12, output_pos=0) + service.acquire_collection(combo=ComboKey(12, 0)) with self.assertRaises(ValueError): - service.acquire_combo_collection(input_pos=0, output_pos=2) + service.acquire_collection(combo=ComboKey(0, 2)) def test_full_collection_still_covers_widened_matrix_in_canonical_order(self) -> None: service, inner, input_switch = _switched_service(input_steps=3) @@ -197,7 +197,7 @@ class ManualComboCaptureUsesTargetedAcquisitionTest(unittest.TestCase): ) session.open() try: - trace = session.capture_current_combo() + trace = session.capture_current_combo().display_trace finally: session.close() first_combo = config.combos[0] @@ -234,6 +234,7 @@ class PerPointStimulusSwitchingConfigTest(unittest.TestCase): config.radar.driver_mode = "native" service = create_matrix_radar_service(config) self.assertIsInstance(service, MultiDeviceLibreVnaService) + assert isinstance(service, MultiDeviceLibreVnaService) self.assertNotIsInstance(service, SwitchedMatrixRadarService) self.assertEqual(service.output_position_count, 4) diff --git a/python_app/tests/test_vna_history_json.py b/python_app/tests/test_vna_history_json.py new file mode 100644 index 0000000..e266069 --- /dev/null +++ b/python_app/tests/test_vna_history_json.py @@ -0,0 +1,87 @@ +"""JSON export keeps real stage data and never guesses across repeated sweep IDs.""" + +from __future__ import annotations + +import unittest + +import numpy as np + +from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection, TraceData +from python_app.storage.npz.vna_history_json import build_vna_history_payload + + +def sweep(cid: int, ns: int, value: float, window: tuple[int, int] = (0, 0)) -> SweepCollection: + trace = TraceData( + ComboKey(0, 0), np.array([1e8, 2e8]), np.full(2, value, complex), + capture_start_ns=window[0], capture_end_ns=window[1], + ) + return SweepCollection(cid, ns, [trace], capture_start_ns=window[0], capture_end_ns=window[1]) + + +def payload(raw: list[SweepCollection], pre: list[SweepCollection], **kwargs): + return build_vna_history_payload(raw, pre, [], input_index=0, output_index=0, **kwargs) + + +class VnaHistoryJsonTest(unittest.TestCase): + def test_missing_stage_is_empty_instead_of_fabricated(self) -> None: + for stage in ("raw", "preprocessed"): + with self.subTest(stage=stage): + raw, pre = ([sweep(1, 100, 7)], []) if stage == "raw" else ([], [sweep(1, 110, 9)]) + history = payload(raw, pre)["sweep_history"] + self.assertEqual(len(history), 1) + self.assertEqual(history[0]["sweep_points"], [[7., 0.], [7., 0.]] if raw else []) + self.assertEqual(history[0]["calibrated_points"], [[9., 0.], [9., 0.]] if pre else []) + + def test_dropped_stages_and_reused_ids_preserve_every_trace_once(self) -> None: + raw = [sweep(1, 100, 1, (80, 90)), sweep(1, 200, 2, (180, 190)), sweep(2, 300, 3, (280, 290))] + pre = [sweep(1, 210, 20, (180, 190)), sweep(3, 410, 40, (380, 390))] + for stage in ("raw", "preprocessed"): + with self.subTest(primary=stage): + history = payload(raw, pre, primary_stage=stage)["sweep_history"] + self.assertEqual(len(history), 4) + self.assertEqual([row["sweep_points"][0][0] if row["sweep_points"] else None for row in history], [1, 2, 3, None]) + self.assertEqual([row["calibrated_points"][0][0] if row["calibrated_points"] else None for row in history], [None, 20, None, 40]) + + def test_conflicting_windows_are_never_paired(self) -> None: + history = payload([sweep(1, 100, 1, (80, 90))], [sweep(1, 110, 10, (81, 91))])["sweep_history"] + self.assertEqual(len(history), 2) + self.assertEqual(history[0]["calibrated_points"], []) + self.assertEqual(history[1]["sweep_points"], []) + + def test_export_timestamp_uses_selected_trace_in_a_switched_collection(self) -> None: + pre = sweep(1, 200, 10, (100, 150)) + pre.capture_start_ns, pre.capture_end_ns = 1, 190 + history = payload([], [pre])["sweep_history"] + self.assertEqual(history[0]["timestamp"], 150 / 1e9) + + def test_join_uses_collection_window_and_capture_order_not_trace_or_stage_time(self) -> None: + raw = sweep(1, 900, 1, (10, 20)) + pre = sweep(1, 1000, 10, (10, 20)) + pre.traces[0].capture_start_ns = 11 + pre.traces[0].capture_end_ns = 19 + fresh = sweep(1, 100, 2, (30, 40)) + history = payload([fresh, raw], [pre])["sweep_history"] + self.assertEqual(len(history), 2) + self.assertEqual(history[0]["sweep_points"], [[1., 0.], [1., 0.]]) + self.assertEqual(history[0]["calibrated_points"], [[10., 0.], [10., 0.]]) + self.assertEqual(history[0]["timestamp"], 19 / 1e9) + self.assertEqual(history[1]["calibrated_points"], []) + + def test_reordered_results_do_not_produce_an_alignment_warning(self) -> None: + pre = [sweep(1, 110, 1, (10, 20)), sweep(2, 210, 2, (30, 40))] + result = [ResultCollection(2, 999, capture_start_ns=30, capture_end_ns=40), + ResultCollection(1, 888, capture_start_ns=10, capture_end_ns=20)] + exported = build_vna_history_payload([], pre, result, input_index=0, output_index=0) + self.assertNotIn("alignment_warning", exported) + + def test_selected_combo_with_many_distinct_ids_has_no_lost_or_duplicated_records(self) -> None: + raw = [sweep(cid, cid * 100, cid, (cid * 100 - 20, cid * 100 - 10)) for cid in range(1, 2001)] + pre = [sweep(cid, cid * 100 + 10, cid * 2, (cid * 100 - 20, cid * 100 - 10)) for cid in range(1, 2001)] + history = payload(raw, pre)["sweep_history"] + self.assertEqual(len(history), 2000) + self.assertEqual(history[-1]["sweep_points"][0][0], 2000) + self.assertEqual(history[-1]["calibrated_points"][0][0], 4000) + + +if __name__ == "__main__": + unittest.main() diff --git a/python_app/tests/test_web_frame_streaming.py b/python_app/tests/test_web_frame_streaming.py new file mode 100644 index 0000000..e5cfc0f --- /dev/null +++ b/python_app/tests/test_web_frame_streaming.py @@ -0,0 +1,108 @@ + +from contextlib import ExitStack +"""Plot capture follows client demand and actual Qt scene changes.""" + +from collections import deque +import unittest +from unittest import mock + +import numpy as np +from PyQt6.QtCore import QSignalBlocker +from PyQt6.QtWidgets import QApplication, QComboBox + +from python_app.tests.gui_test_support import isolated_window + + +class WebPlotCaptureTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.app = QApplication.instance() or QApplication([]) + cls.window = isolated_window(cls.addClassCleanup) + cls.window._pipeline.timer.stop() + cls.window._pipeline.replay.timer.stop() + cls.window.show() + + def setUp(self) -> None: + cleanup = ExitStack() + self.addCleanup(cleanup.close) + self.w = self.window + self.web = self.w._web + self.web.set_frame_streaming(False) + self.grab = cleanup.enter_context(mock.patch.object(self.web, "_grab_png", wraps=self.web._grab_png)) + + def settle(self) -> None: + # Initial paints can settle the axis layout over several Qt event turns. + for _ in range(10): + self.app.processEvents() + self.web._last_grab_s = 0 + self.web._grab_frame_if_due() + + def test_no_clients_never_grabs_and_first_client_gets_current_plot(self) -> None: + self.w._plots.bscan.widget.plot([0, 1], [1, 2]) + self.settle() + self.grab.assert_not_called() + controller = self.web + mode = self.w._view.processing.mode + with QSignalBlocker(mode): + mode.setCurrentText("bscan") + self.w._live.active_mode = "bscan" + self.w._plots.bscan.history_by_combo = {(2, 3): deque([np.array([1., 2.])])} + self.w._plots.sync_display_combos() + self.web.refresh_snapshot() + self.assertEqual(controller.status()["display_combo"], + {"enabled": True, "input": 2, "output": 3, "options": [(2, 3)]}) + self.grab.assert_not_called() + controller.set_frame_streaming(True) + self.settle() + self.assertGreater(self.grab.call_count, 0) + self.assertIsNotNone(self.web._last_png) + calls = self.grab.call_count + self.settle() + self.assertEqual(self.grab.call_count, calls) # grab does not keep itself dirty + controller.set_frame_streaming(False) + self.w._plots.bscan.widget.clear() + self.settle() + self.assertEqual(self.grab.call_count, calls) + + def test_data_clear_zoom_resize_and_sar_grid_trigger_capture_after_idle(self) -> None: + self.web.set_frame_streaming(True) + plot = self.w._plots.bscan.widget + item = plot.getPlotItem() + assert item is not None + view = item.getViewBox() + self.w._plots.stack.setCurrentWidget(plot) + self.settle() + actions = [ + lambda: plot.plot([0, 1, 2], [1, 4, 2]), + lambda: view.setXRange(0.25, 1.5, padding=0), + plot.clear, + lambda: self.w.resize(self.w.width() + 50, self.w.height() + 50), + lambda: self.w._plots.stack.setCurrentWidget(self.w._plots.sar3d.widget), + ] + actions.extend(lambda plot=plot: plot.plot([0, 1], [1, 3]) for plot in self.w._plots.sar3d.plots.values()) + for action in actions: + with self.subTest(action=action): + calls = self.grab.call_count + action() + self.settle() + self.assertGreater(self.grab.call_count, calls) + calls = self.grab.call_count + self.settle() + self.assertEqual(self.grab.call_count, calls) + + def test_throttle_keeps_dirty_and_ignores_changes_emitted_during_grab(self) -> None: + self.web.set_frame_streaming(True) + self.settle() + self.web._mark_dirty() + self.web._last_grab_s = 100 + with mock.patch("python_app.gui.controllers.web_controller.time.monotonic", return_value=100.1): + self.assertIsNone(self.web._grab_frame_if_due()) + self.assertTrue(self.web._dirty) + self.grab.side_effect = lambda: (self.web._mark_dirty(), "changed-png")[1] + with mock.patch("python_app.gui.controllers.web_controller.time.monotonic", return_value=100.3): + self.assertIsNotNone(self.web._grab_frame_if_due()) + self.assertFalse(self.web._dirty) + + +if __name__ == "__main__": + unittest.main() diff --git a/python_app/tests/test_web_streaming.py b/python_app/tests/test_web_streaming.py new file mode 100644 index 0000000..53001e2 --- /dev/null +++ b/python_app/tests/test_web_streaming.py @@ -0,0 +1,66 @@ +"""A slow browser keeps the latest plot, status and settings independently.""" + +import asyncio +import unittest +from unittest import mock + +from fastapi import FastAPI, WebSocket, WebSocketDisconnect + +from python_app.webui.controller import WebController +from python_app.webui.routes import ws +from python_app.webui.streaming import RingBroadcaster + + +class WebStreamingTest(unittest.IsolatedAsyncioTestCase): + async def test_slow_client_keeps_each_type_and_reconnect_gets_current_settings(self) -> None: + controller = mock.Mock(spec=WebController) + controller.status.return_value = {"running": False} + controller.current_live_settings.return_value = [{"name": "gain", "value": 0}] + controller.peek_frame.return_value = {"type": "frame", "seq": 0} + broadcaster = RingBroadcaster(controller) + app = FastAPI() + app.state.controller = controller + app.state.broadcaster = broadcaster + started, release = asyncio.Event(), asyncio.Event() + received: list[dict] = [] + + async def send(message: dict) -> None: + received.append(message) + if len(received) == 1: + started.set() + await release.wait() + if len(received) == 4: + raise WebSocketDisconnect() + + socket = mock.Mock(spec=WebSocket, app=app, accept=mock.AsyncMock(), + close=mock.AsyncMock(), send_json=mock.AsyncMock(side_effect=send)) + with mock.patch("python_app.webui.routes.asyncio.to_thread", mock.AsyncMock( + side_effect=lambda func, *args: func(*args))): + client = asyncio.create_task(ws(socket)) + await asyncio.wait_for(started.wait(), 1) + for value in range(1, 101): + controller.status.return_value = {"running": True, "raw_count": value} + controller.current_live_settings.return_value = [{"name": "gain", "value": value}] + controller.peek_frame.return_value = {"type": "frame", "seq": value} + broadcaster._publish({"type": "status", **controller.status()}) + broadcaster._publish({"type": "settings", "schema": controller.current_live_settings()}) + broadcaster._publish(controller.peek_frame()) + release.set() + await asyncio.wait_for(client, 1) + latest = {message["type"]: message for message in received[1:]} + self.assertEqual(latest, { + "frame": {"type": "frame", "seq": 100}, + "status": {"type": "status", "running": True, "raw_count": 100}, + "settings": {"type": "settings", "schema": [{"name": "gain", "value": 100}]}, + }) + + received.clear() + + async def reconnect_send(message: dict) -> None: + received.append(message) + if len(received) == 3: + raise WebSocketDisconnect() + + socket.send_json.side_effect = reconnect_send + await asyncio.wait_for(ws(socket), 1) + self.assertEqual({message["type"]: message for message in received}, latest) diff --git a/python_app/webui/__init__.py b/python_app/webui/__init__.py index 12832ad..3312a1f 100644 --- a/python_app/webui/__init__.py +++ b/python_app/webui/__init__.py @@ -3,7 +3,7 @@ The web UI streams the live GPR view and exposes the pipeline controls of the desktop app without any Qt dependency. It depends only on the small :class:`~python_app.webui.controller.WebController` contract; the embedded Qt -bridge (``gui/controllers/app_window_web_mixin.py``) implements that contract by +bridge (``gui/controllers/web_controller.py``) implements that contract by forwarding to the AppWindow's existing buttons, so no control flow is duplicated. - :mod:`controller` — the Qt-free control/read contract. diff --git a/python_app/webui/controller.py b/python_app/webui/controller.py index 0d9b0ba..63d4764 100644 --- a/python_app/webui/controller.py +++ b/python_app/webui/controller.py @@ -2,10 +2,9 @@ The web layer (``app``/``routes``/``streaming``) is deliberately free of any Qt or hardware knowledge: it talks only to a :class:`WebController`. The embedded -bridge in ``gui/controllers/app_window_web_mixin.py`` implements this protocol by -forwarding control actions to the AppWindow's *existing* buttons and exposing -read-only snapshots — so the very same desktop logic backs the browser, with no -duplicated control flow. +bridge in ``gui/controllers/web_controller.py`` implements this protocol by +dispatching actions to the shared GUI controllers and exposing read-only +snapshots. The same controller methods serve desktop buttons and the browser. """ from __future__ import annotations @@ -57,8 +56,12 @@ class WebController(Protocol): def remove_last_measurement(self) -> None: """Remove the most recent measurement (the desktop button).""" + def clear_history(self) -> None: + """Clear all runtime measurements (the desktop button).""" + def apply_live_settings(self, fields: dict) -> list: """Apply live processor settings; returns the current settings schema.""" + ... def load_config(self, name: str) -> None: """Load a run-config profile by file name (the desktop "Load Config" button).""" @@ -67,19 +70,25 @@ class WebController(Protocol): """Save up to 1000 recent measurements to ``path``/``name``.""" def select_combo(self, input_pos: int, output_pos: int) -> None: - """Select the matrix B-scan input/output pair shown by the desktop.""" + """Select the B-scan input/output pair shown by the desktop.""" def next_combo(self) -> None: - """Advance the matrix B-scan display to the next input/output pair.""" + """Advance the B-scan display to the next available input/output pair.""" def current_live_settings(self) -> list: """Return the live-settings schema (built from the Qt widgets).""" + ... def list_configs(self) -> list[str]: """Return the run-config file names available to load.""" + ... def status(self) -> dict: """Return a snapshot of pipeline/run state.""" + ... def peek_frame(self) -> dict | None: - """Return the latest rendered-plot frame (PNG of the Qt plot), or ``None``.""" + """Read a cached PNG frame without rendering or blocking; initially ``None``.""" + + def set_frame_streaming(self, enabled: bool) -> None: + """Enable plot capture only while at least one WebSocket client is connected.""" diff --git a/python_app/webui/routes.py b/python_app/webui/routes.py index b77d91d..85c8fc1 100644 --- a/python_app/webui/routes.py +++ b/python_app/webui/routes.py @@ -99,6 +99,13 @@ async def post_remove_last(request: Request) -> dict: return controller.status() +@router.post("/api/clear_history") +async def post_clear_history(request: Request) -> dict: + controller = _controller(request) + await _run_action(controller.clear_history) + return controller.status() + + @router.get("/api/configs") async def get_configs(request: Request) -> dict: return {"names": _controller(request).list_configs()} @@ -157,18 +164,14 @@ async def ws(websocket: WebSocket) -> None: """Stream frames and status to one client until it disconnects.""" await websocket.accept() broadcaster: RingBroadcaster = websocket.app.state.broadcaster - queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=1) - broadcaster.register(queue) + client = broadcaster.register() logger.info("Web UI client connected") try: - frame = await asyncio.to_thread(websocket.app.state.controller.peek_frame) - if frame is not None: - await websocket.send_json(frame) while True: - await websocket.send_json(await queue.get()) + await websocket.send_json(await client.get()) except WebSocketDisconnect: logger.info("Web UI client disconnected") finally: - broadcaster.unregister(queue) + broadcaster.unregister(client) with contextlib.suppress(Exception): await websocket.close() diff --git a/python_app/webui/static/app.js b/python_app/webui/static/app.js index 2bc894c..3851b82 100644 --- a/python_app/webui/static/app.js +++ b/python_app/webui/static/app.js @@ -102,7 +102,8 @@ async function changeDisplayCombo(path, body) { } catch (err) { toast(err.message, true); } finally { - displayComboSelect.disabled = btnNextCombo.disabled = false; + displayComboSelect.disabled = displayComboSelect.options.length === 0; + btnNextCombo.disabled = displayComboSelect.options.length < 2; } } @@ -267,7 +268,8 @@ function buildSettingsForm(schema) { function collectFields() { const out = {}; for (const key in fieldInputs) { - const { el, kind } = fieldInputs[key]; + const { el, kind, dirty } = fieldInputs[key]; + if (!dirty) continue; // unchanged browser values must not overwrite newer desktop edits if (kind === "geometry") out[key] = el.getRecords(); else if (kind === "bool") out[key] = el.checked; else if (kind === "int" || kind === "float") { @@ -296,7 +298,9 @@ function clearDirty() { async function applyOne(field, value) { try { - await api("/api/live_settings", { [field]: value }); + const schema = await api("/api/live_settings", { [field]: value }); + if (fieldInputs[field]) fieldInputs[field].dirty = false; + applySettings(schema); } catch (err) { toast(err.message, true); } @@ -321,8 +325,9 @@ function applySettings(schema) { btnApply.addEventListener("click", async () => { btnApply.disabled = true; try { - await api("/api/live_settings", collectFields()); + const schema = await api("/api/live_settings", collectFields()); clearDirty(); // applied; let live pushes update the form again + applySettings(schema); settingsNote.textContent = "Applied."; } catch (err) { settingsNote.textContent = err.message; @@ -335,7 +340,7 @@ btnApply.addEventListener("click", async () => { btnResetHistory.addEventListener("click", async () => { btnResetHistory.disabled = true; try { - await api("/api/live_settings", { history_command: "clear_all" }); + await api("/api/clear_history"); settingsNote.textContent = "History reset requested."; } catch (err) { settingsNote.textContent = err.message; @@ -464,6 +469,8 @@ function applyStatus(s) { comboOptionsSignature = signature; } displayComboSelect.value = `${combo.input}:${combo.output}`; + displayComboSelect.disabled = options.length === 0; + btnNextCombo.disabled = options.length < 2; } if ("running" in s) { setStat(runningEl, "running", s.running ? "yes" : "no", s.running ? "ok" : "off"); diff --git a/python_app/webui/streaming.py b/python_app/webui/streaming.py index fda9b9a..bcad2f7 100644 --- a/python_app/webui/streaming.py +++ b/python_app/webui/streaming.py @@ -1,10 +1,9 @@ """Fan-out of pipeline result frames and status to connected web clients. -A single broadcaster task polls the :class:`WebController` off the event loop and +A single broadcaster task reads cached :class:`WebController` snapshots and pushes the freshest frame (latest-wins) plus a slower status heartbeat to every -registered client. Each client is a bounded ``asyncio.Queue`` with a drop-oldest -policy, so a slow socket can never stall the producer or the loop — if a client -falls behind it simply skips intermediate frames and always gets the newest. +registered client. Each client keeps one latest message per type, so a slow +socket skips intermediate frames without losing status or settings updates. """ from __future__ import annotations @@ -24,24 +23,55 @@ _FRAME_INTERVAL_S = 0.05 _STATUS_INTERVAL_S = 1.0 +class ClientUpdates: + """At most one pending frame, status and settings update for one socket.""" + + def __init__(self) -> None: + self._pending: dict[str, dict] = {} + self._ready = asyncio.Event() + + def put(self, message: dict) -> None: + self._pending[message["type"]] = message + self._ready.set() + + async def get(self) -> dict: + await self._ready.wait() + message = self._pending.pop(next(iter(self._pending))) + if not self._pending: + self._ready.clear() + return message + + class RingBroadcaster: """Polls the controller and fans frames/status out to all WebSocket clients.""" def __init__(self, controller: WebController) -> None: self._controller = controller - self._clients: set[asyncio.Queue[dict]] = set() + self._clients: set[ClientUpdates] = set() self._task: asyncio.Task[None] | None = None self._last_frame_seq: int | None = None - self._last_settings: dict | None = None + self._last_settings: list | None = None - def register(self, queue: asyncio.Queue[dict]) -> None: - """Add a client queue to receive subsequent frames and status.""" - self._clients.add(queue) + def register(self) -> ClientUpdates: + """Seed every connection with current snapshots, then stream updates.""" + client = ClientUpdates() + frame = self._controller.peek_frame() + if frame is not None: + client.put(frame) + client.put(self._status_message()) + client.put({"type": "settings", "schema": self._controller.current_live_settings()}) + if not self._clients: + self._controller.set_frame_streaming(True) + self._clients.add(client) logger.debug("Registered web client queue (clients=%d)", len(self._clients)) + return client - def unregister(self, queue: asyncio.Queue[dict]) -> None: + def unregister(self, client: ClientUpdates) -> None: """Remove a client queue; safe to call more than once.""" - self._clients.discard(queue) + was_connected = bool(self._clients) + self._clients.discard(client) + if was_connected and not self._clients: + self._controller.set_frame_streaming(False) logger.debug("Unregistered web client queue (clients=%d)", len(self._clients)) def start(self) -> None: @@ -59,6 +89,8 @@ class RingBroadcaster: async def stop(self) -> None: """Cancel the polling task and wait for it to unwind.""" + self._controller.set_frame_streaming(False) + self._clients.clear() if self._task is None: return self._task.cancel() @@ -68,13 +100,9 @@ class RingBroadcaster: logger.info("Ring broadcaster stopped") def _publish(self, message: dict) -> None: - """Push a message to every client, dropping the oldest on a full queue.""" - for queue in self._clients: - if queue.full(): - with contextlib.suppress(asyncio.QueueEmpty): - queue.get_nowait() - with contextlib.suppress(asyncio.QueueFull): - queue.put_nowait(message) + """Replace only older messages of the same type.""" + for client in self._clients: + client.put(message) def _status_message(self) -> dict: """Build a status broadcast from the controller's current state.""" @@ -86,22 +114,21 @@ class RingBroadcaster: next_status = loop.time() while True: try: - # peek_frame may touch shared memory / NumPy, so keep it off the loop. - frame = await loop.run_in_executor(None, self._controller.peek_frame) - if frame is not None and frame["seq"] != self._last_frame_seq: - self._last_frame_seq = frame["seq"] - self._publish(frame) - now = loop.time() - if now >= next_status: + if self._clients and now >= next_status: self._publish(self._status_message()) # Push live settings (Qt -> web) only when they change, so the # browser form mirrors desktop edits in real time without churn. settings = self._controller.current_live_settings() if settings != self._last_settings: - self._last_settings = settings self._publish({"type": "settings", "schema": settings}) + self._last_settings = settings next_status = now + _STATUS_INTERVAL_S + if self._clients: + frame = self._controller.peek_frame() + if frame is not None and frame["seq"] != self._last_frame_seq: + self._publish(frame) + self._last_frame_seq = frame["seq"] except asyncio.CancelledError: raise except Exception: # noqa: BLE001 - one bad frame must not stop streaming diff --git a/python_app/workflows/mimo_calibration.py b/python_app/workflows/mimo_calibration.py index 9a19bf8..26e978a 100644 --- a/python_app/workflows/mimo_calibration.py +++ b/python_app/workflows/mimo_calibration.py @@ -1,33 +1,20 @@ -"""Object-based MIMO S21 corrections estimated after standard preprocessing. - -Archives retain the standalone script's cal_*.npy/cal_meta.json layout. Pair -corrections multiply S21 after through calibration, reference subtraction and -notch filtering; they never alter S11 or the original measurement files. -""" +"""Estimate object-based MIMO S21 factors after through/reference preprocessing.""" from __future__ import annotations -from contextlib import contextmanager -from collections.abc import Callable, Iterable, Sequence -from dataclasses import dataclass +from collections.abc import Callable, Iterable +from dataclasses import asdict, dataclass +import copy import datetime -import json -import logging -from pathlib import Path -import re -import shutil -import tempfile from typing import Any import numpy as np -from numpy.typing import ArrayLike -from python_app.file_io import atomic_write_json -from python_app.models.dataset_model import SweepCollection +from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData +from python_app.storage.npz.serialize import validate_trace_collection +from python_app.workflows.object_calibration import geometry_signature C_LIGHT = 299_792_458.0 -_CAL_ARRAYS = ('freq_hz', 'pair', 'tx', 'rx', 'valid', 'tx_output_pos', 'rx_input_pos', 'antenna_params') -logger = logging.getLogger(__name__) def _ports(values, label): @@ -59,7 +46,7 @@ def _frequency(value, *, uniform=False): @dataclass class Settings: - """Target geometry and estimation options, independent of example globals.""" + """Reflector geometry and phase-estimation settings.""" sphere_radius_m: float = .16 input_idx: tuple[int, ...] = (0, 1, 2, 3) output_idx: tuple[int, ...] = (0, 1, 2, 3) @@ -95,34 +82,10 @@ class Settings: raise ValueError('apply_amplitude must be boolean') -def find_config_profile(start: str | Path, max_levels: int = 8) -> Path: - """Locate the archived acquisition profile above a stage or snapshot folder.""" - path = Path(start) - for _ in range(max_levels): - candidate = path / 'config_profile.json' - if candidate.is_file(): - return candidate - path = path.parent - raise FileNotFoundError(f'config_profile.json not found above {start}') - - -def _profile(path): - config = json.loads(Path(path).read_text(encoding='utf-8')) - if not isinstance(config, dict): - raise ValueError('config_profile.json must contain an object') - return config - - def _geometry(config, input_idx, output_idx): - result = [] + geometry, result = geometry_signature(config), [] for key, identity, ports in (('tx_geometry', 'output_pos', output_idx), ('rx_geometry', 'input_pos', input_idx)): - entries = config.get('gpr', {}).get(key, []) - mapping = {} - for entry in entries: - port = _ports([entry[identity]], identity)[0] - if port in mapping: - raise ValueError(f'duplicate antenna geometry for {identity}={port}') - mapping[port] = _finite_array([entry['x_m'], entry.get('y_m', 0.), entry.get('z_m', 0.)], key, (3,)) + mapping = {entry[identity]: [entry['x_m'], entry['y_m'], entry['z_m']] for entry in geometry[key]} missing = set(ports) - mapping.keys() if missing: raise ValueError(f'{key} is missing physical ports {sorted(missing)}') @@ -130,268 +93,8 @@ def _geometry(config, input_idx, output_idx): return tuple(result) -def load_geometry(config_path: str | Path, input_idx: Sequence[int], output_idx: Sequence[int]) -> tuple[np.ndarray, np.ndarray]: - """Read TX/RX radar coordinates in sorted physical port order.""" - return _geometry(_profile(config_path), sorted(_ports(input_idx, 'input_idx')), sorted(_ports(output_idx, 'output_idx'))) - - -def list_snapshots(root: str | Path, prefixes: Sequence[str] | None = None) -> list[Path]: - """Select complete snapshot directories, requiring unambiguous prefixes.""" - directories = sorted(path for path in Path(root).iterdir() if path.is_dir() and (path / 'meta.json').is_file()) - if prefixes is None: - selected = directories - else: - selected = [] - for prefix in prefixes: - matches = [path for path in directories if path.name.startswith(str(prefix))] - if len(matches) != 1 or matches[0] in selected: - raise ValueError(f'snapshot prefix {prefix!r} must select exactly one distinct snapshot in {root}') - selected.extend(matches) - if not selected: - raise ValueError(f'no snapshots selected in {root}') - return selected - - -def load_s21(snapshot_dir: str | Path, input_idx: Sequence[int], output_idx: Sequence[int]) -> tuple[np.ndarray, np.ndarray]: - """Read a finite TX×RX×frequency matrix on one shared uniform grid.""" - inputs = sorted(_ports(input_idx, 'input_idx')) - outputs = sorted(_ports(output_idx, 'output_idx')) - rows, freq = [], None - for output in outputs: - row = [] - for input_ in inputs: - suffix = f'i{input_}_o{output}' - samples = np.load(Path(snapshot_dir) / f'{suffix}_s21.npy', allow_pickle=False) - axis = _frequency(np.load(Path(snapshot_dir) / f'{suffix}_freq.npy', allow_pickle=False), uniform=True) - if freq is not None and not np.array_equal(axis, freq): - raise ValueError(f'frequency grids differ between pairs in {snapshot_dir}') - freq = axis - row.append(_finite_array(samples, suffix, freq.shape, complex_values=True)) - rows.append(row) - assert freq is not None - return np.array(rows), freq - - -def _source_signature(config) -> dict[str, Any]: - radar = config.get('radar', {}) - s21 = config.get('preprocess', {}).get('s21', {}) - return { - 'radar': {key: radar[key] for key in ('model', 'serial') if key in radar}, - 'slave_serials': radar.get('multi_device', {}).get('slave_serials', []), - 's21_calibration_set': s21.get('calibration', {}).get('set_name', ''), - 's21_reference_set': s21.get('reference', {}).get('set_name', ''), - 'notch': config.get('preprocess', {}).get('notch', {}), - } - - -def validate_calibration(cal: dict[str, Any]) -> None: - """Validate an archive before saving, applying or exporting runtime factors.""" - freq = _frequency(cal['freq_hz'], uniform=True) - tx = _ports(cal['tx_output_pos'], 'tx_output_pos') - rx = _ports(cal['rx_input_pos'], 'rx_input_pos') - if tx != tuple(sorted(tx)) or rx != tuple(sorted(rx)): - raise ValueError('calibration physical ports must be sorted') - T = _finite_array(cal['tx'], 'tx corrections', (len(tx), freq.size), complex_values=True) - R = _finite_array(cal['rx'], 'rx corrections', (len(rx), freq.size), complex_values=True) - pair = _finite_array(cal['pair'], 'pair corrections', (len(tx), len(rx), freq.size), complex_values=True) - if np.any(np.abs(T) == 0) or np.any(np.abs(R) == 0) or np.any(np.abs(pair) == 0): - raise ValueError('calibration corrections must be nonzero') - if not np.allclose(pair, T[:, None, :] * R[None, :, :], rtol=2e-5, atol=1e-7): - raise ValueError('pair corrections disagree with tx/rx corrections') - valid = np.asarray(cal['valid']) - if valid.dtype.kind != 'b' or valid.shape != freq.shape or valid.sum() < 2: - raise ValueError('calibration valid mask must select at least two frequencies') - _finite_array(cal['antenna_params'], 'antenna parameters', (len(tx) + len(rx), 3)) - geometry = cal['meta']['geometry'] - _finite_array(geometry['tx_xyz_m'], 'TX geometry', (len(tx), 3)) - _finite_array(geometry['rx_xyz_m'], 'RX geometry', (len(rx), 3)) - - -def validate_for_config(cal: dict[str, Any], config: dict[str, Any]) -> None: - """Require physical geometry, combo coverage and a compatible phase base.""" - validate_calibration(cal) - tx, rx = _geometry(config, cal['rx_input_pos'], cal['tx_output_pos']) - geometry = cal['meta']['geometry'] - if not (np.allclose(tx, geometry['tx_xyz_m'], rtol=0, atol=1e-9) and - np.allclose(rx, geometry['rx_xyz_m'], rtol=0, atol=1e-9)): - raise ValueError('antenna geometry differs from the object calibration') - for combo in config.get('run', {}).get('combos', []): - input_, output = _ports([combo['input']], 'combo input')[0], _ports([combo['output']], 'combo output')[0] - if input_ not in cal['rx_input_pos'] or output not in cal['tx_output_pos']: - raise ValueError(f'object calibration does not cover combo {combo}') - source = cal['meta'].get('source_preprocess', {}) - current = _source_signature(config) - for key in ('radar', 'slave_serials', 's21_calibration_set'): - if source.get(key) and current.get(key) and source[key] != current[key]: - raise ValueError(f'object calibration source differs: {key}') - - -@contextmanager -def _staged_directory(destination, overwrite, *, preserve_existing=False): - destination = Path(destination) - destination.parent.mkdir(parents=True, exist_ok=True) - if destination.exists() and (not overwrite or not destination.is_dir() or destination.is_symlink()): - raise FileExistsError(f'{destination} already exists; choose another name or overwrite=True') - stage = Path(tempfile.mkdtemp(prefix=f'.{destination.name}.new-', dir=destination.parent)) - backup = None - retain = False - try: - if preserve_existing and destination.exists(): - shutil.copytree(destination, stage, dirs_exist_ok=True, symlinks=True) - yield stage - if destination.exists(): - if not overwrite: - raise FileExistsError(f'{destination} appeared during save; choose another name or overwrite=True') - backup = Path(tempfile.mkdtemp(prefix=f'.{destination.name}.old-', dir=destination.parent)) - backup.rmdir() - destination.replace(backup) - try: - stage.replace(destination) - except BaseException: - if backup is not None: - try: - backup.replace(destination) - except OSError as error: - retain = True - raise RuntimeError(f'calibration rollback failed; originals retained at {backup}, new files at {stage}') from error - raise - finally: - if not retain: - for temporary in (stage, backup): - if temporary is not None and temporary.exists(): - try: - shutil.rmtree(temporary) - except OSError: - logger.exception('Could not remove calibration temporary directory %s', temporary) - - -def save_calibration(cal: dict[str, Any], cal_dir: str | Path, overwrite: bool = False) -> Path: - """Commit a complete legacy archive, restoring its predecessor on failure.""" - validate_calibration(cal) - destination = Path(cal_dir) - if destination.is_dir() and any(destination.glob('cal_*')) and not overwrite: - raise FileExistsError(f'{destination} already contains calibration files; use overwrite=True') - meta: dict[str, Any] = dict(cal['meta'], name=destination.name) - # Older one-target archives used nonstandard JSON NaN for unavailable LOO. - quality = dict(meta.get('quality', {})) - if 'coherence_leave_one_out' in quality: - quality['coherence_leave_one_out'] = [value if value is None or np.isfinite(value) else None for value in quality['coherence_leave_one_out']] - meta['quality'] = quality - with _staged_directory(destination, overwrite or destination.is_dir(), preserve_existing=True) as stage: - for key in _CAL_ARRAYS: - path = stage / f'cal_{key}.npy' - path.unlink(missing_ok=True) - np.save(path, cal[key], allow_pickle=False) - (stage / 'cal_meta.json').unlink(missing_ok=True) - atomic_write_json(stage / 'cal_meta.json', meta) - cal['meta'], cal['dir'] = meta, destination - return destination - - -def load_calibration(cal_dir: str | Path) -> dict[str, Any]: - """Load and validate cal_*.npy arrays plus cal_meta.json without pickle.""" - directory = Path(cal_dir) - cal: dict[str, Any] = {key: np.load(directory / f'cal_{key}.npy', allow_pickle=False) for key in _CAL_ARRAYS} - cal.update(meta=json.loads((directory / 'cal_meta.json').read_text(encoding='utf-8')), dir=directory) - validate_calibration(cal) - return cal - - -def calibration_on_grid(cal: dict[str, Any], freq: ArrayLike) -> np.ndarray: - """Interpolate separable corrections and continue phase smoothly at edges.""" - validate_calibration(cal) - freq = _frequency(freq) - fc = np.asarray(cal['freq_hz']) - if np.array_equal(freq, fc): - return cal['pair'] - - def interp(correction, params): - phase = np.unwrap(np.angle(correction)) - out = np.interp(freq, fc, phase) - slope = 2 * np.pi * params[1] / C_LIGHT - for selected, edge in ((freq < fc[0], 0), (freq > fc[-1], -1)): - out[selected] = phase[edge] + slope * (freq[selected] - fc[edge]) - return np.interp(freq, fc, np.abs(correction)) * np.exp(1j * out) - - n_tx = len(cal['tx_output_pos']) - tx = np.array([interp(value, cal['antenna_params'][index]) for index, value in enumerate(cal['tx'])]) - rx = np.array([interp(value, cal['antenna_params'][n_tx + index]) for index, value in enumerate(cal['rx'])]) - corrections = (tx[:, None, :] * rx[None, :, :]).astype(np.asarray(cal['pair']).dtype) - _finite_array(corrections, 'interpolated corrections', complex_values=True) - return corrections - - -def apply_calibration(s21: ArrayLike, freq: ArrayLike, cal: dict[str, Any]) -> np.ndarray: - """Multiply a TX×RX×frequency S21 matrix by validated pair corrections.""" - corrections = calibration_on_grid(cal, freq) - samples = _finite_array(s21, 'S21', corrections.shape, complex_values=True) - return _finite_array(samples * corrections, 'corrected S21', corrections.shape, complex_values=True) - - -def apply_to_measurement(meas_dir: str | Path, cal: dict[str, Any], overwrite: bool = False) -> tuple[Path, int, int, int]: - """Stage corrected S21 and copied companion files in postprocessed safely.""" - directory = Path(meas_dir) - pre, post = directory / 'preprocessed', directory / 'postprocessed' - if not pre.is_dir(): - raise FileNotFoundError(f'no {pre}') - config = _profile(find_config_profile(pre)) - _reject_corrected_source(config) - validate_for_config(cal, config) - if post.exists() and overwrite and not (post / 'calibration_applied.json').is_file(): - raise FileExistsError(f'{post} has no calibration_applied.json; refusing to replace it') - tx = {int(port): index for index, port in enumerate(cal['tx_output_pos'])} - rx = {int(port): index for index, port in enumerate(cal['rx_input_pos'])} - pattern = re.compile(r'^i(\d+)_o(\d+)_s21\.npy$') - n_snap = n_fix = n_copy = 0 - cache = {} - with _staged_directory(post, overwrite) as stage: - for item in sorted(pre.iterdir()): - if item.is_file(): - shutil.copy2(item, stage / item.name) - n_copy += 1 - continue - out = stage / item.name - out.mkdir() - n_snap += 1 - for path in sorted(item.iterdir()): - match = pattern.match(path.name) - if path.is_dir() or match is None: - if path.is_dir(): - shutil.copytree(path, out / path.name) - else: - shutil.copy2(path, out / path.name) - n_copy += 1 - continue - input_, output = int(match[1]), int(match[2]) - if input_ not in rx or output not in tx: - raise ValueError(f'object calibration does not cover {path.name}') - freq = _frequency(np.load(item / f'i{input_}_o{output}_freq.npy', allow_pickle=False)) - key = freq.tobytes() - if key not in cache: - cache[key] = calibration_on_grid(cal, freq) - samples = np.load(path, allow_pickle=False) - dtype = samples.dtype - samples = _finite_array(samples, path.name, freq.shape, complex_values=True) - corrected = (samples * cache[key][tx[output], rx[input_]]).astype(dtype) - _finite_array(corrected, 'corrected S21', complex_values=True) - np.save(out / path.name, corrected, allow_pickle=False) - n_fix += 1 - atomic_write_json(stage / 'calibration_applied.json', dict( - calibration=cal['meta'].get('name'), calibration_dir=str(cal.get('dir') or ''), - created=datetime.datetime.now().isoformat(timespec='seconds'), source=str(pre), - rule='postprocessed S21 = preprocessed S21 * cal_pair[tx(output_pos), rx(input_pos)]', - frequency_grids=[np.frombuffer(key, dtype=np.float64).tolist() for key in cache])) - return post, n_snap, n_fix, n_copy - - -def _reject_corrected_source(config): - object_calibration = config.get('preprocess', {}).get('s21', {}).get('object_calibration', {}) - if object_calibration.get('set_name') or object_calibration.get('bundle_path'): - raise ValueError('calibration sources must have no object correction; object_calibration is already active') - - def _best_phases(Z): - """Maximize pair _coherence by alternating antenna phases, anchored at TX0/RX0.""" + """Maximize pair coherence by alternating antenna phases, anchored at TX0/RX0.""" t = np.zeros(Z.shape[0]) r = np.zeros(Z.shape[1]) for _ in range(100): @@ -613,20 +316,12 @@ def _mean_samples(samples, check, expected_freq=None): return mean, freq -def _settings(settings): - settings = settings or Settings() - settings.__post_init__() - return settings - - -def _compute(sets, empty, freq, settings, config, config_path, check) -> dict[str, Any]: +def _compute(sets, freq, settings, config, provenance, check) -> tuple[SweepCollection, dict[str, Any]]: tx, rx = _geometry(config, settings.input_idx, settings.output_idx) est = _Estimator(settings, tx, rx, freq, check) for index, record in enumerate(sets): check(f'Estimating target {index + 1}/{len(sets)}') data = record.pop('s21') - if empty is not None: - data = data - empty nominal = _finite_array(record['target_scene_m'], 'target_scene_m', (3,)) echo, refined = est.refine_target(data, est.coarse_range(data, nominal)) if not np.all(np.any(np.abs(echo[:, :, est.ok]) > 0, axis=2)): @@ -642,57 +337,46 @@ def _compute(sets, empty, freq, settings, config, config_path, check) -> dict[st antennas.update({f'Rx{index}': est.antenna_params(value) for index, value in enumerate(R_all)}) T_full = np.array([est.to_full_grid(value, antennas[f'Tx{index}']) for index, value in enumerate(T_all)]) R_full = np.array([est.to_full_grid(value, antennas[f'Rx{index}']) for index, value in enumerate(R_all)]) - unit = (lambda value: np.conj(value) / np.abs(value)) if not settings.apply_amplitude else (lambda value: 1 / value) - saved_pair = unit(T_full[:, None, :] * R_full[None, :, :]).astype(np.complex64) - quality = dict(coherence_before=[], coherence_own=[], coherence_leave_one_out=[], coherence_final=[]) + saved_pair = est.pair_correction(T_full, R_full).astype(np.complex64) + _finite_array(saved_pair, 'pair corrections', complex_values=True) diagnostics = [] final_correction = saved_pair[:, :, est.band] for record in sets: check() before = est.broadband(record['E']) after = est.broadband(record['E'] * final_correction) - quality['coherence_before'].append(_coherence(before)) - quality['coherence_own'].append(_coherence(est.broadband(record['E'] * est.pair_correction(record['T'], record['R'])))) - others = [other for other in sets if other is not record] - quality['coherence_leave_one_out'].append( - _coherence(est.broadband(record['E'] * est.pair_correction(*_combine(others)))) if others else None) - quality['coherence_final'].append(_coherence(after)) - diagnostics.append(dict(phase_before_deg=np.degrees(np.angle(before)).tolist(), - phase_after_deg=np.degrees(np.angle(after)).tolist(), + # Notebook cell 5: each map uses its own summed contribution as phase zero. + phase_before = (np.degrees(np.angle(before) - np.angle(before.sum())) + 180) % 360 - 180 + phase_after = (np.degrees(np.angle(after) - np.angle(after.sum())) + 180) % 360 - 180 + diagnostics.append(dict(phase_before_deg=phase_before.tolist(), + phase_after_deg=phase_after.tolist(), + phase_std_before_deg=float(np.std(phase_before)), + phase_std_after_deg=float(np.std(phase_after)), coherence_before=_coherence(before), coherence_after=_coherence(after))) - valid = np.zeros(freq.size, bool) - valid[np.flatnonzero(est.band)[est.ok]] = True - meta = dict( - created=datetime.datetime.now().isoformat(timespec='seconds'), name=None, - convention=dict(apply='S21_corrected[tx, rx, f] = S21[tx, rx, f] * cal_pair[tx, rx, f]', - index_order='sorted physical tx_output_pos x rx_input_pos', - phase_reference='relative to Tx0 and Rx0; common phase/delay is retained', - amplitude='equalized' if settings.apply_amplitude else 'phase only', - outside_band='linear antenna phase continuation anchored at the band edge'), - geometry=dict(config=str(config_path), tx_xyz_m=tx.tolist(), rx_xyz_m=rx.tolist(), - antenna_height_m=settings.antenna_height_m, pitch_deg=settings.pitch_deg), - estimation=dict(band_hz=[settings.f_min, settings.f_max], gate_half_m=settings.gate_half_m, - gate_taper_m=settings.gate_taper_m, coarse_range_search_m=settings.coarse_range_search_m, - refine_axes=list(settings.refine_axes), refine_range_m=settings.refine_range_m, - phase_model=settings.phase_model, sphere_radius_m=settings.sphere_radius_m), - source_preprocess=_source_signature(config), - targets=[dict(path=record.get('path', ''), snapshots=record.get('snapshots', []), - nominal_scene_m=record['q_nominal'].tolist(), refined_scene_m=record['q'].tolist()) for record in sets], - antennas=antennas, quality=quality, diagnostics=diagnostics) - cal = dict(freq_hz=freq.astype(np.float64), pair=saved_pair, - tx=unit(T_full).astype(np.complex64), rx=unit(R_full).astype(np.complex64), valid=valid, - tx_output_pos=np.array(settings.output_idx, np.int32), rx_input_pos=np.array(settings.input_idx, np.int32), - antenna_params=np.array([[np.radians(value['phase0_deg']), value['delay_path_m'], value['gain_db']] - for value in antennas.values()]), meta=meta, dir=None) - validate_calibration(cal) + metadata = dict( + created=datetime.datetime.now().isoformat(timespec='seconds'), + provenance=copy.deepcopy(provenance), + tx_output_pos=list(settings.output_idx), rx_input_pos=list(settings.input_idx), + geometry=dict(tx_xyz_m=tx.tolist(), rx_xyz_m=rx.tolist()), + settings=asdict(settings), + source_preprocess=dict(reference_name=config.get('preprocess', {}).get('s21', {}).get('reference', {}).get('set_name', ''), + notch=copy.deepcopy(config.get('preprocess', {}).get('notch', {}))), + targets=[dict(captures=record['captures'], nominal_scene_m=record['q_nominal'].tolist(), + refined_scene_m=record['q'].tolist()) for record in sets], + antennas=antennas, diagnostics=diagnostics, + ) + frequency = freq.astype(np.float32) + collection = SweepCollection(0, 0, [ + TraceData(ComboKey(input_, output), frequency, saved_pair[t, r]) + for t, output in enumerate(settings.output_idx) for r, input_ in enumerate(settings.input_idx) + ]) + validate_trace_collection(collection, require_finite=True) check('Object calibration complete') - return cal + return collection, metadata def _collection_samples(collection, settings): - from python_app.storage.npz.serialize import validate_trace_collection - validate_trace_collection(collection, require_finite=True) traces = {(trace.combo.input, trace.combo.output): trace for trace in collection.traces} rows, freq = [], None @@ -712,81 +396,39 @@ def _collection_samples(collection, settings): def compute_from_collections( - cal_sets: Iterable[dict[str, Any]], config_dict: dict[str, Any], - empty_collections: Iterable[SweepCollection] | None = None, settings: Settings | None = None, - *, progress: Callable[[str], None] | None = None, cancelled: Callable[[], bool] | None = None, -) -> dict[str, Any]: - """Estimate corrections from standard-preprocessed sweeps without object correction. + cal_sets: Iterable[dict[str, Any]], config_dict: dict[str, Any], settings: Settings | None = None, + *, provenance: dict[str, Any], progress: Callable[[str], None] | None = None, + cancelled: Callable[[], bool] | None = None, +) -> tuple[SweepCollection, dict[str, Any]]: + """Estimate a saved set from preprocessed sweeps without object correction. - cal_sets contains {collections: [SweepCollection, ...], target_scene_m: XYZ}. - progress accepts a status string; cancelled returns bool and raises - InterruptedError promptly in data, refinement and factorization loops. + Each target provides ``collections`` and ground-relative ``target_scene_m``. + Reference subtraction has already been applied by the live preprocessor. + Cancellation is checked during averaging, refinement and factorization. """ - settings = _settings(settings) - _reject_corrected_source(config_dict) + settings = Settings() if settings is None else settings check = _callbacks(progress, cancelled) sets = list(cal_sets) if not sets: raise ValueError('at least one reflector target is required') - empty, freq = None, None - if empty_collections is not None: - check('Averaging empty scene') - empty, freq = _mean_samples((_collection_samples(value, settings) for value in empty_collections), check) - prepared = [] + prepared, freq = [], None for index, record in enumerate(sets): check(f'Averaging target {index + 1}/{len(sets)}') collections = list(record['collections']) mean, freq = _mean_samples((_collection_samples(value, settings) for value in collections), check, freq) - prepared.append(dict(s21=mean, target_scene_m=record['target_scene_m'], - snapshots=[str(value.collection_id) for value in collections])) - return _compute(prepared, empty, freq, settings, config_dict, 'live capture', check) - - -def compute_calibration( - cal_sets: Iterable[dict[str, Any]], empty_scene_path: str | Path | None = None, settings: Settings | None = None, - *, progress: Callable[[str], None] | None = None, cancelled: Callable[[], bool] | None = None, -) -> dict[str, Any]: - """Legacy archive API backed by the same estimator as live collection capture.""" - settings = _settings(settings) - check = _callbacks(progress, cancelled) - cal_sets = list(cal_sets) - if not cal_sets: - raise ValueError('at least one reflector target is required') - config_path = find_config_profile(cal_sets[0]['path']) - config = _profile(config_path) - _reject_corrected_source(config) - geometry = _geometry(config, settings.input_idx, settings.output_idx) - signature = _source_signature(config) - - def samples(directories): - for directory in directories: - check() - profile_path = find_config_profile(directory) - path = Path(directory) - while True: - if (path / 'calibration_applied.json').is_file(): - raise ValueError(f'object calibration is already applied to archive {directory}') - if path == profile_path.parent: - break - path = path.parent - source = _profile(profile_path) - _reject_corrected_source(source) - current = _geometry(source, settings.input_idx, settings.output_idx) - if any(not np.allclose(first, second, rtol=0, atol=1e-9) for first, second in zip(geometry, current)): - raise ValueError(f'antenna geometry differs in {directory}') - if signature != _source_signature(source): - raise ValueError(f'preprocessing or hardware configuration differs in {directory}') - yield load_s21(directory, settings.input_idx, settings.output_idx) - - empty, freq = None, None - if empty_scene_path is not None: - empty, freq = _mean_samples(samples(list_snapshots(empty_scene_path)), check) - prepared = [] - for record in cal_sets: - directories = list_snapshots(record['path'], record.get('snapshots')) - mean, freq = _mean_samples(samples(directories), check, freq) - prepared.append(dict(s21=mean, target_scene_m=record['target_scene_m'], path=str(record['path']), - snapshots=[directory.name for directory in directories])) - cal = _compute(prepared, empty, freq, settings, config, config_path, check) - cal['meta']['estimation']['empty_scene'] = str(empty_scene_path) if empty_scene_path is not None else None - return cal + prepared.append(dict(s21=mean, target_scene_m=record['target_scene_m'], captures=[ + dict(collection_id=value.collection_id, monotonic_ns=value.monotonic_ns, + capture_start_ns=value.capture_start_ns, capture_end_ns=value.capture_end_ns) + for value in collections + ])) + collection, metadata = _compute(prepared, freq, settings, config_dict, provenance, check) + # Multi-device acquisition keeps every stage for timing. Pairs outside the + # configured geometry carry identity factors, explicitly marked in the set. + corrected = {trace.combo for trace in collection.traces} + unchanged = [trace for trace in collections[0].traces if trace.combo not in corrected] + collection.traces.extend(TraceData( + trace.combo, np.asarray(trace.frequency_hz, dtype=np.float32), + np.ones(trace.frequency_hz.size, dtype=np.complex64), + ) for trace in unchanged) + metadata['unchanged_combos'] = [dict(input=trace.combo.input, output=trace.combo.output) for trace in unchanged] + return collection, metadata diff --git a/python_app/workflows/multi_radar_capture_workflow.py b/python_app/workflows/multi_radar_capture_workflow.py deleted file mode 100644 index 246ba31..0000000 --- a/python_app/workflows/multi_radar_capture_workflow.py +++ /dev/null @@ -1,361 +0,0 @@ -"""Sequential capture workflow that captures each combo across multiple radar sweep variants.""" - -from __future__ import annotations - -from contextlib import suppress -from dataclasses import dataclass -import logging -import time -from typing import cast - -from python_app.hardware_full.matrix_radar_service import MatrixRadarService, create_matrix_radar_service -from python_app.hardware_full.single_radar_service import SingleRadarService, create_single_radar_service -from python_app.hardware_full.switch_service import SwitchService -from python_app.models.dataset_model import SweepCollection, TraceData -from python_app.models.run_config_model import ComboModel, RunConfigModel -from python_app.storage.npz_store import NpzStore -from python_app.workflows.radar_config_variants import RadarConfigVariant -from python_app.workflows.sequential_capture_workflow import ( - MATRIX_RADAR_MANUAL_CAPTURE_KINDS, - SequentialCaptureState, - acquire_matrix_combo_collection, - capture_single_radar_combo, - combine_collections_via_median, - combine_traces_via_median, - select_trace_for_combo, - trace_capture_window, -) - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True, slots=True) -class MultiRadarCaptureBatch: - """One completed combo capture across all configured radar variants.""" - - combo: ComboModel - traces: tuple[TraceData, ...] - variant_labels: tuple[str, ...] - - @property - def display_trace(self) -> TraceData: - """Return the last trace in the batch for preview rendering.""" - return self.traces[-1] - - -@dataclass(frozen=True, slots=True) -class MultiRadarSavedSet: - """One preprocess set persisted for one radar variant.""" - - display_name: str - radar_key: str - trace_count: int - - -class MultiRadarSequentialCaptureSession: - """Capture a full combo matrix for one preprocess asset across multiple radar sweep variants.""" - - def __init__( - self, - *, - base_config: RunConfigModel, - kind: str, - set_name: str, - radar_variants: list[RadarConfigVariant], - median_sweep_count: int = 1, - ) -> None: - """Create capture session for one preprocess asset set and multiple radar variants.""" - if kind not in {"s21_calibration", "s21_reference", "s11_open", "s11_short", "s11_load", "s11_reference"}: - raise RuntimeError(f"Unsupported capture kind: {kind}") - if not set_name: - raise RuntimeError("Set name is required") - if not radar_variants: - raise RuntimeError("At least one radar variant is required") - if int(median_sweep_count) < 1: - raise RuntimeError("median_sweep_count must be >= 1") - - self._base_config = base_config - self._kind = kind - self._set_name = set_name - self._radar_variants = list(radar_variants) - self._median_sweep_count = int(median_sweep_count) - self._is_matrix_radar = base_config.is_matrix_radar - self._manual_matrix_radar_capture = ( - self._is_matrix_radar and kind in MATRIX_RADAR_MANUAL_CAPTURE_KINDS - ) - self._combos = base_config.build_runtime_combos() - if not self._combos: - raise RuntimeError("No switch combinations available for capture") - - self._captured_batches: list[MultiRadarCaptureBatch] = [] - self._traces_by_radar_key = { - variant.radar_key: [] - for variant in self._radar_variants - } - self._next_index = 0 - self._opened = False - - logger.info( - "Multi-radar capture session created: kind=%s set=%s combos=%d variants=%d matrix_radar=%s", - self._kind, - self._set_name, - len(self._combos), - len(self._radar_variants), - self._is_matrix_radar, - ) - - self._radar: MatrixRadarService | SingleRadarService - if self._is_matrix_radar: - self._radar = create_matrix_radar_service(base_config) - self._input_switch = None - self._output_switch = None - else: - self._radar = create_single_radar_service(base_config) - self._input_switch = SwitchService.from_model(base_config.input_switch) - self._output_switch = SwitchService.from_model(base_config.output_switch) - - @property - def kind(self) -> str: - """Return canonical preprocess asset key for this capture session.""" - return self._kind - - @property - def set_name(self) -> str: - """Return destination set name.""" - return self._set_name - - def open(self) -> None: - """Open radar and switch resources.""" - if self._opened: - return - self._opened = True - try: - self._radar.open() - self._radar.configure(self._base_config.radar.sweep) - if self._input_switch is not None: - self._input_switch.open() - if self._output_switch is not None: - self._output_switch.open() - logger.info("Multi-radar capture session opened (kind=%s set=%s)", self._kind, self._set_name) - except Exception: - logger.exception( - "Failed to open multi-radar capture session (kind=%s set=%s)", self._kind, self._set_name - ) - self.close() - raise - - def close(self) -> None: - """Close all opened hardware resources.""" - was_open = self._opened - for resource in (self._output_switch, self._input_switch, self._radar): - if resource is not None: - with suppress(Exception): - resource.close() - self._opened = False - if was_open: - logger.info("Multi-radar capture session closed (kind=%s set=%s)", self._kind, self._set_name) - - def state(self) -> SequentialCaptureState: - """Return current progress snapshot.""" - current_combo = self._current_combo() - return SequentialCaptureState( - kind=self._kind, - set_name=self._set_name, - captured_count=( - self._next_index - if self._is_matrix_radar and not self._manual_matrix_radar_capture - else len(self._captured_batches) - ), - total_count=len(self._combos), - current_combo=current_combo, - can_undo=bool(self._captured_batches), - is_complete=self.is_complete(), - variant_count=len(self._radar_variants), - supports_batch_capture=not self._manual_matrix_radar_capture, - ) - - def capture_current_combo(self) -> MultiRadarCaptureBatch: - """Capture the current combo across all radar variants and advance the combo cursor.""" - if not self._opened: - raise RuntimeError("Capture session is not opened") - combo = self._current_combo() - if combo is None: - raise RuntimeError("Capture session is already complete") - - pending_traces_by_radar_key: dict[str, list[TraceData]] = {} - display_traces: list[TraceData] = [] - variant_labels: list[str] = [] - - if self._is_matrix_radar: - radar = cast(MatrixRadarService, self._radar) - for variant in self._radar_variants: - self._radar.configure(variant.config.radar.sweep) - if self._base_config.runtime.settling_ms > 0: - time.sleep(self._base_config.runtime.settling_ms / 1000.0) - if self._manual_matrix_radar_capture: - # Only this combo is kept, so acquire the smallest collection - # that contains it instead of the full (switch-widened) matrix. - per_sweep_traces = [ - select_trace_for_combo( - acquire_matrix_combo_collection(radar, combo), combo - ) - for _ in range(self._median_sweep_count) - ] - trace = combine_traces_via_median(per_sweep_traces) - pending_traces_by_radar_key[variant.radar_key] = [trace] - display_traces.append(trace) - else: - collections: list[SweepCollection] = [] - for _ in range(self._median_sweep_count): - collection = radar.acquire_collection(collection_id=1) - if not collection.traces: - raise RuntimeError( - f"Matrix radar variant {variant.display_name} returned no traces" - ) - collections.append(collection) - combined_collection = combine_collections_via_median(collections) - pending_traces_by_radar_key[variant.radar_key] = list(combined_collection.traces) - display_traces.append(combined_collection.traces[-1]) - variant_labels.append(variant.display_name) - else: - if self._input_switch is None or self._output_switch is None: - raise RuntimeError("Switches are not initialised for combo capture") - self._output_switch.switch_to(combo.output) - self._input_switch.switch_to(combo.input) - if self._base_config.runtime.settling_ms > 0: - time.sleep(self._base_config.runtime.settling_ms / 1000.0) - - for variant in self._radar_variants: - self._radar.configure(variant.config.radar.sweep) - if self._base_config.runtime.settling_ms > 0: - time.sleep(self._base_config.runtime.settling_ms / 1000.0) - trace = capture_single_radar_combo( - cast(SingleRadarService, self._radar), combo, self._median_sweep_count, - ) - pending_traces_by_radar_key[variant.radar_key] = [trace] - display_traces.append(trace) - variant_labels.append(variant.display_name) - - for radar_key, traces in pending_traces_by_radar_key.items(): - self._traces_by_radar_key[radar_key].extend(traces) - batch = MultiRadarCaptureBatch( - combo=combo, - traces=tuple(display_traces), - variant_labels=tuple(variant_labels), - ) - self._captured_batches.append(batch) - if self._is_matrix_radar and not self._manual_matrix_radar_capture: - self._next_index = len(self._combos) - else: - self._next_index += 1 - logger.debug( - "Captured combo input=%d output=%d across %d variant(s) (%d/%d)", - combo.input, - combo.output, - len(variant_labels), - self._next_index, - len(self._combos), - ) - return batch - - def undo_last_capture(self) -> MultiRadarCaptureBatch: - """Remove the most recently captured combo batch and rewind the cursor.""" - if not self._opened: - raise RuntimeError("Capture session is not opened") - if not self._captured_batches or self._next_index <= 0: - raise RuntimeError("No captured combo is available to undo") - - if self._is_matrix_radar and not self._manual_matrix_radar_capture: - removed_batch = self._captured_batches[-1] - for variant in self._radar_variants: - traces = self._traces_by_radar_key[variant.radar_key] - if len(traces) < len(self._combos): - raise RuntimeError("Capture session state is inconsistent; missing matrix radar traces") - del traces[-len(self._combos) :] - self._next_index = 0 - self._captured_batches.pop() - return removed_batch - - expected_combo = self._combos[self._next_index - 1] - removed_batch = self._captured_batches[-1] - if ( - int(removed_batch.combo.input) != int(expected_combo.input) - or int(removed_batch.combo.output) != int(expected_combo.output) - ): - raise RuntimeError("Capture session state is inconsistent; last batch does not match rewind combo") - - for variant in self._radar_variants: - traces = self._traces_by_radar_key[variant.radar_key] - if not traces: - raise RuntimeError("Capture session state is inconsistent; missing trace during undo") - traces.pop() - - self._next_index -= 1 - self._captured_batches.pop() - return removed_batch - - def last_captured_trace(self) -> TraceData | None: - """Return the most recent trace from the most recent combo batch, if any.""" - if not self._captured_batches: - return None - return self._captured_batches[-1].display_trace - - def captured_batches(self) -> list[MultiRadarCaptureBatch]: - """Return completed combo batches in capture order.""" - return list(self._captured_batches) - - def traces_for_radar_key(self, radar_key: str) -> list[TraceData]: - """Return every captured trace (one per combo) for one radar variant. - - Unlike a batch's ``traces`` (one entry per variant, holding only the - preview trace), this returns the full per-combo set, so matrix radars - expose all ports rather than just the last one. - """ - return list(self._traces_by_radar_key.get(radar_key, [])) - - def radar_variant_count(self) -> int: - """Return how many radar variants are captured per combo.""" - return len(self._radar_variants) - - def is_complete(self) -> bool: - """Return `True` when all combos were captured.""" - return self._next_index >= len(self._combos) - - def finalize(self, store: NpzStore) -> list[MultiRadarSavedSet]: - """Persist completed captures as one preprocess set per radar variant.""" - if not self.is_complete(): - raise RuntimeError("Capture session is not complete") - - monotonic_ns = time.monotonic_ns() - saved_sets: list[MultiRadarSavedSet] = [] - for variant in self._radar_variants: - traces = list(self._traces_by_radar_key[variant.radar_key]) - capture_start_ns, capture_end_ns = trace_capture_window(traces) - collection = SweepCollection( - collection_id=1, - monotonic_ns=monotonic_ns, - traces=traces, - capture_start_ns=capture_start_ns, - capture_end_ns=capture_end_ns, - ) - store.save_set(self._kind, variant.radar_key, self._set_name, collection) - saved_sets.append( - MultiRadarSavedSet( - display_name=variant.display_name, - radar_key=variant.radar_key, - trace_count=len(traces), - ) - ) - logger.info( - "Finalized multi-radar capture kind=%s set=%s into %d variant set(s)", - self._kind, - self._set_name, - len(saved_sets), - ) - return saved_sets - - def _current_combo(self) -> ComboModel | None: - """Return next combo to capture, or `None` if session is complete.""" - if self._next_index >= len(self._combos): - return None - return self._combos[self._next_index] diff --git a/python_app/workflows/kamil_adc_neutral_preprocess.py b/python_app/workflows/neutral_preprocess.py similarity index 78% rename from python_app/workflows/kamil_adc_neutral_preprocess.py rename to python_app/workflows/neutral_preprocess.py index 1b80e73..a29d530 100644 --- a/python_app/workflows/kamil_adc_neutral_preprocess.py +++ b/python_app/workflows/neutral_preprocess.py @@ -1,13 +1,8 @@ -"""Neutral preprocessing-set helpers — the "run without calibration" path. +"""Neutral S21 bundles: unit through-calibration and zero reference. -A neutral pair is a calibration set carrying unit S21 (1+0j) and a reference set -carrying zero S21. The C++ through-calibrator divides measured/calibration and the -reference is subtracted, so applying both leaves the measured S21 untouched. That -lets an operator start the pipeline before any real calibration exists, which the -required-asset check in `_start_run` would otherwise refuse. - -The shortcut is enabled for verified axes: Kamil ADC's processing grid, -synchronized multi-device LibreVNA, and the physical single LibreVNA mock. +Applying these sets leaves measurements unchanged and satisfies pipeline asset +validation before physical calibration is available. The shortcut is enabled for +verified frequency axes: Kamil ADC, multi-device LibreVNA and single-device mock. """ from __future__ import annotations @@ -22,7 +17,7 @@ from python_app.hardware_full.kamil_adc import ( KamilAdcSweepProcessor, ) from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData -from python_app.models.run_config_model import ComboModel, RunConfigModel +from python_app.models.run_config_schema import ComboModel, RunConfigModel logger = logging.getLogger(__name__) @@ -46,8 +41,7 @@ def neutral_frequency_grid_hz(config: RunConfigModel) -> np.ndarray: points = int(config.radar.sweep.points) if points < 1: raise ValueError("Neutral sets require radar.sweep.points >= 1") - # Mirrors both acquisition paths: the native collector seeds this same linspace - # and the mock backend generates it outright. + # The single/multi-device services and mock use this same linspace. return np.linspace( float(config.radar.sweep.start_hz), float(config.radar.sweep.stop_hz), @@ -107,7 +101,6 @@ def _neutral_collection( TraceData( combo=ComboKey(input=int(combo.input), output=int(combo.output)), frequency_hz=frequency_hz.copy(), - s11=np.zeros(point_count, dtype=np.complex64), s21=np.full(point_count, s21_value, dtype=np.complex64), ) for combo in combos diff --git a/python_app/workflows/object_calibration.py b/python_app/workflows/object_calibration.py new file mode 100644 index 0000000..07a1bb9 --- /dev/null +++ b/python_app/workflows/object_calibration.py @@ -0,0 +1,85 @@ +"""Object-calibration sets and the acquisition state that makes them applicable.""" + +from __future__ import annotations + +import hashlib +import struct +from typing import Any + +import numpy as np + +from python_app.models.dataset_model import ComboKey, SweepCollection +from python_app.storage.npz.serialize import validate_trace_collection + + +def geometry_signature(config: dict[str, Any]) -> dict[str, list[dict[str, int | float]]]: + """Normalize physical antenna coordinates independently of table row order.""" + return { + key: sorted(({ + identity: int(entry[identity]), + 'x_m': float(entry['x_m']), 'y_m': float(entry.get('y_m', 0.)), 'z_m': float(entry.get('z_m', 0.)), + } for entry in config.get('gpr', {}).get(key, [])), key=lambda record: record[identity]) + for key, identity in (("tx_geometry", "output_pos"), ("rx_geometry", "input_pos")) + } + + +def through_fingerprint(collection: SweepCollection) -> str: + """Hash physical pairs, float32 frequencies and complex64 S21, excluding timing.""" + validate_trace_collection(collection, require_finite=True) + digest = hashlib.sha256() + for trace in sorted(collection.traces, key=lambda item: (item.combo.input, item.combo.output)): + frequency = np.asarray(trace.frequency_hz, dtype=" dict[str, Any]: + """Freeze the hardware, geometry and through calibration used for a capture.""" + return { + "radar_key": radar_key, + "geometry": geometry_signature(config), + "through_name": config["preprocess"]["s21"]["calibration"]["set_name"].strip(), + "through_hash": through_fingerprint(through), + } + + +def validate_object_set( + collection: SweepCollection, + metadata: dict[str, Any], + config: dict[str, Any], + radar_key: str, + through: SweepCollection, +) -> None: + """Reject corrections whose acquisition state or selected pair grids differ.""" + validate_trace_collection(collection, require_finite=True) + captured = metadata.get("provenance") + if not isinstance(captured, dict): + raise ValueError("Object calibration has no capture provenance; capture it again") + current = capture_provenance(config, radar_key, through) + for field, label in ( + ("radar_key", "radar configuration"), + ("geometry", "antenna geometry"), + ("through_name", "through calibration name"), + ("through_hash", "through calibration content"), + ): + if captured.get(field) != current[field]: + raise ValueError(f"Object calibration {label} differs from the captured state") + corrections = {trace.combo: trace for trace in collection.traces} + through_traces = {trace.combo: trace for trace in through.traces} + for combo in config["run"]["combos"]: + key = ComboKey(input=combo["input"], output=combo["output"]) + if key not in corrections or key not in through_traces: + raise ValueError(f"Object calibration and through must cover input={key.input}, output={key.output}") + correction = corrections[key] + if np.any(correction.s21 == 0): + raise ValueError("Object calibration corrections must be nonzero") + if not np.array_equal( + np.asarray(correction.frequency_hz, dtype=np.float32), + np.asarray(through_traces[key].frequency_hz, dtype=np.float32), + ): + raise ValueError(f"Object calibration frequency grid differs for input={key.input}, output={key.output}") diff --git a/python_app/workflows/radar_config_variants.py b/python_app/workflows/radar_config_variants.py index eb92f81..957efc3 100644 --- a/python_app/workflows/radar_config_variants.py +++ b/python_app/workflows/radar_config_variants.py @@ -7,8 +7,7 @@ import json import logging from pathlib import Path -from python_app.models.run_config_model import RunConfigModel -from python_app.storage.npz_store import radar_key_from_config +from python_app.models.run_config_schema import RunConfigModel logger = logging.getLogger(__name__) @@ -25,7 +24,6 @@ _RADAR_SWEEP_KEYS = ( class RadarConfigVariant: """One capture-ready radar sweep variant loaded from JSON.""" - source_path: Path display_name: str config: RunConfigModel radar_key: str @@ -147,19 +145,8 @@ def _load_radar_config_variant(path: Path, *, base_config: RunConfigModel) -> Ra config_payload["radar"]["sweep"].update(sweep_payload) config = RunConfigModel.from_dict(config_payload) - radar_key = radar_key_from_config( - model_name=config.radar.model, - serial=config.radar.serial, - sweep_start_hz=config.radar.sweep.start_hz, - sweep_stop_hz=config.radar.sweep.stop_hz, - sweep_points=config.radar.sweep.points, - ifbw_hz=config.radar.sweep.if_bandwidth_hz, - power_dbm=config.radar.sweep.power_dbm, - extra_serials=config.radar_key_extra_parts() or None, - ) return RadarConfigVariant( - source_path=path, display_name=path.stem, config=config, - radar_key=radar_key, + radar_key=config.radar_key(), ) diff --git a/python_app/workflows/reference_workflow.py b/python_app/workflows/reference_workflow.py index aba4e5c..ae4f7a5 100644 --- a/python_app/workflows/reference_workflow.py +++ b/python_app/workflows/reference_workflow.py @@ -5,8 +5,8 @@ from __future__ import annotations import logging from python_app.models.dataset_model import SweepCollection -from python_app.models.run_config_model import RunConfigModel -from python_app.storage.npz_store import NpzStore +from python_app.models.run_config_schema import RunConfigModel +from python_app.storage.npz.store import NpzStore from python_app.workflows.sequential_capture_workflow import ( DEFAULT_CALIBRATION_MEDIAN_SWEEP_COUNT, SequentialCaptureSession, @@ -34,6 +34,7 @@ def capture_reference_set( session.open() while not session.is_complete(): session.capture_current_combo() - return session.finalize(store) + saved = session.finalize(store)[0] + return saved.radar_key, saved.collection finally: session.close() diff --git a/python_app/workflows/sequential_capture_workflow.py b/python_app/workflows/sequential_capture_workflow.py index 5b65959..2c67797 100644 --- a/python_app/workflows/sequential_capture_workflow.py +++ b/python_app/workflows/sequential_capture_workflow.py @@ -1,10 +1,9 @@ -"""Sequential capture workflow for preprocess asset dataset creation.""" +"""One capture session for one or more radar sweep variants.""" from __future__ import annotations from contextlib import suppress from dataclasses import dataclass -import logging import time from typing import cast @@ -12,28 +11,23 @@ import numpy as np from python_app.hardware_full.matrix_radar_service import MatrixRadarService, create_matrix_radar_service from python_app.hardware_full.single_radar_service import SingleRadarService, create_single_radar_service -from python_app.hardware_full.switch_service import SwitchService +from python_app.hardware_full.switch_service import SwitchService, select_and_settle from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData -from python_app.models.run_config_model import ComboModel, RunConfigModel -from python_app.storage.npz_store import NpzStore, radar_key_from_config +from python_app.models.run_config_schema import ComboModel, RunConfigModel +from python_app.storage.npz.store import NpzStore +from python_app.workflows.radar_config_variants import RadarConfigVariant -logger = logging.getLogger(__name__) - -MATRIX_RADAR_MANUAL_CAPTURE_KINDS = frozenset({"s21_calibration", "s11_open", "s11_short", "s11_load"}) DEFAULT_CALIBRATION_MEDIAN_SWEEP_COUNT = 5 def trace_capture_window(traces: list[TraceData]) -> tuple[int, int]: - """Return the total measured window, retaining zero when timing was not reported.""" - starts = [int(trace.capture_start_ns) for trace in traces if int(trace.capture_start_ns) > 0] - ends = [int(trace.capture_end_ns) for trace in traces if int(trace.capture_end_ns) > 0] - return min(starts, default=0), max(ends, default=0) + """Span measured trace windows, retaining zero if timing was not reported.""" + return (min((t.capture_start_ns for t in traces if t.capture_start_ns > 0), default=0), + max((t.capture_end_ns for t in traces if t.capture_end_ns > 0), default=0)) @dataclass(slots=True) class SequentialCaptureState: - """Snapshot of sequential capture progress for the GUI/controller.""" - kind: str set_name: str captured_count: int @@ -45,321 +39,178 @@ class SequentialCaptureState: supports_batch_capture: bool = True +@dataclass(frozen=True, slots=True) +class CaptureBatch: + combo: ComboModel + traces: tuple[TraceData, ...] + + @property + def display_trace(self) -> TraceData: + return self.traces[-1] + + +@dataclass(frozen=True, slots=True) +class SavedCaptureSet: + display_name: str + radar_key: str + collection: SweepCollection + + class SequentialCaptureSession: - """Manage hardware and switch stepping for full combo capture sequence.""" + """Capture each antenna pair across variants, committing a complete batch atomically.""" def __init__( - self, - config: RunConfigModel, - kind: str, - set_name: str, - *, - median_sweep_count: int = 1, + self, config: RunConfigModel, kind: str, set_name: str, *, + radar_variants: list[RadarConfigVariant] | None = None, median_sweep_count: int = 1, ) -> None: - """Create capture session for one preprocess asset set.""" - if kind not in {"s21_calibration", "s21_reference", "s11_open", "s11_short", "s11_load", "s11_reference"}: + if kind not in {"s21_calibration", "s21_reference"}: raise RuntimeError(f"Unsupported capture kind: {kind}") if not set_name: raise RuntimeError("Set name is required") - if int(median_sweep_count) < 1: + if median_sweep_count < 1: raise RuntimeError("median_sweep_count must be >= 1") - - self._config = config - self._kind = kind - self._set_name = set_name - self._median_sweep_count = int(median_sweep_count) - self._is_matrix_radar = config.is_matrix_radar - self._manual_matrix_radar_capture = ( - self._is_matrix_radar and kind in MATRIX_RADAR_MANUAL_CAPTURE_KINDS - ) + self.kind, self.set_name = kind, set_name + self._config, self._median_sweep_count = config, median_sweep_count + self._variants = ([RadarConfigVariant("Current radar", config, config.radar_key())] + if radar_variants is None else list(radar_variants)) + if not self._variants: + raise RuntimeError("At least one radar variant is required") self._combos = config.build_runtime_combos() if not self._combos: raise RuntimeError("No switch combinations available for capture") - - self._traces: list[TraceData] = [] - self._next_index = 0 + self._full_matrix = config.is_multi_device and kind == "s21_reference" + self._batches: list[CaptureBatch] = [] + self._traces: dict[str, list[TraceData]] = {v.radar_key: [] for v in self._variants} self._opened = False - - logger.info( - "Sequential capture session created: kind=%s set=%s combos=%d matrix_radar=%s median_sweeps=%d", - self._kind, - self._set_name, - len(self._combos), - self._is_matrix_radar, - self._median_sweep_count, - ) - + self._sweep = self._variants[0].config.radar.sweep self._radar: MatrixRadarService | SingleRadarService - if self._is_matrix_radar: + if config.is_multi_device: self._radar = create_matrix_radar_service(config) - self._input_switch = None - self._output_switch = None + self._input_switch = self._output_switch = None else: self._radar = create_single_radar_service(config) self._input_switch = SwitchService.from_model(config.input_switch) self._output_switch = SwitchService.from_model(config.output_switch) - @property - def kind(self) -> str: - """Return canonical preprocess asset key for this capture session.""" - return self._kind - - @property - def set_name(self) -> str: - """Return destination set name.""" - return self._set_name - def open(self) -> None: - """Open radar and switch resources.""" if self._opened: return self._opened = True try: self._radar.open() - self._radar.configure(self._config.radar.sweep) - if self._input_switch is not None: - self._input_switch.open() - if self._output_switch is not None: - self._output_switch.open() - logger.info("Sequential capture session opened (kind=%s set=%s)", self._kind, self._set_name) + self._radar.configure(self._sweep) + for switch in (self._input_switch, self._output_switch): + if switch is not None: + switch.open() except Exception: - logger.exception("Failed to open sequential capture session (kind=%s set=%s)", self._kind, self._set_name) self.close() raise def close(self) -> None: - """Close all opened hardware resources.""" - was_open = self._opened for resource in (self._output_switch, self._input_switch, self._radar): if resource is not None: with suppress(Exception): resource.close() self._opened = False - if was_open: - logger.info("Sequential capture session closed (kind=%s set=%s)", self._kind, self._set_name) def state(self) -> SequentialCaptureState: - """Return current progress snapshot.""" - current_combo = self._current_combo() + complete = self.is_complete() return SequentialCaptureState( - kind=self._kind, - set_name=self._set_name, - captured_count=len(self._traces), - total_count=len(self._combos), - current_combo=current_combo, - can_undo=bool(self._traces), - is_complete=self.is_complete(), - supports_batch_capture=not self._manual_matrix_radar_capture, + self.kind, self.set_name, len(self._combos) if complete else len(self._batches), + len(self._combos), None if complete else self._combos[len(self._batches)], + bool(self._batches), complete, len(self._variants), + not (self._config.is_multi_device and self.kind == "s21_calibration"), ) - def capture_current_combo(self) -> TraceData: - """Capture one trace for current combo and advance sequence cursor.""" - if not self._opened: - raise RuntimeError("Capture session is not opened") - combo = self._current_combo() - if combo is None: - raise RuntimeError("Capture session is already complete") - - if self._is_matrix_radar: - radar = cast(MatrixRadarService, self._radar) - if self._manual_matrix_radar_capture: - # Only this combo is kept, so acquire the smallest collection that - # contains it instead of the full (switch-widened) matrix. - per_sweep_traces = [ - select_trace_for_combo( - acquire_matrix_combo_collection(radar, combo), combo - ) - for _ in range(self._median_sweep_count) - ] - trace = combine_traces_via_median(per_sweep_traces) - self._traces.append(trace) - self._next_index += 1 - logger.debug("Captured matrix combo input=%d output=%d", combo.input, combo.output) - return trace - - collections: list[SweepCollection] = [] - for _ in range(self._median_sweep_count): - collection = radar.acquire_collection(collection_id=1) - if not collection.traces: - raise RuntimeError("Matrix radar capture returned no traces") - collections.append(collection) - combined_collection = combine_collections_via_median(collections) - self._traces.extend(combined_collection.traces) - self._next_index = len(self._combos) - logger.info("Captured full matrix combo set (%d traces)", len(combined_collection.traces)) - return combined_collection.traces[-1] - - if self._input_switch is None or self._output_switch is None: - raise RuntimeError("Switches are not initialised for combo capture") - self._output_switch.switch_to(combo.output) - self._input_switch.switch_to(combo.input) - if self._config.runtime.settling_ms > 0: - time.sleep(self._config.runtime.settling_ms / 1000.0) - trace = capture_single_radar_combo(cast(SingleRadarService, self._radar), combo, self._median_sweep_count) - self._traces.append(trace) - self._next_index += 1 - logger.debug( - "Captured combo input=%d output=%d (%d/%d)", - combo.input, - combo.output, - self._next_index, - len(self._combos), - ) - return trace - - def undo_last_capture(self) -> TraceData: - """Remove the most recently captured trace and rewind cursor by one combo.""" - if not self._opened: - raise RuntimeError("Capture session is not opened") - if not self._traces or self._next_index <= 0: - raise RuntimeError("No captured combo is available to undo") - - if self._is_matrix_radar and not self._manual_matrix_radar_capture: - if len(self._traces) != len(self._combos): - raise RuntimeError("Capture session state is inconsistent; matrix radar trace matrix is incomplete") - removed_trace = self._traces[-1] - self._traces.clear() - self._next_index = 0 - logger.info("Undid matrix combo set capture (kind=%s set=%s)", self._kind, self._set_name) - return removed_trace - - expected_combo = self._combos[self._next_index - 1] - removed_trace = self._traces[-1] - if ( - int(removed_trace.combo.input) != int(expected_combo.input) - or int(removed_trace.combo.output) != int(expected_combo.output) - ): - raise RuntimeError("Capture session state is inconsistent; last trace does not match rewind combo") - self._next_index -= 1 - self._traces.pop() - logger.debug( - "Undid combo capture input=%d output=%d (%d/%d remaining)", - expected_combo.input, - expected_combo.output, - self._next_index, - len(self._combos), - ) - return removed_trace - - def last_captured_trace(self) -> TraceData | None: - """Return the most recently captured trace, if any.""" - if not self._traces: - return None - return self._traces[-1] - - def captured_traces(self) -> list[TraceData]: - """Return captured traces in acquisition order.""" - return list(self._traces) - def is_complete(self) -> bool: - """Return `True` when all combos were captured.""" - return self._next_index >= len(self._combos) + return bool(self._batches) if self._full_matrix else len(self._batches) == len(self._combos) - def finalize(self, store: NpzStore) -> tuple[str, SweepCollection]: - """Persist completed capture into store and return radar key + collection.""" + def capture_current_combo(self) -> CaptureBatch: + if not self._opened: + raise RuntimeError("Capture session is not opened") + combo = self.state().current_combo + if combo is None: + raise RuntimeError("Capture session is already complete") + if not self._config.is_multi_device: + select_and_settle(self._output_switch, self._input_switch, combo.output, combo.input, + self._config.runtime.settling_ms) + pending: dict[str, list[TraceData]] = {} + for variant in self._variants: + if variant.config.radar.sweep != self._sweep: + self._radar.configure(variant.config.radar.sweep) + self._sweep = variant.config.radar.sweep + if self._config.runtime.settling_ms > 0: + time.sleep(self._config.runtime.settling_ms / 1000.0) + if self._config.is_multi_device: + radar = cast(MatrixRadarService, self._radar) + if self._full_matrix: + collections = [radar.acquire_collection() for _ in range(self._median_sweep_count)] + traces = combine_collections_via_median(collections).traces + if not traces: + raise RuntimeError("Matrix radar capture returned no traces") + else: + traces = [combine_traces_via_median([ + select_trace_for_combo(radar.acquire_collection(combo=ComboKey(combo.input, combo.output)), combo) + for _ in range(self._median_sweep_count)])] + else: + traces = [capture_single_radar_combo(cast(SingleRadarService, self._radar), + combo, self._median_sweep_count)] + pending[variant.radar_key] = traces + for key, traces in pending.items(): + self._traces[key].extend(traces) + batch = CaptureBatch(combo, tuple(traces[-1] for traces in pending.values())) + self._batches.append(batch) + return batch + + def undo_last_capture(self) -> None: + if not self._opened or not self._batches: + raise RuntimeError("No captured combo is available to undo") + for traces in self._traces.values(): + del traces[-(len(self._combos) if self._full_matrix else 1):] + self._batches.pop() + + def last_captured_trace(self) -> TraceData | None: + return self._batches[-1].display_trace if self._batches else None + + def captured_batches(self) -> list[CaptureBatch]: + return list(self._batches) + + def finalize(self, store: NpzStore) -> list[SavedCaptureSet]: if not self.is_complete(): raise RuntimeError("Capture session is not complete") - - capture_start_ns, capture_end_ns = trace_capture_window(self._traces) - collection = SweepCollection( - collection_id=1, - monotonic_ns=time.monotonic_ns(), - traces=list(self._traces), - capture_start_ns=capture_start_ns, - capture_end_ns=capture_end_ns, - ) - radar_key = radar_key_from_config( - model_name=self._config.radar.model, - serial=self._config.radar.serial, - sweep_start_hz=self._config.radar.sweep.start_hz, - sweep_stop_hz=self._config.radar.sweep.stop_hz, - sweep_points=self._config.radar.sweep.points, - ifbw_hz=self._config.radar.sweep.if_bandwidth_hz, - power_dbm=self._config.radar.sweep.power_dbm, - extra_serials=self._config.radar_key_extra_parts() or None, - ) - store.save_set(self._kind, radar_key, self._set_name, collection) - logger.info( - "Finalized capture set kind=%s set=%s radar_key=%s traces=%d", - self._kind, - self._set_name, - radar_key, - len(collection.traces), - ) - return radar_key, collection - - def _current_combo(self) -> ComboModel | None: - """Return next combo to capture, or `None` if session is complete.""" - if self._next_index >= len(self._combos): - return None - return self._combos[self._next_index] + saved = [] + for variant in self._variants: + traces = list(self._traces[variant.radar_key]) + start, end = trace_capture_window(traces) + collection = SweepCollection(1, time.monotonic_ns(), traces, start, end) + store.save_set(self.kind, variant.radar_key, self.set_name, collection) + saved.append(SavedCaptureSet(variant.display_name, variant.radar_key, collection)) + return saved -def capture_single_radar_combo( - radar: SingleRadarService, combo: ComboModel, median_sweep_count: int, -) -> TraceData: - """Capture the selected antenna, discarding buffered data from the previous one.""" - set_active_combo = getattr(radar, "set_active_combo", None) - if callable(set_active_combo): - set_active_combo(ComboKey(combo.input, combo.output)) - # Streaming Kamil sweeps may predate or straddle the switch transition. - drain_after_switch = getattr(radar, "drain_after_switch", None) - if callable(drain_after_switch): - drain_after_switch() +def capture_single_radar_combo(radar: SingleRadarService, combo: ComboModel, count: int) -> TraceData: + radar.prepare_combo(ComboKey(combo.input, combo.output)) traces = [] - for _ in range(median_sweep_count): - sweep_start_ns = time.monotonic_ns() + for _ in range(count): + start = time.monotonic_ns() sweep = radar.acquire() traces.append(TraceData( - combo=ComboKey(combo.input, combo.output), - frequency_hz=np.asarray(sweep.x, dtype=np.float32), - s11=np.asarray(sweep.trace("s11"), dtype=np.complex64), - s21=np.asarray(sweep.trace("s21"), dtype=np.complex64), - capture_start_ns=sweep.capture_start_ns or sweep_start_ns, - capture_end_ns=sweep.capture_end_ns or time.monotonic_ns(), + ComboKey(combo.input, combo.output), np.asarray(sweep.x, dtype=np.float32), + np.asarray(sweep.trace("s21"), dtype=np.complex64), + sweep.capture_start_ns or start, sweep.capture_end_ns or time.monotonic_ns(), )) return combine_traces_via_median(traces) -def acquire_matrix_combo_collection( - radar: MatrixRadarService, - combo: ComboModel, - collection_id: int = 1, -) -> SweepCollection: - """Acquire the smallest matrix collection that contains one combo. - - A switch-widened matrix radar (``SwitchedMatrixRadarService``) can acquire just - the physical switch step carrying the combo, which is several times faster than - the full matrix and keeps the per-combo capture UI responsive. Plain matrix - radars expose only full-matrix acquisition, so they fall back to it. - """ - acquire_combo = getattr(radar, "acquire_combo_collection", None) - if callable(acquire_combo): - collection = cast(SweepCollection, acquire_combo( - input_pos=int(combo.input), - output_pos=int(combo.output), - collection_id=collection_id, - )) - else: - collection = radar.acquire_collection(collection_id=collection_id) - if not collection.traces: - raise RuntimeError("Matrix radar capture returned no traces") - return collection - - def select_trace_for_combo(collection: SweepCollection, combo: ComboModel) -> TraceData: - """Return the trace matching a virtual combo from a full multi-device capture.""" for trace in collection.traces: - if ( - int(trace.combo.input) == int(combo.input) - and int(trace.combo.output) == int(combo.output) - ): + if trace.combo == ComboKey(combo.input, combo.output): return trace raise RuntimeError(f"Multi-device capture is missing trace for input={combo.input}, output={combo.output}") def combine_traces_via_median(traces: list[TraceData]) -> TraceData: - """Return one trace whose S11/S21 are the per-point median of the inputs. + """Return one trace whose S21 is the per-point median of the inputs. A single-element input is returned unchanged. With multiple inputs, the real and imaginary parts of each complex sample are medianed independently so a @@ -392,11 +243,7 @@ def combine_traces_via_median(traces: list[TraceData]) -> TraceData: if not np.array_equal(trace.frequency_hz, first.frequency_hz): raise RuntimeError(f"Median combine frequency-axis mismatch at sweep {index}") - s11_stack = np.stack([np.asarray(t.s11, dtype=np.complex64) for t in traces], axis=0) s21_stack = np.stack([np.asarray(t.s21, dtype=np.complex64) for t in traces], axis=0) - s11_median = ( - np.median(s11_stack.real, axis=0) + 1j * np.median(s11_stack.imag, axis=0) - ).astype(np.complex64) s21_median = ( np.median(s21_stack.real, axis=0) + 1j * np.median(s21_stack.imag, axis=0) ).astype(np.complex64) @@ -405,7 +252,6 @@ def combine_traces_via_median(traces: list[TraceData]) -> TraceData: return TraceData( combo=ComboKey(input=int(combo.input), output=int(combo.output)), frequency_hz=np.asarray(first.frequency_hz, dtype=np.float32), - s11=s11_median, s21=s21_median, capture_start_ns=capture_start_ns, capture_end_ns=capture_end_ns, diff --git a/requirements-cuda.txt b/requirements-cuda.txt deleted file mode 100644 index c489ba1..0000000 --- a/requirements-cuda.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Required for SAR3D on x86_64. Jetson uses JetPack's system CUDA instead. -nvidia-cuda-nvrtc==13.3.33; platform_machine == "x86_64" -nvidia-cuda-runtime==13.3.29; platform_machine == "x86_64" diff --git a/requirements.txt b/requirements.txt index 31f08a2..34ff4b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,10 @@ pyserial>=3.5,<4 PyQt6>=6.6,<6.8; platform_machine == "aarch64" PyQt6>=6.6; platform_machine != "aarch64" pyqtgraph>=0.13.7 -rpi-hardware-pwm>=0.2.2,<1 fastapi>=0.110 # embedded web UI (browser view + controls), served in-process uvicorn[standard]>=0.29 # ASGI server for the web UI, run on a background thread +# nvrtc.h, cuda.h and libnvrtc for the sar3d processor's GPU code (kernels are +# compiled at run time, no nvcc). x86_64 only: on Jetson (aarch64) CUDA comes +# with JetPack at /usr/local/cuda, where the Makefile finds it. +nvidia-cuda-nvrtc==13.3.33; platform_machine == "x86_64" # nvrtc.h + libnvrtc +nvidia-cuda-runtime==13.3.29; platform_machine == "x86_64" # cuda.h (Driver API) diff --git a/run_config.json b/run_config.json index b8134f4..e49a8d5 100644 --- a/run_config.json +++ b/run_config.json @@ -3,7 +3,6 @@ "model": "librevna_multi", "serial": "206830985532", "driver_mode": "mock", - "mock_signal_hz": 5000000.0, "multi_device": { "slave_serials": [ "206930965532", @@ -77,26 +76,22 @@ "port1": { "name": "port1", "driver_mode": "mock", - "driver": "h7992", "radar_port": 1, "positions": 4, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 17, - "pin_b": 27, - "invert_logic": false + "pin_b": 27 }, "port2": { "name": "port2", "driver_mode": "mock", - "driver": "h7992", "radar_port": 2, "positions": 4, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 22, - "pin_b": 23, - "invert_logic": false + "pin_b": 23 } }, "control_button": { @@ -203,26 +198,6 @@ "bundle_path": "" } }, - "s11": { - "calibration": { - "open": { - "set_name": "", - "bundle_path": "" - }, - "short": { - "set_name": "", - "bundle_path": "" - }, - "load": { - "set_name": "", - "bundle_path": "" - } - }, - "reference": { - "set_name": "", - "bundle_path": "" - } - }, "notch": { "enabled": true, "bands_hz": [], @@ -248,13 +223,13 @@ { "output_pos": 2, "x_m": 0.905, - "y_m": 0.0, + "y_m": 0.445, "z_m": 0.0 }, { "output_pos": 3, "x_m": -0.905, - "y_m": 0.0, + "y_m": 0.445, "z_m": 0.0 } ], @@ -268,7 +243,7 @@ { "input_pos": 1, "x_m": 0.485, - "y_m": 0.0, + "y_m": 0.445, "z_m": 0.0 }, { @@ -280,7 +255,7 @@ { "input_pos": 3, "x_m": 0.185, - "y_m": 0.0, + "y_m": 0.445, "z_m": 0.0 } ] @@ -311,5 +286,16 @@ "capacity": 50, "slot_size_bytes": 2097152 } + }, + "gui": { + "processing": { + "bscan": { + "cut_m": 0.0, + "max_depth_m": 8.0 + } + }, + "data_actions": { + "save_path": "python_app/data/snapshots" + } } } diff --git a/run_config_examples/run_config_kamil_adc.example.json b/run_config_examples/run_config_kamil_adc.example.json index 902b6d2..9f35c68 100644 --- a/run_config_examples/run_config_kamil_adc.example.json +++ b/run_config_examples/run_config_kamil_adc.example.json @@ -81,26 +81,22 @@ "port1": { "name": "port1", "driver_mode": "native", - "driver": "h7992", "radar_port": 1, "positions": 2, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 17, - "pin_b": 27, - "invert_logic": false + "pin_b": 27 }, "port2": { "name": "port2", "driver_mode": "native", - "driver": "h7992", "radar_port": 2, "positions": 4, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 22, - "pin_b": 23, - "invert_logic": false + "pin_b": 23 } }, "control_button": { @@ -172,26 +168,6 @@ "bundle_path": "" } }, - "s11": { - "calibration": { - "open": { - "set_name": "", - "bundle_path": "" - }, - "short": { - "set_name": "", - "bundle_path": "" - }, - "load": { - "set_name": "", - "bundle_path": "" - } - }, - "reference": { - "set_name": "", - "bundle_path": "" - } - }, "notch": { "enabled": true, "bands_hz": [], diff --git a/run_config_examples/run_config_librevna.example.json b/run_config_examples/run_config_librevna.example.json index ba2086a..2b77d0a 100644 --- a/run_config_examples/run_config_librevna.example.json +++ b/run_config_examples/run_config_librevna.example.json @@ -57,26 +57,22 @@ "port1": { "name": "port1", "driver_mode": "native", - "driver": "h7992", "radar_port": 1, "positions": 2, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 17, - "pin_b": 27, - "invert_logic": false + "pin_b": 27 }, "port2": { "name": "port2", "driver_mode": "native", - "driver": "h7992", "radar_port": 2, "positions": 4, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 22, - "pin_b": 23, - "invert_logic": false + "pin_b": 23 } }, "control_button": { @@ -148,26 +144,6 @@ "bundle_path": "" } }, - "s11": { - "calibration": { - "open": { - "set_name": "", - "bundle_path": "" - }, - "short": { - "set_name": "", - "bundle_path": "" - }, - "load": { - "set_name": "", - "bundle_path": "" - } - }, - "reference": { - "set_name": "", - "bundle_path": "" - } - }, "notch": { "enabled": true, "bands_hz": [], diff --git a/run_config_examples/run_config_librevna_multi.example.json b/run_config_examples/run_config_librevna_multi.example.json index e59cba7..2f5c898 100644 --- a/run_config_examples/run_config_librevna_multi.example.json +++ b/run_config_examples/run_config_librevna_multi.example.json @@ -1,6 +1,6 @@ { "radar": { - "model": "librevna_multi", + "model": "librevna_multi", "serial": "209E307A5532", "driver_mode": "native", "multi_device": { @@ -60,26 +60,22 @@ "port1": { "name": "port1", "driver_mode": "mock", - "driver": "h7992", "radar_port": 1, "positions": 2, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 17, - "pin_b": 27, - "invert_logic": false + "pin_b": 27 }, "port2": { "name": "port2", "driver_mode": "mock", - "driver": "h7992", "radar_port": 2, "positions": 4, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 22, - "pin_b": 23, - "invert_logic": false + "pin_b": 23 } }, "control_button": { @@ -151,26 +147,6 @@ "bundle_path": "" } }, - "s11": { - "calibration": { - "open": { - "set_name": "", - "bundle_path": "" - }, - "short": { - "set_name": "", - "bundle_path": "" - }, - "load": { - "set_name": "", - "bundle_path": "" - } - }, - "reference": { - "set_name": "", - "bundle_path": "" - } - }, "notch": { "enabled": true, "bands_hz": [], diff --git a/run_config_examples/run_config_librevna_multi.example2.json b/run_config_examples/run_config_librevna_multi.example2.json index 2b13b39..03bb3f6 100644 --- a/run_config_examples/run_config_librevna_multi.example2.json +++ b/run_config_examples/run_config_librevna_multi.example2.json @@ -1,6 +1,6 @@ { "radar": { - "model": "librevna_multi", + "model": "librevna_multi", "serial": "206830985532", "driver_mode": "native", "multi_device": { @@ -60,26 +60,22 @@ "port1": { "name": "port1", "driver_mode": "mock", - "driver": "h7992", "radar_port": 1, "positions": 2, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 17, - "pin_b": 27, - "invert_logic": false + "pin_b": 27 }, "port2": { "name": "port2", "driver_mode": "mock", - "driver": "h7992", "radar_port": 2, "positions": 4, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 22, - "pin_b": 23, - "invert_logic": false + "pin_b": 23 } }, "control_button": { @@ -151,26 +147,6 @@ "bundle_path": "" } }, - "s11": { - "calibration": { - "open": { - "set_name": "", - "bundle_path": "" - }, - "short": { - "set_name": "", - "bundle_path": "" - }, - "load": { - "set_name": "", - "bundle_path": "" - } - }, - "reference": { - "set_name": "", - "bundle_path": "" - } - }, "notch": { "enabled": true, "bands_hz": [], diff --git a/run_config_examples/run_config_librevna_multi_per_point.example.json b/run_config_examples/run_config_librevna_multi_per_point.example.json index d315de7..448899c 100644 --- a/run_config_examples/run_config_librevna_multi_per_point.example.json +++ b/run_config_examples/run_config_librevna_multi_per_point.example.json @@ -76,26 +76,22 @@ "port1": { "name": "port1", "driver_mode": "mock", - "driver": "h7992", "radar_port": 1, "positions": 4, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 17, - "pin_b": 27, - "invert_logic": false + "pin_b": 27 }, "port2": { "name": "port2", "driver_mode": "mock", - "driver": "h7992", "radar_port": 2, "positions": 4, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 22, - "pin_b": 23, - "invert_logic": false + "pin_b": 23 } }, "control_button": { @@ -202,26 +198,6 @@ "bundle_path": "" } }, - "s11": { - "calibration": { - "open": { - "set_name": "", - "bundle_path": "" - }, - "short": { - "set_name": "", - "bundle_path": "" - }, - "load": { - "set_name": "", - "bundle_path": "" - } - }, - "reference": { - "set_name": "", - "bundle_path": "" - } - }, "notch": { "enabled": true, "bands_hz": [], diff --git a/run_config_examples/run_config_librevna_multi_per_point_lowband.example.json b/run_config_examples/run_config_librevna_multi_per_point_lowband.example.json index fe5a6ea..e3d3291 100644 --- a/run_config_examples/run_config_librevna_multi_per_point_lowband.example.json +++ b/run_config_examples/run_config_librevna_multi_per_point_lowband.example.json @@ -76,26 +76,22 @@ "port1": { "name": "port1", "driver_mode": "mock", - "driver": "h7992", "radar_port": 1, "positions": 4, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 17, - "pin_b": 27, - "invert_logic": false + "pin_b": 27 }, "port2": { "name": "port2", "driver_mode": "mock", - "driver": "h7992", "radar_port": 2, "positions": 4, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 22, - "pin_b": 23, - "invert_logic": false + "pin_b": 23 } }, "control_button": { @@ -202,26 +198,6 @@ "bundle_path": "" } }, - "s11": { - "calibration": { - "open": { - "set_name": "", - "bundle_path": "" - }, - "short": { - "set_name": "", - "bundle_path": "" - }, - "load": { - "set_name": "", - "bundle_path": "" - } - }, - "reference": { - "set_name": "", - "bundle_path": "" - } - }, "notch": { "enabled": true, "bands_hz": [], diff --git a/run_config_examples/run_config_replay_vna.example.json b/run_config_examples/run_config_replay_vna.example.json index 6677359..55b5678 100644 --- a/run_config_examples/run_config_replay_vna.example.json +++ b/run_config_examples/run_config_replay_vna.example.json @@ -1,5 +1,5 @@ { - "_comment": "Replay of the recorded VNA_AlGo tank005 dataset through the full pipeline (no hardware). The replay producer feeds the raw ring; pair it with NEUTRAL preprocess sets (calibration=1, reference=0) so the already-preprocessed recording passes through unchanged. Create them via GUI 'Create neutral sets' (name: neutral_replay) or python -m python_app.scripts.create_neutral_preprocess_sets --config .", + "_comment": "Replay of the recorded VNA_AlGo tank005 dataset through the full pipeline (no hardware). The replay producer feeds the raw ring; pair it with NEUTRAL preprocess sets (calibration=1, reference=0) so the already-preprocessed recording passes through unchanged. Create them with the GUI button 'Create Neutral S21 Sets' (name: neutral_replay).", "radar": { "model": "librevna_multi", "serial": "206830985532", @@ -32,26 +32,22 @@ "port1": { "name": "port1", "driver_mode": "mock", - "driver": "h7992", "radar_port": 1, "positions": 2, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 17, - "pin_b": 27, - "invert_logic": false + "pin_b": 27 }, "port2": { "name": "port2", "driver_mode": "mock", - "driver": "h7992", "radar_port": 2, "positions": 4, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 22, - "pin_b": 23, - "invert_logic": false + "pin_b": 23 } }, "control_button": { @@ -78,14 +74,38 @@ "logger_name": "locator_runtime" }, "combos": [ - { "input": 0, "output": 0 }, - { "input": 1, "output": 0 }, - { "input": 2, "output": 0 }, - { "input": 3, "output": 0 }, - { "input": 0, "output": 1 }, - { "input": 1, "output": 1 }, - { "input": 2, "output": 1 }, - { "input": 3, "output": 1 } + { + "input": 0, + "output": 0 + }, + { + "input": 1, + "output": 0 + }, + { + "input": 2, + "output": 0 + }, + { + "input": 3, + "output": 0 + }, + { + "input": 0, + "output": 1 + }, + { + "input": 1, + "output": 1 + }, + { + "input": 2, + "output": 1 + }, + { + "input": 3, + "output": 1 + } ] }, "preprocess": { @@ -99,14 +119,6 @@ "bundle_path": "" } }, - "s11": { - "calibration": { - "open": { "set_name": "", "bundle_path": "" }, - "short": { "set_name": "", "bundle_path": "" }, - "load": { "set_name": "", "bundle_path": "" } - }, - "reference": { "set_name": "", "bundle_path": "" } - }, "notch": { "enabled": false, "bands_hz": [], @@ -117,14 +129,44 @@ "gpr": { "relative_permittivity": 1.0, "tx_geometry": [ - { "output_pos": 0, "x_m": 0.785, "y_m": 0.0, "z_m": 0.0 }, - { "output_pos": 1, "x_m": -0.785, "y_m": 0.0, "z_m": 0.0 } + { + "output_pos": 0, + "x_m": 0.785, + "y_m": 0.0, + "z_m": 0.0 + }, + { + "output_pos": 1, + "x_m": -0.785, + "y_m": 0.0, + "z_m": 0.0 + } ], "rx_geometry": [ - { "input_pos": 0, "x_m": -0.42, "y_m": 0.0, "z_m": 0.0 }, - { "input_pos": 1, "x_m": 0.48, "y_m": 0.0, "z_m": 0.0 }, - { "input_pos": 2, "x_m": 0.18, "y_m": 0.0, "z_m": 0.0 }, - { "input_pos": 3, "x_m": -0.12, "y_m": 0.0, "z_m": 0.0 } + { + "input_pos": 0, + "x_m": -0.42, + "y_m": 0.0, + "z_m": 0.0 + }, + { + "input_pos": 1, + "x_m": 0.48, + "y_m": 0.0, + "z_m": 0.0 + }, + { + "input_pos": 2, + "x_m": 0.18, + "y_m": 0.0, + "z_m": 0.0 + }, + { + "input_pos": 3, + "x_m": -0.12, + "y_m": 0.0, + "z_m": 0.0 + } ] }, "rings": { diff --git a/run_config_examples/run_config_simulator.example.json b/run_config_examples/run_config_simulator.example.json index 1ba15ad..ab1a481 100644 --- a/run_config_examples/run_config_simulator.example.json +++ b/run_config_examples/run_config_simulator.example.json @@ -57,26 +57,22 @@ "port1": { "name": "port1", "driver_mode": "mock", - "driver": "h7992", "radar_port": 1, "positions": 2, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 17, - "pin_b": 27, - "invert_logic": false + "pin_b": 27 }, "port2": { "name": "port2", "driver_mode": "mock", - "driver": "h7992", "radar_port": 2, "positions": 4, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 22, - "pin_b": 23, - "invert_logic": false + "pin_b": 23 } }, "control_button": { @@ -148,26 +144,6 @@ "bundle_path": "" } }, - "s11": { - "calibration": { - "open": { - "set_name": "", - "bundle_path": "" - }, - "short": { - "set_name": "", - "bundle_path": "" - }, - "load": { - "set_name": "", - "bundle_path": "" - } - }, - "reference": { - "set_name": "", - "bundle_path": "" - } - }, "notch": { "enabled": true, "bands_hz": [], @@ -201,7 +177,7 @@ { "input_pos": 1, "x_m": 0.485, - "y_m": 0.0, + "y_m": 0.445, "z_m": 0.0 }, { @@ -213,7 +189,7 @@ { "input_pos": 3, "x_m": 0.185, - "y_m": 0.0, + "y_m": 0.445, "z_m": 0.0 } ] @@ -327,7 +303,6 @@ } }, "data_actions": { - "save_count": 10, "save_path": "python_app/data/snapshots", "save_name": "snapshot_simulator" }, diff --git a/run_configs/run_config.json b/run_configs/run_config.json index 28c09d8..9fd9947 100644 --- a/run_configs/run_config.json +++ b/run_configs/run_config.json @@ -3,7 +3,6 @@ "model": "librevna", "serial": "", "driver_mode": "mock", - "mock_signal_hz": 5000000.0, "multi_device": { "slave_serials": [], "force_external_reference": false, @@ -58,26 +57,22 @@ "port1": { "name": "port1", "driver_mode": "mock", - "driver": "h7992", "radar_port": 1, "positions": 2, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 17, - "pin_b": 27, - "invert_logic": false + "pin_b": 27 }, "port2": { "name": "port2", "driver_mode": "mock", - "driver": "h7992", "radar_port": 2, "positions": 4, "default_position": 0, "gpio_chip": "/dev/gpiochip0", "pin_a": 22, - "pin_b": 23, - "invert_logic": false + "pin_b": 23 } }, "control_button": { @@ -149,26 +144,6 @@ "bundle_path": "" } }, - "s11": { - "calibration": { - "open": { - "set_name": "", - "bundle_path": "" - }, - "short": { - "set_name": "", - "bundle_path": "" - }, - "load": { - "set_name": "", - "bundle_path": "" - } - }, - "reference": { - "set_name": "", - "bundle_path": "" - } - }, "notch": { "enabled": true, "bands_hz": [], @@ -328,7 +303,6 @@ } }, "data_actions": { - "save_count": 10, "save_path": "python_app/data/snapshots", "save_name": "snapshot_simulator" }, diff --git a/start.sh b/start.sh index 54c72c5..1b77bcf 100755 --- a/start.sh +++ b/start.sh @@ -219,34 +219,33 @@ ensure_system_dependencies() { done fi - if command -v pkg-config >/dev/null 2>&1 && pkg-config --exists libusb-1.0 \ - && ((qt_gui_packages_missing == 0)) \ - && python3 -c 'import ctypes; ctypes.CDLL("libatomic.so.1")' >/dev/null 2>&1; then + local runtime_check='import ctypes, ctypes.util; name = ctypes.util.find_library("usb-1.0"); assert name; ctypes.CDLL(name); ctypes.CDLL("libatomic.so.1")' + if ((qt_gui_packages_missing == 0)) && python3 -c "${runtime_check}" >/dev/null 2>&1; then return fi - echo "[start.sh] Installing system dependencies: libusb headers, libatomic and Qt xcb libraries..." + echo "[start.sh] Installing system dependencies: libusb runtime, libatomic and Qt xcb libraries..." if command -v apt-get >/dev/null 2>&1; then run_privileged apt-get update - run_privileged apt-get install -y pkg-config libusb-1.0-0-dev libatomic1 libxcb-cursor0 libxkbcommon-x11-0 + run_privileged apt-get install -y libusb-1.0-0 libatomic1 libxcb-cursor0 libxkbcommon-x11-0 elif command -v dnf >/dev/null 2>&1; then - run_privileged dnf install -y pkgconf-pkg-config libusb1-devel libatomic + run_privileged dnf install -y libusb1 libatomic elif command -v yum >/dev/null 2>&1; then - run_privileged yum install -y pkgconfig libusb1-devel libatomic + run_privileged yum install -y libusb1 libatomic elif command -v pacman >/dev/null 2>&1; then - run_privileged pacman -Sy --needed pkgconf libusb gcc-libs + run_privileged pacman -Sy --needed libusb gcc-libs elif command -v zypper >/dev/null 2>&1; then - run_privileged zypper --non-interactive install pkg-config libusb-1_0-devel libatomic1 + run_privileged zypper --non-interactive install libusb-1_0-0 libatomic1 elif command -v brew >/dev/null 2>&1; then - brew install pkg-config libusb gcc + brew install libusb gcc else echo "Could not detect supported package manager." >&2 - echo "Install manually: pkg-config and libusb development package." >&2 + echo "Install manually: libusb-1.0, libatomic and the Qt xcb runtime libraries." >&2 exit 1 fi - if ! command -v pkg-config >/dev/null 2>&1 || ! pkg-config --exists libusb-1.0; then - echo "libusb-1.0 development package is still unavailable after install attempt." >&2 + if ! python3 -c "${runtime_check}" >/dev/null 2>&1; then + echo "libusb-1.0 or libatomic runtime is still unavailable after install attempt." >&2 exit 1 fi } @@ -361,10 +360,10 @@ run_producer_only() { import os, sys from pathlib import Path from python_app.orchestration.producer_selection import acquisition_command -command = acquisition_command(Path(sys.argv[1]), Path(sys.argv[2])) +command = acquisition_command(Path(sys.argv[1])) print("[start.sh] Starting producer:", command, flush=True) os.execv(command[0], command) -' "${PROJECT_ROOT}" "${config_path}" +' "${config_path}" } stop_headless_service() { @@ -393,10 +392,9 @@ verify_cpp_binaries() { import sys from pathlib import Path from python_app.orchestration.producer_selection import acquisition_command -command = acquisition_command(Path(sys.argv[1]), Path(sys.argv[2])) -print("kamil_adc_collector" if "python_app.scripts.kamil_adc_raw_producer" in command - else "sweep_orchestrator" if len(command) == 3 else "") -' "${PROJECT_ROOT}" "${PROFILE_PATH:-${PROJECT_ROOT}/run_config.json}")" +command = acquisition_command(Path(sys.argv[1])) +print("kamil_adc_collector" if "python_app.scripts.kamil_adc_raw_producer" in command else "") +' "${PROFILE_PATH:-${PROJECT_ROOT}/run_config.json}")" required_bins="${required_bins} ${acquisition_binary}" for bin in ${required_bins}; do if [[ ! -x "${PROJECT_ROOT}/build/bin/${bin}" ]]; then @@ -427,11 +425,10 @@ main() { resolve_profile_path check_environment detect_radar_model - # Skip first-time provisioning (build headers, USB udev rule) in headless + # Skip first-time provisioning (runtime libraries, USB udev rule) in headless # mode: the daemon runs unattended at boot as a non-root user and must not # block on sudo. A fresh machine is provisioned by one interactive launch. - # `all` builds the libusb orchestrator for every model; only the USB access - # rule is device-specific. Python/C++ SHM also needs libatomic. + # LibreVNA uses Python/libusb; Python/C++ SHM needs libatomic. if ((HEADLESS == 0)); then ensure_system_dependencies if [[ "${RADAR_MODEL}" != "kamil_adc" ]]; then ensure_usb_access_rules; fi