"""Physical MIMO estimation, legacy archives and failure-preserving writes.""" from __future__ import annotations import copy import json from pathlib import Path import tempfile import unittest from unittest import mock from typing import Any import numpy as np from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData from python_app.workflows import mimo_calibration as mc class MimoCalibrationTest(unittest.TestCase): def setUp(self): self.settings = mc.Settings(input_idx=(2, 7), output_idx=(1, 5), sphere_radius_m=0., antenna_height_m=1., f_min=2e9, f_max=6e9, refine_axes=(), coarse_range_search_m=0.) self.freq = np.linspace(2e9, 6e9, 129) self.target = (.2, .25, 4.) self.tx = np.array([[-.6, 0., 0.], [.6, 0., 0.]]) self.rx = np.array([[-.3, .1, 0.], [.3, .1, 0.]]) self.config = { 'radar': {'model': 'librevna_multi', 'serial': 'master', 'multi_device': {'slave_serials': ['slave']}}, 'preprocess': {'s21': {'calibration': {'set_name': 'through'}, 'reference': {'set_name': 'empty'}}}, 'gpr': { 'tx_geometry': [dict(output_pos=port, x_m=xyz[0], y_m=xyz[1], z_m=xyz[2]) for port, xyz in zip((1, 5), self.tx)], 'rx_geometry': [dict(input_pos=port, x_m=xyz[0], y_m=xyz[1], z_m=xyz[2]) for port, xyz in zip((2, 7), self.rx)], }, 'run': {'combos': [dict(input=input_, output=output) for output in (1, 5) for input_ in (2, 7)]}, } def signal(self, target=None, *, gains=False): target = self.target if target is None else target pitch = np.radians(self.settings.pitch_deg) y0 = target[1] - self.settings.antenna_height_m radar_target = np.array([target[0], y0 * np.cos(pitch) + target[2] * np.sin(pitch), -y0 * np.sin(pitch) + target[2] * np.cos(pitch)]) vt, vr = self.tx - radar_target, self.rx - radar_target dt, dr = np.linalg.norm(vt, axis=1), np.linalg.norm(vr, axis=1) paths = dt[:, None] + dr[None, :] if self.settings.sphere_radius_m: cosine = (vt @ vr.T) / (dt[:, None] * dr[None, :]) paths -= 2 * self.settings.sphere_radius_m * np.sqrt((1 + cosine) / 2) tx_phase = np.array([0., .7])[:, None] - 2 * np.pi * self.freq * np.array([0., .02])[:, None] / mc.C_LIGHT rx_phase = np.array([0., -.5])[:, None] - 2 * np.pi * self.freq * np.array([0., -.013])[:, None] / mc.C_LIGHT signal = np.exp(-2j * np.pi * self.freq * paths[:, :, None] / mc.C_LIGHT) signal *= np.exp(1j * (tx_phase[:, None, :] + rx_phase[None, :, :] + 1.1)) if gains: signal *= np.array([1., 2.])[:, None, None] * np.array([1., .5])[None, :, None] return signal, paths def collection(self, samples=None, *, freq=None, identity=1): samples = self.signal()[0] if samples is None else samples freq = self.freq if freq is None else freq traces = [TraceData(ComboKey(input_, output), freq.copy(), np.zeros(freq.size, np.complex64), samples[t, r].copy()) for t, output in enumerate((1, 5)) for r, input_ in enumerate((2, 7))] return SweepCollection(identity, identity, traces) def calibration(self): return mc.compute_from_collections([{'collections': [self.collection()], 'target_scene_m': self.target}], self.config, settings=self.settings) def archive(self, root, collection, *, name='0000_id1_ns1'): snapshot = root / 'preprocessed' / name snapshot.mkdir(parents=True) (root / 'config_profile.json').write_text(json.dumps(self.config)) (snapshot / 'meta.json').write_text('{}') for trace in collection.traces: suffix = f'i{trace.combo.input}_o{trace.combo.output}' np.save(snapshot / f'{suffix}_s21.npy', trace.s21) np.save(snapshot / f'{suffix}_freq.npy', trace.frequency_hz) np.save(snapshot / f'{suffix}_s11.npy', trace.s11) return snapshot def test_point_and_sphere_recover_relative_antenna_phases(self): for radius in (0., .16): with self.subTest(radius=radius): self.settings.sphere_radius_m = radius signal, paths = self.signal() cal = self.calibration() corrected = mc.apply_calibration(signal, self.freq, cal) contributions = corrected * np.exp(2j * np.pi * self.freq * paths[:, :, None] / mc.C_LIGHT) phases = np.angle(contributions.mean(axis=2)) np.testing.assert_allclose(phases, np.full((2, 2), 1.1), atol=.004) np.testing.assert_allclose(np.abs(cal['pair']), 1., atol=2e-7) self.assertGreater(cal['meta']['quality']['coherence_final'][0], .9999) self.assertLess(cal['meta']['quality']['coherence_before'][0], .98) diagnostic = cal['meta']['diagnostics'][0] self.assertEqual(np.shape(diagnostic['phase_before_deg']), (2, 2)) np.testing.assert_allclose(diagnostic['phase_after_deg'], np.full((2, 2), np.degrees(1.1)), atol=.25) self.assertIsNone(cal['meta']['quality']['coherence_leave_one_out'][0]) def test_amplitude_equalization_is_explicit(self): self.settings.apply_amplitude = True signal, paths = self.signal(gains=True) cal = mc.compute_from_collections([{'collections': [self.collection(signal)], 'target_scene_m': self.target}], self.config, settings=self.settings) corrected = mc.apply_calibration(signal, self.freq, cal) # Compare amplitudes after removing the physical propagation phase. levels = np.abs((corrected * np.exp(2j * np.pi * self.freq * paths[:, :, None] / mc.C_LIGHT)).mean(axis=2)) self.assertLess(levels.max() / levels.min(), 1.01) self.assertGreater(np.max(np.abs(cal['pair'])), 1.5) def test_archive_and_live_estimation_share_the_same_math(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) signal = self.signal()[0] self.archive(root, self.collection(signal), name='0000_id1_ns1') self.archive(root, self.collection(signal * 2), name='0001_id2_ns2') archived = mc.compute_calibration([{'path': root / 'preprocessed', 'target_scene_m': self.target}], settings=self.settings) live = mc.compute_from_collections([{'collections': [self.collection(signal), self.collection(signal * 2)], 'target_scene_m': self.target}], self.config, settings=self.settings) np.testing.assert_array_equal(archived['pair'], live['pair']) def test_frequency_geometry_channel_and_signal_failures_are_explicit(self): cases = [] incomplete = self.collection(); incomplete.traces.pop(); cases.append(incomplete) duplicate = self.collection(); duplicate.traces.append(duplicate.traces[0]); cases.append(duplicate) mismatched = self.collection(); mismatched.traces[0].frequency_hz += 10000; cases.append(mismatched) nonuniform = self.collection(); nonuniform.traces[0].frequency_hz[20] += 1e6; cases.append(nonuniform) zero = self.collection(np.zeros((2, 2, self.freq.size), complex)); cases.append(zero) invalid = self.collection(); invalid.traces[0].s21[0] = np.nan; cases.append(invalid) for index, collection in enumerate(cases): with self.subTest(collection=index): with self.assertRaises(ValueError): mc.compute_from_collections([{'collections': [collection], 'target_scene_m': self.target}], self.config, settings=self.settings) with self.assertRaisesRegex(ValueError, 'at least one'): mc.compute_from_collections([], self.config, settings=self.settings) def test_config_checks_phase_base_and_geometry_but_allows_new_reference(self): cal = self.calibration() updated = copy.deepcopy(self.config) updated['preprocess']['s21']['reference']['set_name'] = 'new_background' mc.validate_for_config(cal, updated) for edit in ('geometry', 'through', 'combo', 'serial'): updated = copy.deepcopy(self.config) if edit == 'geometry': updated['gpr']['tx_geometry'][0]['x_m'] += .01 elif edit == 'through': updated['preprocess']['s21']['calibration']['set_name'] = 'new_through' elif edit == 'serial': updated['radar']['serial'] = 'another_device' else: updated['run']['combos'].append({'input': 999, 'output': 1}) with self.subTest(edit=edit), self.assertRaises(ValueError): mc.validate_for_config(cal, updated) def test_object_corrected_sources_are_rejected_for_live_and_archives(self): for field in ('set_name', 'bundle_path'): self.config['preprocess']['s21']['object_calibration'] = {field: 'active'} with self.subTest(field=field), self.assertRaisesRegex(ValueError, 'already active'): self.calibration() with tempfile.TemporaryDirectory() as directory: root = Path(directory) self.archive(root, self.collection()) with self.assertRaisesRegex(ValueError, 'already active'): mc.compute_calibration([{'path': root / 'preprocessed', 'target_scene_m': self.target}], settings=self.settings) def test_cancel_is_checked_inside_refinement(self): self.settings.refine_axes = ('z',) state = {'estimating': False, 'checks': 0} def progress(message): state['estimating'] |= message.startswith('Estimating') def cancelled(): if state['estimating']: state['checks'] += 1 return state['checks'] >= 6 with self.assertRaises(InterruptedError): mc.compute_from_collections([{'collections': [self.collection()], 'target_scene_m': self.target}], self.config, settings=self.settings, progress=progress, cancelled=cancelled) self.assertEqual(state['checks'], 6) def test_apply_rejects_broadcasting_and_corrupted_archives(self): cal = self.calibration() with self.assertRaisesRegex(ValueError, 'invalid shape'): mc.apply_calibration(np.ones(self.freq.size), self.freq, cal) cal['pair'][0, 0, 0] *= 2 with self.assertRaisesRegex(ValueError, 'disagree'): mc.validate_calibration(cal) def test_interpolation_is_continuous_at_nonlinear_archive_edges(self): self.settings.phase_model = 'raw' cal = self.calibration() # Keep a consistent separable archive with a deliberately nonlinear phase. phase = .4 * ((self.freq - self.freq.mean()) / 1e9) ** 3 cal['tx'][1] *= np.exp(1j * phase) cal['pair'] = cal['tx'][:, None] * cal['rx'][None] edge = mc.calibration_on_grid(cal, [self.freq[-1], self.freq[-1] + 1]) self.assertLess(abs(np.angle(edge[1, 0, 1] / edge[1, 0, 0])), 1e-6) shifted = mc.calibration_on_grid(cal, self.freq + 10000) self.assertIsNot(shifted, cal['pair']) def test_save_roundtrip_and_write_failure_preserve_the_previous_archive(self): cal = self.calibration() with tempfile.TemporaryDirectory() as directory: destination = Path(directory) / 'cal' destination.mkdir() (destination / 'notes.txt').write_text('preserve user notes') mc.save_calibration(cal, destination) np.testing.assert_array_equal(mc.load_calibration(destination)['pair'], cal['pair']) original = {path.name: path.read_bytes() for path in destination.iterdir()} save = np.save def fail_second(path, *args, **kwargs): if Path(path).name == 'cal_pair.npy': raise OSError('disk full') return save(path, *args, **kwargs) with mock.patch.object(mc.np, 'save', side_effect=fail_second), self.assertRaises(OSError): mc.save_calibration(cal, destination, overwrite=True) self.assertEqual(original, {path.name: path.read_bytes() for path in destination.iterdir()}) self.assertEqual(list(Path(directory).glob('.cal.*')), []) with self.assertRaises(FileExistsError): mc.save_calibration(cal, destination) def test_non_overwriting_save_preserves_archive_published_during_staging(self): with tempfile.TemporaryDirectory() as directory: destination = Path(directory) / 'cal' with self.assertRaises(FileExistsError): with mc._staged_directory(destination, False) as stage: (stage / 'cal_pair.npy').write_bytes(b'new correction') destination.mkdir() (destination / 'cal_pair.npy').write_bytes(b'concurrent correction') (destination / 'cal_meta.json').write_bytes(b'concurrent metadata') self.assertEqual({path.name: path.read_bytes() for path in destination.iterdir()}, { 'cal_pair.npy': b'concurrent correction', 'cal_meta.json': b'concurrent metadata', }) self.assertEqual(list(Path(directory).glob('.cal.*')), []) def test_failed_commit_restores_old_archive_and_failed_rollback_keeps_backups(self): cal = self.calibration() with tempfile.TemporaryDirectory() as directory: destination = Path(directory) / 'cal' mc.save_calibration(cal, destination) original = (destination / 'cal_pair.npy').read_bytes() replace = Path.replace def fail_commit(path, target): if path.name.startswith('.cal.new-'): raise OSError('rename failed') return replace(path, target) with mock.patch.object(Path, 'replace', autospec=True, side_effect=fail_commit), self.assertRaises(OSError): mc.save_calibration(cal, destination, overwrite=True) self.assertEqual((destination / 'cal_pair.npy').read_bytes(), original) def fail_rollback(path, target): if path.name.startswith(('.cal.new-', '.cal.old-')): raise OSError('filesystem unavailable') return replace(path, target) with mock.patch.object(Path, 'replace', autospec=True, side_effect=fail_rollback), self.assertRaisesRegex(RuntimeError, 'originals retained'): mc.save_calibration(cal, destination, overwrite=True) backup = next(Path(directory).glob('.cal.old-*')) self.assertEqual((backup / 'cal_pair.npy').read_bytes(), original) def test_offline_failure_preserves_previous_postprocessed_and_original_data(self): cal = self.calibration() with tempfile.TemporaryDirectory() as directory: root = Path(directory) snapshot = self.archive(root, self.collection()) post = root / 'postprocessed'; post.mkdir() (post / 'calibration_applied.json').write_text('{}') (post / 'old_data.bin').write_bytes(b'old data') original = {path.name: path.read_bytes() for path in snapshot.iterdir()} with mock.patch.object(mc.np, 'save', side_effect=OSError('disk full')), self.assertRaises(OSError): mc.apply_to_measurement(root, cal, overwrite=True) self.assertEqual((post / 'old_data.bin').read_bytes(), b'old data') self.assertEqual(original, {path.name: path.read_bytes() for path in snapshot.iterdir()}) self.assertEqual(list(root.glob('.postprocessed.*')), []) def test_object_corrected_archive_is_rejected_even_when_source_profile_has_object_off(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) snapshot = self.archive(root, self.collection()) post, *_ = mc.apply_to_measurement(root, self.calibration()) for prefixes in (None, [snapshot.name[:4]]): with self.subTest(prefixes=prefixes), self.assertRaisesRegex(ValueError, 'already applied'): mc.compute_calibration([{'path': post, 'snapshots': prefixes, 'target_scene_m': self.target}], settings=self.settings) # A corrected sibling does not change the original preprocessing data. original = mc.compute_calibration([{'path': root / 'preprocessed', 'target_scene_m': self.target}], settings=self.settings) np.testing.assert_array_equal(original['pair'], self.calibration()['pair']) def test_snapshot_local_correction_marker_is_checked_for_target_and_empty_scene(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) snapshot = self.archive(root, self.collection()) (snapshot / 'calibration_applied.json').write_text('{}') targets = [{'path': root / 'preprocessed', 'snapshots': [snapshot.name[:4]], 'target_scene_m': self.target}] with self.assertRaisesRegex(ValueError, 'already applied'): mc.compute_calibration(targets, settings=self.settings) untouched = root / 'untouched' self.archive(untouched, self.collection()) targets = [{'path': untouched / 'preprocessed', 'target_scene_m': self.target}] with self.assertRaisesRegex(ValueError, 'already applied'): mc.compute_calibration(targets, root / 'preprocessed', self.settings) def test_offline_grid_cache_uses_the_entire_axis_and_preserves_s11_and_dtype(self): cal = self.calibration() collection = self.collection(np.ones((2, 2, self.freq.size), np.complex64)) alternate = self.freq.copy() alternate[64] += 1e7 collection.traces[2].frequency_hz = alternate with tempfile.TemporaryDirectory() as directory: root = Path(directory) snapshot = self.archive(root, collection) original_s11 = (snapshot / 'i2_o5_s11.npy').read_bytes() post, count, fixed, _ = mc.apply_to_measurement(root, cal) self.assertEqual((count, fixed), (1, 4)) corrected = np.load(post / snapshot.name / 'i2_o5_s21.npy') self.assertEqual(corrected.dtype, np.complex64) expected = mc.calibration_on_grid(cal, alternate)[1, 0] np.testing.assert_allclose(corrected, expected, atol=1e-7) self.assertGreater(abs(corrected[64] - cal['pair'][1, 0, 64]), .001) self.assertEqual((post / snapshot.name / 'i2_o5_s11.npy').read_bytes(), original_s11) def test_empty_scene_grids_and_geometry_are_checked_before_subtraction(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) target_root, empty_root = root / 'target', root / 'empty' self.archive(target_root, self.collection()) zero = np.zeros((2, 2, self.freq.size), complex) self.archive(empty_root, self.collection(zero)) self.archive(empty_root, self.collection(zero, freq=self.freq + 10000), name='0001_id2_ns2') targets = [{'path': target_root / 'preprocessed', 'target_scene_m': self.target}] with self.assertRaisesRegex(ValueError, 'frequency grids differ'): mc.compute_calibration(targets, empty_root / 'preprocessed', self.settings) profile = copy.deepcopy(self.config) profile['gpr']['rx_geometry'][0]['z_m'] += .1 (empty_root / 'config_profile.json').write_text(json.dumps(profile)) with self.assertRaisesRegex(ValueError, 'antenna geometry differs'): mc.compute_calibration(targets, empty_root / 'preprocessed', self.settings) def test_snapshot_prefixes_must_select_one_distinct_complete_snapshot(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) self.archive(root, self.collection()) self.archive(root, self.collection(), name='0001_id2_ns2') for prefixes in (['000'], ['0000', 'missing'], ['0000', '0000']): with self.subTest(prefixes=prefixes), self.assertRaises(ValueError): mc.list_snapshots(root / 'preprocessed', prefixes) def test_multiple_snapshots_are_averaged_before_one_gated_fft(self): collections = [self.collection(identity=index + 1) for index in range(8)] with mock.patch.object(mc.np.fft, 'ifft', wraps=np.fft.ifft) as transform: mc.compute_from_collections([{'collections': collections, 'target_scene_m': self.target}], self.config, settings=self.settings) self.assertEqual(transform.call_count, 1) def test_original_scipy_refinement_finishes_with_valid_diagnostics(self): self.settings.refine_axes = ('z',) self.settings.coarse_range_search_m = .1 cal = self.calibration() self.assertGreater(cal['meta']['diagnostics'][0]['coherence_after'], .999) refined = cal['meta']['targets'][0]['refined_scene_m'] self.assertTrue(np.all(np.isfinite(refined))) def test_bad_settings_and_duplicate_geometry_fail_early(self): invalid_settings: tuple[dict[str, Any], ...] = ( {'input_idx': (0, 0)}, {'output_idx': (True,)}, {'gate_taper_m': 0.}, {'f_max': 0.}, {'refine_axes': ('z', 'z')}, {'phase_model': 'unknown'}, ) for fields in invalid_settings: with self.subTest(fields=fields), self.assertRaises(ValueError): mc.Settings(**fields) self.config['gpr']['tx_geometry'].append(self.config['gpr']['tx_geometry'][0]) with self.assertRaisesRegex(ValueError, 'duplicate antenna geometry'): self.calibration() if __name__ == '__main__': unittest.main()