awe/radar_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rework radar system

Browse Source
This commit is contained in:
Ayzen
2026-10-06 12:32:26 +03:00
parent 552cdc688a
commit 5bb86b7165
299 changed files with 13537 additions and 27763 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python_app/tests/test_persistence_integrity.py
+127 -76
View File
@@ -2,8 +2,11 @@
from __future__ import annotations
import json
import os
from pathlib import Path
import stat
import tempfile
from typing import cast
import unittest
from unittest import mock
@@ -13,10 +16,9 @@ from python_app.models.dataset_model import (
ComboKey, ResultBlock, ResultCollection, ResultPayload, SweepCollection, TraceData,
)
from python_app.storage.npz.store import NpzStore
from python_app.storage.npz.snapshot_numpy import select_aligned_histories
from python_app.storage.npz.vna_history_json import build_vna_history_payload
from python_app.scripts.replay_raw_producer import _load_dataset
from python_app.scripts.convert_snapshot_to_vna_history import _collection_dirs, _load_stage_records, _build_sweep_history
from python_app.storage.npz.vna_history_json import collection_directories, load_vna_history_snapshot
from python_app.workflows.sequential_capture_workflow import combine_collections_via_median, combine_traces_via_median
@@ -25,7 +27,6 @@ def _sweep(cid: int = 1, ns: int = 20, *, start: int = 10, end: int = 15) -> Swe
collection_id=cid, monotonic_ns=ns, capture_start_ns=start, capture_end_ns=end,
traces=[TraceData(
combo=ComboKey(0, 0), frequency_hz=np.array([1e9, 2e9], dtype=np.float32),
s11=np.array([1 + 2j, 3 - 4j], dtype=np.complex64),
s21=np.array([5 - 6j, 7 + 8j], dtype=np.complex64),
capture_start_ns=start, capture_end_ns=end,
)],
@@ -52,7 +53,7 @@ class PersistenceIntegrityTest(unittest.TestCase):
self.output = self.root / 'output'
def test_all_calibration_and_reference_kinds_preserve_arrays_and_timing(self) -> None:
for kind in ('s21_calibration', 's21_reference', 's11_open', 's11_short', 's11_load', 's11_reference'):
for kind in ('s21_calibration', 's21_reference'):
with self.subTest(kind=kind):
original = _sweep(7, 100)
second = _sweep().traces[0]
@@ -67,22 +68,10 @@ class PersistenceIntegrityTest(unittest.TestCase):
for before, after in zip(original.traces, loaded.traces, strict=True):
self.assertEqual(before.combo, after.combo)
self.assertEqual((after.capture_start_ns, after.capture_end_ns), (10, 15))
for field in ('frequency_hz', 's11', 's21'):
for field in ('frequency_hz', 's21'):
np.testing.assert_array_equal(getattr(before, field), getattr(after, field))
def test_new_set_remains_loadable_without_json_sidecar(self) -> None:
self.store.save_set('s21_reference', 'radar', 'full', _sweep())
(self.root / 'sets/s21_reference/radar/full.json').unlink()
loaded = self.store.load_set('s21_reference', 'radar', 'full')
np.testing.assert_array_equal(loaded.traces[0].s21, _sweep().traces[0].s21)
def test_legacy_npz_json_pair_is_still_loadable(self) -> None:
self.store.save_set('s21_reference', 'radar', 'full', _sweep())
archive = self.root / 'sets/s21_reference/radar/full.npz'
with np.load(archive, allow_pickle=False) as arrays:
legacy = {key: arrays[key] for key in arrays.files if key != 'metadata_json'}
np.savez(archive, **legacy)
self.assertEqual(self.store.load_set('s21_reference', 'radar', 'full').monotonic_ns, 20)
def test_failed_set_commit_preserves_previous_data_and_sidecar(self) -> None:
self.store.save_set('s21_reference', 'radar', 'full', _sweep(1, 20))
@@ -130,13 +119,59 @@ class PersistenceIntegrityTest(unittest.TestCase):
self.assertEqual(self.store.load_set('s21_reference', 'radar', 'full').collection_id, 1)
self.assertEqual(list(self.root.rglob('*.tmp')), [])
def test_set_files_are_synced_before_commit_and_directory_afterwards(self) -> None:
events = []
fsync, replace = os.fsync, Path.replace
def sync_file(descriptor):
events.append('directory' if stat.S_ISDIR(os.fstat(descriptor).st_mode) else 'file')
fsync(descriptor)
def commit(path, destination):
events.append('replace')
return replace(path, destination)
with mock.patch('os.fsync', side_effect=sync_file), mock.patch.object(Path, 'replace', commit):
self.store.save_set('s21_calibration', 'radar', 'full', _sweep())
first_file = events.index('file')
self.assertEqual(events[:first_file], ['directory', 'directory'])
self.assertEqual(events[first_file:], ['file', 'file', 'replace', 'replace', 'directory'])
def test_set_sync_failure_does_not_replace_previous_measurement(self) -> None:
self.store.save_set('s21_calibration', 'radar', 'full', _sweep())
archive = self.root / 'sets/s21_calibration/radar/full.npz'
original = archive.read_bytes()
for failed_sync in (1, 2):
with self.subTest(failed_sync=failed_sync), mock.patch('python_app.storage.npz.store.logger.exception'):
with mock.patch('os.fsync', side_effect=[None] * (failed_sync - 1) + [OSError('sync failed')]):
with self.assertRaises(OSError):
self.store.save_set('s21_calibration', 'radar', 'full', _sweep(2, 30))
self.assertEqual(archive.read_bytes(), original)
self.assertEqual(list(archive.parent.glob('*.tmp')), [])
def test_sets_and_snapshots_preserve_permissions_and_respect_umask(self) -> None:
previous = os.umask(0o027)
try:
self.store.save_set('s21_calibration', 'radar', 'full', _sweep())
directory = self.store.save_runtime_snapshot_numpy(self.output, 'full', [_sweep()], [], [])
finally:
os.umask(previous)
for path in (self.root / 'sets/s21_calibration/radar').iterdir():
self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o640)
path.chmod(0o664)
for path in [directory, *directory.rglob('*')]:
self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o750 if path.is_dir() else 0o640)
self.store.save_set('s21_calibration', 'radar', 'full', _sweep(2, 30))
for path in (self.root / 'sets/s21_calibration/radar').iterdir():
self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o664)
def test_invalid_sets_are_rejected_before_overwriting_valid_data(self) -> None:
self.store.save_set('s21_calibration', 'radar', 'full', _sweep())
invalid = [_sweep(), _sweep(), _sweep(), _sweep()]
invalid[0].traces.append(invalid[0].traces[0])
invalid[1].traces[0].s21 = np.array([1], dtype=np.complex64)
invalid[2].traces[0].frequency_hz = np.array([[1, 2]], dtype=np.float32)
invalid[3].traces[0].s11[0] = complex(float('nan'), 0)
invalid[3].traces[0].s21[0] = complex(float('nan'), 0)
for collection in invalid:
with self.subTest(collection=collection), self.assertRaises(ValueError):
self.store.save_set('s21_calibration', 'radar', 'full', collection)
@@ -150,7 +185,7 @@ class PersistenceIntegrityTest(unittest.TestCase):
def test_invalid_channel_identifiers_are_rejected_without_truncation(self) -> None:
for position in (0.5, True, -1, 2 ** 32):
sweep = _sweep()
sweep.traces[0].combo = ComboKey(position, 0)
sweep.traces[0].combo = ComboKey(cast(int, position), 0)
with self.subTest(position=position), self.assertRaises(ValueError):
self.store.save_set('s21_reference', 'radar', 'invalid', sweep)
self.assertEqual(self.store.list_sets('s21_reference', 'radar'), [])
@@ -173,10 +208,9 @@ class PersistenceIntegrityTest(unittest.TestCase):
raw = [_sweep(1, 10), _sweep(2, 30)]
pre = [_sweep(1, 20), _sweep(3, 40)]
result = [ResultCollection(1, 20)]
directory, summary = self.store.save_runtime_snapshot_numpy(
self.output, 'full', raw, pre, result, 1000, include_unaligned=True,
directory = self.store.save_runtime_snapshot_numpy(
self.output, 'full', raw, pre, result,
)
self.assertEqual((summary['raw_count'], summary['preprocessed_count'], summary['result_count']), (2, 2, 1))
for stage, ids in [('raw', [1, 2]), ('preprocessed', [1, 3]), ('results', [1])]:
metadata = [json.loads(path.read_text()) for path in sorted((directory / stage).glob('*/meta.json'))]
self.assertEqual([item['collection_id'] for item in metadata], ids)
@@ -184,11 +218,15 @@ class PersistenceIntegrityTest(unittest.TestCase):
def test_snapshot_result_payloads_preserve_every_kind_at_both_scopes(self) -> None:
payloads = _payloads()
result = ResultCollection(1, 20, processing_duration_ns=12345,
settings_revision=2**63 + 7, capture_start_ns=2**60 + 3,
capture_end_ns=2**60 + 9, speed_m_s=1.25,
collection_payloads=payloads, blocks=[ResultBlock(ComboKey(0, 0), payloads)])
directory, _ = self.store.save_runtime_snapshot_numpy(self.output, 'full', [], [], [result], 1000)
directory = self.store.save_runtime_snapshot_numpy(self.output, 'full', [], [], [result])
collection_dir = next((directory / 'results').iterdir())
meta = json.loads((collection_dir / 'meta.json').read_text())
self.assertEqual(meta['processing_duration_ns'], 12345)
self.assertEqual((meta['settings_revision'], meta['capture_start_ns'], meta['capture_end_ns'], meta['speed_m_s']),
(2**63 + 7, 2**60 + 3, 2**60 + 9, 1.25))
for base, entries in [(collection_dir, meta['collection_payloads']),
(collection_dir / meta['blocks'][0]['dir'], meta['blocks'][0]['payloads'])]:
self.assertEqual([entry['kind'] for entry in entries], [1, 2, 3, 4])
@@ -204,18 +242,18 @@ class PersistenceIntegrityTest(unittest.TestCase):
for payload in (ResultPayload('unknown', 9), ResultPayload('bad_table', 4, table=np.array([1, 2]))):
with self.subTest(kind=payload.kind), self.assertRaises(ValueError):
self.store.save_runtime_snapshot_numpy(
self.output, 'failed', [_sweep()], [], [ResultCollection(1, 20, collection_payloads=[payload])], 1000,
self.output, 'failed', [_sweep()], [], [ResultCollection(1, 20, collection_payloads=[payload])],
)
self.assertFalse((self.output / 'failed').exists())
self.assertEqual(list(self.output.iterdir()), [])
def test_snapshot_write_failure_can_be_retried_with_same_name(self) -> None:
with mock.patch('numpy.save', side_effect=OSError('disk full')), self.assertRaises(OSError):
self.store.save_runtime_snapshot_numpy(self.output, 'retry', [_sweep()], [], [], 1000)
directory, _ = self.store.save_runtime_snapshot_numpy(self.output, 'retry', [_sweep()], [], [], 1000)
self.store.save_runtime_snapshot_numpy(self.output, 'retry', [_sweep()], [], [])
directory = self.store.save_runtime_snapshot_numpy(self.output, 'retry', [_sweep()], [], [])
self.assertTrue(directory.exists())
with self.assertRaises(FileExistsError):
self.store.save_runtime_snapshot_numpy(self.output, 'retry', [_sweep()], [], [], 1000)
self.store.save_runtime_snapshot_numpy(self.output, 'retry', [_sweep()], [], [])
def test_stream_failure_keeps_prior_chunks_and_retries_without_partial_data(self) -> None:
writer = self.store.create_snapshot_stream(self.output, 'stream')
@@ -229,6 +267,44 @@ class PersistenceIntegrityTest(unittest.TestCase):
self.assertEqual(writer.result_count, 2)
self.assertEqual([path.name.split('_')[0] for path in sorted((writer.directory / 'raw').iterdir())], ['0000', '0001'])
def test_stream_failure_preserves_collections_already_published_in_current_chunk(self) -> None:
writer = self.store.create_snapshot_stream(self.output, 'partial')
good = ResultCollection(1, 20)
bad = ResultCollection(2, 40, collection_payloads=[ResultPayload('invalid', 9)])
with self.assertRaises(ValueError):
writer.append([_sweep(1, 10), _sweep(2, 30)], [], [good, bad])
self.assertEqual(writer.result_count, 1)
self.assertEqual(len(list((writer.directory / 'raw').iterdir())), 2)
self.assertEqual(len(list((writer.directory / 'results').iterdir())), 1)
writer.append([], [], [ResultCollection(2, 40)])
self.assertEqual(writer.result_count, 2)
self.assertEqual([path.name.split('_')[0] for path in sorted((writer.directory / 'results').iterdir())], ['0000', '0001'])
def test_processing_revisions_are_saved_and_cannot_be_reassigned(self) -> None:
settings = {4: {'settings_revision': 4, 'gain': 2}}
directory = self.store.save_runtime_snapshot_numpy(
self.output, 'snapshot', [], [], [], processing_settings=settings,
)
self.assertEqual(json.loads((directory / 'processing_settings.json').read_text()), {'4': settings[4]})
writer = self.store.create_snapshot_stream(self.output, 'stream')
writer.append([], [], [ResultCollection(1, 10)], processing_settings=settings)
later = {5: {'settings_revision': 5, 'gain': 3}}
writer.append([], [], [ResultCollection(2, 20)], processing_settings=later)
path = writer.directory / 'processing_settings.json'
self.assertEqual(json.loads(path.read_text()), {'4': settings[4], '5': later[5]})
with self.assertRaisesRegex(ValueError, 'already has different settings'):
writer.append([], [], [ResultCollection(3, 30)], processing_settings={4: {'gain': 9}})
self.assertEqual(writer.result_count, 2)
self.assertEqual(json.loads(path.read_text()), {'4': settings[4], '5': later[5]})
def test_failed_processing_settings_write_does_not_publish_results(self) -> None:
writer = self.store.create_snapshot_stream(self.output, 'stream')
with mock.patch('python_app.storage.npz.snapshot_numpy.atomic_write_json', side_effect=OSError('disk full')):
with self.assertRaises(OSError):
writer.append([], [], [ResultCollection(1, 10)], processing_settings={1: {'gain': 2}})
self.assertEqual(writer.result_count, 0)
self.assertEqual(list(writer.directory.iterdir()), [])
def test_json_export_preserves_unmatched_raw_sweeps_and_matches_reused_ids_by_timing(self) -> None:
raw_old = _sweep(1, 10, start=1, end=5)
raw_new = _sweep(1, 100, start=90, end=95)
@@ -244,33 +320,37 @@ class PersistenceIntegrityTest(unittest.TestCase):
second = _sweep().traces[0]
second.combo = ComboKey(0, 1)
sweep.traces.append(second)
write_text = Path.write_text
dump = json.dump
writes = 0
def fail_second_file(path, *args, **kwargs):
def fail_second_file(*args, **kwargs):
nonlocal writes
writes += 1
if writes == 2:
raise OSError('disk full')
return write_text(path, *args, **kwargs)
return dump(*args, **kwargs)
with mock.patch.object(Path, 'write_text', fail_second_file), self.assertRaises(OSError):
self.store.save_runtime_vna_history_json_batch(self.output, 'export', [sweep], [], [], 1000)
with mock.patch('python_app.storage.npz.store.json.dump', fail_second_file), self.assertRaises(OSError):
self.store.save_runtime_vna_history_json_batch(self.output, 'export', [sweep], [], [])
self.assertEqual(list(self.output.iterdir()), [])
paths, summary = self.store.save_runtime_vna_history_json_batch(self.output, 'export', [sweep], [], [], 1000, channel='s11')
self.assertEqual(summary['combo_count'], 2)
paths = self.store.save_runtime_vna_history_json_batch(self.output, 'export', [sweep], [], [])
self.assertEqual(len(paths), 2)
self.assertEqual(len(paths), 2)
for path in paths:
self.assertEqual(json.loads(path.read_text())['channel'], 's11')
self.assertEqual(json.loads(path.read_text())['channel'], 's21')
def test_saved_dataset_replay_preserves_s11_and_accepts_legacy_s21_only(self) -> None:
directory, _ = self.store.save_runtime_snapshot_numpy(self.output, 'replay', [_sweep()], [], [], 1000)
dataset = _load_dataset(directory / 'raw', [ComboKey(0, 0)], 2)
np.testing.assert_array_equal(dataset.snapshots[0][0].s11, _sweep().traces[0].s11)
self.assertEqual(dataset.snapshots[0][0].capture_end_ns, 15)
next((directory / 'raw').glob('*/i0_o0_s11.npy')).unlink()
legacy = _load_dataset(directory / 'raw', [ComboKey(0, 0)], 2)
np.testing.assert_array_equal(legacy.snapshots[0][0].s11, np.zeros(2, dtype=np.complex64))
def test_json_export_can_follow_numpy_without_overwriting_either_format(self) -> None:
directory = self.store.save_runtime_snapshot_numpy(self.output, 'both', [_sweep()], [], [])
directory.chmod(0o775)
before = {path: path.read_bytes() for path in directory.rglob('*') if path.is_file()}
paths = self.store.save_runtime_vna_history_json_batch(self.output, 'both', [_sweep()], [], [])
self.assertEqual(stat.S_IMODE(directory.stat().st_mode), 0o775)
self.assertEqual({path: path.read_bytes() for path in before}, before)
exported = paths[0].read_bytes()
with self.assertRaises(FileExistsError):
self.store.save_runtime_vna_history_json_batch(self.output, 'both', [_sweep(2, 30)], [], [])
self.assertEqual(paths[0].read_bytes(), exported)
self.assertEqual({path: path.read_bytes() for path in before}, before)
def test_long_recording_replay_and_converter_use_numeric_directory_order(self) -> None:
writer = self.store.create_snapshot_stream(self.output, 'long')
@@ -280,49 +360,20 @@ class PersistenceIntegrityTest(unittest.TestCase):
folder.rename(folder.with_name(f'{index}_{folder.name.split("_", 1)[1]}'))
dataset = _load_dataset(writer.directory / 'raw', [ComboKey(0, 0)], 2)
self.assertEqual(dataset.timestamps_ns, [10, 20, 30])
self.assertEqual([int(path.name.split('_')[0]) for path in _collection_dirs(writer.directory / 'raw')], [9998, 9999, 10000])
self.assertEqual([int(path.name.split('_')[0]) for path in collection_directories(writer.directory / 'raw')], [9998, 9999, 10000])
def test_offline_and_runtime_json_exports_share_matching_and_preserve_orphan_sweeps(self) -> None:
raw = [_sweep(1, 10, start=1, end=5), _sweep(1, 100, start=90, end=95)]
pre = [_sweep(1, 110, start=90, end=95)]
directory, _ = self.store.save_runtime_snapshot_numpy(
self.output, 'convert', raw, pre, [], 1000, include_unaligned=True,
directory = self.store.save_runtime_snapshot_numpy(
self.output, 'convert', raw, pre, [],
)
records = {
stage: _load_stage_records(directory, stage, input_index=0, output_index=0, channel='s21')
for stage in ('raw', 'preprocessed')
}
offline = _build_sweep_history(records['raw'], records['preprocessed'], channel='s21', primary_stage='preprocessed')
offline = load_vna_history_snapshot(directory, input_index=0, output_index=0)["sweep_history"]
runtime = build_vna_history_payload(raw, pre, [], input_index=0, output_index=0)
self.assertEqual(offline, runtime['sweep_history'])
self.assertEqual(len(offline), 2)
class SnapshotAlignmentTest(unittest.TestCase):
def test_reused_ids_and_missing_frames_do_not_cross_align_runs(self) -> None:
raw_old = _sweep(1, 10, start=1, end=5)
pre_old = _sweep(1, 20, start=1, end=5)
pre_new = _sweep(1, 200, start=150, end=175)
selected_raw, selected_pre, results, summary = select_aligned_histories(
[raw_old], [pre_old, pre_new], [ResultCollection(1, 200)], 1,
)
self.assertEqual(selected_raw, [])
self.assertEqual(selected_pre, [pre_new])
self.assertEqual(summary['raw_missing_count'], 1)
self.assertEqual(results[0].monotonic_ns, 200)
def test_result_does_not_pair_with_wrong_preprocessed_timestamp(self) -> None:
_, pre, _, summary = select_aligned_histories([], [_sweep(1, 20)], [ResultCollection(1, 200)], 1000)
self.assertEqual(pre, [])
self.assertEqual(summary['preprocessed_missing_count'], 1)
def test_independent_stage_limit_is_exactly_requested_tail(self) -> None:
raw = [_sweep(cid=index, ns=index) for index in range(1005)]
selected_raw, _, _, _ = select_aligned_histories(raw, [], [], 1000, include_unaligned=True)
self.assertEqual(len(selected_raw), 1000)
self.assertEqual((selected_raw[0].collection_id, selected_raw[-1].collection_id), (5, 1004))
class CalibrationMedianIntegrityTest(unittest.TestCase):
def test_rejects_frequency_axes_that_changed_between_captures(self) -> None:
first, second = _sweep(), _sweep()