diff --git a/Makefile b/Makefile index 8451370..9bd1806 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,7 @@ DEPFILES := $(sort $(SWEEP_ORCH_OBJS:.o=.d) $(PREPROCESSOR_OBJS:.o=.d) $(DATA_PR NATIVE_TEST_SOURCES := $(COMMON_SOURCES) $(LOCATOR_SOURCES) \ data_acq_and_processing/tests/native_regressions.cpp \ data_acq_and_processing/processing/data_processor/src/processing_live_config.cpp \ - data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp \ + $(filter-out %/main.cpp,$(PREPROC_SOURCES)) \ data_acq_and_processing/sweep_orchestrator/src/sweep_orchestrator.cpp \ $(KAMIL_COLLECTOR_DIR)/src/capture_file_writer.cpp \ $(KAMIL_COLLECTOR_DIR)/src/tty_protocol_writer.cpp diff --git a/README.md b/README.md index 128c33f..791bd54 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ simply skips to the freshest frame: configures the radar, performs the frequency sweep, and publishes raw S11/S21 traces. 2. **Preprocessing** — `data_preprocessor` applies through/OSL calibration, - reference subtraction, and optional frequency-domain notch filtering. + reference subtraction, optional frequency-domain notch filtering, and an + optional object-based MIMO correction of S21. 3. **Processing** — `data_processor` runs the selected processing mode and emits results (pass-through traces, B-scan images, GPR images with detected objects, or 3D SAR views). Detected objects are additionally broadcast to subscribers @@ -223,13 +224,29 @@ result count or Stop; Stop and closing the window flush accepted data. Acquisiti rings use latest-wins delivery, so disk recording cannot recover sweeps overwritten before the GUI receives them. NumPy datasets preserve both S-parameters, capture windows, every supported result payload, and processing time. Failed exports remove -their staging files, leaving existing datasets intact. +their staging files, leaving existing datasets intact. Companion profiles retain +the radar, antenna layout and preprocessing used for acquisition, including when +another profile is loaded after Stop. Calibration and reference sets keep their trace arrays and metadata together in one atomically replaced NPZ archive, with a readable JSON sidecar. Earlier NPZ/JSON pairs remain supported. Invalid trace shapes, duplicate combinations and non-finite calibration/reference values are rejected before a saved set is replaced. Median captures also require identical frequency axes and channel coverage. +In **Preprocessing Panel**, create or select the ordinary S21 calibration and +empty-scene reference first. **Create…** in the **Object Calibration** row opens a separate step: +take fresh sweeps, then enter the measured reflector centre X/Y/Z in metres. +Height above ground, radar pitch and sphere radius follow `mimo_calibration.py`; +several reflector positions may contribute to one calibration. The calculation +shows each pair's phase at the target before/after correction and its coherence. +The selected reference is already subtracted, and the previous object correction +is disabled while these sweeps are acquired. Saved object corrections apply on the +next **Start**, after notch filtering and before any processor; **Off** disables +them. Changing antenna geometry disables an incompatible correction and reloads +acquisition without restarting the application. **Import…** in that row +accepts the example's `cal_*.npy` / `cal_meta.json` format; the example and GUI share +one numerical implementation. + Dataset replay restores both recorded S-parameters and orders long recordings by their numeric index; older S21-only datasets still replay with zero S11. Offline JSON conversion and the GUI share the same sweep-matching code. diff --git a/data_acq_and_processing/common_cpp/config/include/run_config.hpp b/data_acq_and_processing/common_cpp/config/include/run_config.hpp index 7cf50b0..19ca67c 100644 --- a/data_acq_and_processing/common_cpp/config/include/run_config.hpp +++ b/data_acq_and_processing/common_cpp/config/include/run_config.hpp @@ -81,6 +81,7 @@ struct PreprocessAssetConfig { struct S21PreprocessConfig { PreprocessAssetConfig calibration{}; PreprocessAssetConfig reference{}; + PreprocessAssetConfig object_calibration{}; }; struct S11CalibrationPreprocessConfig { diff --git a/data_acq_and_processing/common_cpp/config/src/run_config.cpp b/data_acq_and_processing/common_cpp/config/src/run_config.cpp index 6ce59a6..064919b 100644 --- a/data_acq_and_processing/common_cpp/config/src/run_config.cpp +++ b/data_acq_and_processing/common_cpp/config/src/run_config.cpp @@ -555,6 +555,10 @@ auto load_run_config(const std::string& path) -> RunConfig { const auto* s21_obj = as_object(required_field(*preprocess_obj, "s21"), "preprocess.s21"); config.preprocess.s21.calibration = parse_preprocess_asset(*s21_obj, "calibration", "preprocess.s21"); config.preprocess.s21.reference = parse_preprocess_asset(*s21_obj, "reference", "preprocess.s21"); + if (optional_field(*s21_obj, "object_calibration") != nullptr) { + config.preprocess.s21.object_calibration = + parse_preprocess_asset(*s21_obj, "object_calibration", "preprocess.s21"); + } const auto* s11_obj = as_object(required_field(*preprocess_obj, "s11"), "preprocess.s11"); const auto* s11_calibration_obj = diff --git a/data_acq_and_processing/preprocessing/calibration_master/include/channel_bundle_support.hpp b/data_acq_and_processing/preprocessing/calibration_master/include/channel_bundle_support.hpp index dbfe356..a4c8a03 100644 --- a/data_acq_and_processing/preprocessing/calibration_master/include/channel_bundle_support.hpp +++ b/data_acq_and_processing/preprocessing/calibration_master/include/channel_bundle_support.hpp @@ -50,6 +50,18 @@ class S21ReferenceBundle { std::unordered_map traces_by_combo_{}; }; +// Object-derived MIMO correction, exported on each pair's runtime frequency grid. +// An empty path disables it; enabled bundles must cover every active pair. +class S21ObjectCalibrationBundle { + public: + void load(const std::string& path); + void validate_combos(const std::vector& combos) const; + void apply(ipc::SweepTraceBlock& trace) const; + + private: + std::unordered_map factors_by_combo_{}; +}; + class S11CalibrationBundle { public: struct Coefficients { diff --git a/data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp b/data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp index deee56a..a130748 100644 --- a/data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp +++ b/data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -197,11 +198,11 @@ template auto resolve_required( const std::unordered_map& values_by_combo, const ipc::ComboKey& combo, - const std::string& value_label + const char* value_label ) -> const TValue& { const auto found = values_by_combo.find(combo); if (found == values_by_combo.end()) { - throw std::runtime_error(value_label + " is missing for combo " + combo_to_string(combo)); + throw std::runtime_error(std::string(value_label) + " is missing for combo " + combo_to_string(combo)); } return found->second; } @@ -345,6 +346,38 @@ auto S21ReferenceBundle::resolve(const ipc::ComboKey& combo) const -> const Chan return resolve_required(traces_by_combo_, combo, "S21 reference trace"); } +void S21ObjectCalibrationBundle::load(const std::string& path) { + if (path.empty()) { + factors_by_combo_.clear(); + return; + } + factors_by_combo_ = load_channel_traces(path, "S21 object calibration", RawTraceChannel::S21); +} + +void S21ObjectCalibrationBundle::validate_combos(const std::vector& combos) const { + validate_bundle_combos(factors_by_combo_, combos, "S21 object calibration bundle"); +} + +void S21ObjectCalibrationBundle::apply(ipc::SweepTraceBlock& trace) const { + if (factors_by_combo_.empty()) { return; } + const auto& factors = resolve_required(factors_by_combo_, trace.combo, "S21 object calibration"); + ensure_frequency_axis_match(factors.frequency_hz, trace.frequency_hz, "S21 object calibration"); + if (trace.s21.size() != factors.samples.size()) { + throw std::runtime_error("S21 object calibration vector size mismatch"); + } + for (std::size_t index = 0; index < trace.s21.size(); ++index) { + const auto value = trace.s21[index]; + const auto factor = factors.samples[index]; + const auto corrected = std::complex(value.re, value.im) * std::complex(factor.re, factor.im); + if (!std::isfinite(corrected.real()) || !std::isfinite(corrected.imag()) || + std::abs(corrected.real()) > std::numeric_limits::max() || + std::abs(corrected.imag()) > std::numeric_limits::max()) { + throw std::runtime_error("S21 object calibration result must fit finite float32"); + } + trace.s21[index] = {.re = static_cast(corrected.real()), .im = static_cast(corrected.imag())}; + } +} + void S11CalibrationBundle::load( const std::string& open_path, const std::string& short_path, diff --git a/data_acq_and_processing/preprocessing/data_preprocessor/include/data_preprocessor.hpp b/data_acq_and_processing/preprocessing/data_preprocessor/include/data_preprocessor.hpp index 8ee571f..7e0de76 100644 --- a/data_acq_and_processing/preprocessing/data_preprocessor/include/data_preprocessor.hpp +++ b/data_acq_and_processing/preprocessing/data_preprocessor/include/data_preprocessor.hpp @@ -39,6 +39,7 @@ class DataPreprocessor { const config::RunConfig& config_; CalibrationMaster& calibration_master_; ReferenceMaster& reference_master_; + S21ObjectCalibrationBundle object_calibration_; ipc::ShmRing& raw_ring_; ipc::ShmRing& preprocessed_ring_; ipc::ShmRing* preprocessed_tap_ring_ = nullptr; diff --git a/data_acq_and_processing/preprocessing/data_preprocessor/src/data_preprocessor.cpp b/data_acq_and_processing/preprocessing/data_preprocessor/src/data_preprocessor.cpp index dbf6e08..537b346 100644 --- a/data_acq_and_processing/preprocessing/data_preprocessor/src/data_preprocessor.cpp +++ b/data_acq_and_processing/preprocessing/data_preprocessor/src/data_preprocessor.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include namespace radar::preprocessing { @@ -104,11 +105,14 @@ DataPreprocessor::DataPreprocessor( reference_master_(reference_master), raw_ring_(raw_ring), preprocessed_ring_(preprocessed_ring), - preprocessed_tap_ring_(preprocessed_tap_ring) {} + preprocessed_tap_ring_(preprocessed_tap_ring) { + object_calibration_.load(config_.preprocess.s21.object_calibration.bundle_path); +} void DataPreprocessor::validate_startup_prerequisites() const { calibration_master_.validate_combos(config_.run_combos); reference_master_.validate_combos(config_.run_combos); + object_calibration_.validate_combos(config_.run_combos); } auto DataPreprocessor::try_pop_raw_collection( @@ -188,10 +192,12 @@ auto DataPreprocessor::preprocess_collection(const ipc::RawSweepCollection& raw_ preprocessed.traces.reserve(raw_collection.traces.size()); for (const auto& raw_trace : raw_collection.traces) { - // Pipeline order is fixed: calibration first, then reference subtraction. + // Object-derived corrections act on the completed preprocessing output. const auto calibrated = calibration_master_.apply_to_trace(raw_trace); const auto referenced = reference_master_.apply_to_trace(calibrated); - preprocessed.traces.push_back(apply_notch_filter(referenced, config_.preprocess.notch)); + auto filtered = apply_notch_filter(referenced, config_.preprocess.notch); + object_calibration_.apply(filtered); + preprocessed.traces.push_back(std::move(filtered)); } return preprocessed; diff --git a/data_acq_and_processing/tests/native_regressions.cpp b/data_acq_and_processing/tests/native_regressions.cpp index 746ec0e..ab5c2ec 100644 --- a/data_acq_and_processing/tests/native_regressions.cpp +++ b/data_acq_and_processing/tests/native_regressions.cpp @@ -1,5 +1,6 @@ #include "capture_file_writer.h" #include "channel_bundle_support.hpp" +#include "data_preprocessor.hpp" #include "locator/tcp_server.hpp" #include "processing_live_config.hpp" #include "runtime_wait.hpp" @@ -16,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -60,11 +62,23 @@ struct Workspace { }; struct RingName { - std::string name = "/radar_native_test_" + std::to_string(::getpid()); - RingName() { ipc::ShmRing::unlink_ring(name); } + std::string name; + explicit RingName(const std::string& suffix = {}) + : name("/radar_native_test_" + std::to_string(::getpid()) + suffix) { + ipc::ShmRing::unlink_ring(name); + } ~RingName() { ipc::ShmRing::unlink_ring(name); } }; +auto save_bundle(const Workspace& workspace, const char* name, const ipc::RawSweepCollection& collection) + -> std::string { + const auto path = workspace.path / name; + const auto bytes = ipc::serialize_raw_collection(collection); + std::ofstream(path, std::ios::binary).write( + reinterpret_cast(bytes.data()), static_cast(bytes.size())); + return path.string(); +} + void write_u32(std::vector& bytes, std::size_t offset, std::uint32_t value) { require(offset + sizeof(value) <= bytes.size(), "invalid test offset"); std::memcpy(bytes.data() + offset, &value, sizeof(value)); @@ -137,6 +151,22 @@ void test_configuration(const std::string& seed_path, const Workspace& workspace const auto save = [&](const Json& config) { std::ofstream(config_path) << config; }; save(seed); const auto config = config::load_run_config(config_path.string()); + auto with_object_calibration = seed; + with_object_calibration["preprocess"]["s21"]["object_calibration"] = + Json{{"set_name", "reflector"}, {"bundle_path", "corrections.bin"}}; + save(with_object_calibration); + const auto selected = config::load_run_config(config_path.string()).preprocess.s21.object_calibration; + require(selected.set_name == "reflector" && selected.bundle_path == "corrections.bin", + "object calibration asset did not load"); + with_object_calibration["preprocess"]["s21"].erase("object_calibration"); + save(with_object_calibration); + require(config::load_run_config(config_path.string()).preprocess.s21.object_calibration.bundle_path.empty(), + "absent object calibration was not disabled"); + for (const auto& invalid_asset : {Json(nullptr), Json{{"bundle_path", true}}}) { + with_object_calibration["preprocess"]["s21"]["object_calibration"] = invalid_asset; + save(with_object_calibration); + expect_error([&] { (void)config::load_run_config(config_path.string()); }); + } for (const auto& [pointer, value] : std::array, 8>{{ {"/radar/sweep/start_hz", -1}, {"/radar/sweep/stop_hz", 1e100}, {"/radar/sweep/if_bandwidth_hz", 0}, {"/radar/sweep/points", 0}, @@ -297,12 +327,8 @@ void test_csv_failures(const Workspace& workspace) { void test_asset_reload_preserves_data(const Workspace& workspace) { const auto save = [&](const char* name, float sample) { - const auto path = workspace.path / name; - const auto bytes = ipc::serialize_raw_collection({.traces = { + return save_bundle(workspace, name, {.traces = { {.frequency_hz = {1}, .s11 = {{sample, 0}}, .s21 = {{sample, 0}}}}}); - std::ofstream(path, std::ios::binary).write( - reinterpret_cast(bytes.data()), static_cast(bytes.size())); - return path.string(); }; const auto open = save("open.bin", 1); const auto short_standard = save("short.bin", -1); @@ -328,6 +354,105 @@ void test_asset_reload_preserves_data(const Workspace& workspace) { "failed OSL reload lost previous calibration"); } +void test_object_calibration(const Workspace& workspace) { + const ipc::SweepTraceBlock first{.combo = {7, 3}, .frequency_hz = {1e9F, 2e9F}, + .s11 = {{1, 2}, {3, 4}}, .s21 = {{2, 3}, {4, 5}}, .capture_start_ns = 5, .capture_end_ns = 15}; + const ipc::SweepTraceBlock second{.combo = {1, 9}, .frequency_hz = {3e9F, 4e9F, 5e9F}, + .s11 = {{}, {}, {}}, .s21 = {{6, 7}, {8, 9}, {10, 11}}}; + auto factors_first = first; + factors_first.s21 = {{0, 2}, {2, 0}}; + auto factors_second = second; + factors_second.s21 = {{1, -1}, {-1, 0}, {0, 1}}; + preprocessing::S21ObjectCalibrationBundle correction; + auto trace = first; + correction.apply(trace); + require(ipc::serialize_raw_collection({.traces = {trace}}) == ipc::serialize_raw_collection({.traces = {first}}), + "disabled object calibration changed a trace"); + correction.load(save_bundle(workspace, "object.bin", {.traces = {factors_second, factors_first}})); + correction.validate_combos({first.combo, second.combo}); + expect_error([&] { correction.validate_combos({{0, 0}}); }); + for (auto invalid : {first, second}) { + invalid.combo = {0, 0}; + expect_error([&] { correction.apply(invalid); }); + } + auto mismatch = second; + mismatch.frequency_hz[1] += 1e8F; // Same endpoints and count; interior grid differs. + expect_error([&] { correction.apply(mismatch); }); + mismatch = second; + mismatch.s21.pop_back(); + expect_error([&] { correction.apply(mismatch); }); + auto nonfinite = factors_first; + nonfinite.s21[0].re = std::numeric_limits::quiet_NaN(); + expect_error([&] { correction.load(save_bundle(workspace, "object_nan.bin", {.traces = {nonfinite}})); }); + expect_error([&] { correction.load(save_bundle(workspace, "object_duplicate.bin", {.traces = {factors_first, factors_first}})); }); + auto expected = first; + expected.s21 = {{-6, 4}, {8, 10}}; + trace = first; + correction.apply(trace); + require(ipc::serialize_raw_collection({.traces = {trace}}) == ipc::serialize_raw_collection({.traces = {expected}}), + "object calibration changed S11, timing, pair identity, or multiplication after a failed reload"); + trace = second; + correction.apply(trace); + require(trace.s21[0].re == 13 && trace.s21[0].im == 1 && trace.s21[1].re == -8 && trace.s21[1].im == -9 && + trace.s21[2].re == -11 && trace.s21[2].im == 10, "object calibration mixed pairs or frequency grids"); + trace = first; + trace.s21[0] = {std::numeric_limits::max(), std::numeric_limits::max()}; + expect_error([&] { correction.apply(trace); }); + correction.load(""); + trace = first; + correction.apply(trace); + require(trace.s21[0].re == 2 && trace.s21[0].im == 3, "clearing object calibration left stale factors"); +} + +void test_object_calibration_pipeline(const Workspace& workspace) { + const ipc::SweepTraceBlock trace{.combo = {7, 3}, .frequency_hz = {1e9F, 2e9F}, + .s11 = {{7, 3}, {8, 4}}, .s21 = {{2, 4}, {6, 8}}, .capture_start_ns = 5, .capture_end_ns = 15}; + auto standard = trace; + standard.s21 = {{2, 0}, {2, 0}}; + auto reference = trace; + reference.s21 = {{1, 2}, {2, 2}}; + auto factors = trace; + factors.s21 = {{0, 2}, {5, 1}}; + preprocessing::CalibrationMaster calibration(preprocessing::make_s21_through_calibrator()); + calibration.load_s21_calibration_bundle(save_bundle(workspace, "pipeline_through.bin", {.traces = {standard}})); + preprocessing::ReferenceMaster references; + references.load_s21_reference_bundle(save_bundle(workspace, "pipeline_reference.bin", {.traces = {reference}})); + references.prepare_calibrated(calibration, {trace.combo}); + config::RunConfig config; + config.run_combos = {trace.combo}; + config.preprocess.s21.object_calibration.bundle_path = + save_bundle(workspace, "pipeline_object.bin", {.traces = {factors}}); + config.preprocess.notch = {.enabled = true, .bands_hz = {{2e9F, 2e9F}}, .taper_type = "hard"}; + RingName raw_name("_object_raw"), pre_name("_object_pre"); + auto raw_ring = ipc::ShmRing::open_or_create(raw_name.name, 2, 4096); + auto pre_ring = ipc::ShmRing::open_or_create(pre_name.name, 2, 4096); + preprocessing::DataPreprocessor preprocessor(config, calibration, references, raw_ring, pre_ring); + require(raw_ring.push(ipc::serialize_raw_collection({.collection_id = 41, .capture_start_ns = 5, + .capture_end_ns = 15, .traces = {trace}})), "test raw ring could not hold a sweep"); + std::atomic stop{false}; + std::exception_ptr error; + std::jthread worker([&] { try { preprocessor.run(stop); } catch (...) { error = std::current_exception(); } }); + std::vector bytes; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (bytes.empty() && std::chrono::steady_clock::now() < deadline) { + (void)pre_ring.pop(bytes); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + stop.store(true); + worker.join(); + if (error) { std::rethrow_exception(error); } + require(!bytes.empty(), "object-corrected preprocessed collection was not published"); + const auto result = ipc::deserialize_preprocessed_collection(bytes); + const auto& output = result.traces.at(0); + require(result.collection_id == 41 && result.capture_start_ns == 5 && result.capture_end_ns == 15 && + output.combo == trace.combo && output.capture_start_ns == 5 && output.capture_end_ns == 15, + "object calibration lost capture identity or timing"); + require(output.s21[0].re == -2 && output.s21[0].im == 1 && output.s21[1].re == 0 && output.s21[1].im == 0, + "object calibration ran before existing preprocessing or was applied more than once"); + require(output.s11[0].re == 7 && output.s11[0].im == 3 && output.s11[1].re == 0 && output.s11[1].im == 0, + "object calibration changed S11 or bypassed the notch filter"); +} + void test_locator_packet_errors() { int sockets[2]{}; require(::socketpair(AF_UNIX, SOCK_STREAM, 0, sockets) == 0, "socketpair failed"); @@ -384,6 +509,8 @@ int main(int argc, char** argv) { test_wire_decoders(); test_configuration(argv[1], workspace); test_asset_reload_preserves_data(workspace); + test_object_calibration(workspace); + test_object_calibration_pipeline(workspace); test_ring(); test_partial_driver_open(); test_interruptible_wait(); diff --git a/mimo_calibration.py b/mimo_calibration.py index b244039..571308f 100644 --- a/mimo_calibration.py +++ b/mimo_calibration.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """Фазовая калибровка MIMO-радара по точечному отражателю (шару) и применение её к данным. -Расчёт тот же, что в MIMO_calibration.ipynb, без графиков и диагностики. Формат файлов -калибровки (cal_*.npy, cal_meta.json) и папки postprocessed — тот же. +Пример использует общий расчёт с интерфейсом программы. Формат файлов калибровки +(cal_*.npy, cal_meta.json) и папки postprocessed сохранён. Запуск целиком: python mimo_calibration.py (работает по параметрам из блока ниже) Из другого кода: @@ -13,19 +13,14 @@ s21_corr = mc.apply_calibration(s21, freq, cal) # массив (TX, RX, Nf) в памяти mc.apply_to_measurement(meas_dir, cal) # preprocessed -> postprocessed """ -import datetime -import json -import re -import shutil -from dataclasses import dataclass from pathlib import Path +from statistics import mean -import numpy as np -from scipy.optimize import minimize - -# ╔══════════════════════════════════════════════════════════════════════════════════════╗ -# ║ ЗАДАВАЕМЫЕ ПАРАМЕТРЫ ║ -# ╚══════════════════════════════════════════════════════════════════════════════════════╝ +from python_app.workflows.mimo_calibration import ( + Settings, apply_calibration, apply_to_measurement, calibration_on_grid, + compute_calibration, find_config_profile, load_calibration, load_geometry, + load_s21, save_calibration, +) # --- что делать ------------------------------------------------------------------------- RUN_CALIBRATION = True # True -> посчитать калибровку по CAL_SETS и записать в CAL_DIR; @@ -43,17 +38,14 @@ APPLY_TO_DATA = False # True -> записать исправленные CAL_ROOT = Path('/Users/ivan_root/Downloads/Telegram_dwnld/2026-09-31 balls calib/' 'librevna_multi_0.5-6GHz_1031pts_-3dBm_big_ball_h_0cm_z_5m_x_0cm/preprocessed') CAL_SETS = [ - dict(path=CAL_ROOT, snapshots=['0000'], target_scene_m=(0.0, 0.16, 5.20)), # Здесь нужно указывать координаты, измернные - dict(path=CAL_ROOT, snapshots=['0001'], target_scene_m=(0.0, 0.16, 5.20)), # в системе координат земли (x,y,z) - dict(path=CAL_ROOT, snapshots=['0002'], target_scene_m=(0.0, 0.16, 5.20)), # X - поеперчная координата от центра радара - dict(path=CAL_ROOT, snapshots=['0003'], target_scene_m=(0.0, 0.16, 5.20)), # y - выота от поверхности земли - dict(path=CAL_ROOT, snapshots=['0004'], target_scene_m=(0.0, 0.16, 5.20)), # z - длина вдоль земли от вертикальной плоскости, про -] # проходящей через нижние антенны до центра шара (до переднего края + радиус) + dict(path=CAL_ROOT, snapshots=[f'{index:04d}'], target_scene_m=(0.0, 0.16, 5.20)) + for index in range(5) +] SPHERE_RADIUS_M = 0.16 # радиус шара-отражателя [м]; 0 -> точечная цель EMPTY_SCENE_PATH = Path('/Users/ivan_root/Downloads/Telegram_dwnld/2026-09-31 balls calib/' 'librevna_multi_0.5-6GHz_1031pts_-3dBm_bkgd_h_0cm/preprocessed') - # папка снимков той же сцены БЕЗ отражателя (вычитается их среднее); - # None -> без вычитания + # Дополнительная пустая сцена для архивного примера. + # Если референс уже удалил фон, задайте None. # --- геометрия (как в софте построения изображений) --------------------------------------- INPUT_IDX = [0, 1, 2, 3] # физические входы (Rx), input_pos @@ -87,449 +79,18 @@ APPLY_TO_MEASUREMENTS = [ OVERWRITE_POSTPROCESSED = False # True -> пересоздать postprocessed (только созданный этой калибровкой: # папка без calibration_applied.json не удаляется никогда) -# ╔══════════════════════════════════════════════════════════════════════════════════════╗ -# ║ КОНЕЦ ЗАДАВАЕМЫХ ПАРАМЕТРОВ ║ -# ╚══════════════════════════════════════════════════════════════════════════════════════╝ - -C_LIGHT = 299_792_458.0 - - -@dataclass -class Settings: - """Настройки расчёта; значения по умолчанию — из блока параметров выше.""" - sphere_radius_m: float = SPHERE_RADIUS_M - input_idx: tuple = tuple(INPUT_IDX) - output_idx: tuple = tuple(OUTPUT_IDX) - antenna_height_m: float = ANTENNA_HEIGHT_M - pitch_deg: float = PITCH_DEG - f_min: float = CAL_F_MIN - f_max: float = CAL_F_MAX - gate_half_m: float = GATE_HALF_M - gate_taper_m: float = GATE_TAPER_M - coarse_range_search_m: float = COARSE_RANGE_SEARCH_M - refine_axes: tuple = tuple(REFINE_AXES) - refine_range_m: float = REFINE_RANGE_M - apply_amplitude: bool = APPLY_AMPLITUDE - phase_model: str = PHASE_MODEL - - -# ══════════════════════════════════════════════════════ -# ЗАГРУЗКА -# ══════════════════════════════════════════════════════ -def find_config_profile(start, max_levels=8): - p = Path(start) - for _ in range(max_levels): - if (p / 'config_profile.json').exists(): - return p / 'config_profile.json' - p = p.parent - raise FileNotFoundError(f'config_profile.json не найден выше {start}') - - -def load_geometry(config_path, input_idx, output_idx): - """Координаты антенн в осях радара [м], массивы (N,3) в порядке сортировки физических портов.""" - gpr = json.load(open(config_path, encoding='utf-8'))['gpr'] - tx = {int(e['output_pos']): (e['x_m'], e.get('y_m', 0.0), e.get('z_m', 0.0)) for e in gpr['tx_geometry']} - rx = {int(e['input_pos']): (e['x_m'], e.get('y_m', 0.0), e.get('z_m', 0.0)) for e in gpr['rx_geometry']} - return (np.array([tx[k] for k in sorted(output_idx)], float), - np.array([rx[k] for k in sorted(input_idx)], float)) - - -def list_snapshots(root, prefixes=None): - dirs = sorted(p for p in Path(root).iterdir() if p.is_dir() and (p / 'meta.json').exists()) - if prefixes is None: - return dirs - out = [d for d in dirs if any(d.name.startswith(str(pr)) for pr in prefixes)] - if len(out) != len(prefixes): - raise FileNotFoundError(f'В {root} найдены не все снимки {prefixes}') - return out - - -def load_s21(snapshot_dir, input_idx, output_idx): - """S21 (TX, RX, Nf) и частоты. Индексы — порядок сортировки физических портов.""" - s21, freq = None, None - for a, o in enumerate(sorted(output_idx)): - for b, i in enumerate(sorted(input_idx)): - s = np.load(Path(snapshot_dir) / f'i{i}_o{o}_s21.npy') - f = np.load(Path(snapshot_dir) / f'i{i}_o{o}_freq.npy') - if s21 is None: - s21 = np.zeros((len(output_idx), len(input_idx), s.size), complex); freq = f - elif not np.allclose(f, freq): - raise ValueError(f'Разные частотные сетки у пар в {snapshot_dir}') - s21[a, b] = s - return s21, freq - - -# ══════════════════════════════════════════════════════ -# РАСЧЁТ КАЛИБРОВКИ -# ══════════════════════════════════════════════════════ -def best_phases(Z): - """max |Σ Z_ij·e^{-j(t_i+r_j)}| по фазам антенн, t_0 = r_0 = 0.""" - t = np.zeros(Z.shape[0]); r = np.zeros(Z.shape[1]) - for _ in range(100): - t = np.angle((Z * np.exp(-1j * r)[None, :]).sum(axis=1)) - r = np.angle((Z * np.exp(-1j * t)[:, None]).sum(axis=0)) - t = t - t[0]; r = r - r[0] - return t, r - - -def coherence(Z, t=None, r=None): - """Сфазированность вкладов пар: |Σ| / Σ|·| (1 — все в фазе).""" - if t is not None: - Z = Z * np.exp(-1j * (t[:, None] + r[None, :])) - return float(np.abs(Z.sum()) / (np.abs(Z).sum() + 1e-30)) - - -def circular_median(phases): - """Круговая медиана по первой оси: значение с минимальной суммой угловых расстояний до остальных.""" - d = np.abs(np.angle(np.exp(1j * (phases[:, None] - phases[None, :])))) - best = np.argmin(d.sum(axis=1), axis=0) - return np.take_along_axis(phases, best[None], axis=0)[0] - - -def combine(sets): - """Добавки антенн по всем калибровочным точкам: круговая медиана фаз и медиана усиления.""" - out = [] - for key in ('T', 'R'): - Z = np.array([s[key] for s in sets]) # (точки, антенны, частоты) - out.append(np.exp(np.median(np.log(np.abs(Z)), axis=0)) * np.exp(1j * circular_median(np.angle(Z)))) - return tuple(out) - - -class _Estimator: - """Геометрия, полоса и окно одной калибровки.""" - - def __init__(self, st, tx, rx, freq): - self.st, self.tx, self.rx, self.freq = st, tx, rx, freq - self.n_tx, self.n_rx = len(tx), len(rx) - self.band = (freq >= st.f_min) & (freq <= st.f_max) - self.fb = freq[self.band] - self.win = np.hanning(self.fb.size) - self.ok = self.win > 0.2 # края полосы после деления на окно ненадёжны - self.nfft = 1 << int(np.ceil(np.log2(self.fb.size * 8))) - self.dpath = np.fft.fftfreq(self.nfft, self.fb[1] - self.fb[0]) * C_LIGHT # ось «лишний путь», м - self.gate = np.clip(np.minimum((self.dpath + st.gate_half_m) / st.gate_taper_m, - (st.gate_half_m - self.dpath) / st.gate_taper_m) + 0.5, 0, 1) - - def scene_to_radar(self, q): - """(X, Y над землёй, Z вдоль земли) -> оси радара (x, y, z).""" - p = np.radians(self.st.pitch_deg); y0 = q[1] - self.st.antenna_height_m - return np.array([q[0], y0 * np.cos(p) + q[2] * np.sin(p), -y0 * np.sin(p) + q[2] * np.cos(p)]) - - def pair_paths(self, q_scene): - """Бистатический путь каждой пары до цели (TX, RX). Для шара радиуса a (q_scene — центр) - путь короче на 2a·cos(beta/2), beta — угол между направлениями из центра на Tx и на Rx.""" - a = self.st.sphere_radius_m - pr = self.scene_to_radar(np.asarray(q_scene, float)) - vt = self.tx - pr; vr = self.rx - pr - dt = np.linalg.norm(vt, axis=1); dr = np.linalg.norm(vr, axis=1) - paths = dt[:, None] + dr[None, :] - if a: - cos_beta = (vt @ vr.T) / (dt[:, None] * dr[None, :]) - paths = paths - 2 * a * np.sqrt((1 + cos_beta) / 2) - return paths - - def _unwound(self, s21_band, paths): - return s21_band * np.exp(2j * np.pi * self.fb[None, None, :] * paths[:, :, None] / C_LIGHT) - - def target_echo(self, s21_band, paths): - """Эхо цели E (TX, RX, Nb) со снятым путём: окно ±gate_half_m вокруг заданной точки.""" - h = np.fft.ifft(self._unwound(s21_band, paths) * self.win, self.nfft, axis=2) - return np.fft.fft(h * self.gate, axis=2)[:, :, :self.fb.size] / np.maximum(self.win, 1e-3) - - def broadband(self, E): - """Вклад пары в изображение в точке: сумма эха по надёжной части полосы.""" - return E[:, :, self.ok].sum(axis=2) - - def coarse_range(self, s21, q_nominal): - """Грубая привязка по дальности: максимум суммарной огибающей всех пар в пределах - ±coarse_range_search_m по пути; заданная дальность Z сдвигается под него.""" - q = q_nominal.copy() - if not self.st.coarse_range_search_m: - return q - s = np.mean(s21, axis=0)[:, :, self.band]; sel = np.abs(self.dpath) <= self.st.coarse_range_search_m - for _ in range(3): - env = np.abs(np.fft.ifft(self._unwound(s, self.pair_paths(q)) * self.win, self.nfft, axis=2)).sum(axis=(0, 1)) - q[2] += self.dpath[sel][np.argmax(env[sel])] / 2 - return q - - def refine_target(self, s21, q0): - """Положение цели и широкополосные фазы антенн, максимизирующие сфазированность.""" - P0 = self.pair_paths(q0) - E0 = np.mean([self.target_echo(s[:, :, self.band], P0) for s in s21], axis=0) - axes = [('xyz').index(a) for a in self.st.refine_axes] - - def E_at(v): - q = q0.copy(); q[axes] += v - dP = self.pair_paths(q) - P0 - return E0 * np.exp(2j * np.pi * self.fb[None, None, :] * dP[:, :, None] / C_LIGHT), q - - def score(v, t, r): - if np.any(np.abs(v) > self.st.refine_range_m): - return 0.0 - return coherence(self.broadband(E_at(v)[0]), t, r) - - v = np.zeros(len(axes)) - if axes: - for _ in range(6): - t, r = best_phases(self.broadband(E_at(v)[0])) - simplex = np.vstack([v] + [v + 0.03 * np.eye(len(axes))[k] for k in range(len(axes))]) - v = minimize(lambda w: -score(w, t, r), v, method='Nelder-Mead', - options=dict(xatol=5e-4, fatol=1e-7, initial_simplex=simplex)).x - return E_at(v) - - def factorize(self, E): - """E_ij(f) ≈ g(f)·T_i(f)·R_j(f) на каждой частоте: фазы — best_phases, амплитуды — МНК по log|E|.""" - nb = E.shape[2] - T = np.ones((self.n_tx, nb), complex); R = np.ones((self.n_rx, nb), complex) - L = np.log(np.abs(E) + 1e-30) - for k in range(nb): - t, r = best_phases(E[:, :, k]) - tt = L[:, :, k].mean(axis=1) - L[:, :, k].mean(); rr = L[:, :, k].mean(axis=0) - L[:, :, k].mean() - T[:, k] = np.exp(tt + 1j * t); R[:, k] = np.exp(rr + 1j * r) - return T, R - - def antenna_params(self, Z): - """Фаза, продолженная к 0 Гц, задержка (эквивалентный лишний путь) и усиление антенны.""" - ph = np.unwrap(np.angle(Z[self.ok])); f = self.fb[self.ok] - slope, ph0 = np.polyfit(f, ph, 1) - return dict(phase0_deg=float(np.degrees(ph0)), delay_path_m=float(-slope * C_LIGHT / (2 * np.pi)), - gain_db=float(20 * np.log10(np.mean(np.abs(Z[self.ok])))), - phase_rms_after_linear_deg=float(np.degrees(np.std(ph - (slope * f + ph0))))) - - def phase_model(self, Z): - """Фаза добавки антенны по модели phase_model (подгонка по надёжной части полосы).""" - model = self.st.phase_model - if model == 'raw': - return Z - ph = np.unwrap(np.angle(Z)); f = (self.fb - self.fb.mean()) / 1e9 - if model == 'const': - u = np.exp(1j * np.angle(Z[self.ok])).mean() - return np.abs(Z) * np.exp(1j * np.angle(u)) - deg = {'linear': 1, 'poly3': 3}.get(model) - if deg is None: - raise ValueError(f"phase_model должен быть 'const', 'linear', 'poly3' или 'raw', а не {model!r}") - return np.abs(Z) * np.exp(1j * np.polyval(np.polyfit(f[self.ok], ph[self.ok], deg), f)) - - def pair_correction(self, T, R): - """Поправка пары C_ij(f) = 1/(T_i·R_j); без apply_amplitude — только фаза.""" - Z = T[:, None, :] * R[None, :, :] - return np.conj(Z) / np.abs(Z) if not self.st.apply_amplitude else 1.0 / Z - - def to_full_grid(self, Z_band, params): - """На полную сетку частот: внутри надёжной полосы — оценка, вне её — продолжение с наклоном - линейной модели фазы антенны, пристыкованное к краю полосы без скачка.""" - idx = np.flatnonzero(self.band)[self.ok] - ph_in = np.unwrap(np.angle(Z_band[self.ok])); mag_in = np.abs(Z_band[self.ok]) - slope = -2 * np.pi * params['delay_path_m'] / C_LIGHT # рад/Гц - ph = np.empty(self.freq.size); mag = np.empty(self.freq.size) - ph[idx] = ph_in; mag[idx] = mag_in - lo, hi = idx[0], idx[-1] - ph[:lo] = ph_in[0] + slope * (self.freq[:lo] - self.freq[lo]); mag[:lo] = mag_in[0] - ph[hi + 1:] = ph_in[-1] + slope * (self.freq[hi + 1:] - self.freq[hi]); mag[hi + 1:] = mag_in[-1] - return mag * np.exp(1j * ph) - - -def compute_calibration(cal_sets, empty_scene_path=None, settings=None): - """Калибровка по набору измерений отражателя. Возвращает словарь в том же виде, что load_calibration.""" - st = settings or Settings() - cal_sets = list(cal_sets) - config_path = find_config_profile(cal_sets[0]['path']) - tx, rx = load_geometry(config_path, st.input_idx, st.output_idx) - - freq = None - empty = None - if empty_scene_path is not None: - loaded = [load_s21(d, st.input_idx, st.output_idx) for d in list_snapshots(empty_scene_path)] - empty = np.mean([s for s, _ in loaded], axis=0); freq = loaded[0][1] - - sets = [] - for n, cs in enumerate(cal_sets): - snaps = list_snapshots(cs['path'], cs.get('snapshots')) - data = [] - for d in snaps: - s21, f = load_s21(d, st.input_idx, st.output_idx) - if freq is None: - freq = f - elif f.shape != freq.shape or not np.allclose(f, freq): - raise ValueError(f'Частотная сетка в {d} отличается от остальных калибровочных данных') - cfg = find_config_profile(d) - if cfg != config_path: - tx_d, rx_d = load_geometry(cfg, st.input_idx, st.output_idx) - if not (np.allclose(tx_d, tx) and np.allclose(rx_d, rx)): - raise ValueError(f'Геометрия в {d} отличается от {config_path}') - data.append(s21 - empty if empty is not None else s21) - sets.append(dict(n=n, snaps=[d.name for d in snaps], s21=np.array(data), - q_nominal=np.array(cs['target_scene_m'], float))) - - est = _Estimator(st, tx, rx, freq) - for s in sets: - s['E'], s['q'] = est.refine_target(s['s21'], est.coarse_range(s['s21'], s['q_nominal'])) - s['T'], s['R'] = est.factorize(s['E']) - - T_all, R_all = combine(sets) - T_all = np.array([est.phase_model(z) for z in T_all]); R_all = np.array([est.phase_model(z) for z in R_all]) - ant = {f'Tx{k}': est.antenna_params(T_all[k]) for k in range(est.n_tx)} - ant.update({f'Rx{k}': est.antenna_params(R_all[k]) for k in range(est.n_rx)}) - - # качество: сфазированность пар в точке цели без поправки и с ней - coh_with = lambda s, C: coherence(est.broadband(s['E'] * C)) - quality = dict(coherence_before=[], coherence_own=[], coherence_leave_one_out=[], coherence_final=[]) - for s in sets: - others = [o for o in sets if o is not s] - quality['coherence_before'].append(coh_with(s, 1.0)) - quality['coherence_own'].append(coh_with(s, est.pair_correction(s['T'], s['R']))) - quality['coherence_leave_one_out'].append(coh_with(s, est.pair_correction(*combine(others))) if others else float('nan')) - quality['coherence_final'].append(coh_with(s, est.pair_correction(T_all, R_all))) - - T_full = np.array([est.to_full_grid(T_all[k], ant[f'Tx{k}']) for k in range(est.n_tx)]) - R_full = np.array([est.to_full_grid(R_all[k], ant[f'Rx{k}']) for k in range(est.n_rx)]) - Z_full = T_full[:, None, :] * R_full[None, :, :] - unit = (lambda Z: np.conj(Z) / np.abs(Z)) if not st.apply_amplitude else (lambda Z: 1.0 / Z) - valid = np.zeros(freq.size, bool); valid[np.flatnonzero(est.band)[est.ok]] = True - - meta = dict( - created=datetime.datetime.now().isoformat(timespec='seconds'), name=None, - convention=dict(apply='S21_corrected[tx, rx, f] = S21[tx, rx, f] * cal_pair[tx, rx, f]', - index_order='tx = номер в cal_tx_output_pos (output_pos по возрастанию), rx = cal_rx_input_pos', - files='S21 пары в данных: i{input_pos}_o{output_pos}_s21.npy', - phase_reference='фазы относительно Tx0 и Rx0; общая фаза/задержка не входит', - amplitude='выравнивается' if st.apply_amplitude else 'не меняется (только фаза)', - outside_band='вне cal_valid: продолжение линейной моделью фазы антенны'), - geometry=dict(config=str(config_path), tx_xyz_m=tx.tolist(), rx_xyz_m=rx.tolist(), - antenna_height_m=st.antenna_height_m, pitch_deg=st.pitch_deg), - estimation=dict(band_hz=[st.f_min, st.f_max], gate_half_m=st.gate_half_m, refine_axes=list(st.refine_axes), - phase_model=st.phase_model, sphere_radius_m=st.sphere_radius_m, empty_scene=str(empty_scene_path)), - targets=[dict(path=str(cal_sets[s['n']]['path']), snapshots=s['snaps'], nominal_scene_m=s['q_nominal'].tolist(), - refined_scene_m=s['q'].tolist()) for s in sets], - antennas=ant, quality=quality) - return dict( - freq_hz=freq.astype(np.float64), - pair=unit(Z_full).astype(np.complex64), # (TX, RX, Nf): S21_corr = S21 * pair - tx=unit(T_full).astype(np.complex64), # (TX, Nf): pair = tx[:,None] * rx[None,:] - rx=unit(R_full).astype(np.complex64), # (RX, Nf) - valid=valid, # (Nf,) True — внутри полосы калибровки - tx_output_pos=np.array(sorted(st.output_idx), np.int32), # индекс -> физический выход - rx_input_pos=np.array(sorted(st.input_idx), np.int32), # индекс -> физический вход - antenna_params=np.array([[np.radians(a['phase0_deg']), a['delay_path_m'], a['gain_db']] for a in ant.values()], - np.float64), # (TX+RX, 3): фаза к 0 Гц [рад], путь [м], усиление [дБ] - meta=meta, dir=None) - - -# ══════════════════════════════════════════════════════ -# ФАЙЛЫ КАЛИБРОВКИ -# ══════════════════════════════════════════════════════ -_CAL_ARRAYS = ('freq_hz', 'pair', 'tx', 'rx', 'valid', 'tx_output_pos', 'rx_input_pos', 'antenna_params') - - -def save_calibration(cal, cal_dir, overwrite=False): - cal_dir = Path(cal_dir) - cal_dir.mkdir(parents=True, exist_ok=True) - existing = [p.name for p in cal_dir.glob('cal_*')] - if existing and not overwrite: - raise FileExistsError(f'В {cal_dir} уже есть файлы калибровки ({existing[:3]}...). ' - f'Задайте новое имя или overwrite=True.') - for k in _CAL_ARRAYS: - np.save(cal_dir / f'cal_{k}.npy', cal[k]) - cal['meta']['name'] = cal_dir.name; cal['dir'] = cal_dir - json.dump(cal['meta'], open(cal_dir / 'cal_meta.json', 'w', encoding='utf-8'), ensure_ascii=False, indent=2) - return cal_dir - - -def load_calibration(cal_dir): - cal_dir = Path(cal_dir) - cal = {k: np.load(cal_dir / f'cal_{k}.npy') for k in _CAL_ARRAYS} - cal['meta'] = json.load(open(cal_dir / 'cal_meta.json', encoding='utf-8')) - cal['dir'] = cal_dir - return cal - - -# ══════════════════════════════════════════════════════ -# ПРИМЕНЕНИЕ -# ══════════════════════════════════════════════════════ -def calibration_on_grid(cal, freq): - """Поправка пар (TX, RX, Nf) на частотной сетке freq. Совпадает с сеткой калибровки — берётся как есть; - иначе фаза и модуль поправок антенн интерполируются, вне сетки калибровки — линейная модель фазы.""" - fc = cal['freq_hz'] - if freq.shape == fc.shape and np.allclose(freq, fc): - return cal['pair'] - - def interp(Z, par): - ph = np.unwrap(np.angle(Z)); mag = np.abs(Z) - out_ph = np.interp(freq, fc, ph); out_mag = np.interp(freq, fc, mag) - lin = -(par[0] - 2 * np.pi * freq * par[1] / C_LIGHT) # поправка = −добавка - outside = (freq < fc[0]) | (freq > fc[-1]) - out_ph[outside] = lin[outside] + np.round((out_ph[outside] - lin[outside]) / (2 * np.pi)) * 2 * np.pi - return out_mag * np.exp(1j * out_ph) - - n_tx = cal['tx'].shape[0] - tx = np.array([interp(cal['tx'][k], cal['antenna_params'][k]) for k in range(n_tx)]) - rx = np.array([interp(cal['rx'][k], cal['antenna_params'][n_tx + k]) for k in range(cal['rx'].shape[0])]) - return (tx[:, None, :] * rx[None, :, :]).astype(cal['pair'].dtype) - - -def apply_calibration(s21, freq, cal): - """s21 (TX, RX, Nf) в порядке cal['tx_output_pos'] x cal['rx_input_pos'] -> исправленный массив.""" - return s21 * calibration_on_grid(cal, np.asarray(freq)) - - -def apply_to_measurement(meas_dir, cal, overwrite=False): - """<измерение>/preprocessed -> <измерение>/postprocessed: S21 пар умножаются на поправку, - остальные файлы копируются. Возвращает (папка, снимков, исправлено S21, скопировано файлов).""" - meas_dir = Path(meas_dir); pre = meas_dir / 'preprocessed'; post = meas_dir / 'postprocessed' - if not pre.is_dir(): - raise FileNotFoundError(f'Нет {pre}') - # калибровка годится только для той же сборки антенн - tx_m, rx_m = load_geometry(find_config_profile(pre), cal['rx_input_pos'].tolist(), cal['tx_output_pos'].tolist()) - if not (np.allclose(tx_m, cal['meta']['geometry']['tx_xyz_m']) and np.allclose(rx_m, cal['meta']['geometry']['rx_xyz_m'])): - raise ValueError(f'Геометрия антенн в {meas_dir.name} отличается от геометрии калибровки — применять нельзя') - if post.exists(): - if not overwrite: - raise FileExistsError(f'{post} уже существует. overwrite=True, чтобы пересоздать.') - if not (post / 'calibration_applied.json').exists(): - raise FileExistsError(f'{post} существует, но создан не калибровкой (нет calibration_applied.json) — ' - f'удалять не буду, разберитесь вручную.') - shutil.rmtree(post) - tx_of = {int(p): k for k, p in enumerate(cal['tx_output_pos'])} - rx_of = {int(p): k for k, p in enumerate(cal['rx_input_pos'])} - pat = re.compile(r'^i(\d+)_o(\d+)_s21\.npy$') - n_snap = n_fix = n_copy = 0 - grid_cache = {} - post.mkdir() - for item in sorted(pre.iterdir()): - if item.is_file(): - shutil.copy2(item, post / item.name); n_copy += 1 - continue - out = post / item.name; out.mkdir(); n_snap += 1 - for f in sorted(item.iterdir()): - m = pat.match(f.name) - if f.is_dir() or m is None or int(m.group(2)) not in tx_of or int(m.group(1)) not in rx_of: - if f.is_dir(): - shutil.copytree(f, out / f.name) - else: - shutil.copy2(f, out / f.name) - n_copy += 1 - continue - s21 = np.load(f); freq = np.load(item / f'i{m.group(1)}_o{m.group(2)}_freq.npy') - key = (freq.size, float(freq[0]), float(freq[-1])) - if key not in grid_cache: - grid_cache[key] = calibration_on_grid(cal, freq) - corr = grid_cache[key][tx_of[int(m.group(2))], rx_of[int(m.group(1))]] - np.save(out / f.name, (s21 * corr).astype(s21.dtype)); n_fix += 1 - json.dump(dict(calibration=cal['meta']['name'], calibration_dir=str(Path(cal['dir']).resolve()), - created=datetime.datetime.now().isoformat(timespec='seconds'), source=str(pre), - rule='postprocessed i*_o*_s21 = preprocessed i*_o*_s21 * cal_pair[tx(o), rx(i)]; остальные файлы скопированы', - frequency_grids=[list(k) for k in grid_cache]), - open(post / 'calibration_applied.json', 'w', encoding='utf-8'), ensure_ascii=False, indent=2) - return post, n_snap, n_fix, n_copy - - -def main(): +def main() -> None: if RUN_CALIBRATION: - cal = compute_calibration(CAL_SETS, EMPTY_SCENE_PATH, Settings()) + cal = compute_calibration(CAL_SETS, EMPTY_SCENE_PATH, Settings( + sphere_radius_m=SPHERE_RADIUS_M, input_idx=tuple(INPUT_IDX), output_idx=tuple(OUTPUT_IDX), + antenna_height_m=ANTENNA_HEIGHT_M, pitch_deg=PITCH_DEG, f_min=CAL_F_MIN, f_max=CAL_F_MAX, + gate_half_m=GATE_HALF_M, gate_taper_m=GATE_TAPER_M, coarse_range_search_m=COARSE_RANGE_SEARCH_M, + refine_axes=tuple(REFINE_AXES), refine_range_m=REFINE_RANGE_M, + apply_amplitude=APPLY_AMPLITUDE, phase_model=PHASE_MODEL)) save_calibration(cal, CAL_DIR, overwrite=OVERWRITE) q = cal['meta']['quality'] print(f'Калибровка записана в {CAL_DIR}: сфазированность пар на отражателе ' - f'{np.mean(q["coherence_before"]):.2f} -> {np.mean(q["coherence_final"]):.2f}') + f'{mean(q["coherence_before"]):.2f} -> {mean(q["coherence_final"]):.2f}') if APPLY_TO_DATA: cal = load_calibration(CAL_DIR) targets = APPLY_TO_MEASUREMENTS diff --git a/python_app/gui/app_window.py b/python_app/gui/app_window.py index 09b595e..c494bf2 100644 --- a/python_app/gui/app_window.py +++ b/python_app/gui/app_window.py @@ -791,6 +791,9 @@ class AppWindow( # 0) Stop the GPIO button watcher so a late press cannot start work. self._stop_control_button_watcher() self._resume_pipeline_after_capture = False + controller = getattr(self, "_object_calibration_controller", None) + if controller is not None: + controller.shutdown() # 0) Drop the B-scan replay writer so a pending debounce cannot push into a # ring we are about to tear down. self._bscan_reprocess_timer.stop() diff --git a/python_app/gui/controllers/app_window_config/live_processing_mixin.py b/python_app/gui/controllers/app_window_config/live_processing_mixin.py index bc3d81e..f8ed43d 100644 --- a/python_app/gui/controllers/app_window_config/live_processing_mixin.py +++ b/python_app/gui/controllers/app_window_config/live_processing_mixin.py @@ -17,6 +17,7 @@ from PyQt6.QtWidgets import ( from python_app.gui.runtime.constraints import validate_processing_mode_constraints from python_app.gui.antenna_geometry_editor import AntennaGeometryEditor +from python_app.models.run_config_model import GprRxGeometryModel, GprTxGeometryModel from python_app.orchestration.live_processing_config import ProcessingLiveConfig _GPR_MODES = ("gpr", "legacy_gpr") @@ -320,6 +321,10 @@ class AppWindowLiveProcessingMixin: reprocess_current_result=not self._can_reprocess_history(), ) ) + active = getattr(self, "_active_run_config", None) + if active is not None and self._supervisor.is_running(): + active.gpr.tx_geometry = [GprTxGeometryModel(**entry) for entry in self._gpr_tx_geometry_input.records()] + active.gpr.rx_geometry = [GprRxGeometryModel(**entry) for entry in self._gpr_rx_geometry_input.records()] if getattr(self, "_await_processing_revision", False): self._required_processing_revision = self._processing_settings_revision @@ -379,6 +384,8 @@ class AppWindowLiveProcessingMixin: finally: self._suppress_live_settings_handler = False self._sync_socket_speed_controls() + if {"tx_geometry", "rx_geometry"} & settings.keys(): + self._invalidate_object_calibration_after_geometry_change() if {"tx_geometry", "rx_geometry", "relative_permittivity"} & settings.keys(): self._await_processing_revision = True if "processor_mode" in settings: @@ -395,9 +402,49 @@ class AppWindowLiveProcessingMixin: def _on_gpr_geometry_changed(self) -> None: if getattr(self, "_suppress_live_settings_handler", False): return + self._invalidate_object_calibration_after_geometry_change() self._await_processing_revision = True self._on_processing_live_settings_changed() + def _invalidate_object_calibration_after_geometry_change(self) -> None: + """Disable a correction tied to another antenna layout and reload the preprocessor.""" + selected = self._selected_preprocess_sets.get("s21_object_calibration", "") + was_running = self._supervisor.is_running() + active_config = getattr(self, "_active_run_config", None) + active = active_config.preprocess.s21.object_calibration.set_name if was_running and active_config else "" + if not selected and not active: + return + from python_app.workflows.mimo_calibration import validate_for_config + + geometry = {"gpr": { + "tx_geometry": self._gpr_tx_geometry_input.records(), + "rx_geometry": self._gpr_rx_geometry_input.records(), + }} + selections = [(selected, self._selected_preprocess_radar_key)] + if active: + selections.append((active, self._radar_key(active_config))) + invalid = set() + for name, radar_key in dict.fromkeys(selections): + if not name: + continue + try: + validate_for_config(self._store.load_object_calibration(radar_key, name), geometry) + except Exception as exc: # noqa: BLE001 - reject unavailable or incompatible saved corrections + invalid.add(name) + self._log_warning(f"Object calibration {name} disabled for changed antenna geometry: {exc}") + if not invalid: + return + resume_seq = getattr(self, "_stop_request_seq", 0) + 1 + if active in invalid: + self._stop_run() + if selected in invalid: + self._selected_preprocess_sets["s21_object_calibration"] = "" + self._refresh_preprocess_summary_labels() + if self._preprocess_dialog is not None: + self._preprocess_dialog.set_selected_sets(self._selected_preprocess_sets, emit_signal=False) + if active in invalid and self._pipeline_resume_allowed(resume_seq): + self._start_run() + def _on_processing_live_settings_changed(self, *_args) -> None: """Handle live-processing setting changes and trigger redraw when needed.""" # Web-apply sets many widgets at once; their individual change signals are diff --git a/python_app/gui/controllers/app_window_config/profile_io_mixin.py b/python_app/gui/controllers/app_window_config/profile_io_mixin.py index b3b3069..70197c7 100644 --- a/python_app/gui/controllers/app_window_config/profile_io_mixin.py +++ b/python_app/gui/controllers/app_window_config/profile_io_mixin.py @@ -4,6 +4,7 @@ from __future__ import annotations from collections import deque from contextlib import ExitStack +from copy import deepcopy import json from pathlib import Path @@ -23,12 +24,23 @@ from python_app.orchestration.preprocess_assets import ( class AppWindowConfigProfileIOMixin: """Persist and restore full GUI profiles without restarting the app.""" - def _write_gui_profile_to_path(self, output_path: Path, *, allow_overwrite: bool = True) -> GuiProfileModel: - """Serialize current full GUI profile to `output_path` and return the persisted model.""" + def _write_gui_profile_to_path( + self, output_path: Path, *, allow_overwrite: bool = True, dataset: bool = False, + ) -> GuiProfileModel: + """Save UI settings, preserving the acquisition provenance for recorded data.""" if not allow_overwrite and output_path.exists(): raise FileExistsError(f"Config profile output already exists: {output_path}") profile = self._build_gui_profile() + active = getattr(self, "_active_run_config", None) + if dataset and active is not None: + profile.run_config.radar = deepcopy(active.radar) + profile.run_config.input_switch = deepcopy(active.input_switch) + profile.run_config.output_switch = deepcopy(active.output_switch) + profile.run_config.combos = deepcopy(active.combos) + profile.run_config.preprocess = deepcopy(active.preprocess) + profile.run_config.gpr.tx_geometry = deepcopy(active.gpr.tx_geometry) + profile.run_config.gpr.rx_geometry = deepcopy(active.gpr.rx_geometry) atomic_write_json(output_path, profile.to_dict(), durable=True) return profile @@ -206,6 +218,9 @@ class AppWindowConfigProfileIOMixin: capture sequence or the pipeline is active — the operator must stop first. Shared by the desktop dialog and the web loader so both honor the same rule. """ + if getattr(self, "_object_capture_job", None) is not None: + self._show_error("Wait for object-calibration capture to finish") + return False if self._capture_session is not None: self._show_error( "Cannot load config during active capture sequence", diff --git a/python_app/gui/controllers/app_window_pipeline_mixin.py b/python_app/gui/controllers/app_window_pipeline_mixin.py index 7875858..bf02e50 100644 --- a/python_app/gui/controllers/app_window_pipeline_mixin.py +++ b/python_app/gui/controllers/app_window_pipeline_mixin.py @@ -57,13 +57,16 @@ class AppWindowPipelineMixin: """Start acquisition in single-capture mode.""" self._start_run(single_capture=True) - def _start_run(self, *, single_capture: bool = False) -> None: + def _start_run(self, *, single_capture: bool = False, config_override: RunConfigModel | None = None) -> None: """Start pipeline processes and ring readers.""" if getattr(self, "_closing", False) or getattr(self, "_close_after_stop", False): return if getattr(self, "_stopping_run", False): self._show_error("Wait for pipeline shutdown before starting again") return + if config_override is None and getattr(self, "_object_capture_job", None) is not None: + self._show_error("Wait for object-calibration capture to finish") + return if self._capture_session is not None: self._show_error( "Cannot start pipeline during active capture sequence", @@ -88,7 +91,7 @@ class AppWindowPipelineMixin: try: processor_was_running = self._supervisor.is_processor_running() - config = self._build_config() + config = config_override.clone() if config_override is not None else self._build_config() validate_acquisition_config(config) run_signature = self._build_run_history_signature(config) processor_signature = self._build_processor_run_signature(config) @@ -101,7 +104,8 @@ class AppWindowPipelineMixin: if not processor_was_running: self._reset_runtime_history() - self._validate_processing_mode_constraints(config) + if config_override is None: + self._validate_processing_mode_constraints(config) radar_key = self._radar_key(config) missing_assets = [ @@ -145,14 +149,12 @@ class AppWindowPipelineMixin: f"{', ' + combo_preview if combo_preview else ''}, radar_key={radar_key}" ) - if not single_capture: - should_reset_history = ( - self._history_run_signature is not None and self._history_run_signature != run_signature - ) - if should_reset_history: + if self._history_run_signature is not None and self._history_run_signature != run_signature: + if processor_was_running: self._reset_runtime_history() - self._log("History reset because run settings changed") - self._history_run_signature = run_signature + self._write_live_processing_config(history_command="clear_all", bump_history_seq=True) + self._log("History reset because run settings changed") + self._history_run_signature = run_signature # Open the single-capture acquisition window BEFORE the producer can emit: # any collection the one-shot orchestrator publishes is then strictly newer @@ -211,6 +213,9 @@ class AppWindowPipelineMixin: def _apply_radar_settings(self) -> None: """Apply current radar settings by preconfiguring native device.""" + if getattr(self, "_object_capture_job", None) is not None: + self._show_error("Wait for object-calibration capture to finish") + return if self._capture_session is not None: self._show_error( "Finish or abort capture sequence before applying radar settings", diff --git a/python_app/gui/controllers/app_window_preprocess_mixin.py b/python_app/gui/controllers/app_window_preprocess_mixin.py index 2929738..973b1ef 100644 --- a/python_app/gui/controllers/app_window_preprocess_mixin.py +++ b/python_app/gui/controllers/app_window_preprocess_mixin.py @@ -5,6 +5,7 @@ from __future__ import annotations from PyQt6.QtCore import QTimer from python_app.gui.preprocess_dialog import PreprocessDialog +from python_app.gui.controllers.object_calibration_controller import ObjectCalibrationController from python_app.gui.trace_png_export import export_trace_png from python_app.orchestration.preprocess_assets import ( PREPROCESS_ASSET_SPECS, @@ -124,6 +125,9 @@ class AppWindowPreprocessMixin: """Capture, save, and select a temporary S21 reference with current sweep settings.""" if getattr(self, "_closing", False) or getattr(self, "_stopping_run", False): return + if getattr(self, "_object_capture_job", None) is not None: + self._show_error("Wait for object-calibration capture to finish") + return if self._capture_session is not None: self._show_error( "Cannot capture tmp reference during active capture sequence", @@ -227,6 +231,9 @@ class AppWindowPreprocessMixin: dialog.finalize_sequence_requested.connect(self._finalize_capture_sequence) dialog.abort_sequence_requested.connect(self._abort_capture_sequence) dialog.create_neutral_sets_requested.connect(self._create_neutral_sets) + self._object_calibration_controller = ObjectCalibrationController(self) + dialog.import_object_calibration_requested.connect(self._object_calibration_controller.import_calibration) + dialog.create_object_calibration_requested.connect(self._object_calibration_controller.open_creation) dialog.set_neutral_sets_visible(supports_neutral_preprocess_sets(self._defaults_config)) dialog.set_radar_config_summary( directory_path=self._preprocess_radar_scan_summary.directory_path, @@ -340,6 +347,9 @@ class AppWindowPreprocessMixin: """Start sequential capture session for requested preprocess asset.""" if getattr(self, "_closing", False) or getattr(self, "_stopping_run", False): return + if getattr(self, "_object_capture_job", None) is not None: + self._show_error("Wait for object-calibration capture to finish") + return if self._capture_session is not None: self._show_error("Another capture sequence is already active", details=self._capture_state_details()) return @@ -410,6 +420,9 @@ class AppWindowPreprocessMixin: """Save neutral S21 calibration/reference sets for the current radar settings.""" if getattr(self, "_closing", False) or getattr(self, "_stopping_run", False): return + if getattr(self, "_object_capture_job", None) is not None: + self._show_error("Wait for object-calibration capture to finish") + return if self._capture_session is not None: self._show_error( "Cannot create neutral sets during active capture sequence", diff --git a/python_app/gui/controllers/app_window_recording_mixin.py b/python_app/gui/controllers/app_window_recording_mixin.py index 246e7fa..9bba847 100644 --- a/python_app/gui/controllers/app_window_recording_mixin.py +++ b/python_app/gui/controllers/app_window_recording_mixin.py @@ -71,6 +71,9 @@ class AppWindowRecordingMixin: path/name. Refuses to start a second recording while one is in progress, and reports a name clash up front (before any data is collected). """ + if getattr(self, "_object_capture_job", None) is not None: + self._show_error("Wait for object-calibration capture to finish") + return if self._recording_active: self._show_error( "A recording is already in progress; wait for it to finish before starting another", @@ -100,7 +103,8 @@ class AppWindowRecordingMixin: ) # Adjacent config profile, mirroring the manual save (fail if it clashes). self._write_gui_profile_to_path( - self._snapshot_config_profile_path(writer.directory), allow_overwrite=False + self._snapshot_config_profile_path(writer.directory), allow_overwrite=False, + dataset=self._supervisor.is_running(), ) except Exception as exc: # noqa: BLE001 if writer is not None: @@ -148,6 +152,14 @@ class AppWindowRecordingMixin: shutil.rmtree(writer.directory) self._log("Disk recording canceled because acquisition did not start") return + try: + self._write_gui_profile_to_path( + self._snapshot_config_profile_path(writer.directory), dataset=True, + ) + except Exception as exc: # noqa: BLE001 - keep any accepted data and the initial profile + self._finalize_recording_on_stop() + self._show_exception("Failed to update disk recording config profile", exc) + return self._log(f"Disk recording armed: streaming the next {target} measurement(s) to {writer.directory}") diff --git a/python_app/gui/controllers/app_window_snapshot_mixin.py b/python_app/gui/controllers/app_window_snapshot_mixin.py index 99bc723..2296483 100644 --- a/python_app/gui/controllers/app_window_snapshot_mixin.py +++ b/python_app/gui/controllers/app_window_snapshot_mixin.py @@ -41,6 +41,9 @@ class AppWindowSnapshotMixin: def _save_snapshot(self) -> None: """Save up to 1000 recent runtime measurements as a numpy snapshot.""" + if getattr(self, "_object_capture_job", None) is not None: + self._show_error("Wait for object-calibration capture to finish") + return self._drain_runtime_rings_for_snapshot() if getattr(self, "_closing", False): return @@ -100,7 +103,7 @@ class AppWindowSnapshotMixin: ) config_profile_path = self._snapshot_config_profile_path(snapshot_dir) try: - self._write_gui_profile_to_path(config_profile_path, allow_overwrite=False) + self._write_gui_profile_to_path(config_profile_path, allow_overwrite=False, dataset=True) except Exception as exc: # noqa: BLE001 self._show_error( "Snapshot data was saved, but the adjacent config profile could not be written", @@ -130,6 +133,9 @@ class AppWindowSnapshotMixin: def _save_vna_history_json(self) -> None: """Save one VNA-history JSON per available combo in runtime history.""" + if getattr(self, "_object_capture_job", None) is not None: + self._show_error("Wait for object-calibration capture to finish") + return self._drain_runtime_rings_for_snapshot() if getattr(self, "_closing", False): return @@ -164,7 +170,7 @@ class AppWindowSnapshotMixin: try: # Snapshot and VNA JSON exports intentionally share the same output directory stem. # Refresh the companion GUI profile in place so a prior dataset save does not block JSON export. - self._write_gui_profile_to_path(config_profile_path, allow_overwrite=True) + self._write_gui_profile_to_path(config_profile_path, allow_overwrite=True, dataset=True) except Exception as exc: # noqa: BLE001 exported_preview = "\n".join(str(path) for path in output_paths[:8]) if len(output_paths) > 8: @@ -212,6 +218,9 @@ class AppWindowSnapshotMixin: def _apply_runtime_history_deletion(self, *, remove_last_only: bool) -> None: """Apply destructive runtime-history deletion across readers, caches, and processor replay state.""" + if getattr(self, "_object_capture_job", None) is not None: + self._show_error("Wait for object-calibration capture to finish") + return if self._capture_session is not None: self._show_error( "Cannot modify runtime history during active capture sequence", diff --git a/python_app/gui/controllers/object_calibration_capture.py b/python_app/gui/controllers/object_calibration_capture.py new file mode 100644 index 0000000..a4dfbb6 --- /dev/null +++ b/python_app/gui/controllers/object_calibration_capture.py @@ -0,0 +1,140 @@ +"""Acquire fresh object-calibration sweeps through the existing native preprocessor.""" + +from __future__ import annotations + +import time + +from PyQt6.QtCore import QObject, QTimer, pyqtSignal + +from python_app.models.dataset_model import SweepCollection +from python_app.models.run_config_model import ComboModel + + +class ObjectSweepCapture(QObject): + """Temporarily acquire the full antenna matrix without applying an object correction.""" + + finished = pyqtSignal() + + def __init__(self, owner, dialog) -> None: + if getattr(owner, "_object_capture_job", None) is not None: + raise RuntimeError("Object-calibration capture is already in progress") + super().__init__(dialog) + owner._object_capture_job = self + self._owner = owner + self._dialog = dialog + self._timer = QTimer(self) + self._timer.timeout.connect(self._poll) + self._done = False + self._paused = False + self._started = False + self._resume = False + self._collections: list[SweepCollection] = [] + self._seen: set[tuple[int, int]] = set() + self._stop_seq = getattr(owner, "_stop_request_seq", 0) + QTimer.singleShot(0, self._start) + + def _start(self) -> None: + if self._done: + return + owner = self._owner + try: + if not owner._pipeline_resume_allowed(self._stop_seq): + self.cancel() + return + if getattr(owner, "_closing", False) or owner._capture_session is not None or owner._preprocess_capture_busy: + raise RuntimeError("Finish the active capture before taking object-calibration data") + if owner._single_capture_active or getattr(owner, "_recording_active", False): + raise RuntimeError("Finish the single capture or disk recording before taking calibration data") + self._config = owner._build_config() + if self._config.radar.replay.dataset_dir.strip(): + raise RuntimeError("Object-calibration capture requires live acquisition; disable archive replay") + self._config.preprocess.s21.object_calibration.set_name = "" + self._config.preprocess.s21.object_calibration.bundle_path = "" + tx = sorted({entry.output_pos for entry in self._config.gpr.tx_geometry}) + rx = sorted({entry.input_pos for entry in self._config.gpr.rx_geometry}) + if not tx or not rx: + raise RuntimeError("Configure Tx/Rx antenna positions before taking calibration data") + self._config.combos = [ComboModel(input=i, output=o) for o in tx for i in rx] + self._count = owner._ensure_preprocess_dialog().median_sweep_count() + self._resume = owner._supervisor.is_running() + if self._resume: + self._paused = True + owner._stop_run() + self._stop_seq += 1 + if not owner._pipeline_resume_allowed(self._stop_seq): + self.cancel() + return + self._source_config = self._config.to_dict() + self._start_ns = time.monotonic_ns() + owner._start_run(config_override=self._config) + self._started = owner._supervisor.is_running() + if not self._started: + raise RuntimeError("Calibration acquisition did not start; check the preprocessing sets and log") + self._dialog.set_status(f"Taking calibration data: 0 / {self._count} sweeps") + self._timer.start(50) + except Exception as exc: + self._finish(error=str(exc)) + + def _poll(self) -> None: + owner = self._owner + try: + if not owner._pipeline_resume_allowed(self._stop_seq): + self.cancel() + return + if not owner._supervisor.is_running(): + raise RuntimeError("Calibration acquisition stopped before all sweeps were received") + for collection in owner._pre_history: + identity = (collection.collection_id, collection.monotonic_ns) + starts = [collection.capture_start_ns, *(trace.capture_start_ns for trace in collection.traces)] + acquired_ns = min((start for start in starts if start > 0), default=collection.monotonic_ns) + if acquired_ns < self._start_ns or identity in self._seen: + continue + self._seen.add(identity) + self._collections.append(collection) + if len(self._collections) == self._count: + self._finish() + return + self._dialog.set_status(f"Taking calibration data: {len(self._collections)} / {self._count} sweeps") + if time.monotonic_ns() - self._start_ns > owner._SINGLE_CAPTURE_TIMEOUT_S * 1e9: + raise TimeoutError("Timed out waiting for preprocessed calibration sweeps") + except Exception as exc: + self._finish(error=str(exc)) + + def cancel(self) -> None: + """Cancel this capture and restore the previous acquisition when it is still allowed.""" + self._finish(error="Calibration capture cancelled") + + def _finish(self, *, error: str | None = None) -> None: + if self._done: + return + self._done = True + self._timer.stop() + owner = self._owner + resume_allowed = owner._pipeline_resume_allowed(self._stop_seq) + try: + if self._started and resume_allowed: + owner._stop_run() + self._stop_seq += 1 + if not owner._pipeline_resume_allowed(self._stop_seq): + error = "Calibration capture cancelled" + if error is None: + self._dialog.add_capture(self._collections, self._source_config) + else: + self._dialog.set_status(error) + except Exception as exc: + self._dialog.set_status(f"Calibration capture failed: {exc}") + finally: + if getattr(owner, "_object_capture_job", None) is self: + owner._object_capture_job = None + if self._resume and (self._paused or self._started) and resume_allowed and owner._pipeline_resume_allowed(self._stop_seq): + owner._start_run() + self._dialog.set_capture_pending(False) + self.finished.emit() + self.deleteLater() + + +def capture_object_sweeps(owner, dialog) -> ObjectSweepCapture: + """Start an asynchronous capture; the dialog receives sweeps or a visible error.""" + capture = ObjectSweepCapture(owner, dialog) + dialog.set_capture_pending(True) + return capture diff --git a/python_app/gui/controllers/object_calibration_controller.py b/python_app/gui/controllers/object_calibration_controller.py new file mode 100644 index 0000000..18f929b --- /dev/null +++ b/python_app/gui/controllers/object_calibration_controller.py @@ -0,0 +1,183 @@ +"""GUI coordination for importing and estimating optional object calibration.""" + +from __future__ import annotations + +import queue +import threading +from pathlib import Path + +from PyQt6.QtCore import QObject, QTimer +from PyQt6.QtWidgets import QFileDialog + +from python_app.gui.object_calibration_dialog import ObjectCalibrationDialog + + +def _calculate(groups, config, settings, cancelled, messages, result) -> None: + """The worker handles only Python data; every Qt action stays on the GUI thread.""" + try: + from python_app.workflows.mimo_calibration import Settings, compute_from_collections + + calibration = compute_from_collections( + groups, config, settings=Settings(**settings), + progress=messages.put, cancelled=cancelled.is_set, + ) + result.put(("cancelled", None) if cancelled.is_set() else ("ready", calibration)) + except InterruptedError: + result.put(("cancelled", None)) + except Exception as exc: # noqa: BLE001 - report worker failures on the GUI thread + result.put(("error", exc)) + + +class ObjectCalibrationController(QObject): + """Own one modeless editor and cancellable calculation for an AppWindow.""" + + def __init__(self, owner) -> None: + super().__init__(owner) + self._owner = owner + self._dialog: ObjectCalibrationDialog | None = None + self._capture = None + self._worker: threading.Thread | None = None + self._cancel = threading.Event() + self._messages: queue.Queue = queue.Queue() + self._result: queue.Queue = queue.Queue(maxsize=1) + self._poll = QTimer(self) + self._poll.setInterval(100) + self._poll.timeout.connect(self._poll_calculation) + self._save_name = "" + + def import_calibration(self) -> None: + """Validate and copy a legacy cal_*.npy folder into the app's own store.""" + if getattr(self._owner, "_closing", False): + return + if getattr(self._owner, "_object_capture_job", None) is not None or self._worker is not None: + self._owner._show_error("Wait for the object-calibration operation to finish") + return + panel = self._owner._ensure_preprocess_dialog() + name = panel.set_name() + if not name: + self._owner._show_error("Set name is required") + return + directory = QFileDialog.getExistingDirectory(panel, "Import MIMO calibration") + if not directory: + return + try: + from python_app.workflows.mimo_calibration import load_calibration, validate_for_config + + config = self._owner._build_config() + calibration = load_calibration(Path(directory)) + validate_for_config(calibration, config.to_dict()) + radar_key = self._owner._radar_key(config) + self._owner._store.save_object_calibration(radar_key, name, calibration) + self._select_saved(name) + panel.set_status(f"Object calibration imported: {name}. Applies on the next Start.") + except Exception as exc: # noqa: BLE001 + self._owner._show_exception("Failed to import object calibration", exc) + + def open_creation(self) -> None: + if getattr(self._owner, "_closing", False): + return + if self._dialog is None: + self._dialog = ObjectCalibrationDialog(self._owner._ensure_preprocess_dialog().set_name(), self._owner) + self._dialog.capture_requested.connect(self._capture_data) + self._dialog.calculate_requested.connect(self._start_calculation) + self._dialog.cancel_requested.connect(self.cancel) + self._dialog.show() + self._dialog.raise_() + self._dialog.activateWindow() + + def _capture_data(self) -> None: + if (getattr(self._owner, "_closing", False) or self._capture is not None + or self._worker is not None or self._dialog is None): + return + from python_app.gui.controllers.object_calibration_capture import capture_object_sweeps + + try: + self._capture = capture_object_sweeps(self._owner, self._dialog) + self._capture.finished.connect(self._capture_finished) + except Exception as exc: # noqa: BLE001 + self._dialog.set_capture_pending(False) + self._dialog.set_status(str(exc)) + self._owner._show_exception("Failed to capture object-calibration data", exc) + + def _capture_finished(self) -> None: + self._capture = None + + def _start_calculation(self) -> None: + if getattr(self._owner, "_closing", False) or self._worker is not None or self._capture is not None or self._dialog is None: + return + try: + name = self._dialog.set_name() + if not name: + raise ValueError("Set name is required") + groups, config, settings = self._dialog.calculation_inputs() + self._save_name = name + self._cancel = threading.Event() + self._messages = queue.Queue() + self._result = queue.Queue(maxsize=1) + self._worker = threading.Thread( + target=_calculate, + args=(groups, config, settings, self._cancel, self._messages, self._result), + name="object-calibration", daemon=True, + ) + self._dialog.set_calculating(True) + self._dialog.set_status("Расчёт калибровки…") + self._worker.start() + self._poll.start() + except Exception as exc: # noqa: BLE001 + self._worker = None + self._dialog.set_calculating(False) + self._dialog.set_status(f"Ошибка расчёта: {exc}") + self._owner._show_exception("Failed to calculate object calibration", exc) + + def _poll_calculation(self) -> None: + if self._dialog is None or getattr(self._owner, "_closing", False): + self._poll.stop() + return + while not self._messages.empty(): + self._dialog.set_status(self._messages.get_nowait()) + try: + state, payload = self._result.get_nowait() + except queue.Empty: + return + self._poll.stop() + self._worker = None + self._dialog.set_calculating(False) + if self._cancel.is_set() or state == "cancelled": + self._dialog.set_status("Расчёт отменён. Снятые данные сохранены в этом окне.") + return + if state == "error": + self._dialog.set_status(f"Ошибка расчёта: {payload}") + self._owner._show_exception("Failed to calculate object calibration", payload) + return + try: + from python_app.workflows.mimo_calibration import validate_for_config + + current_config = self._owner._build_config() + validate_for_config(payload, current_config.to_dict()) + radar_key = self._owner._radar_key(current_config) + self._owner._store.save_object_calibration(radar_key, self._save_name, payload) + self._select_saved(self._save_name) + self._owner._ensure_preprocess_dialog().set_set_name(self._save_name) + self._dialog.show_calibration(payload) + self._dialog.set_status(f"Калибровка сохранена: {self._save_name}. Применится при следующем Start.") + except Exception as exc: # noqa: BLE001 + self._dialog.set_status(f"Калибровка не сохранена: {exc}") + self._owner._show_exception("Failed to save object calibration", exc) + + def _select_saved(self, name: str) -> None: + self._owner._selected_preprocess_sets["s21_object_calibration"] = name + self._owner._refresh_sets() + self._owner._log(f"Selected object calibration: {name}") + + def cancel(self) -> None: + self._cancel.set() + if self._capture is not None: + self._capture.cancel() + if self._worker is not None and self._dialog is not None: + self._dialog.set_status("Отмена расчёта…") + + def shutdown(self) -> None: + self.cancel() + self._poll.stop() + if self._dialog is not None: + self._dialog.close() diff --git a/python_app/gui/object_calibration_dialog.py b/python_app/gui/object_calibration_dialog.py new file mode 100644 index 0000000..aafb85d --- /dev/null +++ b/python_app/gui/object_calibration_dialog.py @@ -0,0 +1,293 @@ +"""Live reflector measurements and target-phase diagnostics for MIMO calibration.""" + +from __future__ import annotations + +import copy +import math + +import numpy as np +import pyqtgraph as pg +from PyQt6.QtCore import Qt, pyqtSignal +from PyQt6.QtWidgets import ( + QCheckBox, QComboBox, QDialog, QDoubleSpinBox, QFormLayout, QGroupBox, + QHBoxLayout, QHeaderView, QLabel, QLineEdit, QPushButton, QTableWidget, QTableWidgetItem, + QVBoxLayout, QWidget, +) + +from python_app.models.dataset_model import SweepCollection + + +class ObjectCalibrationDialog(QDialog): + """Collect full-matrix sweeps, then associate each group with measured XYZ.""" + + capture_requested = pyqtSignal() + calculate_requested = pyqtSignal() + cancel_requested = pyqtSignal() + + def __init__(self, set_name: str, parent=None) -> None: + super().__init__(parent) + self.setWindowTitle("MIMO Object Calibration") + self.resize(1000, 850) + self._captures: list[dict] = [] + self._config: dict | None = None + self._capture_pending = self._calculating = False + self._calibration: dict | None = None + layout = QVBoxLayout(self) + name_row = QHBoxLayout() + name_row.addWidget(QLabel("Set name", self)) + self._set_name_input = QLineEdit(set_name, self) + name_row.addWidget(self._set_name_input) + layout.addLayout(name_row) + explanation = QLabel( + "Снимите данные, затем введите физически измеренные координаты центра отражателя: " + "X — поперёк, Y — высота над землёй, Z — вдоль земли. " + "Обычные S21-калибровка и выбранный референс уже применены; " + "существующая MIMO-поправка при съёмке отключена.", self, + ) + explanation.setWordWrap(True) + layout.addWidget(explanation) + self._table = QTableWidget(0, 5, self) + self._table.setHorizontalHeaderLabels(["Набор", "Свипов", "X (м)", "Y (м)", "Z (м)"]) + horizontal, vertical = self._table.horizontalHeader(), self._table.verticalHeader() + assert horizontal is not None and vertical is not None + horizontal.setSectionResizeMode(QHeaderView.ResizeMode.Stretch) + vertical.hide() + self._table.setSelectionBehavior(QTableWidget.SelectionBehavior.SelectRows) + self._table.itemChanged.connect(self._measurements_changed) + layout.addWidget(self._table) + actions = QHBoxLayout() + self._capture_button = QPushButton("Снять данные", self) + self._capture_button.clicked.connect(self.capture_requested.emit) + self._remove_button = QPushButton("Удалить выбранный набор", self) + self._remove_button.clicked.connect(self._remove_capture) + actions.addWidget(self._capture_button) + actions.addWidget(self._remove_button) + actions.addStretch(1) + layout.addLayout(actions) + self._settings: dict[str, QDoubleSpinBox | QCheckBox | QComboBox] = {} + form = QFormLayout() + for key, title, default, low, high in ( + ("sphere_radius_m", "Радиус шара, м (0 — точка)", .16, 0, 100), + ("antenna_height_m", "Высота начала координат антенн над землёй, м", .92, 0, 100), + ("pitch_deg", "Наклон радара вниз, °", 10, -90, 90), + ): + form.addRow(title, self._number(key, default, low, high)) + layout.addLayout(form) + advanced = QGroupBox("Параметры расчёта", self) + advanced.setFlat(True) + advanced.setCheckable(True) + advanced.setChecked(False) + advanced_layout = QVBoxLayout(advanced) + advanced_content = QWidget(advanced) + advanced_form = QFormLayout(advanced_content) + for key, title, default, low, high in ( + ("f_min", "Нижняя частота оценки, МГц", 2000, 0, 100000), + ("f_max", "Верхняя частота оценки, МГц", 6000, 0, 100000), + ("gate_half_m", "Полуширина окна эха по пути, м", .15, .000001, 100), + ("gate_taper_m", "Плавный край окна, м", .03, .000001, 100), + ("coarse_range_search_m", "Грубый поиск по пути, ±м (0 — выкл.)", .6, 0, 100), + ("refine_range_m", "Уточнение координат, ±м", .1, 0, 100), + ): + advanced_form.addRow(title, self._number(key, default, low, high)) + axes = QWidget(advanced_content) + axes_layout = QHBoxLayout(axes) + axes_layout.setContentsMargins(0, 0, 0, 0) + self._refine_axes = {} + for axis in "xyz": + checkbox = QCheckBox(axis.upper(), axes) + checkbox.setChecked(True) + checkbox.toggled.connect(self._measurements_changed) + self._refine_axes[axis] = checkbox + axes_layout.addWidget(checkbox) + advanced_form.addRow("Уточнять координаты", axes) + phase_model = QComboBox(advanced_content) + phase_model.addItems(["const", "linear", "poly3", "raw"]) + phase_model.setCurrentText("linear") + phase_model.currentTextChanged.connect(self._measurements_changed) + self._settings["phase_model"] = phase_model + advanced_form.addRow("Модель фазы антенны", phase_model) + amplitude = QCheckBox("Выравнивать также амплитуды", advanced_content) + amplitude.toggled.connect(self._measurements_changed) + self._settings["apply_amplitude"] = amplitude + advanced_form.addRow(amplitude) + advanced_content.hide() + advanced.toggled.connect(advanced_content.setVisible) + advanced_layout.addWidget(advanced_content) + layout.addWidget(advanced) + self._target_combo = QComboBox(self) + self._target_combo.currentIndexChanged.connect(self._draw_phase_effect) + layout.addWidget(self._target_combo) + self._phase_plot = pg.PlotWidget(background="#0f141c", enableMenu=False) + self._phase_plot.setLabel("left", "Target phase relative to first pair", units="deg") + self._phase_plot.setLabel("bottom", "Tx / Rx pair") + self._phase_plot.showGrid(x=True, y=True, alpha=.2) + self._phase_plot.setMinimumHeight(220) + self._phase_plot.addLegend() + layout.addWidget(self._phase_plot) + self._quality_label = QLabel("Фазы вкладов пар появятся после расчёта.", self) + self._quality_label.setWordWrap(True) + layout.addWidget(self._quality_label) + self._status_label = QLabel("Готово к съёмке", self) + self._status_label.setWordWrap(True) + layout.addWidget(self._status_label) + footer = QHBoxLayout() + self._calculate_button = QPushButton("Рассчитать и сохранить", self) + self._calculate_button.clicked.connect(self.calculate_requested.emit) + self._cancel_button = QPushButton("Отменить операцию", self) + self._cancel_button.clicked.connect(self.cancel_requested.emit) + close_button = QPushButton("Закрыть", self) + close_button.clicked.connect(self.reject) + footer.addWidget(self._calculate_button) + footer.addWidget(self._cancel_button) + footer.addStretch(1) + footer.addWidget(close_button) + layout.addLayout(footer) + self._update_actions() + + def _number(self, key: str, value: float, low: float, high: float) -> QDoubleSpinBox: + field = QDoubleSpinBox(self) + field.setDecimals(6) + field.setRange(low, high) + field.setSingleStep(100 if key in {"f_min", "f_max"} else .1 if key == "pitch_deg" else .01) + field.setValue(value) + field.valueChanged.connect(self._measurements_changed) + self._settings[key] = field + return field + + def add_capture(self, collections: list[SweepCollection], config: dict) -> None: + """Freeze a captured group before the operator enters its target coordinates.""" + if not collections: + raise ValueError("Не получены свипы отражателя") + if self._config is not None and any( + config.get(key) != self._config.get(key) for key in ("radar", "gpr", "preprocess") + ): + raise ValueError("Параметры радара, геометрия или препроцессинг изменились между наборами") + self._config = copy.deepcopy(config) + self._captures.append({"collections": copy.deepcopy(collections)}) + row = self._table.rowCount() + self._table.insertRow(row) + for column, value in enumerate((str(row + 1), str(len(collections)), "", "", "")): + item = QTableWidgetItem(value) + if column < 2: + item.setFlags(item.flags() & ~Qt.ItemFlag.ItemIsEditable) + self._table.setItem(row, column, item) + self._table.setCurrentCell(row, 2) + self.set_status("Данные сняты. Введите координаты центра отражателя для нового набора.") + self._update_actions() + + def calculation_inputs(self) -> tuple[list[dict], dict, dict]: + """Validate XYZ after capture; expose a frozen recipe to the worker.""" + if not self._captures or self._config is None: + raise ValueError("Сначала снимите данные отражателя") + groups = [] + for row, capture in enumerate(self._captures): + try: + coordinates = [] + for column in (2, 3, 4): + item = self._table.item(row, column) + assert item is not None + coordinates.append(float(item.text().strip())) + target = tuple(coordinates) + except ValueError as exc: + raise ValueError(f"Введите X, Y и Z центра отражателя для набора {row + 1}") from exc + if not all(math.isfinite(value) for value in target): + raise ValueError(f"Координаты набора {row + 1} должны быть конечными числами") + groups.append({**capture, "target_scene_m": target}) + settings: dict[str, float | bool | str | tuple[str, ...] | tuple[int, ...]] = { + key: field.value() * (1e6 if key in {"f_min", "f_max"} else 1) if isinstance(field, QDoubleSpinBox) else + field.isChecked() if isinstance(field, QCheckBox) else field.currentText() + for key, field in self._settings.items() + } + settings["refine_axes"] = tuple(axis for axis, field in self._refine_axes.items() if field.isChecked()) + traces = self._captures[0]["collections"][0].traces + settings["input_idx"] = tuple(sorted({trace.combo.input for trace in traces})) + settings["output_idx"] = tuple(sorted({trace.combo.output for trace in traces})) + return groups, copy.deepcopy(self._config), settings + + def set_name(self) -> str: + return self._set_name_input.text().strip() + + def set_capture_pending(self, pending: bool) -> None: + self._capture_pending = pending + self._update_actions() + + def set_calculating(self, calculating: bool) -> None: + self._calculating = calculating + self._update_actions() + + def set_status(self, message: str) -> None: + self._status_label.setText(message) + + def show_calibration(self, calibration: dict) -> None: + self._calibration = calibration + self._target_combo.clear() + for index, target in enumerate(calibration.get("meta", {}).get("targets", [])): + xyz = target.get("nominal_scene_m", []) + self._target_combo.addItem(f"Набор {index + 1}: {xyz}") + self._draw_phase_effect() + + def _draw_phase_effect(self, *_args) -> None: + self._phase_plot.clear() + if self._calibration is None: + return + diagnostics = self._calibration.get("meta", {}).get("diagnostics", []) + index = self._target_combo.currentIndex() + if not 0 <= index < len(diagnostics): + self._quality_label.setText("У этого набора нет диагностики фаз в точке цели.") + return + diagnostic = diagnostics[index] + phases = np.array([diagnostic["phase_before_deg"], diagnostic["phase_after_deg"]]).reshape(2, -1) + before, after = (phases - phases[0, 0] + 180) % 360 - 180 + pairs = [(output, input_pos) for output in self._calibration["tx_output_pos"] + for input_pos in self._calibration["rx_input_pos"]] + xx = np.arange(len(pairs)) + for values, color, label in ((before, "#ed7d31", "До"), (after, "#42c7a5", "После")): + self._phase_plot.plot(xx, values, pen=pg.mkPen(color, width=1.5), symbol="o", + symbolBrush=color, symbolPen=color, name=label) + step = max(1, len(pairs) // 16) + self._phase_plot.getAxis("bottom").setTicks([[ + (k, f"o{pairs[k][0]}/i{pairs[k][1]}") for k in range(0, len(pairs), step) + ]]) + self._quality_label.setText( + f"Согласованность фаз в точке цели: {diagnostic['coherence_before']:.3f} " + f"→ {diagnostic['coherence_after']:.3f} (1 — все вклады в фазе)." + ) + + def _remove_capture(self) -> None: + row = self._table.currentRow() + if row >= 0: + self._captures.pop(row) + self._table.removeRow(row) + for index in range(self._table.rowCount()): + item = self._table.item(index, 0) + assert item is not None + item.setText(str(index + 1)) + if not self._captures: + self._config = None + self._measurements_changed() + + def _measurements_changed(self, *_args) -> None: + self._calibration = None + if hasattr(self, "_phase_plot"): + self._phase_plot.clear() + self._target_combo.clear() + self._quality_label.setText("Параметры изменены — требуется новый расчёт.") + + def _update_actions(self) -> None: + busy = self._capture_pending or self._calculating + self._capture_button.setEnabled(not busy) + self._remove_button.setEnabled(not busy and bool(self._captures)) + self._calculate_button.setEnabled(not busy and bool(self._captures)) + self._cancel_button.setEnabled(busy) + self._table.setEnabled(not busy) + self._set_name_input.setEnabled(not busy) + for field in [*self._settings.values(), *self._refine_axes.values()]: + field.setEnabled(not busy) + + def reject(self) -> None: + self.cancel_requested.emit() + super().reject() + + def closeEvent(self, event) -> None: # noqa: N802 + self.cancel_requested.emit() + super().closeEvent(event) diff --git a/python_app/gui/preprocess_dialog.py b/python_app/gui/preprocess_dialog.py index f4636a2..1fb2982 100644 --- a/python_app/gui/preprocess_dialog.py +++ b/python_app/gui/preprocess_dialog.py @@ -26,7 +26,9 @@ import pyqtgraph as pg from python_app.models.dataset_model import TraceData from python_app.orchestration.preprocess_assets import ( + CAPTURABLE_PREPROCESS_ASSET_KEYS, PREPROCESS_ASSET_SPECS, + REQUIRED_PREPROCESS_ASSET_KEYS, VISIBLE_PREPROCESS_ASSET_KEYS, preprocess_asset_display_name, ) @@ -47,6 +49,8 @@ class PreprocessDialog(QDialog): finalize_sequence_requested = pyqtSignal() abort_sequence_requested = pyqtSignal() create_neutral_sets_requested = pyqtSignal() + import_object_calibration_requested = pyqtSignal() + create_object_calibration_requested = pyqtSignal() def __init__(self, parent=None) -> None: """Initialize window metadata and compose dialog UI.""" @@ -165,10 +169,25 @@ class PreprocessDialog(QDialog): for key in keys: combo = QComboBox(group) combo.setPlaceholderText("") - combo.setCurrentIndex(-1) - combo.currentTextChanged.connect(self._emit_selection_changed) + combo.setProperty("optional", key not in REQUIRED_PREPROCESS_ASSET_KEYS) + self._set_combo_items(combo, [], "") + combo.currentIndexChanged.connect(self._emit_selection_changed) self._set_combos[key] = combo - form.addRow(self._asset_row_label(key), combo) + field = combo + if key == "s21_object_calibration": + field = QWidget(group) + row = QHBoxLayout(field) + row.setContentsMargins(0, 0, 0, 0) + row.addWidget(combo, stretch=1) + self._import_object_calibration_button = QPushButton("Import…", field) + self._import_object_calibration_button.setToolTip("Import an existing cal_*.npy MIMO calibration folder.") + self._import_object_calibration_button.clicked.connect(self.import_object_calibration_requested.emit) + self._create_object_calibration_button = QPushButton("Create…", field) + self._create_object_calibration_button.setToolTip("Calculate a MIMO calibration from reflector measurements.") + self._create_object_calibration_button.clicked.connect(self.create_object_calibration_requested.emit) + row.addWidget(self._import_object_calibration_button) + row.addWidget(self._create_object_calibration_button) + form.addRow(self._asset_row_label(key), field) return group def _build_sequence_group(self) -> QGroupBox: @@ -199,7 +218,7 @@ class PreprocessDialog(QDialog): def _build_sequence_button_grid(self, parent: QGroupBox) -> QGridLayout: """Build per-asset capture start buttons.""" layout = QGridLayout() - for index, key in enumerate(VISIBLE_PREPROCESS_ASSET_KEYS): + for index, key in enumerate(CAPTURABLE_PREPROCESS_ASSET_KEYS): button = QPushButton(f"Start {preprocess_asset_display_name(key)}", parent) button.clicked.connect(lambda _checked=False, asset_key=key: self.start_sequence_requested.emit(asset_key)) layout.addWidget(button, index // 2, index % 2) @@ -316,7 +335,7 @@ class PreprocessDialog(QDialog): def selection_snapshot(self) -> dict[str, str]: """Return currently selected set names keyed by preprocess asset key.""" - return {key: self._set_combos[key].currentText().strip() for key in VISIBLE_PREPROCESS_ASSET_KEYS} + return {key: self._set_combos[key].currentData() or "" for key in VISIBLE_PREPROCESS_ASSET_KEYS} def clear_capture_log(self) -> None: """Clear capture history text box.""" @@ -398,7 +417,7 @@ class PreprocessDialog(QDialog): for key in VISIBLE_PREPROCESS_ASSET_KEYS: combo = self._set_combos[key] with QSignalBlocker(combo): - self._set_combo_items(combo, available_sets.get(key, []), combo.currentText().strip()) + self._set_combo_items(combo, available_sets.get(key, []), combo.currentData() or "") def set_selected_sets(self, selected_sets: dict[str, str], *, emit_signal: bool = True) -> None: """Apply selected set names to all comboboxes and optionally emit update.""" @@ -406,13 +425,10 @@ class PreprocessDialog(QDialog): selected_value = selected_sets.get(key, "") combo = self._set_combos[key] with QSignalBlocker(combo): - if not selected_value: - combo.setCurrentIndex(-1) - continue - index = combo.findText(selected_value) - if index < 0: - combo.addItem(selected_value) - index = combo.findText(selected_value) + index = combo.findData(selected_value) + if index < 0 and selected_value: + combo.addItem(selected_value, selected_value) + index = combo.count() - 1 combo.setCurrentIndex(index) if emit_signal: self._emit_selection_changed() @@ -538,12 +554,15 @@ class PreprocessDialog(QDialog): def _set_combo_items(combo: QComboBox, names: list[str], current_text: str) -> None: """Replace combo contents and keep previous value when still available.""" combo.clear() - combo.addItems(names) + if combo.property("optional"): + combo.addItem("Off", "") + for name in names: + combo.addItem(name, name) if not current_text: - combo.setCurrentIndex(-1) + combo.setCurrentIndex(combo.findData("")) return - index = combo.findText(current_text) + index = combo.findData(current_text) if index < 0: - combo.addItem(current_text) - index = combo.findText(current_text) + combo.addItem(current_text, current_text) + index = combo.count() - 1 combo.setCurrentIndex(index) diff --git a/python_app/models/run_config_codec.py b/python_app/models/run_config_codec.py index 3081ba2..9d41637 100644 --- a/python_app/models/run_config_codec.py +++ b/python_app/models/run_config_codec.py @@ -326,6 +326,10 @@ def run_config_from_dict(payload: dict[str, Any]) -> RunConfigModel: _as_dict(s21_preprocess_payload.get("reference"), "preprocess.s21.reference"), model.preprocess.s21.reference, ) + _load_preprocess_asset( + _as_dict(s21_preprocess_payload.get("object_calibration"), "preprocess.s21.object_calibration"), + model.preprocess.s21.object_calibration, + ) s11_preprocess_payload = _as_dict(preprocess_payload.get("s11"), "preprocess.s11") s11_calibration_payload = _as_dict(s11_preprocess_payload.get("calibration"), "preprocess.s11.calibration") @@ -577,6 +581,10 @@ def run_config_to_dict(model: RunConfigModel) -> dict[str, Any]: "set_name": model.preprocess.s21.reference.set_name, "bundle_path": model.preprocess.s21.reference.bundle_path, }, + "object_calibration": { + "set_name": model.preprocess.s21.object_calibration.set_name, + "bundle_path": model.preprocess.s21.object_calibration.bundle_path, + }, }, "s11": { "calibration": { diff --git a/python_app/models/run_config_schema.py b/python_app/models/run_config_schema.py index ef216cb..00ccf5a 100644 --- a/python_app/models/run_config_schema.py +++ b/python_app/models/run_config_schema.py @@ -275,6 +275,7 @@ class S21PreprocessModel: calibration: PreprocessAssetModel = field(default_factory=PreprocessAssetModel) reference: PreprocessAssetModel = field(default_factory=PreprocessAssetModel) + object_calibration: PreprocessAssetModel = field(default_factory=PreprocessAssetModel) @dataclass(slots=True) diff --git a/python_app/orchestration/config_writer.py b/python_app/orchestration/config_writer.py index 8d51a3c..9ac05bf 100644 --- a/python_app/orchestration/config_writer.py +++ b/python_app/orchestration/config_writer.py @@ -41,7 +41,10 @@ class ConfigWriter: asset = preprocess_asset_model(config, key) bundle_path = self._runtime_dir / spec.runtime_filename logger.debug("Exporting preprocess bundle %s (set=%s) -> %s", key, asset.set_name, bundle_path) - store.export_set_bundle(spec.set_kind, radar_key, asset.set_name, bundle_path) + if key == "s21_object_calibration": + store.export_object_calibration_bundle(radar_key, asset.set_name, bundle_path, config) + else: + store.export_set_bundle(spec.set_kind, radar_key, asset.set_name, bundle_path) asset.bundle_path = str(bundle_path) def write(self, config: RunConfigModel, output_path: Path) -> Path: diff --git a/python_app/orchestration/preprocess_assets.py b/python_app/orchestration/preprocess_assets.py index 8c903be..77decad 100644 --- a/python_app/orchestration/preprocess_assets.py +++ b/python_app/orchestration/preprocess_assets.py @@ -20,6 +20,7 @@ class PreprocessAssetSpec: PREPROCESS_ASSET_SPECS = { "s21_calibration": PreprocessAssetSpec("S21 Calibration", "s21_calibration", "s21_calibration_bundle.bin", "s21"), "s21_reference": PreprocessAssetSpec("S21 Reference", "s21_reference", "s21_reference_bundle.bin", "s21"), + "s21_object_calibration": PreprocessAssetSpec("Object Calibration", "s21_object_calibration", "s21_object_calibration_bundle.bin", "s21"), "s11_open": PreprocessAssetSpec("S11 Open", "s11_open", "s11_open_calibration_bundle.bin", "s11"), "s11_short": PreprocessAssetSpec("S11 Short", "s11_short", "s11_short_calibration_bundle.bin", "s11"), "s11_load": PreprocessAssetSpec("S11 Load", "s11_load", "s11_load_calibration_bundle.bin", "s11"), @@ -30,7 +31,8 @@ PREPROCESS_ASSET_KEYS = tuple(PREPROCESS_ASSET_SPECS.keys()) S21_PREPROCESS_ASSET_KEYS = ("s21_calibration", "s21_reference") S11_PREPROCESS_ASSET_KEYS = ("s11_open", "s11_short", "s11_load", "s11_reference") S11_PREPROCESS_CALIBRATION_KEYS = ("s11_open", "s11_short", "s11_load") -VISIBLE_PREPROCESS_ASSET_KEYS = S21_PREPROCESS_ASSET_KEYS +VISIBLE_PREPROCESS_ASSET_KEYS = (*S21_PREPROCESS_ASSET_KEYS, "s21_object_calibration") +CAPTURABLE_PREPROCESS_ASSET_KEYS = S21_PREPROCESS_ASSET_KEYS REQUIRED_PREPROCESS_ASSET_KEYS = S21_PREPROCESS_ASSET_KEYS @@ -40,6 +42,8 @@ def preprocess_asset_model(config: RunConfigModel, key: str) -> PreprocessAssetM return config.preprocess.s21.calibration if key == "s21_reference": return config.preprocess.s21.reference + if key == "s21_object_calibration": + return config.preprocess.s21.object_calibration if key == "s11_open": return config.preprocess.s11.calibration.open if key == "s11_short": @@ -68,7 +72,9 @@ def preprocess_asset_set_name(config: RunConfigModel, key: str) -> str: def runtime_preprocess_asset_keys(config: RunConfigModel) -> tuple[str, ...]: """Return preprocess assets that should be exported and validated for runtime.""" - enabled_keys = list(REQUIRED_PREPROCESS_ASSET_KEYS) + enabled_keys: list[str] = list(REQUIRED_PREPROCESS_ASSET_KEYS) + if preprocess_asset_set_name(config, "s21_object_calibration"): + enabled_keys.append("s21_object_calibration") has_full_s11_calibration = all( preprocess_asset_set_name(config, key) diff --git a/python_app/storage/npz/store.py b/python_app/storage/npz/store.py index d95130f..baa912f 100644 --- a/python_app/storage/npz/store.py +++ b/python_app/storage/npz/store.py @@ -16,6 +16,7 @@ from zipfile import BadZipFile import numpy as np from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection, TraceData +from python_app.models.run_config_model import RunConfigModel from python_app.storage.npz.paths import radar_key_from_config, sanitize_path_component from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, serialize_trace_collection, validate_trace_collection from python_app.storage.npz.snapshot_numpy import ( @@ -300,10 +301,34 @@ class NpzStore(StoreApi): set_dir = self._set_dir(kind, radar_key) if not set_dir.exists(): return [] + if kind == "s21_object_calibration": + return sorted(path.name for path in set_dir.iterdir() + if not path.name.startswith(".") and (path / "cal_meta.json").is_file()) return sorted(path.stem for path in set_dir.glob("*.npz")) + def save_object_calibration(self, radar_key: str, set_name: str, calibration: dict[str, Any]) -> None: + """Save an object calibration in its shared archive format without overwriting a set.""" + from python_app.workflows.mimo_calibration import save_calibration + + self._validate_set_name(set_name) + destination = self._set_dir("s21_object_calibration", radar_key) / set_name + if destination.exists(): + raise FileExistsError(f"Object calibration already exists: {set_name}") + save_calibration(calibration, destination) + + def load_object_calibration(self, radar_key: str, set_name: str) -> dict[str, Any]: + """Load and validate a saved object calibration, retaining all estimation metadata.""" + from python_app.workflows.mimo_calibration import load_calibration + + self._validate_set_name(set_name) + return load_calibration(self._set_dir("s21_object_calibration", radar_key) / set_name) + def has_combo_coverage(self, kind: str, radar_key: str, set_name: str, combos: list[ComboKey]) -> bool: """Validate that named set covers all required switch combinations.""" + if kind == "s21_object_calibration": + calibration = self.load_object_calibration(radar_key, set_name) + return all(combo.input in calibration["rx_input_pos"] and combo.output in calibration["tx_output_pos"] + for combo in combos) collection = self.load_set(kind, radar_key, set_name) existing = {(trace.combo.input, trace.combo.output) for trace in collection.traces} required = {(combo.input, combo.output) for combo in combos} @@ -312,6 +337,36 @@ class NpzStore(StoreApi): def export_set_bundle(self, kind: str, radar_key: str, set_name: str, output_path: Path) -> Path: """Export named set as binary collection bundle for C++ preprocessing stage.""" collection = self.load_set(kind, radar_key, set_name) + return self._export_collection_bundle(collection, output_path) + + def export_object_calibration_bundle( + self, radar_key: str, set_name: str, output_path: Path, config: RunConfigModel, + ) -> Path: + """Resample pair multipliers on the through-calibration grids used by the acquisition.""" + from python_app.workflows.mimo_calibration import calibration_on_grid, validate_for_config + + calibration = self.load_object_calibration(radar_key, set_name) + validate_for_config(calibration, config.to_dict()) + tx_indices = {int(port): index for index, port in enumerate(calibration["tx_output_pos"])} + rx_indices = {int(port): index for index, port in enumerate(calibration["rx_input_pos"])} + source = self.load_set("s21_calibration", radar_key, config.preprocess.s21.calibration.set_name) + by_combo = {trace.combo: trace for trace in source.traces} + grids: dict[bytes, np.ndarray] = {} + traces: list[TraceData] = [] + for combo in config.combos: + key = ComboKey(input=combo.input, output=combo.output) + trace = by_combo[key] + frequency = trace.frequency_hz + grid_key = frequency.tobytes() + if grid_key not in grids: + grids[grid_key] = calibration_on_grid(calibration, frequency) + correction = grids[grid_key][tx_indices[key.output], rx_indices[key.input]] + traces.append(TraceData(key, frequency, np.zeros(frequency.size, np.complex64), correction)) + return self._export_collection_bundle(SweepCollection(0, 0, traces), output_path) + + def _export_collection_bundle(self, collection: SweepCollection, output_path: Path) -> Path: + """Publish a complete binary trace bundle using one atomic replacement.""" + validate_trace_collection(collection, require_finite=True) output_path.parent.mkdir(parents=True, exist_ok=True) temporary = self._temporary_file(output_path) try: @@ -319,7 +374,7 @@ class NpzStore(StoreApi): temporary.replace(output_path) finally: temporary.unlink(missing_ok=True) - logger.info("Exported set %s/%s/%s bundle to %s", kind, radar_key, set_name, output_path) + logger.info("Exported %d traces to bundle %s", len(collection.traces), output_path) return output_path def save_runtime_snapshot( diff --git a/python_app/tests/test_gui_action_lifecycle.py b/python_app/tests/test_gui_action_lifecycle.py index 687ca75..fb0a16f 100644 --- a/python_app/tests/test_gui_action_lifecycle.py +++ b/python_app/tests/test_gui_action_lifecycle.py @@ -17,12 +17,14 @@ from PyQt6.QtCore import QTimer # noqa: E402 from python_app.gui.app_window import AppWindow # noqa: E402 from python_app.gui.controllers.app_window_config.profile_io_mixin import AppWindowConfigProfileIOMixin # noqa: E402 +from python_app.gui.controllers.app_window_config.live_processing_mixin import AppWindowLiveProcessingMixin # noqa: E402 from python_app.gui.controllers.app_window_pipeline_mixin import AppWindowPipelineMixin # noqa: E402 from python_app.gui.controllers.app_window_preprocess_mixin import AppWindowPreprocessMixin # noqa: E402 from python_app.gui.controllers.app_window_snapshot_mixin import AppWindowSnapshotMixin # noqa: E402 +from python_app.gui.controllers.app_window_recording_mixin import AppWindowRecordingMixin # noqa: E402 from python_app.models.dataset_model import ResultCollection, SweepCollection # noqa: E402 -from python_app.models.gui_profile_model import GuiProfileModel # noqa: E402 -from python_app.models.run_config_model import RunConfigModel # noqa: E402 +from python_app.models.gui_profile_model import GuiProfileModel, GuiStateModel # noqa: E402 +from python_app.models.run_config_model import ComboModel, GprRxGeometryModel, GprTxGeometryModel, RunConfigModel # noqa: E402 from python_app.storage.npz_store import NpzStore # noqa: E402 from python_app.workflows.kamil_adc_neutral_preprocess import build_neutral_s21_sets # noqa: E402 @@ -264,6 +266,111 @@ class CaptureBatchControlTest(unittest.TestCase): class GuiProfileWriteTest(unittest.TestCase): + def test_dataset_profile_uses_actual_preprocessing_and_preserves_current_gui(self) -> None: + for applied, selected in (("", "next_run"), ("applied", "")): + with self.subTest(applied=applied), tempfile.TemporaryDirectory() as folder: + pending = GuiProfileModel(gui=GuiStateModel()) + pending.gui.processing.bscan.max_depth_m = 7.5 + pending.run_config.gpr.tx_geometry = [GprTxGeometryModel(output_pos=0, x_m=.4, y_m=.2, z_m=.1)] + pending.run_config.preprocess.s21.object_calibration.set_name = selected + active = pending.run_config.clone() + active.gpr.tx_geometry[0].x_m = -.4 + active.radar.serial = "acquired_A" + active.radar.sweep.points = 129 + active.input_switch.positions = 2 + active.output_switch.positions = 2 + active.combos = [ComboModel(input=1, output=1)] + pending.run_config.radar.serial = "pending_B" + pending.run_config.radar.sweep.points = 257 + active.preprocess.s21.object_calibration.set_name = applied + active.preprocess.s21.object_calibration.bundle_path = "/bundles/object" if applied else "" + active.preprocess.s21.calibration.set_name = "actual_through" + active.preprocess.s21.calibration.bundle_path = "/bundles/through" + active.preprocess.s21.reference.set_name = "actual_reference" + active.preprocess.s21.reference.bundle_path = "/bundles/reference" + active.preprocess.s11.calibration.open.set_name = "actual_open" + active.preprocess.s11.calibration.open.bundle_path = "/bundles/open" + active.preprocess.s11.reference.set_name = "actual_s11_reference" + active.preprocess.notch.enabled = True + active.preprocess.notch.bands_hz = [(2e9, 2.1e9)] + writer = AppWindowConfigProfileIOMixin() + writer._build_gui_profile = pending.clone + writer._active_run_config = active + dataset_path, settings_path = Path(folder) / "dataset.json", Path(folder) / "settings.json" + saved = writer._write_gui_profile_to_path(dataset_path, dataset=True) + writer._write_gui_profile_to_path(settings_path) + dataset = GuiProfileModel.load_from_path(dataset_path).to_dict() + expected = pending.to_dict() + acquired = active.to_dict() + for key in ("radar", "switches", "run", "preprocess"): + expected[key] = acquired[key] + expected["gpr"]["tx_geometry"] = acquired["gpr"]["tx_geometry"] + expected["gpr"]["rx_geometry"] = acquired["gpr"]["rx_geometry"] + self.assertEqual(dataset, expected) + self.assertEqual(GuiProfileModel.load_from_path(settings_path).to_dict(), pending.to_dict()) + saved.run_config.preprocess.s21.calibration.set_name = "mutated_return_value" + self.assertEqual(active.preprocess.s21.calibration.set_name, "actual_through") + + def test_dataset_profile_retains_original_device_and_geometry_after_loading_another_profile(self) -> None: + acquired = GuiProfileModel() + acquired.run_config.ensure_combos() + acquired.run_config.radar.serial = "device_A" + acquired.run_config.gpr.tx_geometry = [GprTxGeometryModel(output_pos=0, x_m=-.4)] + acquired.run_config.gpr.rx_geometry = [GprRxGeometryModel(input_pos=0, x_m=.4)] + pending = acquired.clone() + pending.run_config.radar.model = "librevna_multi" + pending.run_config.radar.serial = "device_B" + pending.run_config.gpr.tx_geometry[0].x_m = 2. + pending.run_config.gpr.rx_geometry[0].x_m = -2. + writer = AppWindowConfigProfileIOMixin() + writer._build_gui_profile = pending.clone + writer._active_run_config = acquired.run_config + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "dataset.json" + writer._write_gui_profile_to_path(path, dataset=True) + saved = GuiProfileModel.load_from_path(path) + self.assertEqual(saved.run_config.radar, acquired.run_config.radar) + self.assertEqual(saved.run_config.gpr, acquired.run_config.gpr) + self.assertEqual(saved.run_config.combos, acquired.run_config.combos) + + def test_applied_geometry_tracks_successful_live_writes_only_while_acquiring(self) -> None: + for running, failed_write in ((True, False), (False, False), (True, True)): + with self.subTest(running=running, failed_write=failed_write): + active = RunConfigModel() + active.gpr.tx_geometry = [GprTxGeometryModel(output_pos=0, x_m=-.4)] + active.gpr.rx_geometry = [GprRxGeometryModel(input_pos=0, x_m=.4)] + original = active.clone() + writer = SimpleNamespace( + _active_run_config=active, + _supervisor=SimpleNamespace(is_running=lambda: running), + _live_config_writer=SimpleNamespace(write=mock.Mock(side_effect=OSError("disk full") if failed_write else None)), + _live_processing_config=mock.Mock(return_value=object()), + _can_reprocess_history=lambda: False, + _gpr_tx_geometry_input=SimpleNamespace(records=lambda: [dict(output_pos=0, x_m=1., y_m=2., z_m=3.)]), + _gpr_rx_geometry_input=SimpleNamespace(records=lambda: [dict(input_pos=0, x_m=-1., y_m=-2., z_m=-3.)]), + ) + if failed_write: + with self.assertRaises(OSError): + AppWindowLiveProcessingMixin._write_live_processing_config(writer) + else: + AppWindowLiveProcessingMixin._write_live_processing_config(writer) + if running and not failed_write: + self.assertEqual(active.gpr.tx_geometry, [GprTxGeometryModel(0, 1., 2., 3.)]) + self.assertEqual(active.gpr.rx_geometry, [GprRxGeometryModel(0, -1., -2., -3.)]) + else: + self.assertEqual(active.gpr, original.gpr) + + def test_dataset_profile_without_active_run_preserves_ui_selection(self) -> None: + writer = AppWindowConfigProfileIOMixin() + pending = GuiProfileModel() + pending.run_config.preprocess.s21.object_calibration.set_name = "selected" + writer._build_gui_profile = pending.clone + writer._active_run_config = None + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "dataset.json" + writer._write_gui_profile_to_path(path, dataset=True) + self.assertEqual(GuiProfileModel.load_from_path(path).to_dict(), pending.to_dict()) + def test_failed_profile_replace_preserves_existing_profile_and_cleans_temporary(self) -> None: writer = AppWindowConfigProfileIOMixin() writer._build_gui_profile = lambda: GuiProfileModel() @@ -293,5 +400,95 @@ class HistoryCommandRestoreTest(unittest.TestCase): self.assertEqual(AppWindow._load_history_command_seq(path), 9876543210) +class PreprocessHistoryRestartTest(unittest.TestCase): + def _runtime(self): + w = _Runtime() + config = RunConfigModel.from_dict({"radar": {"model": "librevna", "driver_mode": "mock"}}) + config.input_switch.driver = config.output_switch.driver = "h7992" + config.input_switch.radar_port, config.output_switch.radar_port = 1, 2 + config.preprocess.s21.calibration.set_name = "through" + config.preprocess.s21.reference.set_name = "empty" + w._build_config.return_value = config + w._processor_run_signature = w._build_processor_run_signature(config) + w._history_run_signature = w._build_run_history_signature(config) + w._project_root = Path("/tmp/object-calibration-test") + w._store = mock.Mock() + w._store.has_combo_coverage.return_value = True + w._config_writer = mock.Mock() + w._config_writer.write.return_value = w._project_root / "runtime.json" + w._radar_key = mock.Mock(return_value="radar") + w._supervisor.pids.return_value = {} + w._bscan_reprocess_timer = mock.Mock() + for name in ("_validate_processing_mode_constraints", "_prepare_radar_for_native_acquisition", + "_close_replay_ring_writer", "_discard_bscan_replay_results", "_close_readers", + "_drop_pending_ring_payloads", "_reset_runtime_history", "_stop_all_processes", + "_write_live_processing_config"): + setattr(w, name, mock.Mock()) + return w, config + + def test_changed_preprocess_clears_native_history_before_both_start_modes(self) -> None: + for single_capture in (False, True): + with self.subTest(single_capture=single_capture): + w, config = self._runtime() + config.preprocess.s21.object_calibration.set_name = "sphere" + events = [] + w._write_live_processing_config.side_effect = lambda **_kwargs: events.append("clear") + w._supervisor.start.side_effect = lambda *_args, **_kwargs: events.append("start") + with mock.patch("python_app.gui.controllers.app_window_pipeline_mixin.ShmRingReader"): + w._start_run(single_capture=single_capture) + w._show_exception.assert_not_called() + w._write_live_processing_config.assert_called_once_with( + history_command="clear_all", bump_history_seq=True) + self.assertEqual(events, ["clear", "start"]) + w._reset_runtime_history.assert_called_once() + self.assertEqual(w._history_run_signature, w._build_run_history_signature(config)) + + def test_unchanged_preprocess_retains_history_and_existing_processor(self) -> None: + w, config = self._runtime() + with mock.patch("python_app.gui.controllers.app_window_pipeline_mixin.ShmRingReader"): + w._start_run(single_capture=True) + w._show_exception.assert_not_called() + w._write_live_processing_config.assert_not_called() + w._reset_runtime_history.assert_not_called() + w._stop_all_processes.assert_not_called() + + def test_capture_override_is_cloned_and_skips_display_mode_constraints(self) -> None: + w, config = self._runtime() + w._object_capture_job = object() + with mock.patch("python_app.gui.controllers.app_window_pipeline_mixin.ShmRingReader"): + w._start_run(config_override=config) + w._show_exception.assert_not_called() + w._build_config.assert_not_called() + w._validate_processing_mode_constraints.assert_not_called() + exported = w._config_writer.prepare_preprocess_bundles.call_args.args[2] + self.assertIsNot(exported, config) + self.assertFalse(config.runtime.continuous) + self.assertTrue(exported.runtime.continuous) + + def test_regular_start_and_other_hardware_actions_wait_for_object_capture(self) -> None: + for action, args in (("_start_run", ()), ("_capture_tmp_reference", ()), + ("_create_neutral_sets", ()), ("_start_capture_sequence", ("s21_reference",)), + ("_apply_radar_settings", ()), ("_save_snapshot", ()), + ("_save_vna_history_json", ()), ("_remove_last_runtime_history", ()), + ("_clear_all_runtime_history", ())): + with self.subTest(action=action): + w, _ = self._runtime() + w._object_capture_job = object() + getattr(w, action)(*args) + w._build_config.assert_not_called() + w._show_error.assert_called_once() + + def test_profile_load_and_recording_wait_for_object_capture_even_before_pipeline_start(self) -> None: + for action in (AppWindowConfigProfileIOMixin._ensure_ready_to_load_config, + AppWindowRecordingMixin._start_run_with_recording): + with self.subTest(action=action.__name__): + w, _ = self._runtime() + w._object_capture_job = object() + self.assertFalse(action(w)) + w._build_config.assert_not_called() + w._supervisor.is_running.assert_not_called() + w._show_error.assert_called_once() + + if __name__ == "__main__": unittest.main() diff --git a/python_app/tests/test_mimo_calibration.py b/python_app/tests/test_mimo_calibration.py new file mode 100644 index 0000000..5501063 --- /dev/null +++ b/python_app/tests/test_mimo_calibration.py @@ -0,0 +1,384 @@ +"""Physical MIMO estimation, legacy archives and failure-preserving writes.""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path +import tempfile +import unittest +from unittest import mock +from typing import Any + +import numpy as np + +from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData +from python_app.workflows import mimo_calibration as mc + + +class MimoCalibrationTest(unittest.TestCase): + def setUp(self): + self.settings = mc.Settings(input_idx=(2, 7), output_idx=(1, 5), sphere_radius_m=0., + antenna_height_m=1., f_min=2e9, f_max=6e9, + refine_axes=(), coarse_range_search_m=0.) + self.freq = np.linspace(2e9, 6e9, 129) + self.target = (.2, .25, 4.) + self.tx = np.array([[-.6, 0., 0.], [.6, 0., 0.]]) + self.rx = np.array([[-.3, .1, 0.], [.3, .1, 0.]]) + self.config = { + 'radar': {'model': 'librevna_multi', 'serial': 'master', 'multi_device': {'slave_serials': ['slave']}}, + 'preprocess': {'s21': {'calibration': {'set_name': 'through'}, 'reference': {'set_name': 'empty'}}}, + 'gpr': { + 'tx_geometry': [dict(output_pos=port, x_m=xyz[0], y_m=xyz[1], z_m=xyz[2]) for port, xyz in zip((1, 5), self.tx)], + 'rx_geometry': [dict(input_pos=port, x_m=xyz[0], y_m=xyz[1], z_m=xyz[2]) for port, xyz in zip((2, 7), self.rx)], + }, + 'run': {'combos': [dict(input=input_, output=output) for output in (1, 5) for input_ in (2, 7)]}, + } + + def signal(self, target=None, *, gains=False): + target = self.target if target is None else target + pitch = np.radians(self.settings.pitch_deg) + y0 = target[1] - self.settings.antenna_height_m + radar_target = np.array([target[0], y0 * np.cos(pitch) + target[2] * np.sin(pitch), + -y0 * np.sin(pitch) + target[2] * np.cos(pitch)]) + vt, vr = self.tx - radar_target, self.rx - radar_target + dt, dr = np.linalg.norm(vt, axis=1), np.linalg.norm(vr, axis=1) + paths = dt[:, None] + dr[None, :] + if self.settings.sphere_radius_m: + cosine = (vt @ vr.T) / (dt[:, None] * dr[None, :]) + paths -= 2 * self.settings.sphere_radius_m * np.sqrt((1 + cosine) / 2) + tx_phase = np.array([0., .7])[:, None] - 2 * np.pi * self.freq * np.array([0., .02])[:, None] / mc.C_LIGHT + rx_phase = np.array([0., -.5])[:, None] - 2 * np.pi * self.freq * np.array([0., -.013])[:, None] / mc.C_LIGHT + signal = np.exp(-2j * np.pi * self.freq * paths[:, :, None] / mc.C_LIGHT) + signal *= np.exp(1j * (tx_phase[:, None, :] + rx_phase[None, :, :] + 1.1)) + if gains: + signal *= np.array([1., 2.])[:, None, None] * np.array([1., .5])[None, :, None] + return signal, paths + + def collection(self, samples=None, *, freq=None, identity=1): + samples = self.signal()[0] if samples is None else samples + freq = self.freq if freq is None else freq + traces = [TraceData(ComboKey(input_, output), freq.copy(), np.zeros(freq.size, np.complex64), samples[t, r].copy()) + for t, output in enumerate((1, 5)) for r, input_ in enumerate((2, 7))] + return SweepCollection(identity, identity, traces) + + def calibration(self): + return mc.compute_from_collections([{'collections': [self.collection()], 'target_scene_m': self.target}], + self.config, settings=self.settings) + + def archive(self, root, collection, *, name='0000_id1_ns1'): + snapshot = root / 'preprocessed' / name + snapshot.mkdir(parents=True) + (root / 'config_profile.json').write_text(json.dumps(self.config)) + (snapshot / 'meta.json').write_text('{}') + for trace in collection.traces: + suffix = f'i{trace.combo.input}_o{trace.combo.output}' + np.save(snapshot / f'{suffix}_s21.npy', trace.s21) + np.save(snapshot / f'{suffix}_freq.npy', trace.frequency_hz) + np.save(snapshot / f'{suffix}_s11.npy', trace.s11) + return snapshot + + def test_point_and_sphere_recover_relative_antenna_phases(self): + for radius in (0., .16): + with self.subTest(radius=radius): + self.settings.sphere_radius_m = radius + signal, paths = self.signal() + cal = self.calibration() + corrected = mc.apply_calibration(signal, self.freq, cal) + contributions = corrected * np.exp(2j * np.pi * self.freq * paths[:, :, None] / mc.C_LIGHT) + phases = np.angle(contributions.mean(axis=2)) + np.testing.assert_allclose(phases, np.full((2, 2), 1.1), atol=.004) + np.testing.assert_allclose(np.abs(cal['pair']), 1., atol=2e-7) + self.assertGreater(cal['meta']['quality']['coherence_final'][0], .9999) + self.assertLess(cal['meta']['quality']['coherence_before'][0], .98) + diagnostic = cal['meta']['diagnostics'][0] + self.assertEqual(np.shape(diagnostic['phase_before_deg']), (2, 2)) + np.testing.assert_allclose(diagnostic['phase_after_deg'], np.full((2, 2), np.degrees(1.1)), atol=.25) + self.assertIsNone(cal['meta']['quality']['coherence_leave_one_out'][0]) + + def test_amplitude_equalization_is_explicit(self): + self.settings.apply_amplitude = True + signal, paths = self.signal(gains=True) + cal = mc.compute_from_collections([{'collections': [self.collection(signal)], 'target_scene_m': self.target}], + self.config, settings=self.settings) + corrected = mc.apply_calibration(signal, self.freq, cal) + # Compare amplitudes after removing the physical propagation phase. + levels = np.abs((corrected * np.exp(2j * np.pi * self.freq * paths[:, :, None] / mc.C_LIGHT)).mean(axis=2)) + self.assertLess(levels.max() / levels.min(), 1.01) + self.assertGreater(np.max(np.abs(cal['pair'])), 1.5) + + def test_archive_and_live_estimation_share_the_same_math(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + signal = self.signal()[0] + self.archive(root, self.collection(signal), name='0000_id1_ns1') + self.archive(root, self.collection(signal * 2), name='0001_id2_ns2') + archived = mc.compute_calibration([{'path': root / 'preprocessed', 'target_scene_m': self.target}], settings=self.settings) + live = mc.compute_from_collections([{'collections': [self.collection(signal), self.collection(signal * 2)], + 'target_scene_m': self.target}], self.config, settings=self.settings) + np.testing.assert_array_equal(archived['pair'], live['pair']) + + def test_frequency_geometry_channel_and_signal_failures_are_explicit(self): + cases = [] + incomplete = self.collection(); incomplete.traces.pop(); cases.append(incomplete) + duplicate = self.collection(); duplicate.traces.append(duplicate.traces[0]); cases.append(duplicate) + mismatched = self.collection(); mismatched.traces[0].frequency_hz += 10000; cases.append(mismatched) + nonuniform = self.collection(); nonuniform.traces[0].frequency_hz[20] += 1e6; cases.append(nonuniform) + zero = self.collection(np.zeros((2, 2, self.freq.size), complex)); cases.append(zero) + invalid = self.collection(); invalid.traces[0].s21[0] = np.nan; cases.append(invalid) + for index, collection in enumerate(cases): + with self.subTest(collection=index): + with self.assertRaises(ValueError): + mc.compute_from_collections([{'collections': [collection], 'target_scene_m': self.target}], self.config, settings=self.settings) + with self.assertRaisesRegex(ValueError, 'at least one'): + mc.compute_from_collections([], self.config, settings=self.settings) + + def test_config_checks_phase_base_and_geometry_but_allows_new_reference(self): + cal = self.calibration() + updated = copy.deepcopy(self.config) + updated['preprocess']['s21']['reference']['set_name'] = 'new_background' + mc.validate_for_config(cal, updated) + for edit in ('geometry', 'through', 'combo', 'serial'): + updated = copy.deepcopy(self.config) + if edit == 'geometry': + updated['gpr']['tx_geometry'][0]['x_m'] += .01 + elif edit == 'through': + updated['preprocess']['s21']['calibration']['set_name'] = 'new_through' + elif edit == 'serial': + updated['radar']['serial'] = 'another_device' + else: + updated['run']['combos'].append({'input': 999, 'output': 1}) + with self.subTest(edit=edit), self.assertRaises(ValueError): + mc.validate_for_config(cal, updated) + + def test_object_corrected_sources_are_rejected_for_live_and_archives(self): + for field in ('set_name', 'bundle_path'): + self.config['preprocess']['s21']['object_calibration'] = {field: 'active'} + with self.subTest(field=field), self.assertRaisesRegex(ValueError, 'already active'): + self.calibration() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.archive(root, self.collection()) + with self.assertRaisesRegex(ValueError, 'already active'): + mc.compute_calibration([{'path': root / 'preprocessed', 'target_scene_m': self.target}], settings=self.settings) + + def test_cancel_is_checked_inside_refinement(self): + self.settings.refine_axes = ('z',) + state = {'estimating': False, 'checks': 0} + + def progress(message): + state['estimating'] |= message.startswith('Estimating') + + def cancelled(): + if state['estimating']: + state['checks'] += 1 + return state['checks'] >= 6 + + with self.assertRaises(InterruptedError): + mc.compute_from_collections([{'collections': [self.collection()], 'target_scene_m': self.target}], + self.config, settings=self.settings, progress=progress, cancelled=cancelled) + self.assertEqual(state['checks'], 6) + + def test_apply_rejects_broadcasting_and_corrupted_archives(self): + cal = self.calibration() + with self.assertRaisesRegex(ValueError, 'invalid shape'): + mc.apply_calibration(np.ones(self.freq.size), self.freq, cal) + cal['pair'][0, 0, 0] *= 2 + with self.assertRaisesRegex(ValueError, 'disagree'): + mc.validate_calibration(cal) + + def test_interpolation_is_continuous_at_nonlinear_archive_edges(self): + self.settings.phase_model = 'raw' + cal = self.calibration() + # Keep a consistent separable archive with a deliberately nonlinear phase. + phase = .4 * ((self.freq - self.freq.mean()) / 1e9) ** 3 + cal['tx'][1] *= np.exp(1j * phase) + cal['pair'] = cal['tx'][:, None] * cal['rx'][None] + edge = mc.calibration_on_grid(cal, [self.freq[-1], self.freq[-1] + 1]) + self.assertLess(abs(np.angle(edge[1, 0, 1] / edge[1, 0, 0])), 1e-6) + shifted = mc.calibration_on_grid(cal, self.freq + 10000) + self.assertIsNot(shifted, cal['pair']) + + def test_save_roundtrip_and_write_failure_preserve_the_previous_archive(self): + cal = self.calibration() + with tempfile.TemporaryDirectory() as directory: + destination = Path(directory) / 'cal' + destination.mkdir() + (destination / 'notes.txt').write_text('preserve user notes') + mc.save_calibration(cal, destination) + np.testing.assert_array_equal(mc.load_calibration(destination)['pair'], cal['pair']) + original = {path.name: path.read_bytes() for path in destination.iterdir()} + save = np.save + + def fail_second(path, *args, **kwargs): + if Path(path).name == 'cal_pair.npy': + raise OSError('disk full') + return save(path, *args, **kwargs) + + with mock.patch.object(mc.np, 'save', side_effect=fail_second), self.assertRaises(OSError): + mc.save_calibration(cal, destination, overwrite=True) + self.assertEqual(original, {path.name: path.read_bytes() for path in destination.iterdir()}) + self.assertEqual(list(Path(directory).glob('.cal.*')), []) + with self.assertRaises(FileExistsError): + mc.save_calibration(cal, destination) + + def test_non_overwriting_save_preserves_archive_published_during_staging(self): + with tempfile.TemporaryDirectory() as directory: + destination = Path(directory) / 'cal' + with self.assertRaises(FileExistsError): + with mc._staged_directory(destination, False) as stage: + (stage / 'cal_pair.npy').write_bytes(b'new correction') + destination.mkdir() + (destination / 'cal_pair.npy').write_bytes(b'concurrent correction') + (destination / 'cal_meta.json').write_bytes(b'concurrent metadata') + self.assertEqual({path.name: path.read_bytes() for path in destination.iterdir()}, { + 'cal_pair.npy': b'concurrent correction', 'cal_meta.json': b'concurrent metadata', + }) + self.assertEqual(list(Path(directory).glob('.cal.*')), []) + + def test_failed_commit_restores_old_archive_and_failed_rollback_keeps_backups(self): + cal = self.calibration() + with tempfile.TemporaryDirectory() as directory: + destination = Path(directory) / 'cal' + mc.save_calibration(cal, destination) + original = (destination / 'cal_pair.npy').read_bytes() + replace = Path.replace + + def fail_commit(path, target): + if path.name.startswith('.cal.new-'): + raise OSError('rename failed') + return replace(path, target) + + with mock.patch.object(Path, 'replace', autospec=True, side_effect=fail_commit), self.assertRaises(OSError): + mc.save_calibration(cal, destination, overwrite=True) + self.assertEqual((destination / 'cal_pair.npy').read_bytes(), original) + + def fail_rollback(path, target): + if path.name.startswith(('.cal.new-', '.cal.old-')): + raise OSError('filesystem unavailable') + return replace(path, target) + + with mock.patch.object(Path, 'replace', autospec=True, side_effect=fail_rollback), self.assertRaisesRegex(RuntimeError, 'originals retained'): + mc.save_calibration(cal, destination, overwrite=True) + backup = next(Path(directory).glob('.cal.old-*')) + self.assertEqual((backup / 'cal_pair.npy').read_bytes(), original) + + def test_offline_failure_preserves_previous_postprocessed_and_original_data(self): + cal = self.calibration() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + snapshot = self.archive(root, self.collection()) + post = root / 'postprocessed'; post.mkdir() + (post / 'calibration_applied.json').write_text('{}') + (post / 'old_data.bin').write_bytes(b'old data') + original = {path.name: path.read_bytes() for path in snapshot.iterdir()} + with mock.patch.object(mc.np, 'save', side_effect=OSError('disk full')), self.assertRaises(OSError): + mc.apply_to_measurement(root, cal, overwrite=True) + self.assertEqual((post / 'old_data.bin').read_bytes(), b'old data') + self.assertEqual(original, {path.name: path.read_bytes() for path in snapshot.iterdir()}) + self.assertEqual(list(root.glob('.postprocessed.*')), []) + + def test_object_corrected_archive_is_rejected_even_when_source_profile_has_object_off(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + snapshot = self.archive(root, self.collection()) + post, *_ = mc.apply_to_measurement(root, self.calibration()) + for prefixes in (None, [snapshot.name[:4]]): + with self.subTest(prefixes=prefixes), self.assertRaisesRegex(ValueError, 'already applied'): + mc.compute_calibration([{'path': post, 'snapshots': prefixes, 'target_scene_m': self.target}], + settings=self.settings) + # A corrected sibling does not change the original preprocessing data. + original = mc.compute_calibration([{'path': root / 'preprocessed', 'target_scene_m': self.target}], + settings=self.settings) + np.testing.assert_array_equal(original['pair'], self.calibration()['pair']) + + def test_snapshot_local_correction_marker_is_checked_for_target_and_empty_scene(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + snapshot = self.archive(root, self.collection()) + (snapshot / 'calibration_applied.json').write_text('{}') + targets = [{'path': root / 'preprocessed', 'snapshots': [snapshot.name[:4]], + 'target_scene_m': self.target}] + with self.assertRaisesRegex(ValueError, 'already applied'): + mc.compute_calibration(targets, settings=self.settings) + untouched = root / 'untouched' + self.archive(untouched, self.collection()) + targets = [{'path': untouched / 'preprocessed', 'target_scene_m': self.target}] + with self.assertRaisesRegex(ValueError, 'already applied'): + mc.compute_calibration(targets, root / 'preprocessed', self.settings) + + def test_offline_grid_cache_uses_the_entire_axis_and_preserves_s11_and_dtype(self): + cal = self.calibration() + collection = self.collection(np.ones((2, 2, self.freq.size), np.complex64)) + alternate = self.freq.copy() + alternate[64] += 1e7 + collection.traces[2].frequency_hz = alternate + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + snapshot = self.archive(root, collection) + original_s11 = (snapshot / 'i2_o5_s11.npy').read_bytes() + post, count, fixed, _ = mc.apply_to_measurement(root, cal) + self.assertEqual((count, fixed), (1, 4)) + corrected = np.load(post / snapshot.name / 'i2_o5_s21.npy') + self.assertEqual(corrected.dtype, np.complex64) + expected = mc.calibration_on_grid(cal, alternate)[1, 0] + np.testing.assert_allclose(corrected, expected, atol=1e-7) + self.assertGreater(abs(corrected[64] - cal['pair'][1, 0, 64]), .001) + self.assertEqual((post / snapshot.name / 'i2_o5_s11.npy').read_bytes(), original_s11) + + def test_empty_scene_grids_and_geometry_are_checked_before_subtraction(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + target_root, empty_root = root / 'target', root / 'empty' + self.archive(target_root, self.collection()) + zero = np.zeros((2, 2, self.freq.size), complex) + self.archive(empty_root, self.collection(zero)) + self.archive(empty_root, self.collection(zero, freq=self.freq + 10000), name='0001_id2_ns2') + targets = [{'path': target_root / 'preprocessed', 'target_scene_m': self.target}] + with self.assertRaisesRegex(ValueError, 'frequency grids differ'): + mc.compute_calibration(targets, empty_root / 'preprocessed', self.settings) + profile = copy.deepcopy(self.config) + profile['gpr']['rx_geometry'][0]['z_m'] += .1 + (empty_root / 'config_profile.json').write_text(json.dumps(profile)) + with self.assertRaisesRegex(ValueError, 'antenna geometry differs'): + mc.compute_calibration(targets, empty_root / 'preprocessed', self.settings) + + def test_snapshot_prefixes_must_select_one_distinct_complete_snapshot(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.archive(root, self.collection()) + self.archive(root, self.collection(), name='0001_id2_ns2') + for prefixes in (['000'], ['0000', 'missing'], ['0000', '0000']): + with self.subTest(prefixes=prefixes), self.assertRaises(ValueError): + mc.list_snapshots(root / 'preprocessed', prefixes) + + def test_multiple_snapshots_are_averaged_before_one_gated_fft(self): + collections = [self.collection(identity=index + 1) for index in range(8)] + with mock.patch.object(mc.np.fft, 'ifft', wraps=np.fft.ifft) as transform: + mc.compute_from_collections([{'collections': collections, 'target_scene_m': self.target}], + self.config, settings=self.settings) + self.assertEqual(transform.call_count, 1) + + def test_original_scipy_refinement_finishes_with_valid_diagnostics(self): + self.settings.refine_axes = ('z',) + self.settings.coarse_range_search_m = .1 + cal = self.calibration() + self.assertGreater(cal['meta']['diagnostics'][0]['coherence_after'], .999) + refined = cal['meta']['targets'][0]['refined_scene_m'] + self.assertTrue(np.all(np.isfinite(refined))) + + def test_bad_settings_and_duplicate_geometry_fail_early(self): + invalid_settings: tuple[dict[str, Any], ...] = ( + {'input_idx': (0, 0)}, {'output_idx': (True,)}, {'gate_taper_m': 0.}, + {'f_max': 0.}, {'refine_axes': ('z', 'z')}, {'phase_model': 'unknown'}, + ) + for fields in invalid_settings: + with self.subTest(fields=fields), self.assertRaises(ValueError): + mc.Settings(**fields) + self.config['gpr']['tx_geometry'].append(self.config['gpr']['tx_geometry'][0]) + with self.assertRaisesRegex(ValueError, 'duplicate antenna geometry'): + self.calibration() + + +if __name__ == '__main__': + unittest.main() diff --git a/python_app/tests/test_object_calibration_gui.py b/python_app/tests/test_object_calibration_gui.py new file mode 100644 index 0000000..ab0d326 --- /dev/null +++ b/python_app/tests/test_object_calibration_gui.py @@ -0,0 +1,304 @@ +"""Optional selection, live sample input and real target-phase diagnostics.""" + +from __future__ import annotations + +import copy +import os +import queue +import threading +import time +import unittest +from types import SimpleNamespace +from unittest import mock + +import numpy as np + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") +from PyQt6.QtCore import QTimer # noqa: E402 +from PyQt6.QtWidgets import QApplication, QPushButton, QWidget # noqa: E402 + +from python_app.gui.controllers.object_calibration_controller import ObjectCalibrationController, _calculate # noqa: E402 +from python_app.gui.object_calibration_dialog import ObjectCalibrationDialog # noqa: E402 +from python_app.gui.preprocess_dialog import PreprocessDialog # noqa: E402 +from python_app.tests import test_mimo_calibration as numeric_fixture # noqa: E402 + + +def setUpModule() -> None: + global app + app = QApplication.instance() or QApplication([]) + + +class ObjectCalibrationSelectorTest(unittest.TestCase): + def setUp(self) -> None: + self.dialog = PreprocessDialog() + self.addCleanup(self.dialog.close) + + def test_optional_off_is_distinct_from_a_saved_set_named_off(self) -> None: + self.dialog.set_available_sets({"s21_object_calibration": ["Off", "sphere"]}) + changes = mock.Mock() + self.dialog.selection_changed.connect(changes) + combo = self.dialog._set_combos["s21_object_calibration"] + self.assertEqual(combo.currentText(), "Off") + self.assertEqual(self.dialog.selection_snapshot()["s21_object_calibration"], "") + combo.setCurrentIndex(combo.findData("Off")) + self.assertEqual(self.dialog.selection_snapshot()["s21_object_calibration"], "Off") + changes.assert_called_once() + self.dialog.set_available_sets({"s21_object_calibration": ["Off", "sphere"]}) + self.assertEqual(self.dialog.selection_snapshot()["s21_object_calibration"], "Off") + self.dialog.set_selected_sets({"s21_object_calibration": ""}, emit_signal=False) + self.assertEqual(combo.currentText(), "Off") + self.assertEqual(combo.currentData(), "") + + def test_object_actions_do_not_start_a_sequential_hardware_capture(self) -> None: + capture, imported, created = mock.Mock(), mock.Mock(), mock.Mock() + self.dialog.start_sequence_requested.connect(capture) + self.dialog.import_object_calibration_requested.connect(imported) + self.dialog.create_object_calibration_requested.connect(created) + self.dialog._import_object_calibration_button.click() + self.dialog._create_object_calibration_button.click() + imported.assert_called_once() + created.assert_called_once() + capture.assert_not_called() + self.assertEqual([button.text() for button in self.dialog.findChildren(QPushButton) + if button.text().startswith("Start ")], + ["Start S21 Calibration", "Start S21 Reference"]) + + +class ObjectCalibrationDialogTest(unittest.TestCase): + def setUp(self) -> None: + self.fixture = numeric_fixture.MimoCalibrationTest() + self.fixture.setUp() + self.dialog = ObjectCalibrationDialog("sphere") + self.addCleanup(self.dialog.close) + + def _capture_with_target(self) -> None: + self.dialog.add_capture([self.fixture.collection()], self.fixture.config) + for column, coordinate in enumerate(self.fixture.target, start=2): + self.dialog._table.item(0, column).setText(str(coordinate)) + + def test_capture_is_frozen_and_xyz_must_be_entered_after_measurement(self) -> None: + collection = self.fixture.collection() + self.dialog.add_capture([collection], self.fixture.config) + expected = collection.traces[0].s21.copy() + collection.traces[0].s21.fill(0) + with self.assertRaisesRegex(ValueError, "X, Y и Z"): + self.dialog.calculation_inputs() + for column, coordinate in enumerate(self.fixture.target, start=2): + self.dialog._table.item(0, column).setText(str(coordinate)) + groups, config, settings = self.dialog.calculation_inputs() + np.testing.assert_array_equal(groups[0]["collections"][0].traces[0].s21, expected) + self.assertEqual(groups[0]["target_scene_m"], self.fixture.target) + self.assertEqual(config, self.fixture.config) + self.assertEqual(settings["input_idx"], (2, 7)) + self.assertEqual(settings["output_idx"], (1, 5)) + self.dialog._table.item(0, 2).setText("nan") + with self.assertRaisesRegex(ValueError, "конечными"): + self.dialog.calculation_inputs() + + def test_changed_geometry_is_rejected_without_appending_a_capture(self) -> None: + self._capture_with_target() + changed = copy.deepcopy(self.fixture.config) + changed["gpr"]["tx_geometry"][0]["x_m"] += 1 + with self.assertRaisesRegex(ValueError, "изменились"): + self.dialog.add_capture([self.fixture.collection()], changed) + self.assertEqual(self.dialog._table.rowCount(), 1) + + def test_actual_numeric_output_draws_both_target_phase_curves(self) -> None: + calibration = self.fixture.calibration() + self.dialog.show_calibration(calibration) + curves = self.dialog._phase_plot.listDataItems() + self.assertEqual(len(curves), 2) + diagnostic = calibration["meta"]["diagnostics"][0] + phases = np.array([diagnostic["phase_before_deg"], diagnostic["phase_after_deg"]]).reshape(2, -1) + expected = (phases - phases[0, 0] + 180) % 360 - 180 + for curve, phases in zip(curves, expected): + np.testing.assert_allclose(curve.getData()[1], phases) + self.assertLess(np.ptp(curves[1].getData()[1]), .5) + self.assertIn(f"{diagnostic['coherence_before']:.3f}", self.dialog._quality_label.text()) + self.assertIn(f"{diagnostic['coherence_after']:.3f}", self.dialog._quality_label.text()) + + def test_phase_wrap_uses_the_same_first_pair_baseline_for_both_curves(self) -> None: + calibration = self.fixture.calibration() + diagnostic = calibration["meta"]["diagnostics"][0] + diagnostic["phase_before_deg"] = [[179, -179], [178, -178]] + diagnostic["phase_after_deg"] = [[-179, -179], [-179, -179]] + self.dialog.show_calibration(calibration) + before, after = self.dialog._phase_plot.listDataItems() + np.testing.assert_allclose(before.getData()[1], [0, 2, -1, 3]) + np.testing.assert_allclose(after.getData()[1], [2, 2, 2, 2]) + + def test_capture_and_calculation_disable_edits_but_keep_cancel_available(self) -> None: + self._capture_with_target() + for setter in (self.dialog.set_capture_pending, self.dialog.set_calculating): + setter(True) + self.assertFalse(self.dialog._capture_button.isEnabled()) + self.assertFalse(self.dialog._table.isEnabled()) + self.assertTrue(self.dialog._cancel_button.isEnabled()) + setter(False) + self.assertTrue(self.dialog._capture_button.isEnabled()) + self.assertTrue(self.dialog._table.isEnabled()) + + +class ObjectCalibrationWorkerTest(unittest.TestCase): + def _controller_with_data(self): + fixture = numeric_fixture.MimoCalibrationTest() + fixture.setUp() + owner = QWidget() + self.addCleanup(owner.close) + panel = SimpleNamespace(set_name=lambda: "sphere", set_set_name=mock.Mock(), set_status=mock.Mock()) + current_config = SimpleNamespace(to_dict=lambda: copy.deepcopy(fixture.config)) + owner._ensure_preprocess_dialog = mock.Mock(return_value=panel) + owner._build_config = mock.Mock(return_value=current_config) + owner._radar_key = mock.Mock(return_value="current-config-key") + owner._store = mock.Mock() + owner._selected_preprocess_sets = {} + for name in ("_show_exception", "_show_error", "_refresh_sets", "_log"): + setattr(owner, name, mock.Mock()) + controller = ObjectCalibrationController(owner) + controller.open_creation() + self.addCleanup(controller.shutdown) + dialog = controller._dialog + dialog.add_capture([fixture.collection()], fixture.config) + for column, coordinate in enumerate(fixture.target, start=2): + dialog._table.item(0, column).setText(str(coordinate)) + for key in ("sphere_radius_m", "antenna_height_m", "pitch_deg", "coarse_range_search_m"): + dialog._settings[key].setValue(getattr(fixture.settings, key)) + for field in dialog._refine_axes.values(): + field.setChecked(False) + return owner, controller, fixture, current_config + + def _wait_for_completion(self, controller) -> None: + deadline = time.monotonic() + 5 + while controller._worker is not None and time.monotonic() < deadline: + app.processEvents() + time.sleep(.005) + self.assertIsNone(controller._worker, "Background calculation did not complete") + + def test_actual_calculation_runs_off_gui_thread_and_saves_under_current_config(self) -> None: + owner, controller, _, current_config = self._controller_with_data() + from python_app.workflows.mimo_calibration import compute_from_collections + + worker_threads, gui_ticks = [], [] + + def compute(*args, **kwargs): + worker_threads.append(threading.get_ident()) + return compute_from_collections(*args, **kwargs) + + timer = QTimer() + timer.timeout.connect(lambda: gui_ticks.append(threading.get_ident())) + timer.start(1) + self.addCleanup(timer.stop) + with mock.patch("python_app.workflows.mimo_calibration.compute_from_collections", side_effect=compute): + controller._start_calculation() + self.assertTrue(controller._dialog._calculating) + self._wait_for_completion(controller) + owner._show_exception.assert_not_called() + self.assertEqual(len(worker_threads), 1) + self.assertNotEqual(worker_threads[0], threading.get_ident()) + self.assertTrue(gui_ticks) + self.assertEqual(set(gui_ticks), {threading.get_ident()}) + owner._radar_key.assert_called_once_with(current_config) + key, name, calibration = owner._store.save_object_calibration.call_args.args + self.assertEqual((key, name), ("current-config-key", "sphere")) + self.assertEqual(owner._selected_preprocess_sets["s21_object_calibration"], name) + self.assertEqual(len(controller._dialog._phase_plot.listDataItems()), 2) + self.assertGreater(calibration["meta"]["diagnostics"][0]["coherence_after"], .999) + + def test_cancelled_background_calculation_keeps_samples_and_creates_no_saved_set(self) -> None: + owner, controller, _, _ = self._controller_with_data() + entered = threading.Event() + + def compute(*_args, cancelled, progress, **_kwargs): + entered.set() + progress("Safe cancellation boundary") + while not cancelled(): + time.sleep(.005) + raise InterruptedError + + with mock.patch("python_app.workflows.mimo_calibration.compute_from_collections", side_effect=compute): + controller._start_calculation() + self.assertTrue(entered.wait(1)) + controller.cancel() + self._wait_for_completion(controller) + owner._store.save_object_calibration.assert_not_called() + owner._show_exception.assert_not_called() + self.assertEqual(controller._dialog._table.rowCount(), 1) + self.assertTrue(controller._dialog._capture_button.isEnabled()) + self.assertIn("отменён", controller._dialog._status_label.text()) + + def test_thread_start_failure_restores_editor_without_saving(self) -> None: + owner, controller, _, _ = self._controller_with_data() + with mock.patch("python_app.gui.controllers.object_calibration_controller.threading.Thread.start", + side_effect=RuntimeError("thread limit")): + controller._start_calculation() + self.assertIsNone(controller._worker) + self.assertFalse(controller._dialog._calculating) + self.assertTrue(controller._dialog._capture_button.isEnabled()) + owner._store.save_object_calibration.assert_not_called() + owner._show_exception.assert_called_once() + self.assertIn("thread limit", controller._dialog._status_label.text()) + + def test_capture_registration_failure_is_visible_and_does_not_leave_busy_dialog(self) -> None: + owner, controller, _, _ = self._controller_with_data() + with mock.patch("python_app.gui.controllers.object_calibration_capture.capture_object_sweeps", + side_effect=RuntimeError("Capture already running")): + controller._capture_data() + self.assertIsNone(controller._capture) + self.assertTrue(controller._dialog._capture_button.isEnabled()) + self.assertIn("Capture already running", controller._dialog._status_label.text()) + owner._show_exception.assert_called_once() + + def test_import_waits_for_capture_or_calculation_before_opening_file_picker(self) -> None: + owner, controller, _, _ = self._controller_with_data() + for busy in ("capture", "calculation"): + with self.subTest(busy=busy): + owner._object_capture_job = object() if busy == "capture" else None + controller._worker = mock.Mock() if busy == "calculation" else None + with mock.patch("python_app.gui.controllers.object_calibration_controller.QFileDialog.getExistingDirectory") as picker: + controller.import_calibration() + picker.assert_not_called() + owner._show_error.assert_called() + controller._worker = None + owner._object_capture_job = None + + def test_worker_uses_only_data_and_honours_cancellation_without_subtracting_another_background(self) -> None: + fixture = numeric_fixture.MimoCalibrationTest() + fixture.setUp() + cancelled, messages, result = threading.Event(), queue.Queue(), queue.Queue() + cancelled.set() + with mock.patch("python_app.workflows.mimo_calibration.compute_from_collections", + side_effect=InterruptedError) as compute: + _calculate([{"collections": [fixture.collection()], "target_scene_m": fixture.target}], + fixture.config, {}, cancelled, messages, result) + self.assertEqual(result.get_nowait(), ("cancelled", None)) + self.assertNotIn("empty_collections", compute.call_args.kwargs) + + def test_calculate_rejects_invalid_xyz_before_starting_a_worker(self) -> None: + owner = QWidget() + self.addCleanup(owner.close) + owner._ensure_preprocess_dialog = mock.Mock(return_value=SimpleNamespace(set_name=lambda: "sphere")) + owner._show_exception = mock.Mock() + controller = ObjectCalibrationController(owner) + controller.open_creation() + self.addCleanup(controller.shutdown) + fixture = numeric_fixture.MimoCalibrationTest() + fixture.setUp() + controller._dialog.add_capture([fixture.collection()], fixture.config) + with mock.patch("python_app.gui.controllers.object_calibration_controller.threading.Thread") as thread: + controller._start_calculation() + thread.assert_not_called() + owner._show_exception.assert_called_once() + self.assertFalse(controller._dialog._calculating) + + def test_closing_cancels_capture_and_worker_without_waiting_for_a_thread(self) -> None: + owner = QWidget() + self.addCleanup(owner.close) + controller = ObjectCalibrationController(owner) + controller._dialog = ObjectCalibrationDialog("sphere", owner) + controller._capture = mock.Mock() + controller._worker = mock.Mock() + controller.shutdown() + self.assertTrue(controller._cancel.is_set()) + controller._capture.cancel.assert_called_once() + controller._worker.join.assert_not_called() + self.assertFalse(controller._poll.isActive()) diff --git a/python_app/tests/test_object_calibration_runtime.py b/python_app/tests/test_object_calibration_runtime.py new file mode 100644 index 0000000..2ebf212 --- /dev/null +++ b/python_app/tests/test_object_calibration_runtime.py @@ -0,0 +1,319 @@ +"""Object correction persistence, runtime export and capture lifecycle regressions.""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import asdict +import os +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest import mock + +import numpy as np + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") +from PyQt6.QtWidgets import QApplication, QDialog # noqa: E402 + +from python_app.gui.controllers.object_calibration_capture import ObjectSweepCapture # noqa: E402 +from python_app.gui.controllers.app_window_config.live_processing_mixin import AppWindowLiveProcessingMixin # noqa: E402 +from python_app.gui.runtime.history import build_processor_run_signature, build_run_history_signature # noqa: E402 +from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData # noqa: E402 +from python_app.models.run_config_model import ( # noqa: E402 + ComboModel, GprRxGeometryModel, GprTxGeometryModel, RunConfigModel, +) +from python_app.orchestration.config_writer import ConfigWriter # noqa: E402 +from python_app.orchestration.preprocess_assets import runtime_preprocess_asset_keys # noqa: E402 +from python_app.orchestration.shm.decoder import decode_trace_collection # noqa: E402 +from python_app.storage.npz.store import NpzStore # noqa: E402 +from python_app.storage.npz.serialize import RAW_MAGIC # noqa: E402 +from python_app.workflows.mimo_calibration import C_LIGHT, calibration_on_grid # noqa: E402 + + +def setUpModule() -> None: + global _app + _app = QApplication.instance() or QApplication([]) + + +def _config() -> RunConfigModel: + config = RunConfigModel() + config.input_switch.positions = 3 + config.output_switch.positions = 4 + config.combos = [ComboModel(0, 1), ComboModel(2, 3)] + config.gpr.tx_geometry = [GprTxGeometryModel(1, -.4), GprTxGeometryModel(3, .4)] + config.gpr.rx_geometry = [GprRxGeometryModel(0, -.2), GprRxGeometryModel(2, .2)] + config.preprocess.s21.calibration.set_name = "through" + config.preprocess.s21.reference.set_name = "empty" + return config + + +def _calibration(config: RunConfigModel) -> dict: + frequency = np.array([1e8, 2e8, 3e8]) + parameters = np.array([[.3, .02, 0], [-.8, -.01, 0], [.4, .03, 0], [-.1, -.02, 0]]) + corrections = np.exp(-1j * (parameters[:, 0, None] - 2 * np.pi * frequency * parameters[:, 1, None] / C_LIGHT)) + return dict( + freq_hz=frequency, tx=corrections[:2], rx=corrections[2:], + pair=corrections[:2, None] * corrections[None, 2:], valid=np.ones(3, bool), + tx_output_pos=np.array([1, 3], np.int32), rx_input_pos=np.array([0, 2], np.int32), + antenna_params=parameters, dir=None, + meta={"geometry": { + "tx_xyz_m": [[entry.x_m, entry.y_m, entry.z_m] for entry in config.gpr.tx_geometry], + "rx_xyz_m": [[entry.x_m, entry.y_m, entry.z_m] for entry in config.gpr.rx_geometry], + }}, + ) + + +class ObjectCalibrationStoreTest(unittest.TestCase): + def setUp(self) -> None: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.store = NpzStore(self.root / "sets") + self.config = _config() + self.calibration = _calibration(self.config) + traces = [ + TraceData(ComboKey(0, 1), np.array([1e8, 2e8, 3e8], np.float32), np.ones(3), np.ones(3)), + TraceData(ComboKey(2, 3), np.array([1.2e8, 1.8e8, 2.6e8], np.float32), np.ones(3), np.ones(3)), + ] + for kind, name in (("s21_calibration", "through"), ("s21_reference", "empty")): + self.store.save_set(kind, "radar", name, SweepCollection(1, 10, traces)) + + def test_default_disabled_and_legacy_config_round_trip(self) -> None: + payload = self.config.to_dict() + del payload["preprocess"]["s21"]["object_calibration"] + config = RunConfigModel.from_dict(payload) + self.assertEqual(config.preprocess.s21.object_calibration.set_name, "") + self.assertNotIn("s21_object_calibration", runtime_preprocess_asset_keys(config)) + original_run = build_run_history_signature(config) + original_processor = build_processor_run_signature(config) + config.preprocess.s21.object_calibration.set_name = "object" + self.assertEqual(config.clone().preprocess.s21.object_calibration.set_name, "object") + self.assertIn("s21_object_calibration", runtime_preprocess_asset_keys(config)) + self.assertNotEqual(original_run, build_run_history_signature(config)) + self.assertEqual(original_processor, build_processor_run_signature(config)) + + def test_saved_set_retains_coefficients_and_refuses_overwrite(self) -> None: + self.store.save_object_calibration("radar", "object", self.calibration) + self.assertEqual(self.store.list_sets("s21_object_calibration", "radar"), ["object"]) + loaded = self.store.load_object_calibration("radar", "object") + np.testing.assert_array_equal(loaded["pair"], self.calibration["pair"]) + with self.assertRaises(FileExistsError): + self.store.save_object_calibration("radar", "object", self.calibration) + self.assertTrue(self.store.has_combo_coverage("s21_object_calibration", "radar", "object", [ComboKey(2, 3)])) + self.assertFalse(self.store.has_combo_coverage("s21_object_calibration", "radar", "object", [ComboKey(1, 3)])) + with self.assertRaises(ValueError): + self.store.load_object_calibration("radar", "../object") + + def test_writer_exports_physical_pair_factors_on_each_runtime_grid(self) -> None: + self.store.save_object_calibration("radar", "object", self.calibration) + self.config.preprocess.s21.object_calibration.set_name = "object" + ConfigWriter(self.root / "runtime").prepare_preprocess_bundles(self.store, "radar", self.config) + bundle = Path(self.config.preprocess.s21.object_calibration.bundle_path) + collection = decode_trace_collection(bundle.read_bytes(), RAW_MAGIC) + self.assertEqual([trace.combo for trace in collection.traces], [ComboKey(0, 1), ComboKey(2, 3)]) + for index, trace in enumerate(collection.traces): + np.testing.assert_allclose(trace.s21, calibration_on_grid(self.calibration, trace.frequency_hz)[index, index], rtol=1e-6) + np.testing.assert_array_equal(trace.s11, np.zeros(3)) + self.config.preprocess.s21.object_calibration.set_name = "" + ConfigWriter(self.root / "runtime").prepare_preprocess_bundles(self.store, "radar", self.config) + self.assertEqual(self.config.preprocess.s21.object_calibration.bundle_path, "") + + def test_invalid_geometry_or_correction_keeps_previous_export(self) -> None: + self.store.save_object_calibration("radar", "object", self.calibration) + destination = self.root / "factors.bin" + destination.write_bytes(b"previous bundle") + self.config.gpr.tx_geometry[0].z_m = .1 + with self.assertRaisesRegex(ValueError, "geometry"): + self.store.export_object_calibration_bundle("radar", "object", destination, self.config) + self.assertEqual(destination.read_bytes(), b"previous bundle") + broken = deepcopy(self.calibration) + broken["pair"][0, 0, 0] = np.nan + with self.assertRaises(ValueError): + self.store.save_object_calibration("radar", "bad", broken) + self.assertEqual(self.store.list_sets("s21_object_calibration", "radar"), ["object"]) + + +class ObjectCaptureLifecycleTest(unittest.TestCase): + def setUp(self) -> None: + self.dialog = mock.Mock() + self.owner = SimpleNamespace( + _stop_request_seq=0, _capture_session=None, _preprocess_capture_busy=False, + _single_capture_active=False, _recording_active=False, _closing=False, + _build_config=lambda: _config(), _pre_history=[], _SINGLE_CAPTURE_TIMEOUT_S=300, + _ensure_preprocess_dialog=lambda: SimpleNamespace(median_sweep_count=lambda: 2), + _supervisor=SimpleNamespace(is_running=lambda: self.running), + ) + self.running = False + self.owner._pipeline_resume_allowed = lambda seq: not self.owner._closing and seq == self.owner._stop_request_seq + self.owner._stop_run = mock.Mock(side_effect=self._stop) + self.owner._start_run = mock.Mock(side_effect=self._start) + + def _stop(self) -> None: + self.running = False + self.owner._stop_request_seq += 1 + + def _start(self, **_kwargs) -> None: + self.running = True + + def _capture(self) -> ObjectSweepCapture: + with mock.patch("python_app.gui.controllers.object_calibration_capture.QTimer.singleShot"): + view = QDialog() + view.add_capture = self.dialog.add_capture + view.set_status = self.dialog.set_status + view.set_capture_pending = self.dialog.set_capture_pending + self.addCleanup(view.close) + self.view = view + return ObjectSweepCapture(self.owner, view) + + def test_captures_fresh_preprocessed_matrix_without_changing_selected_object(self) -> None: + config = _config() + config.preprocess.s21.object_calibration.set_name = "existing" + self.owner._build_config = lambda: config.clone() + self.running = True + capture = self._capture() + capture._start() + override = self.owner._start_run.call_args.kwargs["config_override"] + self.assertEqual(override.preprocess.s21.object_calibration.set_name, "") + self.assertEqual(len(override.combos), 4) + self.assertEqual(config.preprocess.s21.object_calibration.set_name, "existing") + stale = SweepCollection(1, capture._start_ns - 1) + first, second = SweepCollection(2, capture._start_ns + 1), SweepCollection(3, capture._start_ns + 2) + self.owner._pre_history = [stale, first, first, second] + capture._poll() + self.dialog.add_capture.assert_called_once() + self.assertEqual([c.collection_id for c in self.dialog.add_capture.call_args.args[0]], [2, 3]) + self.assertEqual(self.owner._start_run.call_count, 2) + self.assertEqual(self.owner._start_run.call_args.kwargs, {}) + capture.cancel() + self.assertEqual(self.owner._stop_run.call_count, 2) + + def test_stop_cancels_capture_and_prevents_automatic_resume(self) -> None: + self.running = True + capture = self._capture() + capture._start() + self.owner._stop_run() + capture._poll() + self.dialog.add_capture.assert_not_called() + self.assertEqual(self.owner._start_run.call_count, 1) + self.assertFalse(self.running) + + def test_recording_is_not_interrupted(self) -> None: + self.owner._recording_active = True + capture = self._capture() + capture._start() + self.owner._start_run.assert_not_called() + self.owner._stop_run.assert_not_called() + self.dialog.add_capture.assert_not_called() + self.dialog.set_capture_pending.assert_called_once_with(False) + + def test_stop_before_queued_start_never_opens_acquisition(self) -> None: + capture = self._capture() + self.owner._stop_run() + capture._start() + self.owner._start_run.assert_not_called() + self.assertIsNone(self.owner._object_capture_job) + + def test_stale_capture_does_not_stop_a_new_run_after_explicit_stop(self) -> None: + capture = self._capture() + capture._start() + self.owner._stop_run() + self.owner._start_run() + capture._poll() + self.assertEqual(self.owner._stop_run.call_count, 1) + self.assertTrue(self.running) + self.dialog.add_capture.assert_not_called() + + def test_second_capture_is_rejected_without_mutating_first(self) -> None: + first = self._capture() + with self.assertRaisesRegex(RuntimeError, "already in progress"): + self._capture() + self.assertIs(self.owner._object_capture_job, first) + first.cancel() + + def test_rejected_data_still_restores_original_running_pipeline(self) -> None: + self.running = True + self.dialog.add_capture.side_effect = ValueError("configuration changed") + capture = self._capture() + capture._start() + self.owner._pre_history = [SweepCollection(2, capture._start_ns + 1), SweepCollection(3, capture._start_ns + 2)] + capture._poll() + self.assertTrue(self.running) + self.assertEqual(self.owner._start_run.call_count, 2) + self.assertIsNone(self.owner._object_capture_job) + + def test_capture_lock_is_held_while_stop_drains_events(self) -> None: + capture = self._capture() + capture._start() + locks = [] + original_stop = self.owner._stop_run.side_effect + self.owner._stop_run.side_effect = lambda: (locks.append(self.owner._object_capture_job), original_stop()) + capture.cancel() + self.assertEqual(locks, [capture]) + self.assertIsNone(self.owner._object_capture_job) + + def test_old_acquisition_with_new_preprocessing_timestamp_is_not_captured(self) -> None: + capture = self._capture() + capture._start() + stale = SweepCollection(1, capture._start_ns + 100, capture_start_ns=capture._start_ns - 100) + self.owner._pre_history = [stale] + capture._poll() + self.assertEqual(capture._collections, []) + capture.cancel() + + def test_replay_is_rejected_before_acquisition(self) -> None: + config = _config() + config.radar.replay.dataset_dir = "saved_scene" + self.owner._build_config = lambda: config + capture = self._capture() + capture._start() + self.owner._start_run.assert_not_called() + + +class ObjectGeometryLifecycleTest(unittest.TestCase): + def setUp(self) -> None: + config = _config() + config.preprocess.s21.object_calibration.set_name = "object" + self.tx, self.rx = [asdict(entry) for entry in config.gpr.tx_geometry], [asdict(entry) for entry in config.gpr.rx_geometry] + self.owner = SimpleNamespace( + _selected_preprocess_sets={"s21_object_calibration": "object"}, _selected_preprocess_radar_key="radar", + _store=SimpleNamespace(load_object_calibration=mock.Mock(return_value=_calibration(config))), + _gpr_tx_geometry_input=SimpleNamespace(records=lambda: self.tx), + _gpr_rx_geometry_input=SimpleNamespace(records=lambda: self.rx), + _supervisor=SimpleNamespace(is_running=lambda: True), _preprocess_dialog=mock.Mock(), + _stop_request_seq=0, _refresh_preprocess_summary_labels=mock.Mock(), _start_run=mock.Mock(), _log_warning=mock.Mock(), + _active_run_config=config, _radar_key=lambda _config: "radar", + ) + self.owner._pipeline_resume_allowed = lambda seq: seq == self.owner._stop_request_seq + self.owner._stop_run = mock.Mock(side_effect=lambda: setattr(self.owner, "_stop_request_seq", self.owner._stop_request_seq + 1)) + + def test_same_geometry_retains_object_correction_without_restart(self) -> None: + AppWindowLiveProcessingMixin._invalidate_object_calibration_after_geometry_change(self.owner) + self.assertEqual(self.owner._selected_preprocess_sets["s21_object_calibration"], "object") + self.owner._stop_run.assert_not_called() + self.owner._start_run.assert_not_called() + + def test_changed_geometry_disables_correction_and_reloads_acquisition(self) -> None: + self.tx[0]["z_m"] = .1 + AppWindowLiveProcessingMixin._invalidate_object_calibration_after_geometry_change(self.owner) + self.assertEqual(self.owner._selected_preprocess_sets["s21_object_calibration"], "") + self.owner._stop_run.assert_called_once() + self.owner._start_run.assert_called_once() + self.owner._log_warning.assert_called_once() + + def test_explicit_stop_during_geometry_change_prevents_resume(self) -> None: + self.tx[0]["z_m"] = .1 + self.owner._stop_run.side_effect = lambda: setattr(self.owner, "_stop_request_seq", self.owner._stop_request_seq + 2) + AppWindowLiveProcessingMixin._invalidate_object_calibration_after_geometry_change(self.owner) + self.owner._start_run.assert_not_called() + + def test_pending_off_selection_still_reloads_an_incompatible_active_correction(self) -> None: + self.owner._selected_preprocess_sets["s21_object_calibration"] = "" + self.tx[0]["z_m"] = .1 + AppWindowLiveProcessingMixin._invalidate_object_calibration_after_geometry_change(self.owner) + self.owner._stop_run.assert_called_once() + self.owner._start_run.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/python_app/tests/test_recording_mixin.py b/python_app/tests/test_recording_mixin.py index 2f88dbd..81df524 100644 --- a/python_app/tests/test_recording_mixin.py +++ b/python_app/tests/test_recording_mixin.py @@ -19,7 +19,9 @@ import numpy as np from python_app.gui.controllers import app_window_recording_mixin as rec from python_app.gui.controllers.app_window_recording_mixin import AppWindowRecordingMixin +from python_app.gui.controllers.app_window_config.profile_io_mixin import AppWindowConfigProfileIOMixin from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData +from python_app.models.gui_profile_model import GuiProfileModel from python_app.storage.npz.store import NpzStore @@ -113,7 +115,7 @@ class _Harness(AppWindowRecordingMixin): def _snapshot_config_profile_path(self, directory): return Path(directory) / "config_profile.json" - def _write_gui_profile_to_path(self, path, allow_overwrite) -> None: + def _write_gui_profile_to_path(self, path, allow_overwrite=True, dataset=False) -> None: self.profiles.append((path, allow_overwrite)) def _start_run(self) -> None: @@ -162,7 +164,7 @@ class RecordingArmTest(_RecordingTestBase): h._save_path_input = _Text(str(Path(temporary) / "datasets")) destination = Path(temporary) / "datasets/pref_run" - def write_profile(path, allow_overwrite): + def write_profile(path, allow_overwrite=True, dataset=False): path.write_text("fresh profile", encoding="utf-8") with mock.patch.object(h, "_write_gui_profile_to_path", side_effect=write_profile): @@ -210,6 +212,57 @@ class RecordingArmTest(_RecordingTestBase): self.assertFalse(h.started) self.assertTrue(h._recording_active) + def test_running_recording_profile_uses_applied_object_calibration(self) -> None: + for applied, selected in (("", "next_run"), ("applied", "")): + with self.subTest(applied=applied), tempfile.TemporaryDirectory() as temporary: + h = self._make(count=3, running=True) + h._store = NpzStore(Path(temporary) / "sets") + h._save_path_input = _Text(str(Path(temporary) / "datasets")) + pending = GuiProfileModel() + pending.run_config.preprocess.s21.object_calibration.set_name = selected + h._active_run_config = pending.run_config.clone() + h._active_run_config.preprocess.s21.object_calibration.set_name = applied + h._active_run_config.preprocess.s21.object_calibration.bundle_path = "/bundles/applied" if applied else "" + h._build_gui_profile = pending.clone + with mock.patch.object(h, "_write_gui_profile_to_path", side_effect=lambda *args, **kwargs: + AppWindowConfigProfileIOMixin._write_gui_profile_to_path(h, *args, **kwargs)): + h._start_run_with_recording() + path = Path(temporary) / "datasets/pref_run/config_profile.json" + recorded = GuiProfileModel.load_from_path(path) + self.assertEqual(recorded.run_config.preprocess, h._active_run_config.preprocess) + self.assertFalse(h.started) + self.assertEqual(h.exceptions, []) + + def test_new_recording_profile_uses_new_run_instead_of_previous_history(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + h = self._make(count=3, running=False) + h._store = NpzStore(Path(temporary) / "sets") + h._save_path_input = _Text(str(Path(temporary) / "datasets")) + pending = GuiProfileModel() + pending.run_config.preprocess.s21.object_calibration.set_name = "next_run" + h._active_run_config = pending.run_config.clone() + h._active_run_config.preprocess.s21.object_calibration.set_name = "previous_run" + h._build_gui_profile = pending.clone + path = Path(temporary) / "datasets/pref_run/config_profile.json" + + def start_new_run(): + initial = GuiProfileModel.load_from_path(path) + self.assertEqual(initial.run_config.preprocess, pending.run_config.preprocess) + h._active_run_config = pending.run_config.clone() + h._active_run_config.preprocess.s21.object_calibration.bundle_path = "/bundles/new_object" + h._active_run_config.preprocess.s21.calibration.bundle_path = "/bundles/new_through" + h._supervisor._running = True + + with mock.patch.object(h, "_start_run", side_effect=start_new_run), mock.patch.object( + h, "_write_gui_profile_to_path", side_effect=lambda *args, **kwargs: + AppWindowConfigProfileIOMixin._write_gui_profile_to_path(h, *args, **kwargs), + ): + h._start_run_with_recording() + recorded = GuiProfileModel.load_from_path(path) + self.assertEqual(recorded.run_config.preprocess, h._active_run_config.preprocess) + self.assertEqual(h.exceptions, []) + self.assertTrue(h._recording_active) + def test_arm_refuses_when_destination_exists(self) -> None: h = self._make(count=3, running=True, dest_exists=True) h._start_run_with_recording() @@ -248,7 +301,7 @@ class RecordingArmTest(_RecordingTestBase): traces=[TraceData(ComboKey(0, 0), np.array([1e9], dtype=np.float32), samples, samples)], )) - def write_profile(path, allow_overwrite): + def write_profile(path, allow_overwrite=True, dataset=False): path.write_text('retained configuration', encoding='utf-8') with mock.patch.object(h, '_start_run', side_effect=failed_start), mock.patch.object( diff --git a/python_app/workflows/mimo_calibration.py b/python_app/workflows/mimo_calibration.py new file mode 100644 index 0000000..9a19bf8 --- /dev/null +++ b/python_app/workflows/mimo_calibration.py @@ -0,0 +1,792 @@ +"""Object-based MIMO S21 corrections estimated after standard preprocessing. + +Archives retain the standalone script's cal_*.npy/cal_meta.json layout. Pair +corrections multiply S21 after through calibration, reference subtraction and +notch filtering; they never alter S11 or the original measurement files. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass +import datetime +import json +import logging +from pathlib import Path +import re +import shutil +import tempfile +from typing import Any + +import numpy as np +from numpy.typing import ArrayLike + +from python_app.file_io import atomic_write_json +from python_app.models.dataset_model import SweepCollection + +C_LIGHT = 299_792_458.0 +_CAL_ARRAYS = ('freq_hz', 'pair', 'tx', 'rx', 'valid', 'tx_output_pos', 'rx_input_pos', 'antenna_params') +logger = logging.getLogger(__name__) + + +def _ports(values, label): + array = np.asarray(values) + if array.ndim != 1 or not array.size or array.dtype.kind not in 'iu': + raise ValueError(f'{label} must contain distinct nonnegative integer ports') + if np.any(array < 0) or np.any(array > np.iinfo(np.int32).max) or np.unique(array).size != array.size: + raise ValueError(f'{label} must contain distinct nonnegative integer ports') + return tuple(int(value) for value in array) + + +def _finite_array(value, label, shape=None, *, complex_values=False): + array = np.asarray(value, dtype=complex if complex_values else float) + if (shape is not None and array.shape != shape) or not np.all(np.isfinite(array)): + raise ValueError(f'{label} has an invalid shape or non-finite values') + return array + + +def _frequency(value, *, uniform=False): + freq = _finite_array(value, 'frequency axis') + if freq.ndim != 1 or freq.size < 2 or freq[0] < 0 or np.any(np.diff(freq) <= 0): + raise ValueError('frequency axis must be increasing with at least two samples') + # Dataset frequency axes are float32; account for their quantization at GHz. + tolerance = max(1e-3, 2 * np.finfo(np.float32).eps * float(freq[-1])) + if uniform and not np.allclose(np.diff(freq), np.median(np.diff(freq)), rtol=1e-5, atol=tolerance): + raise ValueError('calibration requires a uniform frequency grid') + return freq + + +@dataclass +class Settings: + """Target geometry and estimation options, independent of example globals.""" + sphere_radius_m: float = .16 + input_idx: tuple[int, ...] = (0, 1, 2, 3) + output_idx: tuple[int, ...] = (0, 1, 2, 3) + antenna_height_m: float = .92 + pitch_deg: float = 10. + f_min: float = 2e9 + f_max: float = 6e9 + gate_half_m: float = .15 + gate_taper_m: float = .03 + coarse_range_search_m: float = .6 + refine_axes: tuple[str, ...] = ('x', 'y', 'z') + refine_range_m: float = .10 + apply_amplitude: bool = False + phase_model: str = 'linear' + + def __post_init__(self): + self.input_idx = tuple(sorted(_ports(self.input_idx, 'input_idx'))) + self.output_idx = tuple(sorted(_ports(self.output_idx, 'output_idx'))) + for name in ('sphere_radius_m', 'antenna_height_m', 'pitch_deg', 'f_min', 'f_max', + 'gate_half_m', 'gate_taper_m', 'coarse_range_search_m', 'refine_range_m'): + value = getattr(self, name) + if isinstance(value, bool) or not np.isfinite(value): + raise ValueError(f'{name} must be finite') + if min(self.sphere_radius_m, self.antenna_height_m, self.coarse_range_search_m, self.refine_range_m, self.f_min) < 0: + raise ValueError('radius, height, frequencies and search ranges must be nonnegative') + if self.f_max <= self.f_min or min(self.gate_half_m, self.gate_taper_m) <= 0: + raise ValueError('calibration band and gate must have positive widths') + if self.phase_model not in ('const', 'linear', 'poly3', 'raw'): + raise ValueError('phase_model must be const, linear, poly3 or raw') + if len(set(self.refine_axes)) != len(self.refine_axes) or any(axis not in 'xyz' or len(axis) != 1 for axis in self.refine_axes): + raise ValueError('refine_axes must contain distinct x/y/z axes') + if not isinstance(self.apply_amplitude, bool): + raise ValueError('apply_amplitude must be boolean') + + +def find_config_profile(start: str | Path, max_levels: int = 8) -> Path: + """Locate the archived acquisition profile above a stage or snapshot folder.""" + path = Path(start) + for _ in range(max_levels): + candidate = path / 'config_profile.json' + if candidate.is_file(): + return candidate + path = path.parent + raise FileNotFoundError(f'config_profile.json not found above {start}') + + +def _profile(path): + config = json.loads(Path(path).read_text(encoding='utf-8')) + if not isinstance(config, dict): + raise ValueError('config_profile.json must contain an object') + return config + + +def _geometry(config, input_idx, output_idx): + result = [] + for key, identity, ports in (('tx_geometry', 'output_pos', output_idx), ('rx_geometry', 'input_pos', input_idx)): + entries = config.get('gpr', {}).get(key, []) + mapping = {} + for entry in entries: + port = _ports([entry[identity]], identity)[0] + if port in mapping: + raise ValueError(f'duplicate antenna geometry for {identity}={port}') + mapping[port] = _finite_array([entry['x_m'], entry.get('y_m', 0.), entry.get('z_m', 0.)], key, (3,)) + missing = set(ports) - mapping.keys() + if missing: + raise ValueError(f'{key} is missing physical ports {sorted(missing)}') + result.append(np.array([mapping[port] for port in ports])) + return tuple(result) + + +def load_geometry(config_path: str | Path, input_idx: Sequence[int], output_idx: Sequence[int]) -> tuple[np.ndarray, np.ndarray]: + """Read TX/RX radar coordinates in sorted physical port order.""" + return _geometry(_profile(config_path), sorted(_ports(input_idx, 'input_idx')), sorted(_ports(output_idx, 'output_idx'))) + + +def list_snapshots(root: str | Path, prefixes: Sequence[str] | None = None) -> list[Path]: + """Select complete snapshot directories, requiring unambiguous prefixes.""" + directories = sorted(path for path in Path(root).iterdir() if path.is_dir() and (path / 'meta.json').is_file()) + if prefixes is None: + selected = directories + else: + selected = [] + for prefix in prefixes: + matches = [path for path in directories if path.name.startswith(str(prefix))] + if len(matches) != 1 or matches[0] in selected: + raise ValueError(f'snapshot prefix {prefix!r} must select exactly one distinct snapshot in {root}') + selected.extend(matches) + if not selected: + raise ValueError(f'no snapshots selected in {root}') + return selected + + +def load_s21(snapshot_dir: str | Path, input_idx: Sequence[int], output_idx: Sequence[int]) -> tuple[np.ndarray, np.ndarray]: + """Read a finite TX×RX×frequency matrix on one shared uniform grid.""" + inputs = sorted(_ports(input_idx, 'input_idx')) + outputs = sorted(_ports(output_idx, 'output_idx')) + rows, freq = [], None + for output in outputs: + row = [] + for input_ in inputs: + suffix = f'i{input_}_o{output}' + samples = np.load(Path(snapshot_dir) / f'{suffix}_s21.npy', allow_pickle=False) + axis = _frequency(np.load(Path(snapshot_dir) / f'{suffix}_freq.npy', allow_pickle=False), uniform=True) + if freq is not None and not np.array_equal(axis, freq): + raise ValueError(f'frequency grids differ between pairs in {snapshot_dir}') + freq = axis + row.append(_finite_array(samples, suffix, freq.shape, complex_values=True)) + rows.append(row) + assert freq is not None + return np.array(rows), freq + + +def _source_signature(config) -> dict[str, Any]: + radar = config.get('radar', {}) + s21 = config.get('preprocess', {}).get('s21', {}) + return { + 'radar': {key: radar[key] for key in ('model', 'serial') if key in radar}, + 'slave_serials': radar.get('multi_device', {}).get('slave_serials', []), + 's21_calibration_set': s21.get('calibration', {}).get('set_name', ''), + 's21_reference_set': s21.get('reference', {}).get('set_name', ''), + 'notch': config.get('preprocess', {}).get('notch', {}), + } + + +def validate_calibration(cal: dict[str, Any]) -> None: + """Validate an archive before saving, applying or exporting runtime factors.""" + freq = _frequency(cal['freq_hz'], uniform=True) + tx = _ports(cal['tx_output_pos'], 'tx_output_pos') + rx = _ports(cal['rx_input_pos'], 'rx_input_pos') + if tx != tuple(sorted(tx)) or rx != tuple(sorted(rx)): + raise ValueError('calibration physical ports must be sorted') + T = _finite_array(cal['tx'], 'tx corrections', (len(tx), freq.size), complex_values=True) + R = _finite_array(cal['rx'], 'rx corrections', (len(rx), freq.size), complex_values=True) + pair = _finite_array(cal['pair'], 'pair corrections', (len(tx), len(rx), freq.size), complex_values=True) + if np.any(np.abs(T) == 0) or np.any(np.abs(R) == 0) or np.any(np.abs(pair) == 0): + raise ValueError('calibration corrections must be nonzero') + if not np.allclose(pair, T[:, None, :] * R[None, :, :], rtol=2e-5, atol=1e-7): + raise ValueError('pair corrections disagree with tx/rx corrections') + valid = np.asarray(cal['valid']) + if valid.dtype.kind != 'b' or valid.shape != freq.shape or valid.sum() < 2: + raise ValueError('calibration valid mask must select at least two frequencies') + _finite_array(cal['antenna_params'], 'antenna parameters', (len(tx) + len(rx), 3)) + geometry = cal['meta']['geometry'] + _finite_array(geometry['tx_xyz_m'], 'TX geometry', (len(tx), 3)) + _finite_array(geometry['rx_xyz_m'], 'RX geometry', (len(rx), 3)) + + +def validate_for_config(cal: dict[str, Any], config: dict[str, Any]) -> None: + """Require physical geometry, combo coverage and a compatible phase base.""" + validate_calibration(cal) + tx, rx = _geometry(config, cal['rx_input_pos'], cal['tx_output_pos']) + geometry = cal['meta']['geometry'] + if not (np.allclose(tx, geometry['tx_xyz_m'], rtol=0, atol=1e-9) and + np.allclose(rx, geometry['rx_xyz_m'], rtol=0, atol=1e-9)): + raise ValueError('antenna geometry differs from the object calibration') + for combo in config.get('run', {}).get('combos', []): + input_, output = _ports([combo['input']], 'combo input')[0], _ports([combo['output']], 'combo output')[0] + if input_ not in cal['rx_input_pos'] or output not in cal['tx_output_pos']: + raise ValueError(f'object calibration does not cover combo {combo}') + source = cal['meta'].get('source_preprocess', {}) + current = _source_signature(config) + for key in ('radar', 'slave_serials', 's21_calibration_set'): + if source.get(key) and current.get(key) and source[key] != current[key]: + raise ValueError(f'object calibration source differs: {key}') + + +@contextmanager +def _staged_directory(destination, overwrite, *, preserve_existing=False): + destination = Path(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists() and (not overwrite or not destination.is_dir() or destination.is_symlink()): + raise FileExistsError(f'{destination} already exists; choose another name or overwrite=True') + stage = Path(tempfile.mkdtemp(prefix=f'.{destination.name}.new-', dir=destination.parent)) + backup = None + retain = False + try: + if preserve_existing and destination.exists(): + shutil.copytree(destination, stage, dirs_exist_ok=True, symlinks=True) + yield stage + if destination.exists(): + if not overwrite: + raise FileExistsError(f'{destination} appeared during save; choose another name or overwrite=True') + backup = Path(tempfile.mkdtemp(prefix=f'.{destination.name}.old-', dir=destination.parent)) + backup.rmdir() + destination.replace(backup) + try: + stage.replace(destination) + except BaseException: + if backup is not None: + try: + backup.replace(destination) + except OSError as error: + retain = True + raise RuntimeError(f'calibration rollback failed; originals retained at {backup}, new files at {stage}') from error + raise + finally: + if not retain: + for temporary in (stage, backup): + if temporary is not None and temporary.exists(): + try: + shutil.rmtree(temporary) + except OSError: + logger.exception('Could not remove calibration temporary directory %s', temporary) + + +def save_calibration(cal: dict[str, Any], cal_dir: str | Path, overwrite: bool = False) -> Path: + """Commit a complete legacy archive, restoring its predecessor on failure.""" + validate_calibration(cal) + destination = Path(cal_dir) + if destination.is_dir() and any(destination.glob('cal_*')) and not overwrite: + raise FileExistsError(f'{destination} already contains calibration files; use overwrite=True') + meta: dict[str, Any] = dict(cal['meta'], name=destination.name) + # Older one-target archives used nonstandard JSON NaN for unavailable LOO. + quality = dict(meta.get('quality', {})) + if 'coherence_leave_one_out' in quality: + quality['coherence_leave_one_out'] = [value if value is None or np.isfinite(value) else None for value in quality['coherence_leave_one_out']] + meta['quality'] = quality + with _staged_directory(destination, overwrite or destination.is_dir(), preserve_existing=True) as stage: + for key in _CAL_ARRAYS: + path = stage / f'cal_{key}.npy' + path.unlink(missing_ok=True) + np.save(path, cal[key], allow_pickle=False) + (stage / 'cal_meta.json').unlink(missing_ok=True) + atomic_write_json(stage / 'cal_meta.json', meta) + cal['meta'], cal['dir'] = meta, destination + return destination + + +def load_calibration(cal_dir: str | Path) -> dict[str, Any]: + """Load and validate cal_*.npy arrays plus cal_meta.json without pickle.""" + directory = Path(cal_dir) + cal: dict[str, Any] = {key: np.load(directory / f'cal_{key}.npy', allow_pickle=False) for key in _CAL_ARRAYS} + cal.update(meta=json.loads((directory / 'cal_meta.json').read_text(encoding='utf-8')), dir=directory) + validate_calibration(cal) + return cal + + +def calibration_on_grid(cal: dict[str, Any], freq: ArrayLike) -> np.ndarray: + """Interpolate separable corrections and continue phase smoothly at edges.""" + validate_calibration(cal) + freq = _frequency(freq) + fc = np.asarray(cal['freq_hz']) + if np.array_equal(freq, fc): + return cal['pair'] + + def interp(correction, params): + phase = np.unwrap(np.angle(correction)) + out = np.interp(freq, fc, phase) + slope = 2 * np.pi * params[1] / C_LIGHT + for selected, edge in ((freq < fc[0], 0), (freq > fc[-1], -1)): + out[selected] = phase[edge] + slope * (freq[selected] - fc[edge]) + return np.interp(freq, fc, np.abs(correction)) * np.exp(1j * out) + + n_tx = len(cal['tx_output_pos']) + tx = np.array([interp(value, cal['antenna_params'][index]) for index, value in enumerate(cal['tx'])]) + rx = np.array([interp(value, cal['antenna_params'][n_tx + index]) for index, value in enumerate(cal['rx'])]) + corrections = (tx[:, None, :] * rx[None, :, :]).astype(np.asarray(cal['pair']).dtype) + _finite_array(corrections, 'interpolated corrections', complex_values=True) + return corrections + + +def apply_calibration(s21: ArrayLike, freq: ArrayLike, cal: dict[str, Any]) -> np.ndarray: + """Multiply a TX×RX×frequency S21 matrix by validated pair corrections.""" + corrections = calibration_on_grid(cal, freq) + samples = _finite_array(s21, 'S21', corrections.shape, complex_values=True) + return _finite_array(samples * corrections, 'corrected S21', corrections.shape, complex_values=True) + + +def apply_to_measurement(meas_dir: str | Path, cal: dict[str, Any], overwrite: bool = False) -> tuple[Path, int, int, int]: + """Stage corrected S21 and copied companion files in postprocessed safely.""" + directory = Path(meas_dir) + pre, post = directory / 'preprocessed', directory / 'postprocessed' + if not pre.is_dir(): + raise FileNotFoundError(f'no {pre}') + config = _profile(find_config_profile(pre)) + _reject_corrected_source(config) + validate_for_config(cal, config) + if post.exists() and overwrite and not (post / 'calibration_applied.json').is_file(): + raise FileExistsError(f'{post} has no calibration_applied.json; refusing to replace it') + tx = {int(port): index for index, port in enumerate(cal['tx_output_pos'])} + rx = {int(port): index for index, port in enumerate(cal['rx_input_pos'])} + pattern = re.compile(r'^i(\d+)_o(\d+)_s21\.npy$') + n_snap = n_fix = n_copy = 0 + cache = {} + with _staged_directory(post, overwrite) as stage: + for item in sorted(pre.iterdir()): + if item.is_file(): + shutil.copy2(item, stage / item.name) + n_copy += 1 + continue + out = stage / item.name + out.mkdir() + n_snap += 1 + for path in sorted(item.iterdir()): + match = pattern.match(path.name) + if path.is_dir() or match is None: + if path.is_dir(): + shutil.copytree(path, out / path.name) + else: + shutil.copy2(path, out / path.name) + n_copy += 1 + continue + input_, output = int(match[1]), int(match[2]) + if input_ not in rx or output not in tx: + raise ValueError(f'object calibration does not cover {path.name}') + freq = _frequency(np.load(item / f'i{input_}_o{output}_freq.npy', allow_pickle=False)) + key = freq.tobytes() + if key not in cache: + cache[key] = calibration_on_grid(cal, freq) + samples = np.load(path, allow_pickle=False) + dtype = samples.dtype + samples = _finite_array(samples, path.name, freq.shape, complex_values=True) + corrected = (samples * cache[key][tx[output], rx[input_]]).astype(dtype) + _finite_array(corrected, 'corrected S21', complex_values=True) + np.save(out / path.name, corrected, allow_pickle=False) + n_fix += 1 + atomic_write_json(stage / 'calibration_applied.json', dict( + calibration=cal['meta'].get('name'), calibration_dir=str(cal.get('dir') or ''), + created=datetime.datetime.now().isoformat(timespec='seconds'), source=str(pre), + rule='postprocessed S21 = preprocessed S21 * cal_pair[tx(output_pos), rx(input_pos)]', + frequency_grids=[np.frombuffer(key, dtype=np.float64).tolist() for key in cache])) + return post, n_snap, n_fix, n_copy + + +def _reject_corrected_source(config): + object_calibration = config.get('preprocess', {}).get('s21', {}).get('object_calibration', {}) + if object_calibration.get('set_name') or object_calibration.get('bundle_path'): + raise ValueError('calibration sources must have no object correction; object_calibration is already active') + + +def _best_phases(Z): + """Maximize pair _coherence by alternating antenna phases, anchored at TX0/RX0.""" + t = np.zeros(Z.shape[0]) + r = np.zeros(Z.shape[1]) + for _ in range(100): + t = np.angle((Z * np.exp(-1j * r)[None, :]).sum(axis=1)) + r = np.angle((Z * np.exp(-1j * t)[:, None]).sum(axis=0)) + t = t - t[0] + r = r - r[0] + return t, r + + +def _coherence(Z, t=None, r=None): + """Return |sum(pair contributions)| / sum(|pair contributions|).""" + if t is not None: + if r is None: + raise ValueError('RX phases must accompany TX phases') + Z = Z * np.exp(-1j * (t[:, None] + r[None, :])) + return float(np.abs(Z.sum()) / (np.abs(Z).sum() + 1e-30)) + + +def _circular_median(phases): + """Choose the phase minimizing circular distances along the first axis.""" + d = np.abs(np.angle(np.exp(1j * (phases[:, None] - phases[None, :])))) + best = np.argmin(d.sum(axis=1), axis=0) + return np.take_along_axis(phases, best[None], axis=0)[0] + + +def _combine(sets): + """Combine targets using circular phase and logarithmic gain medians.""" + out = [] + for key in ('T', 'R'): + Z = np.array([s[key] for s in sets]) # (targets, antennas, frequencies) + out.append(np.exp(np.median(np.log(np.abs(Z)), axis=0)) * np.exp(1j * _circular_median(np.angle(Z)))) + return tuple(out) + + +class _Estimator: + """Geometry, gated target echoes and the original alternating estimator.""" + + def __init__(self, st, tx, rx, freq, check=lambda: None): + self.st, self.tx, self.rx, self.freq = st, tx, rx, freq + self.check = check + self.n_tx, self.n_rx = len(tx), len(rx) + self.band = (freq >= st.f_min) & (freq <= st.f_max) + self.fb = freq[self.band] + if self.fb.size < 4: + raise ValueError('calibration band must contain at least four frequency samples') + self.win = np.hanning(self.fb.size) + self.ok = self.win > 0.2 # Avoid unreliable window division at band edges. + if self.ok.sum() < (4 if st.phase_model == 'poly3' else 2): + raise ValueError('too few reliable frequencies for the requested phase model') + self.nfft = 1 << int(np.ceil(np.log2(self.fb.size * 8))) + self.dpath = np.fft.fftfreq(self.nfft, self.fb[1] - self.fb[0]) * C_LIGHT # Excess propagation path in metres. + self.gate = np.clip(np.minimum((self.dpath + st.gate_half_m) / st.gate_taper_m, + (st.gate_half_m - self.dpath) / st.gate_taper_m) + 0.5, 0, 1) + + def scene_to_radar(self, q): + """Convert ground-relative scene coordinates into radar coordinates.""" + p = np.radians(self.st.pitch_deg) + y0 = q[1] - self.st.antenna_height_m + return np.array([q[0], y0 * np.cos(p) + q[2] * np.sin(p), -y0 * np.sin(p) + q[2] * np.cos(p)]) + + def pair_paths(self, q_scene): + """Return bistatic paths to a point, or sphere surface paths shortened by + 2 * radius * cos(beta/2), using the measured sphere center.""" + a = self.st.sphere_radius_m + pr = self.scene_to_radar(np.asarray(q_scene, float)) + vt = self.tx - pr + vr = self.rx - pr + dt = np.linalg.norm(vt, axis=1) + dr = np.linalg.norm(vr, axis=1) + if np.any(dt <= a) or np.any(dr <= a): + raise ValueError('reflector intersects an antenna or has zero range') + paths = dt[:, None] + dr[None, :] + if a: + cos_beta = (vt @ vr.T) / (dt[:, None] * dr[None, :]) + paths = paths - 2 * a * np.sqrt(np.clip((1 + cos_beta) / 2, 0, 1)) + return paths + + def _unwound(self, s21_band, paths): + return s21_band * np.exp(2j * np.pi * self.fb[None, None, :] * paths[:, :, None] / C_LIGHT) + + def target_echo(self, s21_band, paths): + """Isolate the unwound target echo with the tapered excess-path gate.""" + h = np.fft.ifft(self._unwound(s21_band, paths) * self.win, self.nfft, axis=2) + return np.fft.fft(h * self.gate, axis=2)[:, :, :self.fb.size] / np.maximum(self.win, 1e-3) + + def broadband(self, E): + """Sum reliable-band echo contributions at the target for each pair.""" + return E[:, :, self.ok].sum(axis=2) + + def coarse_range(self, s21, q_nominal): + """Move nominal Z toward the strongest envelope within the path search range.""" + q = q_nominal.copy() + if not self.st.coarse_range_search_m: + return q + s = s21[:, :, self.band] + sel = np.abs(self.dpath) <= self.st.coarse_range_search_m + for _ in range(3): + self.check() + env = np.abs(np.fft.ifft(self._unwound(s, self.pair_paths(q)) * self.win, self.nfft, axis=2)).sum(axis=(0, 1)) + q[2] += self.dpath[sel][np.argmax(env[sel])] / 2 + return q + + def refine_target(self, s21, q0): + """Refine target coordinates and antenna phases by alternating Nelder-Mead fits.""" + P0 = self.pair_paths(q0) + E0 = self.target_echo(s21[:, :, self.band], P0) + axes = [('xyz').index(a) for a in self.st.refine_axes] + + def E_at(v): + q = q0.copy() + q[axes] += v + dP = self.pair_paths(q) - P0 + return E0 * np.exp(2j * np.pi * self.fb[None, None, :] * dP[:, :, None] / C_LIGHT), q + + def score(v, t, r): + self.check() + if np.any(np.abs(v) > self.st.refine_range_m): + return 0.0 + return _coherence(self.broadband(E_at(v)[0]), t, r) + + v = np.zeros(len(axes)) + if axes: + from scipy.optimize import minimize + + for _ in range(6): + self.check() + t, r = _best_phases(self.broadband(E_at(v)[0])) + simplex = np.vstack([v] + [v + 0.03 * np.eye(len(axes))[k] for k in range(len(axes))]) + v = minimize(lambda w: -score(w, t, r), v, method='Nelder-Mead', + options=dict(xatol=5e-4, fatol=1e-7, initial_simplex=simplex)).x + return E_at(v) + + def factorize(self, E): + """Factor E(tx,rx,f) into antenna phases and least-squares log-amplitudes.""" + nb = E.shape[2] + T = np.ones((self.n_tx, nb), complex) + R = np.ones((self.n_rx, nb), complex) + L = np.log(np.abs(E) + 1e-30) + for k in range(nb): + self.check() + t, r = _best_phases(E[:, :, k]) + tt = L[:, :, k].mean(axis=1) - L[:, :, k].mean() + rr = L[:, :, k].mean(axis=0) - L[:, :, k].mean() + T[:, k] = np.exp(tt + 1j * t) + R[:, k] = np.exp(rr + 1j * r) + return T, R + + def antenna_params(self, Z): + """Fit zero-frequency phase, excess delay path and gain for one antenna.""" + ph = np.unwrap(np.angle(Z[self.ok])) + f = self.fb[self.ok] + slope, ph0 = np.polyfit(f, ph, 1) + return dict(phase0_deg=float(np.degrees(ph0)), delay_path_m=float(-slope * C_LIGHT / (2 * np.pi)), + gain_db=float(20 * np.log10(np.mean(np.abs(Z[self.ok])))), + phase_rms_after_linear_deg=float(np.degrees(np.std(ph - (slope * f + ph0))))) + + def phase_model(self, Z): + """Fit the selected antenna phase model over reliable band samples.""" + model = self.st.phase_model + if model == 'raw': + return Z + ph = np.unwrap(np.angle(Z)) + f = (self.fb - self.fb.mean()) / 1e9 + if model == 'const': + u = np.exp(1j * np.angle(Z[self.ok])).mean() + return np.abs(Z) * np.exp(1j * np.angle(u)) + deg = {'linear': 1, 'poly3': 3}[model] + return np.abs(Z) * np.exp(1j * np.polyval(np.polyfit(f[self.ok], ph[self.ok], deg), f)) + + def pair_correction(self, T, R): + """Invert antenna factors, retaining measured amplitudes in phase-only mode.""" + Z = T[:, None, :] * R[None, :, :] + return np.conj(Z) / np.abs(Z) if not self.st.apply_amplitude else 1.0 / Z + + def to_full_grid(self, Z_band, params): + """Extend antenna factors over the full grid with continuous edge-anchored + linear phase and constant edge gain outside the reliable band.""" + idx = np.flatnonzero(self.band)[self.ok] + ph_in = np.unwrap(np.angle(Z_band[self.ok])) + mag_in = np.abs(Z_band[self.ok]) + slope = -2 * np.pi * params['delay_path_m'] / C_LIGHT # Radians per Hz. + ph = np.empty(self.freq.size) + mag = np.empty(self.freq.size) + ph[idx] = ph_in + mag[idx] = mag_in + lo, hi = idx[0], idx[-1] + ph[:lo] = ph_in[0] + slope * (self.freq[:lo] - self.freq[lo]) + mag[:lo] = mag_in[0] + ph[hi + 1:] = ph_in[-1] + slope * (self.freq[hi + 1:] - self.freq[hi]) + mag[hi + 1:] = mag_in[-1] + return mag * np.exp(1j * ph) + + +def _callbacks(progress, cancelled): + def check(message=None): + if cancelled is not None and cancelled(): + raise InterruptedError('object calibration cancelled') + if message is not None and progress is not None: + progress(message) + return check + + +def _mean_samples(samples, check, expected_freq=None): + mean, count, freq = None, 0, expected_freq + for data, axis in samples: + check() + axis = _frequency(axis, uniform=True) + if freq is not None and not np.array_equal(axis, freq): + raise ValueError('frequency grids differ between calibration measurements') + freq = axis + if mean is None: + mean = np.zeros_like(data, dtype=np.complex128) + data = _finite_array(data, 'calibration measurement', mean.shape, complex_values=True) + count += 1 + mean += (data - mean) / count + if mean is None: + raise ValueError('calibration measurements must contain at least one collection') + return mean, freq + + +def _settings(settings): + settings = settings or Settings() + settings.__post_init__() + return settings + + +def _compute(sets, empty, freq, settings, config, config_path, check) -> dict[str, Any]: + tx, rx = _geometry(config, settings.input_idx, settings.output_idx) + est = _Estimator(settings, tx, rx, freq, check) + for index, record in enumerate(sets): + check(f'Estimating target {index + 1}/{len(sets)}') + data = record.pop('s21') + if empty is not None: + data = data - empty + nominal = _finite_array(record['target_scene_m'], 'target_scene_m', (3,)) + echo, refined = est.refine_target(data, est.coarse_range(data, nominal)) + if not np.all(np.any(np.abs(echo[:, :, est.ok]) > 0, axis=2)): + raise ValueError(f'target {index + 1} has no usable echo for one or more antenna pairs') + _finite_array(echo, 'target echo', complex_values=True) + record.update(E=echo, q=refined, q_nominal=nominal) + record['T'], record['R'] = est.factorize(echo) + + T_all, R_all = _combine(sets) + T_all = np.array([est.phase_model(value) for value in T_all]) + R_all = np.array([est.phase_model(value) for value in R_all]) + antennas = {f'Tx{index}': est.antenna_params(value) for index, value in enumerate(T_all)} + antennas.update({f'Rx{index}': est.antenna_params(value) for index, value in enumerate(R_all)}) + T_full = np.array([est.to_full_grid(value, antennas[f'Tx{index}']) for index, value in enumerate(T_all)]) + R_full = np.array([est.to_full_grid(value, antennas[f'Rx{index}']) for index, value in enumerate(R_all)]) + unit = (lambda value: np.conj(value) / np.abs(value)) if not settings.apply_amplitude else (lambda value: 1 / value) + saved_pair = unit(T_full[:, None, :] * R_full[None, :, :]).astype(np.complex64) + quality = dict(coherence_before=[], coherence_own=[], coherence_leave_one_out=[], coherence_final=[]) + diagnostics = [] + final_correction = saved_pair[:, :, est.band] + for record in sets: + check() + before = est.broadband(record['E']) + after = est.broadband(record['E'] * final_correction) + quality['coherence_before'].append(_coherence(before)) + quality['coherence_own'].append(_coherence(est.broadband(record['E'] * est.pair_correction(record['T'], record['R'])))) + others = [other for other in sets if other is not record] + quality['coherence_leave_one_out'].append( + _coherence(est.broadband(record['E'] * est.pair_correction(*_combine(others)))) if others else None) + quality['coherence_final'].append(_coherence(after)) + diagnostics.append(dict(phase_before_deg=np.degrees(np.angle(before)).tolist(), + phase_after_deg=np.degrees(np.angle(after)).tolist(), + coherence_before=_coherence(before), coherence_after=_coherence(after))) + + valid = np.zeros(freq.size, bool) + valid[np.flatnonzero(est.band)[est.ok]] = True + meta = dict( + created=datetime.datetime.now().isoformat(timespec='seconds'), name=None, + convention=dict(apply='S21_corrected[tx, rx, f] = S21[tx, rx, f] * cal_pair[tx, rx, f]', + index_order='sorted physical tx_output_pos x rx_input_pos', + phase_reference='relative to Tx0 and Rx0; common phase/delay is retained', + amplitude='equalized' if settings.apply_amplitude else 'phase only', + outside_band='linear antenna phase continuation anchored at the band edge'), + geometry=dict(config=str(config_path), tx_xyz_m=tx.tolist(), rx_xyz_m=rx.tolist(), + antenna_height_m=settings.antenna_height_m, pitch_deg=settings.pitch_deg), + estimation=dict(band_hz=[settings.f_min, settings.f_max], gate_half_m=settings.gate_half_m, + gate_taper_m=settings.gate_taper_m, coarse_range_search_m=settings.coarse_range_search_m, + refine_axes=list(settings.refine_axes), refine_range_m=settings.refine_range_m, + phase_model=settings.phase_model, sphere_radius_m=settings.sphere_radius_m), + source_preprocess=_source_signature(config), + targets=[dict(path=record.get('path', ''), snapshots=record.get('snapshots', []), + nominal_scene_m=record['q_nominal'].tolist(), refined_scene_m=record['q'].tolist()) for record in sets], + antennas=antennas, quality=quality, diagnostics=diagnostics) + cal = dict(freq_hz=freq.astype(np.float64), pair=saved_pair, + tx=unit(T_full).astype(np.complex64), rx=unit(R_full).astype(np.complex64), valid=valid, + tx_output_pos=np.array(settings.output_idx, np.int32), rx_input_pos=np.array(settings.input_idx, np.int32), + antenna_params=np.array([[np.radians(value['phase0_deg']), value['delay_path_m'], value['gain_db']] + for value in antennas.values()]), meta=meta, dir=None) + validate_calibration(cal) + check('Object calibration complete') + return cal + + +def _collection_samples(collection, settings): + from python_app.storage.npz.serialize import validate_trace_collection + + validate_trace_collection(collection, require_finite=True) + traces = {(trace.combo.input, trace.combo.output): trace for trace in collection.traces} + rows, freq = [], None + for output in settings.output_idx: + row = [] + for input_ in settings.input_idx: + if (input_, output) not in traces: + raise ValueError(f'calibration collection is missing input={input_}, output={output}') + trace = traces[input_, output] + axis = _frequency(trace.frequency_hz, uniform=True) + if freq is not None and not np.array_equal(axis, freq): + raise ValueError('frequency grids differ between collection pairs') + freq = axis + row.append(_finite_array(trace.s21, 'S21', freq.shape, complex_values=True)) + rows.append(row) + return np.array(rows), freq + + +def compute_from_collections( + cal_sets: Iterable[dict[str, Any]], config_dict: dict[str, Any], + empty_collections: Iterable[SweepCollection] | None = None, settings: Settings | None = None, + *, progress: Callable[[str], None] | None = None, cancelled: Callable[[], bool] | None = None, +) -> dict[str, Any]: + """Estimate corrections from standard-preprocessed sweeps without object correction. + + cal_sets contains {collections: [SweepCollection, ...], target_scene_m: XYZ}. + progress accepts a status string; cancelled returns bool and raises + InterruptedError promptly in data, refinement and factorization loops. + """ + settings = _settings(settings) + _reject_corrected_source(config_dict) + check = _callbacks(progress, cancelled) + sets = list(cal_sets) + if not sets: + raise ValueError('at least one reflector target is required') + empty, freq = None, None + if empty_collections is not None: + check('Averaging empty scene') + empty, freq = _mean_samples((_collection_samples(value, settings) for value in empty_collections), check) + prepared = [] + for index, record in enumerate(sets): + check(f'Averaging target {index + 1}/{len(sets)}') + collections = list(record['collections']) + mean, freq = _mean_samples((_collection_samples(value, settings) for value in collections), check, freq) + prepared.append(dict(s21=mean, target_scene_m=record['target_scene_m'], + snapshots=[str(value.collection_id) for value in collections])) + return _compute(prepared, empty, freq, settings, config_dict, 'live capture', check) + + +def compute_calibration( + cal_sets: Iterable[dict[str, Any]], empty_scene_path: str | Path | None = None, settings: Settings | None = None, + *, progress: Callable[[str], None] | None = None, cancelled: Callable[[], bool] | None = None, +) -> dict[str, Any]: + """Legacy archive API backed by the same estimator as live collection capture.""" + settings = _settings(settings) + check = _callbacks(progress, cancelled) + cal_sets = list(cal_sets) + if not cal_sets: + raise ValueError('at least one reflector target is required') + config_path = find_config_profile(cal_sets[0]['path']) + config = _profile(config_path) + _reject_corrected_source(config) + geometry = _geometry(config, settings.input_idx, settings.output_idx) + signature = _source_signature(config) + + def samples(directories): + for directory in directories: + check() + profile_path = find_config_profile(directory) + path = Path(directory) + while True: + if (path / 'calibration_applied.json').is_file(): + raise ValueError(f'object calibration is already applied to archive {directory}') + if path == profile_path.parent: + break + path = path.parent + source = _profile(profile_path) + _reject_corrected_source(source) + current = _geometry(source, settings.input_idx, settings.output_idx) + if any(not np.allclose(first, second, rtol=0, atol=1e-9) for first, second in zip(geometry, current)): + raise ValueError(f'antenna geometry differs in {directory}') + if signature != _source_signature(source): + raise ValueError(f'preprocessing or hardware configuration differs in {directory}') + yield load_s21(directory, settings.input_idx, settings.output_idx) + + empty, freq = None, None + if empty_scene_path is not None: + empty, freq = _mean_samples(samples(list_snapshots(empty_scene_path)), check) + prepared = [] + for record in cal_sets: + directories = list_snapshots(record['path'], record.get('snapshots')) + mean, freq = _mean_samples(samples(directories), check, freq) + prepared.append(dict(s21=mean, target_scene_m=record['target_scene_m'], path=str(record['path']), + snapshots=[directory.name for directory in directories])) + cal = _compute(prepared, empty, freq, settings, config, config_path, check) + cal['meta']['estimation']['empty_scene'] = str(empty_scene_path) if empty_scene_path is not None else None + return cal diff --git a/requirements.txt b/requirements.txt index 71fc52a..31f08a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ numpy>=1.26,<3 +scipy>=1.11,<2 libusb1>=3.1 pyserial>=3.5,<4 # PyQt6 6.8+ ships only manylinux_2_39 aarch64 wheels (glibc >= 2.39); Jetson /