Add object calibration

data_acq_and_processing/common_cpp/config/include/run_config.hpp

@@ -81,6 +81,7 @@ struct PreprocessAssetConfig {
 struct S21PreprocessConfig {
     PreprocessAssetConfig calibration{};
     PreprocessAssetConfig reference{};
+    PreprocessAssetConfig object_calibration{};
 };
 
 struct S11CalibrationPreprocessConfig {

data_acq_and_processing/common_cpp/config/src/run_config.cpp

@@ -555,6 +555,10 @@ auto load_run_config(const std::string& path) -> RunConfig {
         const auto* s21_obj = as_object(required_field(*preprocess_obj, "s21"), "preprocess.s21");
         config.preprocess.s21.calibration = parse_preprocess_asset(*s21_obj, "calibration", "preprocess.s21");
         config.preprocess.s21.reference = parse_preprocess_asset(*s21_obj, "reference", "preprocess.s21");
+        if (optional_field(*s21_obj, "object_calibration") != nullptr) {
+            config.preprocess.s21.object_calibration =
+                parse_preprocess_asset(*s21_obj, "object_calibration", "preprocess.s21");
+        }
 
         const auto* s11_obj = as_object(required_field(*preprocess_obj, "s11"), "preprocess.s11");
         const auto* s11_calibration_obj =

data_acq_and_processing/preprocessing/calibration_master/include/channel_bundle_support.hpp

@@ -50,6 +50,18 @@ class S21ReferenceBundle {
     std::unordered_map<ipc::ComboKey, ChannelTrace, ipc::ComboKeyHash> traces_by_combo_{};
 };
 
+// Object-derived MIMO correction, exported on each pair's runtime frequency grid.
+// An empty path disables it; enabled bundles must cover every active pair.
+class S21ObjectCalibrationBundle {
+  public:
+    void load(const std::string& path);
+    void validate_combos(const std::vector<ipc::ComboKey>& combos) const;
+    void apply(ipc::SweepTraceBlock& trace) const;
+
+  private:
+    std::unordered_map<ipc::ComboKey, ChannelTrace, ipc::ComboKeyHash> factors_by_combo_{};
+};
+
 class S11CalibrationBundle {
   public:
     struct Coefficients {

data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp

@@ -8,6 +8,7 @@
 #include <fstream>
 #include <iostream>
 #include <iterator>
+#include <limits>
 #include <span>
 #include <stdexcept>
 #include <string>
@@ -197,11 +198,11 @@ template <typename TValue>
 auto resolve_required(
     const std::unordered_map<ipc::ComboKey, TValue, ipc::ComboKeyHash>& values_by_combo,
     const ipc::ComboKey& combo,
-    const std::string& value_label
+    const char* value_label
 ) -> const TValue& {
     const auto found = values_by_combo.find(combo);
     if (found == values_by_combo.end()) {
-        throw std::runtime_error(value_label + " is missing for combo " + combo_to_string(combo));
+        throw std::runtime_error(std::string(value_label) + " is missing for combo " + combo_to_string(combo));
     }
     return found->second;
 }
@@ -345,6 +346,38 @@ auto S21ReferenceBundle::resolve(const ipc::ComboKey& combo) const -> const Chan
     return resolve_required(traces_by_combo_, combo, "S21 reference trace");
 }
 
+void S21ObjectCalibrationBundle::load(const std::string& path) {
+    if (path.empty()) {
+        factors_by_combo_.clear();
+        return;
+    }
+    factors_by_combo_ = load_channel_traces(path, "S21 object calibration", RawTraceChannel::S21);
+}
+
+void S21ObjectCalibrationBundle::validate_combos(const std::vector<ipc::ComboKey>& combos) const {
+    validate_bundle_combos(factors_by_combo_, combos, "S21 object calibration bundle");
+}
+
+void S21ObjectCalibrationBundle::apply(ipc::SweepTraceBlock& trace) const {
+    if (factors_by_combo_.empty()) { return; }
+    const auto& factors = resolve_required(factors_by_combo_, trace.combo, "S21 object calibration");
+    ensure_frequency_axis_match(factors.frequency_hz, trace.frequency_hz, "S21 object calibration");
+    if (trace.s21.size() != factors.samples.size()) {
+        throw std::runtime_error("S21 object calibration vector size mismatch");
+    }
+    for (std::size_t index = 0; index < trace.s21.size(); ++index) {
+        const auto value = trace.s21[index];
+        const auto factor = factors.samples[index];
+        const auto corrected = std::complex<double>(value.re, value.im) * std::complex<double>(factor.re, factor.im);
+        if (!std::isfinite(corrected.real()) || !std::isfinite(corrected.imag()) ||
+            std::abs(corrected.real()) > std::numeric_limits<float>::max() ||
+            std::abs(corrected.imag()) > std::numeric_limits<float>::max()) {
+            throw std::runtime_error("S21 object calibration result must fit finite float32");
+        }
+        trace.s21[index] = {.re = static_cast<float>(corrected.real()), .im = static_cast<float>(corrected.imag())};
+    }
+}
+
 void S11CalibrationBundle::load(
     const std::string& open_path,
     const std::string& short_path,

data_acq_and_processing/preprocessing/data_preprocessor/include/data_preprocessor.hpp

@@ -39,6 +39,7 @@ class DataPreprocessor {
     const config::RunConfig& config_;
     CalibrationMaster& calibration_master_;
     ReferenceMaster& reference_master_;
+    S21ObjectCalibrationBundle object_calibration_;
     ipc::ShmRing& raw_ring_;
     ipc::ShmRing& preprocessed_ring_;
     ipc::ShmRing* preprocessed_tap_ring_ = nullptr;

data_acq_and_processing/preprocessing/data_preprocessor/src/data_preprocessor.cpp

@@ -8,6 +8,7 @@
 #include <span>
 #include <stdexcept>
 #include <string_view>
+#include <utility>
 #include <vector>
 
 namespace radar::preprocessing {
@@ -104,11 +105,14 @@ DataPreprocessor::DataPreprocessor(
       reference_master_(reference_master),
       raw_ring_(raw_ring),
       preprocessed_ring_(preprocessed_ring),
-      preprocessed_tap_ring_(preprocessed_tap_ring) {}
+      preprocessed_tap_ring_(preprocessed_tap_ring) {
+    object_calibration_.load(config_.preprocess.s21.object_calibration.bundle_path);
+}
 
 void DataPreprocessor::validate_startup_prerequisites() const {
     calibration_master_.validate_combos(config_.run_combos);
     reference_master_.validate_combos(config_.run_combos);
+    object_calibration_.validate_combos(config_.run_combos);
 }
 
 auto DataPreprocessor::try_pop_raw_collection(
@@ -188,10 +192,12 @@ auto DataPreprocessor::preprocess_collection(const ipc::RawSweepCollection& raw_
     preprocessed.traces.reserve(raw_collection.traces.size());
 
     for (const auto& raw_trace : raw_collection.traces) {
-        // Pipeline order is fixed: calibration first, then reference subtraction.
+        // Object-derived corrections act on the completed preprocessing output.
         const auto calibrated = calibration_master_.apply_to_trace(raw_trace);
         const auto referenced = reference_master_.apply_to_trace(calibrated);
-        preprocessed.traces.push_back(apply_notch_filter(referenced, config_.preprocess.notch));
+        auto filtered = apply_notch_filter(referenced, config_.preprocess.notch);
+        object_calibration_.apply(filtered);
+        preprocessed.traces.push_back(std::move(filtered));
     }
 
     return preprocessed;

data_acq_and_processing/tests/native_regressions.cpp

@@ -1,5 +1,6 @@
 #include "capture_file_writer.h"
 #include "channel_bundle_support.hpp"
+#include "data_preprocessor.hpp"
 #include "locator/tcp_server.hpp"
 #include "processing_live_config.hpp"
 #include "runtime_wait.hpp"
@@ -16,6 +17,7 @@
 #include <atomic>
 #include <chrono>
 #include <cstring>
+#include <exception>
 #include <filesystem>
 #include <fstream>
 #include <iostream>
@@ -60,11 +62,23 @@ struct Workspace {
 };
 
 struct RingName {
-    std::string name = "/radar_native_test_" + std::to_string(::getpid());
-    RingName() { ipc::ShmRing::unlink_ring(name); }
+    std::string name;
+    explicit RingName(const std::string& suffix = {})
+        : name("/radar_native_test_" + std::to_string(::getpid()) + suffix) {
+        ipc::ShmRing::unlink_ring(name);
+    }
     ~RingName() { ipc::ShmRing::unlink_ring(name); }
 };
 
+auto save_bundle(const Workspace& workspace, const char* name, const ipc::RawSweepCollection& collection)
+    -> std::string {
+    const auto path = workspace.path / name;
+    const auto bytes = ipc::serialize_raw_collection(collection);
+    std::ofstream(path, std::ios::binary).write(
+        reinterpret_cast<const char*>(bytes.data()), static_cast<std::streamsize>(bytes.size()));
+    return path.string();
+}
+
 void write_u32(std::vector<std::uint8_t>& bytes, std::size_t offset, std::uint32_t value) {
     require(offset + sizeof(value) <= bytes.size(), "invalid test offset");
     std::memcpy(bytes.data() + offset, &value, sizeof(value));
@@ -137,6 +151,22 @@ void test_configuration(const std::string& seed_path, const Workspace& workspace
     const auto save = [&](const Json& config) { std::ofstream(config_path) << config; };
     save(seed);
     const auto config = config::load_run_config(config_path.string());
+    auto with_object_calibration = seed;
+    with_object_calibration["preprocess"]["s21"]["object_calibration"] =
+        Json{{"set_name", "reflector"}, {"bundle_path", "corrections.bin"}};
+    save(with_object_calibration);
+    const auto selected = config::load_run_config(config_path.string()).preprocess.s21.object_calibration;
+    require(selected.set_name == "reflector" && selected.bundle_path == "corrections.bin",
+        "object calibration asset did not load");
+    with_object_calibration["preprocess"]["s21"].erase("object_calibration");
+    save(with_object_calibration);
+    require(config::load_run_config(config_path.string()).preprocess.s21.object_calibration.bundle_path.empty(),
+        "absent object calibration was not disabled");
+    for (const auto& invalid_asset : {Json(nullptr), Json{{"bundle_path", true}}}) {
+        with_object_calibration["preprocess"]["s21"]["object_calibration"] = invalid_asset;
+        save(with_object_calibration);
+        expect_error([&] { (void)config::load_run_config(config_path.string()); });
+    }
     for (const auto& [pointer, value] : std::array<std::pair<const char*, double>, 8>{{
         {"/radar/sweep/start_hz", -1}, {"/radar/sweep/stop_hz", 1e100},
         {"/radar/sweep/if_bandwidth_hz", 0}, {"/radar/sweep/points", 0},
@@ -297,12 +327,8 @@ void test_csv_failures(const Workspace& workspace) {
 
 void test_asset_reload_preserves_data(const Workspace& workspace) {
     const auto save = [&](const char* name, float sample) {
-        const auto path = workspace.path / name;
-        const auto bytes = ipc::serialize_raw_collection({.traces = {
+        return save_bundle(workspace, name, {.traces = {
             {.frequency_hz = {1}, .s11 = {{sample, 0}}, .s21 = {{sample, 0}}}}});
-        std::ofstream(path, std::ios::binary).write(
-            reinterpret_cast<const char*>(bytes.data()), static_cast<std::streamsize>(bytes.size()));
-        return path.string();
     };
     const auto open = save("open.bin", 1);
     const auto short_standard = save("short.bin", -1);
@@ -328,6 +354,105 @@ void test_asset_reload_preserves_data(const Workspace& workspace) {
         "failed OSL reload lost previous calibration");
 }
 
+void test_object_calibration(const Workspace& workspace) {
+    const ipc::SweepTraceBlock first{.combo = {7, 3}, .frequency_hz = {1e9F, 2e9F},
+        .s11 = {{1, 2}, {3, 4}}, .s21 = {{2, 3}, {4, 5}}, .capture_start_ns = 5, .capture_end_ns = 15};
+    const ipc::SweepTraceBlock second{.combo = {1, 9}, .frequency_hz = {3e9F, 4e9F, 5e9F},
+        .s11 = {{}, {}, {}}, .s21 = {{6, 7}, {8, 9}, {10, 11}}};
+    auto factors_first = first;
+    factors_first.s21 = {{0, 2}, {2, 0}};
+    auto factors_second = second;
+    factors_second.s21 = {{1, -1}, {-1, 0}, {0, 1}};
+    preprocessing::S21ObjectCalibrationBundle correction;
+    auto trace = first;
+    correction.apply(trace);
+    require(ipc::serialize_raw_collection({.traces = {trace}}) == ipc::serialize_raw_collection({.traces = {first}}),
+        "disabled object calibration changed a trace");
+    correction.load(save_bundle(workspace, "object.bin", {.traces = {factors_second, factors_first}}));
+    correction.validate_combos({first.combo, second.combo});
+    expect_error([&] { correction.validate_combos({{0, 0}}); });
+    for (auto invalid : {first, second}) {
+        invalid.combo = {0, 0};
+        expect_error([&] { correction.apply(invalid); });
+    }
+    auto mismatch = second;
+    mismatch.frequency_hz[1] += 1e8F;  // Same endpoints and count; interior grid differs.
+    expect_error([&] { correction.apply(mismatch); });
+    mismatch = second;
+    mismatch.s21.pop_back();
+    expect_error([&] { correction.apply(mismatch); });
+    auto nonfinite = factors_first;
+    nonfinite.s21[0].re = std::numeric_limits<float>::quiet_NaN();
+    expect_error([&] { correction.load(save_bundle(workspace, "object_nan.bin", {.traces = {nonfinite}})); });
+    expect_error([&] { correction.load(save_bundle(workspace, "object_duplicate.bin", {.traces = {factors_first, factors_first}})); });
+    auto expected = first;
+    expected.s21 = {{-6, 4}, {8, 10}};
+    trace = first;
+    correction.apply(trace);
+    require(ipc::serialize_raw_collection({.traces = {trace}}) == ipc::serialize_raw_collection({.traces = {expected}}),
+        "object calibration changed S11, timing, pair identity, or multiplication after a failed reload");
+    trace = second;
+    correction.apply(trace);
+    require(trace.s21[0].re == 13 && trace.s21[0].im == 1 && trace.s21[1].re == -8 && trace.s21[1].im == -9 &&
+            trace.s21[2].re == -11 && trace.s21[2].im == 10, "object calibration mixed pairs or frequency grids");
+    trace = first;
+    trace.s21[0] = {std::numeric_limits<float>::max(), std::numeric_limits<float>::max()};
+    expect_error([&] { correction.apply(trace); });
+    correction.load("");
+    trace = first;
+    correction.apply(trace);
+    require(trace.s21[0].re == 2 && trace.s21[0].im == 3, "clearing object calibration left stale factors");
+}
+
+void test_object_calibration_pipeline(const Workspace& workspace) {
+    const ipc::SweepTraceBlock trace{.combo = {7, 3}, .frequency_hz = {1e9F, 2e9F},
+        .s11 = {{7, 3}, {8, 4}}, .s21 = {{2, 4}, {6, 8}}, .capture_start_ns = 5, .capture_end_ns = 15};
+    auto standard = trace;
+    standard.s21 = {{2, 0}, {2, 0}};
+    auto reference = trace;
+    reference.s21 = {{1, 2}, {2, 2}};
+    auto factors = trace;
+    factors.s21 = {{0, 2}, {5, 1}};
+    preprocessing::CalibrationMaster calibration(preprocessing::make_s21_through_calibrator());
+    calibration.load_s21_calibration_bundle(save_bundle(workspace, "pipeline_through.bin", {.traces = {standard}}));
+    preprocessing::ReferenceMaster references;
+    references.load_s21_reference_bundle(save_bundle(workspace, "pipeline_reference.bin", {.traces = {reference}}));
+    references.prepare_calibrated(calibration, {trace.combo});
+    config::RunConfig config;
+    config.run_combos = {trace.combo};
+    config.preprocess.s21.object_calibration.bundle_path =
+        save_bundle(workspace, "pipeline_object.bin", {.traces = {factors}});
+    config.preprocess.notch = {.enabled = true, .bands_hz = {{2e9F, 2e9F}}, .taper_type = "hard"};
+    RingName raw_name("_object_raw"), pre_name("_object_pre");
+    auto raw_ring = ipc::ShmRing::open_or_create(raw_name.name, 2, 4096);
+    auto pre_ring = ipc::ShmRing::open_or_create(pre_name.name, 2, 4096);
+    preprocessing::DataPreprocessor preprocessor(config, calibration, references, raw_ring, pre_ring);
+    require(raw_ring.push(ipc::serialize_raw_collection({.collection_id = 41, .capture_start_ns = 5,
+        .capture_end_ns = 15, .traces = {trace}})), "test raw ring could not hold a sweep");
+    std::atomic<bool> stop{false};
+    std::exception_ptr error;
+    std::jthread worker([&] { try { preprocessor.run(stop); } catch (...) { error = std::current_exception(); } });
+    std::vector<std::uint8_t> bytes;
+    const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2);
+    while (bytes.empty() && std::chrono::steady_clock::now() < deadline) {
+        (void)pre_ring.pop(bytes);
+        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+    stop.store(true);
+    worker.join();
+    if (error) { std::rethrow_exception(error); }
+    require(!bytes.empty(), "object-corrected preprocessed collection was not published");
+    const auto result = ipc::deserialize_preprocessed_collection(bytes);
+    const auto& output = result.traces.at(0);
+    require(result.collection_id == 41 && result.capture_start_ns == 5 && result.capture_end_ns == 15 &&
+            output.combo == trace.combo && output.capture_start_ns == 5 && output.capture_end_ns == 15,
+        "object calibration lost capture identity or timing");
+    require(output.s21[0].re == -2 && output.s21[0].im == 1 && output.s21[1].re == 0 && output.s21[1].im == 0,
+        "object calibration ran before existing preprocessing or was applied more than once");
+    require(output.s11[0].re == 7 && output.s11[0].im == 3 && output.s11[1].re == 0 && output.s11[1].im == 0,
+        "object calibration changed S11 or bypassed the notch filter");
+}
+
 void test_locator_packet_errors() {
     int sockets[2]{};
     require(::socketpair(AF_UNIX, SOCK_STREAM, 0, sockets) == 0, "socketpair failed");
@@ -384,6 +509,8 @@ int main(int argc, char** argv) {
         test_wire_decoders();
         test_configuration(argv[1], workspace);
         test_asset_reload_preserves_data(workspace);
+        test_object_calibration(workspace);
+        test_object_calibration_pipeline(workspace);
         test_ring();
         test_partial_driver_open();
         test_interruptible_wait();