init commit

data_acq_and_processing/preprocessing/calibration_master/include/calibration_master.hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "calibrator_interface.hpp"
+#include "shared_types.hpp"
+
+namespace radar::preprocessing {
+
+class CalibrationMaster {
+  public:
+    // `calibrator` encapsulates the actual calibration algorithm (v1: through).
+    explicit CalibrationMaster(std::unique_ptr<CalibratorInterface> calibrator);
+
+    // Loads a serialized raw sweep bundle with one calibration standard per combo.
+    void load_bundle(const std::string& path);
+    // Ensures all runtime combos are present in loaded standards.
+    void validate_combos(const std::vector<ipc::ComboKey>& combos) const;
+    // Applies calibration standard corresponding to measured trace combo.
+    [[nodiscard]] auto apply(const ipc::SweepTraceBlock& measured_trace) const -> ipc::SweepTraceBlock;
+
+  private:
+    std::unique_ptr<CalibratorInterface> calibrator_impl_;
+    std::unordered_map<ipc::ComboKey, ipc::SweepTraceBlock, ipc::ComboKeyHash> standards_by_combo_{};
+};
+
+[[nodiscard]] auto make_through_calibrator() -> std::unique_ptr<CalibratorInterface>;
+
+}  // namespace radar::preprocessing

data_acq_and_processing/preprocessing/calibration_master/include/calibrator_interface.hpp

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "shared_types.hpp"
+
+namespace radar::preprocessing {
+
+class CalibratorInterface {
+  public:
+    virtual ~CalibratorInterface() = default;
+
+    [[nodiscard]] virtual auto name() const -> std::string = 0;
+    [[nodiscard]] virtual auto apply(
+        const std::vector<ipc::Complex32>& measured,
+        const std::vector<ipc::Complex32>& calibration
+    ) const -> std::vector<ipc::Complex32> = 0;
+};
+
+}  // namespace radar::preprocessing

data_acq_and_processing/preprocessing/calibration_master/src/calibration_master.cpp

@@ -0,0 +1,101 @@
+#include "calibration_master.hpp"
+
+#include <cstdint>
+#include <fstream>
+#include <iterator>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+namespace radar::preprocessing {
+namespace {
+
+[[nodiscard]] auto combo_to_string(const ipc::ComboKey& combo) -> std::string {
+    return "input=" + std::to_string(combo.input_pos) + " output=" + std::to_string(combo.output_pos);
+}
+
+[[nodiscard]] auto read_binary_file(const std::string& path, const std::string& bundle_label)
+    -> std::vector<std::uint8_t> {
+    std::ifstream stream(path, std::ios::binary);
+    if (!stream.is_open()) {
+        throw std::runtime_error("Failed to open " + bundle_label + " bundle: " + path);
+    }
+
+    return std::vector<std::uint8_t>(std::istreambuf_iterator<char>(stream), std::istreambuf_iterator<char>());
+}
+
+void validate_trace_layout(const ipc::SweepTraceBlock& trace, const std::string& trace_label) {
+    if (trace.frequency_hz.size() != trace.s21.size()) {
+        throw std::runtime_error(
+            trace_label + " frequency/complex vector size mismatch for combo " + combo_to_string(trace.combo)
+        );
+    }
+}
+
+}  // namespace
+
+CalibrationMaster::CalibrationMaster(std::unique_ptr<CalibratorInterface> calibrator)
+    : calibrator_impl_(std::move(calibrator)) {
+    if (!calibrator_impl_) {
+        throw std::runtime_error("CalibrationMaster requires a non-null calibrator");
+    }
+}
+
+void CalibrationMaster::load_bundle(const std::string& path) {
+    if (path.empty()) {
+        throw std::runtime_error("Calibration bundle path must not be empty");
+    }
+
+    const auto bytes = read_binary_file(path, "calibration");
+    if (bytes.empty()) {
+        throw std::runtime_error("Calibration bundle is empty: " + path);
+    }
+
+    const auto collection = ipc::deserialize_raw_collection(bytes);
+    standards_by_combo_.clear();
+    standards_by_combo_.reserve(collection.traces.size());
+    for (const auto& trace : collection.traces) {
+        validate_trace_layout(trace, "Calibration standard");
+        standards_by_combo_.insert_or_assign(trace.combo, trace);
+    }
+
+    if (standards_by_combo_.empty()) {
+        throw std::runtime_error("Calibration bundle does not contain traces: " + path);
+    }
+}
+
+void CalibrationMaster::validate_combos(const std::vector<ipc::ComboKey>& combos) const {
+    for (const auto& combo : combos) {
+        if (!standards_by_combo_.contains(combo)) {
+            throw std::runtime_error("Calibration data is missing for combo " + combo_to_string(combo));
+        }
+    }
+}
+
+auto CalibrationMaster::apply(const ipc::SweepTraceBlock& measured_trace) const -> ipc::SweepTraceBlock {
+    validate_trace_layout(measured_trace, "Measured trace");
+
+    const auto found = standards_by_combo_.find(measured_trace.combo);
+    if (found == standards_by_combo_.end()) {
+        throw std::runtime_error(
+            "Calibration standard is missing for measured combo " + combo_to_string(measured_trace.combo)
+        );
+    }
+
+    const auto& standard = found->second;
+    validate_trace_layout(standard, "Calibration standard");
+    if (measured_trace.s21.size() != standard.s21.size()) {
+        throw std::runtime_error(
+            "Calibration standard point count mismatch for combo " + combo_to_string(measured_trace.combo)
+        );
+    }
+
+    ipc::SweepTraceBlock output{};
+    output.combo = measured_trace.combo;
+    output.frequency_hz = measured_trace.frequency_hz;
+    output.s21 = calibrator_impl_->apply(measured_trace.s21, standard.s21);
+
+    return output;
+}
+
+}  // namespace radar::preprocessing

data_acq_and_processing/preprocessing/calibration_master/src/through_calibrator.cpp

@@ -0,0 +1,76 @@
+#include "calibration_master.hpp"
+
+#include <cstddef>
+#include <memory>
+#include <stdexcept>
+#include <vector>
+
+#include <Eigen/Core>
+
+namespace radar::preprocessing {
+namespace {
+
+class ThroughCalibrator final : public CalibratorInterface {
+  public:
+    [[nodiscard]] auto name() const -> std::string override {
+        return "through";
+    }
+
+    [[nodiscard]] auto apply(
+        const std::vector<ipc::Complex32>& measured,
+        const std::vector<ipc::Complex32>& calibration
+    ) const -> std::vector<ipc::Complex32> override {
+        if (measured.size() != calibration.size()) {
+            throw std::runtime_error("Through calibration vector size mismatch");
+        }
+
+        static_assert(sizeof(ipc::Complex32) == sizeof(float) * 2U, "Complex32 layout must be two contiguous floats");
+
+        constexpr float epsilon = 1e-12F;
+        using InterleavedComplexView = Eigen::Matrix<float, Eigen::Dynamic, 2, Eigen::RowMajor>;
+
+        const auto point_count = static_cast<Eigen::Index>(measured.size());
+        Eigen::Map<const InterleavedComplexView> measured_view(
+            reinterpret_cast<const float*>(measured.data()),
+            point_count,
+            2
+        );
+        Eigen::Map<const InterleavedComplexView> calibration_view(
+            reinterpret_cast<const float*>(calibration.data()),
+            point_count,
+            2
+        );
+
+        const auto measured_re = measured_view.col(0).array();
+        const auto measured_im = measured_view.col(1).array();
+        const auto calibration_re = calibration_view.col(0).array();
+        const auto calibration_im = calibration_view.col(1).array();
+
+        const Eigen::ArrayXf magnitude_squared = calibration_re.square() + calibration_im.square();
+        const auto stable_division_mask = (magnitude_squared > epsilon);
+
+        const Eigen::ArrayXf corrected_re = stable_division_mask.select(
+            ((measured_re * calibration_re) + (measured_im * calibration_im)) / magnitude_squared,
+            measured_re
+        );
+        const Eigen::ArrayXf corrected_im = stable_division_mask.select(
+            ((measured_im * calibration_re) - (measured_re * calibration_im)) / magnitude_squared,
+            measured_im
+        );
+
+        std::vector<ipc::Complex32> corrected(measured.size());
+        Eigen::Map<InterleavedComplexView> corrected_view(reinterpret_cast<float*>(corrected.data()), point_count, 2);
+        corrected_view.col(0) = corrected_re.matrix();
+        corrected_view.col(1) = corrected_im.matrix();
+
+        return corrected;
+    }
+};
+
+}  // namespace
+
+auto make_through_calibrator() -> std::unique_ptr<CalibratorInterface> {
+    return std::make_unique<ThroughCalibrator>();
+}
+
+}  // namespace radar::preprocessing

data_acq_and_processing/preprocessing/data_preprocessor/include/data_preprocessor.hpp

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <atomic>
+#include <cstdint>
+#include <vector>
+
+#include "calibration_master.hpp"
+#include "reference_master.hpp"
+#include "run_config.hpp"
+#include "shm_ring.hpp"
+
+namespace radar::preprocessing {
+
+class DataPreprocessor {
+  public:
+    DataPreprocessor(
+        const config::RunConfig& config,
+        CalibrationMaster& calibration_master,
+        ReferenceMaster& reference_master,
+        ipc::ShmRing& raw_ring,
+        ipc::ShmRing& preprocessed_ring,
+        ipc::ShmRing* preprocessed_tap_ring = nullptr
+    );
+
+    // Main processing loop. Runs until stop flag is set.
+    void run(const std::atomic<bool>& stop_requested);
+
+  private:
+    void validate_startup_prerequisites() const;
+    [[nodiscard]] auto try_pop_raw_collection(
+        std::vector<std::uint8_t>& serialized_raw,
+        ipc::RawSweepCollection* out_collection
+    ) -> bool;
+    void publish_preprocessed_collection(const ipc::PreprocessedCollection& collection);
+
+    [[nodiscard]] auto preprocess_collection(const ipc::RawSweepCollection& raw_collection) const
+        -> ipc::PreprocessedCollection;
+
+    const config::RunConfig& config_;
+    CalibrationMaster& calibration_master_;
+    ReferenceMaster& reference_master_;
+    ipc::ShmRing& raw_ring_;
+    ipc::ShmRing& preprocessed_ring_;
+    ipc::ShmRing* preprocessed_tap_ring_ = nullptr;
+};
+
+}  // namespace radar::preprocessing

data_acq_and_processing/preprocessing/data_preprocessor/src/data_preprocessor.cpp

@@ -0,0 +1,85 @@
+#include "data_preprocessor.hpp"
+
+#include <chrono>
+#include <stdexcept>
+#include <thread>
+#include <vector>
+
+namespace radar::preprocessing {
+
+DataPreprocessor::DataPreprocessor(
+    const config::RunConfig& config,
+    CalibrationMaster& calibration_master,
+    ReferenceMaster& reference_master,
+    ipc::ShmRing& raw_ring,
+    ipc::ShmRing& preprocessed_ring,
+    ipc::ShmRing* preprocessed_tap_ring
+)
+    : config_(config),
+      calibration_master_(calibration_master),
+      reference_master_(reference_master),
+      raw_ring_(raw_ring),
+      preprocessed_ring_(preprocessed_ring),
+      preprocessed_tap_ring_(preprocessed_tap_ring) {}
+
+void DataPreprocessor::validate_startup_prerequisites() const {
+    calibration_master_.validate_combos(config_.run_combos);
+    reference_master_.validate_combos(config_.run_combos);
+}
+
+auto DataPreprocessor::try_pop_raw_collection(
+    std::vector<std::uint8_t>& serialized_raw,
+    ipc::RawSweepCollection* out_collection
+) -> bool {
+    if (!raw_ring_.pop(serialized_raw)) {
+        std::this_thread::sleep_for(std::chrono::milliseconds(config_.runtime.idle_sleep_ms));
+        return false;
+    }
+
+    *out_collection = ipc::deserialize_raw_collection(serialized_raw);
+    return true;
+}
+
+void DataPreprocessor::publish_preprocessed_collection(const ipc::PreprocessedCollection& collection) {
+    const auto serialized_preprocessed = ipc::serialize_preprocessed_collection(collection);
+    if (!preprocessed_ring_.push(serialized_preprocessed)) {
+        throw std::runtime_error("Preprocessed ring slot is too small for serialized collection");
+    }
+    if (preprocessed_tap_ring_ != nullptr && !preprocessed_tap_ring_->push(serialized_preprocessed)) {
+        throw std::runtime_error("Preprocessed tap ring slot is too small for serialized collection");
+    }
+}
+
+void DataPreprocessor::run(const std::atomic<bool>& stop_requested) {
+    validate_startup_prerequisites();
+
+    std::vector<std::uint8_t> serialized_raw{};
+    ipc::RawSweepCollection raw_collection{};
+    while (!stop_requested.load(std::memory_order_relaxed)) {
+        if (!try_pop_raw_collection(serialized_raw, &raw_collection)) {
+            continue;
+        }
+
+        const auto preprocessed_collection = preprocess_collection(raw_collection);
+        publish_preprocessed_collection(preprocessed_collection);
+    }
+}
+
+auto DataPreprocessor::preprocess_collection(const ipc::RawSweepCollection& raw_collection) const
+    -> ipc::PreprocessedCollection {
+    ipc::PreprocessedCollection preprocessed{};
+    preprocessed.collection_id = raw_collection.collection_id;
+    preprocessed.monotonic_ns = ipc::current_monotonic_ns();
+    preprocessed.traces.reserve(raw_collection.traces.size());
+
+    for (const auto& raw_trace : raw_collection.traces) {
+        // Pipeline order is fixed: calibration first, then reference subtraction.
+        const auto calibrated = calibration_master_.apply(raw_trace);
+        const auto referenced = reference_master_.apply(calibrated);
+        preprocessed.traces.push_back(referenced);
+    }
+
+    return preprocessed;
+}
+
+}  // namespace radar::preprocessing

data_acq_and_processing/preprocessing/data_preprocessor/src/main.cpp

@@ -0,0 +1,86 @@
+#include <atomic>
+#include <csignal>
+#include <exception>
+#include <iostream>
+#include <string>
+
+#include "calibration_master.hpp"
+#include "data_preprocessor.hpp"
+#include "reference_master.hpp"
+#include "run_config.hpp"
+#include "shm_ring.hpp"
+
+namespace {
+
+constexpr const char* kDefaultConfigPath = "run_config.json";
+
+std::atomic<bool> g_stop_requested{false};
+
+void signal_handler(int /*signal*/) {
+    g_stop_requested.store(true, std::memory_order_relaxed);
+}
+
+void install_signal_handlers() {
+    std::signal(SIGINT, signal_handler);
+    std::signal(SIGTERM, signal_handler);
+}
+
+[[nodiscard]] auto read_config_path(int argc, char** argv) -> std::string {
+    std::string config_path = kDefaultConfigPath;
+    for (int index = 1; index < argc; ++index) {
+        const std::string arg = argv[index];
+        if (arg == "--config" && (index + 1) < argc) {
+            config_path = argv[++index];
+        }
+    }
+    return config_path;
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+    install_signal_handlers();
+
+    try {
+        const auto config_path = read_config_path(argc, argv);
+        const auto config = radar::config::load_run_config(config_path);
+
+        auto raw_ring = radar::ipc::ShmRing::open_or_create(
+            config.rings.raw.name,
+            config.rings.raw.capacity,
+            config.rings.raw.slot_size_bytes
+        );
+        auto preprocessed_ring = radar::ipc::ShmRing::open_or_create(
+            config.rings.preprocessed.name,
+            config.rings.preprocessed.capacity,
+            config.rings.preprocessed.slot_size_bytes
+        );
+        auto preprocessed_tap_ring = radar::ipc::ShmRing::open_or_create(
+            config.rings.preprocessed_tap.name,
+            config.rings.preprocessed_tap.capacity,
+            config.rings.preprocessed_tap.slot_size_bytes
+        );
+
+        // Load preprocessing assets once before entering run loop.
+        radar::preprocessing::CalibrationMaster calibration_master(radar::preprocessing::make_through_calibrator());
+        calibration_master.load_bundle(config.preprocess.calibration_bundle_path);
+
+        radar::preprocessing::ReferenceMaster reference_master;
+        reference_master.load_bundle(config.preprocess.reference_bundle_path);
+        reference_master.prepare_calibrated(calibration_master);
+
+        radar::preprocessing::DataPreprocessor preprocessor(
+            config,
+            calibration_master,
+            reference_master,
+            raw_ring,
+            preprocessed_ring,
+            &preprocessed_tap_ring
+        );
+        preprocessor.run(g_stop_requested);
+        return 0;
+    } catch (const std::exception& exception) {
+        std::cerr << "data_preprocessor error: " << exception.what() << '\n';
+        return 1;
+    }
+}

data_acq_and_processing/preprocessing/reference_master/include/reference_master.hpp

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "shared_types.hpp"
+
+namespace radar::preprocessing {
+
+class CalibrationMaster;
+
+class ReferenceMaster {
+  public:
+    // Loads a serialized raw sweep bundle with one reference trace per combo.
+    void load_bundle(const std::string& path);
+    // Builds calibrated references in memory using currently loaded raw references.
+    // Must be called after load_bundle() and after calibration standards are loaded.
+    void prepare_calibrated(const CalibrationMaster& calibration_master);
+    // Ensures all runtime combos are present in loaded references.
+    void validate_combos(const std::vector<ipc::ComboKey>& combos) const;
+    // Subtracts per-combo reference trace from calibrated trace.
+    [[nodiscard]] auto apply(const ipc::SweepTraceBlock& calibrated_trace) const -> ipc::SweepTraceBlock;
+
+  private:
+    std::unordered_map<ipc::ComboKey, ipc::SweepTraceBlock, ipc::ComboKeyHash> raw_references_by_combo_{};
+    std::unordered_map<ipc::ComboKey, ipc::SweepTraceBlock, ipc::ComboKeyHash> calibrated_references_by_combo_{};
+};
+
+}  // namespace radar::preprocessing

data_acq_and_processing/preprocessing/reference_master/src/reference_master.cpp

@@ -0,0 +1,141 @@
+#include "reference_master.hpp"
+
+#include <cstdint>
+#include <fstream>
+#include <iterator>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <Eigen/Core>
+
+#include "calibration_master.hpp"
+
+namespace radar::preprocessing {
+namespace {
+
+[[nodiscard]] auto combo_to_string(const ipc::ComboKey& combo) -> std::string {
+    return "input=" + std::to_string(combo.input_pos) + " output=" + std::to_string(combo.output_pos);
+}
+
+[[nodiscard]] auto read_binary_file(const std::string& path, const std::string& bundle_label)
+    -> std::vector<std::uint8_t> {
+    std::ifstream stream(path, std::ios::binary);
+    if (!stream.is_open()) {
+        throw std::runtime_error("Failed to open " + bundle_label + " bundle: " + path);
+    }
+
+    return std::vector<std::uint8_t>(std::istreambuf_iterator<char>(stream), std::istreambuf_iterator<char>());
+}
+
+void validate_trace_layout(const ipc::SweepTraceBlock& trace, const std::string& trace_label) {
+    if (trace.frequency_hz.size() != trace.s21.size()) {
+        throw std::runtime_error(
+            trace_label + " frequency/complex vector size mismatch for combo " + combo_to_string(trace.combo)
+        );
+    }
+}
+
+}  // namespace
+
+void ReferenceMaster::load_bundle(const std::string& path) {
+    if (path.empty()) {
+        throw std::runtime_error("Reference bundle path must not be empty");
+    }
+
+    const auto bytes = read_binary_file(path, "reference");
+    if (bytes.empty()) {
+        throw std::runtime_error("Reference bundle is empty: " + path);
+    }
+
+    const auto collection = ipc::deserialize_raw_collection(bytes);
+    raw_references_by_combo_.clear();
+    raw_references_by_combo_.reserve(collection.traces.size());
+    calibrated_references_by_combo_.clear();
+    for (const auto& trace : collection.traces) {
+        validate_trace_layout(trace, "Reference trace");
+        raw_references_by_combo_.insert_or_assign(trace.combo, trace);
+    }
+
+    if (raw_references_by_combo_.empty()) {
+        throw std::runtime_error("Reference bundle does not contain traces: " + path);
+    }
+}
+
+void ReferenceMaster::prepare_calibrated(const CalibrationMaster& calibration_master) {
+    if (raw_references_by_combo_.empty()) {
+        throw std::runtime_error("Reference bundle must be loaded before prepare_calibrated()");
+    }
+
+    calibrated_references_by_combo_.clear();
+    calibrated_references_by_combo_.reserve(raw_references_by_combo_.size());
+    for (const auto& [combo, raw_reference] : raw_references_by_combo_) {
+        auto calibrated = calibration_master.apply(raw_reference);
+        calibrated.combo = combo;
+        calibrated_references_by_combo_.insert_or_assign(combo, std::move(calibrated));
+    }
+}
+
+void ReferenceMaster::validate_combos(const std::vector<ipc::ComboKey>& combos) const {
+    if (calibrated_references_by_combo_.empty()) {
+        throw std::runtime_error(
+            "Calibrated references are not prepared. Call ReferenceMaster::prepare_calibrated() at startup."
+        );
+    }
+
+    for (const auto& combo : combos) {
+        if (!raw_references_by_combo_.contains(combo)) {
+            throw std::runtime_error("Raw reference data is missing for combo " + combo_to_string(combo));
+        }
+        if (!calibrated_references_by_combo_.contains(combo)) {
+            throw std::runtime_error("Calibrated reference data is missing for combo " + combo_to_string(combo));
+        }
+    }
+}
+
+auto ReferenceMaster::apply(const ipc::SweepTraceBlock& calibrated_trace) const -> ipc::SweepTraceBlock {
+    validate_trace_layout(calibrated_trace, "Calibrated trace");
+
+    const auto found = calibrated_references_by_combo_.find(calibrated_trace.combo);
+    if (found == calibrated_references_by_combo_.end()) {
+        throw std::runtime_error(
+            "Calibrated reference trace is missing for combo " + combo_to_string(calibrated_trace.combo)
+        );
+    }
+
+    const auto& reference = found->second;
+    validate_trace_layout(reference, "Calibrated reference trace");
+    if (calibrated_trace.s21.size() != reference.s21.size()) {
+        throw std::runtime_error(
+            "Reference point count mismatch for combo " + combo_to_string(calibrated_trace.combo)
+        );
+    }
+
+    ipc::SweepTraceBlock output{};
+    output.combo = calibrated_trace.combo;
+    output.frequency_hz = calibrated_trace.frequency_hz;
+    output.s21.resize(calibrated_trace.s21.size());
+
+    static_assert(sizeof(ipc::Complex32) == sizeof(float) * 2U, "Complex32 layout must be two contiguous floats");
+    using InterleavedComplexView = Eigen::Matrix<float, Eigen::Dynamic, 2, Eigen::RowMajor>;
+    const auto point_count = static_cast<Eigen::Index>(calibrated_trace.s21.size());
+
+    Eigen::Map<const InterleavedComplexView> calibrated_view(
+        reinterpret_cast<const float*>(calibrated_trace.s21.data()),
+        point_count,
+        2
+    );
+    Eigen::Map<const InterleavedComplexView> reference_view(
+        reinterpret_cast<const float*>(reference.s21.data()),
+        point_count,
+        2
+    );
+    Eigen::Map<InterleavedComplexView> output_view(reinterpret_cast<float*>(output.s21.data()), point_count, 2);
+
+    output_view = calibrated_view - reference_view;
+
+    return output;
+}
+
+}  // namespace radar::preprocessing