Update GPR and SAR3D processing

data_acq_and_processing/tests/backprojection_fixture.cpp

@@ -0,0 +1,113 @@
+#include "sar3d_processor.hpp"
+#include "gpr_processor.hpp"
+
+#include <algorithm>
+#include <complex>
+#include <iostream>
+#include <vector>
+#include <nlohmann/json.hpp>
+
+using Json = nlohmann::json;
+using namespace radar;
+
+namespace {
+auto complex_samples(const Json& samples) -> std::vector<std::complex<float>> {
+    std::vector<std::complex<float>> values;
+    for (const auto& sample : samples) {
+        values.emplace_back(sample.at(0).get<float>(), sample.at(1).get<float>());
+    }
+    return values;
+}
+
+auto collection(const Json& source) -> ipc::PreprocessedCollection {
+    ipc::PreprocessedCollection result;
+    result.collection_id = source.at("id");
+    result.monotonic_ns = source.at("timestamp");
+    for (const auto& entry : source.at("traces")) {
+        ipc::SweepTraceBlock trace;
+        trace.combo = {entry.at("input"), entry.at("output")};
+        trace.capture_start_ns = entry.at("start");
+        trace.capture_end_ns = entry.at("end");
+        trace.frequency_hz = entry.at("frequency").get<std::vector<float>>();
+        for (const auto& sample : complex_samples(entry.at("s21"))) {
+            trace.s21.push_back({sample.real(), sample.imag()});
+        }
+        if (result.traces.empty() || trace.capture_start_ns < result.capture_start_ns) {
+            result.capture_start_ns = trace.capture_start_ns;
+        }
+        result.capture_end_ns = std::max(result.capture_end_ns, trace.capture_end_ns);
+        result.traces.push_back(std::move(trace));
+    }
+    return result;
+}
+}
+
+// Runs GPR or sar3d on one request or a batch, retaining processor caches. Each request holds the antenna
+// geometry, the live settings and the latest and previous collections; the
+// published payloads go to stdout as JSON.
+int main() {
+    try {
+        const Json source = Json::parse(std::cin);
+        const bool batch = source.is_array();
+        const Json requests = batch ? source : Json::array({source});
+        Json results = Json::array();
+        processing::Sar3dProcessor sar;
+        processing::GprProcessor gpr;
+        for (const auto& input : requests) {
+            config::RunConfig config;
+            config.rings.results.slot_size_bytes = input.value("slot", config.rings.results.slot_size_bytes);
+            for (const auto& row : input.at("tx")) {
+                config.gpr.tx_geometry.push_back({row.at(0), row.at(1), row.at(2), row.at(3)});
+            }
+            for (const auto& row : input.at("rx")) {
+                config.gpr.rx_geometry.push_back({row.at(0), row.at(1), row.at(2), row.at(3)});
+            }
+            processing::ProcessingLiveConfig live;
+            live.gpr_start_freq_mhz = input.at("f_start").get<double>() / 1e6;
+            live.gpr_stop_freq_mhz = input.at("f_stop").get<double>() / 1e6;
+            live.gpr_min_depth_m = input.at("min_depth");
+            live.gpr_max_depth_m = input.at("max_depth");
+            live.gpr_speed_m_s = input.at("speed");
+            live.gpr_look_angle_deg = input.at("pitch");
+            live.sar3d_voxel_m = input.at("voxel");
+            live.sar3d_bp_ground_depth_m = input.at("ground");
+            live.sar3d_frame_radius = input.at("radius");
+            live.sar3d_comp_range_power = input.at("power");
+            live.sar3d_use_compensation = input.at("compensation");
+            live.sar3d_normalize_each_frame = input.at("normalize");
+            live.pair_depth_equalize = input.value("equalize", true);
+            live.gpr_background_subtract_enabled = input.value("subtract_mean", false);
+            live.gpr_background_mean_count = input.value("mean_count", 10U);
+            live.sar3d_x_size_m = input.value("x_size", 0.0F);
+            live.sar3d_y_min_m = input.value("y_min", -0.5F);
+            live.sar3d_y_max_m = input.value("y_max", 2.0F);
+            live.sar3d_z_size_m = input.value("z_size", 0.0F);
+            live.gpr_bp_ground_depth_m = input.at("ground");
+            live.gpr_range_comp_power = input.at("power");
+            live.gpr_motion_mode = input.value("motion", "int_minus");
+            live.gpr_object_approach_min_frames = 1;
+            live.gpr_remove_sidelobe_objects_enabled = false;
+            std::vector<ipc::PreprocessedCollection> previous;
+            for (const auto& member : input.at("previous")) {
+                previous.push_back(collection(member));
+            }
+            const auto latest = collection(input.at("latest"));
+            for (const auto& trace : latest.traces) { config.run_combos.push_back(trace.combo); }
+            processing::ProcessorInterface& processor = input.value("mode", "sar3d") == "gpr"
+                ? static_cast<processing::ProcessorInterface&>(gpr) : static_cast<processing::ProcessorInterface&>(sar);
+            const auto result = processor.process_collection(config, latest, previous, live);
+            Json payloads = Json::array();
+            for (const auto& payload : result.collection_payloads) {
+                payloads.push_back({{"name", payload.processing_name},
+                    {"x", payload.image_x_axis}, {"y", payload.image_y_axis},
+                    {"image", payload.image_values}, {"table", payload.table_values},
+                    {"columns", payload.table_columns}});
+            }
+            results.push_back(Json{{"payloads", payloads}});
+        }
+        std::cout << (batch ? results : results.at(0)) << '\n';
+    } catch (const std::exception& error) {
+        std::cout << Json{{"error", error.what()}} << '\n';
+        return 1;
+    }
+}

data_acq_and_processing/tests/generate_backprojection_reference.py

@@ -1,12 +1,12 @@
-"""Generate small SAR3D references by executing notebook cells 0 and 4.
+"""Generate SAR3D/GPR references from notebook cells 1/14 and Horns coherent BP.
 
-Requires the original, untracked Multi_antenna_3D_clean_v1.ipynb; a missing notebook
+Requires the original, untracked Multi_antenna_demonstrate_depthnorm.ipynb; a missing notebook
 is an error. NumPy/SciPy execute its function/class ASTs unchanged. Only parameter
 values are replaced; plotting imports and the interactive top-level runs are
 excluded. No production SAR implementation or hand-transcribed oracle is used.
 
 Run from the repository root:
-    .venv/bin/python data_acq_and_processing/tests/generate_sar3d_reference.py
+    .venv/bin/python data_acq_and_processing/tests/generate_backprojection_reference.py
 """
 from __future__ import annotations
 
@@ -24,14 +24,14 @@ ROOT = Path(__file__).resolve().parents[2]
 REFERENCE = Path(__file__).with_name("sar3d_reference.npz")
 
 
-def notebook_functions(notebook: Path, parameters: dict) -> tuple[dict, dict]:
-    cells = json.loads(notebook.read_text(encoding="utf-8"))["cells"]
+def reference_functions(source_path: Path, parameters: dict, cells: tuple[int, ...] = (1, 14)) -> tuple[dict, dict]:
+    contents = source_path.read_text(encoding="utf-8")
+    sources = [(str(i), "".join(json.loads(contents)["cells"][i]["source"])) for i in cells] if cells else [("script", contents)]
     namespace = {"__name__": __name__}
     hashes = {}
-    for index in (0, 4):
-        source = "".join(cells[index]["source"])
+    for index, source in sources:
         hashes[str(index)] = hashlib.sha256(source.encode()).hexdigest()
-        tree = ast.parse(source, filename=f"{notebook.name}:cell{index}")
+        tree = ast.parse(source, filename=f"{source_path.name}:cell{index}")
         definitions = []
         for node in tree.body:
             if isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.ImportFrom)):
@@ -45,23 +45,24 @@ def notebook_functions(notebook: Path, parameters: dict) -> tuple[dict, dict]:
                 except (ValueError, TypeError):
                     pass  # Dataset loading and the interactive run are supplied below.
         namespace.update(parameters)
-        exec(compile(ast.Module(definitions, type_ignores=[]), f"{notebook.name}:cell{index}",
+        exec(compile(ast.Module(definitions, type_ignores=[]), f"{source_path.name}:cell{index}",
                      "exec", dont_inherit=True), namespace)
     return namespace, hashes
 
 
-def scene() -> dict:
+def scene(f_start=1e9, f_stop=4e9, points=201) -> dict:
     """Deterministic two-echo input with motion, vertical baselines and four pairs."""
     request = dict(
         tx=[[0, -.4, 0, 0], [1, .45, .2, .05]],
-        rx=[[0, -.2, .1, 0], [1, .3, .35, .02]], f_start=1e9, f_stop=4e9,
+        rx=[[0, -.2, .1, 0], [1, .3, .35, .02]], f_start=f_start, f_stop=f_stop,
         min_depth=1., max_depth=3., speed=.4, pitch=13., ground=.3,
-        voxel=.4, radius=1, power=1.5, compensation=True, normalize=True,
+        voxel=.4, radius=2, power=1.5, compensation=True, normalize=True,
+        equalize=True, subtract_mean=False, mean_count=1, motion="int_minus",
     )
     # Match the float32 settings/geometry accepted by the native fixture.
     for key in ("tx", "rx"):
         request[key] = np.asarray(request[key], dtype=np.float32).astype(float).tolist()
-    frequency = np.linspace(1e9, 4e9, 201, dtype=np.float32)
+    frequency = np.linspace(f_start, f_stop, points, dtype=np.float32)
     members = []
     for frame, offset in enumerate((0, 180_000_000, 510_000_000)):
         traces = []
@@ -91,14 +92,17 @@ def write_dataset(directory: Path, request: dict) -> list[Path]:
     ]
     (directory / "config_profile.json").write_text(json.dumps(profile), encoding="utf-8")
     snapshots = []
-    for index, member in enumerate([*request["previous"], request["latest"]]):
+    members = [*request["previous"], request["latest"]]
+    for index, member in enumerate(members):
+        background = previous_mean({**request, "previous": members[:index]})
         snapshot = directory / f'{index:04d}_id{member["id"]}_ns{member["timestamp"]}'
         snapshot.mkdir()
         metadata = {"traces": []}
         for trace in member["traces"]:
             stem = f'i{trace["input"]}_o{trace["output"]}'
-            values = np.asarray(trace["s21"], dtype=np.float32)
-            np.save(snapshot / f"{stem}_s21.npy", values[:, 0] + 1j * values[:, 1])
+            values = np.asarray(trace["s21"])
+            samples = values[:, 0] + 1j * values[:, 1] - background.get((trace["output"], trace["input"]), 0.)
+            np.save(snapshot / f"{stem}_s21.npy", samples)
             np.save(snapshot / f"{stem}_freq.npy", np.asarray(trace["frequency"], dtype=np.float32))
             metadata["traces"].append(dict(input=trace["input"], output=trace["output"],
                 capture_start_ns=trace["start"], capture_end_ns=trace["end"]))
@@ -109,7 +113,7 @@ def write_dataset(directory: Path, request: dict) -> list[Path]:
 
 def reference(notebook: Path, request: dict, directory: Path) -> tuple[dict, dict]:
     snapshots = write_dataset(directory, request)
-    center = snapshots[max(len(snapshots) - 1 - request["radius"], 0)]
+    center = snapshots[-1]
     parameters = dict(
         INPUT_IDX=[int(row[0]) for row in request["rx"]],
         OUTPUT_IDX=[int(row[0]) for row in request["tx"]],
@@ -120,9 +124,10 @@ def reference(notebook: Path, request: dict, directory: Path) -> tuple[dict, dic
         BP_GROUND_DEPTH_M=request["ground"], COMP_RANGE_POWER=request["power"],
         SAR_VOXEL_M=(request["voxel"],) * 3, SAR_FRAME_RADIUS=request["radius"],
         SAR_USE_COMPENSATION=request["compensation"], SAR_NORMALIZE_EACH_FRAME=request["normalize"],
+        PAIR_DEPTH_EQUALIZE=request["equalize"],
         SAR_SCAN_ROOT=directory, DATA_PATH=center, BG_PATH=directory, v=3e8,
     )
-    ns, hashes = notebook_functions(notebook, parameters)
+    ns, hashes = reference_functions(notebook, parameters)
     tx, rx, order, tx_order, pair_count = ns["load_antenna_config_profile"](
         directory / "config_profile.json", ns["INPUT_IDX"], ns["OUTPUT_IDX"])
     ns["N_tx"], ns["N_rx"] = len(tx), len(rx)
@@ -130,7 +135,9 @@ def reference(notebook: Path, request: dict, directory: Path) -> tuple[dict, dic
         for axis, values in zip("xyz", ns["positions_to_xyz"](positions).T):
             ns[f"{axis}_{kind}"] = values
     antenna_x = np.concatenate((ns["x_tx"], ns["x_rx"]))
-    ns["x_grid_bp"] = np.linspace(antenna_x.min() - 2, antenna_x.max() + 2, ns["NX_BP"])
+    ns["x_ant"] = antenna_x
+    ns["x_min"], ns["x_max"] = antenna_x.min() - 2, antenna_x.max() + 2
+    ns["x_grid_bp"] = np.linspace(ns["x_min"], ns["x_max"], ns["NX_BP"])
     sweep_time, _, _ = ns["estimate_sweep_time_from_meta"](directory, n_pairs=pair_count)
     config = ns["MotionConfig"](
         speed_m_s=ns["SPEED_M_S"], look_angle_deg=ns["LOOK_ANGLE_DEG"],
@@ -143,23 +150,118 @@ def reference(notebook: Path, request: dict, directory: Path) -> tuple[dict, dic
         motion_phase_sign=ns["MOTION_PHASE_SIGN"],
     )
     timestamps, _ = ns["compute_pair_timestamps_for_snapshot"](center, config)
-    result = ns["sar_compute_volume"](center, None, config, timestamps, verbose=False)
+    grid = ns["sar_build_full_region_grid"](config, y_range_m=(request.get("y_min", -.5), request.get("y_max", 2.)))
+    for axis in ("x", "z"):
+        size = request.get(f"{axis}_size", 0.)
+        if size:
+            center_axis = .5 * (grid[axis][0] + grid[axis][-1])
+            grid[axis] = np.linspace(center_axis - size/2, center_axis + size/2, max(round(size/request["voxel"])+1, 2))
+    grid["XX"], grid["ZZ"] = np.meshgrid(grid["x"], grid["z"])
+    ns["x_min"], ns["x_max"] = grid["x"][0], grid["x"][-1]
+    result = ns["sar_compute_volume"](center, None, config, timestamps, grid=grid, verbose=False)
     volume, grid = result["volume"], result["grid"]
     iy, iz, ix = ns["sar_global_max_index"](volume, grid)
+    from scipy.signal import peak_widths
+    widths = []
+    for axis, peak_index, profile in (("x", ix, volume[iy, iz, :]), ("y", iy, volume[:, iz, ix]), ("z", iz, volume[iy, :, ix])):
+        profile = np.asarray(profile, dtype=np.float64)
+        width, *_ = peak_widths(profile, [peak_index], prominence_data=(np.array([profile[peak_index]]),
+                                    np.array([0], dtype=np.intp), np.array([len(profile)-1], dtype=np.intp)))
+        widths.append(max(float(width[0]), 1.) * (grid[axis][1]-grid[axis][0]))
     expected = {axis: grid[axis] for axis in "xyz"}
+    expected["target_width"] = np.asarray(widths)
     expected.update(front=volume.max(axis=1), side=volume.max(axis=2), top=volume.max(axis=0),
                     peak=np.array([result["level"], grid["x"][ix], grid["y"][iy], grid["z"][iz]]))
     return expected, hashes
 
 
+def previous_mean(request: dict) -> dict:
+    samples = {}
+    if request["subtract_mean"] and request["mean_count"]:
+        for member in request["previous"][-request["mean_count"]:]:
+            for trace in member["traces"]:
+                values = np.asarray(trace["s21"])
+                samples.setdefault((trace["output"], trace["input"]), []).append(values[:, 0] + 1j * values[:, 1])
+    return {key: np.mean(values, axis=0) for key, values in samples.items()}
+
+
+def gpr_reference(notebook: Path, script: Path, request: dict) -> tuple[np.ndarray, dict]:
+    """Execute Horns coherent BP unchanged, adding the notebook's equalizer before its scalar norm."""
+    parameters = dict(F_START=request["f_start"], F_STOP=request["f_stop"], MIN_DEPTH=request["min_depth"],
+                      MAX_DEPTH=request["max_depth"], LOOK_ANGLE_DEG=request["pitch"],
+                      BP_GROUND_DEPTH_M=request["ground"], COMP_RANGE_POWER=request["power"], v=3e8)
+    ns, hashes = reference_functions(script, parameters, ())
+    ns["N_tx"], ns["N_rx"] = len(request["tx"]), len(request["rx"])
+    ns["OUTPUT_IDX"], ns["INPUT_IDX"] = [int(row[0]) for row in request["tx"]], [int(row[0]) for row in request["rx"]]
+    for kind in ("tx", "rx"):
+        for axis, values in zip("xyz", np.asarray(request[kind])[:, 1:].T):
+            ns[f"{axis}_{kind}"] = values
+    xs = np.concatenate((ns["x_tx"], ns["x_rx"]))
+    ns["x_ant"] = xs
+    ns["x_min"], ns["x_max"] = xs.min()-2, xs.max()+2
+    ns["x_grid_bp"] = np.linspace(ns["x_min"], ns["x_max"], ns["NX_BP"])
+    ns["z_grid_bp"] = np.linspace(.1, request["max_depth"], ns["NZ_BP"])
+    ns["XX_bp"], ns["ZZ_bp"] = np.meshgrid(ns["x_grid_bp"], ns["z_grid_bp"])
+    ns["depth_gate"] = (ns["ZZ_bp"] >= request["min_depth"]) & (ns["ZZ_bp"] <= request["max_depth"])
+    member = request["latest"]
+    duration = (max(t["end"] for t in member["traces"]) - min(t["start"] for t in member["traces"])) * 1e-9
+    config = ns["MotionConfig"](speed_m_s=request["speed"], look_angle_deg=request["pitch"],
+        tx_per_segment=len(request["tx"]), tx_order_full=tuple(ns["OUTPUT_IDX"]),
+        frame_acq_time_s=duration, n_freq_hint=len(member["traces"][0]["frequency"]),
+        intra_phase_focus_only=request["motion"] == "int_focus", motion_phase_sign=-1.,
+        pair_order_phys=[(t["output"], t["input"]) for t in member["traces"]])
+    clocks, _ = ns["compute_pair_timestamps"](config, ns["INPUT_IDX"], ns["OUTPUT_IDX"])
+    mean = previous_mean(request)
+    H, T = {}, {}
+    for trace in member["traces"]:
+        key = (trace["output"], trace["input"])
+        values = np.asarray(trace["s21"], dtype=np.float32)
+        samples = values[:, 0].astype(float) + 1j * values[:, 1].astype(float)
+        samples -= mean.get(key, 0.)
+        frequency = np.asarray(trace["frequency"], dtype=np.float32)
+        samples, _ = ns["apply_intra_sweep_phase_correction"](samples, frequency, clocks[key], config,
+                                                            ns["v"], ns["F_START"], ns["F_STOP"])
+        t, _, h, _, _ = ns["compute_ascan_bp"](samples, frequency, ns["F_START"], ns["F_STOP"])
+        H[key], T[key] = h, t
+    if request["equalize"]:
+        eq, notebook_hashes = reference_functions(notebook, {**parameters, "PAIR_DEPTH_EQUALIZE": True})
+        hashes.update({f"notebook_cell_{cell}": value for cell, value in notebook_hashes.items()})
+        eq.update(x_ant=xs, x_min=ns["x_min"], x_max=ns["x_max"])
+        H = eq["equalize_pairs_by_depth"](H, T)
+    for key, h in H.items():
+        depth = T[key] * ns["v"] / 2
+        mask = (depth >= request["min_depth"]) & (depth <= request["max_depth"])
+        scale = np.percentile(np.abs(h)[mask] if mask.any() else np.abs(h), 50.)
+        H[key] = h / ((scale if np.isfinite(scale) and scale > 1e-15 else 1.) + 1e-15)
+    raw, _ = ns["backproject_coherent"](H, T)
+    return ns["normalize_bp_map"](raw).astype(np.float32), hashes
+
+
+def generate_gpr(notebook: Path, script: Path, output: Path) -> None:
+    cases = [dict(equalize=False), dict(equalize=True),
+             dict(equalize=True, subtract_mean=True, mean_count=1, speed=-.2, pitch=-7., motion="int_focus", power=2.)]
+    arrays = {}
+    for i, settings in enumerate(cases):
+        request = scene(1.01e9, 4.01e9, 151) if i == 1 else scene()
+        request.update(settings, mode="gpr")
+        arrays[f"case_{i}_image"], hashes = gpr_reference(notebook, script, request)
+        arrays[f"case_{i}_request"] = np.array(json.dumps(request))
+    arrays["provenance"] = np.array(json.dumps(dict(script=script.name, source_sha256=hashes,
+                                                  notebook=notebook.name, numpy=np.__version__, scipy=scipy.__version__)))
+    np.savez_compressed(output, **arrays)
+    print(f"Wrote {output}: {output.stat().st_size} bytes, {len(cases)} GPR references")
+
+
 def generate(notebook: Path, output: Path) -> None:
     # Read up front so a missing notebook cannot produce an incomplete artifact.
     notebook.read_text(encoding="utf-8")
     cases = [
         dict(compensation=True, normalize=True),
-        dict(compensation=True, normalize=False, speed=-.2, pitch=-7., ground=.5),
+        dict(compensation=True, normalize=False, equalize=False, speed=-.2, pitch=-7., ground=.5),
         dict(compensation=False, normalize=True, f_start=1.3e9, f_stop=3.7e9),
         dict(compensation=False, normalize=False, speed=0., pitch=0., radius=0),
+        dict(compensation=True, normalize=True, subtract_mean=True, mean_count=1),
+        dict(compensation=True, normalize=True, x_size=3., z_size=2., y_min=-.2, y_max=1.4),
     ]
     arrays, configurations = {}, []
     for index, settings in enumerate(cases):
@@ -187,9 +289,9 @@ def generate(notebook: Path, output: Path) -> None:
         arrays.update({f"case_{index}_{key}": value for key, value in expected.items()})
     arrays["cases"] = np.array(json.dumps(configurations, separators=(",", ":")))
     arrays["provenance"] = np.array(json.dumps(dict(
-        notebook=notebook.name, cell_source_sha256=hashes, cells=[0, 4], cases=len(cases),
+        notebook=notebook.name, cell_source_sha256=hashes, cells=[1, 14], cases=len(cases),
         numpy=np.__version__, scipy=scipy.__version__,
-        method="Unchanged notebook AST functions; max projections of sar_compute_volume output",
+        method="Unchanged notebook AST functions; per-frame NumPy previous-mean subtraction; max projections of sar_compute_volume output",
     ), sort_keys=True))
     output.parent.mkdir(parents=True, exist_ok=True)
     np.savez_compressed(output, **arrays)
@@ -198,7 +300,10 @@ def generate(notebook: Path, output: Path) -> None:
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--notebook", type=Path, default=ROOT / "Multi_antenna_3D_clean_v1.ipynb")
+    parser.add_argument("--notebook", type=Path, default=ROOT / "Multi_antenna_demonstrate_depthnorm.ipynb")
     parser.add_argument("--output", type=Path, default=REFERENCE)
+    parser.add_argument("--gpr-script", type=Path, default=ROOT / "Horns_motion_4x4_ground.py")
+    parser.add_argument("--gpr-output", type=Path, default=REFERENCE.with_name("gpr_reference.npz"))
     args = parser.parse_args()
     generate(args.notebook, args.output)
+    generate_gpr(args.notebook, args.gpr_script, args.gpr_output)

data_acq_and_processing/tests/native_regressions.cpp

@@ -209,12 +209,14 @@ void test_configuration(const std::string& seed_path, const Workspace& workspace
         expect_error([&] { (void)config::load_run_config(config_path.string()); });
     }
     const auto live_path = workspace.path / "live.json";
-    std::ofstream(live_path) << Json{{"bscan_gain", 2}, {"settings_revision", 11}};
+    std::ofstream(live_path) << Json{{"bscan_gain", 2}, {"settings_revision", 11},
+        {"legacy_gpr_mode", false}, {"relative_permittivity", "obsolete"}};
     processing::ProcessingLiveConfigLoader loader(live_path.string(), config);
     require(loader.refresh_if_needed().bscan_gain == 2, "valid live config did not load");
     const auto revision = loader.revision();
     for (const auto& invalid : {Json{{"bscan_gain", 1e100}},
-             Json{{"gpr_input_positions", {0.0000001}}}, Json{{"bscan_gain", true}}}) {
+             Json{{"gpr_input_positions", {0.0000001}}}, Json{{"bscan_gain", true}},
+             Json{{"processor_mode", "legacy_gpr"}}}) {
         std::ofstream(live_path) << invalid;
         std::this_thread::sleep_for(std::chrono::milliseconds(110));
         const auto current = loader.refresh_if_needed();

data_acq_and_processing/tests/sar3d_fixture.cpp

@@ -1,89 +0,0 @@
-#include "sar3d_processor.hpp"
-
-#include <algorithm>
-#include <complex>
-#include <iostream>
-#include <vector>
-#include <nlohmann/json.hpp>
-
-using Json = nlohmann::json;
-using namespace radar;
-
-namespace {
-auto complex_samples(const Json& samples) -> std::vector<std::complex<float>> {
-    std::vector<std::complex<float>> values;
-    for (const auto& sample : samples) {
-        values.emplace_back(sample.at(0).get<float>(), sample.at(1).get<float>());
-    }
-    return values;
-}
-
-auto collection(const Json& source) -> ipc::PreprocessedCollection {
-    ipc::PreprocessedCollection result;
-    result.collection_id = source.at("id");
-    result.monotonic_ns = source.at("timestamp");
-    for (const auto& entry : source.at("traces")) {
-        ipc::SweepTraceBlock trace;
-        trace.combo = {entry.at("input"), entry.at("output")};
-        trace.capture_start_ns = entry.at("start");
-        trace.capture_end_ns = entry.at("end");
-        trace.frequency_hz = entry.at("frequency").get<std::vector<float>>();
-        for (const auto& sample : complex_samples(entry.at("s21"))) {
-            trace.s21.push_back({sample.real(), sample.imag()});
-        }
-        if (result.traces.empty() || trace.capture_start_ns < result.capture_start_ns) {
-            result.capture_start_ns = trace.capture_start_ns;
-        }
-        result.capture_end_ns = std::max(result.capture_end_ns, trace.capture_end_ns);
-        result.traces.push_back(std::move(trace));
-    }
-    return result;
-}
-}
-
-// Runs the sar3d processor once: the JSON request on stdin holds the antenna
-// geometry, the live settings and the latest and previous collections; the
-// published payloads go to stdout as JSON.
-int main() {
-    try {
-        const Json input = Json::parse(std::cin);
-        config::RunConfig config;
-        config.rings.results.slot_size_bytes = input.value("slot", config.rings.results.slot_size_bytes);
-        for (const auto& row : input.at("tx")) {
-            config.gpr.tx_geometry.push_back({row.at(0), row.at(1), row.at(2), row.at(3)});
-        }
-        for (const auto& row : input.at("rx")) {
-            config.gpr.rx_geometry.push_back({row.at(0), row.at(1), row.at(2), row.at(3)});
-        }
-        processing::ProcessingLiveConfig live;
-        live.gpr_start_freq_mhz = input.at("f_start").get<float>() / 1e6F;
-        live.gpr_stop_freq_mhz = input.at("f_stop").get<float>() / 1e6F;
-        live.gpr_min_depth_m = input.at("min_depth");
-        live.gpr_max_depth_m = input.at("max_depth");
-        live.gpr_speed_m_s = input.at("speed");
-        live.gpr_look_angle_deg = input.at("pitch");
-        live.sar3d_voxel_m = input.at("voxel");
-        live.sar3d_bp_ground_depth_m = input.at("ground");
-        live.sar3d_frame_radius = input.at("radius");
-        live.sar3d_comp_range_power = input.at("power");
-        live.sar3d_use_compensation = input.at("compensation");
-        live.sar3d_normalize_each_frame = input.at("normalize");
-        std::vector<ipc::PreprocessedCollection> previous;
-        for (const auto& member : input.at("previous")) {
-            previous.push_back(collection(member));
-        }
-        processing::Sar3dProcessor processor;
-        const auto result = processor.process_collection(config, collection(input.at("latest")), previous, live);
-        Json payloads = Json::array();
-        for (const auto& payload : result.collection_payloads) {
-            payloads.push_back({{"name", payload.processing_name},
-                {"x", payload.image_x_axis}, {"y", payload.image_y_axis},
-                {"image", payload.image_values}, {"table", payload.table_values},
-                {"columns", payload.table_columns}});
-        }
-        std::cout << Json{{"payloads", payloads}} << '\n';
-    } catch (const std::exception& error) {
-        std::cout << Json{{"error", error.what()}} << '\n';
-        return 1;
-    }
-}

data_acq_and_processing/tests/test_backprojection.py

@@ -1,9 +1,9 @@
 """The sar3d processor on the real CUDA backend: notebook reference and contracts.
 
-Every case runs build/native_tests/sar3d_fixture (SAR3D_FIXTURE overrides it).
+Every case runs build/native_tests/backprojection_fixture (BACKPROJECTION_FIXTURE overrides it).
 Without a usable GPU the processor publishes sar3d_status 3: then only that
 warning is checked and the remaining tests are skipped. The notebook is needed
-only to regenerate sar3d_reference.npz with generate_sar3d_reference.py.
+only to regenerate sar3d_reference.npz with generate_backprojection_reference.py.
 """
 from __future__ import annotations
 
@@ -17,15 +17,18 @@ import unittest
 import numpy as np
 
 ROOT = Path(__file__).resolve().parents[2]
-FIXTURE = Path(os.environ.get("SAR3D_FIXTURE", ROOT / "build/native_tests/sar3d_fixture"))
+FIXTURE = Path(os.environ.get("BACKPROJECTION_FIXTURE", ROOT / "build/native_tests/backprojection_fixture"))
 REFERENCE = Path(__file__).with_name("sar3d_reference.npz")
 VIEWS = ("sar3d_front", "sar3d_side", "sar3d_top", "sar3d_oblique")
 
 
-def run(request: dict) -> dict[str, dict]:
+def run(request: dict | list[dict]) -> dict[str, dict]:
     output = subprocess.run([str(FIXTURE)], input=json.dumps(request), text=True,
                             capture_output=True, timeout=30, check=True)
-    return {item["name"]: item for item in json.loads(output.stdout)["payloads"]}
+    result = json.loads(output.stdout)
+    if isinstance(request, list):
+        result = result[-1]
+    return {item["name"]: item for item in result["payloads"]}
 
 
 def status(payloads: dict[str, dict]) -> tuple[int, str]:
@@ -87,6 +90,8 @@ class Sar3dTest(unittest.TestCase):
                                                rtol=2e-6, atol=1e-6)
                     np.testing.assert_allclose(np.asarray(payload["image"]).reshape(expected.shape),
                                                expected, rtol=2e-3, atol=3e-4)
+                np.testing.assert_allclose(payloads["sar3d_target_width"]["table"],
+                                           self.reference[f"case_{index}_target_width"], rtol=2e-3, atol=2e-4)
                 peak = payloads["sar3d_peak"]["table"]
                 expected_peak = self.reference[f"case_{index}_peak"]
                 np.testing.assert_allclose(peak[1:], expected_peak[1:], rtol=0, atol=2e-6)
@@ -112,6 +117,15 @@ class Sar3dTest(unittest.TestCase):
                 self.assertEqual(code, 2, message)
                 self.assertTrue(message)
 
+    def test_cached_aperture_updates_when_previous_mean_context_changes(self) -> None:
+        request = self.scene(subtract_mean=True, mean_count=1)
+        shorter = copy.deepcopy(request)
+        shorter["previous"] = shorter["previous"][-1:]
+        cached, fresh = run([request, shorter]), run(shorter)
+        self.assertEqual(status(cached)[0], 0, status(cached))
+        for name in VIEWS:
+            np.testing.assert_allclose(cached[name]["image"], fresh[name]["image"], rtol=2e-3, atol=3e-4)
+
     def test_empty_signal_is_reported_without_a_peak(self) -> None:
         request = self.scene(previous=[], radius=0)
         for trace in request["latest"]["traces"]:
@@ -135,5 +149,32 @@ class Sar3dTest(unittest.TestCase):
             self.assertLess(values.max(), 1e-10)
 
 
+class GprTest(unittest.TestCase):
+    def test_changing_tx_rx_counts_rebuilds_the_cached_grid(self) -> None:
+        with np.load(REFERENCE.with_name("gpr_reference.npz"), allow_pickle=False) as reference:
+            base = json.loads(reference["case_0_request"].item())
+        requests = []
+        for tx, rx in ((2, 1), (1, 2)):
+            request = copy.deepcopy(base)
+            request.update(tx=[[i, 0, 0, 0] for i in range(tx)], rx=[[i, 0, 0, 0] for i in range(rx)])
+            request["latest"]["traces"] = [trace for trace in request["latest"]["traces"]
+                                          if trace["output"] < tx and trace["input"] < rx]
+            requests.append(request)
+        np.testing.assert_allclose(run(requests)["gpr_accumulator"]["image"],
+                                   run(requests[-1])["gpr_accumulator"]["image"])
+
+    def test_coherent_ground_bp_matches_horns_and_notebook_equalizer(self) -> None:
+        with np.load(REFERENCE.with_name("gpr_reference.npz"), allow_pickle=False) as reference:
+            for i in range(3):
+                with self.subTest(case=i):
+                    request = json.loads(reference[f"case_{i}_request"].item())
+                    payload = run(request)["gpr_accumulator"]
+                    expected = reference[f"case_{i}_image"]
+                    np.testing.assert_allclose(np.asarray(payload["image"]).reshape(expected.shape), expected,
+                                               rtol=2e-4, atol=3e-5)
+                    np.testing.assert_allclose(payload["y"], np.linspace(.1, request["max_depth"], expected.shape[0]),
+                                               rtol=1e-6, atol=1e-7)
+
+
 if __name__ == "__main__":
     unittest.main()