awe/radar_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Port the SAR3D background gate, CLEAN and object boxes from Multi_depthnorm_clean

Browse Source 
- BG_GATE_ENABLE: frame peak over its median background (sar3d_background); the
  GUI scales the frame with bg_gate_gain and its cloud threshold.
- SAR_CLEAN_ENABLE: CLEAN of a one-frame aperture on a 4x10x4 cm search grid over
  the whole scene; objects in their own scale (sar3d_objects).
- SAR_MARK_ENABLE: boxes by class around object groups (sar3d_marks), with the
  legend under the views.
- SAR_VOXEL_M as three steps X/Y/Z with the notebook defaults 0.025/0.02/0.02.
- GPU backend: image/download/load/project API, per-plane block reduction for the
  maximum, depth gate without FMA; sar3d host code built with -ffp-contract=off.
- References regenerated from notebook cells 1 and 5, with new CLEAN scenes and a
  synthetic box-rule case; docs updated.
This commit is contained in:
Ayzen
2026-10-16 12:08:57 +03:00
parent 9a9b8ce5c7
commit b88d704126
26 changed files with 1895 additions and 407 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data_acq_and_processing/tests/backprojection_fixture.cpp
+58 -17
View File
@@ -1,9 +1,11 @@
#include "sar3d_processor.hpp"
#include "gpr_processor.hpp"
#include "../processing/processors/src/sar3d_clean.hpp"
#include <algorithm>
#include <complex>
#include <iostream>
#include <numbers>
#include <vector>
#include <nlohmann/json.hpp>
@@ -40,11 +42,37 @@ auto collection(const Json& source) -> ipc::PreprocessedCollection {
}
return result;
}
// Boxes around given CLEAN objects, rows [x, y, z, y_low, y_high, level], in a scene with the
// given X and Y bounds and radar placement (sar3d::marks).
auto marks(const Json& input) -> Json {
std::vector<processing::sar3d::CleanObject> objects;
for (const auto& row : input.at("objects")) {
objects.push_back({.index = {}, .position = {row.at(0), row.at(1), row.at(2)},
.y_low = row.at(3), .y_high = row.at(4), .level = row.at(5)});
}
const auto x = input.at("x").get<std::vector<double>>();
const auto y = input.at("y").get<std::vector<double>>();
const processing::sar3d::VolumeRequest frame{
.grid = {x, y, {}},
.pitch_rad = input.at("pitch").get<double>() * (std::numbers::pi / 180.0),
.ground_depth_m = input.at("ground"),
.min_depth_m = input.at("min_depth"),
.max_depth_m = input.at("max_depth"),
};
Json rows = Json::array();
for (const auto& mark : processing::sar3d::marks(objects, frame)) {
Json row = Json::array({static_cast<int>(mark.kind)});
for (const double edge : mark.box) { row.push_back(edge); }
rows.push_back(std::move(row));
}
return Json{{"marks", rows}};
}
}
// Runs GPR or sar3d on one request or a batch, retaining processor caches. Each request holds the antenna
// geometry, the live settings and the latest and previous collections; the
// published payloads go to stdout as JSON.
// published payloads go to stdout as JSON. A request with mode "marks" boxes given objects instead.
int main() {
try {
const Json source = Json::parse(std::cin);
@@ -54,6 +82,10 @@ int main() {
processing::Sar3dProcessor sar;
processing::GprProcessor gpr;
for (const auto& input : requests) {
if (input.value("mode", "sar3d") == "marks") {
results.push_back(marks(input));
continue;
}
config::RunConfig config;
config.rings.results.slot_size_bytes = input.value("slot", config.rings.results.slot_size_bytes);
for (const auto& row : input.at("tx")) {
@@ -62,6 +94,7 @@ int main() {
for (const auto& row : input.at("rx")) {
config.gpr.rx_geometry.push_back({row.at(0), row.at(1), row.at(2), row.at(3)});
}
const bool gpr_mode = input.value("mode", "sar3d") == "gpr";
processing::ProcessingLiveConfig live;
live.gpr_start_freq_mhz = input.at("f_start").get<double>() / 1e6;
live.gpr_stop_freq_mhz = input.at("f_stop").get<double>() / 1e6;
@@ -69,31 +102,39 @@ int main() {
live.gpr_max_depth_m = input.at("max_depth");
live.gpr_speed_m_s = input.at("speed");
live.gpr_look_angle_deg = input.at("pitch");
live.sar3d_voxel_m = input.at("voxel");
live.sar3d_bp_ground_depth_m = input.at("ground");
live.sar3d_frame_radius = input.at("radius");
live.sar3d_comp_range_power = input.at("power");
live.sar3d_use_compensation = input.at("compensation");
live.sar3d_normalize_each_frame = input.at("normalize");
live.pair_depth_equalize = input.value("equalize", true);
live.gpr_background_subtract_enabled = input.value("subtract_mean", false);
live.gpr_background_mean_count = input.value("mean_count", 10U);
live.sar3d_x_size_m = input.value("x_size", 0.0F);
live.sar3d_y_min_m = input.value("y_min", -0.5F);
live.sar3d_y_max_m = input.value("y_max", 2.0F);
live.sar3d_z_size_m = input.value("z_size", 0.0F);
live.gpr_bp_ground_depth_m = input.at("ground");
live.gpr_range_comp_power = input.at("power");
live.gpr_motion_mode = input.value("motion", "int_minus");
live.gpr_object_approach_min_frames = 1;
live.gpr_remove_sidelobe_objects_enabled = false;
if (gpr_mode) {
live.gpr_bp_ground_depth_m = input.at("ground");
live.gpr_range_comp_power = input.at("power");
live.gpr_motion_mode = input.value("motion", "int_minus");
live.gpr_object_approach_min_frames = 1;
live.gpr_remove_sidelobe_objects_enabled = false;
} else {
const auto& voxel = input.at("voxel");
live.sar3d_voxel_x_m = voxel.at(0);
live.sar3d_voxel_y_m = voxel.at(1);
live.sar3d_voxel_z_m = voxel.at(2);
live.sar3d_bp_ground_depth_m = input.at("ground");
live.sar3d_frame_radius = input.at("radius");
live.sar3d_comp_range_power = input.at("power");
live.sar3d_use_compensation = input.at("compensation");
live.sar3d_normalize_each_frame = input.at("normalize");
live.sar3d_x_size_m = input.value("x_size", 0.0F);
live.sar3d_y_min_m = input.value("y_min", -0.5F);
live.sar3d_y_max_m = input.value("y_max", 2.0F);
live.sar3d_z_size_m = input.value("z_size", 0.0F);
live.sar3d_bg_gate_enabled = input.value("bg_gate", live.sar3d_bg_gate_enabled);
live.sar3d_clean_enabled = input.value("clean", live.sar3d_clean_enabled);
}
std::vector<ipc::PreprocessedCollection> previous;
for (const auto& member : input.at("previous")) {
previous.push_back(collection(member));
}
const auto latest = collection(input.at("latest"));
for (const auto& trace : latest.traces) { config.run_combos.push_back(trace.combo); }
processing::ProcessorInterface& processor = input.value("mode", "sar3d") == "gpr"
processing::ProcessorInterface& processor = gpr_mode
? static_cast<processing::ProcessorInterface&>(gpr) : static_cast<processing::ProcessorInterface&>(sar);
const auto result = processor.process_collection(config, latest, previous, live);
Json payloads = Json::array();
data_acq_and_processing/tests/generate_backprojection_reference.py
+131 -25
View File
@@ -1,6 +1,6 @@
"""Generate SAR3D/GPR references from notebook cells 1/14 and Horns coherent BP.
"""Generate SAR3D/GPR references from notebook cells 1/5 and Horns coherent BP.
Requires the original, untracked Multi_antenna_demonstrate_depthnorm.ipynb; a missing notebook
Requires the original, untracked Multi_depthnorm_clean.ipynb; a missing notebook
is an error. NumPy/SciPy execute its function/class ASTs unchanged. Only parameter
values are replaced; plotting imports and the interactive top-level runs are
excluded. No production SAR implementation or hand-transcribed oracle is used.
@@ -16,6 +16,7 @@ import hashlib
import json
from pathlib import Path
import tempfile
from types import SimpleNamespace
import numpy as np
import scipy
@@ -24,7 +25,7 @@ ROOT = Path(__file__).resolve().parents[2]
REFERENCE = Path(__file__).with_name("sar3d_reference.npz")
def reference_functions(source_path: Path, parameters: dict, cells: tuple[int, ...] = (1, 14)) -> tuple[dict, dict]:
def reference_functions(source_path: Path, parameters: dict, cells: tuple[int, ...] = (1, 5)) -> tuple[dict, dict]:
contents = source_path.read_text(encoding="utf-8")
sources = [(str(i), "".join(json.loads(contents)["cells"][i]["source"])) for i in cells] if cells else [("script", contents)]
namespace = {"__name__": __name__}
@@ -56,8 +57,9 @@ def scene(f_start=1e9, f_stop=4e9, points=201) -> dict:
tx=[[0, -.4, 0, 0], [1, .45, .2, .05]],
rx=[[0, -.2, .1, 0], [1, .3, .35, .02]], f_start=f_start, f_stop=f_stop,
min_depth=1., max_depth=3., speed=.4, pitch=13., ground=.3,
voxel=.4, radius=2, power=1.5, compensation=True, normalize=True,
voxel=[.4, .4, .4], radius=2, power=1.5, compensation=True, normalize=True,
equalize=True, subtract_mean=False, mean_count=1, motion="int_minus",
bg_gate=True, clean=False,
)
# Match the float32 settings/geometry accepted by the native fixture.
for key in ("tx", "rx"):
@@ -79,6 +81,45 @@ def scene(f_start=1e9, f_stop=4e9, points=201) -> dict:
return request
# Point reflectors (X, height Y, Z, amplitude) at heights of 0, 0.6 and 0.25 m.
REFLECTORS = [(.3, 0., 2.2, 1.), (-.8, .6, 2.6, .5), (1.3, .25, 1.9, .7)]
def point_scene(targets: list[tuple[float, float, float, float]], noise: float, seed: int, **settings) -> dict:
"""Settings of one static frame of point reflectors (X, height Y, Z, amplitude) in noise, for CLEAN.
Three Tx and three Rx at different heights resolve Y in a single frame. The voxel steps
keep the notebook's proportions, 2.5 : 2 : 2.
"""
request = dict(
tx=[[0, -.6, 0., 0.], [1, 0., .35, .02], [2, .6, .1, 0.]],
rx=[[0, -.3, .2, 0.], [1, .3, .45, .03], [2, .9, 0., .01]],
min_depth=1., max_depth=3.5, speed=0., pitch=13., ground=1., voxel=[.0625, .05, .05], radius=0,
clean=True, previous=[],
)
request.update(settings)
for key in ("tx", "rx"):
request[key] = np.asarray(request[key], dtype=np.float32).astype(float).tolist()
frequency = np.linspace(1e9, 4e9, 201, dtype=np.float32)
cos_p, sin_p = np.cos(np.radians(request["pitch"])), np.sin(np.radians(request["pitch"]))
random = np.random.default_rng(seed)
traces = []
for tx in request["tx"]:
for rx in request["rx"]:
samples = noise * (random.standard_normal(frequency.size) + 1j * random.standard_normal(frequency.size))
for x, height, z, amplitude in targets:
# Scene to radar axes, as sar_scene_to_radar_yz.
y0 = height - request["ground"]
point = np.array([x, y0 * cos_p + z * sin_p, -y0 * sin_p + z * cos_p])
path = np.linalg.norm(point - tx[1:]) + np.linalg.norm(point - rx[1:])
samples += amplitude * np.exp(-2j * np.pi * frequency.astype(float) * path / 3e8)
samples = samples.astype(np.complex64)
traces.append(dict(input=int(rx[0]), output=int(tx[0]), start=10_000_000_000, end=10_165_000_000,
frequency=frequency.tolist(), s21=np.column_stack((samples.real, samples.imag)).tolist()))
request["latest"] = dict(id=1, timestamp=10_165_000_000, traces=traces)
return request
def write_dataset(directory: Path, request: dict) -> list[Path]:
"""Supply the notebook's original config_profile/meta/NPY input contract."""
profile = {"gpr": {}, "run": {"combos": []}}
@@ -115,6 +156,8 @@ def reference(notebook: Path, request: dict, directory: Path) -> tuple[dict, dic
snapshots = write_dataset(directory, request)
center = snapshots[-1]
parameters = dict(
SAR_Y_RANGE_M=(request.get("y_min", -.5), request.get("y_max", 2.)),
BG_GATE_ENABLE=request["bg_gate"], SAR_CLEAN_ENABLE=request["clean"], SAR_MARK_ENABLE=True,
INPUT_IDX=[int(row[0]) for row in request["rx"]],
OUTPUT_IDX=[int(row[0]) for row in request["tx"]],
F_START=request["f_start"], F_STOP=request["f_stop"],
@@ -122,7 +165,7 @@ def reference(notebook: Path, request: dict, directory: Path) -> tuple[dict, dic
PAIR_NORM_DEPTH_MIN=request["min_depth"], PAIR_NORM_DEPTH_MAX=request["max_depth"],
SPEED_M_S=request["speed"], LOOK_ANGLE_DEG=request["pitch"],
BP_GROUND_DEPTH_M=request["ground"], COMP_RANGE_POWER=request["power"],
SAR_VOXEL_M=(request["voxel"],) * 3, SAR_FRAME_RADIUS=request["radius"],
SAR_VOXEL_M=tuple(request["voxel"]), SAR_FRAME_RADIUS=request["radius"],
SAR_USE_COMPENSATION=request["compensation"], SAR_NORMALIZE_EACH_FRAME=request["normalize"],
PAIR_DEPTH_EQUALIZE=request["equalize"],
SAR_SCAN_ROOT=directory, DATA_PATH=center, BG_PATH=directory, v=3e8,
@@ -150,31 +193,76 @@ def reference(notebook: Path, request: dict, directory: Path) -> tuple[dict, dic
motion_phase_sign=ns["MOTION_PHASE_SIGN"],
)
timestamps, _ = ns["compute_pair_timestamps_for_snapshot"](center, config)
grid = ns["sar_build_full_region_grid"](config, y_range_m=(request.get("y_min", -.5), request.get("y_max", 2.)))
for axis in ("x", "z"):
size = request.get(f"{axis}_size", 0.)
grid = ns["sar_build_full_region_grid"](config)
for index, axis in ((0, "x"), (2, "z")):
size, step = request.get(f"{axis}_size", 0.), request["voxel"][index]
if size:
center_axis = .5 * (grid[axis][0] + grid[axis][-1])
grid[axis] = np.linspace(center_axis - size/2, center_axis + size/2, max(round(size/request["voxel"])+1, 2))
grid[axis] = np.linspace(center_axis - size/2, center_axis + size/2, max(round(size/step)+1, 2))
grid["XX"], grid["ZZ"] = np.meshgrid(grid["x"], grid["z"])
ns["x_min"], ns["x_max"] = grid["x"][0], grid["x"][-1]
result = ns["sar_compute_volume"](center, None, config, timestamps, grid=grid, verbose=False)
volume, grid = result["volume"], result["grid"]
iy, iz, ix = ns["sar_global_max_index"](volume, grid)
from scipy.signal import peak_widths
widths = []
for axis, peak_index, profile in (("x", ix, volume[iy, iz, :]), ("y", iy, volume[:, iz, ix]), ("z", iz, volume[iy, :, ix])):
profile = np.asarray(profile, dtype=np.float64)
width, *_ = peak_widths(profile, [peak_index], prominence_data=(np.array([profile[peak_index]]),
np.array([0], dtype=np.intp), np.array([len(profile)-1], dtype=np.intp)))
widths.append(max(float(width[0]), 1.) * (grid[axis][1]-grid[axis][0]))
# What the processor publishes: the frame volume, or the CLEAN display when CLEAN found
# objects, and the peak it marks; the background gain stays to the GUI.
ns["sar_prepare_display"](result, None, timestamps)
objects = result["clean_objects"] or []
volume, grid = result["display_volume"] if objects else result["volume"], result["grid"]
iy, iz, ix = result["display_max_idx"]
expected = {axis: grid[axis] for axis in "xyz"}
expected["target_width"] = np.asarray(widths)
expected.update(front=volume.max(axis=1), side=volume.max(axis=2), top=volume.max(axis=0),
peak=np.array([result["level"], grid["x"][ix], grid["y"][iy], grid["z"][iz]]))
if not objects:
# Width of the response through the marked maximum; objects are marked instead.
from scipy.signal import peak_widths
widths = []
for axis, peak_index, profile in (("x", ix, volume[iy, iz, :]), ("y", iy, volume[:, iz, ix]),
("z", iz, volume[iy, :, ix])):
profile = np.asarray(profile, dtype=np.float64)
width, *_ = peak_widths(profile, [peak_index], prominence_data=(np.array([profile[peak_index]]),
np.array([0], dtype=np.intp), np.array([len(profile)-1], dtype=np.intp)))
widths.append(max(float(width[0]), 1.) * (grid[axis][1]-grid[axis][0]))
expected["target_width"] = np.asarray(widths)
if request["bg_gate"]:
expected["background"] = np.array([result["bg_ratio"], ns["BG_GATE_3D_SHOW"], ns["BG_GATE_3D_FULL"]])
if result["clean_info"] is not None:
expected["clean"] = np.array([result["clean_info"]["steps"], result["clean_info"]["residual_ratio"]])
if objects:
kinds = {"low": 0, "ground": 1, "up": 2, "edge": 3}
expected["objects"] = np.array([[item["x"], item["y"], item["z"], item["level_bg"]] for item in objects])
expected["marks"] = np.array([[kinds[group["kind"]], *group["box"]] for group in result["mark_groups"]])
return expected, hashes
def marks_reference(notebook: Path) -> tuple[dict, np.ndarray]:
"""sar_mark_groups on synthetic CLEAN objects, each of which tests one rule of the boxes.
Returns the fixture's "marks" request and the expected rows [class, X0, X1, Y0, Y1, Z0, Z1].
"""
scene = dict(x=[-2.6, 2.9], y=[-.5, 2.], pitch=13., ground=1., min_depth=1., max_depth=3.5)
objects = [ # [x, y, z, y_low, y_high, level], strongest first
[0., 0., 2., -.1, .3, 60.], # low
[1., .05, 2.2, -.1, .6, 55.], # centre low, top above SAR_MARK_LOW_TOP_M: ground
[-1., .3, 2.4, .1, .5, 50.], # ground
[-.2, .45, 3., .25, .65, 45.], # up while its weak member (last) stays out of the centre
[-1.6, 0., 1.3, -.1, .3, 40.], # a chain of three: the ends are 1.4 m apart, still one box
[-.9, .1, 1.3, -.1, .35, 35.],
[-.2, .05, 1.3, -.15, .3, 30.],
[-2.5, .2, 2., 0., .4, 28.], # near the X minimum: edge
[2.75, .2, 2., 0., .4, 26.], # near the X maximum: edge
[2.2, .1, 3.2, -.1, .3, 24.], # near MAX_DEPTH: edge
[2.2, 0., .85, -.2, .2, 22.], # near MIN_DEPTH: edge
[1.2, 1.7, 2.8, 1.5, 1.9, 20.], # near the top: edge
[.2, -.4, 3.1, -.5, -.25, 15.], # weak member of the "up" box, below 40 % of its top
]
ns, _ = reference_functions(notebook, dict(BP_GROUND_DEPTH_M=scene["ground"], MIN_DEPTH=scene["min_depth"],
MAX_DEPTH=scene["max_depth"]))
groups = ns["sar_mark_groups"](
[dict(zip(("x", "y", "z", "y_lo", "y_hi", "level_bg"), row)) for row in objects],
{axis: np.asarray(scene[axis]) for axis in "xy"}, SimpleNamespace(look_angle_deg=scene["pitch"]))
kinds = {"low": 0, "ground": 1, "up": 2, "edge": 3}
return dict(mode="marks", objects=objects, **scene), np.array([[kinds[g["kind"]], *g["box"]] for g in groups])
def previous_mean(request: dict) -> dict:
samples = {}
if request["subtract_mean"] and request["mean_count"]:
@@ -258,17 +346,32 @@ def generate(notebook: Path, output: Path) -> None:
cases = [
dict(compensation=True, normalize=True),
dict(compensation=True, normalize=False, equalize=False, speed=-.2, pitch=-7., ground=.5),
dict(compensation=False, normalize=True, f_start=1.3e9, f_stop=3.7e9),
# Three different voxel steps: each axis keeps its own.
dict(compensation=False, normalize=True, f_start=1.3e9, f_stop=3.7e9, voxel=[.45, .4, .35]),
dict(compensation=False, normalize=False, speed=0., pitch=0., radius=0),
dict(compensation=True, normalize=True, subtract_mean=True, mean_count=1),
dict(compensation=True, normalize=True, x_size=3., z_size=2., y_min=-.2, y_max=1.4),
# CLEAN stops below its thresholds; the background gate hides the residual.
point_scene(REFLECTORS, noise=.02, seed=1),
# CLEAN stops at SAR_CLEAN_MAX_STEPS; without the gate the residual stays visible.
point_scene(REFLECTORS, noise=.01, seed=3, bg_gate=False),
# The stop at the background of each range (SAR_CLEAN_STOP_RANGE_BG) adds two steps to
# those of the stop at the background of the whole volume.
point_scene([(.2, 0., 2., 1.), (-.5, .1, 4.4, .08)], noise=.01, seed=4, max_depth=5.),
# A narrowed frame: CLEAN still searches the whole scene, so the reflector at X = -0.8 m
# just outside the frame is subtracted instead of its sidelobe becoming an object.
point_scene(REFLECTORS, noise=.02, seed=19, x_size=1.6, z_size=.8),
]
arrays, configurations = {}, []
for index, settings in enumerate(cases):
request = scene()
request.update(settings)
for key in ("min_depth", "max_depth", "speed", "pitch", "ground", "voxel", "power"):
request[key] = float(np.float32(request[key]))
# Match the float32 live settings of the native fixture.
for key in ("min_depth", "max_depth", "speed", "pitch", "ground", "power",
"x_size", "z_size", "y_min", "y_max"):
if key in request:
request[key] = float(np.float32(request[key]))
request["voxel"] = [float(np.float32(step)) for step in request["voxel"]]
if index == 1:
# A separate, unequal capture window per output instead of interleaving.
for member in [*request["previous"], request["latest"]]:
@@ -288,10 +391,13 @@ def generate(notebook: Path, output: Path) -> None:
))
arrays.update({f"case_{index}_{key}": value for key, value in expected.items()})
arrays["cases"] = np.array(json.dumps(configurations, separators=(",", ":")))
marks_request, arrays["marks"] = marks_reference(notebook)
arrays["marks_request"] = np.array(json.dumps(marks_request))
arrays["provenance"] = np.array(json.dumps(dict(
notebook=notebook.name, cell_source_sha256=hashes, cells=[1, 14], cases=len(cases),
notebook=notebook.name, cell_source_sha256=hashes, cells=[1, 5], cases=len(cases),
numpy=np.__version__, scipy=scipy.__version__,
method="Unchanged notebook AST functions; per-frame NumPy previous-mean subtraction; max projections of sar_compute_volume output",
method="Unchanged notebook AST functions; per-frame NumPy previous-mean subtraction; "
"max projections of the sar_prepare_display volume, its background, CLEAN objects and marks",
), sort_keys=True))
output.parent.mkdir(parents=True, exist_ok=True)
np.savez_compressed(output, **arrays)
@@ -300,7 +406,7 @@ def generate(notebook: Path, output: Path) -> None:
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--notebook", type=Path, default=ROOT / "Multi_antenna_demonstrate_depthnorm.ipynb")
parser.add_argument("--notebook", type=Path, default=ROOT / "Multi_depthnorm_clean.ipynb")
parser.add_argument("--output", type=Path, default=REFERENCE)
parser.add_argument("--gpr-script", type=Path, default=ROOT / "Horns_motion_4x4_ground.py")
parser.add_argument("--gpr-output", type=Path, default=REFERENCE.with_name("gpr_reference.npz"))
data_acq_and_processing/tests/gpr_reference.npz
BIN
View File
Binary file not shown.
data_acq_and_processing/tests/native_regressions.cpp
+4 -1
View File
@@ -210,9 +210,12 @@ void test_configuration(const std::string& seed_path, const Workspace& workspace
}
const auto live_path = workspace.path / "live.json";
std::ofstream(live_path) << Json{{"bscan_gain", 2}, {"settings_revision", 11},
{"sar3d_bg_gate_enabled", false}, {"sar3d_clean_enabled", false},
{"legacy_gpr_mode", false}, {"relative_permittivity", "obsolete"}};
processing::ProcessingLiveConfigLoader loader(live_path.string(), config);
require(loader.refresh_if_needed().bscan_gain == 2, "valid live config did not load");
const auto loaded = loader.refresh_if_needed();
require(loaded.bscan_gain == 2 && !loaded.sar3d_bg_gate_enabled && !loaded.sar3d_clean_enabled,
"valid live config did not load");
const auto revision = loader.revision();
for (const auto& invalid : {Json{{"bscan_gain", 1e100}},
Json{{"gpr_input_positions", {0.0000001}}}, Json{{"bscan_gain", true}},
data_acq_and_processing/tests/sar3d_reference.npz
BIN
View File
Binary file not shown.
data_acq_and_processing/tests/test_backprojection.py
+80 -5
View File
@@ -2,8 +2,9 @@
Every case runs build/native_tests/backprojection_fixture (BACKPROJECTION_FIXTURE overrides it).
Without a usable GPU the processor publishes sar3d_status 3: then only that
warning is checked and the remaining tests are skipped. The notebook is needed
only to regenerate sar3d_reference.npz with generate_backprojection_reference.py.
warning and the box rules, which are host code, are checked, and the remaining tests
are skipped. The notebook is needed only to regenerate the references with
generate_backprojection_reference.py.
"""
from __future__ import annotations
@@ -11,6 +12,7 @@ import copy
import json
import os
from pathlib import Path
import re
import subprocess
import unittest
@@ -90,22 +92,80 @@ class Sar3dTest(unittest.TestCase):
rtol=2e-6, atol=1e-6)
np.testing.assert_allclose(np.asarray(payload["image"]).reshape(expected.shape),
expected, rtol=2e-3, atol=3e-4)
np.testing.assert_allclose(payloads["sar3d_target_width"]["table"],
self.reference[f"case_{index}_target_width"], rtol=2e-3, atol=2e-4)
widths = self.reference.get(f"case_{index}_target_width")
if widths is None:
self.assertNotIn("sar3d_target_width", payloads)
else:
np.testing.assert_allclose(payloads["sar3d_target_width"]["table"], widths, rtol=2e-3, atol=2e-4)
peak = payloads["sar3d_peak"]["table"]
expected_peak = self.reference[f"case_{index}_peak"]
np.testing.assert_allclose(peak[1:], expected_peak[1:], rtol=0, atol=2e-6)
np.testing.assert_allclose(peak[0], expected_peak[0], rtol=2e-3, atol=0)
self.check_display(index, payloads)
def check_display(self, index: int, payloads: dict[str, dict]) -> None:
"""Background, CLEAN objects and boxes against sar_prepare_display.
Steps at the noise level may pick a neighbouring A-scan sample where two samples are
equal to float32 precision; the reference scenes were checked to keep their decisions
with the kernels compiled without fused multiply-adds.
"""
background = self.reference.get(f"case_{index}_background")
if background is None:
self.assertNotIn("sar3d_background", payloads)
else:
np.testing.assert_allclose(payloads["sar3d_background"]["table"], background, rtol=2e-3, atol=0)
objects = self.reference.get(f"case_{index}_objects")
if objects is None:
self.assertNotIn("sar3d_objects", payloads)
self.assertNotIn("sar3d_marks", payloads)
else:
table = np.reshape(payloads["sar3d_objects"]["table"], (-1, 4))
np.testing.assert_allclose(table[:, :3], objects[:, :3], rtol=0, atol=2e-6)
np.testing.assert_allclose(table[:, 3], objects[:, 3], rtol=2e-3, atol=0)
np.testing.assert_allclose(np.reshape(payloads["sar3d_marks"]["table"], (-1, 7)),
self.reference[f"case_{index}_marks"], rtol=0, atol=2e-6)
clean = self.reference.get(f"case_{index}_clean")
if clean is not None:
message = status(payloads)[1]
match = re.search(r"\((\d+) subtractions?, residual maximum ([\d.]+) backgrounds\)", message)
assert match is not None, message
self.assertEqual(int(match[1]), clean[0])
self.assertAlmostEqual(float(match[2]), clean[1], delta=0.051)
def test_clean_needs_a_one_frame_aperture(self) -> None:
# The actual aperture decides, as in sar_clean_frame: three frames skip CLEAN, while the
# first frame of a run is a one-frame aperture even with SAR_FRAME_RADIUS = 2.
skipped = run(self.scene(clean=True))
code, message = status(skipped)
self.assertEqual(code, 0, message)
self.assertIn("CLEAN skipped: it needs a one-frame aperture, this one has 3 frames", message)
self.assertNotIn("sar3d_objects", skipped)
code, message = status(run(self.scene(clean=True, previous=[])))
self.assertEqual(code, 0, message)
self.assertIn("; CLEAN: ", message)
def test_boxes_follow_the_notebook_rules(self) -> None:
# Host code only, so it runs without a GPU as well.
request = json.loads(self.reference["marks_request"].item())
output = subprocess.run([str(FIXTURE)], input=json.dumps(request), text=True,
capture_output=True, timeout=30, check=True)
np.testing.assert_allclose(json.loads(output.stdout)["marks"], self.reference["marks"], rtol=0, atol=1e-9)
def test_invalid_input_is_an_error_with_a_reason(self) -> None:
def no_traces(request): request["latest"]["traces"] = []
def empty_window(request): request["latest"]["traces"][0]["end"] = request["latest"]["traces"][0]["start"]
def falling_frequency(request): request["latest"]["traces"][0]["frequency"].reverse()
def overflow(request):
for trace in request["latest"]["traces"]:
samples = np.asarray(trace["s21"])
trace["s21"] = (samples / np.abs(samples).max() * 1e38).tolist()
for name, settings, damage in (
("no traces", {}, no_traces),
("empty capture window", {}, empty_window),
("falling frequency", {}, falling_frequency),
("grid beyond 32-bit indexing", {"voxel": 1e-8}, None),
("volume beyond float32", {"previous": [], "radius": 0, "normalize": False, "equalize": False}, overflow),
("grid beyond 32-bit indexing", {"voxel": [1e-8] * 3}, None),
("vertical look angle", {"pitch": 90}, None),
("result larger than the ring slot", {"slot": 1024}, None),
):
@@ -117,6 +177,20 @@ class Sar3dTest(unittest.TestCase):
self.assertEqual(code, 2, message)
self.assertTrue(message)
def test_views_are_normalized_at_the_peak_of_the_reference_planes(self) -> None:
# Fine grids spread every Y plane over all threads of the per-plane maximum.
for voxel in (.05, .06, .07, .08, .09, .11):
with self.subTest(voxel=voxel):
payloads = run(self.scene(voxel=[voxel] * 3))
self.assertEqual(status(payloads)[0], 0, status(payloads))
front = payloads["sar3d_front"]
x, y = np.asarray(front["x"]), np.asarray(front["y"])
image = np.where(((y >= y[0] + .3) & (y <= y[-1] - .3))[:, None],
np.reshape(front["image"], (len(y), len(x))), -1.)
row, column = np.unravel_index(np.argmax(image), image.shape)
self.assertAlmostEqual(float(image[row, column]), 1., places=5)
np.testing.assert_allclose(payloads["sar3d_peak"]["table"][1:3], [x[column], y[row]], rtol=0, atol=1e-6)
def test_cached_aperture_updates_when_previous_mean_context_changes(self) -> None:
request = self.scene(subtract_mean=True, mean_count=1)
shorter = copy.deepcopy(request)
@@ -133,6 +207,7 @@ class Sar3dTest(unittest.TestCase):
payloads = run(request)
self.assertEqual(status(payloads)[0], 1)
self.assertNotIn("sar3d_peak", payloads)
self.assertNotIn("sar3d_background", payloads)
for name in VIEWS:
self.assertFalse(np.any(payloads[name]["image"]))