"""Generate small SAR3D references by executing notebook cells 0 and 4. Requires the original, untracked Multi_antenna_3D_clean_v1.ipynb; a missing notebook is an error. NumPy/SciPy execute its function/class ASTs unchanged. Only parameter values are replaced; plotting imports and the interactive top-level runs are excluded. No production SAR implementation or hand-transcribed oracle is used. Run from the repository root: .venv/bin/python data_acq_and_processing/tests/generate_sar3d_reference.py """ from __future__ import annotations import argparse import ast import hashlib import json from pathlib import Path import tempfile import numpy as np import scipy ROOT = Path(__file__).resolve().parents[2] REFERENCE = Path(__file__).with_name("sar3d_reference.npz") def notebook_functions(notebook: Path, parameters: dict) -> tuple[dict, dict]: cells = json.loads(notebook.read_text(encoding="utf-8"))["cells"] namespace = {"__name__": __name__} hashes = {} for index in (0, 4): source = "".join(cells[index]["source"]) hashes[str(index)] = hashlib.sha256(source.encode()).hexdigest() tree = ast.parse(source, filename=f"{notebook.name}:cell{index}") definitions = [] for node in tree.body: if isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.ImportFrom)): definitions.append(node) elif isinstance(node, ast.Import): if not any(alias.name.startswith(("matplotlib", "plotly")) for alias in node.names): definitions.append(node) elif isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): try: namespace[node.targets[0].id] = ast.literal_eval(node.value) except (ValueError, TypeError): pass # Dataset loading and the interactive run are supplied below. namespace.update(parameters) exec(compile(ast.Module(definitions, type_ignores=[]), f"{notebook.name}:cell{index}", "exec", dont_inherit=True), namespace) return namespace, hashes def scene() -> dict: """Deterministic two-echo input with motion, vertical baselines and four pairs.""" request = dict( tx=[[0, -.4, 0, 0], [1, .45, .2, .05]], rx=[[0, -.2, .1, 0], [1, .3, .35, .02]], f_start=1e9, f_stop=4e9, min_depth=1., max_depth=3., speed=.4, pitch=13., ground=.3, voxel=.4, radius=1, power=1.5, compensation=True, normalize=True, ) # Match the float32 settings/geometry accepted by the native fixture. for key in ("tx", "rx"): request[key] = np.asarray(request[key], dtype=np.float32).astype(float).tolist() frequency = np.linspace(1e9, 4e9, 201, dtype=np.float32) members = [] for frame, offset in enumerate((0, 180_000_000, 510_000_000)): traces = [] start = 10_000_000_000 + offset for tx in range(2): for rx in range(2): samples = ((1 + .1 * rx) * np.exp(-2j * np.pi * frequency.astype(float) * (4.8 + .08 * tx - .03 * frame) / 3e8) + .35 * np.exp(-2j * np.pi * frequency.astype(float) * (5.6 - .05 * rx) / 3e8)).astype(np.complex64) traces.append(dict(input=rx, output=tx, start=start, end=start + 165_000_000, frequency=frequency.tolist(), s21=np.column_stack((samples.real, samples.imag)).tolist())) members.append(dict(id=frame + 1, timestamp=start + 165_000_000, traces=traces)) request.update(previous=members[:-1], latest=members[-1]) return request def write_dataset(directory: Path, request: dict) -> list[Path]: """Supply the notebook's original config_profile/meta/NPY input contract.""" profile = {"gpr": {}, "run": {"combos": []}} for kind, position in (("tx", "output_pos"), ("rx", "input_pos")): profile["gpr"][f"{kind}_geometry"] = [ {position: int(index), "x_m": x, "y_m": y, "z_m": z} for index, x, y, z in request[kind] ] profile["run"]["combos"] = [ {"input": trace["input"], "output": trace["output"]} for trace in request["latest"]["traces"] ] (directory / "config_profile.json").write_text(json.dumps(profile), encoding="utf-8") snapshots = [] for index, member in enumerate([*request["previous"], request["latest"]]): snapshot = directory / f'{index:04d}_id{member["id"]}_ns{member["timestamp"]}' snapshot.mkdir() metadata = {"traces": []} for trace in member["traces"]: stem = f'i{trace["input"]}_o{trace["output"]}' values = np.asarray(trace["s21"], dtype=np.float32) np.save(snapshot / f"{stem}_s21.npy", values[:, 0] + 1j * values[:, 1]) np.save(snapshot / f"{stem}_freq.npy", np.asarray(trace["frequency"], dtype=np.float32)) metadata["traces"].append(dict(input=trace["input"], output=trace["output"], capture_start_ns=trace["start"], capture_end_ns=trace["end"])) (snapshot / "meta.json").write_text(json.dumps(metadata), encoding="utf-8") snapshots.append(snapshot) return snapshots def reference(notebook: Path, request: dict, directory: Path) -> tuple[dict, dict]: snapshots = write_dataset(directory, request) center = snapshots[max(len(snapshots) - 1 - request["radius"], 0)] parameters = dict( INPUT_IDX=[int(row[0]) for row in request["rx"]], OUTPUT_IDX=[int(row[0]) for row in request["tx"]], F_START=request["f_start"], F_STOP=request["f_stop"], MIN_DEPTH=request["min_depth"], MAX_DEPTH=request["max_depth"], PAIR_NORM_DEPTH_MIN=request["min_depth"], PAIR_NORM_DEPTH_MAX=request["max_depth"], SPEED_M_S=request["speed"], LOOK_ANGLE_DEG=request["pitch"], BP_GROUND_DEPTH_M=request["ground"], COMP_RANGE_POWER=request["power"], SAR_VOXEL_M=(request["voxel"],) * 3, SAR_FRAME_RADIUS=request["radius"], SAR_USE_COMPENSATION=request["compensation"], SAR_NORMALIZE_EACH_FRAME=request["normalize"], SAR_SCAN_ROOT=directory, DATA_PATH=center, BG_PATH=directory, v=3e8, ) ns, hashes = notebook_functions(notebook, parameters) tx, rx, order, tx_order, pair_count = ns["load_antenna_config_profile"]( directory / "config_profile.json", ns["INPUT_IDX"], ns["OUTPUT_IDX"]) ns["N_tx"], ns["N_rx"] = len(tx), len(rx) for kind, positions in (("tx", tx), ("rx", rx)): for axis, values in zip("xyz", ns["positions_to_xyz"](positions).T): ns[f"{axis}_{kind}"] = values antenna_x = np.concatenate((ns["x_tx"], ns["x_rx"])) ns["x_grid_bp"] = np.linspace(antenna_x.min() - 2, antenna_x.max() + 2, ns["NX_BP"]) sweep_time, _, _ = ns["estimate_sweep_time_from_meta"](directory, n_pairs=pair_count) config = ns["MotionConfig"]( speed_m_s=ns["SPEED_M_S"], look_angle_deg=ns["LOOK_ANGLE_DEG"], tx_sweep_time_s=sweep_time, frame_acq_time_s=sweep_time * pair_count, pair_order_phys=order, tx_order_full=tuple(tx_order), n_freq_hint=len(request["latest"]["traces"][0]["frequency"]), apply_pair_motion_correction=ns["APPLY_PAIR_MOTION_CORRECTION"], apply_freq_phase_correction=ns["APPLY_FREQ_PHASE_CORRECTION"], pair_motion_correction_mode=ns["PAIR_MOTION_CORRECTION_MODE"], motion_phase_sign=ns["MOTION_PHASE_SIGN"], ) timestamps, _ = ns["compute_pair_timestamps_for_snapshot"](center, config) result = ns["sar_compute_volume"](center, None, config, timestamps, verbose=False) volume, grid = result["volume"], result["grid"] iy, iz, ix = ns["sar_global_max_index"](volume, grid) expected = {axis: grid[axis] for axis in "xyz"} expected.update(front=volume.max(axis=1), side=volume.max(axis=2), top=volume.max(axis=0), peak=np.array([result["level"], grid["x"][ix], grid["y"][iy], grid["z"][iz]])) return expected, hashes def generate(notebook: Path, output: Path) -> None: # Read up front so a missing notebook cannot produce an incomplete artifact. notebook.read_text(encoding="utf-8") cases = [ dict(compensation=True, normalize=True), dict(compensation=True, normalize=False, speed=-.2, pitch=-7., ground=.5), dict(compensation=False, normalize=True, f_start=1.3e9, f_stop=3.7e9), dict(compensation=False, normalize=False, speed=0., pitch=0., radius=0), ] arrays, configurations = {}, [] for index, settings in enumerate(cases): request = scene() request.update(settings) for key in ("min_depth", "max_depth", "speed", "pitch", "ground", "voxel", "power"): request[key] = float(np.float32(request[key])) if index == 1: # A separate, unequal capture window per output instead of interleaving. for member in [*request["previous"], request["latest"]]: for trace in member["traces"]: if trace["output"] == 0: trace["end"] = trace["start"] + 80_000_000 else: trace["start"] += 95_000_000 with tempfile.TemporaryDirectory(prefix="sar3d_notebook_") as directory: expected, hashes = reference(notebook, request, Path(directory)) if index == 0: arrays["request"] = np.array(json.dumps(request, separators=(",", ":"))) configurations.append(dict( settings={key: request[key] for key in settings}, windows=[[[trace["start"], trace["end"]] for trace in member["traces"]] for member in [*request["previous"], request["latest"]]], )) arrays.update({f"case_{index}_{key}": value for key, value in expected.items()}) arrays["cases"] = np.array(json.dumps(configurations, separators=(",", ":"))) arrays["provenance"] = np.array(json.dumps(dict( notebook=notebook.name, cell_source_sha256=hashes, cells=[0, 4], cases=len(cases), numpy=np.__version__, scipy=scipy.__version__, method="Unchanged notebook AST functions; max projections of sar_compute_volume output", ), sort_keys=True)) output.parent.mkdir(parents=True, exist_ok=True) np.savez_compressed(output, **arrays) print(f"Wrote {output}: {output.stat().st_size} bytes, {len(cases)} notebook references") if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--notebook", type=Path, default=ROOT / "Multi_antenna_3D_clean_v1.ipynb") parser.add_argument("--output", type=Path, default=REFERENCE) args = parser.parse_args() generate(args.notebook, args.output)