Rework radar system
This commit is contained in:
Ayzen
@@ -1,33 +1,20 @@
|
||||
"""Object-based MIMO S21 corrections estimated after standard preprocessing.
|
||||
|
||||
Archives retain the standalone script's cal_*.npy/cal_meta.json layout. Pair
|
||||
corrections multiply S21 after through calibration, reference subtraction and
|
||||
notch filtering; they never alter S11 or the original measurement files.
|
||||
"""
|
||||
"""Estimate object-based MIMO S21 factors after through/reference preprocessing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from collections.abc import Callable, Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import asdict, dataclass
|
||||
import copy
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from numpy.typing import ArrayLike
|
||||
|
||||
from python_app.file_io import atomic_write_json
|
||||
from python_app.models.dataset_model import SweepCollection
|
||||
from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData
|
||||
from python_app.storage.npz.serialize import validate_trace_collection
|
||||
from python_app.workflows.object_calibration import geometry_signature
|
||||
|
||||
C_LIGHT = 299_792_458.0
|
||||
_CAL_ARRAYS = ('freq_hz', 'pair', 'tx', 'rx', 'valid', 'tx_output_pos', 'rx_input_pos', 'antenna_params')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _ports(values, label):
|
||||
@@ -59,7 +46,7 @@ def _frequency(value, *, uniform=False):
|
||||
|
||||
@dataclass
|
||||
class Settings:
|
||||
"""Target geometry and estimation options, independent of example globals."""
|
||||
"""Reflector geometry and phase-estimation settings."""
|
||||
sphere_radius_m: float = .16
|
||||
input_idx: tuple[int, ...] = (0, 1, 2, 3)
|
||||
output_idx: tuple[int, ...] = (0, 1, 2, 3)
|
||||
@@ -95,34 +82,10 @@ class Settings:
|
||||
raise ValueError('apply_amplitude must be boolean')
|
||||
|
||||
|
||||
def find_config_profile(start: str | Path, max_levels: int = 8) -> Path:
|
||||
"""Locate the archived acquisition profile above a stage or snapshot folder."""
|
||||
path = Path(start)
|
||||
for _ in range(max_levels):
|
||||
candidate = path / 'config_profile.json'
|
||||
if candidate.is_file():
|
||||
return candidate
|
||||
path = path.parent
|
||||
raise FileNotFoundError(f'config_profile.json not found above {start}')
|
||||
|
||||
|
||||
def _profile(path):
|
||||
config = json.loads(Path(path).read_text(encoding='utf-8'))
|
||||
if not isinstance(config, dict):
|
||||
raise ValueError('config_profile.json must contain an object')
|
||||
return config
|
||||
|
||||
|
||||
def _geometry(config, input_idx, output_idx):
|
||||
result = []
|
||||
geometry, result = geometry_signature(config), []
|
||||
for key, identity, ports in (('tx_geometry', 'output_pos', output_idx), ('rx_geometry', 'input_pos', input_idx)):
|
||||
entries = config.get('gpr', {}).get(key, [])
|
||||
mapping = {}
|
||||
for entry in entries:
|
||||
port = _ports([entry[identity]], identity)[0]
|
||||
if port in mapping:
|
||||
raise ValueError(f'duplicate antenna geometry for {identity}={port}')
|
||||
mapping[port] = _finite_array([entry['x_m'], entry.get('y_m', 0.), entry.get('z_m', 0.)], key, (3,))
|
||||
mapping = {entry[identity]: [entry['x_m'], entry['y_m'], entry['z_m']] for entry in geometry[key]}
|
||||
missing = set(ports) - mapping.keys()
|
||||
if missing:
|
||||
raise ValueError(f'{key} is missing physical ports {sorted(missing)}')
|
||||
@@ -130,268 +93,8 @@ def _geometry(config, input_idx, output_idx):
|
||||
return tuple(result)
|
||||
|
||||
|
||||
def load_geometry(config_path: str | Path, input_idx: Sequence[int], output_idx: Sequence[int]) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""Read TX/RX radar coordinates in sorted physical port order."""
|
||||
return _geometry(_profile(config_path), sorted(_ports(input_idx, 'input_idx')), sorted(_ports(output_idx, 'output_idx')))
|
||||
|
||||
|
||||
def list_snapshots(root: str | Path, prefixes: Sequence[str] | None = None) -> list[Path]:
|
||||
"""Select complete snapshot directories, requiring unambiguous prefixes."""
|
||||
directories = sorted(path for path in Path(root).iterdir() if path.is_dir() and (path / 'meta.json').is_file())
|
||||
if prefixes is None:
|
||||
selected = directories
|
||||
else:
|
||||
selected = []
|
||||
for prefix in prefixes:
|
||||
matches = [path for path in directories if path.name.startswith(str(prefix))]
|
||||
if len(matches) != 1 or matches[0] in selected:
|
||||
raise ValueError(f'snapshot prefix {prefix!r} must select exactly one distinct snapshot in {root}')
|
||||
selected.extend(matches)
|
||||
if not selected:
|
||||
raise ValueError(f'no snapshots selected in {root}')
|
||||
return selected
|
||||
|
||||
|
||||
def load_s21(snapshot_dir: str | Path, input_idx: Sequence[int], output_idx: Sequence[int]) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""Read a finite TX×RX×frequency matrix on one shared uniform grid."""
|
||||
inputs = sorted(_ports(input_idx, 'input_idx'))
|
||||
outputs = sorted(_ports(output_idx, 'output_idx'))
|
||||
rows, freq = [], None
|
||||
for output in outputs:
|
||||
row = []
|
||||
for input_ in inputs:
|
||||
suffix = f'i{input_}_o{output}'
|
||||
samples = np.load(Path(snapshot_dir) / f'{suffix}_s21.npy', allow_pickle=False)
|
||||
axis = _frequency(np.load(Path(snapshot_dir) / f'{suffix}_freq.npy', allow_pickle=False), uniform=True)
|
||||
if freq is not None and not np.array_equal(axis, freq):
|
||||
raise ValueError(f'frequency grids differ between pairs in {snapshot_dir}')
|
||||
freq = axis
|
||||
row.append(_finite_array(samples, suffix, freq.shape, complex_values=True))
|
||||
rows.append(row)
|
||||
assert freq is not None
|
||||
return np.array(rows), freq
|
||||
|
||||
|
||||
def _source_signature(config) -> dict[str, Any]:
|
||||
radar = config.get('radar', {})
|
||||
s21 = config.get('preprocess', {}).get('s21', {})
|
||||
return {
|
||||
'radar': {key: radar[key] for key in ('model', 'serial') if key in radar},
|
||||
'slave_serials': radar.get('multi_device', {}).get('slave_serials', []),
|
||||
's21_calibration_set': s21.get('calibration', {}).get('set_name', ''),
|
||||
's21_reference_set': s21.get('reference', {}).get('set_name', ''),
|
||||
'notch': config.get('preprocess', {}).get('notch', {}),
|
||||
}
|
||||
|
||||
|
||||
def validate_calibration(cal: dict[str, Any]) -> None:
|
||||
"""Validate an archive before saving, applying or exporting runtime factors."""
|
||||
freq = _frequency(cal['freq_hz'], uniform=True)
|
||||
tx = _ports(cal['tx_output_pos'], 'tx_output_pos')
|
||||
rx = _ports(cal['rx_input_pos'], 'rx_input_pos')
|
||||
if tx != tuple(sorted(tx)) or rx != tuple(sorted(rx)):
|
||||
raise ValueError('calibration physical ports must be sorted')
|
||||
T = _finite_array(cal['tx'], 'tx corrections', (len(tx), freq.size), complex_values=True)
|
||||
R = _finite_array(cal['rx'], 'rx corrections', (len(rx), freq.size), complex_values=True)
|
||||
pair = _finite_array(cal['pair'], 'pair corrections', (len(tx), len(rx), freq.size), complex_values=True)
|
||||
if np.any(np.abs(T) == 0) or np.any(np.abs(R) == 0) or np.any(np.abs(pair) == 0):
|
||||
raise ValueError('calibration corrections must be nonzero')
|
||||
if not np.allclose(pair, T[:, None, :] * R[None, :, :], rtol=2e-5, atol=1e-7):
|
||||
raise ValueError('pair corrections disagree with tx/rx corrections')
|
||||
valid = np.asarray(cal['valid'])
|
||||
if valid.dtype.kind != 'b' or valid.shape != freq.shape or valid.sum() < 2:
|
||||
raise ValueError('calibration valid mask must select at least two frequencies')
|
||||
_finite_array(cal['antenna_params'], 'antenna parameters', (len(tx) + len(rx), 3))
|
||||
geometry = cal['meta']['geometry']
|
||||
_finite_array(geometry['tx_xyz_m'], 'TX geometry', (len(tx), 3))
|
||||
_finite_array(geometry['rx_xyz_m'], 'RX geometry', (len(rx), 3))
|
||||
|
||||
|
||||
def validate_for_config(cal: dict[str, Any], config: dict[str, Any]) -> None:
|
||||
"""Require physical geometry, combo coverage and a compatible phase base."""
|
||||
validate_calibration(cal)
|
||||
tx, rx = _geometry(config, cal['rx_input_pos'], cal['tx_output_pos'])
|
||||
geometry = cal['meta']['geometry']
|
||||
if not (np.allclose(tx, geometry['tx_xyz_m'], rtol=0, atol=1e-9) and
|
||||
np.allclose(rx, geometry['rx_xyz_m'], rtol=0, atol=1e-9)):
|
||||
raise ValueError('antenna geometry differs from the object calibration')
|
||||
for combo in config.get('run', {}).get('combos', []):
|
||||
input_, output = _ports([combo['input']], 'combo input')[0], _ports([combo['output']], 'combo output')[0]
|
||||
if input_ not in cal['rx_input_pos'] or output not in cal['tx_output_pos']:
|
||||
raise ValueError(f'object calibration does not cover combo {combo}')
|
||||
source = cal['meta'].get('source_preprocess', {})
|
||||
current = _source_signature(config)
|
||||
for key in ('radar', 'slave_serials', 's21_calibration_set'):
|
||||
if source.get(key) and current.get(key) and source[key] != current[key]:
|
||||
raise ValueError(f'object calibration source differs: {key}')
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _staged_directory(destination, overwrite, *, preserve_existing=False):
|
||||
destination = Path(destination)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
if destination.exists() and (not overwrite or not destination.is_dir() or destination.is_symlink()):
|
||||
raise FileExistsError(f'{destination} already exists; choose another name or overwrite=True')
|
||||
stage = Path(tempfile.mkdtemp(prefix=f'.{destination.name}.new-', dir=destination.parent))
|
||||
backup = None
|
||||
retain = False
|
||||
try:
|
||||
if preserve_existing and destination.exists():
|
||||
shutil.copytree(destination, stage, dirs_exist_ok=True, symlinks=True)
|
||||
yield stage
|
||||
if destination.exists():
|
||||
if not overwrite:
|
||||
raise FileExistsError(f'{destination} appeared during save; choose another name or overwrite=True')
|
||||
backup = Path(tempfile.mkdtemp(prefix=f'.{destination.name}.old-', dir=destination.parent))
|
||||
backup.rmdir()
|
||||
destination.replace(backup)
|
||||
try:
|
||||
stage.replace(destination)
|
||||
except BaseException:
|
||||
if backup is not None:
|
||||
try:
|
||||
backup.replace(destination)
|
||||
except OSError as error:
|
||||
retain = True
|
||||
raise RuntimeError(f'calibration rollback failed; originals retained at {backup}, new files at {stage}') from error
|
||||
raise
|
||||
finally:
|
||||
if not retain:
|
||||
for temporary in (stage, backup):
|
||||
if temporary is not None and temporary.exists():
|
||||
try:
|
||||
shutil.rmtree(temporary)
|
||||
except OSError:
|
||||
logger.exception('Could not remove calibration temporary directory %s', temporary)
|
||||
|
||||
|
||||
def save_calibration(cal: dict[str, Any], cal_dir: str | Path, overwrite: bool = False) -> Path:
|
||||
"""Commit a complete legacy archive, restoring its predecessor on failure."""
|
||||
validate_calibration(cal)
|
||||
destination = Path(cal_dir)
|
||||
if destination.is_dir() and any(destination.glob('cal_*')) and not overwrite:
|
||||
raise FileExistsError(f'{destination} already contains calibration files; use overwrite=True')
|
||||
meta: dict[str, Any] = dict(cal['meta'], name=destination.name)
|
||||
# Older one-target archives used nonstandard JSON NaN for unavailable LOO.
|
||||
quality = dict(meta.get('quality', {}))
|
||||
if 'coherence_leave_one_out' in quality:
|
||||
quality['coherence_leave_one_out'] = [value if value is None or np.isfinite(value) else None for value in quality['coherence_leave_one_out']]
|
||||
meta['quality'] = quality
|
||||
with _staged_directory(destination, overwrite or destination.is_dir(), preserve_existing=True) as stage:
|
||||
for key in _CAL_ARRAYS:
|
||||
path = stage / f'cal_{key}.npy'
|
||||
path.unlink(missing_ok=True)
|
||||
np.save(path, cal[key], allow_pickle=False)
|
||||
(stage / 'cal_meta.json').unlink(missing_ok=True)
|
||||
atomic_write_json(stage / 'cal_meta.json', meta)
|
||||
cal['meta'], cal['dir'] = meta, destination
|
||||
return destination
|
||||
|
||||
|
||||
def load_calibration(cal_dir: str | Path) -> dict[str, Any]:
|
||||
"""Load and validate cal_*.npy arrays plus cal_meta.json without pickle."""
|
||||
directory = Path(cal_dir)
|
||||
cal: dict[str, Any] = {key: np.load(directory / f'cal_{key}.npy', allow_pickle=False) for key in _CAL_ARRAYS}
|
||||
cal.update(meta=json.loads((directory / 'cal_meta.json').read_text(encoding='utf-8')), dir=directory)
|
||||
validate_calibration(cal)
|
||||
return cal
|
||||
|
||||
|
||||
def calibration_on_grid(cal: dict[str, Any], freq: ArrayLike) -> np.ndarray:
|
||||
"""Interpolate separable corrections and continue phase smoothly at edges."""
|
||||
validate_calibration(cal)
|
||||
freq = _frequency(freq)
|
||||
fc = np.asarray(cal['freq_hz'])
|
||||
if np.array_equal(freq, fc):
|
||||
return cal['pair']
|
||||
|
||||
def interp(correction, params):
|
||||
phase = np.unwrap(np.angle(correction))
|
||||
out = np.interp(freq, fc, phase)
|
||||
slope = 2 * np.pi * params[1] / C_LIGHT
|
||||
for selected, edge in ((freq < fc[0], 0), (freq > fc[-1], -1)):
|
||||
out[selected] = phase[edge] + slope * (freq[selected] - fc[edge])
|
||||
return np.interp(freq, fc, np.abs(correction)) * np.exp(1j * out)
|
||||
|
||||
n_tx = len(cal['tx_output_pos'])
|
||||
tx = np.array([interp(value, cal['antenna_params'][index]) for index, value in enumerate(cal['tx'])])
|
||||
rx = np.array([interp(value, cal['antenna_params'][n_tx + index]) for index, value in enumerate(cal['rx'])])
|
||||
corrections = (tx[:, None, :] * rx[None, :, :]).astype(np.asarray(cal['pair']).dtype)
|
||||
_finite_array(corrections, 'interpolated corrections', complex_values=True)
|
||||
return corrections
|
||||
|
||||
|
||||
def apply_calibration(s21: ArrayLike, freq: ArrayLike, cal: dict[str, Any]) -> np.ndarray:
|
||||
"""Multiply a TX×RX×frequency S21 matrix by validated pair corrections."""
|
||||
corrections = calibration_on_grid(cal, freq)
|
||||
samples = _finite_array(s21, 'S21', corrections.shape, complex_values=True)
|
||||
return _finite_array(samples * corrections, 'corrected S21', corrections.shape, complex_values=True)
|
||||
|
||||
|
||||
def apply_to_measurement(meas_dir: str | Path, cal: dict[str, Any], overwrite: bool = False) -> tuple[Path, int, int, int]:
|
||||
"""Stage corrected S21 and copied companion files in postprocessed safely."""
|
||||
directory = Path(meas_dir)
|
||||
pre, post = directory / 'preprocessed', directory / 'postprocessed'
|
||||
if not pre.is_dir():
|
||||
raise FileNotFoundError(f'no {pre}')
|
||||
config = _profile(find_config_profile(pre))
|
||||
_reject_corrected_source(config)
|
||||
validate_for_config(cal, config)
|
||||
if post.exists() and overwrite and not (post / 'calibration_applied.json').is_file():
|
||||
raise FileExistsError(f'{post} has no calibration_applied.json; refusing to replace it')
|
||||
tx = {int(port): index for index, port in enumerate(cal['tx_output_pos'])}
|
||||
rx = {int(port): index for index, port in enumerate(cal['rx_input_pos'])}
|
||||
pattern = re.compile(r'^i(\d+)_o(\d+)_s21\.npy$')
|
||||
n_snap = n_fix = n_copy = 0
|
||||
cache = {}
|
||||
with _staged_directory(post, overwrite) as stage:
|
||||
for item in sorted(pre.iterdir()):
|
||||
if item.is_file():
|
||||
shutil.copy2(item, stage / item.name)
|
||||
n_copy += 1
|
||||
continue
|
||||
out = stage / item.name
|
||||
out.mkdir()
|
||||
n_snap += 1
|
||||
for path in sorted(item.iterdir()):
|
||||
match = pattern.match(path.name)
|
||||
if path.is_dir() or match is None:
|
||||
if path.is_dir():
|
||||
shutil.copytree(path, out / path.name)
|
||||
else:
|
||||
shutil.copy2(path, out / path.name)
|
||||
n_copy += 1
|
||||
continue
|
||||
input_, output = int(match[1]), int(match[2])
|
||||
if input_ not in rx or output not in tx:
|
||||
raise ValueError(f'object calibration does not cover {path.name}')
|
||||
freq = _frequency(np.load(item / f'i{input_}_o{output}_freq.npy', allow_pickle=False))
|
||||
key = freq.tobytes()
|
||||
if key not in cache:
|
||||
cache[key] = calibration_on_grid(cal, freq)
|
||||
samples = np.load(path, allow_pickle=False)
|
||||
dtype = samples.dtype
|
||||
samples = _finite_array(samples, path.name, freq.shape, complex_values=True)
|
||||
corrected = (samples * cache[key][tx[output], rx[input_]]).astype(dtype)
|
||||
_finite_array(corrected, 'corrected S21', complex_values=True)
|
||||
np.save(out / path.name, corrected, allow_pickle=False)
|
||||
n_fix += 1
|
||||
atomic_write_json(stage / 'calibration_applied.json', dict(
|
||||
calibration=cal['meta'].get('name'), calibration_dir=str(cal.get('dir') or ''),
|
||||
created=datetime.datetime.now().isoformat(timespec='seconds'), source=str(pre),
|
||||
rule='postprocessed S21 = preprocessed S21 * cal_pair[tx(output_pos), rx(input_pos)]',
|
||||
frequency_grids=[np.frombuffer(key, dtype=np.float64).tolist() for key in cache]))
|
||||
return post, n_snap, n_fix, n_copy
|
||||
|
||||
|
||||
def _reject_corrected_source(config):
|
||||
object_calibration = config.get('preprocess', {}).get('s21', {}).get('object_calibration', {})
|
||||
if object_calibration.get('set_name') or object_calibration.get('bundle_path'):
|
||||
raise ValueError('calibration sources must have no object correction; object_calibration is already active')
|
||||
|
||||
|
||||
def _best_phases(Z):
|
||||
"""Maximize pair _coherence by alternating antenna phases, anchored at TX0/RX0."""
|
||||
"""Maximize pair coherence by alternating antenna phases, anchored at TX0/RX0."""
|
||||
t = np.zeros(Z.shape[0])
|
||||
r = np.zeros(Z.shape[1])
|
||||
for _ in range(100):
|
||||
@@ -613,20 +316,12 @@ def _mean_samples(samples, check, expected_freq=None):
|
||||
return mean, freq
|
||||
|
||||
|
||||
def _settings(settings):
|
||||
settings = settings or Settings()
|
||||
settings.__post_init__()
|
||||
return settings
|
||||
|
||||
|
||||
def _compute(sets, empty, freq, settings, config, config_path, check) -> dict[str, Any]:
|
||||
def _compute(sets, freq, settings, config, provenance, check) -> tuple[SweepCollection, dict[str, Any]]:
|
||||
tx, rx = _geometry(config, settings.input_idx, settings.output_idx)
|
||||
est = _Estimator(settings, tx, rx, freq, check)
|
||||
for index, record in enumerate(sets):
|
||||
check(f'Estimating target {index + 1}/{len(sets)}')
|
||||
data = record.pop('s21')
|
||||
if empty is not None:
|
||||
data = data - empty
|
||||
nominal = _finite_array(record['target_scene_m'], 'target_scene_m', (3,))
|
||||
echo, refined = est.refine_target(data, est.coarse_range(data, nominal))
|
||||
if not np.all(np.any(np.abs(echo[:, :, est.ok]) > 0, axis=2)):
|
||||
@@ -642,57 +337,46 @@ def _compute(sets, empty, freq, settings, config, config_path, check) -> dict[st
|
||||
antennas.update({f'Rx{index}': est.antenna_params(value) for index, value in enumerate(R_all)})
|
||||
T_full = np.array([est.to_full_grid(value, antennas[f'Tx{index}']) for index, value in enumerate(T_all)])
|
||||
R_full = np.array([est.to_full_grid(value, antennas[f'Rx{index}']) for index, value in enumerate(R_all)])
|
||||
unit = (lambda value: np.conj(value) / np.abs(value)) if not settings.apply_amplitude else (lambda value: 1 / value)
|
||||
saved_pair = unit(T_full[:, None, :] * R_full[None, :, :]).astype(np.complex64)
|
||||
quality = dict(coherence_before=[], coherence_own=[], coherence_leave_one_out=[], coherence_final=[])
|
||||
saved_pair = est.pair_correction(T_full, R_full).astype(np.complex64)
|
||||
_finite_array(saved_pair, 'pair corrections', complex_values=True)
|
||||
diagnostics = []
|
||||
final_correction = saved_pair[:, :, est.band]
|
||||
for record in sets:
|
||||
check()
|
||||
before = est.broadband(record['E'])
|
||||
after = est.broadband(record['E'] * final_correction)
|
||||
quality['coherence_before'].append(_coherence(before))
|
||||
quality['coherence_own'].append(_coherence(est.broadband(record['E'] * est.pair_correction(record['T'], record['R']))))
|
||||
others = [other for other in sets if other is not record]
|
||||
quality['coherence_leave_one_out'].append(
|
||||
_coherence(est.broadband(record['E'] * est.pair_correction(*_combine(others)))) if others else None)
|
||||
quality['coherence_final'].append(_coherence(after))
|
||||
diagnostics.append(dict(phase_before_deg=np.degrees(np.angle(before)).tolist(),
|
||||
phase_after_deg=np.degrees(np.angle(after)).tolist(),
|
||||
# Notebook cell 5: each map uses its own summed contribution as phase zero.
|
||||
phase_before = (np.degrees(np.angle(before) - np.angle(before.sum())) + 180) % 360 - 180
|
||||
phase_after = (np.degrees(np.angle(after) - np.angle(after.sum())) + 180) % 360 - 180
|
||||
diagnostics.append(dict(phase_before_deg=phase_before.tolist(),
|
||||
phase_after_deg=phase_after.tolist(),
|
||||
phase_std_before_deg=float(np.std(phase_before)),
|
||||
phase_std_after_deg=float(np.std(phase_after)),
|
||||
coherence_before=_coherence(before), coherence_after=_coherence(after)))
|
||||
|
||||
valid = np.zeros(freq.size, bool)
|
||||
valid[np.flatnonzero(est.band)[est.ok]] = True
|
||||
meta = dict(
|
||||
created=datetime.datetime.now().isoformat(timespec='seconds'), name=None,
|
||||
convention=dict(apply='S21_corrected[tx, rx, f] = S21[tx, rx, f] * cal_pair[tx, rx, f]',
|
||||
index_order='sorted physical tx_output_pos x rx_input_pos',
|
||||
phase_reference='relative to Tx0 and Rx0; common phase/delay is retained',
|
||||
amplitude='equalized' if settings.apply_amplitude else 'phase only',
|
||||
outside_band='linear antenna phase continuation anchored at the band edge'),
|
||||
geometry=dict(config=str(config_path), tx_xyz_m=tx.tolist(), rx_xyz_m=rx.tolist(),
|
||||
antenna_height_m=settings.antenna_height_m, pitch_deg=settings.pitch_deg),
|
||||
estimation=dict(band_hz=[settings.f_min, settings.f_max], gate_half_m=settings.gate_half_m,
|
||||
gate_taper_m=settings.gate_taper_m, coarse_range_search_m=settings.coarse_range_search_m,
|
||||
refine_axes=list(settings.refine_axes), refine_range_m=settings.refine_range_m,
|
||||
phase_model=settings.phase_model, sphere_radius_m=settings.sphere_radius_m),
|
||||
source_preprocess=_source_signature(config),
|
||||
targets=[dict(path=record.get('path', ''), snapshots=record.get('snapshots', []),
|
||||
nominal_scene_m=record['q_nominal'].tolist(), refined_scene_m=record['q'].tolist()) for record in sets],
|
||||
antennas=antennas, quality=quality, diagnostics=diagnostics)
|
||||
cal = dict(freq_hz=freq.astype(np.float64), pair=saved_pair,
|
||||
tx=unit(T_full).astype(np.complex64), rx=unit(R_full).astype(np.complex64), valid=valid,
|
||||
tx_output_pos=np.array(settings.output_idx, np.int32), rx_input_pos=np.array(settings.input_idx, np.int32),
|
||||
antenna_params=np.array([[np.radians(value['phase0_deg']), value['delay_path_m'], value['gain_db']]
|
||||
for value in antennas.values()]), meta=meta, dir=None)
|
||||
validate_calibration(cal)
|
||||
metadata = dict(
|
||||
created=datetime.datetime.now().isoformat(timespec='seconds'),
|
||||
provenance=copy.deepcopy(provenance),
|
||||
tx_output_pos=list(settings.output_idx), rx_input_pos=list(settings.input_idx),
|
||||
geometry=dict(tx_xyz_m=tx.tolist(), rx_xyz_m=rx.tolist()),
|
||||
settings=asdict(settings),
|
||||
source_preprocess=dict(reference_name=config.get('preprocess', {}).get('s21', {}).get('reference', {}).get('set_name', ''),
|
||||
notch=copy.deepcopy(config.get('preprocess', {}).get('notch', {}))),
|
||||
targets=[dict(captures=record['captures'], nominal_scene_m=record['q_nominal'].tolist(),
|
||||
refined_scene_m=record['q'].tolist()) for record in sets],
|
||||
antennas=antennas, diagnostics=diagnostics,
|
||||
)
|
||||
frequency = freq.astype(np.float32)
|
||||
collection = SweepCollection(0, 0, [
|
||||
TraceData(ComboKey(input_, output), frequency, saved_pair[t, r])
|
||||
for t, output in enumerate(settings.output_idx) for r, input_ in enumerate(settings.input_idx)
|
||||
])
|
||||
validate_trace_collection(collection, require_finite=True)
|
||||
check('Object calibration complete')
|
||||
return cal
|
||||
return collection, metadata
|
||||
|
||||
|
||||
def _collection_samples(collection, settings):
|
||||
from python_app.storage.npz.serialize import validate_trace_collection
|
||||
|
||||
validate_trace_collection(collection, require_finite=True)
|
||||
traces = {(trace.combo.input, trace.combo.output): trace for trace in collection.traces}
|
||||
rows, freq = [], None
|
||||
@@ -712,81 +396,39 @@ def _collection_samples(collection, settings):
|
||||
|
||||
|
||||
def compute_from_collections(
|
||||
cal_sets: Iterable[dict[str, Any]], config_dict: dict[str, Any],
|
||||
empty_collections: Iterable[SweepCollection] | None = None, settings: Settings | None = None,
|
||||
*, progress: Callable[[str], None] | None = None, cancelled: Callable[[], bool] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Estimate corrections from standard-preprocessed sweeps without object correction.
|
||||
cal_sets: Iterable[dict[str, Any]], config_dict: dict[str, Any], settings: Settings | None = None,
|
||||
*, provenance: dict[str, Any], progress: Callable[[str], None] | None = None,
|
||||
cancelled: Callable[[], bool] | None = None,
|
||||
) -> tuple[SweepCollection, dict[str, Any]]:
|
||||
"""Estimate a saved set from preprocessed sweeps without object correction.
|
||||
|
||||
cal_sets contains {collections: [SweepCollection, ...], target_scene_m: XYZ}.
|
||||
progress accepts a status string; cancelled returns bool and raises
|
||||
InterruptedError promptly in data, refinement and factorization loops.
|
||||
Each target provides ``collections`` and ground-relative ``target_scene_m``.
|
||||
Reference subtraction has already been applied by the live preprocessor.
|
||||
Cancellation is checked during averaging, refinement and factorization.
|
||||
"""
|
||||
settings = _settings(settings)
|
||||
_reject_corrected_source(config_dict)
|
||||
settings = Settings() if settings is None else settings
|
||||
check = _callbacks(progress, cancelled)
|
||||
sets = list(cal_sets)
|
||||
if not sets:
|
||||
raise ValueError('at least one reflector target is required')
|
||||
empty, freq = None, None
|
||||
if empty_collections is not None:
|
||||
check('Averaging empty scene')
|
||||
empty, freq = _mean_samples((_collection_samples(value, settings) for value in empty_collections), check)
|
||||
prepared = []
|
||||
prepared, freq = [], None
|
||||
for index, record in enumerate(sets):
|
||||
check(f'Averaging target {index + 1}/{len(sets)}')
|
||||
collections = list(record['collections'])
|
||||
mean, freq = _mean_samples((_collection_samples(value, settings) for value in collections), check, freq)
|
||||
prepared.append(dict(s21=mean, target_scene_m=record['target_scene_m'],
|
||||
snapshots=[str(value.collection_id) for value in collections]))
|
||||
return _compute(prepared, empty, freq, settings, config_dict, 'live capture', check)
|
||||
|
||||
|
||||
def compute_calibration(
|
||||
cal_sets: Iterable[dict[str, Any]], empty_scene_path: str | Path | None = None, settings: Settings | None = None,
|
||||
*, progress: Callable[[str], None] | None = None, cancelled: Callable[[], bool] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Legacy archive API backed by the same estimator as live collection capture."""
|
||||
settings = _settings(settings)
|
||||
check = _callbacks(progress, cancelled)
|
||||
cal_sets = list(cal_sets)
|
||||
if not cal_sets:
|
||||
raise ValueError('at least one reflector target is required')
|
||||
config_path = find_config_profile(cal_sets[0]['path'])
|
||||
config = _profile(config_path)
|
||||
_reject_corrected_source(config)
|
||||
geometry = _geometry(config, settings.input_idx, settings.output_idx)
|
||||
signature = _source_signature(config)
|
||||
|
||||
def samples(directories):
|
||||
for directory in directories:
|
||||
check()
|
||||
profile_path = find_config_profile(directory)
|
||||
path = Path(directory)
|
||||
while True:
|
||||
if (path / 'calibration_applied.json').is_file():
|
||||
raise ValueError(f'object calibration is already applied to archive {directory}')
|
||||
if path == profile_path.parent:
|
||||
break
|
||||
path = path.parent
|
||||
source = _profile(profile_path)
|
||||
_reject_corrected_source(source)
|
||||
current = _geometry(source, settings.input_idx, settings.output_idx)
|
||||
if any(not np.allclose(first, second, rtol=0, atol=1e-9) for first, second in zip(geometry, current)):
|
||||
raise ValueError(f'antenna geometry differs in {directory}')
|
||||
if signature != _source_signature(source):
|
||||
raise ValueError(f'preprocessing or hardware configuration differs in {directory}')
|
||||
yield load_s21(directory, settings.input_idx, settings.output_idx)
|
||||
|
||||
empty, freq = None, None
|
||||
if empty_scene_path is not None:
|
||||
empty, freq = _mean_samples(samples(list_snapshots(empty_scene_path)), check)
|
||||
prepared = []
|
||||
for record in cal_sets:
|
||||
directories = list_snapshots(record['path'], record.get('snapshots'))
|
||||
mean, freq = _mean_samples(samples(directories), check, freq)
|
||||
prepared.append(dict(s21=mean, target_scene_m=record['target_scene_m'], path=str(record['path']),
|
||||
snapshots=[directory.name for directory in directories]))
|
||||
cal = _compute(prepared, empty, freq, settings, config, config_path, check)
|
||||
cal['meta']['estimation']['empty_scene'] = str(empty_scene_path) if empty_scene_path is not None else None
|
||||
return cal
|
||||
prepared.append(dict(s21=mean, target_scene_m=record['target_scene_m'], captures=[
|
||||
dict(collection_id=value.collection_id, monotonic_ns=value.monotonic_ns,
|
||||
capture_start_ns=value.capture_start_ns, capture_end_ns=value.capture_end_ns)
|
||||
for value in collections
|
||||
]))
|
||||
collection, metadata = _compute(prepared, freq, settings, config_dict, provenance, check)
|
||||
# Multi-device acquisition keeps every stage for timing. Pairs outside the
|
||||
# configured geometry carry identity factors, explicitly marked in the set.
|
||||
corrected = {trace.combo for trace in collection.traces}
|
||||
unchanged = [trace for trace in collections[0].traces if trace.combo not in corrected]
|
||||
collection.traces.extend(TraceData(
|
||||
trace.combo, np.asarray(trace.frequency_hz, dtype=np.float32),
|
||||
np.ones(trace.frequency_hz.size, dtype=np.complex64),
|
||||
) for trace in unchanged)
|
||||
metadata['unchanged_combos'] = [dict(input=trace.combo.input, output=trace.combo.output) for trace in unchanged]
|
||||
return collection, metadata
|
||||
|
||||
Reference in New Issue
Block a user