From aef549ea765635556d3459eff6de34116a2ab196 Mon Sep 17 00:00:00 2001 From: Ayzen Date: Tue, 6 Oct 2026 14:31:07 +0300 Subject: [PATCH] Remove dataset replay --- README.md | 41 +-- .../gui/controllers/pipeline_controller.py | 6 +- .../gui/controllers/preprocess_controller.py | 2 - python_app/models/run_config_codec.py | 13 - python_app/models/run_config_schema.py | 20 -- python_app/models/run_config_validation.py | 8 +- .../orchestration/process_supervisor.py | 1 - .../orchestration/producer_selection.py | 14 +- python_app/scripts/replay_raw_producer.py | 257 ------------------ python_app/tests/test_hardware_lifecycle.py | 38 +-- .../tests/test_object_calibration_runtime.py | 23 +- .../tests/test_orchestration_recovery.py | 13 - .../tests/test_persistence_integrity.py | 5 +- python_app/tests/test_run_config_codec.py | 7 +- .../run_config_replay_vna.example.json | 234 ---------------- 15 files changed, 32 insertions(+), 650 deletions(-) delete mode 100644 python_app/scripts/replay_raw_producer.py delete mode 100644 run_config_examples/run_config_replay_vna.example.json diff --git a/README.md b/README.md index cc39404..951c455 100644 --- a/README.md +++ b/README.md @@ -142,8 +142,8 @@ Run the desktop GUI through the launcher for build and single-instance handling: `start.sh` also supports `--headless` (offscreen, auto-start), `--auto-start`, `--producer-only`, `--skip-build`, `--build-only`, and `--clean-shm`; see `./start.sh --help`. Pass `--profile PATH` so launcher provisioning and the GUI -use the same profile. The producer is selected from `radar.model`, with dataset -replay taking precedence. There is no per-device flag. Provision dependencies +use the same profile. The producer is selected from `radar.model`; there is no +per-device flag. Provision dependencies before unattended use: headless startup does not install missing packages. For development, `.venv/bin/python -m python_app.gui.main` starts the GUI directly, @@ -154,12 +154,6 @@ without the launcher's build step or single-instance lock. `requirements.txt` pins PyQt6 below 6.8 on aarch64 for the older glibc used by Jetson images. CUDA comes from JetPack in `/usr/local/cuda`; pip CUDA packages are installed only on x86_64. Use the same build commands and launcher as above. -For dataset replay, first copy the dataset locally and update -`radar.replay.dataset_dir` in the profile; example paths are machine-specific: - -```bash -./start.sh --profile run_config_examples/run_config_replay_vna.example.json -``` The USB updater can create a fresh venv with an already installed interpreter, for example `bash radar_update/update.sh --python 3.13`. The system Python does @@ -167,12 +161,9 @@ not need to be replaced. ## Running without hardware (emulation) -Two ways to run the full pipeline with no radar attached: - -**Mock producer (synthetic data).** The default `run_config.json` already sets -`radar.driver_mode: "mock"`, so the producer generates synthetic S-parameters. -Mock acquisition supports the LibreVNA models; Kamil ADC uses native acquisition -or recorded-dataset replay without devices. +The default `run_config.json` already sets `radar.driver_mode: "mock"`, so the +producer generates synthetic S-parameters and no radar is needed. Mock acquisition +supports the LibreVNA models; Kamil ADC needs native acquisition. Both LibreVNA modes share one simulator: a point reflector moves back and forth along the antenna array and between depths 3.25–4.25 m over an eight-second cycle. Each S21 uses the actual transmitter–target–receiver distance from the configured @@ -194,26 +185,6 @@ The preprocessor still requires S21 calibration/reference sets. Start the GUI: Open **Preprocessing…**, use **Create Neutral S21 Sets**, then **Start**. This makes identity calibration + zero reference, preserving the simulated S21 samples. -**Dataset replay (recorded scene).** When -`radar.replay.dataset_dir` is set in the profile, the supervisor launches -`python_app.scripts.replay_raw_producer` instead of the normal producer: it -replays the collection directories in that folder into the raw ring. Point the -path at a saved stage such as `dataset/preprocessed`, containing -`NNNN_idK_ns/` directories with `meta.json` and `i*_o*_{freq,s21}.npy`. -Replay follows the recorded cadence scaled by `radar.replay.rate_scale`; -`radar.replay.loop` defaults to true. Trace windows preserve the recorded spacing -for SAR, even with accelerated replay. Each collection starts at its actual publication -time for acquisition identity and retains the original duration for legacy GPR. -For an already-preprocessed recording, select neutral through/reference sets -and disable notch and object correction to preserve the recorded S21 values. -A ready profile for the VNA_AlGo tank005 dataset is provided. Its neutral sets are -created once with the **Create Neutral S21 Sets** button of the preprocessing panel: - -```bash -# interactive GUI (mode sar3d is pre-selected by the profile's gui section): -./start.sh --profile run_config_examples/run_config_replay_vna.example.json -``` - Finer voxels increase SAR3D work as 1/`SAR_VOXEL_M`³. Increase the voxel size when a faster preview matters more than spatial resolution. Processing failures appear above the plots and a failed single capture @@ -518,4 +489,4 @@ GUI profiles, run configs, session state and live processing settings use shared atomic writers with unique temporary files. Profiles and run/session settings sync their file and directory; frequent live settings avoid extra disk syncs. A failed write before replacement leaves the previous file intact. The launcher and supervisor -share acquisition producer selection, with replay taking precedence over the model. +share acquisition producer selection. diff --git a/python_app/gui/controllers/pipeline_controller.py b/python_app/gui/controllers/pipeline_controller.py index 72ec218..9ff7f72 100644 --- a/python_app/gui/controllers/pipeline_controller.py +++ b/python_app/gui/controllers/pipeline_controller.py @@ -318,7 +318,7 @@ class PipelineController: validate_acquisition_config(config) processor_only = self.processor_running and not self.running with self.pause() as operation: - if config.radar.driver_mode == "native" and not config.radar.replay.dataset_dir: + if config.radar.driver_mode == "native": self._refresh_radar_limits() if not operation.valid: return @@ -340,8 +340,8 @@ class PipelineController: @staticmethod def _prepare_radar(config: RunConfigModel) -> None: - if config.radar.replay.dataset_dir or config.radar.driver_mode != "native": - logger.info("Radar pre-configuration skipped (replay or mock mode)") + if config.radar.driver_mode != "native": + logger.info("Radar pre-configuration skipped (mock mode)") elif config.is_multi_device: logger.info("Multi-device raw producer will configure all LibreVNA devices") elif config.is_kamil_adc: diff --git a/python_app/gui/controllers/preprocess_controller.py b/python_app/gui/controllers/preprocess_controller.py index b7e8e15..5f9a744 100644 --- a/python_app/gui/controllers/preprocess_controller.py +++ b/python_app/gui/controllers/preprocess_controller.py @@ -518,8 +518,6 @@ class PreprocessController: config = self._build_config() if not config.rings.preprocessed_tap.name: raise ValueError("Object calibration requires rings.preprocessed_tap; enable it in the run profile") - if config.radar.replay.dataset_dir.strip(): - raise ValueError("Object-calibration capture requires live acquisition; disable archive replay") config.preprocess.s21.object_calibration.set_name = "" config.preprocess.s21.object_calibration.bundle_path = "" tx, rx = sorted({entry.output_pos for entry in config.gpr.tx_geometry}), sorted({entry.input_pos for entry in config.gpr.rx_geometry}) diff --git a/python_app/models/run_config_codec.py b/python_app/models/run_config_codec.py index 03be625..4dffb08 100644 --- a/python_app/models/run_config_codec.py +++ b/python_app/models/run_config_codec.py @@ -106,7 +106,6 @@ def run_config_from_dict(payload: object) -> RunConfigModel: "run.locator_server", ) multi_device_payload = _as_dict(radar_payload.get("multi_device"), "radar.multi_device") - replay_payload = _as_dict(radar_payload.get("replay"), "radar.replay") kamil_adc_payload = _as_dict(radar_payload.get("kamil_adc"), "radar.kamil_adc") laser_control_payload = _as_dict(radar_payload.get("laser_control"), "radar.laser_control") @@ -163,13 +162,6 @@ def run_config_from_dict(payload: object) -> RunConfigModel: "stimulus_switch_positions", model.radar.multi_device.stimulus_switch_positions, ) - model.radar.replay.dataset_dir = _read_str( - replay_payload, "dataset_dir", model.radar.replay.dataset_dir - ) - model.radar.replay.loop = _read_bool(replay_payload, "loop", model.radar.replay.loop) - model.radar.replay.rate_scale = _read_float( - replay_payload, "rate_scale", model.radar.replay.rate_scale - ) model.radar.kamil_adc.project_dir = _read_str( kamil_adc_payload, "project_dir", model.radar.kamil_adc.project_dir ) @@ -443,11 +435,6 @@ def run_config_to_dict(model: RunConfigModel) -> dict[str, Any]: "input_switch_positions": model.radar.multi_device.input_switch_positions, "stimulus_switch_positions": model.radar.multi_device.stimulus_switch_positions }, - "replay": { - "dataset_dir": model.radar.replay.dataset_dir, - "loop": model.radar.replay.loop, - "rate_scale": model.radar.replay.rate_scale, - }, "kamil_adc": { "project_dir": model.radar.kamil_adc.project_dir, "executable_path": model.radar.kamil_adc.executable_path, diff --git a/python_app/models/run_config_schema.py b/python_app/models/run_config_schema.py index 160da42..d24e587 100644 --- a/python_app/models/run_config_schema.py +++ b/python_app/models/run_config_schema.py @@ -55,25 +55,6 @@ class RadarMultiDeviceModel: stimulus_switch_positions: int = 1 -@dataclass(slots=True) -class RadarReplayModel: - """Recorded-dataset replay settings (no hardware). - - When ``dataset_dir`` is non-empty the supervisor launches - ``python_app.scripts.replay_raw_producer`` instead of the model's normal - acquisition producer. The dataset is a directory of snapshot folders named - ``NNNN_idK_ns``, each holding ``i{input}_o{output}_s21.npy`` / - ``_freq.npy`` files (the VNA_AlGo preprocessed-capture layout). - """ - - dataset_dir: str = "" - # Restart from the first snapshot when the dataset ends; the replay - # timeline keeps advancing monotonically across the seam. - loop: bool = True - # Playback speed multiplier: 1.0 replays at the recorded frame cadence. - rate_scale: float = 1.0 - - @dataclass(slots=True) class KamilAdcPhaseCalibrationModel: """Affine law mapping the reference signal's unwrapped phase to frequency. @@ -177,7 +158,6 @@ class RadarModel: driver_mode: str = "mock" sweep: RadarSweepModel = field(default_factory=RadarSweepModel) multi_device: RadarMultiDeviceModel = field(default_factory=RadarMultiDeviceModel) - replay: RadarReplayModel = field(default_factory=RadarReplayModel) kamil_adc: KamilAdcModel = field(default_factory=KamilAdcModel) laser_control: LaserControlModel = field(default_factory=LaserControlModel) diff --git a/python_app/models/run_config_validation.py b/python_app/models/run_config_validation.py index 55f0b35..31f2d16 100644 --- a/python_app/models/run_config_validation.py +++ b/python_app/models/run_config_validation.py @@ -124,13 +124,7 @@ def validate_ring_endpoint(ring: RingEndpointModel) -> None: def validate_acquisition_config(config: RunConfigModel) -> None: - """Reject unusable acquisition settings before starting processes or hardware. - - Replay uses the recorded trace dimensions and needs no physical devices; its - producer checks the actual payload size before publishing a frame. - """ - if config.radar.replay.dataset_dir: - return + """Reject unusable acquisition settings before starting processes or hardware.""" mode = config.radar.driver_mode if mode not in ("mock", "native"): raise ValueError("radar.driver_mode must be mock or native") diff --git a/python_app/orchestration/process_supervisor.py b/python_app/orchestration/process_supervisor.py index 54ffe9c..6f221ee 100644 --- a/python_app/orchestration/process_supervisor.py +++ b/python_app/orchestration/process_supervisor.py @@ -556,7 +556,6 @@ class ProcessSupervisor: producer_modules = { "python_app.scripts.matrix_raw_producer", "python_app.scripts.kamil_adc_raw_producer", - "python_app.scripts.replay_raw_producer", } try: raw = (Path("/proc") / str(pid) / "cmdline").read_bytes() diff --git a/python_app/orchestration/producer_selection.py b/python_app/orchestration/producer_selection.py index c554f64..70c0169 100644 --- a/python_app/orchestration/producer_selection.py +++ b/python_app/orchestration/producer_selection.py @@ -11,21 +11,15 @@ from python_app.models.json_fields import json_object, read_str def acquisition_command(config_path: Path) -> list[str]: - """Select replay first, then the configured model's acquisition producer.""" + """Select the configured model's acquisition producer.""" payload = json_object(json.loads(config_path.read_text(encoding="utf-8")), "run config") radar = json_object(payload.get("radar"), "radar") - replay = json_object(radar.get("replay"), "radar.replay") - dataset = read_str(replay, "dataset_dir", "") model = read_str(radar, "model", RadarModel().model) producers = { "librevna": "matrix_raw_producer", "librevna_multi": "matrix_raw_producer", "kamil_adc": "kamil_adc_raw_producer", } - if dataset.strip(): - module = "replay_raw_producer" - else: - if model not in producers: - raise ValueError(f"Unsupported radar model: {model}") - module = producers[model] - return [sys.executable, "-m", f"python_app.scripts.{module}", "--config", str(config_path)] + if model not in producers: + raise ValueError(f"Unsupported radar model: {model}") + return [sys.executable, "-m", f"python_app.scripts.{producers[model]}", "--config", str(config_path)] diff --git a/python_app/scripts/replay_raw_producer.py b/python_app/scripts/replay_raw_producer.py deleted file mode 100644 index c8266e8..0000000 --- a/python_app/scripts/replay_raw_producer.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Raw producer that replays a recorded snapshot dataset (no hardware). - -Selected by the supervisor instead of the model's normal acquisition producer -when ``radar.replay.dataset_dir`` is non-empty. The dataset is a directory of -snapshot folders named ``NNNN_idK_ns`` (the VNA_AlGo -preprocessed-capture layout), each holding ``i{input}_o{output}_s21.npy`` and -``i{input}_o{output}_freq.npy`` files for every run combo. - -The recorded data is already background-subtracted, so the profile must pair -this producer with NEUTRAL preprocess sets (calibration = 1, reference = 0, -no notch) — then the C++ preprocessor passes the traces through bit-exactly -and the processors (e.g. sar3d) see the recorded scene. - -Timeline: snapshot timestamps are re-anchored to this process's monotonic -clock, preserving the recorded inter-frame gaps. On loop, the seam advances by -the median inter-frame gap, so motion-model consumers (sar3d aperture offsets) -never see time jump backwards. ``collection_id`` increases forever. -""" - -from __future__ import annotations - -from contextlib import ExitStack -import json -import logging -from pathlib import Path -import re -import statistics -import time - -import numpy as np - -from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData -from python_app.orchestration.shm.decoder import RAW_MAGIC -from python_app.scripts.producer_common import RawPublisher, load_producer_config -from python_app.storage.npz.serialize import serialize_trace_collection - -logger = logging.getLogger(__name__) - -_SNAPSHOT_NAME = re.compile(r"^(\d+)_id\d+_ns(\d+)$") - - -class ReplayDataset: - """Recorded snapshots preloaded into memory, with their recorded timeline.""" - - def __init__(self, snapshots: list[list[TraceData]], timestamps_ns: list[int]) -> None: - self.snapshots = snapshots - self.timestamps_ns = timestamps_ns - gaps = [b - a for a, b in zip(timestamps_ns, timestamps_ns[1:])] - # Median gap doubles as the loop-seam step and the capture-window span; - # a single-snapshot dataset falls back to an arbitrary sane cadence. - self.median_gap_ns = int(statistics.median(gaps)) if gaps else 160_000_000 - - def __len__(self) -> int: - return len(self.snapshots) - - -def _load_dataset(dataset_dir: Path, combos: list[ComboKey], expected_points: int) -> ReplayDataset: - """Preload every snapshot's traces; fail loudly on a malformed dataset.""" - if not dataset_dir.is_dir(): - raise RuntimeError(f"radar.replay.dataset_dir is not a directory: {dataset_dir}") - - named: list[tuple[int, str, int]] = [] - for entry in dataset_dir.iterdir(): - match = _SNAPSHOT_NAME.match(entry.name) - if entry.is_dir() and match: - named.append((int(match.group(1)), entry.name, int(match.group(2)))) - if not named: - raise RuntimeError( - f"No snapshot folders (NNNN_idK_ns) found in {dataset_dir}" - ) - - snapshots: list[list[TraceData]] = [] - timestamps_ns: list[int] = [] - for _, name, timestamp_ns in sorted(named): - folder = dataset_dir / name - # Recorded per-trace capture windows from the snapshot's meta.json: the - # sar3d segmented timing model reconstructs the acquisition segments - # from them. Missing/partial meta leaves the zero "unmeasured" sentinel - # and the consumer falls back to its uniform-segment model. - windows: dict[tuple[int, int], tuple[int, int]] = {} - meta_path = folder / "meta.json" - if meta_path.is_file(): - try: - meta = json.loads(meta_path.read_text()) - for entry in meta.get("traces", []): - windows[(int(entry["input"]), int(entry["output"]))] = ( - int(entry["capture_start_ns"]), - int(entry["capture_end_ns"]), - ) - except (OSError, ValueError, KeyError, TypeError): - windows = {} - traces: list[TraceData] = [] - for combo in combos: - s21_path = folder / f"i{combo.input}_o{combo.output}_s21.npy" - freq_path = folder / f"i{combo.input}_o{combo.output}_freq.npy" - if not s21_path.is_file() or not freq_path.is_file(): - raise RuntimeError( - f"Snapshot {name} is missing files for combo " - f"input={combo.input} output={combo.output}: expected " - f"{s21_path.name} and {freq_path.name}. Check run.combos vs the dataset." - ) - s21 = np.load(s21_path, allow_pickle=False).astype(np.complex64, copy=False) - freq = np.load(freq_path, allow_pickle=False).astype(np.float32, copy=False) - if freq.ndim != 1 or s21.shape != freq.shape: - raise RuntimeError(f"Snapshot {name}: frequency and S21 must be equally sized 1D arrays") - if expected_points and s21.size != expected_points: - raise RuntimeError( - f"{s21_path}: {s21.size} points, but radar.sweep.points = " - f"{expected_points}. Align the profile's sweep with the dataset." - ) - window = windows.get((combo.input, combo.output), (0, 0)) - traces.append( - TraceData( - combo=combo, - frequency_hz=freq, - s21=s21, - capture_start_ns=window[0], - capture_end_ns=window[1], - ) - ) - snapshots.append(traces) - timestamps_ns.append(timestamp_ns) - - return ReplayDataset(snapshots, timestamps_ns) - - -def _build_collection( - dataset: ReplayDataset, - index: int, - collection_id: int, - frame_ns: int, -) -> SweepCollection: - """Wrap one preloaded snapshot in a SweepCollection at the given frame time. - - Recorded per-trace capture windows are re-anchored by the same shift as the - frame time, preserving the real intra-frame segment layout. Traces without - recorded windows get a synthetic window spanning the median frame gap. - """ - span_ns = dataset.median_gap_ns - start_ns = frame_ns - span_ns // 2 - end_ns = frame_ns + span_ns // 2 - shift_ns = frame_ns - dataset.timestamps_ns[index] - traces = [] - for source in dataset.snapshots[index]: - if source.capture_end_ns > source.capture_start_ns: - capture = (source.capture_start_ns + shift_ns, source.capture_end_ns + shift_ns) - else: - capture = (start_ns, end_ns) - traces.append( - TraceData( - combo=source.combo, - frequency_hz=source.frequency_hz, - s21=source.s21, - capture_start_ns=capture[0], - capture_end_ns=capture[1], - ) - ) - return SweepCollection( - collection_id=collection_id, - monotonic_ns=frame_ns, - traces=traces, - capture_start_ns=min(t.capture_start_ns for t in traces), - capture_end_ns=max(t.capture_end_ns for t in traces), - ) - - -def main() -> int: - """Replay the configured dataset into the raw SHM rings until stopped.""" - _, config, stop_requested = load_producer_config("Replay a recorded dataset to raw SHM rings") - - replay = config.radar.replay - if not replay.dataset_dir: - raise RuntimeError("replay_raw_producer requires a non-empty radar.replay.dataset_dir") - rate_scale = float(replay.rate_scale) if replay.rate_scale > 0 else 1.0 - combos = [ComboKey(input=combo.input, output=combo.output) for combo in config.combos] - - # expanduser: profiles are copied between machines, so "~/..." dataset paths - # must resolve against the local home instead of failing as a literal "~". - dataset = _load_dataset( - Path(replay.dataset_dir).expanduser(), combos, int(config.radar.sweep.points) - ) - logger.info( - "Replay producer starting: dataset=%s, snapshots=%d, combos=%d, " - "median_gap=%.3f s, loop=%s, rate_scale=%.2f", - replay.dataset_dir, len(dataset), len(combos), - dataset.median_gap_ns / 1e9, replay.loop, rate_scale, - ) - - cleanup = ExitStack() - try: - publisher = RawPublisher(config.rings, cleanup) - - payload_size = len(serialize_trace_collection( - _build_collection(dataset, 0, 1, time.monotonic_ns()), RAW_MAGIC, require_capture_window=True, - )) - for name in ("raw", "raw_tap", "preprocessed", "preprocessed_tap"): - ring = getattr(config.rings, name) - if name.endswith("_tap") and not ring.name: - continue - if ring.slot_size_bytes < payload_size: - raise ValueError(f"Replay requires {payload_size} bytes per slot; ring {ring.name} has {ring.slot_size_bytes}") - - base_ns = time.monotonic_ns() - wall_start_s = time.monotonic() - first_ts = dataset.timestamps_ns[0] - cycle_span_ns = dataset.timestamps_ns[-1] - first_ts + dataset.median_gap_ns - - collection_id = 1 - cycle = 0 - while not stop_requested.is_set(): - for index in range(len(dataset)): - if stop_requested.is_set(): - break - # Monotonic replay timeline: recorded offsets re-anchored at - # base_ns, each loop shifted by one full cycle span. - frame_ns = ( - base_ns - + (dataset.timestamps_ns[index] - first_ts) - + cycle * cycle_span_ns - ) - # Publish on the recorded cadence (scaled); waiting via the stop - # event keeps shutdown prompt. - deadline_s = wall_start_s + (frame_ns - base_ns) / 1e9 / rate_scale - delay_s = deadline_s - time.monotonic() - if delay_s > 0 and stop_requested.wait(delay_s): - break - - collection = _build_collection(dataset, index, collection_id, frame_ns) - # Acquisition identity uses real monotonic time, even at accelerated - # replay rates. Keep the recorded duration for legacy GPR and the - # virtual per-trace timeline for SAR aperture/motion reconstruction. - duration_ns = collection.capture_end_ns - collection.capture_start_ns - collection.capture_start_ns = time.monotonic_ns() - collection.capture_end_ns = collection.capture_start_ns + duration_ns - if not publisher.publish(collection): - raise RuntimeError(f"Replay raw ring rejected collection {collection_id}") - if collection_id == 1 or collection_id % 50 == 0: - logger.info( - "replay collection %d published (snapshot %d/%d, cycle %d)", - collection_id, index + 1, len(dataset), cycle, - ) - collection_id += 1 - if not config.runtime.continuous: - stop_requested.set() - break - if not replay.loop: - break - cycle += 1 - finally: - cleanup.close() - - logger.info("replay raw producer stopped") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_app/tests/test_hardware_lifecycle.py b/python_app/tests/test_hardware_lifecycle.py index ee6b204..462cfac 100644 --- a/python_app/tests/test_hardware_lifecycle.py +++ b/python_app/tests/test_hardware_lifecycle.py @@ -28,9 +28,9 @@ from python_app.hardware_full.librevna_driver.transport.usb import USBTransport from python_app.hardware_full.switch_drivers.h7992_driver import H7992Driver from python_app.hardware_full.switched_matrix_radar_service import SwitchedMatrixRadarService from python_app.models.run_config_schema import RadarSweepModel, RunConfigModel -from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData +from python_app.models.dataset_model import SweepCollection from python_app.orchestration.shm.decoder import RAW_MAGIC, decode_trace_collection -from python_app.scripts import kamil_adc_raw_producer, matrix_raw_producer, replay_raw_producer, producer_common +from python_app.scripts import kamil_adc_raw_producer, matrix_raw_producer, producer_common from python_app.workflows.radar_config_variants import scan_radar_config_variants @@ -297,42 +297,16 @@ class TtyLifecycleTest(unittest.TestCase): class ProducerCleanupTest(unittest.TestCase): - def test_replay_rejects_small_slots_before_publishing_any_data(self) -> None: - module = replay_raw_producer - config = RunConfigModel() - config.radar.replay.dataset_dir = "/test/data" - config.rings.raw.slot_size_bytes = 1 - config.rings.raw_tap.name = "/test_raw_tap" - trace = TraceData( - ComboKey(0, 0), np.array([1, 2], dtype=np.float32), - np.array([1, 2], dtype=np.complex64), - ) - dataset = module.ReplayDataset([[trace]], [20]) - raw, tap = Mock(), Mock() - with patch.object(producer_common.RunConfigModel, "load_from_path", return_value=config), patch.object(producer_common.signal, "signal"), patch.object(module, "_load_dataset", return_value=dataset), patch.object(producer_common, "ShmRingWriter", side_effect=[raw, tap]), patch("sys.argv", [module.__name__, "--config", "/test/config.json"]): - with self.assertRaisesRegex(ValueError, "Replay requires .* bytes per slot"): - module.main() - raw.push.assert_not_called() - tap.push.assert_not_called() - raw.close.assert_called_once() - tap.close.assert_called_once() - def test_partial_ring_initialization_closes_first_writer(self) -> None: - for module, model in ((matrix_raw_producer, "librevna_multi"), (kamil_adc_raw_producer, "kamil_adc"), (replay_raw_producer, "librevna")): + for module, model in ((matrix_raw_producer, "librevna_multi"), (kamil_adc_raw_producer, "kamil_adc")): with self.subTest(producer=module.__name__): config = RunConfigModel() config.radar.model = model - config.radar.replay.dataset_dir = "/test/data" config.rings.raw_tap.name = "/test_raw_tap" writer = Mock() - with patch.object(producer_common.RunConfigModel, "load_from_path", return_value=config), patch.object(producer_common.signal, "signal"), patch.object(producer_common, "ShmRingWriter", side_effect=[writer, OSError("second ring failed")]), patch("sys.argv", [module.__name__, "--config", "/test/config.json"]): - if module is replay_raw_producer: - with patch.object(module, "_load_dataset", return_value=Mock(median_gap_ns=1, __len__=Mock(return_value=1))): - with self.assertRaisesRegex(OSError, "second ring failed"): - module.main() - else: - with self.assertRaisesRegex(OSError, "second ring failed"): - module.main() + with patch.object(producer_common.RunConfigModel, "load_from_path", return_value=config), patch.object(producer_common, "validate_acquisition_config"), patch.object(producer_common.signal, "signal"), patch.object(producer_common, "ShmRingWriter", side_effect=[writer, OSError("second ring failed")]), patch("sys.argv", [module.__name__, "--config", "/test/config.json"]): + with self.assertRaisesRegex(OSError, "second ring failed"): + module.main() writer.close.assert_called_once() def test_disabled_tap_opens_and_publishes_only_the_required_raw_ring(self) -> None: diff --git a/python_app/tests/test_object_calibration_runtime.py b/python_app/tests/test_object_calibration_runtime.py index e419238..59d9df1 100644 --- a/python_app/tests/test_object_calibration_runtime.py +++ b/python_app/tests/test_object_calibration_runtime.py @@ -119,20 +119,15 @@ class ObjectCaptureLifecycleTest(unittest.TestCase): capture._start() return capture - def test_missing_tap_and_replay_report_error_before_pausing(self) -> None: - for invalid in ("tap", "replay"): - with self.subTest(invalid=invalid): - self.config = _config() - if invalid == "tap": - self.config.rings.preprocessed_tap.name = "" - else: - self.config.radar.replay.dataset_dir = "/recorded" - self.controller.capture_data() - self.assertIsNone(self.controller.capture) - self.assertFalse(self.dialog._capture_pending) - self.stop.assert_not_called() - self.start.assert_not_called() - self.assertEqual(self.error.call_count, 2) + def test_missing_tap_reports_error_before_pausing(self) -> None: + self.config = _config() + self.config.rings.preprocessed_tap.name = "" + self.controller.capture_data() + self.assertIsNone(self.controller.capture) + self.assertFalse(self.dialog._capture_pending) + self.stop.assert_not_called() + self.start.assert_not_called() + self.error.assert_called_once() def test_preprocess_refresh_signals_report_invalid_pending_config(self) -> None: dialog = self.preprocess.ensure_dialog() diff --git a/python_app/tests/test_orchestration_recovery.py b/python_app/tests/test_orchestration_recovery.py index e83c0fd..4a1b428 100644 --- a/python_app/tests/test_orchestration_recovery.py +++ b/python_app/tests/test_orchestration_recovery.py @@ -137,19 +137,6 @@ class RadarModelCommandTest(unittest.TestCase): return path - def test_null_replay_directory_does_not_select_replay(self) -> None: - path = self._config_with_model("librevna") - path.write_text(json.dumps({"radar": {"replay": {"dataset_dir": None}}})) - self.assertEqual(self.supervisor._acquisition_command(path)[:3], - [sys.executable, "-m", "python_app.scripts.matrix_raw_producer"]) - - def test_replay_overrides_every_model(self) -> None: - for model in ("librevna", "librevna_multi", "kamil_adc"): - path = self._config_with_model(model) - path.write_text(json.dumps({"radar": {"model": model, "replay": {"dataset_dir": "capture"}}})) - self.assertEqual(self.supervisor._acquisition_command(path)[:3], - [sys.executable, "-m", "python_app.scripts.replay_raw_producer"]) - def test_matrix_producer_for_multi(self) -> None: cmd = self.supervisor._acquisition_command(self._config_with_model("librevna_multi")) self.assertEqual(cmd[:3], [sys.executable, "-m", "python_app.scripts.matrix_raw_producer"]) diff --git a/python_app/tests/test_persistence_integrity.py b/python_app/tests/test_persistence_integrity.py index 1f3a52f..15ad577 100644 --- a/python_app/tests/test_persistence_integrity.py +++ b/python_app/tests/test_persistence_integrity.py @@ -17,7 +17,6 @@ from python_app.models.dataset_model import ( ) from python_app.storage.npz.store import NpzStore from python_app.storage.npz.vna_history_json import build_vna_history_payload -from python_app.scripts.replay_raw_producer import _load_dataset from python_app.storage.npz.vna_history_json import collection_directories, load_vna_history_snapshot from python_app.workflows.sequential_capture_workflow import combine_collections_via_median, combine_traces_via_median @@ -352,14 +351,12 @@ class PersistenceIntegrityTest(unittest.TestCase): self.assertEqual(paths[0].read_bytes(), exported) self.assertEqual({path: path.read_bytes() for path in before}, before) - def test_long_recording_replay_and_converter_use_numeric_directory_order(self) -> None: + def test_long_recording_converter_uses_numeric_directory_order(self) -> None: writer = self.store.create_snapshot_stream(self.output, 'long') writer.append([_sweep(1, 10), _sweep(2, 20), _sweep(3, 30)], [], []) folders = sorted((writer.directory / 'raw').iterdir()) for folder, index in zip(folders, (9998, 9999, 10000), strict=True): folder.rename(folder.with_name(f'{index}_{folder.name.split("_", 1)[1]}')) - dataset = _load_dataset(writer.directory / 'raw', [ComboKey(0, 0)], 2) - self.assertEqual(dataset.timestamps_ns, [10, 20, 30]) self.assertEqual([int(path.name.split('_')[0]) for path in collection_directories(writer.directory / 'raw')], [9998, 9999, 10000]) def test_offline_and_runtime_json_exports_share_matching_and_preserve_orphan_sweeps(self) -> None: diff --git a/python_app/tests/test_run_config_codec.py b/python_app/tests/test_run_config_codec.py index 231cd76..7aed108 100644 --- a/python_app/tests/test_run_config_codec.py +++ b/python_app/tests/test_run_config_codec.py @@ -149,12 +149,9 @@ class AcquisitionValidationTest(unittest.TestCase): ring.slot_size_bytes = original validate_acquisition_config(config) - def test_kamil_requires_native_mode_except_for_replay(self) -> None: - config = self.config("kamil_adc") + def test_kamil_requires_native_mode(self) -> None: with self.assertRaisesRegex(ValueError, "native"): - validate_acquisition_config(config) - config.radar.replay.dataset_dir = "recorded" - validate_acquisition_config(config) + validate_acquisition_config(self.config("kamil_adc")) def test_matrix_validates_physical_switch_instead_of_virtual_axis(self) -> None: config = self.config("librevna_multi") diff --git a/run_config_examples/run_config_replay_vna.example.json b/run_config_examples/run_config_replay_vna.example.json deleted file mode 100644 index 55b5678..0000000 --- a/run_config_examples/run_config_replay_vna.example.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "_comment": "Replay of the recorded VNA_AlGo tank005 dataset through the full pipeline (no hardware). The replay producer feeds the raw ring; pair it with NEUTRAL preprocess sets (calibration=1, reference=0) so the already-preprocessed recording passes through unchanged. Create them with the GUI button 'Create Neutral S21 Sets' (name: neutral_replay).", - "radar": { - "model": "librevna_multi", - "serial": "206830985532", - "driver_mode": "mock", - "multi_device": { - "slave_serials": [ - "206930965532", - "206930A15532" - ], - "force_external_reference": true, - "recovery_attempts": 3, - "output_switch_positions": 1, - "input_switch_positions": 1, - "stimulus_switch_positions": 1 - }, - "replay": { - "dataset_dir": "/home/awe/Documents/VNA_AlGo/librevna_multi_2-6GHz_751pts_-3dBm_tank005/librevna_multi_2-6GHz_751pts_-3dBm_tank005/preprocessed", - "loop": true, - "rate_scale": 1.0 - }, - "sweep": { - "start_hz": 2000000000.0, - "stop_hz": 6000000000.0, - "if_bandwidth_hz": 50000.0, - "stimulus_power_dbm": -3.0, - "points": 751 - } - }, - "switches": { - "port1": { - "name": "port1", - "driver_mode": "mock", - "radar_port": 1, - "positions": 2, - "default_position": 0, - "gpio_chip": "/dev/gpiochip0", - "pin_a": 17, - "pin_b": 27 - }, - "port2": { - "name": "port2", - "driver_mode": "mock", - "radar_port": 2, - "positions": 4, - "default_position": 0, - "gpio_chip": "/dev/gpiochip0", - "pin_a": 22, - "pin_b": 23 - } - }, - "control_button": { - "enabled": false, - "gpio_chip": "/dev/gpiochip0", - "pin": -1, - "active_low": true, - "bias": "", - "debounce_ms": 50, - "action": "capture_tmp_reference" - }, - "run": { - "settling_ms": 0, - "idle_sleep_ms": 2, - "continuous": true, - "processing_live_config_path": "python_app/runtime/processing_live.json", - "locator_server": { - "device_id": 3, - "protocol_version": 1, - "host": "0.0.0.0", - "port": 8888, - "max_payload_bytes": 65536, - "client_queue_size": 32, - "logger_name": "locator_runtime" - }, - "combos": [ - { - "input": 0, - "output": 0 - }, - { - "input": 1, - "output": 0 - }, - { - "input": 2, - "output": 0 - }, - { - "input": 3, - "output": 0 - }, - { - "input": 0, - "output": 1 - }, - { - "input": 1, - "output": 1 - }, - { - "input": 2, - "output": 1 - }, - { - "input": 3, - "output": 1 - } - ] - }, - "preprocess": { - "s21": { - "calibration": { - "set_name": "neutral_replay", - "bundle_path": "" - }, - "reference": { - "set_name": "neutral_replay", - "bundle_path": "" - } - }, - "notch": { - "enabled": false, - "bands_hz": [], - "taper_width_hz": 40000000.0, - "taper_type": "cosine" - } - }, - "gpr": { - "relative_permittivity": 1.0, - "tx_geometry": [ - { - "output_pos": 0, - "x_m": 0.785, - "y_m": 0.0, - "z_m": 0.0 - }, - { - "output_pos": 1, - "x_m": -0.785, - "y_m": 0.0, - "z_m": 0.0 - } - ], - "rx_geometry": [ - { - "input_pos": 0, - "x_m": -0.42, - "y_m": 0.0, - "z_m": 0.0 - }, - { - "input_pos": 1, - "x_m": 0.48, - "y_m": 0.0, - "z_m": 0.0 - }, - { - "input_pos": 2, - "x_m": 0.18, - "y_m": 0.0, - "z_m": 0.0 - }, - { - "input_pos": 3, - "x_m": -0.12, - "y_m": 0.0, - "z_m": 0.0 - } - ] - }, - "rings": { - "raw": { - "name": "/radar_raw_simulator", - "capacity": 50, - "slot_size_bytes": 2097152 - }, - "raw_tap": { - "name": "/radar_raw_tap_simulator", - "capacity": 50, - "slot_size_bytes": 2097152 - }, - "preprocessed": { - "name": "/radar_preprocessed_simulator", - "capacity": 50, - "slot_size_bytes": 2097152 - }, - "preprocessed_tap": { - "name": "/radar_preprocessed_tap_simulator", - "capacity": 50, - "slot_size_bytes": 2097152 - }, - "results": { - "name": "/radar_results_simulator", - "capacity": 50, - "slot_size_bytes": 2097152 - } - }, - "gui": { - "version": 1, - "switches": { - "combo_mode": "text", - "combos_text": "0:0,1:0,2:0,3:0,0:1,1:1,2:1,3:1", - "single_input": "0", - "single_output": "0" - }, - "processing": { - "selected_mode": "sar3d", - "sar3d": { - "input_positions": "0,1,2,3", - "output_positions": "0,1", - "min_depth_m": 3.2, - "max_depth_m": 10.0, - "start_freq_mhz": 2000.0, - "stop_freq_mhz": 6000.0, - "speed_m_s": 0.854, - "ignore_socket_speed_enabled": true, - "look_angle_deg": 6.0, - "frame_radius": 6, - "voxel_m": 0.02, - "bp_ground_depth_m": 1.25, - "comp_range_power": 0.3, - "use_compensation": true, - "normalize_each_frame": true - } - }, - "preprocess_dialog": { - "set_name": "neutral_replay", - "radar_config_dir": "", - "use_all_radar_configs": false, - "median_sweep_count": 5 - } - } -}