diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..e31c405 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "cmake.sourceDirectory": "/home/ayzen/Documents/radar_system/data_acq_and_processing/gpr_cpp" +} \ No newline at end of file diff --git a/Makefile b/Makefile index 0e83b4a..8451370 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,8 @@ CXX := g++ +PYTHON ?= .venv/bin/python CXXFLAGS := -std=c++20 -O2 -Wall -Wextra -Wpedantic -pthread -MMD -MP LDFLAGS := -pthread -lrt ORCH_LDFLAGS := $(LDFLAGS) -lusb-1.0 -VISA_CXXFLAGS ?= -I/usr/include/ni-visa -VISA_LDFLAGS := $(LDFLAGS) -lvisa BUILD_DIR := build BIN_DIR := $(BUILD_DIR)/bin @@ -11,9 +10,9 @@ BIN_DIR := $(BUILD_DIR)/bin # --- CUDA for the sar3d processor (optional). Its kernels are compiled at run # time by NVRTC and launched through the CUDA driver API, so no nvcc is needed: # only nvrtc.h, cuda.h, libnvrtc and the NVIDIA driver's libcuda. Headers and -# libnvrtc come from the pip nvidia-cuda-* packages in .venv (requirements.txt, +# libnvrtc come from the pip nvidia-cuda-* packages in .venv (requirements-cuda.txt, # x86_64) or from a system toolkit in /usr/local/cuda (Jetson). Without them -# data_processor still builds; sar3d then reports that CUDA is missing. +# data_processor still builds; selecting sar3d shows a CUDA warning. CUDA_ROOTS := $(wildcard $(CURDIR)/.venv/lib/python3*/site-packages/nvidia/*) /usr/local/cuda CUDA_NVRTC_HEADER := $(firstword $(wildcard $(addsuffix /include/nvrtc.h,$(CUDA_ROOTS)))) CUDA_DRIVER_HEADER := $(firstword $(wildcard $(addsuffix /include/cuda.h,$(CUDA_ROOTS)))) @@ -54,7 +53,6 @@ ORCH_SOURCES := \ data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_lifecycle.cpp \ data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_transport.cpp \ data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_protocol.cpp \ - data_acq_and_processing/sweep_orchestrator/device_drivers/radar/remote_compact_m_k209/remote_compact_m_k209_driver.cpp \ data_acq_and_processing/sweep_orchestrator/device_drivers/switches/h7992_minimal_driver.cpp \ data_acq_and_processing/sweep_orchestrator/device_drivers/switches/hmc349a_minimal_driver.cpp \ data_acq_and_processing/sweep_orchestrator/src/sweep_orchestrator.cpp \ @@ -120,17 +118,72 @@ KAMIL_BUILD_DIR := $(BUILD_DIR)/kamil KAMIL_COLLECTOR_OBJS := $(addprefix $(KAMIL_BUILD_DIR)/,$(notdir $(KAMIL_COLLECTOR_SOURCES:.cpp=.o))) DEPFILES := $(sort $(SWEEP_ORCH_OBJS:.o=.d) $(PREPROCESSOR_OBJS:.o=.d) $(DATA_PROCESSOR_OBJS:.o=.d) $(KAMIL_COLLECTOR_OBJS:.o=.d)) +NATIVE_TEST_SOURCES := $(COMMON_SOURCES) $(LOCATOR_SOURCES) \ + data_acq_and_processing/tests/native_regressions.cpp \ + data_acq_and_processing/processing/data_processor/src/processing_live_config.cpp \ + data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp \ + data_acq_and_processing/sweep_orchestrator/src/sweep_orchestrator.cpp \ + $(KAMIL_COLLECTOR_DIR)/src/capture_file_writer.cpp \ + $(KAMIL_COLLECTOR_DIR)/src/tty_protocol_writer.cpp +NATIVE_BUILD_DIR := $(BUILD_DIR)/native_tests +NATIVE_TEST_OBJS := $(addprefix $(NATIVE_BUILD_DIR)/,$(NATIVE_TEST_SOURCES:.cpp=.o)) +NATIVE_CXXFLAGS := $(CXXFLAGS) -O1 -g $(if $(NATIVE_SANITIZERS),-fsanitize=$(NATIVE_SANITIZERS) -fno-omit-frame-pointer) +NATIVE_FLAGS_STAMP := $(NATIVE_BUILD_DIR)/compiler.flags +$(shell mkdir -p $(NATIVE_BUILD_DIR) && echo '$(NATIVE_CXXFLAGS)' | cmp -s - $(NATIVE_FLAGS_STAMP) || echo '$(NATIVE_CXXFLAGS)' > $(NATIVE_FLAGS_STAMP)) +DEPFILES += $(NATIVE_TEST_OBJS:.o=.d) + TARGETS := \ $(BIN_DIR)/sweep_orchestrator \ $(BIN_DIR)/data_preprocessor \ $(BIN_DIR)/data_processor -.PHONY: all clean kamil_adc_collector +.PHONY: all clean kamil_adc_collector check-native check-sar3d all: $(TARGETS) kamil_adc_collector: $(BIN_DIR)/kamil_adc_collector +check-native: $(NATIVE_BUILD_DIR)/native_regressions + $(NATIVE_BUILD_DIR)/native_regressions run_config_examples/run_config_simulator.example.json + +SAR3D_TEST_SOURCES := data_acq_and_processing/tests/sar3d_fixture.cpp \ + data_acq_and_processing/processing/processors/src/sar3d_processor.cpp \ + data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp +SAR3D_TEST_OBJS := $(addprefix $(NATIVE_BUILD_DIR)/,$(SAR3D_TEST_SOURCES:.cpp=.o)) +SAR3D_EMULATION_OBJ := $(NATIVE_BUILD_DIR)/data_acq_and_processing/tests/sar3d_kernel_emulation.o +SAR3D_KERNEL_TEST_OBJ := $(NATIVE_BUILD_DIR)/data_acq_and_processing/tests/sar3d_kernel_regressions.o +SAR3D_EMULATION_HEADERS := $(NATIVE_BUILD_DIR)/sar3d_kernels.inc $(NATIVE_BUILD_DIR)/sar3d_helpers.inc +DEPFILES += $(SAR3D_TEST_OBJS:.o=.d) $(SAR3D_EMULATION_OBJ:.o=.d) $(SAR3D_KERNEL_TEST_OBJ:.o=.d) + +check-sar3d: $(NATIVE_BUILD_DIR)/sar3d_fixture $(NATIVE_BUILD_DIR)/sar3d_math_fixture $(NATIVE_BUILD_DIR)/sar3d_kernel_regressions + $(NATIVE_BUILD_DIR)/sar3d_kernel_regressions + $(PYTHON) data_acq_and_processing/tests/test_sar3d_oracle.py + +$(NATIVE_BUILD_DIR)/sar3d_fixture: $(SAR3D_TEST_OBJS) + $(CXX) $(NATIVE_CXXFLAGS) $^ -o $@ $(LDFLAGS) $(SAR3D_LDFLAGS) + +$(NATIVE_BUILD_DIR)/sar3d_math_fixture: $(filter-out $(NATIVE_BUILD_DIR)/data_acq_and_processing/processing/processors/src/sar3d_gpu.o,$(SAR3D_TEST_OBJS)) $(SAR3D_EMULATION_OBJ) + $(CXX) $(NATIVE_CXXFLAGS) $^ -o $@ $(LDFLAGS) + +$(NATIVE_BUILD_DIR)/sar3d_kernel_regressions: $(SAR3D_KERNEL_TEST_OBJ) $(SAR3D_EMULATION_OBJ) + $(CXX) $(NATIVE_CXXFLAGS) $^ -o $@ $(LDFLAGS) + +$(SAR3D_EMULATION_HEADERS) &: data_acq_and_processing/tests/generate_sar3d_emulation.py data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp data_acq_and_processing/processing/processors/src/sar3d_gpu.hpp + $(PYTHON) data_acq_and_processing/tests/generate_sar3d_emulation.py $(NATIVE_BUILD_DIR) + +$(SAR3D_EMULATION_OBJ): NATIVE_CXXFLAGS += -I$(NATIVE_BUILD_DIR) +$(SAR3D_EMULATION_OBJ): $(SAR3D_EMULATION_HEADERS) + +$(NATIVE_BUILD_DIR)/data_acq_and_processing/processing/processors/src/sar3d_gpu.o: NATIVE_CXXFLAGS += $(SAR3D_CXXFLAGS) +$(NATIVE_BUILD_DIR)/data_acq_and_processing/processing/processors/src/sar3d_gpu.o: $(SAR3D_CUDA_STAMP) + +$(NATIVE_BUILD_DIR)/native_regressions: $(NATIVE_TEST_OBJS) + $(CXX) $(NATIVE_CXXFLAGS) $^ -o $@ $(LDFLAGS) -lutil + +$(NATIVE_BUILD_DIR)/%.o: %.cpp $(NATIVE_FLAGS_STAMP) + @mkdir -p $(dir $@) + $(CXX) $(NATIVE_CXXFLAGS) $(INCLUDES) -I$(KAMIL_COLLECTOR_DIR)/include -c $< -o $@ + $(BIN_DIR)/sweep_orchestrator: $(SWEEP_ORCH_OBJS) @mkdir -p $(BIN_DIR) $(CXX) $(SWEEP_ORCH_OBJS) -o $@ $(ORCH_LDFLAGS) @@ -142,13 +195,13 @@ $(BIN_DIR)/data_preprocessor: $(PREPROCESSOR_OBJS) $(BIN_DIR)/data_processor: $(DATA_PROCESSOR_OBJS) $(SAR3D_CUDA_STAMP) @mkdir -p $(BIN_DIR) $(CXX) $(DATA_PROCESSOR_OBJS) -o $@ $(LDFLAGS) $(SAR3D_LDFLAGS) - $(if $(SAR3D_LDFLAGS),,@echo "note: CUDA not found, data_processor built without the sar3d GPU backend") + $(if $(SAR3D_LDFLAGS),,@echo "note: CUDA not found, selecting sar3d will show a warning") $(BUILD_DIR)/data_acq_and_processing/processing/processors/src/sar3d_gpu.o: CXXFLAGS += $(SAR3D_CXXFLAGS) $(BUILD_DIR)/data_acq_and_processing/processing/processors/src/sar3d_gpu.o: $(SAR3D_CUDA_STAMP) # The collector compiles into its own build/kamil/ tree with collector-only -# includes (no VISA), so the shared driver/config sources it reuses never collide +# includes, so the shared driver/config sources it reuses never collide # with the orchestrator's objects of the same name. Basenames are unique, so a # vpath lets one pattern rule find every source. vpath %.cpp $(sort $(dir $(KAMIL_COLLECTOR_SOURCES))) @@ -163,7 +216,7 @@ $(BIN_DIR)/kamil_adc_collector: $(KAMIL_COLLECTOR_OBJS) $(BUILD_DIR)/%.o: %.cpp @mkdir -p $(dir $@) - $(CXX) $(CXXFLAGS) $(VISA_CXXFLAGS) $(INCLUDES) -c $< -o $@ + $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ -include $(DEPFILES) diff --git a/README.md b/README.md index 01acc7f..128c33f 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,6 @@ controls plus a pixel-identical live plot through an embedded web UI. - **librevna** — single LibreVNA. - **librevna_multi** — synchronized multi-device LibreVNA. -- **compact_m_k209** — Compact-M K209 / S2VNA (including Raspberry Pi remote mode). -- **sn9000** — SNVNA SN9000 over VISA (pure-Python `@py` backend). - **kamil_adc** — external Kamil ADC acquisition process, with optional laser control. - **mock / simulator** — driver-less mode for development without hardware. @@ -56,13 +54,28 @@ controls plus a pixel-identical live plot through an embedded web UI. - **gpr** — coherent back-projection GPR imaging with object detection. - **legacy_gpr** — ellipse-intersection MIMO localization (point and extended/region modes). - **sar3d** — 3D SAR of the reference notebook `Multi_antenna_3D_clean_v1.ipynb` - on an NVIDIA GPU: front (X-Y), side (Z-Y), top (X-Z) and 3D views of the imaged + on an NVIDIA GPU through CUDA: front (X-Y), side (Z-Y), top (X-Z) + and oblique maximum projections of the imaged volume. Its settings carry the notebook's parameter names; the scene window is fixed (X −2…2 m, height −0.5…2 m above the ground), its range extent follows `MIN_DEPTH`/`MAX_DEPTH`, and the voxels are cubes with edge `SAR_VOXEL_M` (default 2 cm). The radar motion is corrected from each trace's capture timestamps, so the sweep timing needs no parameters. +For matrix-radar B-scans, **Next combination** advances output first, then input, +and wraps to the first pair. The desktop fields and browser channel selector +redraw the selected cached B-scan immediately without restarting acquisition. +**Save Last 1000** and **Save S21 JSON** use a fixed limit of 1000 recent +measurements from each available stage; older GUI profiles may still contain +`save_count`, which is ignored. **Start + Record** retains its separate count for +recording future measurements to disk. + +Tx/Rx antenna positions are edited in tables with switch position and X/Y/Z in +metres, on both the desktop and web interface. Coordinate and legacy-GPR medium +changes apply to the running processor and recompute its retained result. Profiles +preserve all three coordinates. GPR/SAR views wait for a result marked with the +current settings revision, so an older in-flight result cannot overwrite new geometry. + ## Repository layout | Path | Contents | @@ -72,22 +85,26 @@ controls plus a pixel-identical live plot through an embedded web UI. | `run_config_examples/` | Ready-made configuration profiles, one per hardware setup. | | `deploy/` | systemd daemon installer, and the USB update kit for the Pi and Jetson (`pack-usb.sh`, `update.sh`). | | `build/bin/` | Compiled C++ binaries. | -| `device_firmware/`, `docs/` | Device firmware and vendor manuals. | +| `device_firmware/` | Device firmware and protocol documentation. | ## Requirements - Linux (POSIX shared memory; GPIO on Raspberry Pi). The deployment target is a **Raspberry Pi 5**; hardware PWM gating for the generator sweep is Pi 5 specific (GPIO12/13/18/19, `dtoverlay=pwm-2chan`). -- A C++20 compiler and `make` — Eigen and nlohmann/json are vendored, so no extra C++ dependencies are needed. +- A C++20 compiler and `make`; Eigen and nlohmann/json are vendored. +- `libusb-1.0` development headers and `libatomic.so.1` (`libatomic1` on Debian/Ubuntu). + Python ring readers/writers use lock-free atomic loads, stores and cursor CAS + through libatomic, including memory barriers for ARM. Slot sizes must be multiples + of 8 bytes, and the complete ring mapping must not exceed 1 TiB. - **Python 3.10 or newer**, in a virtual environment. -- For the `sar3d` processor: an NVIDIA GPU with its driver (`libcuda`). The +- For `sar3d`: an NVIDIA GPU with its driver (`libcuda`). The kernels are compiled at run time by NVRTC, so there is no nvcc step; `make` needs only `nvrtc.h`, `cuda.h` and `libnvrtc`, taken from the pip - `nvidia-cuda-*` packages in `.venv` (`requirements.txt`, x86_64; CUDA 13 covers - Turing and newer GPUs) or from a system toolkit in `/usr/local/cuda` (Jetson). - Without them everything else still builds, and sar3d reports that CUDA is - missing — a Raspberry Pi needs none of it. + `nvidia-cuda-*` packages in `.venv` (`requirements-cuda.txt`, x86_64) or from + a system toolkit in `/usr/local/cuda` (Jetson). + Without these dependencies or a usable GPU, SAR3D shows a warning above the + plots and does not calculate an image. Other processing modes remain available. ## Build and run @@ -103,6 +120,8 @@ make # build the C++ binaries into build/bin/ `make` builds the sar3d GPU code when it finds CUDA (see Requirements) and prints a note when it does not. After installing the CUDA requirements into an existing venv, run `make` again: it notices the change and rebuilds that part. +For SAR3D on x86_64, install the optional packages with +`.venv/bin/python -m pip install -r requirements-cuda.txt`, then run `make`. Run the desktop GUI: @@ -153,17 +172,27 @@ Two ways to run the full pipeline with no radar attached: **Mock producer (synthetic data).** The default `run_config.json` already sets `radar.driver_mode: "mock"`, so the producer generates synthetic S-parameters. -The preprocessor still requires S21 calibration/reference sets; create them once: +Mock acquisition supports the LibreVNA models; Kamil ADC uses native acquisition +or recorded-dataset replay without devices. +Both LibreVNA modes share one simulator: a point reflector moves back and forth +along the antenna array and between depths 3.25–4.25 m over an eight-second cycle. +Each S21 uses the actual transmitter–target–receiver distance from the configured +XYZ coordinates; coincident antennas produce identical signals. All combinations +share the same target position and measured capture window, with collections paced +at 50 ms intervals. B-scan's default range is 0–8 m; stored profiles retain their +chosen range. The simulator models a stationary radar, so radar speed should be zero. +The scene uses the antenna positions from its startup profile; live geometry edits +change the reconstruction of captured sweeps, whose samples remain unchanged. +The preprocessor still requires S21 calibration/reference sets. Start the GUI: ```bash -.venv/bin/python -m python_app.scripts.manual_smoke_run --duration 1.0 # creates smoke_cal/smoke_ref -./start.sh # then in the GUI: Preprocessing Panel -> pick the sets -> Start +./start.sh ``` -(Alternatively use the GUI's "Create neutral sets" button, which makes -identity calibration + zero reference so preprocessing is a pass-through.) +In **Preprocessing Panel**, use **Create neutral sets**, then **Start**. This makes +identity calibration + zero reference, preserving the simulated S21 samples. -**Dataset replay (recorded scene, meaningful sar3d picture).** When +**Dataset replay (recorded scene).** When `radar.replay.dataset_dir` is set in the profile, the supervisor launches `python_app.scripts.replay_raw_producer` instead of the normal producer: it replays recorded snapshots into the raw ring at the recorded cadence, looping @@ -179,15 +208,32 @@ A ready profile for the VNA_AlGo tank005 dataset is provided: ./start.sh --profile run_config_examples/run_config_replay_vna.example.json ``` -The sar3d processor keeps about 75 MB of buffers on the GPU at the default -2 cm voxel and depth 2…8 m (plus the CUDA context). The volume grows as -1/`SAR_VOXEL_M`³: 1 cm takes about 600 MB and ran about 6 times slower in -testing. If the GPU has no room, it logs the CUDA error once and publishes empty -results until memory frees. +Finer voxels increase SAR3D work as 1/`SAR_VOXEL_M`³. Increase the voxel size when +a faster preview matters more than spatial resolution. Processing failures appear +above the plots and a failed single capture +finishes with its explanation while retaining the acquired data. The web interface is served in-process at `http://:8080`. It mirrors the live plot and offers Start / Stop / Capture controls and the active processor settings. +Manual dataset and JSON export keep the last 1000 available collections from each +stage independently, including raw or preprocessed sweeps without a result. +Streaming recording keeps every collection received after arming until the requested +result count or Stop; Stop and closing the window flush accepted data. Acquisition +rings use latest-wins delivery, so disk recording cannot recover sweeps overwritten +before the GUI receives them. NumPy datasets preserve both S-parameters, capture +windows, every supported result payload, and processing time. Failed exports remove +their staging files, leaving existing datasets intact. + +Calibration and reference sets keep their trace arrays and metadata together in one +atomically replaced NPZ archive, with a readable JSON sidecar. Earlier NPZ/JSON pairs +remain supported. Invalid trace shapes, duplicate combinations and non-finite +calibration/reference values are rejected before a saved set is replaced. +Median captures also require identical frequency axes and channel coverage. +Dataset replay restores both recorded S-parameters and orders long recordings by +their numeric index; older S21-only datasets still replay with zero S11. Offline +JSON conversion and the GUI share the same sweep-matching code. + ## Configuration The active profile is `run_config.json`. It selects the radar model and sweep, the @@ -217,13 +263,34 @@ the current version is renamed to `radar_system`, the next free number. Kept it: `run_config.json`, profiles in `run_config_examples/` changed or added on the device (the release version of a changed one lands next to it as `.new`), `python_app/runtime/` without logs, `python_app/data*` and any other top-level -folder that is not part of the release, as hard links. Then it carries the venv over -(or builds a new one with `--python`), builds the C++ part and restarts +file or folder that is not part of the release, as hard links. The venv is copied +independently so package updates cannot modify the rollback environment (or a new +one is built with `--python`). It builds the C++ pipeline and ADC collector and restarts `radar.service` if it was running. To go back: `rm -rf radar_system && mv radar_system radar_system`. ## Tests ```bash -.venv/bin/python -m unittest discover -s python_app/tests +QT_QPA_PLATFORM=offscreen .venv/bin/python -m unittest discover -s python_app/tests +make check-native +make check-sar3d PYTHON=.venv/bin/python +ASAN_OPTIONS=detect_leaks=0 make check-native NATIVE_SANITIZERS=address,undefined ``` + +The tests cover failed writes and settings commits, recordings during manual Save, +repeated Start/Stop/Close, web-action cancellation, malformed IPC frames, hardware +resource cleanup with mock failures, and an isolated deployment update. Native tests +also exercise mutated frames, ring concurrency, locator input and ADC spool recovery. +Firmware host tests check cart timer rollover and SBUS packing; they do not validate +physical timing. Real devices, GPIO and the CUDA backend need their own hardware run. + +Invalid device/switch settings and undersized raw/preprocessed slots fail before +acquisition starts. Oversized processed results report the required slot size in +the log; switching to a mode that fits resumes processing without a restart. + +GUI profiles, run configs, session state and live processing settings use shared +atomic writers with unique temporary files. Profiles and run/session settings sync +their file and directory; frequent live settings avoid extra disk syncs. A failed +write before replacement leaves the previous file intact. The launcher and supervisor +share acquisition producer selection, with replay taking precedence over the model. diff --git a/data_acq_and_processing/common_cpp/config/include/run_config.hpp b/data_acq_and_processing/common_cpp/config/include/run_config.hpp index 0f4d43a..7cf50b0 100644 --- a/data_acq_and_processing/common_cpp/config/include/run_config.hpp +++ b/data_acq_and_processing/common_cpp/config/include/run_config.hpp @@ -32,10 +32,7 @@ struct RadarConfig { // Radar device identity and runtime mode. std::string model = "librevna"; std::string serial{}; - std::string remote_host = "127.0.0.1"; - std::uint32_t remote_port = 50209; DriverMode driver_mode = DriverMode::Mock; - float mock_signal_hz = 5'000'000.0F; RadarSweepSettings sweep{}; }; @@ -134,7 +131,7 @@ struct GprRxGeometry { }; struct GprConfig { - // Stable collection-level GPR processing settings. + // Initial GPR settings; the data_processor can override them through live config. float relative_permittivity = 1.0F; std::vector tx_geometry{}; std::vector rx_geometry{}; @@ -154,5 +151,6 @@ struct RunConfig { }; [[nodiscard]] auto load_run_config(const std::string& path) -> RunConfig; +void validate_gpr_config(const GprConfig& config, const RunConfig& run_config); } // namespace radar::config diff --git a/data_acq_and_processing/common_cpp/config/include/runtime_wait.hpp b/data_acq_and_processing/common_cpp/config/include/runtime_wait.hpp new file mode 100644 index 0000000..efa5160 --- /dev/null +++ b/data_acq_and_processing/common_cpp/config/include/runtime_wait.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace radar::config { +// Returns true when stop interrupts the delay, checking at least every 50 ms. +inline auto wait_or_stop(std::uint32_t delay_ms, const std::atomic& stop) -> bool { + std::uint32_t waited = 0; + while (waited < delay_ms && !stop.load(std::memory_order_relaxed)) { + const auto chunk = std::min(50U, delay_ms - waited); + std::this_thread::sleep_for(std::chrono::milliseconds(chunk)); + waited += chunk; + } + return stop.load(std::memory_order_relaxed); +} +} // namespace radar::config diff --git a/data_acq_and_processing/common_cpp/config/src/run_config.cpp b/data_acq_and_processing/common_cpp/config/src/run_config.cpp index 4ce65c2..6ce59a6 100644 --- a/data_acq_and_processing/common_cpp/config/src/run_config.cpp +++ b/data_acq_and_processing/common_cpp/config/src/run_config.cpp @@ -65,7 +65,18 @@ using Json = nlohmann::json; if (!value.is_number()) { throw std::runtime_error("Expected JSON number at " + context); } - return value.get(); + const auto numeric = value.get(); + if (!std::isfinite(numeric)) { + throw std::runtime_error("Expected finite number at " + context); + } + return numeric; +} + +[[nodiscard]] auto number_to_f32(double value, const std::string& context) -> float { + if (!std::isfinite(value) || std::fabs(value) > std::numeric_limits::max()) { + throw std::runtime_error("Value out of finite float32 range at " + context); + } + return static_cast(value); } [[nodiscard]] auto number_to_u32(double value, const std::string& context) -> std::uint32_t { @@ -74,7 +85,7 @@ using Json = nlohmann::json; } const auto rounded = std::round(value); - if (std::fabs(value - rounded) > 0.000001) { + if (value != rounded) { throw std::runtime_error("Expected integer value at " + context); } return static_cast(rounded); @@ -82,7 +93,7 @@ using Json = nlohmann::json; [[nodiscard]] auto number_to_i32(double value, const std::string& context) -> std::int32_t { const auto rounded = std::round(value); - if (std::fabs(value - rounded) > 0.000001) { + if (value != rounded) { throw std::runtime_error("Expected integer value at " + context); } @@ -135,7 +146,7 @@ using Json = nlohmann::json; [[nodiscard]] auto optional_f32(const Json& object, const std::string& key, float fallback) -> float { if (const auto* value = optional_field(object, key); value != nullptr) { - return static_cast(as_number(*value, key)); + return number_to_f32(as_number(*value, key), key); } return fallback; } @@ -173,8 +184,8 @@ using Json = nlohmann::json; } PreprocessNotchBandConfig band{}; - band.low_hz = static_cast(as_number((*band_array)[0], context + ".bands_hz[][0]")); - band.high_hz = static_cast(as_number((*band_array)[1], context + ".bands_hz[][1]")); + band.low_hz = number_to_f32(as_number((*band_array)[0], context + ".bands_hz[][0]"), context); + band.high_hz = number_to_f32(as_number((*band_array)[1], context + ".bands_hz[][1]"), context); notch.bands_hz.push_back(band); } } @@ -242,6 +253,9 @@ using Json = nlohmann::json; } void validate_radar_sweep(const RadarSweepSettings& sweep) { + if (sweep.start_hz < 0.0F || sweep.stop_hz < 0.0F || !(sweep.if_bandwidth_hz > 0.0F)) { + throw std::runtime_error("radar.sweep frequencies must be non-negative and IF bandwidth must be > 0"); + } if (sweep.points == 0U) { throw std::runtime_error("radar.sweep.points must be > 0"); } @@ -297,32 +311,6 @@ void validate_combo_key(const ipc::ComboKey& key, const RunConfig& config) { } } -void validate_gpr_config(const GprConfig& config, const RunConfig& run_config) { - if (!(config.relative_permittivity > 0.0F)) { - throw std::runtime_error("gpr.relative_permittivity must be > 0"); - } - - std::unordered_set seen_tx_positions{}; - for (const auto& entry : config.tx_geometry) { - if (entry.output_pos >= run_config.output_switch.positions) { - throw std::runtime_error("gpr.tx_geometry output_pos is out of range"); - } - if (!seen_tx_positions.insert(entry.output_pos).second) { - throw std::runtime_error("gpr.tx_geometry contains duplicate output_pos"); - } - } - - std::unordered_set seen_rx_positions{}; - for (const auto& entry : config.rx_geometry) { - if (entry.input_pos >= run_config.input_switch.positions) { - throw std::runtime_error("gpr.rx_geometry input_pos is out of range"); - } - if (!seen_rx_positions.insert(entry.input_pos).second) { - throw std::runtime_error("gpr.rx_geometry contains duplicate input_pos"); - } - } -} - [[nodiscard]] auto parse_gpr_config(const Json& object, const RunConfig& run_config) -> GprConfig { const auto* gpr_obj = as_object(object, "gpr"); GprConfig config{}; @@ -406,6 +394,9 @@ void validate_gpr_config(const GprConfig& config, const RunConfig& run_config) { if (endpoint.slot_size_bytes == 0U) { throw std::runtime_error("Ring slot_size_bytes must be > 0 for " + key); } + if (endpoint.slot_size_bytes % 8U != 0U) { + throw std::runtime_error("Ring slot_size_bytes must be a multiple of 8 for " + key); + } if (endpoint.name.empty() || endpoint.name.front() != '/') { throw std::runtime_error("Ring name must start with '/' for " + key); } @@ -415,6 +406,39 @@ void validate_gpr_config(const GprConfig& config, const RunConfig& run_config) { } // namespace +void validate_gpr_config(const GprConfig& config, const RunConfig& run_config) { + if (!std::isfinite(config.relative_permittivity) || !(config.relative_permittivity > 0.0F)) { + throw std::runtime_error("gpr.relative_permittivity must be > 0"); + } + + std::unordered_set seen_tx_positions{}; + for (const auto& entry : config.tx_geometry) { + if (!std::isfinite(entry.x_m) || !std::isfinite(entry.y_m) || !std::isfinite(entry.z_m)) { + throw std::runtime_error("gpr.tx_geometry coordinates must be finite"); + } + if (entry.output_pos >= run_config.output_switch.positions) { + throw std::runtime_error("gpr.tx_geometry output_pos is out of range"); + } + if (!seen_tx_positions.insert(entry.output_pos).second) { + throw std::runtime_error("gpr.tx_geometry contains duplicate output_pos"); + } + } + + std::unordered_set seen_rx_positions{}; + for (const auto& entry : config.rx_geometry) { + if (!std::isfinite(entry.x_m) || !std::isfinite(entry.y_m) || !std::isfinite(entry.z_m)) { + throw std::runtime_error("gpr.rx_geometry coordinates must be finite"); + } + if (entry.input_pos >= run_config.input_switch.positions) { + throw std::runtime_error("gpr.rx_geometry input_pos is out of range"); + } + if (!seen_rx_positions.insert(entry.input_pos).second) { + throw std::runtime_error("gpr.rx_geometry contains duplicate input_pos"); + } + } +} + + auto load_run_config(const std::string& path) -> RunConfig { std::ifstream stream(path); if (!stream.is_open()) { @@ -440,30 +464,21 @@ auto load_run_config(const std::string& path) -> RunConfig { const auto* radar_obj = as_object(required_field(*root_obj, "radar"), "radar"); config.radar.model = optional_string(*radar_obj, "model", "librevna"); config.radar.serial = optional_string(*radar_obj, "serial", ""); - config.radar.remote_host = optional_string(*radar_obj, "remote_host", "127.0.0.1"); - config.radar.remote_port = optional_u32(*radar_obj, "remote_port", 50209); - if (config.radar.remote_host.empty()) { - throw std::runtime_error("radar.remote_host must not be empty"); - } - if (config.radar.remote_port == 0U || config.radar.remote_port > 65535U) { - throw std::runtime_error("radar.remote_port must be in 1..65535"); - } config.radar.driver_mode = parse_driver_mode(optional_string(*radar_obj, "driver_mode", "mock")); - config.radar.mock_signal_hz = optional_f32(*radar_obj, "mock_signal_hz", 5'000'000.0F); const auto* sweep_obj = as_object(required_field(*radar_obj, "sweep"), "radar.sweep"); config.radar.sweep.start_hz = - static_cast(as_number(required_field(*sweep_obj, "start_hz"), "radar.sweep.start_hz")); + number_to_f32(as_number(required_field(*sweep_obj, "start_hz"), "radar.sweep.start_hz"), "radar.sweep.start_hz"); config.radar.sweep.stop_hz = - static_cast(as_number(required_field(*sweep_obj, "stop_hz"), "radar.sweep.stop_hz")); + number_to_f32(as_number(required_field(*sweep_obj, "stop_hz"), "radar.sweep.stop_hz"), "radar.sweep.stop_hz"); if (const auto* points_value = optional_field(*sweep_obj, "points"); points_value != nullptr) { config.radar.sweep.points = number_to_u32(as_number(*points_value, "radar.sweep.points"), "radar.sweep.points"); } else if (config.radar.model != "kamil_adc") { throw std::runtime_error("Missing required config field: points"); } - config.radar.sweep.if_bandwidth_hz = static_cast( - as_number(required_field(*sweep_obj, "if_bandwidth_hz"), "radar.sweep.if_bandwidth_hz") + config.radar.sweep.if_bandwidth_hz = number_to_f32( + as_number(required_field(*sweep_obj, "if_bandwidth_hz"), "radar.sweep.if_bandwidth_hz"), "radar.sweep.if_bandwidth_hz" ); config.radar.sweep.power_dbm = optional_f32(*sweep_obj, "stimulus_power_dbm", -10.0F); validate_radar_sweep(config.radar.sweep); diff --git a/data_acq_and_processing/common_cpp/ipc/include/shared_types.hpp b/data_acq_and_processing/common_cpp/ipc/include/shared_types.hpp index de6548e..65721e4 100644 --- a/data_acq_and_processing/common_cpp/ipc/include/shared_types.hpp +++ b/data_acq_and_processing/common_cpp/ipc/include/shared_types.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -26,7 +27,8 @@ struct ComboKey { struct ComboKeyHash { [[nodiscard]] auto operator()(const ComboKey& key) const noexcept -> std::size_t { - return (static_cast(key.input_pos) << 32U) ^ static_cast(key.output_pos); + const auto packed = (static_cast(key.input_pos) << 32U) | key.output_pos; + return std::hash{}(packed); } }; diff --git a/data_acq_and_processing/common_cpp/ipc/include/shm_ring.hpp b/data_acq_and_processing/common_cpp/ipc/include/shm_ring.hpp index c1b42d7..a86a20b 100644 --- a/data_acq_and_processing/common_cpp/ipc/include/shm_ring.hpp +++ b/data_acq_and_processing/common_cpp/ipc/include/shm_ring.hpp @@ -53,6 +53,7 @@ class ShmRing { [[nodiscard]] auto slot_stride_bytes() const -> std::size_t; [[nodiscard]] auto slot_header(std::uint64_t sequence) const -> SlotHeader*; [[nodiscard]] auto slot_payload(SlotHeader* slot) const -> std::uint8_t*; + void advance_read_seq(std::uint64_t target); void close(); static void validate_name(const std::string& name); diff --git a/data_acq_and_processing/common_cpp/ipc/src/shared_types.cpp b/data_acq_and_processing/common_cpp/ipc/src/shared_types.cpp index 244e7ac..42590a9 100644 --- a/data_acq_and_processing/common_cpp/ipc/src/shared_types.cpp +++ b/data_acq_and_processing/common_cpp/ipc/src/shared_types.cpp @@ -95,7 +95,7 @@ class BinaryReader { private: void ensure_available(std::size_t size) const { - if (offset_ + size > bytes_.size()) { + if (size > bytes_.size() - offset_) { throw std::runtime_error("Unexpected end of serialized payload"); } } @@ -132,9 +132,8 @@ void write_trace_block(BinaryWriter& writer, const SweepTraceBlock& trace) { // request gigabytes and crash the process (length_error/bad_alloc/OOM-kill). The // payload itself is already bounded by the ring slot size, so any count whose // minimum encoding cannot fit in the bytes still available is structurally invalid. -void require_count_fits(std::uint32_t count, std::size_t min_bytes_each, BinaryReader& reader) { - if (min_bytes_each != 0U - && static_cast(count) * min_bytes_each > reader.remaining_bytes()) { +void require_count_fits(std::uint64_t count, std::size_t min_bytes_each, BinaryReader& reader) { + if (min_bytes_each != 0U && count > reader.remaining_bytes() / min_bytes_each) { throw std::runtime_error("Declared element count exceeds remaining payload bytes"); } } @@ -257,6 +256,7 @@ void write_trace_result_payload(BinaryWriter& writer, const ResultPayload& paylo auto read_trace_result_payload(BinaryReader& reader, ResultPayload* payload) -> void { const auto point_count = reader.read(); + require_count_fits(point_count, 12U, reader); payload->frequency_hz.reserve(point_count); payload->trace.reserve(point_count); @@ -296,10 +296,12 @@ void write_image_result_payload(BinaryWriter& writer, const ResultPayload& paylo auto read_image_result_payload(BinaryReader& reader, ResultPayload* payload) -> void { const auto x_count = reader.read(); const auto y_count = reader.read(); + const auto value_count = static_cast(x_count) * y_count; + require_count_fits(value_count + x_count + y_count, sizeof(float), reader); payload->image_x_axis.reserve(x_count); payload->image_y_axis.reserve(y_count); - payload->image_values.reserve(static_cast(x_count) * static_cast(y_count)); + payload->image_values.reserve(static_cast(value_count)); for (std::uint32_t index = 0; index < x_count; ++index) { payload->image_x_axis.push_back(reader.read()); @@ -307,7 +309,7 @@ auto read_image_result_payload(BinaryReader& reader, ResultPayload* payload) -> for (std::uint32_t index = 0; index < y_count; ++index) { payload->image_y_axis.push_back(reader.read()); } - for (std::size_t index = 0; index < static_cast(x_count) * static_cast(y_count); ++index) { + for (std::size_t index = 0; index < value_count; ++index) { payload->image_values.push_back(reader.read()); } } @@ -334,9 +336,14 @@ void write_table_result_payload(BinaryWriter& writer, const ResultPayload& paylo auto read_table_result_payload(BinaryReader& reader, ResultPayload* payload) -> void { const auto column_count = reader.read(); const auto row_count = reader.read(); + if (column_count == 0U && row_count != 0U) { + throw std::runtime_error("Result table cannot have rows without columns"); + } + const auto value_count = static_cast(column_count) * row_count; + require_count_fits(value_count, sizeof(float), reader); payload->table_columns = column_count; - payload->table_values.reserve(static_cast(column_count) * static_cast(row_count)); - for (std::size_t index = 0; index < static_cast(column_count) * static_cast(row_count); ++index) { + payload->table_values.reserve(static_cast(value_count)); + for (std::size_t index = 0; index < value_count; ++index) { payload->table_values.push_back(reader.read()); } } @@ -402,6 +409,7 @@ void write_result_block(BinaryWriter& writer, const ResultBlock& block) { block.combo.output_pos = reader.read(); const auto payload_count = reader.read(); + require_count_fits(payload_count, 3U, reader); block.payloads.reserve(payload_count); for (std::uint32_t index = 0; index < payload_count; ++index) { block.payloads.push_back(read_result_payload(reader)); @@ -478,11 +486,13 @@ auto deserialize_result_collection(std::span bytes) -> Resul const auto collection_payload_count = reader.read(); const auto block_count = reader.read(); + require_count_fits(collection_payload_count, 3U, reader); collection.collection_payloads.reserve(collection_payload_count); for (std::uint32_t index = 0; index < collection_payload_count; ++index) { collection.collection_payloads.push_back(read_result_payload(reader)); } + require_count_fits(block_count, 12U, reader); collection.blocks.reserve(block_count); for (std::uint32_t index = 0; index < block_count; ++index) { collection.blocks.push_back(read_result_block(reader)); diff --git a/data_acq_and_processing/common_cpp/ipc/src/shm_ring.cpp b/data_acq_and_processing/common_cpp/ipc/src/shm_ring.cpp index 4d94610..a37659e 100644 --- a/data_acq_and_processing/common_cpp/ipc/src/shm_ring.cpp +++ b/data_acq_and_processing/common_cpp/ipc/src/shm_ring.cpp @@ -30,7 +30,7 @@ struct alignas(64) ShmRing::Header { std::atomic dropped{}; }; -struct alignas(16) ShmRing::SlotHeader { +struct alignas(8) ShmRing::SlotHeader { std::uint32_t payload_size = 0; std::uint32_t reserved = 0; std::uint64_t sequence = 0; @@ -42,6 +42,7 @@ constexpr std::string_view kRingMagic = "RDRRING2"; constexpr std::uint32_t kRingVersion = 1; constexpr auto kHeaderInitWaitTimeout = std::chrono::milliseconds(1000); constexpr auto kHeaderInitPollInterval = std::chrono::milliseconds(2); +constexpr std::uint64_t kMaxMappedSize = 1ULL << 40U; [[nodiscard]] auto checked_u64_diff(std::uint64_t left, std::uint64_t right) -> std::uint64_t { return left >= right ? left - right : 0; @@ -134,6 +135,22 @@ auto ShmRing::open_or_create( if (slot_size_bytes == 0U) { throw std::runtime_error("Ring slot size must be > 0"); } + if (slot_size_bytes % alignof(SlotHeader) != 0U) { + throw std::runtime_error("Ring slot size must be a multiple of 8"); + } + + const auto stride = sizeof(SlotHeader) + static_cast(slot_size_bytes); + if (stride > (kMaxMappedSize - sizeof(Header)) / capacity) { + throw std::runtime_error("Shared memory ring exceeds 1 TiB for " + name); + } + const auto mapped_size_u64 = sizeof(Header) + stride * capacity; + if (mapped_size_u64 > kMaxMappedSize + || mapped_size_u64 > std::numeric_limits::max() + || mapped_size_u64 > static_cast(std::numeric_limits::max())) { + throw std::runtime_error("Shared memory ring geometry overflows for " + name); + } + const auto mapped_size = static_cast(mapped_size_u64); + bool created = false; int fd = shm_open(name.c_str(), O_RDWR | O_CREAT | O_EXCL, 0660); @@ -149,13 +166,28 @@ auto ShmRing::open_or_create( ScopedFd scoped_fd(fd); - const auto slot_stride = sizeof(SlotHeader) + static_cast(slot_size_bytes); - const auto mapped_size = sizeof(Header) + slot_stride * capacity; if (created) { if (ftruncate(fd, static_cast(mapped_size)) != 0) { throw errno_message("Failed to resize shared memory ring", name); } + } else { + // shm_open publishes the name before the creator calls ftruncate. Mapping + // that zero/short file and reading its header would terminate us with SIGBUS. + const auto deadline = std::chrono::steady_clock::now() + kHeaderInitWaitTimeout; + while (true) { + struct stat info {}; + if (fstat(fd, &info) != 0) { + throw errno_message("Failed to stat shared memory ring", name); + } + if (info.st_size >= 0 && static_cast(info.st_size) >= mapped_size) { + break; + } + if (std::chrono::steady_clock::now() >= deadline) { + throw std::runtime_error("Shared memory ring size is too small for " + name); + } + std::this_thread::sleep_for(kHeaderInitPollInterval); + } } void* mapped = mmap(nullptr, mapped_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); @@ -167,7 +199,9 @@ auto ShmRing::open_or_create( auto* header = static_cast(mapped); if (created) { - std::memset(mapped, 0, mapped_size); + // ftruncate already supplies zero-filled slots; leave them demand-paged + // instead of touching the full ring before any payload is published. + std::memset(mapped, 0, sizeof(Header)); std::memcpy(header->magic, kRingMagic.data(), kRingMagic.size()); header->version = kRingVersion; header->capacity = capacity; @@ -319,14 +353,13 @@ auto ShmRing::push(std::span payload) -> bool { } auto* slot = slot_header(write_seq); - // Seqlock publish: write the payload + size FIRST, then store the slot sequence - // LAST (after a release fence). A consumer that observes sequence == write_seq+1 - // is therefore guaranteed a fully-written payload. (Publishing the sequence before - // the copy, as before, let a consumer copy a half-written slot.) + // Invalidate the old publication before overwriting its bytes: a slow consumer + // can otherwise accept a mixture of old and new data under the old sequence. + std::atomic_ref(slot->sequence).store(0U, std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_seq_cst); slot->payload_size = static_cast(payload.size()); std::memcpy(slot_payload(slot), payload.data(), payload.size()); - std::atomic_thread_fence(std::memory_order_release); - slot->sequence = write_seq + 1U; + std::atomic_ref(slot->sequence).store(write_seq + 1U, std::memory_order_release); std::atomic_thread_fence(std::memory_order_release); header_->write_seq.store(write_seq + 1U, std::memory_order_release); @@ -345,9 +378,9 @@ auto ShmRing::pop(std::vector& payload) -> bool { } auto* slot = slot_header(read_seq); - if (slot->sequence != read_seq + 1U) { + if (std::atomic_ref(slot->sequence).load(std::memory_order_acquire) != read_seq + 1U) { // Slot not published for this seq, or already overwritten (lapped). Resync. - header_->read_seq.store(write_seq, std::memory_order_release); + advance_read_seq(write_seq); return false; } @@ -355,7 +388,7 @@ auto ShmRing::pop(std::vector& payload) -> bool { if (payload_size > header_->slot_size_bytes) { // Corrupt/torn slot: skip it (resync) rather than throw — a single bad slot // must never abort the long-running consumer. - header_->read_seq.store(write_seq, std::memory_order_release); + advance_read_seq(write_seq); return false; } @@ -366,15 +399,21 @@ auto ShmRing::pop(std::vector& payload) -> bool { // Seqlock verify: if the slot sequence changed during the copy, the producer // lapped us mid-read and the payload is torn — discard and resync. std::atomic_thread_fence(std::memory_order_acquire); - if (slot->sequence != read_seq + 1U) { - header_->read_seq.store(write_seq, std::memory_order_release); + if (std::atomic_ref(slot->sequence).load(std::memory_order_acquire) != read_seq + 1U) { + advance_read_seq(write_seq); return false; } - header_->read_seq.store(read_seq + 1U, std::memory_order_release); + advance_read_seq(read_seq + 1U); return true; } +void ShmRing::advance_read_seq(std::uint64_t target) { + auto current = header_->read_seq.load(std::memory_order_relaxed); + while (current < target && !header_->read_seq.compare_exchange_weak( + current, target, std::memory_order_release, std::memory_order_relaxed)) {} +} + auto ShmRing::slot_stride_bytes() const -> std::size_t { return sizeof(SlotHeader) + header_->slot_size_bytes; } @@ -422,18 +461,18 @@ void ShmRing::validate_geometry(const Header& header, std::size_t mapped_size, c if (slot_size_bytes == 0U) { throw std::runtime_error("Shared memory ring slot size is zero for " + name); } + if (slot_size_bytes % alignof(SlotHeader) != 0U) { + throw std::runtime_error("Shared memory ring slot size must be a multiple of 8 for " + name); + } - constexpr auto kMax = std::numeric_limits::max(); - const auto slot_stride = sizeof(SlotHeader) + static_cast(slot_size_bytes); - // Guard each multiply/add against std::size_t overflow before computing expected_size. - if (slot_stride > kMax / static_cast(capacity)) { + const auto stride = sizeof(SlotHeader) + static_cast(slot_size_bytes); + if (stride > (kMaxMappedSize - sizeof(Header)) / capacity) { + throw std::runtime_error("Shared memory ring exceeds 1 TiB for " + name); + } + const auto expected_size = sizeof(Header) + stride * capacity; + if (expected_size > kMaxMappedSize || expected_size > std::numeric_limits::max()) { throw std::runtime_error("Shared memory ring geometry overflows for " + name); } - const auto slots_size = slot_stride * static_cast(capacity); - if (slots_size > kMax - sizeof(Header)) { - throw std::runtime_error("Shared memory ring geometry overflows for " + name); - } - const auto expected_size = sizeof(Header) + slots_size; if (mapped_size < expected_size) { throw std::runtime_error("Shared memory ring size is too small for " + name); diff --git a/data_acq_and_processing/kamil_adc_collector/include/capture_file_writer.h b/data_acq_and_processing/kamil_adc_collector/include/capture_file_writer.h index 350b69f..8d24ae9 100644 --- a/data_acq_and_processing/kamil_adc_collector/include/capture_file_writer.h +++ b/data_acq_and_processing/kamil_adc_collector/include/capture_file_writer.h @@ -27,9 +27,7 @@ public: void initialize_live_plot() const; void initialize_csv(std::size_t channel_count, bool has_di1_trace) const; - void append_csv_packet(const CapturePacket& packet, - double frame_freq_hz, - std::size_t& global_frame_index) const; + void append_csv_packet(const CapturePacket& packet) const; void update_live_plot(const CapturePacket& packet, std::size_t packets_seen, diff --git a/data_acq_and_processing/kamil_adc_collector/src/capture_file_writer.cpp b/data_acq_and_processing/kamil_adc_collector/src/capture_file_writer.cpp index e3dcbf1..8ab2491 100644 --- a/data_acq_and_processing/kamil_adc_collector/src/capture_file_writer.cpp +++ b/data_acq_and_processing/kamil_adc_collector/src/capture_file_writer.cpp @@ -234,12 +234,20 @@ std::string csv_spool_path(const std::string& csv_path) { return replace_extension(csv_path, ".capture_spool.bin"); } +void finish_file(std::ofstream& stream, const std::string& path) { + stream.close(); + if (!stream) { + fail_write("Cannot finish writing file: " + path); + } +} + void write_text_file(const std::string& path, const std::string& text) { std::ofstream file(path, std::ios::binary); if (!file) { fail_write("Cannot open file for writing: " + path); } file << text; + finish_file(file, path); } void write_live_data_script(const std::string& path, const std::string& data_json) { @@ -584,6 +592,7 @@ void write_live_html_document(const std::string& path, const std::string& data_s << " \n" << "\n" << "\n"; + finish_file(html, path); } [[noreturn]] void fail_write(const std::string& message) { @@ -629,11 +638,14 @@ void CaptureFileWriter::initialize_live_plot() const { } void CaptureFileWriter::initialize_csv(std::size_t channel_count, bool has_di1_trace) const { + if (channel_count != 1U && channel_count != 2U) { + fail_write("Capture channel count must be 1 or 2"); + } std::ofstream csv_file(csv_path_, std::ios::binary | std::ios::trunc); if (!csv_file) { fail_write("Cannot prepare CSV output file: " + csv_path_); } - csv_file << ""; + finish_file(csv_file, csv_path_); const std::string spool_path = csv_spool_path(csv_path_); std::ofstream spool(spool_path, std::ios::binary | std::ios::trunc); @@ -647,16 +659,15 @@ void CaptureFileWriter::initialize_csv(std::size_t channel_count, bool has_di1_t static_cast(has_di1_trace ? 1U : 0U) }; spool.write(reinterpret_cast(&header), sizeof(header)); - if (!spool) { - fail_write("Cannot initialize CSV spool: " + spool_path); - } + finish_file(spool, spool_path); } -void CaptureFileWriter::append_csv_packet(const CapturePacket& packet, - double frame_freq_hz, - std::size_t& global_frame_index) const { - (void) frame_freq_hz; - (void) global_frame_index; +void CaptureFileWriter::append_csv_packet(const CapturePacket& packet) const { + if ((packet.channel_count != 1U && packet.channel_count != 2U) + || (packet.channel_count == 2U && packet.ch1.size() != packet.ch2.size()) + || (packet.has_di1_trace && packet.di1.size() != packet.ch1.size())) { + fail_write("Capture packet channel lengths are inconsistent"); + } const std::string spool_path = csv_spool_path(csv_path_); std::ofstream spool(spool_path, std::ios::binary | std::ios::app); @@ -689,9 +700,7 @@ void CaptureFileWriter::append_csv_packet(const CapturePacket& packet, static_cast(frames * sizeof(uint8_t))); } } - if (!spool) { - fail_write("Cannot append packet to CSV spool: " + spool_path); - } + finish_file(spool, spool_path); } void CaptureFileWriter::update_live_plot(const CapturePacket& packet, @@ -763,7 +772,7 @@ void CaptureFileWriter::finalize_csv_from_spool(double frame_freq_hz) const { CsvSpoolPacketHeader packet_header{}; spool.read(reinterpret_cast(&packet_header), sizeof(packet_header)); if (!spool) { - if (spool.eof()) { + if (spool.eof() && spool.gcount() == 0) { break; } fail_write("Cannot read packet header from CSV spool: " + spool_path); @@ -773,6 +782,17 @@ void CaptureFileWriter::finalize_csv_from_spool(double frame_freq_hz) const { const std::size_t channel_count = static_cast(packet_header.channel_count); const std::size_t frames = static_cast(packet_header.frame_count); const bool has_di1_trace = packet_header.has_di1_trace != 0U; + if (channel_count != 1U && channel_count != 2U) { + fail_write("Invalid channel count in CSV spool: " + spool_path); + } + const auto data_position = spool.tellg(); + spool.seekg(0, std::ios::end); + const auto remaining = spool.tellg() - data_position; + spool.seekg(data_position); + const auto bytes_per_frame = channel_count * sizeof(double) + (has_di1_trace ? 1U : 0U); + if (!spool || remaining < 0 || packet_header.frame_count > static_cast(remaining) / bytes_per_frame) { + fail_write("Truncated or invalid packet data in CSV spool: " + spool_path); + } if (!header_written) { csv << "packet_index,frame_index,global_frame_index,time_s,packet_time_s,ch1_v"; @@ -839,6 +859,13 @@ void CaptureFileWriter::finalize_csv_from_spool(double frame_freq_hz) const { } csv.flush(); + if (!csv) { + fail_write("Cannot flush CSV output; capture spool retained: " + csv_path_); + } + csv.close(); + if (!csv) { + fail_write("Cannot close CSV output; capture spool retained: " + csv_path_); + } std::remove(spool_path.c_str()); } @@ -1016,4 +1043,5 @@ void CaptureFileWriter::write_svg(const std::vector& packets, << "\" font-size=\"14\" font-family=\"Segoe UI, Arial, sans-serif\" fill=\"#203040\">DI1\n"; } file << "\n"; + finish_file(file, svg_path_); } diff --git a/data_acq_and_processing/kamil_adc_collector/src/main.cpp b/data_acq_and_processing/kamil_adc_collector/src/main.cpp index ce954e2..258a278 100644 --- a/data_acq_and_processing/kamil_adc_collector/src/main.cpp +++ b/data_acq_and_processing/kamil_adc_collector/src/main.cpp @@ -156,7 +156,8 @@ uint32_t parse_u32(const std::string& text, const std::string& field_name) { const std::string clean = trim_copy(text); char* end = nullptr; const auto value = std::strtoull(clean.c_str(), &end, 0); - if ((end == clean.c_str()) || (*end != '\0') || (value > std::numeric_limits::max())) { + if ((!clean.empty() && clean.front() == '-') || (end == clean.c_str()) + || (*end != '\0') || (value > std::numeric_limits::max())) { fail("Invalid integer for " + field_name + ": " + text); } return static_cast(value); @@ -191,15 +192,6 @@ uint32_t parse_ipv4(const std::string& text) { return (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]; } -std::string ipv4_to_string(uint32_t ip_addr) { - std::ostringstream out; - out << ((ip_addr >> 24) & 0xFF) << '.' - << ((ip_addr >> 16) & 0xFF) << '.' - << ((ip_addr >> 8) & 0xFF) << '.' - << (ip_addr & 0xFF); - return out.str(); -} - uint32_t parse_range(const std::string& text) { const double value = parse_double(text, "range"); if (std::fabs(value - 10.0) < 1e-9) { @@ -2033,8 +2025,14 @@ int run(const Config& cfg) { } expect_ok(api, api.StreamsEnable(device.hnd, enabled_streams), "Enable data streams"); + const double target_frame_count = (cfg.duration_ms / 1000.0) * actual_frame_freq_hz; + if (!std::isfinite(target_frame_count) || target_frame_count < 0.0 + || target_frame_count >= static_cast(std::numeric_limits::max()) + || target_frame_count > static_cast(std::numeric_limits::max())) { + fail("Capture duration and frame clock produce an unsupported frame count"); + } const std::size_t target_frames = std::max( - 1, static_cast(std::llround((cfg.duration_ms / 1000.0) * actual_frame_freq_hz))); + 1, static_cast(std::llround(target_frame_count))); std::cout << "Capture settings:\n" << " clock source: " << sync_mode_to_string(cfg.sync_mode, false) << "\n" @@ -2306,7 +2304,6 @@ int run(const Config& cfg) { if (!fast_tty_avg_stream_mode) { current_packet.reset(target_frames, cfg.channel_count); } - std::size_t csv_global_frame_index = 0; std::size_t packet_avg_steps = 0; std::size_t fast_packet_frames = 0; uint64_t packet_clock_count = 0; @@ -2776,7 +2773,7 @@ int run(const Config& cfg) { } std::cout << "\n"; - writer->append_csv_packet(packet, actual_frame_freq_hz, csv_global_frame_index); + writer->append_csv_packet(packet); packets.push_back(std::move(packet)); if ((cfg.svg_history_packets != 0U) && (packets.size() > cfg.svg_history_packets)) { packets.pop_front(); diff --git a/data_acq_and_processing/kamil_adc_collector/src/tty_protocol_writer.cpp b/data_acq_and_processing/kamil_adc_collector/src/tty_protocol_writer.cpp index dbca921..49ba20b 100644 --- a/data_acq_and_processing/kamil_adc_collector/src/tty_protocol_writer.cpp +++ b/data_acq_and_processing/kamil_adc_collector/src/tty_protocol_writer.cpp @@ -196,7 +196,7 @@ TtyProtocolWriter::TtyProtocolWriter(std::string path, std::size_t ring_capacity : path_(std::move(path)), impl_(std::make_unique(ring_capacity_bytes)) { if (is_character_device_path(path_)) { - impl_->fd = ::open(path_.c_str(), O_WRONLY | O_NOCTTY); + impl_->fd = ::open(path_.c_str(), O_WRONLY | O_NOCTTY | O_NONBLOCK); if (impl_->fd < 0) { throw std::runtime_error(io_error("Cannot open tty output", path_)); } @@ -233,7 +233,20 @@ TtyProtocolWriter::TtyProtocolWriter(std::string path, std::size_t ring_capacity } } - impl_->worker = std::thread([this]() { worker_loop(); }); + try { + const auto flags = ::fcntl(impl_->fd, F_GETFL); + if (flags < 0 || ::fcntl(impl_->fd, F_SETFL, flags | O_NONBLOCK) < 0) { + throw std::runtime_error(io_error("Cannot make tty output nonblocking", path_)); + } + impl_->worker = std::thread([this]() { worker_loop(); }); + } catch (...) { + if (impl_->owns_link) { + ::unlink(path_.c_str()); + } + close_fd_if_open(impl_->slave_fd); + close_fd_if_open(impl_->fd); + throw; + } } TtyProtocolWriter::~TtyProtocolWriter() { @@ -363,12 +376,12 @@ void TtyProtocolWriter::shutdown() { impl_->stop_requested = true; } - close_fd_if_open(impl_->fd); impl_->data_ready_cv.notify_all(); if (impl_->worker.joinable()) { impl_->worker.join(); } + close_fd_if_open(impl_->fd); } void TtyProtocolWriter::enqueue_frame(uint16_t word0, uint16_t word1, uint16_t word2, uint16_t word3) { @@ -402,6 +415,12 @@ void TtyProtocolWriter::worker_loop() { const std::uint8_t* bytes = write_batch.data(); std::size_t remaining = batch_count * kFrameByteCount; while (remaining != 0U) { + { + std::lock_guard lock(impl_->mutex); + if (impl_->stop_requested) { + return; + } + } const ssize_t written = ::write(impl_->fd, bytes, remaining); if (written < 0) { if (errno == EINTR) { diff --git a/data_acq_and_processing/preprocessing/calibration_master/include/channel_bundle_support.hpp b/data_acq_and_processing/preprocessing/calibration_master/include/channel_bundle_support.hpp index 4a85258..dbfe356 100644 --- a/data_acq_and_processing/preprocessing/calibration_master/include/channel_bundle_support.hpp +++ b/data_acq_and_processing/preprocessing/calibration_master/include/channel_bundle_support.hpp @@ -16,6 +16,9 @@ struct ChannelTrace { std::vector samples{}; }; +void validate_channel_trace_layout(const ChannelTrace& trace, const std::string& label, const ipc::ComboKey& combo); +[[nodiscard]] auto frequency_axes_match(std::span left, std::span right) -> bool; + class S21CalibrationBundle { public: void load(const std::string& path); diff --git a/data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp b/data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp index ac8e945..deee56a 100644 --- a/data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp +++ b/data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include "calibrator_interface.hpp" @@ -59,6 +60,8 @@ void validate_raw_trace_layout(const ipc::SweepTraceBlock& trace, const std::str } } +} // namespace + void validate_channel_trace_layout( const ChannelTrace& trace, const std::string& trace_label, @@ -69,6 +72,12 @@ void validate_channel_trace_layout( trace_label + " frequency/channel vector size mismatch for combo " + combo_to_string(combo) ); } + for (std::size_t i = 0; i < trace.samples.size(); ++i) { + if (!std::isfinite(trace.frequency_hz[i]) || !std::isfinite(trace.samples[i].re) + || !std::isfinite(trace.samples[i].im)) { + throw std::runtime_error(trace_label + " contains non-finite values for combo " + combo_to_string(combo)); + } + } } [[nodiscard]] auto frequency_axes_match(std::span left, std::span right) -> bool { @@ -77,6 +86,9 @@ void validate_channel_trace_layout( } for (std::size_t index = 0; index < left.size(); ++index) { + if (!std::isfinite(left[index]) || !std::isfinite(right[index])) { + return false; + } const auto delta = std::fabs(left[index] - right[index]); const auto scale = std::max(std::fabs(left[index]), std::fabs(right[index])); const auto tolerance = std::max(kFrequencyAbsoluteTolerance, scale * kFrequencyRelativeTolerance); @@ -87,6 +99,8 @@ void validate_channel_trace_layout( return true; } +namespace { + void ensure_frequency_axis_match( std::span expected, std::span actual, @@ -122,6 +136,9 @@ void ensure_frequency_axis_match( const std::string& bundle_label ) -> ChannelTrace { validate_raw_trace_layout(trace, bundle_label + " trace"); + if (trace.frequency_hz.empty()) { + throw std::runtime_error(bundle_label + " trace contains no frequency samples"); + } ChannelTrace channel_trace{}; channel_trace.frequency_hz = trace.frequency_hz; @@ -275,8 +292,6 @@ void ensure_combo_coverage( } // namespace void S21CalibrationBundle::load(const std::string& path) { - traces_by_combo_.clear(); - if (path.empty()) { throw std::runtime_error("S21 calibration bundle path must not be empty"); } @@ -311,8 +326,6 @@ auto S21CalibrationBundle::resolve(const ipc::ComboKey& combo) const -> const Ch } void S21ReferenceBundle::load(const std::string& path) { - traces_by_combo_.clear(); - if (path.empty()) { throw std::runtime_error("S21 reference bundle path must not be empty"); } @@ -337,9 +350,8 @@ void S11CalibrationBundle::load( const std::string& short_path, const std::string& load_path ) { - coefficients_by_combo_.clear(); - if (open_path.empty() && short_path.empty() && load_path.empty()) { + coefficients_by_combo_.clear(); // #38: S11 calibration disabled because no OSL paths were provided; surface it instead of silently skipping. std::cerr << "S11 calibration: no open/short/load bundle paths provided; reflection correction disabled\n"; return; @@ -357,13 +369,15 @@ void S11CalibrationBundle::load( ensure_combo_coverage(short_traces, open_traces, "S11 short calibration", "S11 open calibration"); ensure_combo_coverage(load_traces, open_traces, "S11 load calibration", "S11 open calibration"); - coefficients_by_combo_.reserve(open_traces.size()); + decltype(coefficients_by_combo_) coefficients; + coefficients.reserve(open_traces.size()); for (const auto& [combo, open_trace] : open_traces) { - coefficients_by_combo_.insert_or_assign( + coefficients.insert_or_assign( combo, solve_osl_coefficients(combo, open_trace, short_traces.at(combo), load_traces.at(combo)) ); } + coefficients_by_combo_ = std::move(coefficients); } auto S11CalibrationBundle::is_enabled() const noexcept -> bool { @@ -422,9 +436,8 @@ auto S11CalibrationBundle::resolve(const ipc::ComboKey& combo) const -> const Co } void S11ReferenceBundle::load(const std::string& path) { - traces_by_combo_.clear(); - if (path.empty()) { + traces_by_combo_.clear(); // #38: S11 reference disabled because no path was provided; surface it instead of silently skipping. std::cerr << "S11 reference: no bundle path provided; reference normalization disabled\n"; return; diff --git a/data_acq_and_processing/preprocessing/data_preprocessor/src/data_preprocessor.cpp b/data_acq_and_processing/preprocessing/data_preprocessor/src/data_preprocessor.cpp index 44449dd..dbf6e08 100644 --- a/data_acq_and_processing/preprocessing/data_preprocessor/src/data_preprocessor.cpp +++ b/data_acq_and_processing/preprocessing/data_preprocessor/src/data_preprocessor.cpp @@ -1,14 +1,13 @@ #include "data_preprocessor.hpp" +#include "runtime_wait.hpp" #include -#include #include #include #include #include #include #include -#include #include namespace radar::preprocessing { @@ -117,7 +116,6 @@ auto DataPreprocessor::try_pop_raw_collection( ipc::RawSweepCollection* out_collection ) -> bool { if (!raw_ring_.pop(serialized_raw)) { - std::this_thread::sleep_for(std::chrono::milliseconds(config_.runtime.idle_sleep_ms)); return false; } @@ -145,6 +143,7 @@ void DataPreprocessor::run(const std::atomic& stop_requested) { while (!stop_requested.load(std::memory_order_relaxed)) { try { if (!try_pop_raw_collection(serialized_raw, &raw_collection)) { + (void)config::wait_or_stop(config_.runtime.idle_sleep_ms, stop_requested); continue; } const auto preprocessed_collection = preprocess_collection(raw_collection); @@ -174,7 +173,7 @@ void DataPreprocessor::run(const std::atomic& stop_requested) { // Back-off floored at 1ms (the empty-ring sleep is never reached while a // persistent error keeps the ring non-empty) so the catch cannot busy-spin // a CPU core even when idle_sleep_ms is configured to 0. - std::this_thread::sleep_for(std::chrono::milliseconds(std::max(1, config_.runtime.idle_sleep_ms))); + (void)config::wait_or_stop(std::max(1U, config_.runtime.idle_sleep_ms), stop_requested); } } } diff --git a/data_acq_and_processing/preprocessing/reference_master/src/reference_master.cpp b/data_acq_and_processing/preprocessing/reference_master/src/reference_master.cpp index accad52..7c6bf62 100644 --- a/data_acq_and_processing/preprocessing/reference_master/src/reference_master.cpp +++ b/data_acq_and_processing/preprocessing/reference_master/src/reference_master.cpp @@ -1,7 +1,5 @@ #include "reference_master.hpp" -#include -#include #include #include #include @@ -15,41 +13,10 @@ namespace radar::preprocessing { namespace { -constexpr float kFrequencyRelativeTolerance = 1e-5F; -constexpr float kFrequencyAbsoluteTolerance = 1e-3F; - [[nodiscard]] auto combo_to_string(const ipc::ComboKey& combo) -> std::string { return "input=" + std::to_string(combo.input_pos) + " output=" + std::to_string(combo.output_pos); } -void validate_channel_trace_layout( - const ChannelTrace& trace, - const std::string& trace_label, - const ipc::ComboKey& combo -) { - if (trace.frequency_hz.size() != trace.samples.size()) { - throw std::runtime_error( - trace_label + " frequency/channel vector size mismatch for combo " + combo_to_string(combo) - ); - } -} - -[[nodiscard]] auto frequency_axes_match(std::span left, std::span right) -> bool { - if (left.size() != right.size()) { - return false; - } - - for (std::size_t index = 0; index < left.size(); ++index) { - const auto delta = std::fabs(left[index] - right[index]); - const auto scale = std::max(std::fabs(left[index]), std::fabs(right[index])); - const auto tolerance = std::max(kFrequencyAbsoluteTolerance, scale * kFrequencyRelativeTolerance); - if (delta > tolerance) { - return false; - } - } - return true; -} - } // namespace void ReferenceMaster::load_s21_reference_bundle(const std::string& path) { diff --git a/data_acq_and_processing/processing/data_processor/include/data_processor.hpp b/data_acq_and_processing/processing/data_processor/include/data_processor.hpp index 777094d..e209e57 100644 --- a/data_acq_and_processing/processing/data_processor/include/data_processor.hpp +++ b/data_acq_and_processing/processing/data_processor/include/data_processor.hpp @@ -54,6 +54,7 @@ class DataProcessor { void publish_locator(const ipc::ResultCollection& collection, const ProcessingLiveConfig& live_config); const config::RunConfig& config_; + config::RunConfig processing_config_; ipc::ShmRing& preprocessed_ring_; ipc::ShmRing& results_ring_; ProcessorRegistry processors_{}; diff --git a/data_acq_and_processing/processing/data_processor/include/processing_live_config.hpp b/data_acq_and_processing/processing/data_processor/include/processing_live_config.hpp index a21323a..3074803 100644 --- a/data_acq_and_processing/processing/data_processor/include/processing_live_config.hpp +++ b/data_acq_and_processing/processing/data_processor/include/processing_live_config.hpp @@ -2,9 +2,12 @@ #include #include +#include #include #include +#include "run_config.hpp" + namespace radar::processing { enum class HistoryCommand { @@ -34,12 +37,14 @@ struct ProcessingLiveConfig { float pass_through_y_max_db = 0.0F; std::string bscan_axis = "abs"; std::string bscan_channel = "s21"; - float bscan_cut_m = 0.824F; - float bscan_max_depth_m = 1.0F; + float bscan_cut_m = 0.0F; + float bscan_max_depth_m = 8.0F; float bscan_gain = 1.0F; float bscan_start_freq_mhz = 100.0F; float bscan_stop_freq_mhz = 8800.0F; LegacyGprMode legacy_gpr_mode = LegacyGprMode::Point; + // Absent in legacy live files: use the initial run_config geometry and medium. + std::optional gpr{}; std::vector gpr_input_positions{}; std::vector gpr_output_positions{}; float gpr_min_depth_m = 2.0F; @@ -108,11 +113,12 @@ struct ProcessingLiveConfig { bool reprocess_current_result = true; std::uint64_t history_command_seq = 0; HistoryCommand history_command = HistoryCommand::None; + std::uint64_t settings_revision = 0; }; class ProcessingLiveConfigLoader { public: - explicit ProcessingLiveConfigLoader(std::string path); + ProcessingLiveConfigLoader(std::string path, const config::RunConfig& run_config); [[nodiscard]] auto current() const -> ProcessingLiveConfig; [[nodiscard]] auto refresh_if_needed() -> ProcessingLiveConfig; @@ -122,6 +128,7 @@ class ProcessingLiveConfigLoader { [[nodiscard]] auto read_file_text() const -> std::string; std::string path_{}; + const config::RunConfig& run_config_; ProcessingLiveConfig current_{}; std::string last_json_text_{}; bool has_last_json_text_ = false; diff --git a/data_acq_and_processing/processing/data_processor/src/data_processor.cpp b/data_acq_and_processing/processing/data_processor/src/data_processor.cpp index 5f532aa..3e76a67 100644 --- a/data_acq_and_processing/processing/data_processor/src/data_processor.cpp +++ b/data_acq_and_processing/processing/data_processor/src/data_processor.cpp @@ -1,4 +1,5 @@ #include "data_processor.hpp" +#include "runtime_wait.hpp" #include #include @@ -7,7 +8,6 @@ #include #include #include -#include #include #include "bscan_processor.hpp" @@ -27,10 +27,15 @@ constexpr std::size_t kBscanReplayWindow = 50U; return std::min({preprocessed_capacity, results_capacity, kBscanReplayWindow}); } -void publish_result_collection(const ipc::ResultCollection& collection, ipc::ShmRing& results_ring) { +void publish_result_collection( + const ipc::ResultCollection& collection, ipc::ShmRing& results_ring, std::uint64_t& oversized_results +) { const auto out_bytes = ipc::serialize_result_collection(collection); - if (!results_ring.push(out_bytes)) { - throw std::runtime_error("Results ring slot is too small for serialized collection"); + if (!results_ring.push(out_bytes) && ++oversized_results % 100U == 1U) { + std::cerr << "data_processor: result " << collection.collection_id << " requires " << out_bytes.size() + << " bytes, results ring slot holds " << results_ring.slot_size_bytes() + << "; increase rings.results.slot_size_bytes or reduce processing output" + << " (dropped total=" << oversized_results << ")\n"; } } @@ -44,11 +49,12 @@ DataProcessor::DataProcessor( radar::locator::TcpServer* locator_server ) : config_(config), + processing_config_(config), preprocessed_ring_(preprocessed_ring), results_ring_(results_ring), processors_(std::move(processors)), default_processor_mode_(kDefaultProcessorMode), - live_config_loader_(config.runtime.processing_live_config_path), + live_config_loader_(config.runtime.processing_live_config_path, config), locator_server_(locator_server) { if (processors_.empty()) { throw std::runtime_error("DataProcessor requires at least one processor"); @@ -66,6 +72,7 @@ void DataProcessor::run(const std::atomic& stop_requested) { std::uint64_t last_applied_history_command_seq = 0; std::uint64_t error_count = 0; std::uint64_t consecutive_errors = 0; + std::uint64_t oversized_results = 0; // track the socket-fed speed used for the last reprocess so a change // arriving without a live-config revision bump still triggers a reprocess of // the current result (gated below by reprocess_current_result). @@ -82,6 +89,9 @@ void DataProcessor::run(const std::atomic& stop_requested) { live_config.gpr_speed_m_s = static_cast(*current_socket_speed); } const auto live_revision = live_config_loader_.revision(); + if (live_revision != last_replayed_revision) { + processing_config_.gpr = live_config.gpr.value_or(config_.gpr); + } auto& processor = resolve_processor(live_config); const bool socket_speed_dirty = current_socket_speed != last_reprocessed_socket_speed; @@ -108,7 +118,7 @@ void DataProcessor::run(const std::atomic& stop_requested) { processor, live_config ); - publish_result_collection(replay_result, results_ring_); + publish_result_collection(replay_result, results_ring_, oversized_results); publish_locator(replay_result, live_config); } } else if (!preprocessed_history.empty()) { @@ -118,7 +128,7 @@ void DataProcessor::run(const std::atomic& stop_requested) { processor, live_config ); - publish_result_collection(replay_result, results_ring_); + publish_result_collection(replay_result, results_ring_, oversized_results); publish_locator(replay_result, live_config); } last_replayed_revision = live_revision; @@ -156,7 +166,7 @@ void DataProcessor::run(const std::atomic& stop_requested) { processor, live_config ); - publish_result_collection(result_collection, results_ring_); + publish_result_collection(result_collection, results_ring_, oversized_results); if (!fed_back) { publish_locator(result_collection, live_config); } @@ -164,14 +174,8 @@ void DataProcessor::run(const std::atomic& stop_requested) { continue; } - std::this_thread::sleep_for(std::chrono::milliseconds(config_.runtime.idle_sleep_ms)); + (void)config::wait_or_stop(config_.runtime.idle_sleep_ms, stop_requested); } catch (const std::exception& exc) { - // A ring-slot-too-small failure is a permanent config error (the slot - // cannot hold a serialized collection): swallowing it would silently - // drop every collection forever, so fail fast and let the operator fix it. - if (std::string(exc.what()).find("slot is too small") != std::string::npos) { - throw; - } // A single bad collection (torn ring slot, decode/processing error) must // not kill the long-running processor: drop it and keep going. ++error_count; @@ -189,7 +193,7 @@ void DataProcessor::run(const std::atomic& stop_requested) { } // Back-off (floored at 1ms even if idle_sleep_ms==0) so a persistent error // cannot busy-spin a core or flood the log. - std::this_thread::sleep_for(std::chrono::milliseconds(std::max(1, config_.runtime.idle_sleep_ms))); + (void)config::wait_or_stop(std::max(1U, config_.runtime.idle_sleep_ms), stop_requested); } } } @@ -201,11 +205,20 @@ auto DataProcessor::process_collection( const ProcessingLiveConfig& live_config ) -> ipc::ResultCollection { const auto started_at = std::chrono::steady_clock::now(); - auto result = processor.process_collection(config_, preprocessed, previous_collections, live_config); + auto result = processor.process_collection(processing_config_, preprocessed, previous_collections, live_config); const auto finished_at = std::chrono::steady_clock::now(); result.processing_duration_ns = static_cast( std::chrono::duration_cast(finished_at - started_at).count() ); + // Four 16-bit words retain the uint64 exactly in the existing float32 table format. + ipc::ResultPayload revision{}; + revision.processing_name = "processing_revision"; + revision.kind = ipc::ResultKind::TableF32; + revision.table_columns = 4U; + for (unsigned shift = 0; shift < 64; shift += 16) { + revision.table_values.push_back(static_cast((live_config.settings_revision >> shift) & 0xffffU)); + } + result.collection_payloads.push_back(std::move(revision)); return result; } diff --git a/data_acq_and_processing/processing/data_processor/src/processing_live_config.cpp b/data_acq_and_processing/processing/data_processor/src/processing_live_config.cpp index 2ef9432..da57661 100644 --- a/data_acq_and_processing/processing/data_processor/src/processing_live_config.cpp +++ b/data_acq_and_processing/processing/data_processor/src/processing_live_config.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -92,16 +93,55 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s throw std::runtime_error(field_name + " must be one of: s21, s11"); } +[[nodiscard]] auto parse_f32(const Json& value, const std::string& field_name) -> float { + if (!value.is_number()) { + throw std::runtime_error(field_name + " must be number"); + } + const auto number = value.get(); + if (!std::isfinite(number) || std::fabs(number) > std::numeric_limits::max()) { + throw std::runtime_error(field_name + " must fit finite float32"); + } + return static_cast(number); +} + +template +void assign_optional(const Json& root, const char* key, T& target) { + const auto found = root.find(key); + if (found == root.end()) { + return; + } + const auto field_name = std::string("processing.") + key; + if constexpr (std::is_same_v) { + target = parse_f32(*found, field_name); + } else { + const bool valid = std::is_same_v ? found->is_boolean() : found->is_string(); + if (!valid) { + throw std::runtime_error(field_name + " has invalid type"); + } + target = found->get(); + } +} + [[nodiscard]] auto parse_u64_number(const Json& value, const std::string& field_name) -> std::uint64_t { if (!value.is_number()) { throw std::runtime_error(field_name + " must be number"); } + if (value.is_number_unsigned()) { + return value.get(); + } + if (value.is_number_integer()) { + const auto integer = value.get(); + if (integer < 0) { + throw std::runtime_error(field_name + " is out of uint64 range"); + } + return static_cast(integer); + } const double numeric = value.get(); - if (numeric < 0.0 || numeric > static_cast(std::numeric_limits::max())) { + if (!std::isfinite(numeric) || numeric < 0.0 || numeric >= static_cast(std::numeric_limits::max())) { throw std::runtime_error(field_name + " is out of uint64 range"); } const double rounded = std::round(numeric); - if (std::fabs(numeric - rounded) > 1e-6) { + if (numeric != rounded) { throw std::runtime_error(field_name + " must be integer"); } return static_cast(rounded); @@ -128,7 +168,31 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s return result; } -[[nodiscard]] auto parse_live_config(const std::string& json_text, const std::string& path) -> ProcessingLiveConfig { +template +auto parse_geometry(const Json& value, const char* position_field, std::uint32_t Entry::*position_member) + -> std::vector { + if (!value.is_array()) { + throw std::runtime_error("processing geometry must be an array"); + } + std::vector entries{}; + entries.reserve(value.size()); + for (const auto& record : value) { + if (!record.is_object()) { + throw std::runtime_error("processing geometry entry must be an object"); + } + Entry entry{}; + entry.*position_member = parse_u32_number(record.at(position_field), position_field); + entry.x_m = parse_f32(record.at("x_m"), "processing.x_m"); + assign_optional(record, "y_m", entry.y_m); + assign_optional(record, "z_m", entry.z_m); + entries.push_back(entry); + } + return entries; +} + +[[nodiscard]] auto parse_live_config( + const std::string& json_text, const std::string& path, const config::RunConfig& run_config +) -> ProcessingLiveConfig { if (json_text.empty()) { throw std::runtime_error("Processing live config is empty: " + path); } @@ -145,12 +209,23 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s } ProcessingLiveConfig config{}; - if (const auto found = root.find("processor_mode"); found != root.end()) { - if (!found->is_string()) { - throw std::runtime_error("processing.processor_mode must be string"); - } - config.processor_mode = found->get(); + if (const auto found = root.find("settings_revision"); found != root.end()) { + config.settings_revision = parse_u64_number(*found, "processing.settings_revision"); } + if (root.contains("relative_permittivity") || root.contains("tx_geometry") || root.contains("rx_geometry")) { + config.gpr = run_config.gpr; + if (const auto found = root.find("relative_permittivity"); found != root.end()) { + config.gpr->relative_permittivity = parse_f32(*found, "processing.relative_permittivity"); + } + if (const auto found = root.find("tx_geometry"); found != root.end()) { + config.gpr->tx_geometry = parse_geometry(*found, "output_pos", &config::GprTxGeometry::output_pos); + } + if (const auto found = root.find("rx_geometry"); found != root.end()) { + config.gpr->rx_geometry = parse_geometry(*found, "input_pos", &config::GprRxGeometry::input_pos); + } + config::validate_gpr_config(*config.gpr, run_config); + } + assign_optional(root, "processor_mode", config.processor_mode); if (const auto found = root.find("pass_through_channel"); found != root.end()) { if (!found->is_string()) { throw std::runtime_error("processing.pass_through_channel must be string"); @@ -158,66 +233,21 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s config.pass_through_channel = parse_s_parameter_channel(found->get(), "processing.pass_through_channel"); } - if (const auto found = root.find("pass_through_fixed_y_enabled"); found != root.end()) { - if (!found->is_boolean()) { - throw std::runtime_error("processing.pass_through_fixed_y_enabled must be bool"); - } - config.pass_through_fixed_y_enabled = found->get(); - } - if (const auto found = root.find("pass_through_y_min_db"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.pass_through_y_min_db must be number"); - } - config.pass_through_y_min_db = static_cast(found->get()); - } - if (const auto found = root.find("pass_through_y_max_db"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.pass_through_y_max_db must be number"); - } - config.pass_through_y_max_db = static_cast(found->get()); - } - if (const auto found = root.find("bscan_axis"); found != root.end()) { - if (!found->is_string()) { - throw std::runtime_error("processing.bscan_axis must be string"); - } - config.bscan_axis = found->get(); - } + assign_optional(root, "pass_through_fixed_y_enabled", config.pass_through_fixed_y_enabled); + assign_optional(root, "pass_through_y_min_db", config.pass_through_y_min_db); + assign_optional(root, "pass_through_y_max_db", config.pass_through_y_max_db); + assign_optional(root, "bscan_axis", config.bscan_axis); if (const auto found = root.find("bscan_channel"); found != root.end()) { if (!found->is_string()) { throw std::runtime_error("processing.bscan_channel must be string"); } config.bscan_channel = parse_s_parameter_channel(found->get(), "processing.bscan_channel"); } - if (const auto found = root.find("bscan_cut_m"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.bscan_cut_m must be number"); - } - config.bscan_cut_m = static_cast(found->get()); - } - if (const auto found = root.find("bscan_max_depth_m"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.bscan_max_depth_m must be number"); - } - config.bscan_max_depth_m = static_cast(found->get()); - } - if (const auto found = root.find("bscan_gain"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.bscan_gain must be number"); - } - config.bscan_gain = static_cast(found->get()); - } - if (const auto found = root.find("bscan_start_freq_mhz"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.bscan_start_freq_mhz must be number"); - } - config.bscan_start_freq_mhz = static_cast(found->get()); - } - if (const auto found = root.find("bscan_stop_freq_mhz"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.bscan_stop_freq_mhz must be number"); - } - config.bscan_stop_freq_mhz = static_cast(found->get()); - } + assign_optional(root, "bscan_cut_m", config.bscan_cut_m); + assign_optional(root, "bscan_max_depth_m", config.bscan_max_depth_m); + assign_optional(root, "bscan_gain", config.bscan_gain); + assign_optional(root, "bscan_start_freq_mhz", config.bscan_start_freq_mhz); + assign_optional(root, "bscan_stop_freq_mhz", config.bscan_stop_freq_mhz); if (const auto found = root.find("legacy_gpr_mode"); found != root.end()) { if (!found->is_string()) { throw std::runtime_error("processing.legacy_gpr_mode must be string"); @@ -237,36 +267,11 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s if (const auto found = root.find("gpr_output_positions"); found != root.end()) { config.gpr_output_positions = parse_u32_array(*found, "processing.gpr_output_positions"); } - if (const auto found = root.find("gpr_min_depth_m"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.gpr_min_depth_m must be number"); - } - config.gpr_min_depth_m = static_cast(found->get()); - } - if (const auto found = root.find("gpr_max_depth_m"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.gpr_max_depth_m must be number"); - } - config.gpr_max_depth_m = static_cast(found->get()); - } - if (const auto found = root.find("gpr_range_comp_power"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.gpr_range_comp_power must be number"); - } - config.gpr_range_comp_power = static_cast(found->get()); - } - if (const auto found = root.find("gpr_object_min_frac"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.gpr_object_min_frac must be number"); - } - config.gpr_object_min_frac = static_cast(found->get()); - } - if (const auto found = root.find("gpr_comp_power"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.gpr_comp_power must be number"); - } - config.gpr_comp_power = static_cast(found->get()); - } + assign_optional(root, "gpr_min_depth_m", config.gpr_min_depth_m); + assign_optional(root, "gpr_max_depth_m", config.gpr_max_depth_m); + assign_optional(root, "gpr_range_comp_power", config.gpr_range_comp_power); + assign_optional(root, "gpr_object_min_frac", config.gpr_object_min_frac); + assign_optional(root, "gpr_comp_power", config.gpr_comp_power); if (const auto found = root.find("gpr_score_mode"); found != root.end()) { if (!found->is_string()) { throw std::runtime_error("processing.gpr_score_mode must be string"); @@ -279,30 +284,10 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s } config.gpr_motion_mode = parse_gpr_motion_mode(found->get(), "processing.gpr_motion_mode"); } - if (const auto found = root.find("gpr_speed_m_s"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.gpr_speed_m_s must be number"); - } - config.gpr_speed_m_s = static_cast(found->get()); - } - if (const auto found = root.find("gpr_look_angle_deg"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.gpr_look_angle_deg must be number"); - } - config.gpr_look_angle_deg = static_cast(found->get()); - } - if (const auto found = root.find("gpr_direction_sign"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.gpr_direction_sign must be number"); - } - config.gpr_direction_sign = static_cast(found->get()); - } - if (const auto found = root.find("gpr_apply_freq_phase_correction"); found != root.end()) { - if (!found->is_boolean()) { - throw std::runtime_error("processing.gpr_apply_freq_phase_correction must be bool"); - } - config.gpr_apply_freq_phase_correction = found->get(); - } + assign_optional(root, "gpr_speed_m_s", config.gpr_speed_m_s); + assign_optional(root, "gpr_look_angle_deg", config.gpr_look_angle_deg); + assign_optional(root, "gpr_direction_sign", config.gpr_direction_sign); + assign_optional(root, "gpr_apply_freq_phase_correction", config.gpr_apply_freq_phase_correction); if (const auto found = root.find("gpr_reference_mode"); found != root.end()) { if (!found->is_string()) { throw std::runtime_error("processing.gpr_reference_mode must be string"); @@ -310,70 +295,25 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s config.gpr_reference_mode = parse_legacy_gpr_reference_mode(found->get(), "processing.gpr_reference_mode"); } - if (const auto found = root.find("gpr_snr_thresh"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.gpr_snr_thresh must be number"); - } - config.gpr_snr_thresh = static_cast(found->get()); - } - if (const auto found = root.find("gpr_snr_comp_max"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.gpr_snr_comp_max must be number"); - } - config.gpr_snr_comp_max = static_cast(found->get()); - } - if (const auto found = root.find("gpr_start_freq_mhz"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.gpr_start_freq_mhz must be number"); - } - config.gpr_start_freq_mhz = static_cast(found->get()); - } - if (const auto found = root.find("gpr_stop_freq_mhz"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.gpr_stop_freq_mhz must be number"); - } - config.gpr_stop_freq_mhz = static_cast(found->get()); - } - if (const auto found = root.find("gpr_background_subtract_enabled"); found != root.end()) { - if (!found->is_boolean()) { - throw std::runtime_error("processing.gpr_background_subtract_enabled must be bool"); - } - config.gpr_background_subtract_enabled = found->get(); - } + assign_optional(root, "gpr_snr_thresh", config.gpr_snr_thresh); + assign_optional(root, "gpr_snr_comp_max", config.gpr_snr_comp_max); + assign_optional(root, "gpr_start_freq_mhz", config.gpr_start_freq_mhz); + assign_optional(root, "gpr_stop_freq_mhz", config.gpr_stop_freq_mhz); + assign_optional(root, "gpr_background_subtract_enabled", config.gpr_background_subtract_enabled); if (const auto found = root.find("gpr_background_mean_count"); found != root.end()) { config.gpr_background_mean_count = parse_u32_number(*found, "processing.gpr_background_mean_count"); } - if (const auto found = root.find("gpr_remove_sidelobe_objects_enabled"); found != root.end()) { - if (!found->is_boolean()) { - throw std::runtime_error("processing.gpr_remove_sidelobe_objects_enabled must be bool"); - } - config.gpr_remove_sidelobe_objects_enabled = found->get(); - } - if (const auto found = root.find("gpr_imaging_plane_y_m"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.gpr_imaging_plane_y_m must be number"); - } - config.gpr_imaging_plane_y_m = static_cast(found->get()); - } + assign_optional(root, "gpr_remove_sidelobe_objects_enabled", config.gpr_remove_sidelobe_objects_enabled); + assign_optional(root, "gpr_imaging_plane_y_m", config.gpr_imaging_plane_y_m); for (const auto& [key, target] : { std::pair{"gpr_visible_x_min_m", &config.gpr_visible_x_min_m}, std::pair{"gpr_visible_x_max_m", &config.gpr_visible_x_max_m}, std::pair{"gpr_visible_z_min_m", &config.gpr_visible_z_min_m}, std::pair{"gpr_visible_z_max_m", &config.gpr_visible_z_max_m}, }) { - if (const auto found = root.find(key); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error(std::string("processing.") + key + " must be number"); - } - *target = static_cast(found->get()); - } - } - if (const auto found = root.find("legacy_gpr_min_visible_pair_count"); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error("processing.legacy_gpr_min_visible_pair_count must be number"); - } - config.legacy_gpr_min_visible_pair_count = static_cast(found->get()); + assign_optional(root, key, *target); } + assign_optional(root, "legacy_gpr_min_visible_pair_count", config.legacy_gpr_min_visible_pair_count); if (const auto found = root.find("gpr_max_detected_objects_to_draw"); found != root.end()) { config.gpr_max_detected_objects_to_draw = parse_u32_number(*found, "processing.gpr_max_detected_objects_to_draw"); @@ -394,36 +334,16 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s std::pair{"sar3d_bp_ground_depth_m", &config.sar3d_bp_ground_depth_m}, std::pair{"sar3d_comp_range_power", &config.sar3d_comp_range_power}, }) { - if (const auto found = root.find(key); found != root.end()) { - if (!found->is_number()) { - throw std::runtime_error(std::string("processing.") + key + " must be number"); - } - *target = static_cast(found->get()); - } + assign_optional(root, key, *target); } for (const auto& [key, target] : { std::pair{"sar3d_use_compensation", &config.sar3d_use_compensation}, std::pair{"sar3d_normalize_each_frame", &config.sar3d_normalize_each_frame}, }) { - if (const auto found = root.find(key); found != root.end()) { - if (!found->is_boolean()) { - throw std::runtime_error(std::string("processing.") + key + " must be bool"); - } - *target = found->get(); - } - } - if (const auto found = root.find("ignore_socket_speed"); found != root.end()) { - if (!found->is_boolean()) { - throw std::runtime_error("processing.ignore_socket_speed must be bool"); - } - config.ignore_socket_speed = found->get(); - } - if (const auto found = root.find("reprocess_current_result"); found != root.end()) { - if (!found->is_boolean()) { - throw std::runtime_error("processing.reprocess_current_result must be bool"); - } - config.reprocess_current_result = found->get(); + assign_optional(root, key, *target); } + assign_optional(root, "ignore_socket_speed", config.ignore_socket_speed); + assign_optional(root, "reprocess_current_result", config.reprocess_current_result); if (const auto found = root.find("history_command_seq"); found != root.end()) { config.history_command_seq = parse_u64_number(*found, "processing.history_command_seq"); } @@ -438,8 +358,9 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s } // namespace -ProcessingLiveConfigLoader::ProcessingLiveConfigLoader(std::string path) +ProcessingLiveConfigLoader::ProcessingLiveConfigLoader(std::string path, const config::RunConfig& run_config) : path_(std::move(path)), + run_config_(run_config), next_check_at_(std::chrono::steady_clock::now()) { if (path_.empty()) { throw std::runtime_error("Processing live config path must not be empty"); @@ -472,7 +393,7 @@ auto ProcessingLiveConfigLoader::refresh_if_needed() -> ProcessingLiveConfig { return current_; } - current_ = parse_live_config(json_text, path_); + current_ = parse_live_config(json_text, path_, run_config_); last_json_text_ = json_text; has_last_json_text_ = true; ++revision_; diff --git a/data_acq_and_processing/processing/locator/include/locator/tcp_server.hpp b/data_acq_and_processing/processing/locator/include/locator/tcp_server.hpp index a9500b1..d0d261e 100644 --- a/data_acq_and_processing/processing/locator/include/locator/tcp_server.hpp +++ b/data_acq_and_processing/processing/locator/include/locator/tcp_server.hpp @@ -88,6 +88,7 @@ class ClientSession { private: void writer_loop(); void reader_loop(std::atomic& shared_vlc_slot, std::atomic& shared_vlc_at_ns); + void finish_loop(); int socket_fd_; std::string peer_name_; @@ -148,11 +149,7 @@ class TcpServer { void acceptor_loop(); void enroll_client(std::unique_ptr session); void broadcast_packet(const std::vector& packet); - void reap_finished_clients(); - // Best-effort reap from the publish path: acquires clients_mutex_ via try-lock - // (skips if contended) and joins exited sessions outside the lock. Keeps - // clients_ bounded even when no new connections arrive to trigger a reap. - void try_reap_finished_clients(); + void reap_finished_clients(bool wait_for_lock = true); void cache_latest_packet(std::vector packet); [[nodiscard]] auto latest_packet_copy() const -> std::optional>; diff --git a/data_acq_and_processing/processing/locator/src/payload_builder.cpp b/data_acq_and_processing/processing/locator/src/payload_builder.cpp index f3f1bd2..4611a99 100644 --- a/data_acq_and_processing/processing/locator/src/payload_builder.cpp +++ b/data_acq_and_processing/processing/locator/src/payload_builder.cpp @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include #include #include @@ -96,17 +98,6 @@ constexpr std::uint32_t kMinTableColumns = 3U; // [x_m, z_m, score, ...] return stream.str(); } -// Wall-clock epoch milliseconds at packet-build time. Unlike the human-readable -// `tim` field (which has no date and rolls over at midnight), this is a -// monotonically increasing generation stamp consumers can use to compute the -// true age of a snapshot and detect a stalled pipeline. -[[nodiscard]] auto epoch_millis_now() -> std::int64_t { - using Clock = std::chrono::system_clock; - return std::chrono::duration_cast( - Clock::now().time_since_epoch() - ).count(); -} - void append_u32_little_endian(std::vector& buffer, std::uint32_t value) { buffer.push_back(static_cast(value & 0xFFU)); buffer.push_back(static_cast((value >> 8U) & 0xFFU)); @@ -200,15 +191,6 @@ auto build_payload_json( {"obs", std::move(obs_array)}, }; - // `gen` is a build-time generation stamp (epoch ms) that lets a consumer compute - // a packet's true age and flag a cached/stalled snapshot — something `tim` cannot - // express, having no date and rolling over at midnight. It is currently NOT sent; - // set this flag to true to re-enable it (epoch_millis_now() is kept ready for that). - constexpr bool kEmitGenerationStamp = false; - if (kEmitGenerationStamp) { - root["gen"] = epoch_millis_now(); - } - return root.dump(); } @@ -216,6 +198,9 @@ auto encode_packet( const std::string& payload_json, std::uint32_t device_id ) -> std::vector { + if (payload_json.size() > std::numeric_limits::max()) { + throw std::runtime_error("Locator packet payload exceeds uint32 wire limit"); + } const auto payload_size = static_cast(payload_json.size()); std::vector packet; packet.reserve(static_cast(8U) + payload_json.size()); diff --git a/data_acq_and_processing/processing/locator/src/tcp_server.cpp b/data_acq_and_processing/processing/locator/src/tcp_server.cpp index e13628c..4158a51 100644 --- a/data_acq_and_processing/processing/locator/src/tcp_server.cpp +++ b/data_acq_and_processing/processing/locator/src/tcp_server.cpp @@ -225,10 +225,16 @@ void ClientSession::start( std::atomic& shared_vlc_slot, std::atomic& shared_vlc_at_ns ) { - writer_thread_ = std::thread([this]() { writer_loop(); }); - reader_thread_ = std::thread([this, &shared_vlc_slot, &shared_vlc_at_ns]() { - reader_loop(shared_vlc_slot, shared_vlc_at_ns); - }); + try { + writer_thread_ = std::thread([this]() { writer_loop(); }); + reader_thread_ = std::thread([this, &shared_vlc_slot, &shared_vlc_at_ns]() { + reader_loop(shared_vlc_slot, shared_vlc_at_ns); + }); + } catch (...) { + request_stop(); + join(); + throw; + } } void ClientSession::enqueue(std::vector packet) { @@ -278,14 +284,7 @@ void ClientSession::writer_loop() { break; } } - request_stop(); - // Mark exited only once BOTH loops have finished; otherwise a writer-only - // death (e.g. a send timeout while the reader still blocks on recv) would - // never be reaped. request_stop() above shuts the socket so the reader's - // recv() returns promptly. - if (loops_finished_.fetch_add(1, std::memory_order_acq_rel) + 1U == 2U) { - exited_.store(true, std::memory_order_release); - } + finish_loop(); } void ClientSession::reader_loop( @@ -295,51 +294,58 @@ void ClientSession::reader_loop( std::array header_buffer{}; std::vector payload_buffer; - while (!stop_requested_.load(std::memory_order_acquire)) { - if (!read_exact(socket_fd_, header_buffer.data(), header_buffer.size(), stop_requested_)) { - break; - } - const auto payload_len = decode_u32_little_endian(header_buffer.data() + 4U); - if (payload_len > max_payload_bytes_) { - log_warning( - "closing client " + peer_name_ + " after payload size " - + std::to_string(payload_len) + " exceeded the configured limit" - ); - break; - } + try { + while (!stop_requested_.load(std::memory_order_acquire)) { + if (!read_exact(socket_fd_, header_buffer.data(), header_buffer.size(), stop_requested_)) { + break; + } + const auto payload_len = decode_u32_little_endian(header_buffer.data() + 4U); + if (payload_len > max_payload_bytes_) { + log_warning( + "closing client " + peer_name_ + " after payload size " + + std::to_string(payload_len) + " exceeded the configured limit" + ); + break; + } - payload_buffer.assign(payload_len, std::uint8_t{0}); - if (payload_len > 0U - && !read_exact(socket_fd_, payload_buffer.data(), payload_len, stop_requested_)) { - break; - } + payload_buffer.assign(payload_len, std::uint8_t{0}); + if (payload_len > 0U + && !read_exact(socket_fd_, payload_buffer.data(), payload_len, stop_requested_)) { + break; + } - try { - const auto json = Json::parse(payload_buffer.begin(), payload_buffer.end()); - if (json.is_object()) { - const auto found = json.find("vlc"); - if (found != json.end() && found->is_number()) { - const double value = found->get(); - if (std::isfinite(value)) { - // Stamp the value first, then the time, so a reader that - // observes a fresh timestamp also observes the matching value. - shared_vlc_slot.store(value, std::memory_order_release); - const auto now_ns = std::chrono::duration_cast( - std::chrono::steady_clock::now().time_since_epoch() - ).count(); - shared_vlc_at_ns.store(now_ns, std::memory_order_release); + try { + const auto json = Json::parse(payload_buffer.begin(), payload_buffer.end()); + if (json.is_object()) { + const auto found = json.find("vlc"); + if (found != json.end() && found->is_number()) { + const double value = found->get(); + if (std::isfinite(value) && std::fabs(value) <= std::numeric_limits::max()) { + // Stamp the value first, then the time, so a reader that + // observes a fresh timestamp also observes the matching value. + shared_vlc_slot.store(value, std::memory_order_release); + const auto now_ns = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch() + ).count(); + shared_vlc_at_ns.store(now_ns, std::memory_order_release); + } } } + } catch (const Json::exception& error) { + log_warning( + "ignoring malformed packet from " + peer_name_ + ": " + error.what() + ); } - } catch (const Json::parse_error& error) { - log_warning( - "ignoring malformed packet from " + peer_name_ + ": " + error.what() - ); } + } catch (const std::exception& error) { + log_warning("closing client " + peer_name_ + ": " + error.what()); } + finish_loop(); +} + +void ClientSession::finish_loop() { request_stop(); - // Mark exited only once BOTH loops have finished (see writer_loop), so a - // reader-only death still waits for the writer before this session is reaped. + // A session is reapable after both threads have stopped using its socket. if (loops_finished_.fetch_add(1, std::memory_order_acq_rel) + 1U == 2U) { exited_.store(true, std::memory_order_release); } @@ -417,7 +423,13 @@ void TcpServer::start() { } listen_fd_ = fd; - acceptor_thread_ = std::thread([this]() { acceptor_loop(); }); + try { + acceptor_thread_ = std::thread([this]() { acceptor_loop(); }); + } catch (...) { + shutdown_and_close(listen_fd_); + running_.store(false, std::memory_order_release); + throw; + } } void TcpServer::stop() { @@ -427,13 +439,14 @@ void TcpServer::stop() { if (listen_fd_ >= 0) { (void)::shutdown(listen_fd_, SHUT_RDWR); - (void)::close(listen_fd_); - listen_fd_ = -1; } if (acceptor_thread_.joinable()) { acceptor_thread_.join(); } + // Keep the descriptor stable until accept() has returned: closing it earlier + // permits fd reuse and races the acceptor's read of listen_fd_. + shutdown_and_close(listen_fd_); std::vector> sessions; { @@ -466,7 +479,7 @@ void TcpServer::publish(const ipc::ResultCollection& collection, const FilterPar // Reap exited sessions on the publish path too, so clients_ stays bounded // even when no new connections arrive to trigger an accept-time reap. Uses a // try-lock and joins outside the mutex, so publish() never blocks on it. - try_reap_finished_clients(); + reap_finished_clients(false); } auto TcpServer::latest_socket_speed() const -> std::optional { @@ -525,19 +538,27 @@ void TcpServer::acceptor_loop() { reap_finished_clients(); - apply_socket_keepalive(client_fd); - auto session = std::make_unique( - client_fd, - format_peer(peer_addr), - config_.client_queue_size, - config_.max_payload_bytes - ); - const auto snapshot = latest_packet_copy(); - if (snapshot.has_value()) { - session->enqueue(*snapshot); + std::unique_ptr session; + bool socket_owned = false; + try { + apply_socket_keepalive(client_fd); + session = std::make_unique( + client_fd, format_peer(peer_addr), config_.client_queue_size, + config_.max_payload_bytes + ); + socket_owned = true; + const auto snapshot = latest_packet_copy(); + if (snapshot.has_value()) { + session->enqueue(*snapshot); + } + session->start(latest_socket_speed_, latest_socket_speed_at_ns_); + enroll_client(std::move(session)); + } catch (const std::exception& error) { + if (!socket_owned) { + ::close(client_fd); + } + log_warning(std::string("cannot start client: ") + error.what()); } - session->start(latest_socket_speed_, latest_socket_speed_at_ns_); - enroll_client(std::move(session)); } } @@ -553,35 +574,13 @@ void TcpServer::broadcast_packet(const std::vector& packet) { } } -void TcpServer::reap_finished_clients() { +void TcpServer::reap_finished_clients(bool wait_for_lock) { std::vector> to_join; { - std::lock_guard guard(clients_mutex_); - auto first_dead = std::partition( - clients_.begin(), - clients_.end(), - [](const std::unique_ptr& session) { - return !session->has_exited(); - } - ); - for (auto it = first_dead; it != clients_.end(); ++it) { - to_join.push_back(std::move(*it)); - } - clients_.erase(first_dead, clients_.end()); - } - for (auto& session : to_join) { - session->request_stop(); - session->join(); - } -} - -void TcpServer::try_reap_finished_clients() { - std::vector> to_join; - { - // try_to_lock: if the acceptor already holds the mutex (it reaps too), - // skip this round rather than block the publish path. - std::unique_lock guard(clients_mutex_, std::try_to_lock); - if (!guard.owns_lock()) { + std::unique_lock guard(clients_mutex_, std::defer_lock); + if (wait_for_lock) { + guard.lock(); + } else if (!guard.try_lock()) { return; } auto first_dead = std::partition( diff --git a/data_acq_and_processing/processing/processors/src/gpr_legacy_processor.ipp b/data_acq_and_processing/processing/processors/src/gpr_legacy_processor.ipp index aa5b386..df595ae 100644 --- a/data_acq_and_processing/processing/processors/src/gpr_legacy_processor.ipp +++ b/data_acq_and_processing/processing/processors/src/gpr_legacy_processor.ipp @@ -20,10 +20,10 @@ // 5. Optional extended-mode region detection for diffuse reflectors. // // Sweep-event model — radar topology matters: -// * Matrix radars (`librevna_multi`, `sn9000`) fire one Tx at a time and +// * Matrix radars (`librevna_multi`) fire one Tx at a time and // receive on all Rx channels in parallel. A run of 8 traces is just 2 // Tx-events; all (tx_k, *) pairs share one timestamp. -// * Sequential radars (single librevna with switches, kamil_adc, k209…) +// * Sequential radars (single librevna with switches, kamil_adc) // measure each pair separately. A run of 8 traces is 8 separate sweeps; // every pair has its own timestamp. // Both cases reduce to: there are N events in the frame, each event lasts @@ -283,7 +283,7 @@ enum class LegacyPeakDomain { } [[nodiscard]] auto is_matrix_radar_model(const std::string& model) -> bool { - return model == "librevna_multi" || model == "sn9000"; + return model == "librevna_multi"; } // Assign an `event_index` to every selected pair. The mapping depends on the diff --git a/data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp b/data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp index 40f7654..57af839 100644 --- a/data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp +++ b/data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp @@ -1,19 +1,76 @@ #include "sar3d_gpu.hpp" +#include +#include +#include +#include +#include #include +#include #ifdef SAR3D_WITH_CUDA #include #include -#include -#include -#include -#include #include #endif namespace radar::processing::sar3d { +namespace { + +constexpr std::size_t kMaxIndexedItems = static_cast(INT_MAX); + +auto checked_product(std::size_t a, std::size_t b, std::size_t limit, const char* what) -> std::size_t { + if (a == 0U || b == 0U || a > limit / b) { + throw std::runtime_error(std::string("sar3d ") + what + " is too large or empty"); + } + return a * b; +} + +#ifdef SAR3D_WITH_CUDA +auto gaussian_taps(double sigma) -> std::vector { + if (!(sigma > 0.0) || !std::isfinite(sigma) || sigma > 100.0) { + throw std::runtime_error("sar3d smoothing sigma must be between 0 and 100"); + } + const int radius = static_cast(4.0 * sigma + 0.5); + std::vector weights; + double total = 0.0; + for (int k = -radius; k <= radius; ++k) { + weights.push_back(std::exp(-0.5 * k * k / (sigma * sigma))); + total += weights.back(); + } + std::vector taps; + for (const double weight : weights) { + taps.push_back(static_cast(weight / total)); + } + return taps; +} + +auto reference_plane(const VolumeRequest& request, std::size_t iy) -> bool { + const double margin = request.norm_edge_margin_m; + if (request.y.size() == 2U || request.y.back() - request.y.front() <= 2.5 * margin) { + return true; + } + return request.y[iy] >= request.y.front() + margin && request.y[iy] <= request.y.back() - margin; +} + +void choose_peak(Projections& result, const VolumeRequest& request, + std::span levels, std::span indices = {}) { + for (std::size_t iy = 0U; iy < request.y.size(); ++iy) { + if (!std::isfinite(levels[iy])) { + throw std::runtime_error("sar3d volume contains a non-finite value"); + } + if (reference_plane(request, iy) && levels[iy] > result.level) { + result.level = levels[iy]; + if (!indices.empty()) { + result.peak = {iy, indices[iy] / request.x.size(), indices[iy] % request.x.size()}; + } + } + } +} + +#endif +} // namespace #ifdef SAR3D_WITH_CUDA namespace { @@ -39,8 +96,9 @@ extern "C" __global__ void inverse_dft( const float2* __restrict__ bands, const int points, const int first_bin, const int size, const int items, float2* __restrict__ ascans) { - const int item = blockIdx.x * blockDim.x + threadIdx.x; - if (item >= items) return; + const unsigned index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= (unsigned)items) return; + const int item = (int)index; const unsigned k = (unsigned)(item % size); const unsigned mask = (unsigned)size - 1u; const float2* band = bands + (item / size) * points; @@ -72,8 +130,9 @@ extern "C" __global__ void backproject( const float comp_power, const float comp_max, float* __restrict__ volume) { - const int voxel = blockIdx.x * blockDim.x + threadIdx.x; - if (voxel >= voxels) return; + const unsigned index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= (unsigned)voxels) return; + const int voxel = (int)index; const int row = voxel / nx; // Scene -> radar axes (sar_scene_to_radar_yz). The depth gate is decided in // double so voxels on its edge are classified like in the reference. @@ -87,7 +146,7 @@ extern "C" __global__ void backproject( const float px = (float)gx[voxel - row * nx]; const float py = (float)(y0 * cos_p + zs * sin_p); const float pz = (float)depth; - const float last = (float)(samples - 1); + const double last = (double)(samples - 1); float re = 0.0f, im = 0.0f, weights = 0.0f; for (int c = 0; c < count; ++c) { const Contribution k = contributions[c]; @@ -96,7 +155,7 @@ extern "C" __global__ void backproject( const float rt = sqrtf(dxt * dxt + dyt * dyt + dzt * dzt); const float rr = sqrtf(dxr * dxr + dyr * dyr + dzr * dzr); const float position = (rt + rr) * samples_per_metre; - if (position > last) continue; + if (!isfinite(position) || position < 0.0f || (double)position > last) continue; const int i = min((int)position, samples - 2); const float frac = position - (float)i; const float2* row_ascan = ascans + (long long)c * samples; @@ -111,7 +170,7 @@ extern "C" __global__ void backproject( im += w * (a.y + frac * (b.y - a.y)); weights += k.weight; } - volume[voxel] = sqrtf(re * re + im * im) / (weights + 1e-12f); + volume[voxel] = hypotf(re, im) / (weights + 1e-12f); } // One pass of scipy.ndimage.gaussian_filter1d (mode 'reflect') along Z or X. @@ -120,8 +179,9 @@ extern "C" __global__ void smooth( const float* __restrict__ taps, const int radius, const int nx, const int nz, const int voxels, const int along_z) { - const int voxel = blockIdx.x * blockDim.x + threadIdx.x; - if (voxel >= voxels) return; + const unsigned thread = blockIdx.x * blockDim.x + threadIdx.x; + if (thread >= (unsigned)voxels) return; + const int voxel = (int)thread; const int length = along_z ? nz : nx; const int index = along_z ? (voxel / nx) % nz : voxel % nx; const int stride = along_z ? nx : 1; @@ -134,17 +194,36 @@ extern "C" __global__ void smooth( dst[voxel] = sum; } -// dst[o] = max over r of src[(o / inner) * outer_stride + (o % inner) * inner_stride + r * reduce_stride]. +// Strides select the displayed axes and the axis reduced to its maximum. extern "C" __global__ void max_project( const float* __restrict__ src, float* __restrict__ dst, const int outputs, const int inner, const int outer_stride, const int inner_stride, const int reduce, const int reduce_stride) { - const int o = blockIdx.x * blockDim.x + threadIdx.x; - if (o >= outputs) return; - const float* p = src + (o / inner) * outer_stride + (o % inner) * inner_stride; - float m = 0.0f; - for (int r = 0; r < reduce; ++r) m = fmaxf(m, p[r * reduce_stride]); - dst[o] = m; + const unsigned thread = blockIdx.x * blockDim.x + threadIdx.x; + if (thread >= (unsigned)outputs) return; + const int o = (int)thread; + const float* row = src + (o / inner) * outer_stride + (o % inner) * inner_stride; + float maximum = 0.0f; + for (int index = 0; index < reduce; ++index) maximum = fmaxf(maximum, row[index * reduce_stride]); + dst[o] = maximum; +} + +// Keep the first joint (y,z,x) maximum. Separate projections cannot locate it. +extern "C" __global__ void peak_planes( + const float* volume, const int plane, const int ny, float* levels, unsigned int* indices) +{ + const unsigned thread = blockIdx.x * blockDim.x + threadIdx.x; + if (thread >= (unsigned)ny) return; + const int y = (int)thread; + const float* row = volume + y * plane; + float level = 0.0f; + unsigned int index = 0u; + for (int i = 0; i < plane; ++i) { + if (!isfinite(row[i])) { levels[y] = row[i]; indices[y] = (unsigned int)i; return; } + if (row[i] > level) { level = row[i]; index = (unsigned int)i; } + } + levels[y] = level; + indices[y] = index; } // Orthographic max projection. The volume is non-negative, so float bit @@ -158,8 +237,9 @@ extern "C" __global__ void project_oblique( const double u0, const double v0, const double inv_pixel, const int nu, const int nv, unsigned int* __restrict__ image) { - const int voxel = blockIdx.x * blockDim.x + threadIdx.x; - if (voxel >= voxels) return; + const unsigned index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= (unsigned)voxels) return; + const int voxel = (int)index; const float value = volume[voxel]; if (!(value > 0.0f)) return; const int row = voxel / nx; @@ -175,11 +255,13 @@ extern "C" __global__ void project_oblique( constexpr int kBlockSize = 256; -void check(CUresult result, const char* what) { +void check(CUresult result, const char* what, bool device_required = false) { if (result != CUDA_SUCCESS) { const char* text = nullptr; cuGetErrorString(result, &text); - throw std::runtime_error(std::string{"CUDA "} + what + ": " + (text != nullptr ? text : "unknown error")); + const auto message = std::string{"CUDA "} + what + ": " + (text != nullptr ? text : "unknown error"); + if (device_required) { throw CudaUnavailable(message); } + throw std::runtime_error(message); } } @@ -299,20 +381,20 @@ template template void launch(CUfunction kernel, int items, Args... args) { void* parameters[] = {static_cast(&args)...}; - const auto blocks = static_cast((items + kBlockSize - 1) / kBlockSize); + const auto blocks = (static_cast(items) + kBlockSize - 1U) / kBlockSize; check(cuLaunchKernel(kernel, blocks, 1U, 1U, kBlockSize, 1U, 1U, 0U, nullptr, parameters, nullptr), "launch"); } } // namespace -struct Gpu::Impl { +struct CudaBackend { // Declared first, destroyed last: buffers and the module need the context. struct Context { CUdevice device = 0; CUcontext handle = nullptr; Context() { - check(cuInit(0U), "init"); - check(cuDeviceGet(&device, 0), "device"); + check(cuInit(0U), "init", true); + check(cuDeviceGet(&device, 0), "device", true); check(cuDevicePrimaryCtxRetain(&handle, device), "context"); if (const CUresult result = cuCtxSetCurrent(handle); result != CUDA_SUCCESS) { cuDevicePrimaryCtxRelease(device); @@ -331,10 +413,11 @@ struct Gpu::Impl { return kernel; } } module{compile_kernels(context.device)}; - CUfunction inverse_dft = module.function("inverse_dft"); + CUfunction inverse_dft_kernel = module.function("inverse_dft"); CUfunction backproject = module.function("backproject"); CUfunction smooth = module.function("smooth"); CUfunction max_project = module.function("max_project"); + CUfunction peak_planes = module.function("peak_planes"); CUfunction project_oblique = module.function("project_oblique"); int smooth_radius = 0; DeviceBuffer taps; @@ -345,50 +428,36 @@ struct Gpu::Impl { DeviceBuffer volume; DeviceBuffer scratch; DeviceBuffer images; + DeviceBuffer peak_stats; - ~Impl() { cuCtxSetCurrent(context.handle); } + ~CudaBackend() { cuCtxSetCurrent(context.handle); } + explicit CudaBackend(std::span weights) { + smooth_radius = static_cast(weights.size() / 2U); + taps.upload(weights); + } + auto inverse_dft(std::span>, std::size_t, std::size_t, std::size_t) -> std::vector>; + auto project(const VolumeRequest&) -> Projections; }; -Gpu::Gpu(double smooth_sigma_voxels) : impl_(std::make_unique()) { - // scipy.ndimage.gaussian_filter1d taps: radius int(4 sigma + 0.5), normalized. - const int radius = static_cast(4.0 * smooth_sigma_voxels + 0.5); - std::vector weights{}; - for (int k = -radius; k <= radius; ++k) { - weights.push_back(std::exp(-0.5 * k * k / (smooth_sigma_voxels * smooth_sigma_voxels))); - } - double total = 0.0; - for (const double weight : weights) { - total += weight; - } - std::vector taps{}; - for (const double weight : weights) { - taps.push_back(static_cast(weight / total)); - } - impl_->smooth_radius = radius; - impl_->taps.upload(std::span(taps)); -} - -Gpu::~Gpu() = default; - -auto Gpu::inverse_dft( +auto CudaBackend::inverse_dft( std::span> bands, std::size_t points, std::size_t first_bin, std::size_t size ) -> std::vector> { - Impl& gpu = *impl_; + CudaBackend& gpu = *this; check(cuCtxSetCurrent(gpu.context.handle), "context"); const std::size_t samples = bands.size() / points * size; const int items = checked_int(samples, "A-scan stack"); const CUdeviceptr input = gpu.bands.upload(bands); const CUdeviceptr output = gpu.ascans.reserve(sizeof(std::complex) * samples); - launch(gpu.inverse_dft, items, input, checked_int(points, "band"), checked_int(first_bin, "first bin"), + launch(gpu.inverse_dft_kernel, items, input, checked_int(points, "band"), checked_int(first_bin, "first bin"), checked_int(size, "A-scan"), items, output); return download>(output, samples); } -auto Gpu::project(const VolumeRequest& request) -> Projections { - Impl& gpu = *impl_; +auto CudaBackend::project(const VolumeRequest& request) -> Projections { + CudaBackend& gpu = *this; check(cuCtxSetCurrent(gpu.context.handle), "context"); const int nx = checked_int(request.x.size(), "grid"); @@ -413,6 +482,12 @@ auto Gpu::project(const VolumeRequest& request) -> Projections { std::cos(request.pitch_rad), std::sin(request.pitch_rad), request.ground_depth_m, request.min_depth_m, request.max_depth_m, static_cast(request.samples_per_metre), static_cast(request.use_compensation), request.comp_range_power, request.comp_weight_max, volume); + const int plane = nz * nx; + const auto stats = gpu.peak_stats.reserve(2U * sizeof(float) * static_cast(ny)); + launch(gpu.peak_planes, ny, volume, plane, ny, stats, stats + sizeof(float) * static_cast(ny)); + Projections raw_peak; + const auto raw_levels = download(stats, static_cast(ny)); + choose_peak(raw_peak, request, raw_levels); // Separable Gaussian with sigma (0, s, s) over [y][z][x]: Z first, then X, like scipy. const CUdeviceptr taps = gpu.taps.pointer(); launch(gpu.smooth, voxels, volume, scratch, taps, gpu.smooth_radius, nx, nz, voxels, 1); @@ -423,49 +498,112 @@ auto Gpu::project(const VolumeRequest& request) -> Projections { const int side_size = ny * nz; const int top_size = nz * nx; const int oblique_size = checked_int(static_cast(view.nv) * static_cast(view.nu), "view"); - const auto image_floats = static_cast(front_size + side_size + top_size + oblique_size); + const auto image_floats = static_cast(front_size) + side_size + top_size + oblique_size + 2U * ny; const CUdeviceptr front = gpu.images.reserve(sizeof(float) * image_floats); const CUdeviceptr side = front + sizeof(float) * static_cast(front_size); const CUdeviceptr top = side + sizeof(float) * static_cast(side_size); const CUdeviceptr oblique = top + sizeof(float) * static_cast(top_size); - const int plane = nz * nx; + const CUdeviceptr levels = oblique + sizeof(float) * static_cast(oblique_size); + const CUdeviceptr indices = levels + sizeof(float) * static_cast(ny); + launch(gpu.peak_planes, ny, volume, plane, ny, levels, indices); + Projections result; + result.raw_level = raw_peak.level; + const auto plane_levels = download(levels, static_cast(ny)); + const auto plane_indices = download(indices, static_cast(ny)); + choose_peak(result, request, plane_levels, plane_indices); launch(gpu.max_project, front_size, volume, front, front_size, nx, plane, 1, nz, nx); launch(gpu.max_project, side_size, volume, side, side_size, nz, plane, nx, nx, 1); launch(gpu.max_project, top_size, volume, top, top_size, nx, nx, 1, ny, plane); check(cuMemsetD32(oblique, 0U, static_cast(oblique_size)), "clear"); launch(gpu.project_oblique, voxels, volume, gx, gy, gz, nx, nz, voxels, view.u[0], view.u[1], view.u[2], view.v[0], view.v[1], view.v[2], view.u0, view.v0, 1.0 / view.pixel_m, view.nu, view.nv, oblique); - - return Projections{ - .front = download(front, static_cast(front_size)), - .side = download(side, static_cast(side_size)), - .top = download(top, static_cast(top_size)), - .oblique = download(oblique, static_cast(oblique_size)), - }; -} - -#else // !SAR3D_WITH_CUDA - -struct Gpu::Impl {}; - -Gpu::Gpu(double /*smooth_sigma_voxels*/) { - throw std::runtime_error( - "data_processor was built without CUDA (nvrtc.h, cuda.h, libnvrtc or libcuda not found), " - "so sar3d cannot run; install the CUDA requirements and rebuild" - ); -} - -Gpu::~Gpu() = default; - -auto Gpu::inverse_dft(std::span>, std::size_t, std::size_t, std::size_t) - -> std::vector> { - return {}; -} - -auto Gpu::project(const VolumeRequest& /*request*/) -> Projections { - return {}; + result.front = download(front, static_cast(front_size)); + result.side = download(side, static_cast(side_size)); + result.top = download(top, static_cast(top_size)); + result.oblique = download(oblique, static_cast(oblique_size)); + return result; } #endif +struct Engine::Impl { +#ifdef SAR3D_WITH_CUDA + std::unique_ptr cuda; +#endif +}; + +Engine::Engine(double smooth_sigma_voxels) : impl_(std::make_unique()) { +#ifdef SAR3D_WITH_CUDA + impl_->cuda = std::make_unique(gaussian_taps(smooth_sigma_voxels)); +#else + (void)smooth_sigma_voxels; + throw CudaUnavailable("SAR3D requires an NVIDIA GPU with CUDA; this program was built without " + "nvrtc.h, cuda.h, libnvrtc or libcuda. Install CUDA and rebuild."); +#endif +} + +Engine::~Engine() = default; + +auto Engine::inverse_dft(std::span> bands, std::size_t points, + std::size_t first_bin, std::size_t size) -> std::vector> { + if (points == 0U || bands.empty() || bands.size() % points != 0U || !std::has_single_bit(size) || + size > kMaxIndexedItems || first_bin >= size || points > size - first_bin) { + throw std::runtime_error("sar3d invalid inverse FFT dimensions"); + } + (void)checked_product(bands.size() / points, size, kMaxIndexedItems, "A-scan stack"); +#ifdef SAR3D_WITH_CUDA + return impl_->cuda->inverse_dft(bands, points, first_bin, size); +#else + throw CudaUnavailable("SAR3D requires an NVIDIA GPU with CUDA"); +#endif +} + +auto Engine::project(const VolumeRequest& request) -> Projections { + const auto nx = request.x.size(), ny = request.y.size(), nz = request.z.size(); + const auto plane = checked_product(nx, nz, kMaxIndexedItems, "grid plane"); + (void)checked_product(plane, ny, kMaxIndexedItems, "grid (32-bit indexing; increase SAR_VOXEL_M)"); + const auto& view = request.oblique; + if (nx < 2U || ny < 2U || nz < 2U || request.samples_per_ascan < 2U || + request.contributions.empty() || request.samples_per_metre <= 0.0 || + !std::isfinite(request.samples_per_metre) || request.samples_per_metre > std::numeric_limits::max() || + view.nu <= 0 || view.nv <= 0 || !(view.pixel_m > 0.0) || !std::isfinite(view.pixel_m)) { + throw std::runtime_error("sar3d invalid grid, aperture, A-scan or oblique dimensions"); + } + const auto scan_count = checked_product(request.contributions.size(), request.samples_per_ascan, + kMaxIndexedItems, "A-scan stack"); + if (request.ascans.size() != scan_count) { + throw std::runtime_error("sar3d A-scan stack does not match the aperture"); + } + for (const auto axis : {request.x, request.y, request.z}) { + for (const double coordinate : axis) { + if (!std::isfinite(coordinate) || std::abs(coordinate) > std::numeric_limits::max()) { + throw std::runtime_error("sar3d grid contains an unsupported coordinate"); + } + } + } + for (const auto& sample : request.ascans) { + if (!std::isfinite(sample.real()) || !std::isfinite(sample.imag())) { + throw std::runtime_error("sar3d A-scan contains a non-finite sample"); + } + } + for (const auto& term : request.contributions) { + if (!std::isfinite(term.weight) || term.weight < 0.0F || + !std::isfinite(term.inv_reference_ranges) || term.inv_reference_ranges <= 0.0F) { + throw std::runtime_error("sar3d aperture contains an invalid weight or reference range"); + } + for (const auto point : {term.tx, term.rx}) { + if (!std::ranges::all_of(point, [](float value) { return std::isfinite(value); })) { + throw std::runtime_error("sar3d aperture contains a non-finite antenna position"); + } + } + } + (void)checked_product(static_cast(view.nu), static_cast(view.nv), + kMaxIndexedItems, "oblique image"); +#ifdef SAR3D_WITH_CUDA + return impl_->cuda->project(request); +#else + throw CudaUnavailable("SAR3D requires an NVIDIA GPU with CUDA"); +#endif +} + } // namespace radar::processing::sar3d diff --git a/data_acq_and_processing/processing/processors/src/sar3d_gpu.hpp b/data_acq_and_processing/processing/processors/src/sar3d_gpu.hpp index e3e4007..1ff4cb9 100644 --- a/data_acq_and_processing/processing/processors/src/sar3d_gpu.hpp +++ b/data_acq_and_processing/processing/processors/src/sar3d_gpu.hpp @@ -5,13 +5,19 @@ #include #include #include +#include #include -// GPU half of the sar3d processor: A-scans of the band spectra, and the +// Numerical backend of the sar3d processor: A-scans of the band spectra, and the // backprojection of an aperture into the 3D volume, smoothed and projected. -// Only the projections of the volume leave the GPU. +// Only the projections and peak leave the backend. namespace radar::processing::sar3d { +class CudaUnavailable : public std::runtime_error { + public: + using std::runtime_error::runtime_error; +}; + // One (aperture frame, Tx/Rx pair) term of the backprojection sum. The layout // is shared with the CUDA kernel. struct Contribution { @@ -51,25 +57,29 @@ struct VolumeRequest { bool use_compensation = true; float comp_range_power = 0.0F; float comp_weight_max = 0.0F; + double norm_edge_margin_m = 0.0; ObliqueView oblique{}; }; -// Max-projections of the smoothed, not yet normalized volume (row-major). +// Maximum projections of the smoothed volume (row-major), before normalization. struct Projections { std::vector front; // [y][x], maximum along Z std::vector side; // [y][z], maximum along X std::vector top; // [z][x], maximum along Y std::vector oblique; // [v][u] + float level = 0.0F; + float raw_level = 0.0F; + std::array peak{}; // joint voxel maximum, [y, z, x] }; -// CUDA context with the kernels compiled for the installed GPU. Construction -// throws when no usable GPU is present or the processor was built without CUDA. -class Gpu { +// CUDA context and runtime-compiled kernels. A missing device raises +// CudaUnavailable; computation and compilation failures remain ordinary errors. +class Engine { public: - explicit Gpu(double smooth_sigma_voxels); - ~Gpu(); - Gpu(const Gpu&) = delete; - auto operator=(const Gpu&) -> Gpu& = delete; + explicit Engine(double smooth_sigma_voxels); + ~Engine(); + Engine(const Engine&) = delete; + auto operator=(const Engine&) -> Engine& = delete; // numpy.fft.ifft of each band spectrum placed at bins [first_bin, first_bin + // points) of a `size`-point spectrum; `bands` holds the bands back to back. diff --git a/data_acq_and_processing/processing/processors/src/sar3d_processor.cpp b/data_acq_and_processing/processing/processors/src/sar3d_processor.cpp index a91ff84..af846fc 100644 --- a/data_acq_and_processing/processing/processors/src/sar3d_processor.cpp +++ b/data_acq_and_processing/processing/processors/src/sar3d_processor.cpp @@ -7,11 +7,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -158,22 +160,26 @@ void select_antennas( struct TraceClock { double start_s = 0.0; // sweep start double event_s = 0.0; // sweep duration / (stages * points) - long long stage = 0; - long long stages = 1; + std::size_t stage = 0U; + std::size_t stages = 1U; [[nodiscard]] auto sample_time_s(std::size_t point) const -> double { - return start_s + static_cast(stages * static_cast(point) + stage) * event_s; + return start_s + (static_cast(stages) * point + stage) * event_s; } }; struct CollectionTiming { - double center_s = 0.0; // absolute acquisition centre, positions the frame in the aperture + std::uint64_t center_ns = 0U; // integer centre avoids overflow and loss of relative timing + double center_fraction_ns = 0.0; std::vector clocks; // per collection trace }; // Timing from the per-trace capture windows (snapshot_acquisition_timing); every // trace takes part, so the stages do not depend on the INPUT_IDX/OUTPUT_IDX choice. [[nodiscard]] auto collection_timing(const ipc::PreprocessedCollection& collection) -> CollectionTiming { + if (collection.traces.empty()) { + throw std::runtime_error("collection has no traces"); + } std::map, std::vector> sweep_outputs{}; std::uint64_t first = std::numeric_limits::max(); std::uint64_t last = 0U; @@ -181,6 +187,9 @@ struct CollectionTiming { if (trace.capture_end_ns <= trace.capture_start_ns) { throw std::runtime_error("collection has traces without a capture window"); } + if (trace.frequency_hz.empty()) { + throw std::runtime_error("collection has a trace without frequency points"); + } sweep_outputs[{trace.capture_start_ns, trace.capture_end_ns}].push_back(trace.combo.output_pos); first = std::min(first, trace.capture_start_ns); last = std::max(last, trace.capture_end_ns); @@ -191,18 +200,19 @@ struct CollectionTiming { } CollectionTiming timing{}; - const auto center_x2_ns = static_cast(first + last); - timing.center_s = 0.5e-9 * static_cast(center_x2_ns); + const auto duration_ns = last - first; + timing.center_ns = first + duration_ns / 2U; + timing.center_fraction_ns = 0.5 * static_cast(duration_ns % 2U); for (const auto& trace : collection.traces) { const auto& stages = sweep_outputs.at({trace.capture_start_ns, trace.capture_end_ns}); const auto stage = std::find(stages.begin(), stages.end(), trace.combo.output_pos) - stages.begin(); TraceClock clock{}; - clock.start_s = - 0.5e-9 * static_cast(2 * static_cast(trace.capture_start_ns) - center_x2_ns); + clock.start_s = 1e-9 * (static_cast(trace.capture_start_ns - first) - + 0.5 * static_cast(duration_ns)); clock.stage = stage; - clock.stages = static_cast(stages.size()); + clock.stages = stages.size(); clock.event_s = 1e-9 * static_cast(trace.capture_end_ns - trace.capture_start_ns) / - static_cast(clock.stages * static_cast(trace.frequency_hz.size())); + static_cast(clock.stages) / trace.frequency_hz.size(); timing.clocks.push_back(clock); } return timing; @@ -226,6 +236,10 @@ struct Band { } std::vector points{}; for (std::size_t point = 0U; point < frequency.size(); ++point) { + if (!std::isfinite(frequency[point]) || frequency[point] < 0.0F || + (point > 0U && frequency[point] <= frequency[point - 1U])) { + throw std::runtime_error("trace frequencies must be finite, non-negative and increasing"); + } if (frequency[point] >= settings.f_start_hz && frequency[point] <= settings.f_stop_hz) { points.push_back(point); } @@ -242,7 +256,15 @@ struct Band { if (!(band.df_hz > 0.0)) { throw std::runtime_error("trace frequencies do not increase"); } - band.first_bin = static_cast(std::nearbyint(frequency[points.front()] / band.df_hz)); + // Bound before converting to an integer or calling bit_ceil. CUDA uses + // 32-bit indexing; pathological sub-Hz steps must not wrap CUDA indices. + constexpr std::size_t max_fft = 1U << 30U; // largest power of two inside CUDA's signed index + const double first_bin = std::nearbyint(frequency[points.front()] / band.df_hz); + if (!std::isfinite(first_bin) || first_bin < 0.0 || + first_bin + points.size() - 1U > max_fft / (2U * kOversample)) { + throw std::runtime_error("frequency band exceeds the 32-bit CUDA A-scan indexing"); + } + band.first_bin = static_cast(first_bin); band.size = std::bit_ceil(2U * (band.first_bin + points.size() - 1U)) * kOversample; // The radar moves SPEED_M_S * cos(LOOK_ANGLE_DEG) along range; its offset @@ -255,6 +277,9 @@ struct Band { const std::size_t point = points[index]; const double hann = 0.5 - 0.5 * std::cos(2.0 * pi * static_cast(index) / last); const std::complex sample(trace.s21[point].re, trace.s21[point].im); + if (!std::isfinite(sample.real()) || !std::isfinite(sample.imag())) { + throw std::runtime_error("S21 contains a non-finite sample"); + } band.spectrum.emplace_back( sample * std::polar(hann, phase_per_hz_s * frequency[point] * clock.sample_time_s(point)) ); @@ -289,7 +314,8 @@ void normalize_ascan(std::span> ascan, double dt_s, const Se // A-scans of every selected Tx/Rx pair of one collection, (Tx, Rx) order. struct Frame { - double center_s = 0.0; + std::uint64_t center_ns = 0U; + double center_fraction_ns = 0.0; double dt_s = 0.0; std::size_t samples = 0U; // per A-scan std::vector> ascans{}; // [pair][sample] @@ -298,7 +324,7 @@ struct Frame { [[nodiscard]] auto compute_frame( const ipc::PreprocessedCollection& collection, const Settings& settings, - sar3d::Gpu& gpu + sar3d::Engine& backend ) -> Frame { const auto timing = collection_timing(collection); std::vector bands{}; @@ -327,10 +353,11 @@ struct Frame { spectra.insert(spectra.end(), band.spectrum.begin(), band.spectrum.end()); } Frame frame{}; - frame.center_s = timing.center_s; + frame.center_ns = timing.center_ns; + frame.center_fraction_ns = timing.center_fraction_ns; frame.samples = first.size; frame.dt_s = 1.0 / (static_cast(first.size) * first.df_hz); - frame.ascans = gpu.inverse_dft(spectra, first.spectrum.size(), first.first_bin, first.size); + frame.ascans = backend.inverse_dft(spectra, first.spectrum.size(), first.first_bin, first.size); if (settings.normalize_each_frame) { for (std::size_t offset = 0U; offset < frame.ascans.size(); offset += frame.samples) { normalize_ascan(std::span(frame.ascans).subspan(offset, frame.samples), frame.dt_s, settings); @@ -350,9 +377,11 @@ struct Frame { return axis; } -// Axis with about the given pitch (sar_build_full_region_grid: round(span/pitch)+1 points). -[[nodiscard]] auto grid_axis(double start, double stop, double pitch) -> std::vector { - return linspace(start, stop, static_cast(std::max(std::nearbyint((stop - start) / pitch) + 1.0, 2.0))); +[[nodiscard]] auto checked_axis_size(double count) -> std::size_t { + if (!std::isfinite(count) || count < 1.0 || count > std::numeric_limits::max()) { + throw std::runtime_error("SAR axis exceeds 32-bit kernel indexing; increase SAR_VOXEL_M"); + } + return static_cast(count); } struct Grid { @@ -367,6 +396,9 @@ struct Grid { [[nodiscard]] auto scene_grid(const Settings& settings) -> Grid { const double cos_p = std::cos(settings.look_angle_rad); const double sin_p = std::sin(settings.look_angle_rad); + if (std::abs(cos_p) < 1e-6) { + throw std::runtime_error("SAR grid is undefined at a radar look angle of 90 degrees"); + } double z_min = std::numeric_limits::infinity(); double z_max = -std::numeric_limits::infinity(); for (const double height : {kSceneYMin, kSceneYMax}) { @@ -377,11 +409,19 @@ struct Grid { z_max = std::max(z_max, z); } } - return { - grid_axis(kSceneXMin, kSceneXMax, settings.voxel_m), - grid_axis(kSceneYMin, kSceneYMax, settings.voxel_m), - grid_axis(z_min, z_max, settings.voxel_m), + // Match sar_build_full_region_grid and reject oversized products before + // allocating the axes. Every downstream kernel uses signed 32-bit indices. + const auto axis_size = [&](double span) { + return checked_axis_size(std::max(std::nearbyint(span / settings.voxel_m) + 1.0, 2.0)); }; + const auto nx = axis_size(kSceneXMax - kSceneXMin); + const auto ny = axis_size(kSceneYMax - kSceneYMin); + const auto nz = axis_size(z_max - z_min); + if (nx > static_cast(std::numeric_limits::max()) / ny / nz) { + throw std::runtime_error("SAR grid exceeds 32-bit kernel indexing; increase SAR_VOXEL_M"); + } + return {linspace(kSceneXMin, kSceneXMax, nx), linspace(kSceneYMin, kSceneYMax, ny), + linspace(z_min, z_max, nz)}; } // Cosine taper over the aperture: 1 at the centre frame, SAR_EDGE_FRAME_WEIGHT at @@ -463,8 +503,11 @@ struct Oblique { } } } - view.nu = static_cast(std::lround((u_max - view.u0) / view.pixel_m)) + 1; - view.nv = static_cast(std::lround((v_max - view.v0) / view.pixel_m)) + 1; + view.nu = static_cast(checked_axis_size(std::round((u_max - view.u0) / view.pixel_m) + 1.0)); + view.nv = static_cast(checked_axis_size(std::round((v_max - view.v0) / view.pixel_m) + 1.0)); + if (view.nu > std::numeric_limits::max() / view.nv) { + throw std::runtime_error("SAR oblique image exceeds 32-bit kernel indexing; increase SAR_VOXEL_M"); + } for (int index = 0; index < view.nu; ++index) { oblique.u.push_back(view.u0 + index * view.pixel_m); } @@ -479,6 +522,11 @@ struct Oblique { } [[nodiscard]] auto to_float(const std::array& point) -> std::array { + for (const double coordinate : point) { + if (!std::isfinite(coordinate) || std::abs(coordinate) > std::numeric_limits::max()) { + throw std::runtime_error("SAR antenna motion exceeds the supported coordinate range"); + } + } return {static_cast(point[0]), static_cast(point[1]), static_cast(point[2])}; } @@ -511,6 +559,15 @@ struct Oblique { return payload; } +[[nodiscard]] auto status_payload(float code, const std::string& message) -> ipc::ResultPayload { + std::vector values{code}; + for (const unsigned char byte : message) { + values.push_back(byte); + } + const auto columns = static_cast(values.size()); + return table_payload("sar3d_status", columns, std::move(values)); +} + } // namespace struct Sar3dProcessor::State { @@ -521,44 +578,42 @@ struct Sar3dProcessor::State { return {collection.collection_id, collection.monotonic_ns}; } - // A-scans by sweep for `frames_key`; null marks a sweep that could not be - // processed, so its error is reported once and not retried. - std::map> frames{}; + struct CachedFrame { + std::shared_ptr value; + std::string error; + }; + std::map frames{}; AscanKey frames_key{}; - std::unique_ptr gpu{}; - std::string gpu_error{}; + std::unique_ptr engine{}; + std::exception_ptr engine_error{}; std::string last_error{}; std::uint64_t repeated_errors = 0U; - auto frame(const ipc::PreprocessedCollection& collection, const Settings& settings, sar3d::Gpu& gpu) + auto frame(const ipc::PreprocessedCollection& collection, const Settings& settings, sar3d::Engine& backend) -> std::shared_ptr { if (const auto found = frames.find(sweep_key(collection)); found != frames.end()) { - return found->second; + if (found->second.value == nullptr) { report_error(found->second.error); } + return found->second.value; } std::shared_ptr frame{}; + std::string failure; try { - frame = std::make_shared(compute_frame(collection, settings, gpu)); + frame = std::make_shared(compute_frame(collection, settings, backend)); } catch (const std::exception& error) { - report_error(error.what()); + failure = error.what(); + report_error(failure); } - frames.emplace(sweep_key(collection), frame); + frames.emplace(sweep_key(collection), CachedFrame{frame, std::move(failure)}); return frame; } - // Set up once: compiling the kernels takes seconds, so a failure is kept - // instead of being retried on every frame. - auto gpu_engine() -> sar3d::Gpu& { - if (gpu == nullptr && gpu_error.empty()) { - try { - gpu = std::make_unique(kSmoothSigma); - } catch (const std::exception& error) { - gpu_error = error.what(); - } + auto numerical_engine() -> sar3d::Engine& { + if (engine == nullptr && engine_error == nullptr) { + try { engine = std::make_unique(kSmoothSigma); } + catch (...) { engine_error = std::current_exception(); } } - if (gpu == nullptr) { - throw std::runtime_error(gpu_error); - } - return *gpu; + if (engine == nullptr) { std::rethrow_exception(engine_error); } + return *engine; } auto render( @@ -566,23 +621,29 @@ struct Sar3dProcessor::State { std::span previous, const Settings& settings ) -> std::vector { - auto& gpu = gpu_engine(); + auto& backend = numerical_engine(); if (settings.ascan_key() != frames_key) { frames.clear(); frames_key = settings.ascan_key(); } - const auto newest = frame(collection, settings, gpu); + const auto history_count = settings.frame_radius >= previous.size() / 2U + previous.size() % 2U + ? previous.size() : 2U * settings.frame_radius; + const auto recent = previous.last(history_count); + std::set retained{sweep_key(collection)}; + for (const auto& member : recent) { retained.insert(sweep_key(member)); } + // Prune even when the newest frame is invalid, so a bad input stream + // cannot grow the failed-frame cache forever. + std::erase_if(frames, [&](const auto& entry) { return !retained.contains(entry.first); }); + const auto newest = frame(collection, settings, backend); if (newest == nullptr) { - return {}; + return {status_payload(2.0F, last_error)}; } // Aperture: the newest 2r+1 collections, centred r collections before // the newest one, so the image trails the input by r frames. Collections // that left the aperture are dropped from the cache. std::vector> aperture{}; - std::map> kept{{sweep_key(collection), newest}}; - for (const auto& member : previous.last(std::min(previous.size(), 2U * settings.frame_radius))) { - auto member_frame = frame(member, settings, gpu); - kept.emplace(sweep_key(member), member_frame); + for (const auto& member : recent) { + auto member_frame = frame(member, settings, backend); const bool same_axis = member_frame != nullptr && member_frame->samples == newest->samples && member_frame->dt_s == newest->dt_s; if (same_axis) { @@ -590,14 +651,16 @@ struct Sar3dProcessor::State { } } aperture.push_back(newest); - frames = std::move(kept); const std::size_t centre_index = aperture.size() > settings.frame_radius ? aperture.size() - 1U - settings.frame_radius : 0U; const Frame& centre = *aperture[centre_index]; std::vector offsets_s{}; for (const auto& member : aperture) { - offsets_s.push_back(member->center_s - centre.center_s); + const double delta_ns = member->center_ns >= centre.center_ns + ? static_cast(member->center_ns - centre.center_ns) + : -static_cast(centre.center_ns - member->center_ns); + offsets_s.push_back(1e-9 * (delta_ns + member->center_fraction_ns - centre.center_fraction_ns)); } const auto weights = frame_weights(offsets_s); const Grid grid = scene_grid(settings); @@ -637,8 +700,9 @@ struct Sar3dProcessor::State { // A Tx+Rx path is convex, so the box corners bound it: samples beyond the // longest path are never read and are not sent to the GPU. const double samples_per_metre = 1.0 / (kWaveSpeed * newest->dt_s); - const std::size_t samples = - std::min(newest->samples, static_cast(std::ceil(longest_path * samples_per_metre)) + 2U); + const double needed = std::ceil(longest_path * samples_per_metre) + 2.0; + const std::size_t samples = needed >= newest->samples + ? newest->samples : static_cast(needed); std::vector> ascans{}; for (const auto& member : aperture) { const std::span> rows = member->ascans; @@ -649,7 +713,7 @@ struct Sar3dProcessor::State { } const Oblique oblique = oblique_view(grid, settings.voxel_m); - auto images = gpu.project({ + auto images = backend.project({ .x = grid.x, .y = grid.y, .z = grid.z, @@ -664,6 +728,7 @@ struct Sar3dProcessor::State { .use_compensation = settings.use_compensation, .comp_range_power = settings.comp_range_power, .comp_weight_max = kCompRangeWeightMax, + .norm_edge_margin_m = kNormEdgeMargin, .oblique = oblique.view, }); @@ -671,30 +736,17 @@ struct Sar3dProcessor::State { // edges, where a constant-range ridge leaves the grid and would set a // false maximum (sar_norm_reference_mask, sar_finish_volume, // sar_global_max_index). The level is the volume maximum, so the - // published projections peak at 1. - const std::size_t nx = grid.x.size(); - const std::size_t nz = grid.z.size(); - float level = 0.0F; - std::size_t peak_x = 0U; - std::size_t peak_y = 0U; - for (std::size_t iy = 0U; iy < grid.y.size(); ++iy) { - if (grid.y[iy] < grid.y.front() + kNormEdgeMargin || grid.y[iy] > grid.y.back() - kNormEdgeMargin) { - continue; - } - for (std::size_t ix = 0U; ix < nx; ++ix) { - if (images.front[iy * nx + ix] > level) { - level = images.front[iy * nx + ix]; - peak_x = ix; - peak_y = iy; - } - } - } - const auto side_row = std::span(images.side).subspan(peak_y * nz, nz); - const auto peak_z = static_cast(std::ranges::max_element(side_row) - side_row.begin()); - const float scale = level > 0.0F ? 1.0F / level : 0.0F; + // Values at excluded Y edges can exceed 1, as in sar_finish_volume. + const float level = images.level; + const auto [peak_y, peak_z, peak_x] = images.peak; + // The notebook normalizes before and after the linear Gaussian pass. + // Combining both factors in double preserves its epsilons and avoids + // float32 reciprocal overflow for a weak, nonzero signal. + const double normalization = static_cast(level) + (images.raw_level + 1e-12) * 1e-12; for (auto* image : {&images.front, &images.side, &images.top, &images.oblique}) { for (auto& value : *image) { - value *= scale; + value = static_cast(static_cast(value) / normalization); + if (!std::isfinite(value)) { throw std::runtime_error("SAR normalization exceeds the float32 range"); } } } @@ -707,10 +759,12 @@ struct Sar3dProcessor::State { basis.insert(basis.end(), oblique.view.v.begin(), oblique.view.v.end()); payloads.push_back(table_payload("sar3d_oblique_basis", 3U, std::move(basis))); if (level > 0.0F) { - payloads.push_back(table_payload("sar3d_peak", 4U, {level, static_cast(grid.x[peak_x]), + payloads.push_back(table_payload("sar3d_peak", 4U, {static_cast(normalization), static_cast(grid.x[peak_x]), static_cast(grid.y[peak_y]), static_cast(grid.z[peak_z])})); } + payloads.push_back(status_payload(level > 0.0F ? 0.0F : 1.0F, + level > 0.0F ? "CUDA" : "No signal inside the selected depth range")); return payloads; } @@ -748,10 +802,13 @@ auto Sar3dProcessor::process_collection( try { results.collection_payloads = state_->render(collection, previous_collections, read_settings(run_config, live_config)); - } catch (const std::exception& error) { - // A data or CUDA error must not stop the data_processor loop: publish an - // empty result and keep going. + } catch (const sar3d::CudaUnavailable& error) { state_->report_error(error.what()); + results.collection_payloads = {status_payload(3.0F, error.what())}; + } catch (const std::exception& error) { + // Keep the processing loop alive and make the reason visible in the GUI. + state_->report_error(error.what()); + results.collection_payloads = {status_payload(2.0F, error.what())}; } return results; } diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/interfaces/radar_driver.hpp b/data_acq_and_processing/sweep_orchestrator/device_drivers/interfaces/radar_driver.hpp index 059da61..ccc7288 100644 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/interfaces/radar_driver.hpp +++ b/data_acq_and_processing/sweep_orchestrator/device_drivers/interfaces/radar_driver.hpp @@ -35,18 +35,6 @@ class RadarDriver { /** @brief Acquire one sweep containing the forward traces exposed by the driver. */ [[nodiscard]] virtual auto acquire_sweep() -> SweepTrace = 0; - /** - * @brief Announce which switch combo the next `acquire_sweep` belongs to. - * - * Real radars are agnostic to this because the switch state itself decides - * what they see. Mock drivers use it to synthesise per-combo variation so - * downstream plots show eight distinct traces for an eight-combo run - * instead of eight identical curves stacked on top of each other. - * - * Default implementation is a no-op so production drivers do not need to - * override. - */ - virtual void set_active_combo(const ipc::ComboKey& /*combo*/) {} }; } // namespace radar::drivers diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/compact_m_k209/compact_m_k209_driver.cpp b/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/compact_m_k209/compact_m_k209_driver.cpp deleted file mode 100644 index fd86703..0000000 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/compact_m_k209/compact_m_k209_driver.cpp +++ /dev/null @@ -1,414 +0,0 @@ -#include "../compact_m_k209_driver.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace radar::drivers { -namespace { - -constexpr std::uint32_t kFloatBytes = 4U; -constexpr std::uint32_t kComplexScalarCount = 2U; -constexpr std::uint64_t kMaxBinaryBlockBytes = 512ULL * 1024ULL * 1024ULL; - -[[nodiscard]] auto visa_resource(const std::string& resource) -> ViRsrc { - return reinterpret_cast(const_cast(resource.c_str())); -} - -[[nodiscard]] auto visa_buffer(std::uint8_t* data) -> ViBuf { - return reinterpret_cast(data); -} - -[[nodiscard]] auto visa_const_buffer(const char* data) -> ViBuf { - return reinterpret_cast(const_cast(data)); -} - -[[nodiscard]] auto is_line_ending(std::uint8_t value) -> bool { - return value == static_cast('\n') || value == static_cast('\r'); -} - -[[nodiscard]] auto is_response_separator(std::uint8_t value) -> bool { - return is_line_ending(value) || value == static_cast(';'); -} - -[[nodiscard]] auto checked_vi_count(std::uint64_t value) -> ViUInt32 { - const auto max_count = static_cast(std::numeric_limits::max()); - return static_cast(std::min(value, max_count)); -} - -} // namespace - -CompactMK209Driver::CompactMK209Driver(CompactMK209DriverSettings settings) : settings_(std::move(settings)) {} - -CompactMK209Driver::~CompactMK209Driver() { - try { - close(); - } catch (...) { - } -} - -void CompactMK209Driver::open() { - if (is_open_) { - return; - } - if (settings_.resource.empty()) { - throw std::runtime_error("K209 VISA resource must not be empty"); - } - if (settings_.sweep.points < 2U) { - throw std::runtime_error("K209 sweep points must be >= 2"); - } - if (settings_.sweep.stop_hz < settings_.sweep.start_hz) { - throw std::runtime_error("K209 sweep stop_hz must be >= start_hz"); - } - if (!(settings_.sweep.if_bandwidth_hz > 0.0F)) { - throw std::runtime_error("K209 IF bandwidth must be > 0"); - } - - ViStatus status = viOpenDefaultRM(&resource_manager_); - if (status < VI_SUCCESS) { - throw std::runtime_error("Failed to open VISA resource manager: status=" + std::to_string(status)); - } - - try { - check_status( - viOpen(resource_manager_, visa_resource(settings_.resource), VI_NULL, VI_NULL, &instrument_), - "open K209 VISA resource" - ); - check_status( - viSetAttribute(instrument_, VI_ATTR_TMO_VALUE, static_cast(settings_.timeout_ms)), - "set K209 VISA timeout" - ); - check_status( - viSetAttribute(instrument_, VI_ATTR_TERMCHAR_EN, static_cast(VI_FALSE)), - "disable K209 VISA termchar" - ); - write_command("*CLS"); - configure_device(); - } catch (...) { - close(); - throw; - } - - is_open_ = true; -} - -void CompactMK209Driver::close() { - if (instrument_ != VI_NULL) { - viClose(instrument_); - instrument_ = VI_NULL; - } - if (resource_manager_ != VI_NULL) { - viClose(resource_manager_); - resource_manager_ = VI_NULL; - } - is_open_ = false; -} - -auto CompactMK209Driver::acquire_sweep() -> SweepTrace { - require_open(); - - auto traces = query_sweep_trace_pair(settings_.sweep.points); - - SweepTrace trace{}; - trace.frequency_hz = frequency_hz_; - trace.s11 = complex_trace_from_interleaved(traces.first); - trace.s21 = complex_trace_from_interleaved(traces.second); - return trace; -} - -auto CompactMK209Driver::query_identity() -> std::string { - require_open(); - return trim_ascii(query_string("*IDN?")); -} - -auto CompactMK209Driver::query_system_error() -> std::string { - require_open(); - return trim_ascii(query_string("SYST:ERR?")); -} - -void CompactMK209Driver::configure_device() { - if (settings_.preset_on_open) { - write_command("SYST:PRES"); - expect_operation_complete("*OPC?", "wait for K209 preset"); - } - - write_command("SENS:FREQ:STAR " + format_frequency(settings_.sweep.start_hz)); - write_command("SENS:FREQ:STOP " + format_frequency(settings_.sweep.stop_hz)); - write_command("SENS:SWE:POIN " + std::to_string(settings_.sweep.points)); - write_command("SENS:SWE:POIN:TIME 0"); - write_command("SENS:BAND " + format_frequency(settings_.sweep.if_bandwidth_hz)); - write_command("SOUR:POW " + format_power(settings_.sweep.power_dbm)); - write_command("SENS:AVER OFF"); - write_command("CALC:PAR1:DEF S21"); - write_command("CALC:PAR1:SEL"); - write_command("FORM:DATA REAL32"); - write_command("FORM:BORD SWAP"); - write_command("INIT:CONT ON"); - write_command("TRIG:SOUR BUS"); - expect_operation_complete("*OPC?", "wait for K209 setup"); - - frequency_hz_ = query_float_array("SENS:FREQ:DATA?", settings_.sweep.points); -} - -void CompactMK209Driver::expect_operation_complete(const std::string& command, const std::string& context) { - const auto response = trim_ascii(query_string(command)); - if (response != "1") { - throw std::runtime_error(context + " returned unexpected *OPC? response: " + response); - } -} - -void CompactMK209Driver::write_command(const std::string& command) { - require_open(); - - const std::string message = command + "\n"; - ViUInt32 written = 0; - check_status( - viWrite(instrument_, visa_const_buffer(message.data()), static_cast(message.size()), &written), - "write K209 command: " + command - ); - if (written != message.size()) { - throw std::runtime_error("Incomplete K209 command write: " + command); - } -} - -auto CompactMK209Driver::query_string(const std::string& command) -> std::string { - write_command(command); - return read_line(); -} - -auto CompactMK209Driver::query_float_array( - const std::string& command, - std::uint32_t expected_values -) -> std::vector { - write_command(command); - return read_float_array_response(command, expected_values); -} - -auto CompactMK209Driver::query_sweep_trace_pair(std::uint32_t points) - -> std::pair, std::vector> { - write_command("TRIG:SING;*OPC?;:SENS:DATA:CORR? S11;:SENS:DATA:CORR? S21"); - const auto opc_response = read_ascii_token(); - if (opc_response != "1") { - throw std::runtime_error("K209 sweep returned unexpected *OPC? response: " + opc_response); - } - - return { - read_float_array_response("SENS:DATA:CORR? S11", points * kComplexScalarCount), - read_float_array_response("SENS:DATA:CORR? S21", points * kComplexScalarCount), - }; -} - -auto CompactMK209Driver::read_float_array_response( - const std::string& context, - std::uint32_t expected_values -) -> std::vector { - auto values = float_array_from_little_endian(read_ieee_block()); - if (values.size() != expected_values) { - throw std::runtime_error( - "K209 response for " + context + " returned " + std::to_string(values.size()) + - " float32 values, expected " + std::to_string(expected_values) - ); - } - return values; -} - -auto CompactMK209Driver::read_ascii_token() -> std::string { - require_open(); - - std::string token{}; - token.reserve(32); - - while (true) { - const auto byte = read_byte(); - if (is_response_separator(byte)) { - if (!token.empty()) { - return token; - } - continue; - } - token.push_back(static_cast(byte)); - if (token.size() > 64U * 1024U) { - throw std::runtime_error("K209 ASCII response token is too long"); - } - } -} - -auto CompactMK209Driver::read_line() -> std::string { - require_open(); - - std::string line{}; - line.reserve(128); - bool has_content = false; - - while (true) { - const auto byte = read_byte(); - if (!has_content && is_line_ending(byte)) { - continue; - } - if (byte == static_cast('\n')) { - break; - } - if (byte != static_cast('\r')) { - line.push_back(static_cast(byte)); - has_content = true; - } - if (line.size() > 64U * 1024U) { - throw std::runtime_error("K209 ASCII response line is too long"); - } - } - - return line; -} - -auto CompactMK209Driver::read_byte() -> std::uint8_t { - auto bytes = read_exact(1); - return bytes[0]; -} - -auto CompactMK209Driver::read_exact(std::uint64_t size) -> std::vector { - require_open(); - - std::vector bytes(size); - std::uint64_t offset = 0; - while (offset < size) { - ViUInt32 transferred = 0; - const auto count = checked_vi_count(size - offset); - check_status( - viRead(instrument_, visa_buffer(bytes.data() + offset), count, &transferred), - "read K209 response bytes" - ); - if (transferred == 0U) { - throw std::runtime_error("K209 VISA read returned zero bytes before expected payload was complete"); - } - offset += transferred; - } - return bytes; -} - -auto CompactMK209Driver::read_ieee_block() -> std::vector { - std::uint8_t marker = read_byte(); - while (is_response_separator(marker)) { - marker = read_byte(); - } - if (marker != static_cast('#')) { - throw std::runtime_error("K209 binary response does not start with IEEE block marker '#'"); - } - - const auto digit = read_byte(); - if (digit != static_cast('8')) { - throw std::runtime_error("K209 binary response uses unsupported IEEE block header width"); - } - - const auto size_text_bytes = read_exact(8); - std::uint64_t payload_size = 0; - for (const auto byte : size_text_bytes) { - if (!std::isdigit(static_cast(byte))) { - throw std::runtime_error("K209 binary response has non-numeric payload size"); - } - payload_size = (payload_size * 10ULL) + static_cast(byte - static_cast('0')); - } - if (payload_size == 0U || payload_size > kMaxBinaryBlockBytes) { - throw std::runtime_error("K209 binary response payload size is out of allowed range"); - } - if ((payload_size % kFloatBytes) != 0U) { - throw std::runtime_error("K209 binary response payload size is not aligned to float32 values"); - } - - return read_exact(payload_size); -} - -void CompactMK209Driver::check_status(ViStatus status, const std::string& context) const { - if (status >= VI_SUCCESS) { - return; - } - throw std::runtime_error(context + ": " + status_message(status)); -} - -auto CompactMK209Driver::status_message(ViStatus status) const -> std::string { - std::array description{}; - const auto session = instrument_ != VI_NULL ? instrument_ : resource_manager_; - if (session != VI_NULL && viStatusDesc(session, status, description.data()) >= VI_SUCCESS) { - return std::string(description.data()) + " (status=" + std::to_string(status) + ")"; - } - return "VISA status=" + std::to_string(status); -} - -void CompactMK209Driver::require_open() const { - if (instrument_ == VI_NULL) { - throw std::runtime_error("K209 VISA instrument is not open"); - } -} - -auto CompactMK209Driver::float_array_from_little_endian(std::vector bytes) -> std::vector { - if ((bytes.size() % kFloatBytes) != 0U) { - throw std::runtime_error("K209 binary float payload is not aligned to 4 bytes"); - } - - std::vector values(bytes.size() / kFloatBytes, 0.0F); - for (std::size_t index = 0; index < values.size(); ++index) { - const auto offset = index * kFloatBytes; - const std::uint32_t bits = static_cast(bytes[offset]) | - (static_cast(bytes[offset + 1U]) << 8U) | - (static_cast(bytes[offset + 2U]) << 16U) | - (static_cast(bytes[offset + 3U]) << 24U); - std::memcpy(&values[index], &bits, sizeof(float)); - } - return values; -} - -auto CompactMK209Driver::complex_trace_from_interleaved(const std::vector& values) - -> std::vector { - if ((values.size() % kComplexScalarCount) != 0U) { - throw std::runtime_error("K209 complex trace payload has odd scalar count"); - } - - std::vector trace{}; - trace.reserve(values.size() / kComplexScalarCount); - for (std::size_t index = 0; index < values.size(); index += kComplexScalarCount) { - trace.push_back(ipc::Complex32{.re = values[index], .im = values[index + 1U]}); - } - return trace; -} - -auto CompactMK209Driver::format_frequency(float value_hz) -> std::string { - if (!std::isfinite(value_hz)) { - throw std::runtime_error("K209 frequency value is not finite"); - } - - std::ostringstream stream; - stream.precision(9); - stream << std::fixed << value_hz; - return stream.str(); -} - -auto CompactMK209Driver::format_power(float value_dbm) -> std::string { - if (!std::isfinite(value_dbm)) { - throw std::runtime_error("K209 power value is not finite"); - } - - std::ostringstream stream; - stream.precision(3); - stream << std::fixed << value_dbm; - return stream.str(); -} - -auto CompactMK209Driver::trim_ascii(std::string value) -> std::string { - const auto first = std::find_if_not(value.begin(), value.end(), [](unsigned char ch) { - return std::isspace(ch) != 0; - }); - const auto last = std::find_if_not(value.rbegin(), value.rend(), [](unsigned char ch) { - return std::isspace(ch) != 0; - }).base(); - if (first >= last) { - return {}; - } - return std::string(first, last); -} - -} // namespace radar::drivers diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/compact_m_k209_driver.hpp b/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/compact_m_k209_driver.hpp deleted file mode 100644 index 3bd86b8..0000000 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/compact_m_k209_driver.hpp +++ /dev/null @@ -1,71 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include - -#include "radar_driver.hpp" -#include "run_config.hpp" - -namespace radar::drivers { - -struct CompactMK209DriverSettings { - std::string resource{}; - config::RadarSweepSettings sweep{}; - std::uint32_t timeout_ms = 20'000; - bool preset_on_open = true; -}; - -class CompactMK209Driver final : public RadarDriver { - public: - explicit CompactMK209Driver(CompactMK209DriverSettings settings); - ~CompactMK209Driver() override; - - void open() override; - void close() override; - [[nodiscard]] auto acquire_sweep() -> SweepTrace override; - - [[nodiscard]] auto query_identity() -> std::string; - [[nodiscard]] auto query_system_error() -> std::string; - - private: - void configure_device(); - void expect_operation_complete(const std::string& command, const std::string& context); - - void write_command(const std::string& command); - [[nodiscard]] auto query_string(const std::string& command) -> std::string; - [[nodiscard]] auto query_float_array(const std::string& command, std::uint32_t expected_values) - -> std::vector; - [[nodiscard]] auto query_sweep_trace_pair(std::uint32_t points) - -> std::pair, std::vector>; - [[nodiscard]] auto read_float_array_response(const std::string& context, std::uint32_t expected_values) - -> std::vector; - - [[nodiscard]] auto read_ascii_token() -> std::string; - [[nodiscard]] auto read_line() -> std::string; - [[nodiscard]] auto read_byte() -> std::uint8_t; - [[nodiscard]] auto read_exact(std::uint64_t size) -> std::vector; - [[nodiscard]] auto read_ieee_block() -> std::vector; - - void check_status(ViStatus status, const std::string& context) const; - [[nodiscard]] auto status_message(ViStatus status) const -> std::string; - void require_open() const; - - [[nodiscard]] static auto float_array_from_little_endian(std::vector bytes) -> std::vector; - [[nodiscard]] static auto complex_trace_from_interleaved(const std::vector& values) - -> std::vector; - [[nodiscard]] static auto format_frequency(float value_hz) -> std::string; - [[nodiscard]] static auto format_power(float value_dbm) -> std::string; - [[nodiscard]] static auto trim_ascii(std::string value) -> std::string; - - CompactMK209DriverSettings settings_{}; - std::vector frequency_hz_{}; - bool is_open_ = false; - ViSession resource_manager_ = VI_NULL; - ViSession instrument_ = VI_NULL; -}; - -} // namespace radar::drivers diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_lifecycle.cpp b/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_lifecycle.cpp index a4932c0..c9bb66d 100644 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_lifecycle.cpp +++ b/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_lifecycle.cpp @@ -3,15 +3,11 @@ #include #include #include -#include -#include #include #include -#include #include #include #include -#include #include #include @@ -43,60 +39,6 @@ constexpr std::uint32_t kNativeAcquireMaxAttempts = 3U; ); } -// One synthetic GPR reflector. `range_m` is its physical depth, `reflection` -// is the dimensionless complex reflection coefficient (|Γ| ≤ 1). -struct MockTarget { - float range_m; - float reflection_magnitude; -}; - -// Three reflectors at GPR-typical depths: a strong near-surface scatterer, -// a mid-depth target, and a weak deeper one. The magnitudes are tuned so the -// summed S21 stays within unit modulus across the sweep band. -constexpr std::array kMockTargets = {{ - {0.45F, 0.55F}, - {1.30F, 0.30F}, - {2.90F, 0.18F}, -}}; - -// Group velocity in moderately wet soil (≈ c / sqrt(εr), εr ≈ 4). The choice -// is what maps reflector depth to round-trip phase delay; it is held constant -// to keep the simulator deterministic. -constexpr float kGroundVelocityMps = 1.5e8F; - -// Soil attenuation grows with frequency. Calibrated so a target at 3 m sees -// roughly −20 dB extra loss at 6 GHz on top of geometric spreading. -constexpr float kAttenuationCoeffPerMeterAtRefHz = 0.22F; -constexpr float kAttenuationReferenceHz = 6e9F; - -// Antenna mismatch dominates S11: simulate one shallow reflection right at -// the connector, plus a small amount of cross-coupling from S21 targets. -constexpr float kS11ConnectorReflection = 0.55F; -constexpr float kS11ConnectorRangeM = 0.02F; -constexpr float kS11CrossCouplingFactor = 0.06F; - -// Noise floor in linear voltage units. Real LibreVNA hits ~ −90 dB at 1 kHz -// IFBW; pick something a touch noisier so the GPR processor has to work. -constexpr float kNoiseAmplitudeLinear = 0.004F; - -// Minimum simulated dwell so 0-point or pathological configs don't busy-loop. -constexpr auto kMockMinimumSweepDuration = std::chrono::microseconds(50); - -// Compute the dwell time the mock pretends to spend on the device. Mirrors -// the real LibreVNA contract: per-point dwell ≈ 1 / IFBW. Capped so absurd -// configs (IFBW ≈ 0 or huge sweeps) cannot freeze the producer for hours. -[[nodiscard]] auto mock_target_sweep_duration(const config::RadarSweepSettings& sweep) - -> std::chrono::nanoseconds { - const float points = std::max(1.0F, static_cast(sweep.points)); - const float if_bw = std::max(1.0F, sweep.if_bandwidth_hz); - const double seconds = static_cast(points) / static_cast(if_bw); - const auto duration_ns = std::chrono::nanoseconds( - static_cast(seconds * 1e9) - ); - constexpr auto kHardCap = std::chrono::seconds(5); - return std::clamp(duration_ns, kMockMinimumSweepDuration, kHardCap); -} - [[nodiscard]] auto is_retryable_native_acquire_error(std::string_view message) -> bool { // A stop request must not be retried: drain to the caller immediately (#26). if (message.find("aborted by stop request") != std::string_view::npos) { @@ -139,9 +81,7 @@ void LibreVnaMinimalDriver::open() { throw std::runtime_error("Radar sweep points must be > 0"); } - if (settings_.mode == config::DriverMode::Native) { - open_native(); - } + open_native(); is_open_ = true; } @@ -151,166 +91,58 @@ void LibreVnaMinimalDriver::close() { return; } - if (settings_.mode == config::DriverMode::Native) { - close_native(); - } + close_native(); is_open_ = false; } -void LibreVnaMinimalDriver::set_active_combo(const ipc::ComboKey& combo) { - active_combo_ = combo; -} - auto LibreVnaMinimalDriver::acquire_sweep() -> SweepTrace { if (!is_open_) { throw std::runtime_error("Radar driver is not open"); } - ++sweep_index_; - switch (settings_.mode) { - case config::DriverMode::Mock: - return acquire_mock(); - case config::DriverMode::Native: { - std::exception_ptr last_exception{}; - std::string last_message{}; + std::exception_ptr last_exception{}; + std::string last_message{}; - for (std::uint32_t attempt = 1; attempt <= kNativeAcquireMaxAttempts; ++attempt) { - try { - return acquire_native(); - } catch (const std::exception& exception) { - last_exception = std::current_exception(); - last_message = exception.what(); + for (std::uint32_t attempt = 1; attempt <= kNativeAcquireMaxAttempts; ++attempt) { + try { + return acquire_native(); + } catch (const std::exception& exception) { + last_exception = std::current_exception(); + last_message = exception.what(); - if (!is_retryable_native_acquire_error(last_message) || attempt == kNativeAcquireMaxAttempts) { - break; - } - if (detail::native_stop_requested()) { - break; // #26: do not waste reconnect attempts while shutting down. - } - - // #31: recover from transient USB/protocol stalls without - // destroying the libusb_context. Release only the interface - // and handle, then reopen against the surviving context so - // we ride out a brief device re-enumeration instead of - // re-initialising libusb from scratch on every retry. - rx_buffer_.clear(); - packet_queue_.clear(); - if (usb_handle_ != nullptr && interface_claimed_) { - libusb_release_interface(usb_handle_, detail::kUsbInterface); - interface_claimed_ = false; - } - if (usb_handle_ != nullptr) { - libusb_close(usb_handle_); - usb_handle_ = nullptr; - } - protocol_version_ = 0; - device_num_ports_ = 0; - open_native(); // Reuses the surviving usb_context_ and retries discovery. - } + if (!is_retryable_native_acquire_error(last_message) || attempt == kNativeAcquireMaxAttempts) { + break; + } + if (detail::native_stop_requested()) { + break; // #26: do not waste reconnect attempts while shutting down. } - if (last_exception != nullptr) { - std::rethrow_exception(last_exception); + // #31: recover from transient USB/protocol stalls without + // destroying the libusb_context. Release only the interface + // and handle, then reopen against the surviving context so + // we ride out a brief device re-enumeration instead of + // re-initialising libusb from scratch on every retry. + rx_buffer_.clear(); + packet_queue_.clear(); + if (usb_handle_ != nullptr && interface_claimed_) { + libusb_release_interface(usb_handle_, detail::kUsbInterface); + interface_claimed_ = false; } - throw std::runtime_error("Native acquisition failed without exception detail"); + if (usb_handle_ != nullptr) { + libusb_close(usb_handle_); + usb_handle_ = nullptr; + } + protocol_version_ = 0; + device_num_ports_ = 0; + open_native(); // Reuses the surviving usb_context_ and retries discovery. } - default: - throw std::runtime_error("Unsupported radar driver mode"); - } -} - -auto LibreVnaMinimalDriver::acquire_mock() -> SweepTrace { - // Synthesise a frequency-domain VNA response containing several discrete - // reflectors at known depths, plus light Gaussian noise. The sweep dwells - // for a realistic duration derived from the configured IF bandwidth so - // downstream stages cannot be flooded faster than a real device would - // produce data. - const auto started_at = std::chrono::steady_clock::now(); - const auto target_duration = mock_target_sweep_duration(settings_.sweep); - - SweepTrace trace{}; - trace.frequency_hz.reserve(settings_.sweep.points); - trace.s11.reserve(settings_.sweep.points); - trace.s21.reserve(settings_.sweep.points); - - const float span_hz = settings_.sweep.stop_hz - settings_.sweep.start_hz; - const float denominator = - settings_.sweep.points > 1U ? static_cast(settings_.sweep.points - 1U) : 1.0F; - - // A tiny per-sweep range drift gives the rendered B-scan a visible motion - // signature so the simulator does not look frozen. - const float range_drift_m = - 0.01F * std::sin(0.07F * static_cast(sweep_index_)); - - // Combo-dependent variation. Without this the mock returns near-identical - // S21 for every (input, output) combo and an eight-combo pass-through plot - // collapses into a single visible trace. The factors below are arbitrary - // but chosen small enough that the overall response stays in a reasonable - // band and large enough that each pair is visually distinct. - const auto input_pos = static_cast(active_combo_.input_pos); - const auto output_pos = static_cast(active_combo_.output_pos); - const float combo_amplitude_gain = 0.55F + 0.08F * input_pos + 0.05F * output_pos; - const float combo_phase_offset = 0.4F * input_pos + 0.9F * output_pos; - const float combo_range_offset_m = 0.05F * input_pos + 0.12F * output_pos; - - // Deterministic-per-sweep noise so two consecutive frames look distinct - // but the test stays reproducible for any given sweep index. - std::mt19937 noise_engine( - static_cast(0x9E3779B9ULL ^ sweep_index_) - ); - std::normal_distribution noise_dist(0.0F, kNoiseAmplitudeLinear); - - for (std::uint32_t point = 0; point < settings_.sweep.points; ++point) { - const float ratio = static_cast(point) / denominator; - const float frequency_hz = settings_.sweep.start_hz + span_hz * ratio; - const float frequency_scale = frequency_hz / kAttenuationReferenceHz; - - std::complex s21_total{0.0F, 0.0F}; - std::complex s11_total{0.0F, 0.0F}; - - for (const auto& target : kMockTargets) { - const float range_m = target.range_m + range_drift_m + combo_range_offset_m; - // Round-trip phase: 2π·f·(2R/v). - const float round_trip_phase = - 2.0F * detail::kPi * frequency_hz * (2.0F * range_m / kGroundVelocityMps); - // Geometric spreading: 1/(1 + R) keeps near-zero ranges finite. - const float spreading = 1.0F / (1.0F + range_m); - // Frequency-dependent soil attenuation in linear amplitude. - const float attenuation = - std::exp(-kAttenuationCoeffPerMeterAtRefHz * range_m * frequency_scale); - - const std::complex contribution = std::polar( - target.reflection_magnitude * spreading * attenuation * combo_amplitude_gain, - -round_trip_phase + combo_phase_offset - ); - s21_total += contribution; - s11_total += kS11CrossCouplingFactor * contribution; - } - - // Antenna mismatch dominates the near-field S11 response. - const float antenna_phase = - 2.0F * detail::kPi * frequency_hz * (2.0F * kS11ConnectorRangeM / kGroundVelocityMps); - s11_total += std::polar(kS11ConnectorReflection, -antenna_phase); - - // Independent noise per channel; complex variance ≈ kNoiseAmplitudeLinear². - s21_total += std::complex(noise_dist(noise_engine), noise_dist(noise_engine)); - s11_total += std::complex(noise_dist(noise_engine), noise_dist(noise_engine)); - - trace.frequency_hz.push_back(frequency_hz); - trace.s11.push_back({.re = s11_total.real(), .im = s11_total.imag()}); - trace.s21.push_back({.re = s21_total.real(), .im = s21_total.imag()}); } - // Honour the IFBW-derived dwell time. If generation already took longer - // than the simulated device would have needed (huge `points` × CPU jitter) - // we skip the sleep so the producer does not fall further behind. - const auto elapsed = std::chrono::steady_clock::now() - started_at; - if (elapsed < target_duration) { - std::this_thread::sleep_for(target_duration - elapsed); + if (last_exception != nullptr) { + std::rethrow_exception(last_exception); } - - return trace; + throw std::runtime_error("Native acquisition failed without exception detail"); } auto LibreVnaMinimalDriver::acquire_native() -> SweepTrace { diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna_minimal_driver.hpp b/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna_minimal_driver.hpp index 633b137..e6d4bb7 100644 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna_minimal_driver.hpp +++ b/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna_minimal_driver.hpp @@ -23,10 +23,8 @@ namespace radar::drivers { * @brief Runtime settings for minimal LibreVNA acquisition. */ struct LibreVnaMinimalDriverSettings { - config::DriverMode mode = config::DriverMode::Mock; std::string serial{}; config::RadarSweepSettings sweep{}; - float mock_signal_hz = 5'000'000.0F; }; /** @@ -43,7 +41,6 @@ class LibreVnaMinimalDriver final : public RadarDriver { void open() override; void close() override; [[nodiscard]] auto acquire_sweep() -> SweepTrace override; - void set_active_combo(const ipc::ComboKey& combo) override; private: /** @@ -66,7 +63,6 @@ class LibreVnaMinimalDriver final : public RadarDriver { bool has_s21 = false; }; - [[nodiscard]] auto acquire_mock() -> SweepTrace; [[nodiscard]] auto acquire_native() -> SweepTrace; void open_native(); @@ -92,11 +88,6 @@ class LibreVnaMinimalDriver final : public RadarDriver { LibreVnaMinimalDriverSettings settings_{}; bool is_open_ = false; - std::uint64_t sweep_index_ = 0; - // Latest combo announced by the orchestrator. Used by the mock backend to - // give each (input, output) pair a slightly different reflectivity profile - // so a multi-combo run does not render as eight identical traces. - ipc::ComboKey active_combo_{}; libusb_context* usb_context_ = nullptr; libusb_device_handle* usb_handle_ = nullptr; diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/remote_compact_m_k209/remote_compact_m_k209_driver.cpp b/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/remote_compact_m_k209/remote_compact_m_k209_driver.cpp deleted file mode 100644 index 06024d6..0000000 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/remote_compact_m_k209/remote_compact_m_k209_driver.cpp +++ /dev/null @@ -1,271 +0,0 @@ -#include "../remote_compact_m_k209_driver.hpp" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace radar::drivers { -namespace { - -constexpr std::uint8_t kCommandConfigure = static_cast('C'); -constexpr std::uint8_t kCommandAcquire = static_cast('A'); -constexpr std::uint8_t kStatusOk = static_cast('O'); -constexpr std::uint8_t kStatusError = static_cast('E'); - -[[nodiscard]] auto system_error_message(const std::string& context) -> std::string { - return context + ": " + std::strerror(errno); -} - -void append_u32_be(std::vector& buffer, std::uint32_t value) { - buffer.push_back(static_cast((value >> 24U) & 0xFFU)); - buffer.push_back(static_cast((value >> 16U) & 0xFFU)); - buffer.push_back(static_cast((value >> 8U) & 0xFFU)); - buffer.push_back(static_cast(value & 0xFFU)); -} - -void append_u64_be(std::vector& buffer, std::uint64_t value) { - for (int shift = 56; shift >= 0; shift -= 8) { - buffer.push_back(static_cast((value >> static_cast(shift)) & 0xFFU)); - } -} - -void append_double_be(std::vector& buffer, double value) { - std::uint64_t bits = 0; - static_assert(sizeof(bits) == sizeof(value)); - std::memcpy(&bits, &value, sizeof(bits)); - append_u64_be(buffer, bits); -} - -[[nodiscard]] auto parse_u32_be(const std::uint8_t* data) -> std::uint32_t { - return (static_cast(data[0]) << 24U) | - (static_cast(data[1]) << 16U) | - (static_cast(data[2]) << 8U) | - static_cast(data[3]); -} - -void send_all(int socket_fd, const void* data, std::size_t size) { - const auto* cursor = static_cast(data); - auto remaining = size; - while (remaining > 0U) { - const auto sent = ::send(socket_fd, cursor, remaining, MSG_NOSIGNAL); - if (sent < 0) { - throw std::runtime_error(system_error_message("send to K209 remote server failed")); - } - if (sent == 0) { - throw std::runtime_error("send to K209 remote server returned zero bytes"); - } - cursor += sent; - remaining -= static_cast(sent); - } -} - -void recv_exact(int socket_fd, void* data, std::size_t size) { - auto* cursor = static_cast(data); - auto remaining = size; - while (remaining > 0U) { - const auto received = ::recv(socket_fd, cursor, remaining, 0); - if (received < 0) { - throw std::runtime_error(system_error_message("read from K209 remote server failed")); - } - if (received == 0) { - throw std::runtime_error("K209 remote server closed the connection"); - } - cursor += received; - remaining -= static_cast(received); - } -} - -[[nodiscard]] auto recv_u32(int socket_fd) -> std::uint32_t { - std::uint8_t bytes[4]{}; - recv_exact(socket_fd, bytes, sizeof(bytes)); - return parse_u32_be(bytes); -} - -void read_status(int socket_fd) { - std::uint8_t status = 0; - recv_exact(socket_fd, &status, 1); - if (status == kStatusOk) { - return; - } - if (status == kStatusError) { - const auto message_size = recv_u32(socket_fd); - std::string message(message_size, '\0'); - if (message_size > 0U) { - recv_exact(socket_fd, message.data(), message.size()); - } - throw std::runtime_error("K209 remote server error: " + message); - } - throw std::runtime_error("K209 remote server returned invalid status byte"); -} - -[[nodiscard]] auto recv_float32_array(int socket_fd, std::uint32_t expected_values, const std::string& context) - -> std::vector { - const auto payload_size = recv_u32(socket_fd); - const auto expected_size = expected_values * static_cast(sizeof(float)); - if (payload_size != expected_size) { - throw std::runtime_error( - "K209 remote " + context + " payload has " + std::to_string(payload_size) + - " bytes, expected " + std::to_string(expected_size) - ); - } - - std::vector values(expected_values); - if (payload_size > 0U) { - recv_exact(socket_fd, values.data(), payload_size); - } - return values; -} - -[[nodiscard]] auto connect_socket(const std::string& host, std::uint32_t port, std::uint32_t timeout_ms) -> int { - addrinfo hints{}; - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - hints.ai_protocol = IPPROTO_TCP; - - addrinfo* result = nullptr; - const auto port_text = std::to_string(port); - const auto rc = ::getaddrinfo(host.c_str(), port_text.c_str(), &hints, &result); - if (rc != 0) { - throw std::runtime_error("getaddrinfo failed for K209 remote host " + host + ": " + ::gai_strerror(rc)); - } - - int connected_fd = -1; - std::string last_error; - for (auto* item = result; item != nullptr; item = item->ai_next) { - const int fd = ::socket(item->ai_family, item->ai_socktype, item->ai_protocol); - if (fd < 0) { - last_error = system_error_message("socket"); - continue; - } - - const int one = 1; - static_cast(::setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one))); - timeval timeout{}; - timeout.tv_sec = static_cast(timeout_ms / 1000U); - timeout.tv_usec = static_cast((timeout_ms % 1000U) * 1000U); - static_cast(::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout))); - static_cast(::setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout))); - - if (::connect(fd, item->ai_addr, item->ai_addrlen) == 0) { - connected_fd = fd; - break; - } - last_error = system_error_message("connect"); - ::close(fd); - } - ::freeaddrinfo(result); - - if (connected_fd < 0) { - throw std::runtime_error("Failed to connect to K209 remote server " + host + ":" + port_text + ": " + last_error); - } - return connected_fd; -} - -[[nodiscard]] auto complex_from_interleaved(const std::vector& values, std::uint32_t points, const std::string& context) - -> std::vector { - if (values.size() != static_cast(points) * 2U) { - throw std::runtime_error("K209 remote " + context + " returned unexpected scalar count"); - } - - std::vector output(points); - for (std::uint32_t index = 0; index < points; ++index) { - output[index] = ipc::Complex32{ - .re = values[static_cast(index) * 2U], - .im = values[static_cast(index) * 2U + 1U], - }; - } - return output; -} - -} // namespace - -RemoteCompactMK209Driver::RemoteCompactMK209Driver(RemoteCompactMK209DriverSettings settings) - : settings_(std::move(settings)) { - if (settings_.host.empty()) { - throw std::runtime_error("K209 remote host must not be empty"); - } - if (settings_.port == 0U || settings_.port > 65535U) { - throw std::runtime_error("K209 remote port must be in 1..65535"); - } - if (settings_.timeout_ms == 0U) { - throw std::runtime_error("K209 remote timeout_ms must be > 0"); - } -} - -RemoteCompactMK209Driver::~RemoteCompactMK209Driver() { - try { - close(); - } catch (...) { - } -} - -void RemoteCompactMK209Driver::open() { - if (socket_fd_ >= 0) { - return; - } - - socket_fd_ = connect_socket(settings_.host, settings_.port, settings_.timeout_ms); - - try { - std::vector payload{}; - payload.reserve(1U + 8U + 8U + 4U + 8U + 8U); - payload.push_back(kCommandConfigure); - append_double_be(payload, static_cast(settings_.sweep.start_hz)); - append_double_be(payload, static_cast(settings_.sweep.stop_hz)); - append_u32_be(payload, settings_.sweep.points); - append_double_be(payload, static_cast(settings_.sweep.if_bandwidth_hz)); - append_double_be(payload, static_cast(settings_.sweep.power_dbm)); - send_all(socket_fd_, payload.data(), payload.size()); - - read_status(socket_fd_); - const auto points = recv_u32(socket_fd_); - if (points != settings_.sweep.points) { - throw std::runtime_error("K209 remote server returned unexpected configured point count"); - } - frequency_hz_ = recv_float32_array(socket_fd_, points, "frequency"); - } catch (...) { - close(); - throw; - } -} - -void RemoteCompactMK209Driver::close() { - if (socket_fd_ >= 0) { - ::close(socket_fd_); - socket_fd_ = -1; - } - frequency_hz_.clear(); -} - -auto RemoteCompactMK209Driver::acquire_sweep() -> SweepTrace { - if (socket_fd_ < 0) { - throw std::runtime_error("K209 remote driver is not open"); - } - const std::uint8_t command = kCommandAcquire; - send_all(socket_fd_, &command, 1); - read_status(socket_fd_); - - const auto points = recv_u32(socket_fd_); - if (points != settings_.sweep.points) { - throw std::runtime_error("K209 remote server returned unexpected sweep point count"); - } - const auto s11_values = recv_float32_array(socket_fd_, points * 2U, "S11"); - const auto s21_values = recv_float32_array(socket_fd_, points * 2U, "S21"); - - return SweepTrace{ - .frequency_hz = frequency_hz_, - .s11 = complex_from_interleaved(s11_values, points, "S11"), - .s21 = complex_from_interleaved(s21_values, points, "S21"), - }; -} - -} // namespace radar::drivers diff --git a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/remote_compact_m_k209_driver.hpp b/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/remote_compact_m_k209_driver.hpp deleted file mode 100644 index 3f8a827..0000000 --- a/data_acq_and_processing/sweep_orchestrator/device_drivers/radar/remote_compact_m_k209_driver.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "radar_driver.hpp" -#include "run_config.hpp" - -namespace radar::drivers { - -struct RemoteCompactMK209DriverSettings { - std::string host = "127.0.0.1"; - std::uint32_t port = 50209; - config::RadarSweepSettings sweep{}; - std::uint32_t timeout_ms = 20'000; -}; - -class RemoteCompactMK209Driver final : public RadarDriver { - public: - explicit RemoteCompactMK209Driver(RemoteCompactMK209DriverSettings settings); - ~RemoteCompactMK209Driver() override; - - void open() override; - void close() override; - [[nodiscard]] auto acquire_sweep() -> SweepTrace override; - - private: - RemoteCompactMK209DriverSettings settings_{}; - int socket_fd_ = -1; - std::vector frequency_hz_{}; -}; - -} // namespace radar::drivers diff --git a/data_acq_and_processing/sweep_orchestrator/src/main.cpp b/data_acq_and_processing/sweep_orchestrator/src/main.cpp index e5d6573..9b1408c 100644 --- a/data_acq_and_processing/sweep_orchestrator/src/main.cpp +++ b/data_acq_and_processing/sweep_orchestrator/src/main.cpp @@ -10,7 +10,6 @@ #include "hmc349a_minimal_driver.hpp" #include "librevna_minimal_driver.hpp" #include "radar_driver.hpp" -#include "remote_compact_m_k209_driver.hpp" #include "run_config.hpp" #include "shm_ring.hpp" #include "sweep_orchestrator.hpp" @@ -20,7 +19,6 @@ namespace { constexpr const char* kDefaultConfigPath = "run_config.json"; constexpr const char* kLibreVnaModel = "librevna"; constexpr const char* kLibreVnaMultiModel = "librevna_multi"; -constexpr const char* kCompactMK209Model = "compact_m_k209"; std::atomic g_stop_requested{false}; void signal_handler(int /*signal*/) { @@ -48,26 +46,13 @@ void install_signal_handlers() { [[nodiscard]] auto make_radar_driver(const radar::config::RadarConfig& config) -> std::unique_ptr { if (config.model == kLibreVnaModel || config.model.empty()) { + if (config.driver_mode != radar::config::DriverMode::Native) { + throw std::runtime_error("LibreVNA simulation uses python_app.scripts.matrix_raw_producer; launch through start.sh"); + } return std::make_unique( radar::drivers::LibreVnaMinimalDriverSettings{ - .mode = config.driver_mode, .serial = config.serial, .sweep = config.sweep, - .mock_signal_hz = config.mock_signal_hz, - } - ); - } - - if (config.model == kCompactMK209Model) { - if (config.driver_mode != radar::config::DriverMode::Native) { - throw std::runtime_error("compact_m_k209 requires radar.driver_mode='native'"); - } - return std::make_unique( - radar::drivers::RemoteCompactMK209DriverSettings{ - .host = config.remote_host, - .port = config.remote_port, - .sweep = config.sweep, - .timeout_ms = 20'000, } ); } @@ -76,10 +61,6 @@ void install_signal_handlers() { throw std::runtime_error("librevna_multi is handled by python_app.scripts.matrix_raw_producer"); } - if (config.model == "sn9000") { - throw std::runtime_error("sn9000 is handled by python_app.scripts.matrix_raw_producer"); - } - throw std::runtime_error("Unsupported radar.model for sweep_orchestrator: " + config.model); } diff --git a/data_acq_and_processing/sweep_orchestrator/src/sweep_orchestrator.cpp b/data_acq_and_processing/sweep_orchestrator/src/sweep_orchestrator.cpp index 76661ba..1509348 100644 --- a/data_acq_and_processing/sweep_orchestrator/src/sweep_orchestrator.cpp +++ b/data_acq_and_processing/sweep_orchestrator/src/sweep_orchestrator.cpp @@ -1,12 +1,11 @@ #include "sweep_orchestrator.hpp" +#include "runtime_wait.hpp" #include -#include #include #include #include #include -#include #include namespace radar::acq { @@ -26,28 +25,6 @@ constexpr std::uint64_t kOpenRetryLogEvery = 30U; return stop_requested.load(std::memory_order_relaxed); } -void sleep_if_needed_ms(std::uint32_t delay_ms) { - if (delay_ms == 0U) { - return; - } - std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); -} - -// Interruptible sleep: returns true if a stop was requested while waiting. -[[nodiscard]] auto interruptible_sleep_ms(std::uint32_t delay_ms, const std::atomic& stop_requested) -> bool { - constexpr std::uint32_t kPollMs = 50U; - std::uint32_t waited = 0U; - while (waited < delay_ms) { - if (should_stop(stop_requested)) { - return true; - } - const std::uint32_t chunk = std::min(kPollMs, delay_ms - waited); - std::this_thread::sleep_for(std::chrono::milliseconds(chunk)); - waited += chunk; - } - return should_stop(stop_requested); -} - void validate_sweep(const drivers::SweepTrace& sweep) { if (sweep.frequency_hz.size() != sweep.s11.size() || sweep.frequency_hz.size() != sweep.s21.size()) { throw std::runtime_error("Radar driver returned inconsistent sweep vectors"); @@ -66,30 +43,14 @@ class DriverLifecycleGuard { output_switch_driver_(output_switch_driver) {} ~DriverLifecycleGuard() noexcept { - if (!active_) { - return; - } - - // Shutdown should never hide the primary error path. - try { - output_switch_driver_.close(); - } catch (...) { - } - try { - input_switch_driver_.close(); - } catch (...) { - } - try { - radar_driver_.close(); - } catch (...) { - } + close_all(); } void open_all() { + active_ = true; // A later open failure still owns partially opened devices. radar_driver_.open(); input_switch_driver_.open(); output_switch_driver_.open(); - active_ = true; } // Open all devices, retrying forever with capped exponential backoff until success or @@ -118,7 +79,7 @@ class DriverLifecycleGuard { << "); retrying up to every " << (kOpenRetryMaxMs / 1'000U) << "s until present: " << exc.what() << '\n'; } - if (interruptible_sleep_ms(delay_ms, stop_requested)) { + if (config::wait_or_stop(delay_ms, stop_requested)) { return false; } delay_ms = std::min(delay_ms * 2U, kOpenRetryMaxMs); @@ -127,14 +88,23 @@ class DriverLifecycleGuard { return false; } - void close_all() { + void close_all() noexcept { if (!active_) { return; } - output_switch_driver_.close(); - input_switch_driver_.close(); - radar_driver_.close(); active_ = false; + const auto close = [](auto& driver) { + try { + driver.close(); + } catch (const std::exception& error) { + std::cerr << "sweep_orchestrator: driver cleanup failed: " << error.what() << '\n'; + } catch (...) { + std::cerr << "sweep_orchestrator: driver cleanup failed with an unknown error\n"; + } + }; + close(output_switch_driver_); + close(input_switch_driver_); + close(radar_driver_); } private: @@ -238,7 +208,7 @@ void SweepOrchestrator::run(const std::atomic& stop_requested) { if (!config_.runtime.continuous) { break; } - sleep_if_needed_ms(config_.runtime.idle_sleep_ms); + (void)config::wait_or_stop(config_.runtime.idle_sleep_ms, stop_requested); continue; } @@ -289,11 +259,11 @@ auto SweepOrchestrator::acquire_one_collection( // 1) switch output path, 2) switch input path, 3) optional settle. output_switch_driver_.switch_to(combo.output_pos); input_switch_driver_.switch_to(combo.input_pos); - sleep_if_needed_ms(config_.runtime.settling_ms); + if (config::wait_or_stop(config_.runtime.settling_ms, stop_requested)) { + interrupted = true; + break; + } - // Production drivers ignore this; mock drivers use it to give every - // (input, output) pair its own synthetic response. - radar_driver_.set_active_combo(combo); // Stamp around the sweep only: the switch drive and settling above belong to // neither the previous combo nor this one, so excluding them keeps the window // an honest "when was this combo actually measured". diff --git a/data_acq_and_processing/tests/generate_sar3d_emulation.py b/data_acq_and_processing/tests/generate_sar3d_emulation.py new file mode 100644 index 0000000..4adabee --- /dev/null +++ b/data_acq_and_processing/tests/generate_sar3d_emulation.py @@ -0,0 +1,18 @@ +"""Extract production CUDA kernels and host helpers for the test-only adapter.""" + +from pathlib import Path +import re +import sys + + +source = (Path(__file__).resolve().parents[1] / "processing/processors/src/sar3d_gpu.cpp").read_text() +kernel = re.search(r'kKernelSource\s*=\s*R"CUDA\((.*?)\)CUDA";', source, re.DOTALL) +if kernel is None: + raise RuntimeError("SAR CUDA kernel literal was not found") +first = source.index("constexpr std::size_t kMaxIndexedItems") +last = source.index("} // namespace", first) +destination = Path(sys.argv[1]) +destination.mkdir(parents=True, exist_ok=True) +(destination / "sar3d_kernels.inc").write_text(kernel.group(1)) +helpers = re.sub(r"^#(?:ifdef SAR3D_WITH_CUDA|endif)$", "", source[first:last], flags=re.MULTILINE) +(destination / "sar3d_helpers.inc").write_text(helpers) diff --git a/data_acq_and_processing/tests/native_regressions.cpp b/data_acq_and_processing/tests/native_regressions.cpp new file mode 100644 index 0000000..746ec0e --- /dev/null +++ b/data_acq_and_processing/tests/native_regressions.cpp @@ -0,0 +1,398 @@ +#include "capture_file_writer.h" +#include "channel_bundle_support.hpp" +#include "locator/tcp_server.hpp" +#include "processing_live_config.hpp" +#include "runtime_wait.hpp" +#include "shm_ring.hpp" +#include "sweep_orchestrator.hpp" +#include "tty_protocol_writer.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace { +using namespace radar; +using Json = nlohmann::json; + +void require(bool condition, const char* message) { + if (!condition) { + throw std::runtime_error(message); + } +} + +template +void expect_error(Callable operation) { + try { + operation(); + } catch (const std::bad_alloc&) { + throw; // A malformed count must be rejected before allocating. + } catch (const std::exception&) { + return; + } + throw std::runtime_error("Expected operation to reject malformed input"); +} + +struct Workspace { + std::filesystem::path path; + Workspace() { + char pattern[] = "/tmp/radar-native-XXXXXX"; + const auto* created = ::mkdtemp(pattern); + require(created != nullptr, "mkdtemp failed"); + path = created; + } + ~Workspace() { std::filesystem::remove_all(path); } +}; + +struct RingName { + std::string name = "/radar_native_test_" + std::to_string(::getpid()); + RingName() { ipc::ShmRing::unlink_ring(name); } + ~RingName() { ipc::ShmRing::unlink_ring(name); } +}; + +void write_u32(std::vector& bytes, std::size_t offset, std::uint32_t value) { + require(offset + sizeof(value) <= bytes.size(), "invalid test offset"); + std::memcpy(bytes.data() + offset, &value, sizeof(value)); +} + +void test_wire_decoders() { + ipc::RawSweepCollection raw{.collection_id = 7, .monotonic_ns = 42, + .capture_start_ns = 21, .capture_end_ns = 40, + .traces = {{.combo = {1, 2}, .frequency_hz = {1, 2}, + .s11 = {{1, 2}, {3, 4}}, .s21 = {{5, 6}, {7, 8}}, + .capture_start_ns = 25, .capture_end_ns = 39}}}; + ipc::ResultCollection results{.collection_id = 7, .processing_duration_ns = 17, + .collection_payloads = { + {.kind = ipc::ResultKind::TraceComplex, .frequency_hz = {1}, .trace = {{2, 3}}}, + {.kind = ipc::ResultKind::ScalarF32, .scalar_value = 4}, + {.kind = ipc::ResultKind::ImageF32, .image_x_axis = {1, 2}, + .image_y_axis = {3}, .image_values = {4, 5}}, + {.kind = ipc::ResultKind::TableF32, .table_columns = 2, .table_values = {6, 7}}, + {.kind = ipc::ResultKind::ImageF32, .image_y_axis = {1}}, + {.kind = ipc::ResultKind::TableF32, .table_columns = 3}}, + .blocks = {{.combo = {1, 2}, .payloads = {{.kind = ipc::ResultKind::ScalarF32}}}}}; + const auto raw_bytes = ipc::serialize_raw_collection(raw); + const auto pre_bytes = ipc::serialize_preprocessed_collection(raw); + const auto result_bytes = ipc::serialize_result_collection(results); + require(ipc::serialize_raw_collection(ipc::deserialize_raw_collection(raw_bytes)) == raw_bytes, + "raw roundtrip changed data"); + require(ipc::serialize_preprocessed_collection(ipc::deserialize_preprocessed_collection(pre_bytes)) == pre_bytes, + "preprocessed roundtrip changed data"); + require(ipc::serialize_result_collection(ipc::deserialize_result_collection(result_bytes)) == result_bytes, + "result roundtrip changed data or empty shape"); + + for (std::size_t size = 0; size < result_bytes.size(); ++size) { + expect_error([&] { (void)ipc::deserialize_result_collection(std::span(result_bytes).first(size)); }); + } + for (std::size_t count_offset : {28U, 32U, 39U}) { + auto corrupt = result_bytes; + write_u32(corrupt, count_offset, std::numeric_limits::max()); + expect_error([&] { (void)ipc::deserialize_result_collection(corrupt); }); + } + for (auto kind : {ipc::ResultKind::ImageF32, ipc::ResultKind::TableF32}) { + ipc::ResultCollection empty{.collection_payloads = {{.kind = kind}}}; + auto corrupt = ipc::serialize_result_collection(empty); + write_u32(corrupt, 39, std::numeric_limits::max()); + write_u32(corrupt, 43, std::numeric_limits::max()); + expect_error([&] { (void)ipc::deserialize_result_collection(corrupt); }); + } + // Deterministic random mutations include wire-compatible valid frames as seeds. + std::mt19937 random(0x52414452U); + for (unsigned iteration = 0; iteration < 10'000U; ++iteration) { + auto bytes = iteration % 3U == 0U ? raw_bytes : iteration % 3U == 1U ? pre_bytes : result_bytes; + for (unsigned edit = 0; edit < 1U + random() % 5U; ++edit) { + bytes[random() % bytes.size()] = static_cast(random()); + } + if (iteration % 4U == 0U) { + bytes.resize(random() % bytes.size()); + } + try { + if (iteration % 3U == 0U) { (void)ipc::deserialize_raw_collection(bytes); } + else if (iteration % 3U == 1U) { (void)ipc::deserialize_preprocessed_collection(bytes); } + else { (void)ipc::deserialize_result_collection(bytes); } + } catch (const std::bad_alloc&) { throw; } + catch (const std::runtime_error&) {} + } +} + +void test_configuration(const std::string& seed_path, const Workspace& workspace) { + std::ifstream input(seed_path); + const auto seed = Json::parse(input); + const auto config_path = workspace.path / "config.json"; + const auto save = [&](const Json& config) { std::ofstream(config_path) << config; }; + save(seed); + const auto config = config::load_run_config(config_path.string()); + for (const auto& [pointer, value] : std::array, 8>{{ + {"/radar/sweep/start_hz", -1}, {"/radar/sweep/stop_hz", 1e100}, + {"/radar/sweep/if_bandwidth_hz", 0}, {"/radar/sweep/points", 0}, + {"/radar/sweep/points", 4294967296.0}, {"/run/combos/0/input", 0.0000001}, + {"/rings/raw/slot_size_bytes", 65}, {"/gpr/rx_geometry/0/x_m", 1e100}}}) { + auto invalid = seed; + invalid[Json::json_pointer(pointer)] = value; + save(invalid); + expect_error([&] { (void)config::load_run_config(config_path.string()); }); + } + const auto live_path = workspace.path / "live.json"; + std::ofstream(live_path) << Json{{"bscan_gain", 2}, {"settings_revision", 11}}; + processing::ProcessingLiveConfigLoader loader(live_path.string(), config); + require(loader.refresh_if_needed().bscan_gain == 2, "valid live config did not load"); + const auto revision = loader.revision(); + for (const auto& invalid : {Json{{"bscan_gain", 1e100}}, + Json{{"gpr_input_positions", {0.0000001}}}, Json{{"bscan_gain", true}}}) { + std::ofstream(live_path) << invalid; + std::this_thread::sleep_for(std::chrono::milliseconds(110)); + const auto current = loader.refresh_if_needed(); + require(current.bscan_gain == 2 && loader.revision() == revision, + "invalid live config replaced the last valid settings"); + } + const std::array nan_axis{std::numeric_limits::quiet_NaN()}; + require(!preprocessing::frequency_axes_match(nan_axis, nan_axis), "NaN frequency axes matched"); + preprocessing::ChannelTrace invalid_trace{.frequency_hz = {1}, + .samples = {{std::numeric_limits::infinity(), 0}}}; + expect_error([&] { preprocessing::validate_channel_trace_layout(invalid_trace, "test", {}); }); +} + +void test_ring() { + RingName name; + expect_error([&] { (void)ipc::ShmRing::open_or_create(name.name, 2, 65); }); + expect_error([&] { (void)ipc::ShmRing::open_or_create(name.name, 0xffffffffU, 0xfffffff8U); }); + auto ring = ipc::ShmRing::open_or_create(name.name, 2, 4096); + const auto child = ::fork(); + require(child >= 0, "fork failed"); + if (child == 0) { + std::array words{}; + for (std::uint64_t sequence = 1; sequence <= 20'000U; ++sequence) { + words.fill(sequence); + if (!ring.push({reinterpret_cast(words.data()), sizeof(words)})) { + ::_exit(2); + } + } + ::_exit(0); + } + std::uint64_t previous = 0; + std::vector payload; + int status = 0; + bool finished = false; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + try { + while (true) { + if (ring.pop(payload)) { + require(payload.size() == 4096, "stress payload changed size"); + std::uint64_t sequence = 0; + std::memcpy(&sequence, payload.data(), sizeof(sequence)); + require(sequence > previous, "stress ring yielded duplicate or older data"); + for (std::size_t offset = 0; offset < payload.size(); offset += sizeof(sequence)) { + std::uint64_t word = 0; + std::memcpy(&word, payload.data() + offset, sizeof(word)); + require(word == sequence, "stress ring accepted a torn payload"); + } + previous = sequence; + } else if (finished) { break; } + finished = finished || ::waitpid(child, &status, WNOHANG) == child; + require(std::chrono::steady_clock::now() < deadline, "ring stress timed out"); + } + require(WIFEXITED(status) && WEXITSTATUS(status) == 0 && previous != 0, + "ring stress writer failed"); + } catch (...) { + ::kill(child, SIGKILL); + if (!finished) { (void)::waitpid(child, &status, 0); } + throw; + } +} + +void test_partial_driver_open() { + struct Radar final : drivers::RadarDriver { + unsigned closes = 0; + void open() override {} + void close() override { ++closes; } + auto acquire_sweep() -> drivers::SweepTrace override { return {}; } + } radar; + struct Switch final : drivers::SwitchDriver { + std::atomic& stop; + bool fail_open = false; + bool fail_close = false; + unsigned closes = 0; + explicit Switch(std::atomic& stop_flag) : stop(stop_flag) {} + void open() override { + if (fail_open) { stop.store(true); throw std::runtime_error("open failure"); } + } + void close() override { ++closes; if (fail_close) { throw std::runtime_error("close failure"); } } + auto position_count() const -> std::uint32_t override { return 1; } + void switch_to(std::uint32_t) override {} + }; + std::atomic stop{false}; + Switch input(stop), output(stop); + input.fail_open = true; + output.fail_close = true; + config::RunConfig config; + config.radar.sweep.points = 1; + config.run_combos = {{0, 0}}; + RingName name; + auto ring = ipc::ShmRing::open_or_create(name.name, 2, 256); + acq::SweepOrchestrator(config, radar, input, output, ring).run(stop); + require(radar.closes == 1 && input.closes == 1 && output.closes == 1, + "partial driver startup did not close each driver exactly once"); +} + +void test_interruptible_wait() { + std::atomic stop{false}; + std::jthread request_stop([&] { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + stop.store(true); + }); + const auto start = std::chrono::steady_clock::now(); + require(config::wait_or_stop(0xffffffffU, stop), "large wait ignored stop"); + require(std::chrono::steady_clock::now() - start < std::chrono::seconds(1), + "large wait delayed shutdown"); +} + +void test_csv_failures(const Workspace& workspace) { + const auto csv = workspace.path / "capture.csv"; + const auto spool = workspace.path / "capture.capture_spool.bin"; + CaptureFileWriter writer(csv.string(), (workspace.path / "capture.svg").string(), "", ""); + writer.initialize_csv(2, false); + CapturePacket packet{.ch1 = {1, 2}, .ch2 = {3, 4}, .di1 = {}}; + auto invalid_packet = packet; + invalid_packet.ch2.pop_back(); + expect_error([&] { writer.append_csv_packet(invalid_packet); }); + require(std::filesystem::file_size(spool) == 16, "invalid packet partially changed the spool"); + writer.append_csv_packet(packet); + std::filesystem::remove(csv); + std::filesystem::create_symlink("/dev/full", csv); + expect_error([&] { writer.write({packet}, 1000, 10); }); + require(std::filesystem::file_size(spool) > 16, "failed CSV write deleted the recovery spool"); + std::filesystem::remove(csv); + writer.write({packet}, 1000, 10); + require(!std::filesystem::exists(spool), "successful CSV export retained the spool"); + std::ifstream csv_input(csv); + const std::string csv_text((std::istreambuf_iterator(csv_input)), {}); + require(csv_text.find("1.000000000,3.000000000") != std::string::npos, + "CSV retry lost channel samples"); + writer.initialize_csv(2, false); + std::ofstream(spool, std::ios::binary | std::ios::app).put('x'); + expect_error([&] { writer.write({}, 1000, 10); }); + require(std::filesystem::exists(spool), "truncated packet header deleted the spool"); + writer.initialize_csv(2, false); + const std::array corrupt_packet{0, 2, 0xffffffffffffffffULL, 0}; + std::ofstream(spool, std::ios::binary | std::ios::app).write( + reinterpret_cast(corrupt_packet.data()), sizeof(corrupt_packet)); + expect_error([&] { writer.write({}, 1000, 10); }); + require(std::filesystem::exists(spool), "oversized packet count deleted the spool"); +} + +void test_asset_reload_preserves_data(const Workspace& workspace) { + const auto save = [&](const char* name, float sample) { + const auto path = workspace.path / name; + const auto bytes = ipc::serialize_raw_collection({.traces = { + {.frequency_hz = {1}, .s11 = {{sample, 0}}, .s21 = {{sample, 0}}}}}); + std::ofstream(path, std::ios::binary).write( + reinterpret_cast(bytes.data()), static_cast(bytes.size())); + return path.string(); + }; + const auto open = save("open.bin", 1); + const auto short_standard = save("short.bin", -1); + const auto load = save("load.bin", 0); + const auto nan = save("nan.bin", std::numeric_limits::quiet_NaN()); + preprocessing::S21ReferenceBundle s21_reference; + s21_reference.load(open); + expect_error([&] { s21_reference.load(nan); }); + require(s21_reference.resolve({}).samples.front().re == 1, "failed S21 reload lost previous reference"); + preprocessing::S21CalibrationBundle s21_calibration; + s21_calibration.load(open); + expect_error([&] { s21_calibration.load(nan); }); + require(s21_calibration.is_enabled(), "failed S21 reload disabled previous calibration"); + preprocessing::S11ReferenceBundle s11_reference; + s11_reference.load(open); + expect_error([&] { s11_reference.load(nan); }); + require(s11_reference.resolve({}).samples.front().re == 1, "failed S11 reload lost previous reference"); + preprocessing::S11CalibrationBundle s11_calibration; + s11_calibration.load(open, short_standard, load); + expect_error([&] { s11_calibration.load(open, open, load); }); + const std::array axis{1}; + require(s11_calibration.apply({}, axis, {{0.5, 0}}).front().re == 0.5F, + "failed OSL reload lost previous calibration"); +} + +void test_locator_packet_errors() { + int sockets[2]{}; + require(::socketpair(AF_UNIX, SOCK_STREAM, 0, sockets) == 0, "socketpair failed"); + std::atomic speed{0}; + std::atomic at_ns{0}; + locator::ClientSession client(sockets[0], "regression", 2, 1024); + client.start(speed, at_ns); + const auto send_json = [&](const std::string& json) { + std::vector bytes(8 + json.size()); + write_u32(bytes, 4, static_cast(json.size())); + std::memcpy(bytes.data() + 8, json.data(), json.size()); + const auto sent = ::send(sockets[1], bytes.data(), bytes.size(), MSG_NOSIGNAL); + if (sent != static_cast(bytes.size())) { + throw std::runtime_error(std::string("send test packet failed: ") + std::strerror(errno)); + } + }; + send_json("{\"vlc\":1e400}"); + send_json("{\"vlc\":1e308}"); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + require(at_ns.load() == 0 && speed.load() == 0, "invalid socket speed reached the processor"); + send_json("{\"vlc\":3.5}"); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (at_ns.load() == 0 && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + require(speed.load() == 3.5 && at_ns.load() > 0, "locator stopped after oversized JSON number"); + client.request_stop(); + client.join(); + ::close(sockets[1]); + require(client.has_exited(), "locator session did not finish both loops"); + locator::LocatorServerConfig config{.host = "127.0.0.1", .port = 0}; + locator::TcpServer server(config); + for (unsigned iteration = 0; iteration < 30; ++iteration) { server.start(); server.stop(); } +} + +void test_unread_tty_shutdown(const Workspace& workspace) { + TtyProtocolWriter writer((workspace.path / "tty").string(), 160'000); + const std::vector words(80'000, 42); + writer.enqueue_encoded_frames(words.data(), words.size() / 4U); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + const auto start = std::chrono::steady_clock::now(); + writer.shutdown(); + require(std::chrono::steady_clock::now() - start < std::chrono::seconds(1), + "unread tty blocked shutdown"); + writer.shutdown(); +} + +} // namespace + +int main(int argc, char** argv) { + try { + require(argc == 2, "Pass the simulator config fixture path"); + Workspace workspace; + test_wire_decoders(); + test_configuration(argv[1], workspace); + test_asset_reload_preserves_data(workspace); + test_ring(); + test_partial_driver_open(); + test_interruptible_wait(); + test_csv_failures(workspace); + test_locator_packet_errors(); + test_unread_tty_shutdown(workspace); + std::cout << "native regressions passed (10,000 decoder mutations; 20,000 concurrent ring frames)\n"; + } catch (const std::exception& error) { + std::cerr << "native regression failed: " << error.what() << '\n'; + return 1; + } +} diff --git a/data_acq_and_processing/tests/sar3d_fixture.cpp b/data_acq_and_processing/tests/sar3d_fixture.cpp new file mode 100644 index 0000000..6fdf010 --- /dev/null +++ b/data_acq_and_processing/tests/sar3d_fixture.cpp @@ -0,0 +1,90 @@ +#include "sar3d_processor.hpp" +#include "../processing/processors/src/sar3d_gpu.hpp" + +#include +#include + +using Json = nlohmann::json; +using namespace radar; + +namespace { +auto complex_samples(const Json& samples) -> std::vector> { + std::vector> values; + for (const auto& sample : samples) { + values.emplace_back(sample.at(0).get(), sample.at(1).get()); + } + return values; +} + +auto collection(const Json& source) -> ipc::PreprocessedCollection { + ipc::PreprocessedCollection result; + result.collection_id = source.at("id"); + result.monotonic_ns = source.at("timestamp"); + for (const auto& entry : source.at("traces")) { + ipc::SweepTraceBlock trace; + trace.combo = {entry.at("input"), entry.at("output")}; + trace.capture_start_ns = entry.at("start"); + trace.capture_end_ns = entry.at("end"); + trace.frequency_hz = entry.at("frequency").get>(); + for (const auto& sample : complex_samples(entry.at("s21"))) { + trace.s21.push_back({sample.real(), sample.imag()}); + } + result.traces.push_back(std::move(trace)); + } + return result; +} +} + +int main() { + try { + const Json input = Json::parse(std::cin); + Json output; + if (input.at("operation") == "processor") { + config::RunConfig config; + for (const auto& row : input.at("tx")) { + config.gpr.tx_geometry.push_back({row.at(0), row.at(1), row.at(2), row.at(3)}); + } + for (const auto& row : input.at("rx")) { + config.gpr.rx_geometry.push_back({row.at(0), row.at(1), row.at(2), row.at(3)}); + } + processing::ProcessingLiveConfig live; + live.gpr_start_freq_mhz = input.at("f_start").get() / 1e6F; + live.gpr_stop_freq_mhz = input.at("f_stop").get() / 1e6F; + live.gpr_min_depth_m = input.at("min_depth"); + live.gpr_max_depth_m = input.at("max_depth"); + live.gpr_speed_m_s = input.at("speed"); + live.gpr_look_angle_deg = input.at("pitch"); + live.sar3d_voxel_m = input.at("voxel"); + live.sar3d_bp_ground_depth_m = input.at("ground"); + live.sar3d_frame_radius = input.at("radius"); + live.sar3d_comp_range_power = input.at("power"); + live.sar3d_use_compensation = input.at("compensation"); + live.sar3d_normalize_each_frame = input.at("normalize"); + std::vector previous; + for (const auto& member : input.at("previous")) { + previous.push_back(collection(member)); + } + processing::Sar3dProcessor processor; + const auto result = processor.process_collection(config, collection(input.at("latest")), previous, live); + output["payloads"] = Json::array(); + for (const auto& payload : result.collection_payloads) { + output["payloads"].push_back({{"name", payload.processing_name}, + {"x", payload.image_x_axis}, {"y", payload.image_y_axis}, + {"image", payload.image_values}, {"table", payload.table_values}, + {"columns", payload.table_columns}}); + } + } else { + processing::sar3d::Engine engine(1.5); + const auto samples = engine.inverse_dft(complex_samples(input.at("bands")), + input.at("points"), input.at("first_bin"), input.at("size")); + output["ascans"] = Json::array(); + for (const auto& sample : samples) { + output["ascans"].push_back({sample.real(), sample.imag()}); + } + } + std::cout << output << '\n'; + } catch (const std::exception& error) { + std::cout << Json{{"error", error.what()}} << '\n'; + return 1; + } +} diff --git a/data_acq_and_processing/tests/sar3d_kernel_emulation.cpp b/data_acq_and_processing/tests/sar3d_kernel_emulation.cpp new file mode 100644 index 0000000..c8469a8 --- /dev/null +++ b/data_acq_and_processing/tests/sar3d_kernel_emulation.cpp @@ -0,0 +1,136 @@ +// Test-only execution of the exact CUDA kernel source, one logical thread at a time. +// Linked solely into sar3d_math_fixture; never used as a production backend. +#include "../processing/processors/src/sar3d_gpu.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#define __global__ +using std::isfinite; +using std::min; +struct Index { unsigned x; }; +Index blockIdx{0}, blockDim{1}, threadIdx{0}; +struct float2 { float x, y; }; +auto make_float2(float x, float y) -> float2 { return {x, y}; } +void sincospif(float x, float* sine, float* cosine) { + *sine = std::sin(x * 3.14159265358979323846F); + *cosine = std::cos(x * 3.14159265358979323846F); +} +auto __powf(float x, float y) -> float { return std::pow(x, y); } +auto __float_as_uint(float x) -> unsigned int { return std::bit_cast(x); } +auto __double2int_rn(double x) -> int { return static_cast(std::nearbyint(x)); } +void atomicMax(unsigned int* destination, unsigned int value) { *destination = std::max(*destination, value); } +#include "sar3d_kernels.inc" + +auto probe_ascan_upper_bound() -> float { + blockIdx.x = 0; + const double x[]{1073741824.0}, zero[]{0.0}; + const ::Contribution contribution{{}, {}, 1.0F, 1.0F}; + float volume = -1.0F; + ::backproject(nullptr, INT_MAX, &contribution, 1, x, zero, zero, 1, 1, 1, + 1.0, 0.0, 0.0, 0.0, 1.0, 1.0F, 0, 0.0F, 0.0F, &volume); + return volume; +} + +namespace { +template +void each(int count, Operation operation) { + for (blockIdx.x = 0; blockIdx.x < static_cast(count); ++blockIdx.x) { operation(); } +} + +auto float_pairs(std::span> values) -> std::vector { + std::vector pairs; + pairs.reserve(values.size()); + for (const auto value : values) { pairs.push_back({value.real(), value.imag()}); } + return pairs; +} +} + +namespace radar::processing::sar3d { +namespace { +#include "sar3d_helpers.inc" +} + +struct Engine::Impl { std::vector taps; }; +Engine::Engine(double sigma) : impl_(std::make_unique()) { impl_->taps = gaussian_taps(sigma); } +Engine::~Engine() = default; + +auto Engine::inverse_dft(std::span> bands, std::size_t points, + std::size_t first_bin, std::size_t size) -> std::vector> { + if (points == 0U || bands.empty() || bands.size() % points != 0U || !std::has_single_bit(size) || + size > kMaxIndexedItems || first_bin >= size || points > size - first_bin) { + throw std::runtime_error("sar3d invalid inverse FFT dimensions"); + } + const auto count = checked_product(bands.size() / points, size, kMaxIndexedItems, "A-scan stack"); + const auto input = float_pairs(bands); + std::vector output(count); + each(static_cast(count), [&] { + ::inverse_dft(input.data(), points, first_bin, size, count, output.data()); + }); + std::vector> result; + result.reserve(count); + for (const auto value : output) { result.emplace_back(value.x, value.y); } + return result; +} + +auto Engine::project(const VolumeRequest& request) -> Projections { + const int nx = request.x.size(), ny = request.y.size(), nz = request.z.size(); + const int plane = checked_product(nx, nz, kMaxIndexedItems, "grid plane"); + const int voxels = checked_product(plane, ny, kMaxIndexedItems, "grid"); + std::vector<::Contribution> contributions; + for (const auto& term : request.contributions) { + contributions.push_back(std::bit_cast<::Contribution>(term)); + } + const auto ascans = float_pairs(request.ascans); + std::vector volume(voxels), scratch(voxels), levels(ny); + std::vector indices(ny); + each(voxels, [&] { + ::backproject(ascans.data(), request.samples_per_ascan, contributions.data(), contributions.size(), + request.x.data(), request.y.data(), request.z.data(), nx, nz, voxels, + std::cos(request.pitch_rad), std::sin(request.pitch_rad), request.ground_depth_m, + request.min_depth_m, request.max_depth_m, request.samples_per_metre, + request.use_compensation, request.comp_range_power, request.comp_weight_max, volume.data()); + }); + each(ny, [&] { ::peak_planes(volume.data(), plane, ny, levels.data(), indices.data()); }); + Projections raw; + choose_peak(raw, request, levels); + const int radius = impl_->taps.size() / 2; + each(voxels, [&] { + ::smooth(volume.data(), scratch.data(), impl_->taps.data(), radius, nx, nz, voxels, 1); + }); + each(voxels, [&] { + ::smooth(scratch.data(), volume.data(), impl_->taps.data(), radius, nx, nz, voxels, 0); + }); + each(ny, [&] { ::peak_planes(volume.data(), plane, ny, levels.data(), indices.data()); }); + Projections result; + result.raw_level = raw.level; + choose_peak(result, request, levels, indices); + result.front.resize(ny * nx); + result.side.resize(ny * nz); + result.top.resize(plane); + const auto& view = request.oblique; + std::vector oblique(checked_product(view.nu, view.nv, kMaxIndexedItems, "oblique image")); + each(result.front.size(), [&] { + ::max_project(volume.data(), result.front.data(), result.front.size(), nx, plane, 1, nz, nx); + }); + each(result.side.size(), [&] { + ::max_project(volume.data(), result.side.data(), result.side.size(), nz, plane, nx, nx, 1); + }); + each(result.top.size(), [&] { + ::max_project(volume.data(), result.top.data(), result.top.size(), nx, nx, 1, ny, plane); + }); + each(voxels, [&] { + ::project_oblique(volume.data(), request.x.data(), request.y.data(), request.z.data(), nx, nz, voxels, + view.u[0], view.u[1], view.u[2], view.v[0], view.v[1], view.v[2], view.u0, view.v0, + 1 / view.pixel_m, view.nu, view.nv, oblique.data()); + }); + result.oblique.reserve(oblique.size()); + for (const auto value : oblique) { result.oblique.push_back(std::bit_cast(value)); } + return result; +} +} diff --git a/data_acq_and_processing/tests/sar3d_kernel_regressions.cpp b/data_acq_and_processing/tests/sar3d_kernel_regressions.cpp new file mode 100644 index 0000000..f77c5d6 --- /dev/null +++ b/data_acq_and_processing/tests/sar3d_kernel_regressions.cpp @@ -0,0 +1,85 @@ +// Kernel arithmetic checks linked only with the host emulation fixture. +#include "../processing/processors/src/sar3d_gpu.hpp" + +#include +#include +#include +#include +#include + +extern "C" void peak_planes(const float*, int, int, float*, unsigned int*); +struct Index { unsigned x; }; +extern Index blockIdx; +auto probe_ascan_upper_bound() -> float; + +namespace { +void rejects_unsigned_padding() { + blockIdx.x = 0x80000000U; + peak_planes(nullptr, 1, 1, nullptr, nullptr); +} + +void rejects_rounded_sample_bound() { + if (probe_ascan_upper_bound() != 0.0F) { + throw std::runtime_error("out-of-range float A-scan position was accepted"); + } +} + +void rejects_nan_peak() { + blockIdx.x = 0; + const float volume[]{std::numeric_limits::quiet_NaN(), 0.0F}; + float level = -1.0F; + unsigned int index = 0U; + peak_planes(volume, 2, 1, &level, &index); + if (std::isfinite(level)) { throw std::runtime_error("peak kernel silently ignored NaN"); } +} + +void keeps_first_joint_peak() { + blockIdx.x = 0; + const float volume[]{0, 1, 0, 1, 0, 0}; + float level = 0; + unsigned int index = 0U; + peak_planes(volume, 6, 1, &level, &index); + if (level != 1.0F || index != 1U) { throw std::runtime_error("joint peak tie chose a nonexistent voxel"); } +} + +void rejects_finite_coherent_overflow() { + using namespace radar::processing::sar3d; + Engine engine(1.5); + const std::vector x{-1, 1}, y{-.5, .5}, z{2, 3}; + std::vector terms(3); + for (auto& term : terms) { term.weight = 1; term.inv_reference_ranges = 1; } + std::vector> scans(3 * 64); + for (int i = 0; i < 64; ++i) { + scans[i] = {1e38F, 0}; + scans[64 + i] = {-1e38F, 0}; + scans[128 + i] = {1, 0}; + } + const VolumeRequest request{ + .x = x, .y = y, .z = z, .min_depth_m = 1, .max_depth_m = 4, + .contributions = terms, .ascans = scans, .samples_per_ascan = 64, .samples_per_metre = 1, + .use_compensation = true, .comp_range_power = 1, .comp_weight_max = 5, .norm_edge_margin_m = .3, + .oblique = {.u = {1, 0, 0}, .v = {0, 0, 1}, .u0 = -1, .v0 = 2, .pixel_m = 1, .nu = 3, .nv = 2}, + }; + try { + (void)engine.project(request); + } catch (const std::runtime_error& error) { + if (std::string_view(error.what()).find("non-finite") == std::string_view::npos) { throw; } + return; + } + throw std::runtime_error("finite coherent overflow was silently accepted as no signal"); +} +} + +int main() { + try { + rejects_unsigned_padding(); + rejects_rounded_sample_bound(); + rejects_nan_peak(); + keeps_first_joint_peak(); + rejects_finite_coherent_overflow(); + std::cout << "SAR CUDA kernel arithmetic regressions passed\n"; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/data_acq_and_processing/tests/test_sar3d_oracle.py b/data_acq_and_processing/tests/test_sar3d_oracle.py new file mode 100644 index 0000000..0a73fed --- /dev/null +++ b/data_acq_and_processing/tests/test_sar3d_oracle.py @@ -0,0 +1,297 @@ +"""Compare the native SAR port with notebook cells 0/4, without dataset or GPU fixtures.""" +from __future__ import annotations + +import json +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + +import numpy as np + +ROOT = Path(__file__).resolve().parents[2] +FIXTURE = ROOT / "build/native_tests/sar3d_math_fixture" +CUDA_FIXTURE = ROOT / "build/native_tests/sar3d_fixture" +C = 3e8 + + +def smooth(volume): + # scipy gaussian_filter(sigma=(0,1.5,1.5), mode='reflect', truncate=4). + taps = np.exp(-.5 * (np.arange(-6, 7) / 1.5) ** 2) + taps /= taps.sum() + for axis in (1, 2): + volume = np.apply_along_axis(lambda row: np.convolve( + np.pad(row, 6, mode="symmetric"), taps, mode="valid"), axis, volume) + return volume + + +def native(request, executable=None): + run = subprocess.run([str(executable or FIXTURE)], input=json.dumps(request), text=True, + capture_output=True, timeout=30, check=False) + if not run.stdout: + raise AssertionError(run.stderr) + return json.loads(run.stdout) + + +def fixture(base=2**64 - 3_000_000_000): + request = dict(operation="processor", tx=[[0, -.4, 0, 0], [1, .1, .2, 0], [2, .45, .1, .05]], + rx=[[0, -.2, .1, 0], [1, .3, .35, .02]], f_start=1e9, f_stop=4e9, + min_depth=1., max_depth=3., speed=.4, pitch=13., ground=.3, + voxel=.4, radius=1, power=1.5, compensation=True, normalize=True) + members = [] + frequency = np.linspace(1e9, 4e9, 64, dtype=np.float32) + for frame, offset in enumerate([0, 180_000_000, 510_000_000]): + traces = [] + for tx in range(3): + for rx in range(2): + # Two transmitters interleave in one real capture window; the + # third has a longer independent segment, matching meta timing. + start = base + offset + (0 if tx < 2 else 70_000_000) + end = base + offset + (60_000_000 if tx < 2 else 165_000_001) + samples = ((1 + .1 * rx) * np.exp(-2j * np.pi * frequency * + (4.8 + .08 * tx - .03 * frame) / C) + + .35 * np.exp(-2j * np.pi * frequency * (5.6 - .05 * rx) / C)).astype(np.complex64) + traces.append(dict(input=rx, output=tx, start=start, end=end, + frequency=frequency.tolist(), s21=np.column_stack((samples.real, samples.imag)).tolist())) + members.append(dict(id=frame + 1, timestamp=base + offset, traces=traces)) + request.update(previous=members[:-1], latest=members[-1]) + return request + + +def oracle(request): + """Notebook ground/frequency-domain mode, with actual per-trace capture windows.""" + pitch = np.deg2rad(float(np.float32(request["pitch"]))) + cos_p, sin_p = np.cos(pitch), np.sin(pitch) + speed, voxel, ground = (float(np.float32(request[key])) for key in ("speed", "voxel", "ground")) + axis = lambda lo, hi: np.linspace(lo, hi, max(round((hi - lo) / voxel) + 1, 2)) + x, y = axis(-2, 2), axis(-.5, 2) + limits = [(depth + (height - ground) * sin_p) / cos_p + for height in (y[0], y[-1]) for depth in (request["min_depth"], request["max_depth"])] + z = axis(min(limits), max(limits)) + xx, zz = np.meshgrid(x, z) + frames, centres = [], [] + for member in request["previous"] + [request["latest"]]: + traces = member["traces"] + first, last = min(t["start"] for t in traces), max(t["end"] for t in traces) + centres.append((first + last) / 2 if first < 2**53 else (first, last)) + ascans = [] + for trace in traces: + frequency = np.asarray(trace["frequency"], dtype=np.float32) + samples = np.asarray(trace["s21"]) + samples = samples[:, 0] + 1j * samples[:, 1] + mask = (frequency >= request["f_start"]) & (frequency <= request["f_stop"]) + points = np.flatnonzero(mask) + stages = sorted({t["output"] for t in traces + if (t["start"], t["end"]) == (trace["start"], trace["end"])}) + # Relative integer subtraction retains precision at uint64 limits. + t = ((trace["start"] - first) - .5 * (last - first)) * 1e-9 + t += (len(stages) * points + stages.index(trace["output"])) * ( + (trace["end"] - trace["start"]) * 1e-9 / len(stages) / len(frequency)) + band = samples[mask] * np.exp(-4j * np.pi * speed * cos_p * frequency[mask] * t / C) + df = float(np.median(np.diff(frequency[mask].astype(float)))) + k0, n = round(float(frequency[mask][0]) / df), len(band) + size = 2 ** (2 * (k0 + n - 1) - 1).bit_length() * 8 + spectrum = np.zeros(size, dtype=complex) + spectrum[k0:k0+n] = band * np.hanning(n) + h = np.fft.ifft(spectrum) + dt = 1 / (size * df) + if request["normalize"]: + depth = np.arange(size) * dt * C / 2 + gate = (depth >= request["min_depth"]) & (depth <= request["max_depth"]) + scale = np.percentile(np.abs(h)[gate], 50) + h /= (scale if np.isfinite(scale) and scale > 1e-15 else 1) + 1e-15 + ascans.append(h) + frames.append(ascans) + # All members use identical grids. Native centre trails latest by radius. + centre_index = max(len(frames) - 1 - request["radius"], 0) + def relative(a, b): + if isinstance(a, tuple): + return .5e-9 * ((a[0] - b[0]) + (a[1] - b[1])) + return 1e-9 * (a - b) + offsets = np.array([relative(t, centres[centre_index]) for t in centres]) + farthest = np.max(np.abs(offsets)) + weights = np.ones(len(frames)) if farthest <= 1e-12 else .8 + .1 * (1 + np.cos(np.pi * abs(offsets) / farthest)) + acc = np.zeros((len(y), len(z), len(x)), dtype=complex) + count = np.zeros(acc.shape) + tx = np.asarray(request["tx"], dtype=np.float32)[:, 1:].astype(float) + rx = np.asarray(request["rx"], dtype=np.float32)[:, 1:].astype(float) + for scans, offset, weight in zip(frames, offsets, weights): + shift = np.array([0, speed * offset * sin_p, speed * offset * cos_p]) + for iy, height in enumerate(y): + yr, zr = (height - ground) * cos_p + zz * sin_p, -(height - ground) * sin_p + zz * cos_p + depth_ok = (zr >= request["min_depth"]) & (zr <= request["max_depth"]) + for it, antenna_t in enumerate(tx): + for ir, antenna_r in enumerate(rx): + def distance(antenna): + return np.sqrt((xx-antenna[0])**2 + (yr-antenna[1])**2 + (zr-antenna[2])**2) + rt, rr = distance(antenna_t + shift), distance(antenna_r + shift) + position = (rt + rr) / (C * dt) + h = scans[it * len(rx) + ir] + valid = depth_ok & (position <= len(h) - 1) + sample = np.interp(position, np.arange(len(h)), h) + if request["compensation"]: + point = np.array([.5 * (antenna_t[0] + antenna_r[0]), + (-ground + 5 * sin_p) / cos_p, 5]) + ref = 1 / (np.linalg.norm(point-antenna_t) * np.linalg.norm(point-antenna_r) + 1e-12) + 1e-30 + ratio = (1 / (rt * rr + 1e-12) + 1e-30) / ref + sample *= np.clip(1 / (ratio ** request["power"] + 1e-12), 0, 5) + acc[iy] += weight * np.where(valid, sample, 0) + count[iy] += weight * valid + eligible = (y >= y[0]+.3) & (y <= y[-1]-.3) + raw_volume = np.abs(acc / (count + 1e-12)) + raw_level = raw_volume[eligible].max() + 1e-12 + volume = smooth(raw_volume / raw_level) + smooth_level = volume[eligible].max() + 1e-12 + level = raw_level * smooth_level + peak = np.unravel_index(np.argmax(np.where(eligible[:, None, None], volume, -1)), volume.shape) + volume /= smooth_level + iy, iz, ix = peak + eye = np.array([1.7, -2.2, .85]) + forward = -eye / np.linalg.norm(eye) + right = np.cross(forward, [0, 0, 1]) + right /= np.linalg.norm(right) + up = np.cross(right, forward) + u, v = right[[0, 2, 1]], up[[0, 2, 1]] + points = np.array(np.meshgrid(x, y, z, indexing="ij")).reshape(3, -1).T + us, vs = points @ u, points @ v + u0, v0 = us.min(), vs.min() + nu, nv = round((us.max() - u0) / voxel) + 1, round((vs.max() - v0) / voxel) + 1 + oblique = np.zeros((nv, nu)) + pu, pv = np.rint((us-u0)/voxel).astype(int), np.rint((vs-v0)/voxel).astype(int) + valid = (pu >= 0) & (pu < nu) & (pv >= 0) & (pv < nv) + np.maximum.at(oblique, (pv[valid], pu[valid]), volume.transpose(2, 0, 1).ravel()[valid]) + return dict(x=x, y=y, z=z, volume=volume, level=level, peak=[x[ix], y[iy], z[iz]], + peak_index=peak, oblique=oblique, u=u0 + np.arange(nu)*voxel, v=v0 + np.arange(nv)*voxel) + + +class Sar3dOracleTest(unittest.TestCase): + def test_cuda_branch_cpp_syntax(self): + # Hardware-free name/type structure check. Variadic API placeholders do + # not validate the NVIDIA ABI or replace testing NVRTC on an actual GPU. + source_path = ROOT / "data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp" + source = source_path.read_text() + with tempfile.TemporaryDirectory(prefix="radar_cuda_syntax_") as directory: + path = Path(directory) + functions = set(re.findall(r"\b(cu[A-Z]\w*)\s*\(", source)) + constants = set(re.findall(r"\bCU_DEVICE_ATTRIBUTE_\w+", source)) + (path / "cuda.h").write_text( + "#pragma once\n#include \nenum CUresult {CUDA_SUCCESS=0};\n" + "using CUdevice=int;using CUdeviceptr=std::uintptr_t;using CUcontext=struct Context*;" + "using CUmodule=struct Module*;using CUfunction=struct Function*;\n" + + "".join(f"constexpr int {name}=0;\n" for name in constants) + + "".join(f"inline CUresult {name}(...) {{return CUDA_SUCCESS;}}\n" for name in functions)) + functions = set(re.findall(r"\b(nvrtc[A-Z]\w*)\s*\(", source)) | { + "nvrtcGetCUBINSize", "nvrtcGetPTXSize", "nvrtcGetCUBIN", "nvrtcGetPTX"} + (path / "nvrtc.h").write_text( + "#pragma once\nenum nvrtcResult {NVRTC_SUCCESS=0};using nvrtcProgram=struct Program*;\n" + 'inline const char* nvrtcGetErrorString(...) {return "stub";}\n' + + "".join(f"inline nvrtcResult {name}(...) {{return NVRTC_SUCCESS;}}\n" + for name in functions if name != "nvrtcGetErrorString")) + result = subprocess.run(["g++", "-std=c++20", "-pthread", "-fsyntax-only", "-Wall", "-Wextra", + "-Wpedantic", "-Werror", "-DSAR3D_WITH_CUDA", f"-I{path}", str(source_path)], + text=True, capture_output=True, timeout=30, check=False) + self.assertEqual(result.returncode, 0, result.stderr) + + def ready(self, result): + payloads = {p["name"]: p for p in result["payloads"]} + status = payloads["sar3d_status"]["table"] + self.assertEqual(payloads["sar3d_status"]["columns"], len(status)) + self.assertEqual(status[0], 0) + return payloads + + def test_inverse_fft_matches_numpy(self): + random = np.random.default_rng(209) + bands = random.normal(size=(2, 48)) + 1j * random.normal(size=(2, 48)) + result = native(dict(operation="ifft", bands=np.column_stack((bands.ravel().real, bands.ravel().imag)).tolist(), + points=48, first_bin=17, size=512)) + actual = np.asarray(result["ascans"]) + spectrum = np.zeros((2, 512), dtype=complex) + spectrum[:, 17:65] = bands.astype(np.complex64) + expected = np.fft.ifft(spectrum).ravel() + np.testing.assert_allclose(actual[:, 0] + 1j * actual[:, 1], expected, rtol=3e-4, atol=2e-7) + + def test_production_backend_reports_cuda_warning(self): + result = native(fixture(), CUDA_FIXTURE) + payloads = {p["name"]: p for p in result["payloads"]} + status = payloads["sar3d_status"]["table"] + if status[0] == 3: + self.assertIn("CUDA", bytes(map(int, status[1:])).decode("utf-8")) + self.assertFalse(any(item["image"] for item in payloads.values())) + else: + # On a CUDA-equipped test host, also run the real compiled backend. + self.ready(result) + + def test_motion_volume_matches_notebook(self): + for compensation, normalize in ((True, True), (True, False), (False, True), (False, False)): + with self.subTest(compensation=compensation, normalize=normalize): + request = fixture() + request.update(compensation=compensation, normalize=normalize) + payloads = self.ready(native(request)) + expected = oracle(request) + for name, image, x, y in ( + ("sar3d_front", expected["volume"].max(axis=1), expected["x"], expected["y"]), + ("sar3d_side", expected["volume"].max(axis=2), expected["z"], expected["y"]), + ("sar3d_top", expected["volume"].max(axis=0), expected["x"], expected["z"]), + ("sar3d_oblique", expected["oblique"], expected["u"], expected["v"])): + np.testing.assert_allclose(payloads[name]["x"], x, rtol=2e-6, atol=1e-6) + np.testing.assert_allclose(payloads[name]["y"], y, rtol=2e-6, atol=1e-6) + np.testing.assert_allclose(np.asarray(payloads[name]["image"]).reshape(image.shape), image, + rtol=2e-3, atol=3e-4) + np.testing.assert_allclose(payloads["sar3d_peak"]["table"][1:], expected["peak"], atol=2e-6) + np.testing.assert_allclose(payloads["sar3d_peak"]["table"][0], expected["level"], rtol=2e-3) + + def test_errors_are_published(self): + for malformed in ("empty", "capture", "frequency", "axis", "angle"): + with self.subTest(malformed=malformed): + request = fixture() + if malformed == "empty": + request["latest"]["traces"] = [] + elif malformed == "capture": + request["latest"]["traces"][0]["end"] = request["latest"]["traces"][0]["start"] + elif malformed == "frequency": + request["latest"]["traces"][0]["frequency"].reverse() + elif malformed == "axis": + request["voxel"] = 1e-8 + else: + request["pitch"] = 90 + result = native(request) + payloads = {p["name"]: p for p in result["payloads"]} + status = payloads["sar3d_status"]["table"] + self.assertEqual(payloads["sar3d_status"]["columns"], len(status)) + self.assertEqual(status[0], 2) + self.assertTrue(bytes(map(int, status[1:])).decode()) + + def test_empty_signal_is_reported(self): + request = fixture() + request.update(previous=[], radius=0) + for trace in request["latest"]["traces"]: + trace["s21"] = [[0, 0]] * len(trace["frequency"]) + payloads = {p["name"]: p for p in native(request)["payloads"]} + self.assertEqual(payloads["sar3d_status"]["table"][0], 1) + self.assertNotIn("sar3d_peak", payloads) + for name in ("sar3d_front", "sar3d_side", "sar3d_top", "sar3d_oblique"): + self.assertTrue(np.all(np.asarray(payloads[name]["image"]) == 0)) + + def test_invalid_fft_dimensions_are_rejected(self): + for points, first_bin, size in ((0, 0, 8), (1, 0, 7), (1, 8, 8), (2, 7, 8), (1, 0, 2**31)): + with self.subTest(points=points, first_bin=first_bin, size=size): + result = native(dict(operation="ifft", bands=[[1, 0]], + points=points, first_bin=first_bin, size=size)) + self.assertIn("error", result) + + def test_weak_signal_stays_finite(self): + request = fixture() + request.update(previous=[], radius=0, normalize=False) + for trace in request["latest"]["traces"]: + trace["s21"] = (np.asarray(trace["s21"]) * 1e-38).tolist() + payloads = self.ready(native(request)) + for name in ("sar3d_front", "sar3d_side", "sar3d_top", "sar3d_oblique"): + values = np.asarray(payloads[name]["image"]) + self.assertTrue(np.isfinite(values).all()) + self.assertGreater(values.max(), 0) + self.assertLess(values.max(), 1e-10) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/pack-usb.sh b/deploy/pack-usb.sh index dafc181..4d613c0 100755 --- a/deploy/pack-usb.sh +++ b/deploy/pack-usb.sh @@ -9,9 +9,9 @@ set -euo pipefail PROJECT_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" -# What runs on the Pi or Jetson; device_firmware/ and docs/ stay on this machine. +# What runs on the Pi or Jetson; device_firmware/ stays on this machine. HOST_PATHS=(data_acq_and_processing deploy python_app run_config_examples Makefile - requirements.txt run_config.json start.sh locator_test_client.py) + "requirements*.txt" run_config.json start.sh locator_test_client.py) drive="${1:-}" if [[ -z "${drive}" ]]; then diff --git a/deploy/update.sh b/deploy/update.sh index 92fabbc..a555c49 100755 --- a/deploy/update.sh +++ b/deploy/update.sh @@ -35,6 +35,7 @@ while (( $# > 0 )); do *) archive="$1"; shift ;; esac done +if [[ -n "${archive}" ]]; then archive="$(realpath -- "${archive}")"; fi cd "${HOME}" # never stand in the directory that is about to be renamed # --- check everything before touching the project ------------------------------ @@ -149,8 +150,8 @@ if [[ -n "${backup}" ]]; then # writes new files instead of rewriting old ones. data=() for entry in "${backup}"/python_app/data*; do data+=("python_app/${entry##*/}"); done - for dir in "${backup}"/*/; do - name="$(basename -- "${dir}")" + for entry in "${backup}"/*; do + name="${entry##*/}" case "${name}" in build|graphify-out|device_firmware|docs) continue ;; esac [[ -e "${target}/${name}" ]] || data+=("${name}") done @@ -168,7 +169,9 @@ fi venv="${target}/.venv" if [[ -z "${python}" && -n "${backup}" && -x "${backup}/.venv/bin/python" ]]; then say "carrying the venv over" - cp -al -- "${backup}/.venv" "${venv}" + # pip and generated caches may rewrite existing files: the rollback copy + # must own its files independently of the environment being upgraded. + cp -a -- "${backup}/.venv" "${venv}" "${venv}/bin/python" -m pip install --quiet -r "${target}/requirements.txt" else python="${python:-$(command -v python3)}" @@ -178,7 +181,7 @@ else "${venv}/bin/python" -m pip install -r "${target}/requirements.txt" fi say "building the C++ part" -make -C "${target}" -j"$(nproc)" all +make -C "${target}" -j"$(nproc)" all kamil_adc_collector echo "${archive##*/}" > "${target}/RELEASE" trap - ERR diff --git a/device_firmware/cart_firmware/src/main.cpp b/device_firmware/cart_firmware/src/main.cpp index 6af2c78..4bba73d 100644 --- a/device_firmware/cart_firmware/src/main.cpp +++ b/device_firmware/cart_firmware/src/main.cpp @@ -8,15 +8,7 @@ static const int X_FWD = 650, X_MID = 2014, X_BACK = 3378; // PA0 (плечи static const int Y_RIGHT = 479, Y_MID = 1981, Y_LEFT = 3586; // PA1 static const int DEADZONE = 15; // отсечка дребезга вокруг нейтрали -// ---- SBUS-значения старого пульта ---- static const uint16_t SBUS_MID = 1024; -static const uint16_t CH2_MIN = 433, CH2_MAX = 1643; // назад..вперёд -static const uint16_t CH4_MIN = 446, CH4_MAX = 1654; // влево..вправо - -// ---- expo: 0 = линейно, 1 = максимально мягкая нейтраль ---- -static const float EXPO_K = 0.0f; -// ---- общий масштаб выхода: 1.0 = диапазон старого пульта ---- -static const float RANGE_SCALE = 0.75f; // симметричные плечи: вперёд и назад дают одинаковый максимум static const uint16_t CH2_SPAN = 591; // min(1643-1024, 1024-433) @@ -75,15 +67,6 @@ static float axisNorm(int adc, int lowEnd, int mid, int highEnd) { return constrain(x, -1.0f, 1.0f); } -// expo-кривая: гасит чувствительность у нейтрали, сохраняет края -static float expo(float x) { - return EXPO_K * x * x * x + (1.0f - EXPO_K) * x; -} - -static uint16_t toSbus(float x, uint16_t span) { - return (uint16_t)(SBUS_MID + lroundf(x * span)); -} - // стик [-1..1] -> SBUS-значение канала: ниже MOVE_START — нейтраль, выше — // линейно от порога срабатывания приёмника (db) до POWER_FRAC*span на полном стике static uint16_t driveToSbus(float x, uint16_t span, uint16_t db, @@ -129,11 +112,11 @@ void setup() { } void loop() { - static uint32_t next = 0; + static uint32_t last = millis() - FRAME_PERIOD_MS; uint32_t now = millis(); - if (now < next) + if (uint32_t(now - last) < FRAME_PERIOD_MS) return; - next = now + FRAME_PERIOD_MS; + last = now; int adcX = readAvg(PIN_X); int adcY = readAvg(PIN_Y); diff --git a/docs/S2VNA Operating and Programing Manual ver 26.1 2.pdf b/docs/S2VNA Operating and Programing Manual ver 26.1 2.pdf deleted file mode 100644 index 0b2f724..0000000 Binary files a/docs/S2VNA Operating and Programing Manual ver 26.1 2.pdf and /dev/null differ diff --git a/docs/Руководство_по_эксплуатации_SNVNA_часть_2_ver_26.1.pdf b/docs/Руководство_по_эксплуатации_SNVNA_часть_2_ver_26.1.pdf deleted file mode 100644 index 9b32032..0000000 Binary files a/docs/Руководство_по_эксплуатации_SNVNA_часть_2_ver_26.1.pdf and /dev/null differ diff --git a/locator_test_client.py b/locator_test_client.py index ad284c5..dc1b89e 100644 --- a/locator_test_client.py +++ b/locator_test_client.py @@ -3,8 +3,8 @@ import random import socket import struct import threading -import time -from typing import Any, Dict, Tuple +from contextlib import suppress +from typing import Any HOST = "127.0.0.1" PORT = 8888 @@ -18,7 +18,7 @@ MAX_PAYLOAD_BYTES = 64 * 1024 HEADER_STRUCT = struct.Struct(" bytes: +def encode_packet(payload: dict[str, Any], device_id: int) -> bytes: """Encode a JSON payload using the protocol binary header.""" payload_bytes = json.dumps( payload, @@ -28,12 +28,17 @@ def encode_packet(payload: Dict[str, Any], device_id: int) -> bytes: return HEADER_STRUCT.pack(device_id, len(payload_bytes)) + payload_bytes -def recv_exactly(sock: socket.socket, size: int) -> bytes: +def recv_exactly(sock: socket.socket, size: int, stop_event: threading.Event | None = None) -> bytes: """Receive an exact number of bytes from the socket.""" chunks = bytearray() while len(chunks) < size: - chunk = sock.recv(size - len(chunks)) + if stop_event is not None and stop_event.is_set(): + raise ConnectionError("Client stopped.") + try: + chunk = sock.recv(size - len(chunks)) + except socket.timeout: + continue # Preserve a partial header/payload across timeout polls. if not chunk: raise ConnectionError("Connection closed by peer.") chunks.extend(chunk) @@ -41,9 +46,9 @@ def recv_exactly(sock: socket.socket, size: int) -> bytes: return bytes(chunks) -def read_packet(sock: socket.socket) -> Tuple[int, Any]: +def read_packet(sock: socket.socket, stop_event: threading.Event | None = None) -> tuple[int, Any]: """Read and decode a single packet from the server.""" - header_bytes = recv_exactly(sock, HEADER_STRUCT.size) + header_bytes = recv_exactly(sock, HEADER_STRUCT.size, stop_event) device_id, payload_length = HEADER_STRUCT.unpack(header_bytes) if payload_length > MAX_PAYLOAD_BYTES: @@ -52,10 +57,11 @@ def read_packet(sock: socket.socket) -> Tuple[int, Any]: % (payload_length, MAX_PAYLOAD_BYTES) ) - payload_bytes = recv_exactly(sock, payload_length) + payload_bytes = recv_exactly(sock, payload_length, stop_event) payload = json.loads(payload_bytes.decode("utf-8")) return device_id, payload + def send_vlc_message(sock: socket.socket, device_id: int, vlc: float) -> None: """Send one test client payload to the server.""" payload = {"vlc": vlc} @@ -67,10 +73,8 @@ def receive_loop(sock: socket.socket, stop_event: threading.Event) -> None: """Print all packets received from the locator server.""" while not stop_event.is_set(): try: - device_id, payload = read_packet(sock) - except socket.timeout: - continue - except (ConnectionError, OSError, ValueError, json.JSONDecodeError) as error: + device_id, payload = read_packet(sock, stop_event) + except (OSError, ValueError) as error: if not stop_event.is_set(): print("Receiver stopped:", error) stop_event.set() @@ -89,12 +93,10 @@ def send_loop(sock: socket.socket, stop_event: threading.Event) -> None: ) if SEND_INTERVAL_SECONDS <= 0: - while not stop_event.is_set(): - time.sleep(0.1) + stop_event.wait() return - while not stop_event.is_set(): - time.sleep(SEND_INTERVAL_SECONDS) + while not stop_event.wait(SEND_INTERVAL_SECONDS): send_vlc_message( sock, CLIENT_DEVICE_ID, @@ -124,8 +126,13 @@ def main() -> None: send_loop(sock, stop_event) except KeyboardInterrupt: print("Stopping client.") + except OSError as error: + print("Sender stopped:", error) finally: stop_event.set() + with suppress(OSError): + sock.shutdown(socket.SHUT_RDWR) + receiver.join(RECV_TIMEOUT_SECONDS + 1.0) if __name__ == "__main__": diff --git a/mimo_calibration.py b/mimo_calibration.py new file mode 100644 index 0000000..b244039 --- /dev/null +++ b/mimo_calibration.py @@ -0,0 +1,544 @@ +#!/usr/bin/env python3 +"""Фазовая калибровка MIMO-радара по точечному отражателю (шару) и применение её к данным. + +Расчёт тот же, что в MIMO_calibration.ipynb, без графиков и диагностики. Формат файлов +калибровки (cal_*.npy, cal_meta.json) и папки postprocessed — тот же. + +Запуск целиком: python mimo_calibration.py (работает по параметрам из блока ниже) +Из другого кода: + import mimo_calibration as mc + cal = mc.compute_calibration(cal_sets, empty_scene_path, mc.Settings(...)) + mc.save_calibration(cal, cal_dir) + cal = mc.load_calibration(cal_dir) + s21_corr = mc.apply_calibration(s21, freq, cal) # массив (TX, RX, Nf) в памяти + mc.apply_to_measurement(meas_dir, cal) # preprocessed -> postprocessed +""" +import datetime +import json +import re +import shutil +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +from scipy.optimize import minimize + +# ╔══════════════════════════════════════════════════════════════════════════════════════╗ +# ║ ЗАДАВАЕМЫЕ ПАРАМЕТРЫ ║ +# ╚══════════════════════════════════════════════════════════════════════════════════════╝ + +# --- что делать ------------------------------------------------------------------------- +RUN_CALIBRATION = True # True -> посчитать калибровку по CAL_SETS и записать в CAL_DIR; + # False -> взять готовую калибровку из CAL_DIR +APPLY_TO_DATA = False # True -> записать исправленные данные в <измерение>/postprocessed + +# --- калибровочные измерения ------------------------------------------------------------ +# Каждая запись — одно положение отражателя. snapshots: список префиксов снимков ('0003') +# или None (все снимки папки). Несколько снимков одной записи усредняются. +# target_scene_m = (X, Y, Z) в осях сцены, метры; для шара — координаты его ЦЕНТРА, +# измеренные физически (а не взятые с некалиброванного изображения): +# X — поперёк, вправо, от начала координат антенн (config_profile, x = 0); +# Y — высота центра отражателя над землёй; +# Z — расстояние вдоль земли от точки под началом координат антенн до отражателя. +CAL_ROOT = Path('/Users/ivan_root/Downloads/Telegram_dwnld/2026-09-31 balls calib/' + 'librevna_multi_0.5-6GHz_1031pts_-3dBm_big_ball_h_0cm_z_5m_x_0cm/preprocessed') +CAL_SETS = [ + dict(path=CAL_ROOT, snapshots=['0000'], target_scene_m=(0.0, 0.16, 5.20)), # Здесь нужно указывать координаты, измернные + dict(path=CAL_ROOT, snapshots=['0001'], target_scene_m=(0.0, 0.16, 5.20)), # в системе координат земли (x,y,z) + dict(path=CAL_ROOT, snapshots=['0002'], target_scene_m=(0.0, 0.16, 5.20)), # X - поеперчная координата от центра радара + dict(path=CAL_ROOT, snapshots=['0003'], target_scene_m=(0.0, 0.16, 5.20)), # y - выота от поверхности земли + dict(path=CAL_ROOT, snapshots=['0004'], target_scene_m=(0.0, 0.16, 5.20)), # z - длина вдоль земли от вертикальной плоскости, про +] # проходящей через нижние антенны до центра шара (до переднего края + радиус) +SPHERE_RADIUS_M = 0.16 # радиус шара-отражателя [м]; 0 -> точечная цель +EMPTY_SCENE_PATH = Path('/Users/ivan_root/Downloads/Telegram_dwnld/2026-09-31 balls calib/' + 'librevna_multi_0.5-6GHz_1031pts_-3dBm_bkgd_h_0cm/preprocessed') + # папка снимков той же сцены БЕЗ отражателя (вычитается их среднее); + # None -> без вычитания + +# --- геометрия (как в софте построения изображений) --------------------------------------- +INPUT_IDX = [0, 1, 2, 3] # физические входы (Rx), input_pos +OUTPUT_IDX = [0, 1, 2, 3] # физические выходы (Tx), output_pos +ANTENNA_HEIGHT_M = 0.92 # высота начала координат антенн (нижний ряд) над землёй [м] +PITCH_DEG = 10.0 # наклон оси радара ниже горизонта [град] + +# --- оценка ----------------------------------------------------------------------------- +CAL_F_MIN = 2.0e9 # полоса, по которой оценивается калибровка [Гц] +CAL_F_MAX = 6.0e9 +GATE_HALF_M = 0.15 # окно вокруг эха цели, ± по бистатическому пути [м] +GATE_TAPER_M = 0.03 # плавные края окна [м] +COARSE_RANGE_SEARCH_M = 0.6 # грубый поиск эха по дальности перед точной подгонкой, ± по пути [м]; + # 0 -> выключен +REFINE_AXES = ('x', 'y', 'z') # какие координаты цели уточнять по данным; если X и Y измерены + # точно — ('z',) +REFINE_RANGE_M = 0.10 # уточнение не дальше этого от найденной точки [м] +APPLY_AMPLITUDE = False # True -> поправка выравнивает и амплитуды антенн +PHASE_MODEL = 'linear' # модель фазы антенны по частоте: 'const' | 'linear' | 'poly3' | 'raw' + +# --- куда писать калибровку --------------------------------------------------------------- +CAL_NAME = 'cal_20260931_big_ball_h0_script' +CAL_DIR = Path(__file__).resolve().parent / 'results' / 'calibration' / CAL_NAME +OVERWRITE = True # True -> перезаписать файлы калибровки в CAL_DIR + +# --- к каким измерениям применять --------------------------------------------------------- +# Список папок измерений (тех, где лежат preprocessed, raw, results). +APPLY_TO_MEASUREMENTS = [ + # '/путь/к/измерению', +] +OVERWRITE_POSTPROCESSED = False # True -> пересоздать postprocessed (только созданный этой калибровкой: + # папка без calibration_applied.json не удаляется никогда) + +# ╔══════════════════════════════════════════════════════════════════════════════════════╗ +# ║ КОНЕЦ ЗАДАВАЕМЫХ ПАРАМЕТРОВ ║ +# ╚══════════════════════════════════════════════════════════════════════════════════════╝ + +C_LIGHT = 299_792_458.0 + + +@dataclass +class Settings: + """Настройки расчёта; значения по умолчанию — из блока параметров выше.""" + sphere_radius_m: float = SPHERE_RADIUS_M + input_idx: tuple = tuple(INPUT_IDX) + output_idx: tuple = tuple(OUTPUT_IDX) + antenna_height_m: float = ANTENNA_HEIGHT_M + pitch_deg: float = PITCH_DEG + f_min: float = CAL_F_MIN + f_max: float = CAL_F_MAX + gate_half_m: float = GATE_HALF_M + gate_taper_m: float = GATE_TAPER_M + coarse_range_search_m: float = COARSE_RANGE_SEARCH_M + refine_axes: tuple = tuple(REFINE_AXES) + refine_range_m: float = REFINE_RANGE_M + apply_amplitude: bool = APPLY_AMPLITUDE + phase_model: str = PHASE_MODEL + + +# ══════════════════════════════════════════════════════ +# ЗАГРУЗКА +# ══════════════════════════════════════════════════════ +def find_config_profile(start, max_levels=8): + p = Path(start) + for _ in range(max_levels): + if (p / 'config_profile.json').exists(): + return p / 'config_profile.json' + p = p.parent + raise FileNotFoundError(f'config_profile.json не найден выше {start}') + + +def load_geometry(config_path, input_idx, output_idx): + """Координаты антенн в осях радара [м], массивы (N,3) в порядке сортировки физических портов.""" + gpr = json.load(open(config_path, encoding='utf-8'))['gpr'] + tx = {int(e['output_pos']): (e['x_m'], e.get('y_m', 0.0), e.get('z_m', 0.0)) for e in gpr['tx_geometry']} + rx = {int(e['input_pos']): (e['x_m'], e.get('y_m', 0.0), e.get('z_m', 0.0)) for e in gpr['rx_geometry']} + return (np.array([tx[k] for k in sorted(output_idx)], float), + np.array([rx[k] for k in sorted(input_idx)], float)) + + +def list_snapshots(root, prefixes=None): + dirs = sorted(p for p in Path(root).iterdir() if p.is_dir() and (p / 'meta.json').exists()) + if prefixes is None: + return dirs + out = [d for d in dirs if any(d.name.startswith(str(pr)) for pr in prefixes)] + if len(out) != len(prefixes): + raise FileNotFoundError(f'В {root} найдены не все снимки {prefixes}') + return out + + +def load_s21(snapshot_dir, input_idx, output_idx): + """S21 (TX, RX, Nf) и частоты. Индексы — порядок сортировки физических портов.""" + s21, freq = None, None + for a, o in enumerate(sorted(output_idx)): + for b, i in enumerate(sorted(input_idx)): + s = np.load(Path(snapshot_dir) / f'i{i}_o{o}_s21.npy') + f = np.load(Path(snapshot_dir) / f'i{i}_o{o}_freq.npy') + if s21 is None: + s21 = np.zeros((len(output_idx), len(input_idx), s.size), complex); freq = f + elif not np.allclose(f, freq): + raise ValueError(f'Разные частотные сетки у пар в {snapshot_dir}') + s21[a, b] = s + return s21, freq + + +# ══════════════════════════════════════════════════════ +# РАСЧЁТ КАЛИБРОВКИ +# ══════════════════════════════════════════════════════ +def best_phases(Z): + """max |Σ Z_ij·e^{-j(t_i+r_j)}| по фазам антенн, t_0 = r_0 = 0.""" + t = np.zeros(Z.shape[0]); r = np.zeros(Z.shape[1]) + for _ in range(100): + t = np.angle((Z * np.exp(-1j * r)[None, :]).sum(axis=1)) + r = np.angle((Z * np.exp(-1j * t)[:, None]).sum(axis=0)) + t = t - t[0]; r = r - r[0] + return t, r + + +def coherence(Z, t=None, r=None): + """Сфазированность вкладов пар: |Σ| / Σ|·| (1 — все в фазе).""" + if t is not None: + Z = Z * np.exp(-1j * (t[:, None] + r[None, :])) + return float(np.abs(Z.sum()) / (np.abs(Z).sum() + 1e-30)) + + +def circular_median(phases): + """Круговая медиана по первой оси: значение с минимальной суммой угловых расстояний до остальных.""" + d = np.abs(np.angle(np.exp(1j * (phases[:, None] - phases[None, :])))) + best = np.argmin(d.sum(axis=1), axis=0) + return np.take_along_axis(phases, best[None], axis=0)[0] + + +def combine(sets): + """Добавки антенн по всем калибровочным точкам: круговая медиана фаз и медиана усиления.""" + out = [] + for key in ('T', 'R'): + Z = np.array([s[key] for s in sets]) # (точки, антенны, частоты) + out.append(np.exp(np.median(np.log(np.abs(Z)), axis=0)) * np.exp(1j * circular_median(np.angle(Z)))) + return tuple(out) + + +class _Estimator: + """Геометрия, полоса и окно одной калибровки.""" + + def __init__(self, st, tx, rx, freq): + self.st, self.tx, self.rx, self.freq = st, tx, rx, freq + self.n_tx, self.n_rx = len(tx), len(rx) + self.band = (freq >= st.f_min) & (freq <= st.f_max) + self.fb = freq[self.band] + self.win = np.hanning(self.fb.size) + self.ok = self.win > 0.2 # края полосы после деления на окно ненадёжны + self.nfft = 1 << int(np.ceil(np.log2(self.fb.size * 8))) + self.dpath = np.fft.fftfreq(self.nfft, self.fb[1] - self.fb[0]) * C_LIGHT # ось «лишний путь», м + self.gate = np.clip(np.minimum((self.dpath + st.gate_half_m) / st.gate_taper_m, + (st.gate_half_m - self.dpath) / st.gate_taper_m) + 0.5, 0, 1) + + def scene_to_radar(self, q): + """(X, Y над землёй, Z вдоль земли) -> оси радара (x, y, z).""" + p = np.radians(self.st.pitch_deg); y0 = q[1] - self.st.antenna_height_m + return np.array([q[0], y0 * np.cos(p) + q[2] * np.sin(p), -y0 * np.sin(p) + q[2] * np.cos(p)]) + + def pair_paths(self, q_scene): + """Бистатический путь каждой пары до цели (TX, RX). Для шара радиуса a (q_scene — центр) + путь короче на 2a·cos(beta/2), beta — угол между направлениями из центра на Tx и на Rx.""" + a = self.st.sphere_radius_m + pr = self.scene_to_radar(np.asarray(q_scene, float)) + vt = self.tx - pr; vr = self.rx - pr + dt = np.linalg.norm(vt, axis=1); dr = np.linalg.norm(vr, axis=1) + paths = dt[:, None] + dr[None, :] + if a: + cos_beta = (vt @ vr.T) / (dt[:, None] * dr[None, :]) + paths = paths - 2 * a * np.sqrt((1 + cos_beta) / 2) + return paths + + def _unwound(self, s21_band, paths): + return s21_band * np.exp(2j * np.pi * self.fb[None, None, :] * paths[:, :, None] / C_LIGHT) + + def target_echo(self, s21_band, paths): + """Эхо цели E (TX, RX, Nb) со снятым путём: окно ±gate_half_m вокруг заданной точки.""" + h = np.fft.ifft(self._unwound(s21_band, paths) * self.win, self.nfft, axis=2) + return np.fft.fft(h * self.gate, axis=2)[:, :, :self.fb.size] / np.maximum(self.win, 1e-3) + + def broadband(self, E): + """Вклад пары в изображение в точке: сумма эха по надёжной части полосы.""" + return E[:, :, self.ok].sum(axis=2) + + def coarse_range(self, s21, q_nominal): + """Грубая привязка по дальности: максимум суммарной огибающей всех пар в пределах + ±coarse_range_search_m по пути; заданная дальность Z сдвигается под него.""" + q = q_nominal.copy() + if not self.st.coarse_range_search_m: + return q + s = np.mean(s21, axis=0)[:, :, self.band]; sel = np.abs(self.dpath) <= self.st.coarse_range_search_m + for _ in range(3): + env = np.abs(np.fft.ifft(self._unwound(s, self.pair_paths(q)) * self.win, self.nfft, axis=2)).sum(axis=(0, 1)) + q[2] += self.dpath[sel][np.argmax(env[sel])] / 2 + return q + + def refine_target(self, s21, q0): + """Положение цели и широкополосные фазы антенн, максимизирующие сфазированность.""" + P0 = self.pair_paths(q0) + E0 = np.mean([self.target_echo(s[:, :, self.band], P0) for s in s21], axis=0) + axes = [('xyz').index(a) for a in self.st.refine_axes] + + def E_at(v): + q = q0.copy(); q[axes] += v + dP = self.pair_paths(q) - P0 + return E0 * np.exp(2j * np.pi * self.fb[None, None, :] * dP[:, :, None] / C_LIGHT), q + + def score(v, t, r): + if np.any(np.abs(v) > self.st.refine_range_m): + return 0.0 + return coherence(self.broadband(E_at(v)[0]), t, r) + + v = np.zeros(len(axes)) + if axes: + for _ in range(6): + t, r = best_phases(self.broadband(E_at(v)[0])) + simplex = np.vstack([v] + [v + 0.03 * np.eye(len(axes))[k] for k in range(len(axes))]) + v = minimize(lambda w: -score(w, t, r), v, method='Nelder-Mead', + options=dict(xatol=5e-4, fatol=1e-7, initial_simplex=simplex)).x + return E_at(v) + + def factorize(self, E): + """E_ij(f) ≈ g(f)·T_i(f)·R_j(f) на каждой частоте: фазы — best_phases, амплитуды — МНК по log|E|.""" + nb = E.shape[2] + T = np.ones((self.n_tx, nb), complex); R = np.ones((self.n_rx, nb), complex) + L = np.log(np.abs(E) + 1e-30) + for k in range(nb): + t, r = best_phases(E[:, :, k]) + tt = L[:, :, k].mean(axis=1) - L[:, :, k].mean(); rr = L[:, :, k].mean(axis=0) - L[:, :, k].mean() + T[:, k] = np.exp(tt + 1j * t); R[:, k] = np.exp(rr + 1j * r) + return T, R + + def antenna_params(self, Z): + """Фаза, продолженная к 0 Гц, задержка (эквивалентный лишний путь) и усиление антенны.""" + ph = np.unwrap(np.angle(Z[self.ok])); f = self.fb[self.ok] + slope, ph0 = np.polyfit(f, ph, 1) + return dict(phase0_deg=float(np.degrees(ph0)), delay_path_m=float(-slope * C_LIGHT / (2 * np.pi)), + gain_db=float(20 * np.log10(np.mean(np.abs(Z[self.ok])))), + phase_rms_after_linear_deg=float(np.degrees(np.std(ph - (slope * f + ph0))))) + + def phase_model(self, Z): + """Фаза добавки антенны по модели phase_model (подгонка по надёжной части полосы).""" + model = self.st.phase_model + if model == 'raw': + return Z + ph = np.unwrap(np.angle(Z)); f = (self.fb - self.fb.mean()) / 1e9 + if model == 'const': + u = np.exp(1j * np.angle(Z[self.ok])).mean() + return np.abs(Z) * np.exp(1j * np.angle(u)) + deg = {'linear': 1, 'poly3': 3}.get(model) + if deg is None: + raise ValueError(f"phase_model должен быть 'const', 'linear', 'poly3' или 'raw', а не {model!r}") + return np.abs(Z) * np.exp(1j * np.polyval(np.polyfit(f[self.ok], ph[self.ok], deg), f)) + + def pair_correction(self, T, R): + """Поправка пары C_ij(f) = 1/(T_i·R_j); без apply_amplitude — только фаза.""" + Z = T[:, None, :] * R[None, :, :] + return np.conj(Z) / np.abs(Z) if not self.st.apply_amplitude else 1.0 / Z + + def to_full_grid(self, Z_band, params): + """На полную сетку частот: внутри надёжной полосы — оценка, вне её — продолжение с наклоном + линейной модели фазы антенны, пристыкованное к краю полосы без скачка.""" + idx = np.flatnonzero(self.band)[self.ok] + ph_in = np.unwrap(np.angle(Z_band[self.ok])); mag_in = np.abs(Z_band[self.ok]) + slope = -2 * np.pi * params['delay_path_m'] / C_LIGHT # рад/Гц + ph = np.empty(self.freq.size); mag = np.empty(self.freq.size) + ph[idx] = ph_in; mag[idx] = mag_in + lo, hi = idx[0], idx[-1] + ph[:lo] = ph_in[0] + slope * (self.freq[:lo] - self.freq[lo]); mag[:lo] = mag_in[0] + ph[hi + 1:] = ph_in[-1] + slope * (self.freq[hi + 1:] - self.freq[hi]); mag[hi + 1:] = mag_in[-1] + return mag * np.exp(1j * ph) + + +def compute_calibration(cal_sets, empty_scene_path=None, settings=None): + """Калибровка по набору измерений отражателя. Возвращает словарь в том же виде, что load_calibration.""" + st = settings or Settings() + cal_sets = list(cal_sets) + config_path = find_config_profile(cal_sets[0]['path']) + tx, rx = load_geometry(config_path, st.input_idx, st.output_idx) + + freq = None + empty = None + if empty_scene_path is not None: + loaded = [load_s21(d, st.input_idx, st.output_idx) for d in list_snapshots(empty_scene_path)] + empty = np.mean([s for s, _ in loaded], axis=0); freq = loaded[0][1] + + sets = [] + for n, cs in enumerate(cal_sets): + snaps = list_snapshots(cs['path'], cs.get('snapshots')) + data = [] + for d in snaps: + s21, f = load_s21(d, st.input_idx, st.output_idx) + if freq is None: + freq = f + elif f.shape != freq.shape or not np.allclose(f, freq): + raise ValueError(f'Частотная сетка в {d} отличается от остальных калибровочных данных') + cfg = find_config_profile(d) + if cfg != config_path: + tx_d, rx_d = load_geometry(cfg, st.input_idx, st.output_idx) + if not (np.allclose(tx_d, tx) and np.allclose(rx_d, rx)): + raise ValueError(f'Геометрия в {d} отличается от {config_path}') + data.append(s21 - empty if empty is not None else s21) + sets.append(dict(n=n, snaps=[d.name for d in snaps], s21=np.array(data), + q_nominal=np.array(cs['target_scene_m'], float))) + + est = _Estimator(st, tx, rx, freq) + for s in sets: + s['E'], s['q'] = est.refine_target(s['s21'], est.coarse_range(s['s21'], s['q_nominal'])) + s['T'], s['R'] = est.factorize(s['E']) + + T_all, R_all = combine(sets) + T_all = np.array([est.phase_model(z) for z in T_all]); R_all = np.array([est.phase_model(z) for z in R_all]) + ant = {f'Tx{k}': est.antenna_params(T_all[k]) for k in range(est.n_tx)} + ant.update({f'Rx{k}': est.antenna_params(R_all[k]) for k in range(est.n_rx)}) + + # качество: сфазированность пар в точке цели без поправки и с ней + coh_with = lambda s, C: coherence(est.broadband(s['E'] * C)) + quality = dict(coherence_before=[], coherence_own=[], coherence_leave_one_out=[], coherence_final=[]) + for s in sets: + others = [o for o in sets if o is not s] + quality['coherence_before'].append(coh_with(s, 1.0)) + quality['coherence_own'].append(coh_with(s, est.pair_correction(s['T'], s['R']))) + quality['coherence_leave_one_out'].append(coh_with(s, est.pair_correction(*combine(others))) if others else float('nan')) + quality['coherence_final'].append(coh_with(s, est.pair_correction(T_all, R_all))) + + T_full = np.array([est.to_full_grid(T_all[k], ant[f'Tx{k}']) for k in range(est.n_tx)]) + R_full = np.array([est.to_full_grid(R_all[k], ant[f'Rx{k}']) for k in range(est.n_rx)]) + Z_full = T_full[:, None, :] * R_full[None, :, :] + unit = (lambda Z: np.conj(Z) / np.abs(Z)) if not st.apply_amplitude else (lambda Z: 1.0 / Z) + valid = np.zeros(freq.size, bool); valid[np.flatnonzero(est.band)[est.ok]] = True + + meta = dict( + created=datetime.datetime.now().isoformat(timespec='seconds'), name=None, + convention=dict(apply='S21_corrected[tx, rx, f] = S21[tx, rx, f] * cal_pair[tx, rx, f]', + index_order='tx = номер в cal_tx_output_pos (output_pos по возрастанию), rx = cal_rx_input_pos', + files='S21 пары в данных: i{input_pos}_o{output_pos}_s21.npy', + phase_reference='фазы относительно Tx0 и Rx0; общая фаза/задержка не входит', + amplitude='выравнивается' if st.apply_amplitude else 'не меняется (только фаза)', + outside_band='вне cal_valid: продолжение линейной моделью фазы антенны'), + geometry=dict(config=str(config_path), tx_xyz_m=tx.tolist(), rx_xyz_m=rx.tolist(), + antenna_height_m=st.antenna_height_m, pitch_deg=st.pitch_deg), + estimation=dict(band_hz=[st.f_min, st.f_max], gate_half_m=st.gate_half_m, refine_axes=list(st.refine_axes), + phase_model=st.phase_model, sphere_radius_m=st.sphere_radius_m, empty_scene=str(empty_scene_path)), + targets=[dict(path=str(cal_sets[s['n']]['path']), snapshots=s['snaps'], nominal_scene_m=s['q_nominal'].tolist(), + refined_scene_m=s['q'].tolist()) for s in sets], + antennas=ant, quality=quality) + return dict( + freq_hz=freq.astype(np.float64), + pair=unit(Z_full).astype(np.complex64), # (TX, RX, Nf): S21_corr = S21 * pair + tx=unit(T_full).astype(np.complex64), # (TX, Nf): pair = tx[:,None] * rx[None,:] + rx=unit(R_full).astype(np.complex64), # (RX, Nf) + valid=valid, # (Nf,) True — внутри полосы калибровки + tx_output_pos=np.array(sorted(st.output_idx), np.int32), # индекс -> физический выход + rx_input_pos=np.array(sorted(st.input_idx), np.int32), # индекс -> физический вход + antenna_params=np.array([[np.radians(a['phase0_deg']), a['delay_path_m'], a['gain_db']] for a in ant.values()], + np.float64), # (TX+RX, 3): фаза к 0 Гц [рад], путь [м], усиление [дБ] + meta=meta, dir=None) + + +# ══════════════════════════════════════════════════════ +# ФАЙЛЫ КАЛИБРОВКИ +# ══════════════════════════════════════════════════════ +_CAL_ARRAYS = ('freq_hz', 'pair', 'tx', 'rx', 'valid', 'tx_output_pos', 'rx_input_pos', 'antenna_params') + + +def save_calibration(cal, cal_dir, overwrite=False): + cal_dir = Path(cal_dir) + cal_dir.mkdir(parents=True, exist_ok=True) + existing = [p.name for p in cal_dir.glob('cal_*')] + if existing and not overwrite: + raise FileExistsError(f'В {cal_dir} уже есть файлы калибровки ({existing[:3]}...). ' + f'Задайте новое имя или overwrite=True.') + for k in _CAL_ARRAYS: + np.save(cal_dir / f'cal_{k}.npy', cal[k]) + cal['meta']['name'] = cal_dir.name; cal['dir'] = cal_dir + json.dump(cal['meta'], open(cal_dir / 'cal_meta.json', 'w', encoding='utf-8'), ensure_ascii=False, indent=2) + return cal_dir + + +def load_calibration(cal_dir): + cal_dir = Path(cal_dir) + cal = {k: np.load(cal_dir / f'cal_{k}.npy') for k in _CAL_ARRAYS} + cal['meta'] = json.load(open(cal_dir / 'cal_meta.json', encoding='utf-8')) + cal['dir'] = cal_dir + return cal + + +# ══════════════════════════════════════════════════════ +# ПРИМЕНЕНИЕ +# ══════════════════════════════════════════════════════ +def calibration_on_grid(cal, freq): + """Поправка пар (TX, RX, Nf) на частотной сетке freq. Совпадает с сеткой калибровки — берётся как есть; + иначе фаза и модуль поправок антенн интерполируются, вне сетки калибровки — линейная модель фазы.""" + fc = cal['freq_hz'] + if freq.shape == fc.shape and np.allclose(freq, fc): + return cal['pair'] + + def interp(Z, par): + ph = np.unwrap(np.angle(Z)); mag = np.abs(Z) + out_ph = np.interp(freq, fc, ph); out_mag = np.interp(freq, fc, mag) + lin = -(par[0] - 2 * np.pi * freq * par[1] / C_LIGHT) # поправка = −добавка + outside = (freq < fc[0]) | (freq > fc[-1]) + out_ph[outside] = lin[outside] + np.round((out_ph[outside] - lin[outside]) / (2 * np.pi)) * 2 * np.pi + return out_mag * np.exp(1j * out_ph) + + n_tx = cal['tx'].shape[0] + tx = np.array([interp(cal['tx'][k], cal['antenna_params'][k]) for k in range(n_tx)]) + rx = np.array([interp(cal['rx'][k], cal['antenna_params'][n_tx + k]) for k in range(cal['rx'].shape[0])]) + return (tx[:, None, :] * rx[None, :, :]).astype(cal['pair'].dtype) + + +def apply_calibration(s21, freq, cal): + """s21 (TX, RX, Nf) в порядке cal['tx_output_pos'] x cal['rx_input_pos'] -> исправленный массив.""" + return s21 * calibration_on_grid(cal, np.asarray(freq)) + + +def apply_to_measurement(meas_dir, cal, overwrite=False): + """<измерение>/preprocessed -> <измерение>/postprocessed: S21 пар умножаются на поправку, + остальные файлы копируются. Возвращает (папка, снимков, исправлено S21, скопировано файлов).""" + meas_dir = Path(meas_dir); pre = meas_dir / 'preprocessed'; post = meas_dir / 'postprocessed' + if not pre.is_dir(): + raise FileNotFoundError(f'Нет {pre}') + # калибровка годится только для той же сборки антенн + tx_m, rx_m = load_geometry(find_config_profile(pre), cal['rx_input_pos'].tolist(), cal['tx_output_pos'].tolist()) + if not (np.allclose(tx_m, cal['meta']['geometry']['tx_xyz_m']) and np.allclose(rx_m, cal['meta']['geometry']['rx_xyz_m'])): + raise ValueError(f'Геометрия антенн в {meas_dir.name} отличается от геометрии калибровки — применять нельзя') + if post.exists(): + if not overwrite: + raise FileExistsError(f'{post} уже существует. overwrite=True, чтобы пересоздать.') + if not (post / 'calibration_applied.json').exists(): + raise FileExistsError(f'{post} существует, но создан не калибровкой (нет calibration_applied.json) — ' + f'удалять не буду, разберитесь вручную.') + shutil.rmtree(post) + tx_of = {int(p): k for k, p in enumerate(cal['tx_output_pos'])} + rx_of = {int(p): k for k, p in enumerate(cal['rx_input_pos'])} + pat = re.compile(r'^i(\d+)_o(\d+)_s21\.npy$') + n_snap = n_fix = n_copy = 0 + grid_cache = {} + post.mkdir() + for item in sorted(pre.iterdir()): + if item.is_file(): + shutil.copy2(item, post / item.name); n_copy += 1 + continue + out = post / item.name; out.mkdir(); n_snap += 1 + for f in sorted(item.iterdir()): + m = pat.match(f.name) + if f.is_dir() or m is None or int(m.group(2)) not in tx_of or int(m.group(1)) not in rx_of: + if f.is_dir(): + shutil.copytree(f, out / f.name) + else: + shutil.copy2(f, out / f.name) + n_copy += 1 + continue + s21 = np.load(f); freq = np.load(item / f'i{m.group(1)}_o{m.group(2)}_freq.npy') + key = (freq.size, float(freq[0]), float(freq[-1])) + if key not in grid_cache: + grid_cache[key] = calibration_on_grid(cal, freq) + corr = grid_cache[key][tx_of[int(m.group(2))], rx_of[int(m.group(1))]] + np.save(out / f.name, (s21 * corr).astype(s21.dtype)); n_fix += 1 + json.dump(dict(calibration=cal['meta']['name'], calibration_dir=str(Path(cal['dir']).resolve()), + created=datetime.datetime.now().isoformat(timespec='seconds'), source=str(pre), + rule='postprocessed i*_o*_s21 = preprocessed i*_o*_s21 * cal_pair[tx(o), rx(i)]; остальные файлы скопированы', + frequency_grids=[list(k) for k in grid_cache]), + open(post / 'calibration_applied.json', 'w', encoding='utf-8'), ensure_ascii=False, indent=2) + return post, n_snap, n_fix, n_copy + + +def main(): + if RUN_CALIBRATION: + cal = compute_calibration(CAL_SETS, EMPTY_SCENE_PATH, Settings()) + save_calibration(cal, CAL_DIR, overwrite=OVERWRITE) + q = cal['meta']['quality'] + print(f'Калибровка записана в {CAL_DIR}: сфазированность пар на отражателе ' + f'{np.mean(q["coherence_before"]):.2f} -> {np.mean(q["coherence_final"]):.2f}') + if APPLY_TO_DATA: + cal = load_calibration(CAL_DIR) + targets = APPLY_TO_MEASUREMENTS + if isinstance(targets, (str, Path)): # один путь вместо списка + targets = [targets] + for meas in targets: + post, n_snap, n_fix, _ = apply_to_measurement(meas, cal, OVERWRITE_POSTPROCESSED) + print(f'{Path(meas).name}: {n_snap} снимков, исправлено S21 {n_fix} -> {post}') + + +if __name__ == '__main__': + main() diff --git a/python_app/file_io.py b/python_app/file_io.py new file mode 100644 index 0000000..df0609c --- /dev/null +++ b/python_app/file_io.py @@ -0,0 +1,47 @@ +"""Atomic writes for JSON settings shared by GUI and pipeline processes.""" + +from __future__ import annotations + +import json +import os +from contextlib import suppress +from pathlib import Path +import tempfile + + +def atomic_write_json(path: Path, payload: object, *, durable: bool = False) -> Path: + """Serialize strict JSON before opening any files, then atomically commit it.""" + return atomic_write_text(path, json.dumps(payload, indent=2, allow_nan=False), durable=durable) + + +def atomic_write_text(path: Path, text: str, *, durable: bool = False) -> Path: + """Replace a text file only after a complete write, cleaning up failed writes. + + Each call owns its temporary file, so concurrent writers cannot truncate or + rename one another's work. Durable writes also sync the file and directory; + frequent live settings can omit those syncs to avoid unnecessary disk I/O. + """ + path.parent.mkdir(parents=True, exist_ok=True) + temporary = None + try: + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=path.parent, + prefix=f".{path.name}.", suffix=".tmp", delete=False, + ) as handle: + temporary = Path(handle.name) + handle.write(text) + if durable: + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + if durable: + directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + finally: + if temporary is not None: + with suppress(OSError): + temporary.unlink(missing_ok=True) + return path diff --git a/python_app/generator_sweep/config.py b/python_app/generator_sweep/config.py index ed8f35a..5214d10 100644 --- a/python_app/generator_sweep/config.py +++ b/python_app/generator_sweep/config.py @@ -4,6 +4,7 @@ from __future__ import annotations from dataclasses import dataclass import logging +import math logger = logging.getLogger(__name__) @@ -121,17 +122,13 @@ def _build_frequency_grid(model: GeneratorSweepConfig) -> tuple[int, ...]: if has_step: _validate_positive_int(step_hz, "generator_sweep.step_hz") - frequencies: list[int] = [] - current_hz = start_hz - while current_hz < stop_hz: - frequencies.append(current_hz) - current_hz += step_hz - if not frequencies or frequencies[-1] != stop_hz: - frequencies.append(stop_hz) - return tuple(frequencies) + _require((stop_hz - start_hz + step_hz - 1) // step_hz + 1 <= 1_000_000, + "generator sweep grid must not exceed 1000000 frequencies") + return (*range(start_hz, stop_hz, step_hz), stop_hz) _validate_positive_int(points, "generator_sweep.points") _require(points >= 2, "generator_sweep.points must be >= 2 when start_hz != stop_hz") + _require(points <= 1_000_000, "generator sweep grid must not exceed 1000000 frequencies") span_hz = stop_hz - start_hz frequencies = tuple( int(round(start_hz + (span_hz * index) / float(points - 1))) @@ -148,6 +145,16 @@ def _build_frequency_grid(model: GeneratorSweepConfig) -> tuple[int, ...]: def resolve_generator_sweep_config(model: GeneratorSweepConfig) -> ResolvedGeneratorSweepConfig: """Validate and resolve runtime configuration for generator sweep mode.""" + for name in ("points", "settle_timeout_ms", "post_lock_delay_us", "pwm_frequency_hz", + "sweep_start_pin", "curr_step_pin", "pwm_pin", "curr_step_initial_level", + "strict_protocol_version", "port"): + value = getattr(model, name) + _require(type(value) is int, f"generator_sweep.{name} must be an integer") + for name in ("start_hz", "stop_hz", "step_hz", "hold_time_ms", "status_poll_ms", "power_dbm", "pwm_duty_cycle"): + value = getattr(model, name) + _require(type(value) in (int, float) and math.isfinite(value), + f"generator_sweep.{name} must be a finite number") + _require(model.points >= 0 and model.step_hz >= 0, "generator_sweep points and step_hz must be >= 0") log_level = _resolve_log_level(model.log_level) frequencies_hz = _build_frequency_grid(model) hold_time_ms = float(model.hold_time_ms) diff --git a/python_app/generator_sweep/runner.py b/python_app/generator_sweep/runner.py index ffdad82..117dd2b 100644 --- a/python_app/generator_sweep/runner.py +++ b/python_app/generator_sweep/runner.py @@ -142,24 +142,19 @@ class GeneratorSweepRunner: def _close(self) -> None: """Best-effort cleanup of PWM, GPIO, and device state.""" - self._pwm.disable() - - device = self._device + device, self._device = self._device, None + cleanup = [self._pwm.disable] if device is not None: + cleanup.extend([ + lambda: device.generator.set_idle(timeout_s=1.0) if device.is_connected else None, + device.disconnect, + ]) + cleanup.extend([self._markers.reset_to_idle, self._markers.close, self._pwm.close]) + for action in cleanup: try: - if device.is_connected: - device.generator.set_idle(timeout_s=1.0) - except Exception as exc: # noqa: BLE001 - logger.warning("Failed to set LibreVNA idle mode during shutdown: %s", exc) - finally: - device.disconnect() - self._device = None - - try: - self._markers.reset_to_idle() - finally: - self._markers.close() - self._pwm.close() + action() + except Exception as exc: + logger.warning("Generator sweep cleanup failed: %s", exc) def _run_single_sweep(self) -> None: """Execute one pass from the first configured frequency to the last.""" diff --git a/python_app/gui/antenna_geometry_editor.py b/python_app/gui/antenna_geometry_editor.py new file mode 100644 index 0000000..4fd4f6b --- /dev/null +++ b/python_app/gui/antenna_geometry_editor.py @@ -0,0 +1,133 @@ +"""Table editor for antenna switch positions and coordinates in metres.""" + +from __future__ import annotations + +from dataclasses import asdict, is_dataclass +import math + +from PyQt6.QtCore import QLocale, QSignalBlocker, pyqtSignal +from PyQt6.QtGui import QDoubleValidator, QIntValidator +from PyQt6.QtWidgets import ( + QAbstractItemView, QHeaderView, QHBoxLayout, QLineEdit, QMessageBox, QPushButton, + QStyledItemDelegate, QTableWidget, QTableWidgetItem, QVBoxLayout, QWidget, +) + + +class _CoordinateDelegate(QStyledItemDelegate): + def createEditor(self, parent, option, index): + editor = QLineEdit(parent) + validator = ( + QIntValidator(0, self.parent().property("position_count") - 1, editor) + if index.column() == 0 else QDoubleValidator(editor) + ) + validator.setLocale(QLocale.c()) + editor.setValidator(validator) + return editor + + +class AntennaGeometryEditor(QWidget): + """Keep geometry explicit and preserve all three coordinates on profile round trips.""" + + changed = pyqtSignal() + + def __init__(self, position_field: str, entries, *, position_count: int, parent=None) -> None: + super().__init__(parent) + self.position_field = position_field + self._table = QTableWidget(0, 4, self) + self.set_position_count(position_count) + self._table.setHorizontalHeaderLabels([ + "Output" if position_field == "output_pos" else "Input", "X (m)", "Y (m)", "Z (m)", + ]) + self._table.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeMode.Stretch) + self._table.verticalHeader().hide() + self._table.setSelectionBehavior(QAbstractItemView.SelectionBehavior.SelectRows) + self._table.setItemDelegate(_CoordinateDelegate(self._table)) + self._table.setMinimumHeight(130) + self._table.setMaximumHeight(220) + layout = QVBoxLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + layout.addWidget(self._table) + buttons = QHBoxLayout() + self._add_button = QPushButton("Add antenna", self) + self._remove_button = QPushButton("Remove selected", self) + buttons.addWidget(self._add_button) + buttons.addWidget(self._remove_button) + layout.addLayout(buttons) + self.set_records(entries) + self._table.itemChanged.connect(self._on_item_changed) + self._add_button.clicked.connect(self._add_row) + self._remove_button.clicked.connect(self._remove_rows) + + def records(self) -> list[dict]: + records = [] + for row in range(self._table.rowCount()): + records.append(dict(zip( + (self.position_field, "x_m", "y_m", "z_m"), + (self._table.item(row, column).text() for column in range(4)), + strict=True, + ))) + return self._normalize(records) + + def _normalize(self, entries) -> list[dict]: + records = [] + positions = set() + for entry in entries: + entry = asdict(entry) if is_dataclass(entry) else entry + if any(isinstance(entry.get(key), bool) for key in (self.position_field, "x_m", "y_m", "z_m")): + raise ValueError("Antenna positions and coordinates must be numbers, not booleans") + position = int(entry[self.position_field]) + if float(entry[self.position_field]) != position or not 0 <= position < self._table.property("position_count"): + raise ValueError("Antenna position is outside the configured switch positions") + if position in positions: + raise ValueError(f"Duplicate antenna position: {position}") + positions.add(position) + coordinates = {key: float(entry.get(key, 0.0)) for key in ("x_m", "y_m", "z_m")} + if any(not math.isfinite(value) or abs(value) > 3.4e38 for value in coordinates.values()): + raise ValueError("Antenna coordinates must be finite numbers") + records.append({self.position_field: position, **coordinates}) + return records + + def set_position_count(self, count: int) -> None: + self.position_count = int(count) + self._table.setProperty("position_count", int(count)) + + def set_records(self, entries) -> None: + records = self._normalize(entries) + with QSignalBlocker(self._table): + self._table.setRowCount(len(records)) + for row, record in enumerate(records): + for column, value in enumerate(record.values()): + self._table.setItem(row, column, QTableWidgetItem(str(value))) + self._valid_records = records + self._add_button.setEnabled(len(records) < self._table.property("position_count")) + self.changed.emit() + + def _on_item_changed(self, _item) -> None: + try: + self._valid_records = self.records() + except ValueError as exc: + with QSignalBlocker(self): + self.set_records(self._valid_records) + QMessageBox.warning(self, "Invalid antenna geometry", str(exc)) + return + self.changed.emit() + + def _add_row(self) -> None: + records = self.records() + if len(records) >= self._table.property("position_count"): + return + positions = {record[self.position_field] for record in records} + position = next(index for index in range(len(positions) + 1) if index not in positions) + self.set_records([*records, {self.position_field: position}]) + self._table.setCurrentCell(len(records), 1) + self._table.editItem(self._table.item(len(records), 1)) + + def _remove_rows(self) -> None: + rows = {index.row() for index in self._table.selectionModel().selectedRows()} + if rows: + with QSignalBlocker(self._table): + for row in sorted(rows, reverse=True): + self._table.removeRow(row) + self._valid_records = self.records() + self._add_button.setEnabled(True) + self.changed.emit() diff --git a/python_app/gui/app_window.py b/python_app/gui/app_window.py index 71078a9..09b595e 100644 --- a/python_app/gui/app_window.py +++ b/python_app/gui/app_window.py @@ -23,6 +23,7 @@ from PyQt6.QtGui import QTextCursor from PyQt6.QtWidgets import QApplication, QMainWindow, QMessageBox from python_app.gui.controllers.app_window_config_mixin import AppWindowConfigMixin +from python_app.gui.controllers.app_window_config.state_builders import GUI_SAVE_HISTORY_LIMIT from python_app.gui.controllers.app_window_control_button_mixin import AppWindowControlButtonMixin from python_app.gui.controllers.app_window_pipeline_mixin import AppWindowPipelineMixin from python_app.gui.controllers.app_window_plot_mixin import AppWindowPlotMixin @@ -160,6 +161,9 @@ class AppWindow( self._pending_startup_log_entries: list[tuple[str, str, str | None]] = [] # Guards closeEvent against re-entrant teardown (e.g. a second signal). self._closing = False + self._close_after_stop = False + self._stopping_run = False + self._stop_request_seq = 0 def _init_logging(self) -> None: """Configure the application logger and the bridge that feeds the GUI panel. @@ -336,11 +340,10 @@ class AppWindow( def _init_history_state(self) -> None: """Initialize runtime history buffers and render-cache state.""" - bscan_cpp_replay_window = self._cpp_bscan_replay_window_from_config() - save_history_limit = self._save_history_limit_from_config() - self._raw_history: deque[SweepCollection] = deque(maxlen=save_history_limit) - self._pre_history: deque[SweepCollection] = deque(maxlen=save_history_limit) - self._result_history: deque[ResultCollection] = deque(maxlen=save_history_limit) + bscan_cpp_replay_window = self._cpp_bscan_replay_window_for_config(self._defaults_config) + self._raw_history: deque[SweepCollection] = deque(maxlen=GUI_SAVE_HISTORY_LIMIT) + self._pre_history: deque[SweepCollection] = deque(maxlen=GUI_SAVE_HISTORY_LIMIT) + self._result_history: deque[ResultCollection] = deque(maxlen=GUI_SAVE_HISTORY_LIMIT) # Sequence id must survive GUI restarts so history commands stay monotonic. self._history_command_seq = self._load_history_command_seq(self._live_config_writer.path) @@ -371,20 +374,11 @@ class AppWindow( self._gpr_region_contours = [] self._gpr_geometry_signature = None self._gpr_selected_geometry = None - self._phase_viewbox = None self._history_run_signature = None self._processor_run_signature = None self._active_processing_mode = "pass_through" self._radar_limits: dict[str, float | int] | None = None - def _cpp_bscan_replay_window_from_config(self) -> int: - """Return the C++ B-scan replay window for the active config.""" - return self._cpp_bscan_replay_window_for_config(self._defaults_config) - - def _save_history_limit_from_config(self) -> int: - """Return maxlen for GUI snapshot-save deques (independent of ring capacities).""" - return self._save_history_limit_for_config(self._defaults_config) - def _init_runtime_limits(self) -> None: """Initialize read/drain loop limits used by polling and snapshot code.""" self._max_pop_per_poll = 256 @@ -638,30 +632,6 @@ class AppWindow( ) return "
" + "".join(body_parts) + "
" - def _append_log_entry( - self, - level: str, - text: str, - *, - details: str | None = None, - once_key: str | None = None, - ) -> None: - """Append formatted log entry with timestamp and optional details.""" - if once_key is not None: - if once_key in self._logged_once_keys: - return - self._logged_once_keys.add(once_key) - - cursor = self._log_box.textCursor() - cursor.movePosition(QTextCursor.MoveOperation.End) - self._log_box.setTextCursor(cursor) - self._log_box.insertHtml(self._render_log_entry_html(level, text, details)) - self._log_box.insertPlainText("\n") - self._log_box.ensureCursorVisible() - - if level.upper() == "ERROR" and hasattr(self, "_status_label"): - self._status_label.setText("Status: error") - def _on_log_level_selected(self, level_text: str) -> None: """Apply the chosen log level immediately (sub-level logs stop being generated). @@ -746,12 +716,8 @@ class AppWindow( # Missing or malformed file should not block startup. return 0 - raw_value = payload.get("history_command_seq", 0) - if isinstance(raw_value, bool): - return 0 - if isinstance(raw_value, (int, float)): - return max(0, int(raw_value)) - return 0 + raw_value = payload.get("history_command_seq", 0) if isinstance(payload, dict) else 0 + return raw_value if type(raw_value) is int and 0 <= raw_value < (1 << 64) - 1 else 0 def _capture_web_action_error(self, message: str) -> bool: """Record the first error of an in-flight web action so the browser can show it. @@ -804,18 +770,24 @@ class AppWindow( Guarded against re-entrancy so a second close signal (or a ``window.close()`` after the event loop has already returned) does not run teardown twice. """ + if self._stopping_run: + # A drain pumps Qt events. Finish ingesting/finalizing before teardown + # can close the same readers and recording writer beneath that drain. + self._close_after_stop = True + self._stop_all_requested = True + event.ignore() + return if self._closing: # Re-entrant close (second signal, or window.close() after the event loop # already returned): teardown is in progress or done — do nothing more. super().closeEvent(event) return self._closing = True + self._timer.stop() self._log("Window closing; shutting down runtime.") try: # 0) Stop the web server first so a late request cannot start work. self._shutdown_web_ui() - # 0) Stop the disk-recording writer thread so it is not orphaned. - self._shutdown_recording() # 0) Stop the GPIO button watcher so a late press cannot start work. self._stop_control_button_watcher() self._resume_pipeline_after_capture = False @@ -825,11 +797,15 @@ class AppWindow( self._close_replay_ring_writer() # 1) Abort active capture first (releases exclusive hardware resources). self._abort_capture_sequence(resume_pipeline=False) + # Drain acquisition stages before finalizing the last recording chunk. + if self._recording_active: + self._stop_run() # 2) Stop all managed processes/readers. self._stop_all_processes() # 3) Close auxiliary dialog windows. if self._preprocess_dialog is not None: self._preprocess_dialog.close() finally: + self._shutdown_recording() self._log_debug("Window teardown finished.") super().closeEvent(event) diff --git a/python_app/gui/controllers/app_window_config/live_processing_mixin.py b/python_app/gui/controllers/app_window_config/live_processing_mixin.py index 4624733..bc3d81e 100644 --- a/python_app/gui/controllers/app_window_config/live_processing_mixin.py +++ b/python_app/gui/controllers/app_window_config/live_processing_mixin.py @@ -2,17 +2,21 @@ from __future__ import annotations +from contextlib import ExitStack +import math +import time + from PyQt6.QtCore import QSignalBlocker from PyQt6.QtWidgets import ( QCheckBox, QComboBox, QDoubleSpinBox, QLineEdit, - QPlainTextEdit, QSpinBox, ) from python_app.gui.runtime.constraints import validate_processing_mode_constraints +from python_app.gui.antenna_geometry_editor import AntennaGeometryEditor from python_app.orchestration.live_processing_config import ProcessingLiveConfig _GPR_MODES = ("gpr", "legacy_gpr") @@ -56,6 +60,9 @@ _WEB_LIVE_SCHEMA = [ ("bscan_start_freq_mhz", "B-scan", ("bscan",), _attr("_bscan_start_freq_mhz")), ("bscan_stop_freq_mhz", "B-scan", ("bscan",), _attr("_bscan_stop_freq_mhz")), ("legacy_gpr_mode", "Mode", ("legacy_gpr",), _attr("_legacy_gpr_config_mode")), + ("relative_permittivity", "Geometry & medium", ("legacy_gpr",), _attr("_gpr_relative_permittivity")), + ("tx_geometry", "Geometry & medium", _GPR_LIKE_MODES, _attr("_gpr_tx_geometry_input")), + ("rx_geometry", "Geometry & medium", _GPR_LIKE_MODES, _attr("_gpr_rx_geometry_input")), ("gpr_input_positions", "Geometry & depth", _GPR_LIKE_MODES, _dual("_gpr_input_positions_input", {"legacy_gpr": "_legacy_gpr_input_positions_input", "sar3d": "_sar3d_input_positions_input"})), ("gpr_output_positions", "Geometry & depth", _GPR_LIKE_MODES, _dual("_gpr_output_positions_input", {"legacy_gpr": "_legacy_gpr_output_positions_input", "sar3d": "_sar3d_output_positions_input"})), ("gpr_min_depth_m", "Geometry & depth", _GPR_LIKE_MODES, _dual("_gpr_min_depth_m", {"legacy_gpr": "_legacy_gpr_min_depth_m", "sar3d": "_sar3d_min_depth_m"})), @@ -107,6 +114,7 @@ _NON_WIDGET_LIVE_FIELDS = frozenset({ "reprocess_current_result", "history_command", "history_command_seq", + "settings_revision", }) # Display toggles: GUI-only rendering choices (kept in the GUI profile, not the live @@ -118,21 +126,10 @@ _WEB_DISPLAY_SCHEMA = [ ("combo_filter", "Display", ("pass_through",), _attr("_pass_through_combo_filter_input")), ] -# Stable run_config fields (GPR medium and antenna geometry). They live in run_config and -# take effect only when the pipeline (re)starts — not hot-reloaded — so the web marks them -# "applies on Start" and editing them just updates the widget for the next start. -_WEB_STABLE_SCHEMA = [ - # Coherent BP fixes the medium to eps_r = 1 (Horns_motion_3libre.py), so relative - # permittivity is a legacy-GPR-only knob; BP ignores it. - ("relative_permittivity", "Geometry & medium", ("legacy_gpr",), _attr("_gpr_relative_permittivity")), - ("tx_geometry", "Geometry & medium", _GPR_LIKE_MODES, _attr("_gpr_tx_geometry_input")), - ("rx_geometry", "Geometry & medium", _GPR_LIKE_MODES, _attr("_gpr_rx_geometry_input")), -] - -# Every field the web form renders and may write back, across all three categories. +# Every field the web form renders and may write back. _WEB_FORM_GETTERS = { field: getter - for source in (_WEB_LIVE_SCHEMA, _WEB_DISPLAY_SCHEMA, _WEB_STABLE_SCHEMA) + for source in (_WEB_LIVE_SCHEMA, _WEB_DISPLAY_SCHEMA) for field, _group, _modes, getter in source } @@ -157,8 +154,9 @@ def _web_field_schema(field: str, group: str, widget) -> dict | None: return {**base, "kind": "float", "value": float(widget.value()), "min": float(widget.minimum()), "max": float(widget.maximum()), "step": float(widget.singleStep()), "decimals": int(widget.decimals())} - if isinstance(widget, QPlainTextEdit): - return {**base, "kind": "textarea", "value": widget.toPlainText()} + if isinstance(widget, AntennaGeometryEditor): + return {**base, "kind": "geometry", "value": widget.records(), + "position_field": widget.position_field, "position_count": widget.position_count} if isinstance(widget, QLineEdit): return {**base, "kind": "text", "value": widget.text()} return None @@ -180,48 +178,47 @@ def _read_live_widget_value(widget): return float(widget.value()) if isinstance(widget, QLineEdit): return widget.text() + if isinstance(widget, AntennaGeometryEditor): + return widget.records() raise TypeError(f"Unsupported live-settings widget: {type(widget).__name__}") def _build_web_settings_schema(window) -> list[dict]: """Build the settings schema for the active mode straight from the Qt widgets. - Covers all three widget categories — live settings, GUI display toggles, and stable - run_config fields — so the web form shows exactly what the desktop processing panel - does. Stable fields are tagged ``applies_on_start`` (they take effect on the next run). + Covers live settings and GUI display toggles, matching the desktop panel. """ mode = window._processing_mode.currentText() schema: list[dict] = [] - for source, extra in ( - (_WEB_LIVE_SCHEMA, {}), - (_WEB_DISPLAY_SCHEMA, {}), - (_WEB_STABLE_SCHEMA, {"applies_on_start": True}), - ): + for source in (_WEB_LIVE_SCHEMA, _WEB_DISPLAY_SCHEMA): for field, group, modes, getter in source: if modes is not None and mode not in modes: continue entry = _web_field_schema(field, group, getter(window)) if entry is not None: - schema.append({**entry, **extra}) + schema.append(entry) return schema -def _set_web_live_field(window, field: str, value) -> None: - """Write one web value into the desktop widget that feeds ``field`` (any category).""" - getter = _WEB_FORM_GETTERS.get(field) - if getter is None: - return - widget = getter(window) +def _set_widget_value(window, widget, value) -> None: + """Validate and write a web value, also used when rolling back a rejected batch.""" if isinstance(widget, QComboBox): + if widget.findText(str(value)) < 0: + raise ValueError(f"Unknown setting option: {value}") window._set_combo_current_text(widget, str(value)) elif isinstance(widget, QCheckBox): - widget.setChecked(bool(value)) - elif isinstance(widget, QSpinBox): - widget.setValue(int(float(value))) - elif isinstance(widget, QDoubleSpinBox): - widget.setValue(float(value)) - elif isinstance(widget, QPlainTextEdit): - widget.setPlainText(str(value)) + if not isinstance(value, bool): + raise ValueError("Checkbox value must be true or false") + widget.setChecked(value) + elif isinstance(widget, (QSpinBox, QDoubleSpinBox)): + numeric = float(value) + if not math.isfinite(numeric) or not widget.minimum() <= numeric <= widget.maximum(): + raise ValueError("Numeric setting is outside its allowed range") + if isinstance(widget, QSpinBox) and numeric != int(numeric): + raise ValueError("Setting requires an integer") + widget.setValue(int(numeric) if isinstance(widget, QSpinBox) else numeric) + elif isinstance(widget, AntennaGeometryEditor): + widget.set_records(value) elif isinstance(widget, QLineEdit): widget.setText(",".join(str(int(v)) for v in value) if isinstance(value, list) else str(value)) @@ -297,6 +294,7 @@ class AppWindowLiveProcessingMixin: reprocess_current_result=bool(reprocess_current_result), history_command=str(history_command), history_command_seq=int(self._history_command_seq), + settings_revision=getattr(self, "_processing_settings_revision", 0), **values, ) @@ -315,12 +313,15 @@ class AppWindowLiveProcessingMixin: """ if bump_history_seq: self._history_command_seq += 1 + self._processing_settings_revision = time.monotonic_ns() self._live_config_writer.write( self._live_processing_config( history_command=history_command, reprocess_current_result=not self._can_reprocess_history(), ) ) + if getattr(self, "_await_processing_revision", False): + self._required_processing_revision = self._processing_settings_revision def _apply_web_live_settings(self, fields: dict) -> None: """Apply web-requested live settings through the SAME path as a desktop edit. @@ -339,6 +340,10 @@ class AppWindowLiveProcessingMixin: """ try: history_command = str(fields.get("history_command", "none")) + if history_command not in {"none", "clear_all", "remove_last"}: + raise ValueError("Unknown history command") + if history_command != "none" and set(fields) != {"history_command"}: + raise ValueError("History commands must be sent separately from settings") if history_command == "clear_all": self._clear_all_runtime_history() return @@ -356,15 +361,30 @@ class AppWindowLiveProcessingMixin: self._log_debug(f"Applying web live settings: fields={sorted(settings)}.") self._suppress_live_settings_handler = True try: - # processor_mode first: dual-sourced gpr_* fields route to the gpr or - # legacy_gpr widget based on the active mode. - if "processor_mode" in settings: - _set_web_live_field(self, "processor_mode", settings.pop("processor_mode")) - for name, value in settings.items(): - _set_web_live_field(self, name, value) + with ExitStack() as blockers: + snapshots = [] + try: + # Set the mode under a blocker before resolving per-mode fields. + ordered = sorted(settings, key=lambda name: name != "processor_mode") + for name in ordered: + widget = _WEB_FORM_GETTERS[name](self) + blockers.enter_context(QSignalBlocker(widget)) + snapshots.append((widget, _read_live_widget_value(widget))) + _set_widget_value(self, widget, settings[name]) + self._validate_processing_mode_selection(self._processing_mode.currentText()) + except Exception: + for widget, value in reversed(snapshots): + _set_widget_value(self, widget, value) + raise finally: self._suppress_live_settings_handler = False - self._on_processing_live_settings_changed() + self._sync_socket_speed_controls() + if {"tx_geometry", "rx_geometry", "relative_permittivity"} & settings.keys(): + self._await_processing_revision = True + if "processor_mode" in settings: + self._on_processing_mode_changed(self._processing_mode.currentText()) + else: + self._on_processing_live_settings_changed() except Exception as exc: # noqa: BLE001 self._show_exception("Failed to apply web live settings", exc) @@ -372,6 +392,12 @@ class AppWindowLiveProcessingMixin: """Return the live-settings schema for the active mode (read from widgets).""" return _build_web_settings_schema(self) + def _on_gpr_geometry_changed(self) -> None: + if getattr(self, "_suppress_live_settings_handler", False): + return + self._await_processing_revision = True + self._on_processing_live_settings_changed() + def _on_processing_live_settings_changed(self, *_args) -> None: """Handle live-processing setting changes and trigger redraw when needed.""" # Web-apply sets many widgets at once; their individual change signals are @@ -399,11 +425,12 @@ class AppWindowLiveProcessingMixin: collection = latest if collection is None and self._result_history: collection = self._result_history[-1] - if collection is None or not self._draw_results(collection): - if current_mode == "sar3d": - self._clear_sar3d_plot() - else: - self._clear_gpr_plot() + if current_mode == "sar3d": + self._clear_sar3d_plot() + if collection is not None: + self._draw_results(collection) + elif collection is None or not self._draw_results(collection): + self._clear_gpr_plot() elif self._result_history: self._draw_results(self._result_history[-1]) else: @@ -589,9 +616,9 @@ class AppWindowLiveProcessingMixin: self._clear_gpr_plot() return if self._processing_mode.currentText() == "sar3d": - if self._result_history and self._draw_results(self._result_history[-1]): - return self._clear_sar3d_plot() + if self._result_history: + self._draw_results(self._result_history[-1]) return if self._result_history: self._draw_results(self._result_history[-1]) diff --git a/python_app/gui/controllers/app_window_config/profile_io_mixin.py b/python_app/gui/controllers/app_window_config/profile_io_mixin.py index 4de81ca..b3b3069 100644 --- a/python_app/gui/controllers/app_window_config/profile_io_mixin.py +++ b/python_app/gui/controllers/app_window_config/profile_io_mixin.py @@ -10,6 +10,8 @@ from pathlib import Path from PyQt6.QtCore import QSignalBlocker from PyQt6.QtWidgets import QFileDialog +from python_app.file_io import atomic_write_json +from python_app.gui.controllers.app_window_config.state_builders import GUI_SAVE_HISTORY_LIMIT from python_app.models.gui_profile_model import GuiProfileModel from python_app.models.run_config_model import RunConfigModel from python_app.orchestration.preprocess_assets import ( @@ -27,8 +29,7 @@ class AppWindowConfigProfileIOMixin: raise FileExistsError(f"Config profile output already exists: {output_path}") profile = self._build_gui_profile() - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(json.dumps(profile.to_dict(), indent=2), encoding="utf-8") + atomic_write_json(output_path, profile.to_dict(), durable=True) return profile def _set_combo_selection_mode(self, mode: str) -> None: @@ -42,6 +43,7 @@ class AppWindowConfigProfileIOMixin: self._single_combo_input.setEnabled(True) self._run_combos_select_button.setEnabled(False) self._single_combo_select_button.setEnabled(False) + self._next_combo_button.setEnabled(True) return text_selected = mode != "single" @@ -52,6 +54,32 @@ class AppWindowConfigProfileIOMixin: self._single_combo_input.setEnabled(not text_selected) self._run_combos_select_button.setEnabled(True) self._single_combo_select_button.setEnabled(True) + self._next_combo_button.setEnabled(not text_selected) + + def _select_display_combo(self, input_pos: int, output_pos: int) -> None: + """Select an available pair and redraw its cached B-scan immediately.""" + key = (int(input_pos), int(output_pos)) + if key not in self._available_display_combo_keys(): + raise ValueError(f"Unknown channel combination: {key[0]}:{key[1]}") + self._single_combo_input.setText(str(key[0])) + self._single_combo_output.setText(str(key[1])) + self._on_bscan_display_combo_changed() + + def _select_next_combo(self) -> None: + """Advance output first, then input, wrapping after the last configured pair.""" + available = self._available_display_combo_keys() + if not available: + return + try: + current = (int(self._single_combo_input.text()), int(self._single_combo_output.text())) + index = available.index(current) + 1 + except ValueError: + index = 0 + self._select_display_combo(*available[index % len(available)]) + + def _available_display_combo_keys(self) -> list[tuple[int, int]]: + """Return configured pairs ordered by input, then output for display navigation.""" + return sorted((combo.input, combo.output) for combo in self._defaults_config.build_runtime_combos()) def _is_matrix_radar_model_selected(self) -> bool: """Return whether the loaded config targets a matrix-mode radar acquisition.""" @@ -124,11 +152,10 @@ class AppWindowConfigProfileIOMixin: C++ replay window still follows ring capacities, since it bounds how much of the history the processor can re-publish coherently. """ - save_history_limit = self._save_history_limit_for_config(config) bscan_cpp_replay_window = self._cpp_bscan_replay_window_for_config(config) - self._raw_history = deque(self._raw_history, maxlen=save_history_limit) - self._pre_history = deque(self._pre_history, maxlen=save_history_limit) - self._result_history = deque(self._result_history, maxlen=save_history_limit) + self._raw_history = deque(self._raw_history, maxlen=GUI_SAVE_HISTORY_LIMIT) + self._pre_history = deque(self._pre_history, maxlen=GUI_SAVE_HISTORY_LIMIT) + self._result_history = deque(self._result_history, maxlen=GUI_SAVE_HISTORY_LIMIT) self._bscan_cpp_replay_window = bscan_cpp_replay_window self._clear_bscan_plot_history() @@ -351,7 +378,6 @@ class AppWindowConfigProfileIOMixin: self._sar3d_comp_range_power, self._sar3d_use_compensation, self._sar3d_normalize_each_frame, - self._save_count, self._record_count, self._save_path_input, self._save_name_input, @@ -457,18 +483,10 @@ class AppWindowConfigProfileIOMixin: self._bscan_history_window.setValue(int(gui_state.processing.bscan.history_window_scans)) self._gpr_relative_permittivity.setValue(float(config.gpr.relative_permittivity)) - self._gpr_tx_geometry_input.setPlainText( - "\n".join( - f"{int(entry.output_pos)} {float(entry.x_m):g}" - for entry in config.gpr.tx_geometry - ) - ) - self._gpr_rx_geometry_input.setPlainText( - "\n".join( - f"{int(entry.input_pos)} {float(entry.x_m):g}" - for entry in config.gpr.rx_geometry - ) - ) + self._gpr_tx_geometry_input.set_position_count(config.output_switch.positions) + self._gpr_rx_geometry_input.set_position_count(config.input_switch.positions) + self._gpr_tx_geometry_input.set_records(config.gpr.tx_geometry) + self._gpr_rx_geometry_input.set_records(config.gpr.rx_geometry) self._gpr_input_positions_input.setText(str(gui_state.processing.gpr.input_positions)) self._gpr_output_positions_input.setText(str(gui_state.processing.gpr.output_positions)) self._gpr_min_depth_m.setValue(float(gui_state.processing.gpr.min_depth_m)) @@ -555,7 +573,6 @@ class AppWindowConfigProfileIOMixin: self._sar3d_normalize_each_frame.setChecked(bool(sar3d_state.normalize_each_frame)) self._sync_socket_speed_controls() - self._save_count.setValue(int(gui_state.data_actions.save_count)) self._record_count.setValue(int(gui_state.data_actions.record_count)) self._save_path_input.setText(str(gui_state.data_actions.save_path)) self._save_name_input.setText(str(gui_state.data_actions.save_name)) @@ -590,6 +607,7 @@ class AppWindowConfigProfileIOMixin: self._apply_initial_radar_limits() if self._preprocess_dialog is not None: self._refresh_sets() + self._await_processing_revision = True self._on_processing_mode_changed(gui_state.processing.selected_mode) self._update_history_indicator() self._remember_active_profile_path(profile_path) diff --git a/python_app/gui/controllers/app_window_config/radar_limits_mixin.py b/python_app/gui/controllers/app_window_config/radar_limits_mixin.py index 8009a4f..27a3ba0 100644 --- a/python_app/gui/controllers/app_window_config/radar_limits_mixin.py +++ b/python_app/gui/controllers/app_window_config/radar_limits_mixin.py @@ -28,9 +28,6 @@ class AppWindowRadarLimitsMixin: else: radar_service = create_single_radar_service(config) - if isinstance(radar_service, LibreVnaService) and not radar_service.driver_available: - raise RuntimeError("LibreVNA Python driver is not available for device limits query") - limits = radar_service.read_device_limits() return self._apply_radar_limits_to_ui(limits) diff --git a/python_app/gui/controllers/app_window_config/state_builders.py b/python_app/gui/controllers/app_window_config/state_builders.py index b7ba641..7a2708d 100644 --- a/python_app/gui/controllers/app_window_config/state_builders.py +++ b/python_app/gui/controllers/app_window_config/state_builders.py @@ -81,52 +81,6 @@ class AppWindowConfigStateBuildersMixin: env[key] = value return env - @staticmethod - def _parse_geometry_line_coordinates(parts: list[str]) -> tuple[float, float, float]: - """Parse 1/2/3 trailing coordinate fields into (x, y, z); missing axes default to 0.""" - x_m = float(parts[0]) - y_m = float(parts[1]) if len(parts) >= 2 else 0.0 - z_m = float(parts[2]) if len(parts) >= 3 else 0.0 - return x_m, y_m, z_m - - @staticmethod - def _parse_gpr_tx_geometry_text(text: str) -> list[GprTxGeometryModel]: - """Parse line-based Tx geometry editor text: `output_pos x_m [y_m] [z_m]`.""" - entries: list[GprTxGeometryModel] = [] - for line_number, raw_line in enumerate(text.splitlines(), start=1): - line = raw_line.strip() - if not line: - continue - parts = line.split() - if len(parts) < 2 or len(parts) > 4: - raise ValueError( - f"Invalid Tx geometry line {line_number}: expected `output_pos x_m [y_m] [z_m]`" - ) - x_m, y_m, z_m = AppWindowConfigStateBuildersMixin._parse_geometry_line_coordinates(parts[1:]) - entries.append( - GprTxGeometryModel(output_pos=int(parts[0]), x_m=x_m, y_m=y_m, z_m=z_m) - ) - return entries - - @staticmethod - def _parse_gpr_rx_geometry_text(text: str) -> list[GprRxGeometryModel]: - """Parse line-based Rx geometry editor text: `input_pos x_m [y_m] [z_m]`.""" - entries: list[GprRxGeometryModel] = [] - for line_number, raw_line in enumerate(text.splitlines(), start=1): - line = raw_line.strip() - if not line: - continue - parts = line.split() - if len(parts) < 2 or len(parts) > 4: - raise ValueError( - f"Invalid Rx geometry line {line_number}: expected `input_pos x_m [y_m] [z_m]`" - ) - x_m, y_m, z_m = AppWindowConfigStateBuildersMixin._parse_geometry_line_coordinates(parts[1:]) - entries.append( - GprRxGeometryModel(input_pos=int(parts[0]), x_m=x_m, y_m=y_m, z_m=z_m) - ) - return entries - @staticmethod def _format_combos_text_from_config(config: RunConfigModel) -> str: """Render configured combos for UI text editor, keeping full matrix as empty.""" @@ -192,18 +146,6 @@ class AppWindowConfigStateBuildersMixin: ), ) - @staticmethod - def _save_history_limit_for_config(config: RunConfigModel) -> int: - """Return maxlen for GUI snapshot-save deques. - - Independent of ring capacities — see :data:`GUI_SAVE_HISTORY_LIMIT`. - The `config` argument is kept for symmetry with - :meth:`_cpp_bscan_replay_window_for_config` and possible future per-profile - overrides. - """ - del config - return GUI_SAVE_HISTORY_LIMIT - def _default_gui_state_for_config(self, config: RunConfigModel) -> GuiStateModel: """Build fallback GUI-only defaults for a stable run config.""" default_combo = config.combos[0] if config.combos else ComboModel(input=0, output=0) @@ -217,74 +159,17 @@ class AppWindowConfigStateBuildersMixin: single_output=str(int(default_combo.output)), ), processing=GuiProcessingStateModel( - selected_mode="pass_through", - pass_through=GuiPassThroughStateModel( - show_magnitude=True, - show_phase=True, - unwrap_phase=False, - combo_filter="", - fixed_y_enabled=False, - y_min_db=-100.0, - y_max_db=0.0, - ), - bscan=GuiBscanStateModel( - axis="abs", - cut_m=0.824, - max_depth_m=1.0, - gain=1.0, - start_freq_mhz=100.0, - stop_freq_mhz=8800.0, - subtract_mean_ascan=False, - ), gpr=GuiGprStateModel( input_positions=self._default_gpr_input_positions_from_config(config), output_positions=self._default_gpr_output_positions_from_config(config), - min_depth_m=2.0, - max_depth_m=14.0, - range_comp_power=0.1, - object_min_frac=0.7, - score_mode="combined", - motion_mode="int_minus", - look_angle_deg=0.0, - direction_sign=1.0, - speed_m_s=0.0, - ignore_socket_speed_enabled=False, - max_detected_objects_to_draw=5, - draw_top_m_objects=2, - object_approach_min_frames=3, - start_freq_mhz=3000.0, - stop_freq_mhz=6000.0, - background_subtract_enabled=True, - background_mean_count=10, - remove_sidelobe_objects_enabled=True, - imaging_plane_y_m=0.0, - render_mode="heatmap", visible_x_min_m=default_gpr_x_min_m, visible_x_max_m=default_gpr_x_max_m, - visible_z_min_m=0.0, - visible_z_max_m=14.0, ), legacy_gpr=GuiLegacyGprStateModel( input_positions=self._default_gpr_input_positions_from_config(config), output_positions=self._default_gpr_output_positions_from_config(config), - min_depth_m=2.0, - max_depth_m=14.0, - comp_power=0.2, - start_freq_mhz=3000.0, - stop_freq_mhz=6000.0, - speed_m_s=0.0, - ignore_socket_speed_enabled=False, - look_angle_deg=0.0, - snr_thresh=4.5, - snr_comp_max=25.0, - background_subtract_enabled=True, - background_mean_count=10, - render_mode="heatmap", - min_visible_pair_count=1, visible_x_min_m=default_gpr_x_min_m, visible_x_max_m=default_gpr_x_max_m, - visible_z_min_m=0.0, - visible_z_max_m=14.0, ), sar3d=GuiSar3dStateModel( input_positions=self._default_gpr_input_positions_from_config(config), @@ -292,15 +177,7 @@ class AppWindowConfigStateBuildersMixin: ), ), data_actions=GuiDataActionsStateModel( - save_count=10, save_path=str(self._project_root / "python_app/data/snapshots"), - save_name="snapshot_manual", - record_count=100, - ), - preprocess_dialog=GuiPreprocessDialogStateModel( - set_name="set_001", - radar_config_dir="", - use_all_radar_configs=False, ), ) @@ -442,7 +319,6 @@ class AppWindowConfigStateBuildersMixin: ), ), data_actions=GuiDataActionsStateModel( - save_count=int(self._save_count.value()), save_path=self._save_path_input.text().strip(), save_name=self._save_name_input.text().strip(), record_count=int(self._record_count.value()), @@ -502,8 +378,8 @@ class AppWindowConfigStateBuildersMixin: for key in VISIBLE_PREPROCESS_ASSET_KEYS: preprocess_asset_model(config, key).set_name = self._selected_preprocess_sets.get(key, "") config.gpr.relative_permittivity = float(self._gpr_relative_permittivity.value()) - config.gpr.tx_geometry = self._parse_gpr_tx_geometry_text(self._gpr_tx_geometry_input.toPlainText()) - config.gpr.rx_geometry = self._parse_gpr_rx_geometry_text(self._gpr_rx_geometry_input.toPlainText()) + config.gpr.tx_geometry = [GprTxGeometryModel(**entry) for entry in self._gpr_tx_geometry_input.records()] + config.gpr.rx_geometry = [GprRxGeometryModel(**entry) for entry in self._gpr_rx_geometry_input.records()] validate_gpr_model( config.gpr, input_switch_positions=config.input_switch.positions, diff --git a/python_app/gui/controllers/app_window_pipeline_mixin.py b/python_app/gui/controllers/app_window_pipeline_mixin.py index 052bc5c..7875858 100644 --- a/python_app/gui/controllers/app_window_pipeline_mixin.py +++ b/python_app/gui/controllers/app_window_pipeline_mixin.py @@ -7,6 +7,7 @@ import time from PyQt6.QtCore import QCoreApplication, QEventLoop from python_app.gui.runtime.constraints import validate_processing_mode_constraints +from python_app.gui.controllers.app_window_plot.sar3d_plot_mixin import sar3d_status from python_app.gui.runtime.history import ( build_processor_run_signature, build_run_history_signature, @@ -16,6 +17,7 @@ from python_app.hardware_full.kamil_adc import apply_kamil_adc_laser_control from python_app.hardware_full.single_radar_service import create_single_radar_service from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection from python_app.models.run_config_model import RunConfigModel +from python_app.models.run_config_validation import validate_acquisition_config from python_app.orchestration.preprocess_assets import ( PREPROCESS_ASSET_SPECS, REQUIRED_PREPROCESS_ASSET_KEYS, @@ -57,6 +59,11 @@ class AppWindowPipelineMixin: def _start_run(self, *, single_capture: bool = False) -> None: """Start pipeline processes and ring readers.""" + if getattr(self, "_closing", False) or getattr(self, "_close_after_stop", False): + return + if getattr(self, "_stopping_run", False): + self._show_error("Wait for pipeline shutdown before starting again") + return if self._capture_session is not None: self._show_error( "Cannot start pipeline during active capture sequence", @@ -76,15 +83,20 @@ class AppWindowPipelineMixin: return self._log("Stopping continuous pipeline before single capture") self._stop_run() + if getattr(self, "_closing", False): + return try: processor_was_running = self._supervisor.is_processor_running() config = self._build_config() + validate_acquisition_config(config) run_signature = self._build_run_history_signature(config) processor_signature = self._build_processor_run_signature(config) if self._processor_requires_restart(processor_signature): self._log("Restarting data_processor because stable run settings changed") self._stop_all_processes() + if getattr(self, "_closing", False): + return processor_was_running = False if not processor_was_running: self._reset_runtime_history() @@ -207,13 +219,17 @@ class AppWindowPipelineMixin: return was_running = self._supervisor.is_running() + resume_seq = getattr(self, "_stop_request_seq", 0) + 1 processor_only_running = self._supervisor.is_processor_running() and not was_running if was_running: self._stop_run() + if not self._pipeline_resume_allowed(resume_seq): + return try: config = self._build_config() - if config.radar.driver_mode == "native": + validate_acquisition_config(config) + if config.radar.driver_mode == "native" and not config.radar.replay.dataset_dir: self._refresh_radar_limits_from_device() processor_signature = self._build_processor_run_signature(config) if processor_only_running and self._processor_requires_restart(processor_signature): @@ -241,13 +257,14 @@ class AppWindowPipelineMixin: except Exception as exc: # noqa: BLE001 self._show_exception("Failed to apply radar settings", exc) finally: - if was_running: + if was_running and self._pipeline_resume_allowed(resume_seq): self._start_run() def _prepare_radar_for_native_acquisition(self, config: RunConfigModel) -> None: """Preconfigure native single-radar hardware using current sweep settings.""" - if config.radar.model == RunConfigModel.COMPACT_M_K209_MODEL and config.radar.driver_mode != "native": - raise RuntimeError("Compact-M K209 requires radar.driver_mode='native'") + if config.radar.replay.dataset_dir: + self._log("Radar pre-configuration skipped (dataset replay)") + return if config.radar.driver_mode != "native": self._log("Radar pre-configuration skipped (mock mode)") return @@ -265,9 +282,6 @@ class AppWindowPipelineMixin: return radar_service = create_single_radar_service(config) - if not getattr(radar_service, "driver_available", True): - raise RuntimeError("LibreVNA Python driver is not available for native pre-configuration") - try: radar_service.open() radar_service.configure(config.radar.sweep) @@ -278,49 +292,77 @@ class AppWindowPipelineMixin: def _stop_run(self) -> None: """Stop acquisition-side processes and close readers as needed.""" - self._pipeline_should_run = False # an explicit stop disables crash auto-restart - was_running = self._supervisor.is_running() - if was_running: - self._supervisor.stop_orchestrator() - self._drain_rings_until_quiet(timeout_s=0.35, poll_s=0.02) - self._supervisor.stop_preprocessor() - self._drain_rings_until_quiet(timeout_s=0.25, poll_s=0.02) - else: - self._supervisor.stop() - self._drain_rings_once_for_history() - # Flush any in-progress disk recording so its buffered (not-yet-full) chunk is - # written rather than lost; the drains above already captured the last data. - self._finalize_recording_on_stop() - keep_results_reader = self._supervisor.is_processor_running() - self._close_readers(keep_results=keep_results_reader) - self._single_capture_active = False - self._single_capture_start_ns = None - self._single_capture_seen_raw = False - self._single_capture_target_collection_id = None - self._update_history_indicator() - self._status_label.setText("Status: idle") - if was_running: - if keep_results_reader: - self._log("Acquisition stopped (data_processor kept running)") - else: - self._log("Pipeline stopped") + self._stop_request_seq = getattr(self, "_stop_request_seq", 0) + 1 + self._resume_pipeline_after_capture = False + self._stop_pipeline(keep_processor=True) def _stop_all_processes(self) -> None: """Stop all managed pipeline processes and close all readers.""" - self._pipeline_should_run = False # an explicit stop disables crash auto-restart - was_running = self._supervisor.is_running() or self._supervisor.is_processor_running() - self._supervisor.stop_all() - self._drain_rings_until_quiet(timeout_s=0.25, poll_s=0.02) - self._close_readers(keep_results=False) - self._single_capture_active = False - self._single_capture_start_ns = None - self._single_capture_seen_raw = False - self._single_capture_target_collection_id = None - self._processor_run_signature = None - self._update_history_indicator() - self._status_label.setText("Status: idle") - if was_running: - self._log("All pipeline processes stopped") + self._stop_pipeline(keep_processor=False) + + def _stop_pipeline(self, *, keep_processor: bool) -> None: + """Share shutdown/drain/finalization, protecting event-loop re-entry.""" + self._pipeline_should_run = False + if getattr(self, "_stopping_run", False): + self._stop_all_requested |= not keep_processor + return + self._stopping_run = True + self._stop_all_requested = not keep_processor + try: + drain_error = None + try: + was_running = self._supervisor.is_running() + if keep_processor and was_running: + self._supervisor.stop_orchestrator() + self._drain_rings_until_quiet(timeout_s=0.35, poll_s=0.02) + self._supervisor.stop_preprocessor() + self._drain_rings_until_quiet(timeout_s=0.25, poll_s=0.02) + elif keep_processor: + self._supervisor.stop() + self._drain_rings_once_for_history() + if self._stop_all_requested: + self._supervisor.stop_all() + self._drain_rings_until_quiet(timeout_s=0.25, poll_s=0.02) + except Exception as exc: # noqa: BLE001 - corrupt data must not prevent Stop + drain_error = exc + self._stop_request_seq = getattr(self, "_stop_request_seq", 0) + 1 + if self._stop_all_requested: + self._supervisor.stop_all() + else: + self._supervisor.stop() + if self._stop_all_requested: + self._processor_run_signature = None + self._finalize_recording_on_stop() + keep_results = not self._stop_all_requested and self._supervisor.is_processor_running() + result_ring_name = getattr(self._result_reader, "_ring_name", None) if drain_error else None + self._close_readers(keep_results=keep_results and drain_error is None) + self._single_capture_active = False + self._single_capture_start_ns = None + self._single_capture_seen_raw = False + self._single_capture_target_collection_id = None + self._update_history_indicator() + self._status_label.setText("Status: idle") + self._log("Acquisition stopped (data_processor kept running)" if keep_results else "Pipeline stopped") + if drain_error is not None: + self._status_label.setText("Status: error") + self._show_exception("Pipeline stopped, but final ring data could not be read", drain_error) + if keep_results and result_ring_name: + self._result_reader = ShmRingReader(result_ring_name) + except Exception as exc: # noqa: BLE001 - this is also a direct Qt button slot + self._stop_request_seq = getattr(self, "_stop_request_seq", 0) + 1 + self._status_label.setText("Status: error") + self._show_exception("Failed to stop pipeline", exc) + finally: + self._stopping_run = False + if getattr(self, "_close_after_stop", False): + self._close_after_stop = False + self.close() + + def _pipeline_resume_allowed(self, stop_seq: int) -> bool: + """A Stop or Close received during a temporary pause cancels its restart.""" + return not getattr(self, "_closing", False) and not getattr(self, "_close_after_stop", False) and ( + getattr(self, "_stop_request_seq", 0) == stop_seq + ) def _close_readers(self, *, keep_results: bool = False) -> None: """Close active ring readers; keep the results reader when ``keep_results``.""" @@ -342,6 +384,8 @@ class AppWindowPipelineMixin: def _poll_rings(self) -> None: """Poll readers, ingest history, and trigger rendering.""" + if getattr(self, "_closing", False) or getattr(self, "_stopping_run", False): + return self._web_update_snapshot() # guarded internally; never raises self._handle_process_exit_reports() @@ -366,12 +410,10 @@ class AppWindowPipelineMixin: # crash-storm budget (it only caps consecutive crash-restarts). self._pipeline_restart_count = 0 render_started_ns = time.monotonic_ns() - self._draw_preferred_collection(result_latest=result_latest) + self._draw_results(result_latest) self._pipeline_metrics.record( "rendering", time.monotonic_ns() - render_started_ns ) - else: - self._draw_preferred_collection(result_latest=None) except Exception as exc: # noqa: BLE001 self._handle_reader_poll_error(exc) @@ -395,7 +437,12 @@ class AppWindowPipelineMixin: self._log_exception("Failed to collect process exit reports", exc, level="ERROR") return - if unexpected and getattr(self, "_pipeline_should_run", False): + if getattr(self, "_pipeline_should_run", False) and ( + unexpected + or not {"data_processor", "data_preprocessor", "sweep_orchestrator"}.issubset(self._supervisor.pids()) + ): + # Exit reports are consumed once. A missing child must keep triggering + # recovery after a back-off delay or a failed start with no reports left. self._recover_pipeline_after_crash() def _recover_pipeline_after_crash(self) -> None: @@ -441,7 +488,7 @@ class AppWindowPipelineMixin: self._drop_pending_ring_payloads(include_results=True) self._status_label.setText("Status: running") self._log("Pipeline auto-restarted after crash.") - except Exception as exc: # noqa: BLE001 - retry on the next crash signal + except Exception as exc: # noqa: BLE001 - retry on subsequent polls self._log_exception("Pipeline auto-restart failed; will retry", exc, level="ERROR") def _check_single_capture_deadline(self) -> None: @@ -529,8 +576,15 @@ class AppWindowPipelineMixin: return False self._draw_results(target_result) - self._log("Single capture completed") + status = sar3d_status(target_result) if self._processing_mode.currentText() == "sar3d" else None self._stop_run() + if status is not None and status[0] in (1, 2, 3): + state = {1: "no signal in selected range", 2: "error", 3: "warning"}[status[0]] + self._status_label.setText(f"Status: {state}") + log = self._log_error if status[0] == 2 else self._log_warning + log(f"Single capture: SAR3D {state}: {status[1]}") + else: + self._log("Single capture completed") return True def _find_single_capture_target_result(self) -> ResultCollection | None: @@ -551,12 +605,12 @@ class AppWindowPipelineMixin: return None - def _read_all_raw(self) -> SweepCollection | None: + def _read_all_raw(self, *, max_collections: int | None = None) -> SweepCollection | None: """Read available raw collections from raw ring.""" if self._raw_reader is None: raise RuntimeError("Raw ring reader is not initialised") latest: SweepCollection | None = None - for _ in range(self._max_pop_per_poll): + for _ in range(self._max_pop_per_poll if max_collections is None else max_collections): collection = self._raw_reader.pop_raw_collection() if collection is None: break @@ -575,24 +629,27 @@ class AppWindowPipelineMixin: self._single_capture_target_collection_id = collection.collection_id return latest - def _read_all_preprocessed(self) -> None: + def _read_all_preprocessed(self, *, max_collections: int | None = None) -> SweepCollection | None: """Read available preprocessed collections from preprocessed ring.""" if self._pre_reader is None: return - for _ in range(self._max_pop_per_poll): + latest: SweepCollection | None = None + for _ in range(self._max_pop_per_poll if max_collections is None else max_collections): collection = self._pre_reader.pop_preprocessed_collection() if collection is None: break self._pre_history.append(collection) self._record_collection("preprocessed", collection) + latest = collection + return latest - def _read_all_results(self) -> ResultCollection | None: + def _read_all_results(self, *, max_collections: int | None = None) -> ResultCollection | None: """Read available result collections from results ring.""" if self._result_reader is None: raise RuntimeError("Result ring reader is not initialised") latest: ResultCollection | None = None - for _ in range(self._max_pop_per_poll): + for _ in range(self._max_pop_per_poll if max_collections is None else max_collections): collection = self._result_reader.pop_result_collection() if collection is None: break @@ -627,36 +684,19 @@ class AppWindowPipelineMixin: # residual in tiny slices so we neither busy-spin nor block too long. time.sleep(min(0.002, max(0.0, deadline - time.monotonic()))) - def _drain_rings_once_for_history(self) -> None: + def _drain_rings_once_for_history(self) -> bool: """Perform one non-blocking read pass to extend histories.""" - if self._raw_reader is not None: - self._read_all_raw() - self._read_all_preprocessed() - if self._result_reader is not None: - self._read_all_results() + raw = self._read_all_raw() if self._raw_reader is not None else None + preprocessed = self._read_all_preprocessed() + result = self._read_all_results() if self._result_reader is not None else None + return raw is not None or preprocessed is not None or result is not None def _drain_rings_until_quiet(self, *, timeout_s: float, poll_s: float) -> None: - """Drain rings until history sizes stabilize or timeout expires.""" + """Drain rings until readers are quiet or timeout expires, even at full history capacity.""" deadline = time.monotonic() + timeout_s stable_rounds = 0 - previous = ( - len(self._raw_history), - len(self._pre_history), - len(self._result_history), - ) - while time.monotonic() < deadline and stable_rounds < 2: - self._drain_rings_once_for_history() - current = ( - len(self._raw_history), - len(self._pre_history), - len(self._result_history), - ) - if current == previous: - stable_rounds += 1 - else: - stable_rounds = 0 - previous = current + stable_rounds = 0 if self._drain_rings_once_for_history() else stable_rounds + 1 # Event-loop-friendly wait so timers/signals keep firing during drain. self._pump_events_during_drain(poll_s) @@ -734,4 +774,3 @@ class AppWindowPipelineMixin: config, self._live_processing_config(), ) - diff --git a/python_app/gui/controllers/app_window_plot/bscan_plot_mixin.py b/python_app/gui/controllers/app_window_plot/bscan_plot_mixin.py index 01dbb08..77cfc05 100644 --- a/python_app/gui/controllers/app_window_plot/bscan_plot_mixin.py +++ b/python_app/gui/controllers/app_window_plot/bscan_plot_mixin.py @@ -198,7 +198,7 @@ class AppWindowBscanPlotMixin: def _draw_bscan_heatmap(self, _collection: ResultCollection) -> bool: """Draw B-scan image rebuilt from processed result history.""" - self._disable_phase_axis() + self._trace_phase_plot.clear() self._sync_bscan_history_from_results() return self._draw_bscan_heatmap_from_history() @@ -256,8 +256,8 @@ class AppWindowBscanPlotMixin: image_item.setRect(QRectF(x_min, depth_min, sweep_width, depth_span)) axis_mode = self._bscan_axis.currentText() - image_item.setLookupTable(self._bscan_lookup_table(axis_mode)) - image_item.setLevels(self._bscan_levels(sweeps, axis_mode)) + image_item.setLookupTable(bscan_lookup_table(axis_mode)) + image_item.setLevels(bscan_levels(sweeps, axis_mode)) self._bscan_plot.clear() self._configure_bscan_plot_axes() @@ -463,53 +463,8 @@ class AppWindowBscanPlotMixin: except ValueError: return None - def _bscan_lookup_table(self, axis_mode: str) -> np.ndarray: - """Return lookup table for current B-scan axis mode.""" - return bscan_lookup_table(axis_mode) - - @staticmethod - def _build_lut(stops: list[str], *, size: int = 256) -> np.ndarray: - """Backward-compatible wrapper around LUT builder.""" - return build_lut(stops, size=size) - - @staticmethod - def _bscan_levels(sweeps: np.ndarray, axis_mode: str) -> tuple[float, float]: - """Return display levels for B-scan image.""" - return bscan_levels(sweeps, axis_mode) - def _clear_bscan_plot_history(self) -> None: """Drop cached B-scan history and invalidate cache signature.""" self._bscan_history_by_combo.clear() self._bscan_depth_axis_by_combo.clear() self._bscan_render_signature = None - - def _ensure_phase_view_box(self) -> pg.ViewBox: - """Create or return secondary right-axis ViewBox for phase curves.""" - plot_item = self._bscan_plot.getPlotItem() - phase_view_box = self._phase_viewbox - if phase_view_box is None: - phase_view_box = pg.ViewBox() - self._phase_viewbox = phase_view_box - plot_item.scene().addItem(phase_view_box) - plot_item.getAxis("right").linkToView(phase_view_box) - phase_view_box.setXLink(plot_item.vb) - plot_item.vb.sigResized.connect(self._update_phase_view_box_geometry) - self._update_phase_view_box_geometry() - return phase_view_box - - def _update_phase_view_box_geometry(self) -> None: - """Keep right-axis ViewBox geometry in sync with main plot ViewBox.""" - phase_view_box = self._phase_viewbox - if phase_view_box is None: - return - plot_item = self._bscan_plot.getPlotItem() - phase_view_box.setGeometry(plot_item.vb.sceneBoundingRect()) - phase_view_box.linkedViewChanged(plot_item.vb, phase_view_box.XAxis) - - def _clear_phase_overlay(self) -> None: - """Remove all phase curves from secondary ViewBox.""" - self._trace_phase_plot.clear() - - def _disable_phase_axis(self) -> None: - """Hide right axis and clear phase overlay when phase is not rendered.""" - self._clear_phase_overlay() diff --git a/python_app/gui/controllers/app_window_plot/gpr_plot_mixin.py b/python_app/gui/controllers/app_window_plot/gpr_plot_mixin.py index 6947feb..e59c5ba 100644 --- a/python_app/gui/controllers/app_window_plot/gpr_plot_mixin.py +++ b/python_app/gui/controllers/app_window_plot/gpr_plot_mixin.py @@ -6,6 +6,7 @@ from PyQt6.QtCore import QRectF import numpy as np import pyqtgraph as pg +from python_app.gui.controllers.app_window_plot.bscan_plot_mixin import build_lut from python_app.models.dataset_model import ResultCollection from python_app.orchestration.gpr_locator import ( collection_payload_by_name as gpr_collection_payload_by_name, @@ -95,7 +96,7 @@ class AppWindowGprPlotMixin: self._configure_gpr_plot_axes() if self._gpr_lookup_table is None: - self._gpr_lookup_table = self._build_lut(["#081c15", "#1b4332", "#ffd166", "#f94144"]) + self._gpr_lookup_table = build_lut(["#081c15", "#1b4332", "#ffd166", "#f94144"]) self._gpr_image_item = pg.ImageItem(axisOrder="row-major") self._gpr_image_item.setZValue(0) @@ -179,29 +180,29 @@ class AppWindowGprPlotMixin: output_positions_text = self._gpr_output_positions_input.text() requested_inputs = tuple(self._parse_csv_int_list(input_positions_text)) requested_outputs = tuple(self._parse_csv_int_list(output_positions_text)) + tx_entries = self._gpr_tx_geometry_input.records() + rx_entries = self._gpr_rx_geometry_input.records() signature = ( - self._gpr_tx_geometry_input.toPlainText(), - self._gpr_rx_geometry_input.toPlainText(), + tuple(tuple(entry.values()) for entry in tx_entries), + tuple(tuple(entry.values()) for entry in rx_entries), requested_inputs, requested_outputs, ) if signature == self._gpr_geometry_signature and self._gpr_selected_geometry is not None: return self._gpr_selected_geometry - tx_entries = self._parse_gpr_tx_geometry_text(signature[0]) - rx_entries = self._parse_gpr_rx_geometry_text(signature[1]) requested_input_set = set(requested_inputs) requested_output_set = set(requested_outputs) - rx_entries = sorted(rx_entries, key=lambda entry: int(entry.input_pos)) - tx_entries = sorted(tx_entries, key=lambda entry: int(entry.output_pos)) + rx_entries = sorted(rx_entries, key=lambda entry: entry["input_pos"]) + tx_entries = sorted(tx_entries, key=lambda entry: entry["output_pos"]) if requested_input_set: - rx_entries = [entry for entry in rx_entries if int(entry.input_pos) in requested_input_set] + rx_entries = [entry for entry in rx_entries if entry["input_pos"] in requested_input_set] if requested_output_set: - tx_entries = [entry for entry in tx_entries if int(entry.output_pos) in requested_output_set] + tx_entries = [entry for entry in tx_entries if entry["output_pos"] in requested_output_set] - x_tx = np.asarray([float(entry.x_m) for entry in tx_entries], dtype=np.float32) - x_rx = np.asarray([float(entry.x_m) for entry in rx_entries], dtype=np.float32) + x_tx = np.asarray([entry["x_m"] for entry in tx_entries], dtype=np.float32) + x_rx = np.asarray([entry["x_m"] for entry in rx_entries], dtype=np.float32) self._gpr_geometry_signature = signature self._gpr_selected_geometry = (x_tx, x_rx) return self._gpr_selected_geometry diff --git a/python_app/gui/controllers/app_window_plot/sar3d_plot_mixin.py b/python_app/gui/controllers/app_window_plot/sar3d_plot_mixin.py index 35ab550..fb22b5d 100644 --- a/python_app/gui/controllers/app_window_plot/sar3d_plot_mixin.py +++ b/python_app/gui/controllers/app_window_plot/sar3d_plot_mixin.py @@ -1,14 +1,17 @@ """3D SAR views of the volume published by the sar3d processor. Scene axes: X across, Y height above the ground (Y = 0 is the ground), Z along -the ground away from the radar. Payloads, the volume normalized to a peak of 1: +the ground away from the radar. Payloads share normalization by the maximum +away from the Y boundaries: - ``sar3d_front``, ``sar3d_side``, ``sar3d_top``: maximum projections onto X-Y (looking ahead from the radar), Z-Y (from the side) and X-Z (from above); - ``sar3d_oblique``: orthographic projection along the camera of the notebook's 3D scene; ``sar3d_oblique_basis`` (2x3) gives its image axes u and v as scene directions, which places the scene box, the ground and the maximum on it; -- ``sar3d_peak``: [level, X, Y, Z] of the volume maximum before normalization. +- ``sar3d_peak``: [level, X, Y, Z] of that maximum before normalization. +- ``sar3d_status``: [code, UTF-8 bytes...] with ready CUDA (0), no signal (1), + processing error (2), or CUDA unavailable (3), including the explanation. """ from __future__ import annotations @@ -22,7 +25,7 @@ from PyQt6.QtCore import QRectF, Qt from python_app.gui.controllers.app_window_plot.bscan_plot_mixin import build_lut from python_app.models.dataset_model import ResultCollection -# Colours start at this fraction of the maximum; weaker voxels stay transparent, +# Colours start at this fraction of the normalization reference; weaker voxels stay transparent, # like those under the threshold of the notebook's point cloud. DISPLAY_THRESHOLD = 0.75 @@ -31,7 +34,7 @@ SAR3D_VIEWS: dict[str, tuple[str, str, str, tuple[int, int] | None]] = { "sar3d_front": ("Front (X-Y)", "X", "Y", (0, 1)), "sar3d_side": ("Side (Z-Y)", "Z", "Y", (2, 1)), "sar3d_top": ("Top (X-Z)", "X", "Z", (0, 2)), - "sar3d_oblique": ("3D", "", "", None), + "sar3d_oblique": ("3D projection", "", "", None), } # Corners of the scene box indexed by bits (x, y, z); an edge joins corners one bit apart. @@ -40,6 +43,22 @@ _BOX_EDGES = [(a, b) for a, b in itertools.combinations(range(8), 2) if bin(a ^ _AXIS_EDGES = {"X": (0, 1), "Y": (1, 3), "Z": (1, 5)} +def sar3d_status(collection: ResultCollection) -> tuple[int, str] | None: + """Decode processor status without letting malformed telemetry break a Qt slot.""" + payload = next((item for item in collection.collection_payloads + if item.processing_name == "sar3d_status" and item.kind == 4), None) + if payload is None: + return None + table = payload.table + if table.ndim != 2 or table.shape[0] != 1 or table.shape[1] == 0: + return 2, "Invalid SAR3D status payload" + row = table[0] + if (not np.isfinite(row).all() or not np.equal(row, np.floor(row)).all() + or row[0] not in (0, 1, 2, 3) or np.any((row[1:] < 0) | (row[1:] > 255))): + return 2, "Invalid SAR3D status payload" + return int(row[0]), bytes(row[1:].astype(np.uint8)).decode("utf-8", errors="replace") + + def _image_rect(x_axis: np.ndarray, y_axis: np.ndarray) -> QRectF: """Rectangle covering the pixels centred on the axis samples.""" dx = (x_axis[-1] - x_axis[0]) / max(len(x_axis) - 1, 1) @@ -97,7 +116,7 @@ class AppWindowSar3dPlotMixin: oblique.addItem(label) self._sar3d_axis_labels[axis_name] = label - def _clear_sar3d_plot(self) -> None: + def _clear_sar3d_plot(self, message: str = "SAR3D: waiting for measurements") -> None: """Clear all four 3D SAR views.""" for name, (title, *_rest) in SAR3D_VIEWS.items(): self._sar3d_images[name].hide() @@ -105,13 +124,21 @@ class AppWindowSar3dPlotMixin: self._sar3d_plots[name].setTitle(title) self._sar3d_box.setData([], []) self._sar3d_ground.setData([], []) + for label in self._sar3d_axis_labels.values(): + label.hide() + self._sar3d_status_label.setText(message) def _draw_sar3d_projections(self, collection: ResultCollection) -> bool: - """Draw the 3D SAR views of a result collection; return success.""" + """Draw four maximum projections of the volume; return success.""" + status = sar3d_status(collection) + if status is not None and status[0] in (1, 2, 3): + state = {1: "no signal", 2: "error", 3: "warning"}[status[0]] + self._clear_sar3d_plot(f"SAR3D: {state} — {status[1]}") + return False payloads = {name: self._collection_payload_by_name(collection, name, kind=3) for name in SAR3D_VIEWS} basis = self._collection_payload_by_name(collection, "sar3d_oblique_basis", kind=4) if basis is None or any(payload is None or payload.image.size == 0 for payload in payloads.values()): - self._clear_sar3d_plot() + self._clear_sar3d_plot("SAR3D: waiting for projection data") return False peak = self._collection_payload_by_name(collection, "sar3d_peak", kind=4) level, *peak_xyz = peak.table[0] if peak is not None else (None,) @@ -147,12 +174,14 @@ class AppWindowSar3dPlotMixin: for axis_name, (start, end) in _AXIS_EDGES.items(): middle = (projected[start] + projected[end]) / 2 self._sar3d_axis_labels[axis_name].setPos(*(middle + 0.08 * (middle - centre))) # just outside + self._sar3d_axis_labels[axis_name].show() (x0, x1), _, (z0, z1) = bounds ground = (np.array([[x0, 0.0, z0], [x1, 0.0, z0], [x1, 0.0, z1], [x0, 0.0, z1], [x0, 0.0, z0]]) @ basis.table.T).T self._sar3d_ground.setData(ground[0], ground[1]) if level is not None: x, y, z = peak_xyz self._sar3d_plots["sar3d_oblique"].setTitle( - f"3D | max X {x:+.2f}, Y {y:+.2f}, Z {z:.2f} m | level {level:.3g}" + f"3D projection | max X {x:+.2f}, Y {y:+.2f}, Z {z:.2f} m | level {level:.3g}" ) + self._sar3d_status_label.setText(f"SAR3D: ready — {status[1]}" if status else "SAR3D: ready") return True diff --git a/python_app/gui/controllers/app_window_plot_mixin.py b/python_app/gui/controllers/app_window_plot_mixin.py index 94f0832..13e21c5 100644 --- a/python_app/gui/controllers/app_window_plot_mixin.py +++ b/python_app/gui/controllers/app_window_plot_mixin.py @@ -21,18 +21,19 @@ class AppWindowPlotMixin( ): """Routes plotting to trace, B-scan, GPR, or 3D SAR-specific mixins.""" - def _draw_preferred_collection( - self, - *, - result_latest: ResultCollection | None, - ) -> None: - """Draw latest available result collection if present.""" - if result_latest is None: - return - self._draw_results(result_latest) - def _draw_results(self, collection: ResultCollection) -> bool: """Draw collection based on currently selected processing mode.""" + expected_revision = getattr(self, "_required_processing_revision", None) + if expected_revision is not None and self._is_gpr_like_processing_mode(self._processing_mode.currentText()): + revision = next((payload for payload in collection.collection_payloads + if payload.processing_name == "processing_revision" and payload.kind == 4), None) + if revision is None or revision.table.shape != (1, 4): + return False + if any(not 0 <= word <= 0xffff or word != int(word) for word in revision.table[0]): + return False + actual_revision = sum(int(word) << (index * 16) for index, word in enumerate(revision.table[0])) + if actual_revision != expected_revision: + return False if self._processing_mode.currentText() == "bscan": return self._draw_bscan_heatmap(collection) if self._processing_mode.currentText() in {"gpr", "legacy_gpr"}: @@ -42,11 +43,11 @@ class AppWindowPlotMixin( return self._draw_trace_lines(collection) def _result_collection_has_trace(self, collection: ResultCollection) -> bool: - """Return `True` when collection contains at least one trace payload.""" - if collection.collection_payloads: + """Recognize processed output, including an empty trace after a valid depth/frequency cut.""" + if any(payload.processing_name != "processing_revision" for payload in collection.collection_payloads): return True for block in collection.blocks: for payload in block.payloads: - if payload.kind == 1 and payload.trace.size > 0: + if payload.kind == 1: return True return False diff --git a/python_app/gui/controllers/app_window_preprocess_mixin.py b/python_app/gui/controllers/app_window_preprocess_mixin.py index e4b9d42..2929738 100644 --- a/python_app/gui/controllers/app_window_preprocess_mixin.py +++ b/python_app/gui/controllers/app_window_preprocess_mixin.py @@ -122,6 +122,8 @@ class AppWindowPreprocessMixin: def _capture_tmp_reference(self) -> None: """Capture, save, and select a temporary S21 reference with current sweep settings.""" + if getattr(self, "_closing", False) or getattr(self, "_stopping_run", False): + return if self._capture_session is not None: self._show_error( "Cannot capture tmp reference during active capture sequence", @@ -138,12 +140,15 @@ class AppWindowPreprocessMixin: pipeline_was_running = self._supervisor.is_running() pipeline_was_paused = False + resume_seq = getattr(self, "_stop_request_seq", 0) + 1 try: if pipeline_was_running: self._log("Pipeline paused for tmp reference capture") self._stop_run() pipeline_was_paused = True + if not self._pipeline_resume_allowed(resume_seq): + return config = self._build_config() radar_key = self._radar_key(config) @@ -194,7 +199,7 @@ class AppWindowPreprocessMixin: except Exception as exc: # noqa: BLE001 self._show_exception("Failed to capture tmp reference", exc) finally: - if pipeline_was_paused: + if pipeline_was_paused and self._pipeline_resume_allowed(resume_seq): self._start_run() self._control_button_busy = False @@ -333,6 +338,8 @@ class AppWindowPreprocessMixin: def _start_capture_sequence(self, kind: str) -> None: """Start sequential capture session for requested preprocess asset.""" + if getattr(self, "_closing", False) or getattr(self, "_stopping_run", False): + return if self._capture_session is not None: self._show_error("Another capture sequence is already active", details=self._capture_state_details()) return @@ -344,9 +351,12 @@ class AppWindowPreprocessMixin: return was_running = self._supervisor.is_running() + resume_seq = getattr(self, "_stop_request_seq", 0) + 1 if was_running: self._log("Pipeline paused for exclusive hardware capture") self._stop_run() + if not self._pipeline_resume_allowed(resume_seq): + return self._resume_pipeline_after_capture = was_running @@ -398,6 +408,8 @@ class AppWindowPreprocessMixin: def _create_neutral_sets(self) -> None: """Save neutral S21 calibration/reference sets for the current radar settings.""" + if getattr(self, "_closing", False) or getattr(self, "_stopping_run", False): + return if self._capture_session is not None: self._show_error( "Cannot create neutral sets during active capture sequence", @@ -412,11 +424,12 @@ class AppWindowPreprocessMixin: return pipeline_was_paused = False + resume_seq = getattr(self, "_stop_request_seq", 0) + 1 try: config = self._build_config() if not supports_neutral_preprocess_sets(config): self._show_error( - "Neutral S21 sets are available only for kamil_adc and librevna_multi" + "Neutral S21 sets are not supported for this radar configuration" ) return @@ -435,11 +448,14 @@ class AppWindowPreprocessMixin: self._log("Pipeline paused for neutral-set creation") self._stop_run() pipeline_was_paused = True + if not self._pipeline_resume_allowed(resume_seq): + return calibration, reference = build_neutral_s21_sets(config) point_count = int(calibration.traces[0].frequency_hz.size) - self._store.save_set("s21_calibration", radar_key, set_name, calibration) - self._store.save_set("s21_reference", radar_key, set_name, reference) + self._store.save_sets(radar_key, set_name, { + "s21_calibration": calibration, "s21_reference": reference, + }) self._selected_preprocess_sets["s21_calibration"] = set_name self._selected_preprocess_sets["s21_reference"] = set_name @@ -458,7 +474,7 @@ class AppWindowPreprocessMixin: except Exception as exc: # noqa: BLE001 self._show_exception("Failed to create neutral S21 sets", exc) finally: - if pipeline_was_paused: + if pipeline_was_paused and self._pipeline_resume_allowed(resume_seq): self._start_run() def _build_single_radar_capture_session( @@ -556,19 +572,24 @@ class AppWindowPreprocessMixin: display_name = preprocess_asset_display_name(session.kind) dialog = self._ensure_preprocess_dialog() + stop_seq = getattr(self, "_stop_request_seq", 0) dialog.set_status(f"{display_name} batch capture started") self._log( f"{display_name} batch capture started: remaining=" f"{session.state().total_count - session.state().captured_count}" ) try: - while not session.is_complete(): + while self._capture_session is session and self._pipeline_resume_allowed(stop_seq) and not session.is_complete(): try: capture_result = session.capture_current_combo() except Exception as exc: # noqa: BLE001 self._on_capture_combo_failed(session, exc) return self._record_preprocess_capture(session, capture_result) + if not session.is_complete(): + self._pump_events_during_drain(0.01) + if self._capture_session is session and not session.is_complete(): + dialog.set_status(f"{display_name} batch capture paused; previous captures kept") finally: self._end_preprocess_capture() @@ -578,7 +599,7 @@ class AppWindowPreprocessMixin: Returns False for a duplicate request (e.g. a click delivered while an error dialog inside a capture pumps the event loop). """ - if self._preprocess_capture_busy: + if getattr(self, "_closing", False) or self._preprocess_capture_busy: self._log("Preprocess combo capture already in progress; ignoring duplicate request.") return False self._preprocess_capture_busy = True diff --git a/python_app/gui/controllers/app_window_recording_mixin.py b/python_app/gui/controllers/app_window_recording_mixin.py index 232b668..246e7fa 100644 --- a/python_app/gui/controllers/app_window_recording_mixin.py +++ b/python_app/gui/controllers/app_window_recording_mixin.py @@ -11,11 +11,11 @@ Key properties: and freed, and at most :data:`_RECORDING_QUEUE_CHUNKS` chunks are ever in flight, so RAM does not grow with the recording length. * **Off the GUI thread.** A single background writer thread does all disk I/O; the GUI - poll tick only hands it chunks. The writes never stall acquisition or the web UI. If + poll tick only hands it chunks. If the disk genuinely cannot keep up, the bounded queue applies brief backpressure rather than growing memory without limit. -* **Aligned at capture.** Each result is paired with its raw/preprocessed collection by - ``collection_id`` as it arrives, so no cross-stage buffering is needed. +* **Complete received stages.** Raw, preprocessed and result collections are saved + independently with their original timestamps, including sweeps without a result. * **Only new sweeps.** Collections produced before the arm instant are ignored (gated on ``monotonic_ns``), so a pre-existing ring backlog is not recorded. * **No double-arm.** Arming is refused while a recording is still in progress. @@ -29,6 +29,7 @@ from contextlib import suppress import logging from pathlib import Path import queue +import shutil import threading import time @@ -39,9 +40,6 @@ _RECORDING_CHUNK_SIZE = 200 # Max chunks queued for the writer thread before the producer applies backpressure; # bounds in-flight memory to roughly this many chunks of measurements. _RECORDING_QUEUE_CHUNKS = 8 -# Safety bound on raw/preprocessed collections still waiting for their result (normally -# a handful). If the processor stalls, the oldest are dropped so the maps stay bounded. -_RECORDING_MAX_PENDING = 512 class AppWindowRecordingMixin: @@ -54,13 +52,12 @@ class AppWindowRecordingMixin: self._recording_target = 0 self._recording_since_ns = 0 self._recording_enqueued = 0 + self._recording_received_count = 0 self._recording_writer = None self._recording_queue: queue.Queue | None = None self._recording_thread: threading.Thread | None = None self._recording_stop = threading.Event() self._recording_write_error: str | None = None - self._recording_pending_raw: dict[int, object] = {} - self._recording_pending_pre: dict[int, object] = {} self._recording_chunk_raw: list = [] self._recording_chunk_pre: list = [] self._recording_chunk_results: list = [] @@ -82,6 +79,9 @@ class AppWindowRecordingMixin: return target = int(self._record_count.value()) + if target <= 0: + self._show_error("Recording count must be greater than zero") + return destination = self._snapshot_destination_dir() if destination.exists(): @@ -91,6 +91,7 @@ class AppWindowRecordingMixin: ) return + writer = None try: writer = self._store.create_snapshot_stream( Path(self._save_path_input.text().strip()).expanduser(), @@ -102,6 +103,9 @@ class AppWindowRecordingMixin: self._snapshot_config_profile_path(writer.directory), allow_overwrite=False ) except Exception as exc: # noqa: BLE001 + if writer is not None: + with suppress(OSError): + shutil.rmtree(writer.directory) self._show_exception("Failed to start disk recording", exc) return @@ -109,26 +113,41 @@ class AppWindowRecordingMixin: self._recording_target = target self._recording_since_ns = time.monotonic_ns() self._recording_enqueued = 0 + self._recording_received_count = 0 self._recording_write_error = None - self._recording_pending_raw = {} - self._recording_pending_pre = {} self._recording_chunk_raw = [] self._recording_chunk_pre = [] self._recording_chunk_results = [] self._recording_queue = queue.Queue(maxsize=_RECORDING_QUEUE_CHUNKS) self._recording_stop = threading.Event() - self._recording_thread = threading.Thread( - target=self._recording_writer_loop, - args=(writer, self._recording_queue, self._recording_stop), - name="disk-recorder", - daemon=True, - ) - self._recording_thread.start() + try: + self._recording_thread = threading.Thread( + target=self._recording_writer_loop, + args=(writer, self._recording_queue, self._recording_stop), + name="disk-recorder", + daemon=False, + ) + self._recording_thread.start() + except Exception as exc: # noqa: BLE001 - thread resource exhaustion must reach the operator + self._init_recording_state() + with suppress(OSError): + shutil.rmtree(writer.directory) + self._show_exception("Failed to start disk recording", exc) + return self._recording_active = True self._recording_collecting = True if not self._supervisor.is_running(): self._start_run() + if not self._supervisor.is_running(): + # Startup can fail after producing data. Flush it, and release the + # destination only when no collection was ever accepted. + self._finalize_recording_on_stop() + if not self._recording_active and not self._recording_received_count: + with suppress(OSError): + shutil.rmtree(writer.directory) + self._log("Disk recording canceled because acquisition did not start") + return self._log(f"Disk recording armed: streaming the next {target} measurement(s) to {writer.directory}") @@ -139,7 +158,7 @@ class AppWindowRecordingMixin: ) -> None: """Write queued chunks to disk until a sentinel, a stop request, or an error. - Runs on a daemon thread; touches only the writer (file I/O) and logging — never + Runs on a worker thread; touches only the writer (file I/O) and logging — never Qt. A write failure is recorded for the GUI thread to surface and ends the loop. """ while not stop.is_set(): @@ -160,75 +179,62 @@ class AppWindowRecordingMixin: # -- capture (called from the ring-read hooks on the poll tick) ----------- def _record_collection(self, stage: str, collection) -> None: - """Stream one arriving collection while recording; hand chunks to the writer thread. - - ``stage`` is ``"raw"``, ``"preprocessed"`` or ``"results"``. Raw/preprocessed - collections are held by ``collection_id`` until their result arrives; the result - pairs them and appends an aligned measurement to the current chunk. - """ + """Save each received stage, retaining raw/preprocessed sweeps without a result.""" if not self._recording_collecting: return - produced_ns = int(getattr(collection, "monotonic_ns", 0) or 0) if produced_ns and produced_ns < self._recording_since_ns: return if stage == "raw": - self._stash_pending(self._recording_pending_raw, collection) + self._recording_chunk_raw.append(collection) elif stage == "preprocessed": - self._stash_pending(self._recording_pending_pre, collection) + self._recording_chunk_pre.append(collection) elif stage == "results": - self._record_result(collection) - - @staticmethod - def _stash_pending(pending: dict, collection) -> None: - """Hold a raw/preprocessed collection by id until its result arrives (bounded).""" - pending[int(collection.collection_id)] = collection - while len(pending) > _RECORDING_MAX_PENDING: - pending.pop(next(iter(pending))) # drop oldest; its result never came - - def _record_result(self, result) -> None: - """Pair a result with its raw/preprocessed, buffer it, and flush/finish as needed.""" - if self._recording_collected_count() >= self._recording_target: - return # already have the full target accepted - - collection_id = int(result.collection_id) - raw = self._recording_pending_raw.pop(collection_id, None) - preprocessed = self._recording_pending_pre.pop(collection_id, None) - if raw is not None: - self._recording_chunk_raw.append(raw) - if preprocessed is not None: - self._recording_chunk_pre.append(preprocessed) - self._recording_chunk_results.append(result) - - if len(self._recording_chunk_results) >= _RECORDING_CHUNK_SIZE: - if not self._flush_recording_chunk(): + if self._recording_collected_count() >= self._recording_target: return + self._recording_chunk_results.append(collection) + else: + raise ValueError(f"Unknown recording stage: {stage}") + self._recording_received_count += 1 + + chunk_size = max(len(self._recording_chunk_raw), len(self._recording_chunk_pre), len(self._recording_chunk_results)) + if chunk_size >= _RECORDING_CHUNK_SIZE and not self._flush_recording_chunk(): + return if self._recording_collected_count() >= self._recording_target: - if not self._flush_recording_chunk(): - return - self._stop_collecting() + if self._flush_recording_chunk(): + self._stop_collecting() + + def _enqueue_recording_work(self, work) -> bool: + """Apply bounded backpressure while checking that the consumer remains healthy.""" + while self._recording_thread is not None: + if self._recording_write_error is not None or not self._recording_thread.is_alive(): + self._abort_recording_after_write_error() + return False + try: + self._recording_queue.put(work, timeout=0.1) + return True + except queue.Full: + continue + return False def _flush_recording_chunk(self) -> bool: - """Hand the buffered chunk to the writer thread. Returns ``False`` if it aborted.""" - if not self._recording_chunk_results: + """Queue all received stages, including a chunk containing no results yet.""" + if not (self._recording_chunk_raw or self._recording_chunk_pre or self._recording_chunk_results): return True - if self._recording_write_error is not None or not self._recording_thread.is_alive(): - self._abort_recording_after_write_error() # never block on a dead consumer - return False chunk = (self._recording_chunk_raw, self._recording_chunk_pre, self._recording_chunk_results) + if not self._enqueue_recording_work(chunk): + return False self._recording_enqueued += len(self._recording_chunk_results) self._recording_chunk_raw = [] self._recording_chunk_pre = [] self._recording_chunk_results = [] - self._recording_queue.put(chunk) # brief backpressure only if the disk lags return True def _stop_collecting(self) -> None: - """Stop accepting measurements and tell the writer to drain and exit.""" + """Queue an exit sentinel after every accepted chunk, including a full queue.""" self._recording_collecting = False - with suppress(Exception): - self._recording_queue.put_nowait(None) # sentinel after the last chunk + self._enqueue_recording_work(None) # -- completion / teardown (GUI thread) ----------------------------------- @@ -249,35 +255,19 @@ class AppWindowRecordingMixin: self._log(f"Disk recording complete: {written} measurement(s) written to {directory}") def _finalize_recording_on_stop(self) -> None: - """Flush the partial chunk and finish the recording when the run is stopped. - - Pressing Stop mid-recording writes everything collected so far — including a - not-yet-full chunk — to disk and ends the recording, so no buffered measurement - is lost. The dataset then holds exactly what was seen before Stop. Called from - ``_stop_run`` after its ring drains, so the last in-flight measurements are - already captured. The writer is joined briefly (bounded, like the stop drains) - so completion is deterministic without waiting on a later poll tick. - """ - if not self._recording_active or not self._recording_collecting: + """Flush accepted data on Stop and retain the writer until its queue is drained.""" + if not self._recording_active: return - if not self._flush_recording_chunk(): - return # writer already failed and was surfaced/reset - self._stop_collecting() - + if self._recording_collecting: + if not self._flush_recording_chunk(): + return + self._stop_collecting() thread = self._recording_thread - directory = self._recording_writer.directory if self._recording_writer is not None else "?" if thread is not None: thread.join(timeout=3.0) - write_error = self._recording_write_error - written = self._recording_enqueued - self._reset_recording_state() - if write_error is not None: - self._show_error( - "Disk recording stopped with a write error", - details=f"dataset={directory}\nerror={write_error}", - ) - else: - self._log(f"Disk recording stopped: {written} measurement(s) written to {directory}") + self._poll_recording_writer() + if self._recording_active: + self._log("Disk recording is finishing queued writes; the dataset is not complete yet") def _abort_recording_after_write_error(self) -> None: """Surface a writer-thread failure on the GUI thread and disarm.""" @@ -302,21 +292,17 @@ class AppWindowRecordingMixin: self._recording_thread = None self._recording_write_error = None self._recording_enqueued = 0 - self._recording_pending_raw = {} - self._recording_pending_pre = {} self._recording_chunk_raw = [] self._recording_chunk_pre = [] self._recording_chunk_results = [] def _shutdown_recording(self) -> None: - """Stop the writer thread on app teardown, briefly joining so it is not orphaned.""" - thread = self._recording_thread - self._recording_stop.set() - if self._recording_queue is not None: - with suppress(Exception): - self._recording_queue.put_nowait(None) - if thread is not None and thread.is_alive(): - thread.join(timeout=2.0) + """Flush on close; a non-daemon writer also drains after the GUI event loop exits.""" + self._finalize_recording_on_stop() + if self._recording_active: + # Do not reset a live writer: setting its stop flag would discard its queue. + # Python waits for this non-daemon thread during process exit. + return self._reset_recording_state() # -- status --------------------------------------------------------------- diff --git a/python_app/gui/controllers/app_window_snapshot_mixin.py b/python_app/gui/controllers/app_window_snapshot_mixin.py index de7e876..99bc723 100644 --- a/python_app/gui/controllers/app_window_snapshot_mixin.py +++ b/python_app/gui/controllers/app_window_snapshot_mixin.py @@ -6,7 +6,8 @@ from pathlib import Path import time from PyQt6.QtWidgets import QFileDialog -from python_app.gui.runtime.history import record_result_history, remove_last_aligned_histories +from python_app.gui.controllers.app_window_config.state_builders import GUI_SAVE_HISTORY_LIMIT +from python_app.gui.runtime.history import remove_last_aligned_histories from python_app.storage.npz.paths import radar_config_filename_prefix @@ -39,8 +40,10 @@ class AppWindowSnapshotMixin: return output_dir / "config_profile.json" def _save_snapshot(self) -> None: - """Save the last-N runtime measurements as a numpy snapshot (the "Save Dataset" button).""" + """Save up to 1000 recent runtime measurements as a numpy snapshot.""" self._drain_runtime_rings_for_snapshot() + if getattr(self, "_closing", False): + return if not self._raw_history and not self._pre_history and not self._result_history: self._show_error("No runtime data is available for save", details=self._runtime_history_details()) @@ -50,7 +53,8 @@ class AppWindowSnapshotMixin: list(self._raw_history), list(self._pre_history), list(self._result_history), - int(self._save_count.value()), + GUI_SAVE_HISTORY_LIMIT, + include_unaligned=True, ) def _snapshot_destination_dir(self) -> Path: @@ -72,10 +76,12 @@ class AppWindowSnapshotMixin: preprocessed_history: list, result_history: list, last_n: int, + *, + include_unaligned: bool = False, ) -> None: - """Save the given aligned histories as a numpy snapshot + adjacent config profile. + """Save the given histories as a numpy snapshot + adjacent config profile. - Shared by the manual "Save Dataset" button (which passes the runtime history) + Shared by the manual "Save Last 1000" button (which passes the runtime history) and the on-the-fly disk recorder (which passes the freshly recorded buffers), so both produce byte-identical dataset layouts and identical error reporting. """ @@ -90,6 +96,7 @@ class AppWindowSnapshotMixin: result_history, last_n, name_prefix=self._radar_config_name_prefix(), + include_unaligned=include_unaligned, ) config_profile_path = self._snapshot_config_profile_path(snapshot_dir) try: @@ -124,13 +131,15 @@ class AppWindowSnapshotMixin: def _save_vna_history_json(self) -> None: """Save one VNA-history JSON per available combo in runtime history.""" self._drain_runtime_rings_for_snapshot() + if getattr(self, "_closing", False): + return if not self._raw_history and not self._pre_history and not self._result_history: self._show_error("No runtime data is available for save", details=self._runtime_history_details()) return try: - last_n = int(self._save_count.value()) + last_n = GUI_SAVE_HISTORY_LIMIT channel = "s21" output_root = Path(self._save_path_input.text().strip()).expanduser() output_name = self._save_name_input.text().strip() @@ -144,6 +153,7 @@ class AppWindowSnapshotMixin: channel=channel, primary_stage="preprocessed", name_prefix=self._radar_config_name_prefix(), + include_unaligned=True, ) output_stem = str(summary.get("output_stem", "")).strip() if not output_stem: @@ -210,6 +220,7 @@ class AppWindowSnapshotMixin: return resume_acquisition = self._supervisor.is_running() + resume_seq = getattr(self, "_stop_request_seq", 0) + 1 dropped_raw = 0 dropped_pre = 0 dropped_results = 0 @@ -220,6 +231,8 @@ class AppWindowSnapshotMixin: try: if resume_acquisition: self._stop_run() + if not self._pipeline_resume_allowed(resume_seq): + return dropped_raw = self._raw_reader.drop_all() if self._raw_reader is not None else 0 dropped_pre = self._pre_reader.drop_all() if self._pre_reader is not None else 0 @@ -260,7 +273,7 @@ class AppWindowSnapshotMixin: f"results={dropped_results}" ) - if resume_acquisition: + if resume_acquisition and self._pipeline_resume_allowed(resume_seq): self._start_run() except Exception as exc: # noqa: BLE001 self._show_exception(f"Failed to {error_action}", exc) @@ -280,64 +293,28 @@ class AppWindowSnapshotMixin: if self._raw_reader is None and self._pre_reader is None and self._result_reader is None: return try: - start_raw_count = len(self._raw_history) - start_pre_count = len(self._pre_history) - start_result_count = len(self._result_history) + readers = (self._raw_reader, self._pre_reader, self._result_reader) + observed = [False, False, False] deadline = time.monotonic() + 1.2 - while True: - progress = False + while time.monotonic() < deadline and not getattr(self, "_closing", False): + limit = self._max_pop_per_snapshot_drain + raw = self._read_all_raw(max_collections=limit) if self._raw_reader is not None else None + preprocessed = self._read_all_preprocessed(max_collections=limit) + result = self._read_all_results(max_collections=limit) if self._result_reader is not None else None + latest = (raw, preprocessed, result) + observed = [seen or collection is not None for seen, collection in zip(observed, latest)] - if self._raw_reader is not None: - for _ in range(self._max_pop_per_snapshot_drain): - collection = self._raw_reader.pop_raw_collection() - if collection is None: - break - self._raw_history.append(collection) - progress = True - - if self._pre_reader is not None: - for _ in range(self._max_pop_per_snapshot_drain): - collection = self._pre_reader.pop_preprocessed_collection() - if collection is None: - break - self._pre_history.append(collection) - progress = True - - if self._result_reader is not None: - for _ in range(self._max_pop_per_snapshot_drain): - collection = self._result_reader.pop_result_collection() - if collection is None: - break - record_result_history(self._result_history, collection) - progress = True - - if progress: + if any(collection is not None for collection in latest): continue - raw_count = len(self._raw_history) - pre_count = len(self._pre_history) - result_count = len(self._result_history) - - missing_raw = self._raw_reader is not None and raw_count == start_raw_count - missing_pre = self._pre_reader is not None and pre_count == start_pre_count - missing_results = self._result_reader is not None and result_count == start_result_count - got_results = result_count > start_result_count - got_raw_or_pre = raw_count > start_raw_count or pre_count > start_pre_count - # Event-loop-friendly waits so the keepalive/watchdog timers and # queued signals keep firing instead of freezing the headless daemon. - if got_results and (missing_raw or missing_pre) and time.monotonic() < deadline: - self._pump_events_during_drain(0.01) - continue - if got_raw_or_pre and missing_results and time.monotonic() < deadline: - self._pump_events_during_drain(0.01) - continue - if ( - self._result_reader is not None - and max(raw_count, pre_count) > result_count - and time.monotonic() < deadline - ): + missing_stage = any(reader is not None and not seen for reader, seen in zip(readers, observed)) + awaiting_results = self._result_reader is not None and ( + max(len(self._raw_history), len(self._pre_history)) > len(self._result_history) + ) + if (any(observed) and missing_stage) or awaiting_results: self._pump_events_during_drain(0.01) continue break diff --git a/python_app/gui/controllers/app_window_ui_mixin.py b/python_app/gui/controllers/app_window_ui_mixin.py index d7d2665..ad3baa4 100644 --- a/python_app/gui/controllers/app_window_ui_mixin.py +++ b/python_app/gui/controllers/app_window_ui_mixin.py @@ -10,6 +10,7 @@ Layout is intentionally split into two independent plot surfaces: from __future__ import annotations +from PyQt6.QtCore import Qt from PyQt6.QtWidgets import ( QComboBox, QFrame, @@ -145,13 +146,22 @@ class AppWindowUiMixin: def _build_sar3d_plot_page(self) -> None: """Create the 3D SAR page: front, side, top and 3D views in a 2x2 grid.""" self._sar3d_plots_container = QWidget() - layout = QGridLayout(self._sar3d_plots_container) + layout = QVBoxLayout(self._sar3d_plots_container) layout.setContentsMargins(0, 0, 0, 0) layout.setSpacing(6) + self._sar3d_status_label = QLabel() + self._sar3d_status_label.setTextFormat(Qt.TextFormat.PlainText) + self._sar3d_status_label.setWordWrap(True) + layout.addWidget(self._sar3d_status_label) + self._sar3d_plot_grid = QWidget() + grid = QGridLayout(self._sar3d_plot_grid) + grid.setContentsMargins(0, 0, 0, 0) + grid.setSpacing(6) + layout.addWidget(self._sar3d_plot_grid, stretch=1) self._sar3d_plots = {} for index, name in enumerate(SAR3D_VIEWS): self._sar3d_plots[name] = self._create_plot_widget(background="#0f141c") - layout.addWidget(self._sar3d_plots[name], index // 2, index % 2) + grid.addWidget(self._sar3d_plots[name], index // 2, index % 2) self._init_sar3d_views() self._clear_sar3d_plot() self._plot_stack.addWidget(self._sar3d_plots_container) diff --git a/python_app/gui/controllers/app_window_web_mixin.py b/python_app/gui/controllers/app_window_web_mixin.py index 5bbb6fc..c088ec6 100644 --- a/python_app/gui/controllers/app_window_web_mixin.py +++ b/python_app/gui/controllers/app_window_web_mixin.py @@ -44,17 +44,42 @@ class _WebActionCall: a request/response so failures reach the browser. """ - __slots__ = ("done", "error") + __slots__ = ("done", "error", "_lock", "_started") def __init__(self) -> None: self.done = threading.Event() self.error: str | None = None + self._lock = threading.Lock() + self._started = False + + def begin(self) -> bool: + with self._lock: + if self.done.is_set() or self._started: + return False + self._started = True + return True + + def finish(self, error: str | None = None) -> None: + with self._lock: + if not self.done.is_set(): + self.error = error + self.done.set() + + def cancel(self, error: str, *, queued_only: bool = True) -> bool: + with self._lock: + if self.done.is_set(): + return True + if queued_only and self._started: + return False + self.error = error + self.done.set() + return True # Headless has no shown window, so give the offscreen window a usable size for the # grabbed plot. In GUI mode the user's real (shown) window size is used as-is. _HEADLESS_PLOT_SIZE = (1600, 900) # Grab/encode the plot at most this often (the plot only changes per result/redraw). _GRAB_INTERVAL_S = 0.2 -# Field names a web client may apply: the form's live/display/stable fields plus the +# Field names a web client may apply: the form's live/display fields plus the # history command. Derived from the single schema so it can never drift from the form. _LIVE_FIELD_NAMES = web_apply_field_names() @@ -89,8 +114,7 @@ class AppWindowWebController(QObject): """ # Control signals carry a trailing _WebActionCall the GUI slot fills in, so the web - # thread can block on the real outcome. apply_settings stays fire-and-forget: it is - # validated up front and returns the live schema, not a pass/fail. + # thread can block on the real outcome and receive the updated state. start_requested = pyqtSignal(object) stop_requested = pyqtSignal(object) remove_last_requested = pyqtSignal(object) @@ -99,7 +123,9 @@ class AppWindowWebController(QObject): start_recording_requested = pyqtSignal(str, str, int, object) load_config_requested = pyqtSignal(str, object) save_dataset_requested = pyqtSignal(str, str, object) - apply_settings_requested = pyqtSignal(dict) + select_combo_requested = pyqtSignal(int, int, object) + next_combo_requested = pyqtSignal(object) + apply_settings_requested = pyqtSignal(dict, object) def __init__(self, run_configs_dir: Path, parent: QObject | None = None) -> None: super().__init__(parent) @@ -107,10 +133,13 @@ class AppWindowWebController(QObject): self._status: dict = {} self._live_settings: list = [] self._frame: dict | None = None + self._pending_lock = threading.Lock() + self._pending_calls: set[_WebActionCall] = set() + self._closed = False # -- Snapshot refresh (Qt main thread) ----------------------------------- - def update_snapshot(self, *, status: dict, live_settings: dict, frame: dict | None) -> None: + def update_snapshot(self, *, status: dict, live_settings: list, frame: dict | None) -> None: """Replace the served snapshots; ``frame`` only when a new one was grabbed.""" self._status = status self._live_settings = live_settings @@ -143,11 +172,28 @@ class AppWindowWebController(QObject): action surfaced an error or did not finish within the timeout. """ call = _WebActionCall() - signal.emit(*args, call) - if not call.done.wait(_WEB_ACTION_TIMEOUT_S): - raise WebActionError("The desktop did not complete the action in time") - if call.error is not None: - raise WebActionError(call.error) + with self._pending_lock: + if self._closed: + raise WebActionError("The application is closing") + self._pending_calls.add(call) + try: + signal.emit(*args, call) + if not call.done.wait(_WEB_ACTION_TIMEOUT_S): + if not call.cancel("The desktop did not start the action in time; the request was cancelled"): + raise WebActionError("The desktop action is still running; check its status before retrying") + if call.error is not None: + raise WebActionError(call.error) + finally: + with self._pending_lock: + self._pending_calls.discard(call) + + def shutdown(self) -> None: + """Release waiting HTTP workers and reject queued controls during teardown.""" + with self._pending_lock: + self._closed = True + pending = tuple(self._pending_calls) + for call in pending: + call.cancel("The application is closing", queued_only=False) def start(self) -> None: self._dispatch(self.start_requested) @@ -166,13 +212,15 @@ class AppWindowWebController(QObject): def start_recording(self, path: str, name: str, count: int) -> None: """Arm a run + disk recording of the next ``count`` measurements (the desktop button).""" - self._dispatch(self.start_recording_requested, path, name, int(count)) + if type(count) is not int or count < 1: + raise ValueError("Recording count must be a positive integer") + self._dispatch(self.start_recording_requested, path, name, count) - def apply_live_settings(self, fields: dict) -> dict: + def apply_live_settings(self, fields: dict) -> list: unknown = set(fields) - _LIVE_FIELD_NAMES if unknown: raise ValueError(f"Unknown live-settings fields: {', '.join(sorted(unknown))}") - self.apply_settings_requested.emit(dict(fields)) + self._dispatch(self.apply_settings_requested, dict(fields)) return self.current_live_settings() def load_config(self, name: str) -> None: @@ -187,9 +235,19 @@ class AppWindowWebController(QObject): self._dispatch(self.load_config_requested, name) def save_dataset(self, path: str, name: str) -> None: - """Save the runtime dataset to ``path``/``name`` (the desktop "Save Dataset" button).""" + """Save the last 1000 runtime measurements to ``path``/``name``.""" self._dispatch(self.save_dataset_requested, path, name) + def select_combo(self, input_pos: int, output_pos: int) -> None: + """Select the matrix B-scan channel through the desktop display controls.""" + if any(type(pos) is not int or pos < 0 for pos in (input_pos, output_pos)): + raise ValueError("Channel positions must be nonnegative integers") + self._dispatch(self.select_combo_requested, input_pos, output_pos) + + def next_combo(self) -> None: + """Advance the matrix B-scan display to the next configured channel pair.""" + self._dispatch(self.next_combo_requested) + class AppWindowWebMixin: """Start/stop the embedded web server and feed it the live plot + settings.""" @@ -245,7 +303,15 @@ class AppWindowWebMixin: controller.save_dataset_requested.connect( lambda path, name, call: self._run_web_action(call, self._save_web_dataset, path, name) ) - controller.apply_settings_requested.connect(self._apply_web_live_settings) + controller.select_combo_requested.connect( + lambda inp, out, call: self._run_web_action(call, self._select_display_combo, inp, out) + ) + controller.next_combo_requested.connect( + lambda call: self._run_web_action(call, self._select_next_combo) + ) + controller.apply_settings_requested.connect( + lambda fields, call: self._run_web_action(call, self._apply_web_live_settings, fields) + ) self._web_controller = controller self._web_update_snapshot() # seed snapshots before the first request @@ -268,17 +334,24 @@ class AppWindowWebMixin: modal error dialog pumping the event loop in GUI mode — saves and restores the capture slot, so each action only sees its own first error. """ + if getattr(self, "_closing", False): + call.cancel("The application is closing", queued_only=False) + return + if not call.begin(): + return previous_capture = self._web_action_error_capture capture: list[str] = [] self._web_action_error_capture = capture + error = None try: action(*args) - call.error = capture[0] if capture else None + error = capture[0] if capture else None + self._web_update_snapshot() except Exception as exc: # noqa: BLE001 - handlers self-report; this is a backstop - call.error = self._exception_summary(exc) + error = self._exception_summary(exc) finally: self._web_action_error_capture = previous_capture - call.done.set() + call.finish(error) def _load_web_config(self, name: str) -> None: """Load a run config chosen in the browser through the shared desktop load path. @@ -289,12 +362,11 @@ class AppWindowWebMixin: """ config_path = _safe_run_config_path(self._run_configs_dir, name) if config_path is None: - self._log_warning(f"Ignored web request to load unknown run config: {name}") - return + raise ValueError(f"Unknown run config: {name}") self._load_config_from_path(config_path) def _save_web_dataset(self, path: str, name: str) -> None: - """Save the dataset from the browser via the same handler as the desktop button. + """Save the last 1000 measurements via the shared desktop handler. The web fields are remote editors of the shared save-path/name widgets: the name is mirrored as-is (blank is valid — it yields a timestamped file), while a blank @@ -313,11 +385,15 @@ class AppWindowWebMixin: path keeps the configured destination), the record count is applied to the shared spinbox, then the unchanged desktop arming action runs. """ + if type(count) is not int or not self._record_count.minimum() <= count <= self._record_count.maximum(): + raise ValueError( + f"Recording count must be an integer from {self._record_count.minimum()} " + f"to {self._record_count.maximum()}" + ) if path.strip(): self._save_path_input.setText(path) self._save_name_input.setText(name) - if count >= 1: - self._record_count.setValue(min(count, self._record_count.maximum())) + self._record_count.setValue(count) self._start_run_with_recording() def _web_update_snapshot(self) -> None: @@ -349,6 +425,16 @@ class AppWindowWebMixin: "raw_count": len(self._raw_history), "preprocessed_count": len(self._pre_history), "result_count": len(self._result_history), + "sar3d_status": self._sar3d_status_label.text() + if self._processing_mode.currentText() == "sar3d" else "", + "display_combo": { + "enabled": self._is_matrix_radar_model_selected() + and self._processing_mode.currentText() == "bscan", + "input": self._single_combo_input.text(), + "output": self._single_combo_output.text(), + "options": self._available_display_combo_keys() + if self._is_matrix_radar_model_selected() else [], + }, }, live_settings=self._web_settings_schema(), frame=self._web_grab_frame_if_due(), @@ -377,8 +463,10 @@ class AppWindowWebMixin: } def _grab_plot_png_b64(self) -> str | None: - """Render the currently-visible plot page to a base64 PNG (exactly as shown).""" + """Render the active plots; SAR3D status is sent separately as web text.""" widget = self._plot_stack.currentWidget() + if widget is self._sar3d_plots_container: + widget = self._sar3d_plot_grid if widget is None or widget.width() <= 0 or widget.height() <= 0: return None pixmap = widget.grab() @@ -391,6 +479,9 @@ class AppWindowWebMixin: def _shutdown_web_ui(self) -> None: """Stop the server (and its broadcaster) during teardown.""" + controller = getattr(self, "_web_controller", None) + if controller is not None: + controller.shutdown() server = getattr(self, "_web_server", None) if server is not None: with contextlib.suppress(Exception): diff --git a/python_app/gui/controllers/sections/data_actions_section.py b/python_app/gui/controllers/sections/data_actions_section.py index fdb49a7..4f459c7 100644 --- a/python_app/gui/controllers/sections/data_actions_section.py +++ b/python_app/gui/controllers/sections/data_actions_section.py @@ -22,7 +22,8 @@ def build_data_actions_group(owner) -> QGroupBox: layout.setSpacing(8) data_defaults = owner._gui_defaults.data_actions - save_button = QPushButton("Save Dataset") + save_button = QPushButton("Save Last 1000") + save_button.setToolTip("Save up to 1000 most recent measurements with all available data stages.") save_button.clicked.connect(owner._save_snapshot) save_button.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) save_vna_json_button = QPushButton("Save S21 JSON") @@ -43,10 +44,6 @@ def build_data_actions_group(owner) -> QGroupBox: ) record_button.clicked.connect(owner._start_run_with_recording) record_button.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) - owner._save_count = QSpinBox() - owner._save_count.setMinimum(1) - owner._save_count.setMaximum(10_000) - owner._save_count.setValue(int(data_defaults.save_count)) owner._record_count = QSpinBox() owner._record_count.setMinimum(1) owner._record_count.setMaximum(1_000_000) @@ -64,13 +61,6 @@ def build_data_actions_group(owner) -> QGroupBox: button_grid.setColumnStretch(1, 1) layout.addLayout(button_grid) - count_row = QHBoxLayout() - count_row.setSpacing(8) - count_row.addWidget(QLabel("Number of Measurements to Save")) - count_row.addWidget(owner._save_count) - count_row.addStretch(1) - layout.addLayout(count_row) - record_count_row = QHBoxLayout() record_count_row.setSpacing(8) record_count_row.addWidget(QLabel("Number of Sweeps to Record")) diff --git a/python_app/gui/controllers/sections/processing_section.py b/python_app/gui/controllers/sections/processing_section.py index 29b0689..12bbbb8 100644 --- a/python_app/gui/controllers/sections/processing_section.py +++ b/python_app/gui/controllers/sections/processing_section.py @@ -8,9 +8,7 @@ from PyQt6.QtWidgets import ( QDoubleSpinBox, QFormLayout, QGroupBox, - QLabel, QLineEdit, - QPlainTextEdit, QSizePolicy, QSpinBox, QStackedWidget, @@ -20,6 +18,7 @@ from PyQt6.QtWidgets import ( from python_app.gui.controllers.app_window_config.state_builders import GUI_SAVE_HISTORY_LIMIT from python_app.gui.controllers.sections.layout_helpers import FormRow, build_two_column_form_widget +from python_app.gui.antenna_geometry_editor import AntennaGeometryEditor # Same semantics on every GPR-like page; the data_processor applies them. @@ -30,31 +29,6 @@ _IGNORE_SOCKET_SPEED_TOOLTIP = ( ) -def _format_geometry_row(position: int, x_m: float, y_m: float, z_m: float) -> str: - """Render one geometry row, trimming trailing zero y/z so 1D layouts stay compact.""" - if z_m != 0.0: - return f"{position} {x_m:g} {y_m:g} {z_m:g}" - if y_m != 0.0: - return f"{position} {x_m:g} {y_m:g}" - return f"{position} {x_m:g}" - - -def _format_tx_geometry(owner) -> str: - """Render Tx geometry defaults into editable line-based text.""" - return "\n".join( - _format_geometry_row(int(entry.output_pos), float(entry.x_m), float(entry.y_m), float(entry.z_m)) - for entry in owner._defaults_config.gpr.tx_geometry - ) - - -def _format_rx_geometry(owner) -> str: - """Render Rx geometry defaults into editable line-based text.""" - return "\n".join( - _format_geometry_row(int(entry.input_pos), float(entry.x_m), float(entry.y_m), float(entry.z_m)) - for entry in owner._defaults_config.gpr.rx_geometry - ) - - def _build_processing_mode_page(parent: QStackedWidget, rows: list[FormRow], *, split_index: int | None = None) -> QWidget: """Create one processing-mode page with a two-column form body.""" page = QWidget(parent) @@ -199,21 +173,19 @@ def build_processing_group(owner) -> QGroupBox: # Medium permittivity is a legacy-GPR-only knob (shown on the legacy page below). # Coherent BP fixes the medium to eps_r = 1 (Horns_motion_3libre.py), so it is not - # offered there. Applies on the next pipeline start (a run_config field). + # offered there. owner._gpr_relative_permittivity = QDoubleSpinBox() owner._gpr_relative_permittivity.setDecimals(4) owner._gpr_relative_permittivity.setRange(0.0001, 1000.0) owner._gpr_relative_permittivity.setSingleStep(0.05) - owner._gpr_relative_permittivity.setToolTip("Applied on the next pipeline start (Save Config and restart).") owner._gpr_relative_permittivity.setValue(float(gpr_defaults.relative_permittivity)) - owner._gpr_tx_geometry_input = QPlainTextEdit(_format_tx_geometry(owner)) - owner._gpr_tx_geometry_input.setPlaceholderText("output_pos x_m [y_m] [z_m]") - owner._gpr_tx_geometry_input.setFixedHeight(78) - - owner._gpr_rx_geometry_input = QPlainTextEdit(_format_rx_geometry(owner)) - owner._gpr_rx_geometry_input.setPlaceholderText("input_pos x_m [y_m] [z_m]") - owner._gpr_rx_geometry_input.setFixedHeight(78) + owner._gpr_tx_geometry_input = AntennaGeometryEditor( + "output_pos", gpr_defaults.tx_geometry, position_count=owner._defaults_config.output_switch.positions, + ) + owner._gpr_rx_geometry_input = AntennaGeometryEditor( + "input_pos", gpr_defaults.rx_geometry, position_count=owner._defaults_config.input_switch.positions, + ) owner._gpr_common_page = _build_processing_mode_page( group, @@ -224,9 +196,9 @@ def build_processing_group(owner) -> QGroupBox: split_index=1, ) - owner._gpr_geometry_hint = QLabel("To apply Tx/Rx geometry changes: Save Config and restart the app") - owner._gpr_geometry_hint.setWordWrap(True) - owner._gpr_common_page.layout().addWidget(owner._gpr_geometry_hint) + owner._gpr_tx_geometry_input.changed.connect(owner._on_gpr_geometry_changed) + owner._gpr_rx_geometry_input.changed.connect(owner._on_gpr_geometry_changed) + owner._gpr_relative_permittivity.valueChanged.connect(owner._on_gpr_geometry_changed) owner._gpr_input_positions_input = QLineEdit(str(gpr_live_defaults.input_positions)) owner._gpr_input_positions_input.setPlaceholderText("0,1,2") diff --git a/python_app/gui/controllers/sections/switch_section.py b/python_app/gui/controllers/sections/switch_section.py index e71d123..5accb05 100644 --- a/python_app/gui/controllers/sections/switch_section.py +++ b/python_app/gui/controllers/sections/switch_section.py @@ -45,6 +45,8 @@ def build_switch_group(owner) -> QGroupBox: owner._single_combo_input.setPlaceholderText("0") owner._single_combo_select_button = QPushButton("Select") owner._single_combo_select_button.setCheckable(True) + owner._next_combo_button = QPushButton("Next combination") + owner._next_combo_button.setToolTip("Advance through input/output pairs and redraw the selected B-scan.") single_row = QHBoxLayout() single_row.setSpacing(8) @@ -54,6 +56,7 @@ def build_switch_group(owner) -> QGroupBox: single_row.addWidget(QLabel("Output")) single_row.addWidget(owner._single_combo_output) single_row.addWidget(owner._single_combo_select_button) + single_row.addWidget(owner._next_combo_button) layout.addLayout(single_row) owner._combo_select_group = QButtonGroup(group) @@ -63,6 +66,7 @@ def build_switch_group(owner) -> QGroupBox: owner._run_combos_select_button.clicked.connect(lambda: owner._set_combo_selection_mode("text")) owner._single_combo_select_button.clicked.connect(lambda: owner._set_combo_selection_mode("single")) + owner._next_combo_button.clicked.connect(owner._select_next_combo) # The live processing config has no combo: picking another one only changes which # B-scan is shown (and, outside matrix mode, the combo of the next run). owner._single_combo_output.editingFinished.connect(owner._on_bscan_display_combo_changed) diff --git a/python_app/hardware_full/compact_m_k209_service.py b/python_app/hardware_full/compact_m_k209_service.py deleted file mode 100644 index 08d65f2..0000000 --- a/python_app/hardware_full/compact_m_k209_service.py +++ /dev/null @@ -1,274 +0,0 @@ -"""VISA HiSLIP driver for Compact-M K209 / S2VNA analyzers.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any - -import numpy as np -import pyvisa - -from python_app.hardware_full.librevna_driver.models import SweepResult -from python_app.models.run_config_model import RadarSweepModel - - -@dataclass(slots=True) -class CompactMK209InterleavedSweep: - """Raw corrected K209 traces as interleaved REAL32 arrays.""" - - frequency_hz: np.ndarray - s11_values: np.ndarray - s21_values: np.ndarray - - -@dataclass(slots=True) -class CompactMK209Service: - """Acquire corrected S11/S21 sweeps from a K209 analyzer through VISA HiSLIP.""" - - resource: str - timeout_ms: int = 20_000 - preset_on_open: bool = True - visa_library: str = "@ivi" - _resource_manager: pyvisa.ResourceManager | None = field(init=False, default=None, repr=False) - _instrument: Any | None = field(init=False, default=None, repr=False) - _settings: RadarSweepModel | None = field(init=False, default=None, repr=False) - _frequency_hz: np.ndarray | None = field(init=False, default=None, repr=False) - - def __post_init__(self) -> None: - """Normalize constructor values.""" - self.resource = str(self.resource).strip() - if not self.resource: - raise ValueError("K209 VISA resource must not be empty") - self.timeout_ms = int(self.timeout_ms) - if self.timeout_ms <= 0: - raise ValueError("K209 timeout_ms must be > 0") - self.visa_library = str(self.visa_library).strip() or "@ivi" - if self.visa_library == "@py" or self.visa_library.endswith("@py"): - raise ValueError("K209 requires an IVI/Vendor VISA backend, not pyvisa-py") - - @property - def is_open(self) -> bool: - """Return whether the VISA session is open.""" - return self._instrument is not None - - def open(self) -> None: - """Open VISA resource and apply stored sweep settings when available.""" - if self._instrument is not None: - return - - try: - self._resource_manager = pyvisa.ResourceManager(self.visa_library) - self._instrument = self._resource_manager.open_resource(self.resource) - self._instrument.timeout = self.timeout_ms - self._instrument.write_termination = "\n" - self._instrument.read_termination = None - self._instrument.chunk_size = max(int(getattr(self._instrument, "chunk_size", 20_480)), 8 * 1024 * 1024) - self._instrument.write("*CLS") - - if self._settings is not None: - self._apply_configuration(self._settings) - except Exception: - self.close() - raise - - def close(self) -> None: - """Close VISA sessions.""" - if self._instrument is not None: - self._instrument.close() - self._instrument = None - if self._resource_manager is not None: - self._resource_manager.close() - self._resource_manager = None - - def __enter__(self) -> CompactMK209Service: - """Open and return this service.""" - self.open() - return self - - def __exit__(self, exc_type: object, exc: object, traceback: object) -> None: - """Close VISA resources.""" - self.close() - - def query_identity(self) -> str: - """Read analyzer identity string.""" - instrument = self._require_instrument() - return str(instrument.query("*IDN?")).strip() - - def query_system_error(self) -> str: - """Read one analyzer SCPI error queue entry.""" - instrument = self._require_instrument() - return str(instrument.query("SYST:ERR?")).strip() - - def configure(self, sweep: RadarSweepModel) -> None: - """Store and apply sweep settings.""" - self._validate_sweep(sweep) - self._settings = sweep - self._frequency_hz = None - if self._instrument is None: - return - self._apply_configuration(sweep) - - def read_device_limits(self) -> dict[str, float | int]: - """Read analyzer limits through SCPI capability/service queries.""" - opened_here = self._instrument is None - try: - if opened_here: - self.open() - instrument = self._require_instrument() - return { - "min_frequency_hz": float(instrument.query("SERV:SWE:FREQ:MIN?")), - "max_frequency_hz": float(instrument.query("SERV:SWE:FREQ:MAX?")), - "min_ifbw_hz": float(instrument.query("SYST:CAP:IFBW:MIN?")), - "max_ifbw_hz": float(instrument.query("SYST:CAP:IFBW:MAX?")), - "max_points": int(float(instrument.query("SERV:SWE:POIN?"))), - "min_power_dbm": float(instrument.query("SERV:SWE:POW:MIN?")), - "max_power_dbm": float(instrument.query("SERV:SWE:POW:MAX?")), - } - finally: - if opened_here: - self.close() - - def frequency_axis(self) -> np.ndarray: - """Return the configured frequency axis.""" - if self._frequency_hz is None: - raise RuntimeError("K209 frequency axis is not configured") - return self._frequency_hz - - def acquire_interleaved(self) -> CompactMK209InterleavedSweep: - """Acquire one corrected sweep without converting interleaved arrays.""" - if self._settings is None: - raise RuntimeError("K209 service is not configured") - if self._frequency_hz is None: - raise RuntimeError("K209 frequency axis is not configured") - points = int(self._settings.points) - - s11_values, s21_values = self._query_sweep_trace_pair(points) - return CompactMK209InterleavedSweep( - frequency_hz=self._frequency_hz, - s11_values=s11_values, - s21_values=s21_values, - ) - - def acquire(self) -> SweepResult: - """Acquire one corrected S11/S21 sweep.""" - raw = self.acquire_interleaved() - - return SweepResult( - x=raw.frequency_hz.copy(), - traces={ - "s11": self._complex_from_interleaved(raw.s11_values), - "s21": self._complex_from_interleaved(raw.s21_values), - }, - ) - - def _apply_configuration(self, sweep: RadarSweepModel) -> None: - instrument = self._require_instrument() - if self.preset_on_open: - instrument.write("SYST:PRES") - self._expect_opc("*OPC?", context="K209 preset") - - instrument.write(f"SENS:FREQ:STAR {float(sweep.start_hz):.9f}") - instrument.write(f"SENS:FREQ:STOP {float(sweep.stop_hz):.9f}") - instrument.write(f"SENS:SWE:POIN {int(sweep.points)}") - instrument.write("SENS:SWE:POIN:TIME 0") - instrument.write(f"SENS:BAND {float(sweep.if_bandwidth_hz):.9f}") - instrument.write(f"SOUR:POW {float(sweep.power_dbm):.3f}") - instrument.write("SENS:AVER OFF") - instrument.write("CALC:PAR1:DEF S21") - instrument.write("CALC:PAR1:SEL") - instrument.write("FORM:DATA REAL32") - instrument.write("FORM:BORD SWAP") - instrument.write("INIT:CONT ON") - instrument.write("TRIG:SOUR BUS") - self._expect_opc("*OPC?", context="K209 setup") - self._frequency_hz = self._query_float32_array("SENS:FREQ:DATA?", int(sweep.points)) - - def _expect_opc(self, command: str, *, context: str) -> None: - instrument = self._require_instrument() - response = str(instrument.query(command)).strip() - if response != "1": - raise RuntimeError(f"{context} returned unexpected *OPC? response: {response!r}") - - def _query_float32_array(self, command: str, expected_values: int) -> np.ndarray: - instrument = self._require_instrument() - instrument.write(command) - return self._read_float32_block(command, expected_values) - - def _query_sweep_trace_pair(self, points: int) -> tuple[np.ndarray, np.ndarray]: - instrument = self._require_instrument() - instrument.write("TRIG:SING;*OPC?;:SENS:DATA:CORR? S11;:SENS:DATA:CORR? S21") - response = self._read_ascii_token() - if response != "1": - raise RuntimeError(f"K209 sweep returned unexpected *OPC? response: {response!r}") - return ( - self._read_float32_block("SENS:DATA:CORR? S11", points * 2), - self._read_float32_block("SENS:DATA:CORR? S21", points * 2), - ) - - def _read_ascii_token(self) -> str: - token = bytearray() - while True: - byte = self._read_response_bytes(1) - if byte in (b";", b"\n", b"\r"): - if token: - return token.decode("ascii").strip() - continue - token.extend(byte) - if len(token) > 64 * 1024: - raise RuntimeError("K209 ASCII response token is too long") - - def _read_float32_block(self, context: str, expected_values: int) -> np.ndarray: - marker = self._read_response_bytes(1) - while marker in (b";", b"\n", b"\r"): - marker = self._read_response_bytes(1) - if marker != b"#": - raise RuntimeError(f"K209 response for {context!r} does not start with IEEE block marker") - - width = self._read_response_bytes(1) - if width != b"8": - raise RuntimeError(f"K209 response for {context!r} uses unsupported IEEE block header width") - - payload_size = int(self._read_response_bytes(8).decode("ascii")) - expected_size = expected_values * np.dtype(np.float32).itemsize - if payload_size != expected_size: - raise RuntimeError( - f"K209 response for {context!r} returned {payload_size} payload bytes, " - f"expected {expected_size}" - ) - - payload = self._read_response_bytes(payload_size) - array = np.frombuffer(payload, dtype=" bytes: - instrument = self._require_instrument() - data = instrument.read_bytes(count, break_on_termchar=False) - if len(data) != count: - raise RuntimeError(f"K209 response ended after {len(data)} bytes, expected {count}") - return data - - def _require_instrument(self) -> Any: - if self._instrument is None: - raise RuntimeError("K209 VISA instrument is not open") - return self._instrument - - @staticmethod - def _validate_sweep(sweep: RadarSweepModel) -> None: - if int(sweep.points) < 2: - raise ValueError("K209 sweep points must be >= 2") - if float(sweep.stop_hz) < float(sweep.start_hz): - raise ValueError("K209 sweep stop_hz must be >= start_hz") - if float(sweep.if_bandwidth_hz) <= 0.0: - raise ValueError("K209 IF bandwidth must be > 0") - - @staticmethod - def _complex_from_interleaved(values: np.ndarray) -> np.ndarray: - if values.size % 2 != 0: - raise RuntimeError("K209 complex trace payload has odd scalar count") - reshaped = np.asarray(values, dtype=np.float32).reshape((-1, 2)) - return (reshaped[:, 0] + 1j * reshaped[:, 1]).astype(np.complex64) diff --git a/python_app/hardware_full/k209_remote_protocol.py b/python_app/hardware_full/k209_remote_protocol.py deleted file mode 100644 index 34db875..0000000 --- a/python_app/hardware_full/k209_remote_protocol.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Binary TCP protocol shared by the K209 remote server and Python client.""" - -from __future__ import annotations - -import socket -import struct -from typing import BinaryIO - -import numpy as np - -COMMAND_IDENTITY = b"I" -COMMAND_LIMITS = b"L" -COMMAND_CONFIGURE = b"C" -COMMAND_ACQUIRE = b"A" - -STATUS_OK = b"O" -STATUS_ERROR = b"E" - -DEFAULT_REMOTE_HOST = "127.0.0.1" -DEFAULT_REMOTE_PORT = 50209 - -CONFIG_STRUCT = struct.Struct("!ddIdd") -LIMITS_STRUCT = struct.Struct("!ddddIdd") -U32_STRUCT = struct.Struct("!I") - - -def recv_exact(stream: socket.socket | BinaryIO, size: int) -> bytes: - """Read exactly `size` bytes from a socket-like object.""" - chunks: list[bytes] = [] - remaining = int(size) - while remaining > 0: - chunk = stream.recv(remaining) if isinstance(stream, socket.socket) else stream.read(remaining) - if not chunk: - raise ConnectionError(f"K209 remote connection closed with {remaining} bytes pending") - chunks.append(chunk) - remaining -= len(chunk) - return b"".join(chunks) - - -def send_all(stream: socket.socket | BinaryIO, payload: bytes) -> None: - """Write all payload bytes to a socket-like object.""" - if isinstance(stream, socket.socket): - stream.sendall(payload) - return - stream.write(payload) - - -def send_u32(stream: socket.socket | BinaryIO, value: int) -> None: - """Send one network-order uint32.""" - send_all(stream, U32_STRUCT.pack(int(value))) - - -def recv_u32(stream: socket.socket | BinaryIO) -> int: - """Read one network-order uint32.""" - return int(U32_STRUCT.unpack(recv_exact(stream, U32_STRUCT.size))[0]) - - -def send_error(stream: socket.socket | BinaryIO, message: str) -> None: - """Send protocol error response.""" - payload = str(message).encode("utf-8", errors="replace") - send_all(stream, STATUS_ERROR) - send_u32(stream, len(payload)) - send_all(stream, payload) - - -def read_status(stream: socket.socket | BinaryIO) -> None: - """Read response status and raise remote error when needed.""" - status = recv_exact(stream, 1) - if status == STATUS_OK: - return - if status == STATUS_ERROR: - message = recv_exact(stream, recv_u32(stream)).decode("utf-8", errors="replace") - raise RuntimeError(f"K209 remote server error: {message}") - raise RuntimeError(f"K209 remote server returned invalid status byte: {status!r}") - - -def send_float32_array(stream: socket.socket | BinaryIO, values: np.ndarray) -> None: - """Send a float32 array as a length-prefixed little-endian payload.""" - payload = np.asarray(values, dtype=" np.ndarray: - """Read a length-prefixed little-endian float32 array.""" - payload_size = recv_u32(stream) - expected_size = int(expected_values) * np.dtype(np.float32).itemsize - if payload_size != expected_size: - raise RuntimeError(f"K209 remote payload has {payload_size} bytes, expected {expected_size}") - return np.frombuffer(recv_exact(stream, payload_size), dtype=" None: @@ -120,6 +126,8 @@ class KamilAdcTtyReader: deadline = time.monotonic() + float(timeout_s) with self._mailbox_cv: while True: + if self._stop_event.is_set(): + raise RuntimeError("Kamil ADC TTY reader is closed") # Deliver a pending sweep first: if the reader both published a # sweep and then died, the consumer still sees the good data and # only meets the error on the next call. @@ -156,6 +164,8 @@ class KamilAdcTtyReader: deadline = time.monotonic() + float(timeout_s) with self._mailbox_cv: while True: + if self._stop_event.is_set(): + raise RuntimeError("Kamil ADC TTY reader is closed") sweep = self._combo_slots.pop(combo, None) if sweep is not None: return sweep diff --git a/python_app/hardware_full/laser_control/monitoring/session.py b/python_app/hardware_full/laser_control/monitoring/session.py index 71bf951..fead238 100644 --- a/python_app/hardware_full/laser_control/monitoring/session.py +++ b/python_app/hardware_full/laser_control/monitoring/session.py @@ -20,6 +20,8 @@ from dataclasses import asdict, dataclass from pathlib import Path from typing import Union +from python_app.file_io import atomic_write_json + _PathLike = Union[str, os.PathLike[str]] # Default IPC locations. Both are overridable via CLI/API so several runs can use @@ -46,12 +48,7 @@ class LaserVariationSession: Writes to a temp file in the same directory then renames, so a concurrent checker never observes a half-written file. """ - dest = Path(path) - dest.parent.mkdir(parents=True, exist_ok=True) - tmp = dest.with_name(f"{dest.name}.{os.getpid()}.tmp") - tmp.write_text(json.dumps(asdict(self), indent=2), encoding="utf-8") - os.replace(tmp, dest) - return dest + return atomic_write_json(Path(path), asdict(self)) @classmethod def load(cls, path: _PathLike = DEFAULT_SESSION_PATH) -> "LaserVariationSession": diff --git a/python_app/hardware_full/laser_control/protocol.py b/python_app/hardware_full/laser_control/protocol.py index 283d29f..8ed2f3f 100644 --- a/python_app/hardware_full/laser_control/protocol.py +++ b/python_app/hardware_full/laser_control/protocol.py @@ -196,6 +196,8 @@ class Protocol: the bus. Note the lock is advisory (``flock``): every program that opens this port must also use ``exclusive=True`` for the guarantee to hold. """ + if self.is_connected: + return port = self._port_name or self._detect_port() try: self._serial = serial.Serial( @@ -219,9 +221,10 @@ class Protocol: def disconnect(self) -> None: """Close the serial port if open.""" - if self._serial and self._serial.is_open: + connection, self._serial = self._serial, None + if connection is not None and connection.is_open: logger.debug("Closing laser serial port") - self._serial.close() + connection.close() @property def is_connected(self) -> bool: @@ -246,37 +249,6 @@ class Protocol: raise CommunicationError("Serial port is not connected") return self._serial.read(length) - # ---- Static encoding helpers (no connection required) --------------- - - @staticmethod - def flipfour(value: int) -> int: - """Byte-swap a 16-bit integer (little-endian word swap).""" - return ((value & 0xFF) << 8) | ((value >> 8) & 0xFF) - - @staticmethod - def pack_float(value: float) -> bytes: - return struct.pack(' bytes: - return struct.pack(' int: - """Return the XOR CRC over all 16-bit words except word 0 and the CRC field. - - The command-code word (word 0) is excluded, matching the firmware's CRC - expectation. - """ - hex_str = data.hex() - words = [hex_str[i:i+4] for i in range(0, len(hex_str), 4)] - # Word 0 (command code) is excluded from the CRC. - crc_words = words[1:] - result = int(crc_words[0], 16) - for w in crc_words[1:]: - result ^= int(w, 16) - return result - # ---- Command encoders ----------------------------------------------- @staticmethod @@ -404,11 +376,6 @@ class Protocol: """Build DEFAULT_ENABLE command (0x2222) — reset device.""" return bytearray.fromhex(_flipfour(_int_to_hex4(CMD_DEFAULT_ENABLE))) - @staticmethod - def encode_remove_file() -> bytes: - """Build REMOVE_FILE command (0x5555) — delete saved data.""" - return bytearray.fromhex(_flipfour(_int_to_hex4(CMD_REMOVE_FILE))) - # ---- Response decoders ----------------------------------------------- @staticmethod @@ -477,4 +444,4 @@ class Protocol: @staticmethod def state_to_description(state_hex_str: str) -> str: """Return human-readable description for a state hex string.""" - return STATE_DESCRIPTIONS.get(state_hex_str, "Unknown or reserved error.") \ No newline at end of file + return STATE_DESCRIPTIONS.get(state_hex_str, "Unknown or reserved error.") diff --git a/python_app/hardware_full/librevna_backends.py b/python_app/hardware_full/librevna_backends.py index 61d66bb..9c2a813 100644 --- a/python_app/hardware_full/librevna_backends.py +++ b/python_app/hardware_full/librevna_backends.py @@ -3,8 +3,8 @@ from __future__ import annotations from dataclasses import dataclass, field -import math -from typing import Any, Protocol +from contextlib import suppress +from typing import Any import numpy as np @@ -12,29 +12,6 @@ from python_app.hardware_full.librevna_driver.models import SweepResult from python_app.models.run_config_model import RadarSweepModel -class LibreVnaBackend(Protocol): - """Minimal backend contract required by `LibreVnaService`.""" - - def open(self) -> None: - """Open backend resources.""" - - def close(self) -> None: - """Close backend resources.""" - - @property - def is_open(self) -> bool: - """Return whether backend currently holds open runtime resources.""" - - def configure(self, sweep: RadarSweepModel) -> None: - """Apply sweep settings.""" - - def read_device_limits(self) -> dict[str, float | int]: - """Query runtime device limits.""" - - def acquire(self) -> SweepResult: - """Acquire one sweep with all available traces.""" - - @dataclass(slots=True) class NativeLibreVnaBackend: """Native backend using direct USB LibreVNA Python driver.""" @@ -43,15 +20,13 @@ class NativeLibreVnaBackend: strict_protocol_version: int _device: Any | None = field(init=False, default=None, repr=False) _settings: RadarSweepModel | None = field(init=False, default=None, repr=False) - _libre_vna_device: type[Any] | None = field(init=False, default=None, repr=False) - _vna_sweep_settings: type[Any] | None = field(init=False, default=None, repr=False) + _libre_vna_device: type[Any] = field(init=False, repr=False) + _vna_sweep_settings: type[Any] = field(init=False, repr=False) def __post_init__(self) -> None: """Import native driver classes lazily and initialize backend state.""" from python_app.hardware_full.librevna_driver import LibreVNADevice, VNASweepSettings - self._device: LibreVNADevice | None = None - self._settings: RadarSweepModel | None = None self._libre_vna_device = LibreVNADevice self._vna_sweep_settings = VNASweepSettings @@ -60,8 +35,13 @@ class NativeLibreVnaBackend: if self._device is not None: return - self._device = self._libre_vna_device() - self._device.connect(serial=self.serial, strict_protocol_version=self.strict_protocol_version, timeout_s=2.0) + device = self._device = self._libre_vna_device() + try: + device.connect(serial=self.serial, strict_protocol_version=self.strict_protocol_version, timeout_s=2.0) + except Exception: + with suppress(Exception): + self.close() + raise @property def is_open(self) -> bool: @@ -72,8 +52,8 @@ class NativeLibreVnaBackend: """Close device connection when open.""" if self._device is None: return - self._device.disconnect() - self._device = None + device, self._device = self._device, None + device.disconnect() def configure(self, sweep: RadarSweepModel) -> None: """Store and apply sweep settings to connected hardware.""" @@ -124,58 +104,3 @@ class NativeLibreVnaBackend: "s21": np.asarray(result.trace("s21"), dtype=np.complex64), }, ) - - -@dataclass(slots=True) -class MockLibreVnaBackend: - """Synthetic backend used for local development and tests.""" - _settings: RadarSweepModel | None = field(init=False, default=None, repr=False) - _mock_phase: float = field(init=False, default=0.0, repr=False) - - def __post_init__(self) -> None: - """Initialize mock backend state.""" - self._settings = None - self._mock_phase = 0.0 - - def open(self) -> None: - """No-op for mock backend.""" - - def close(self) -> None: - """No-op for mock backend.""" - - @property - def is_open(self) -> bool: - """Mock backend does not hold external resources.""" - return False - - def configure(self, sweep: RadarSweepModel) -> None: - """Store sweep settings used by synthetic acquisition.""" - self._settings = sweep - - def read_device_limits(self) -> dict[str, float | int]: - """Mock backend does not support native device limits queries.""" - raise RuntimeError("LibreVNA Python driver is not available") - - def acquire(self) -> SweepResult: - """Generate synthetic S11 and S21 values using deterministic envelopes.""" - if self._settings is None: - raise RuntimeError("Radar service is not configured") - - points = self._settings.points - freq = np.linspace(self._settings.start_hz, self._settings.stop_hz, points, dtype=np.float32) - phase = (2.0 * math.pi * np.linspace(0.0, 1.0, points, dtype=np.float32)) + self._mock_phase - s21_envelope = 0.6 + 0.4 * np.sin(phase * 0.5) - s11_envelope = 0.25 + 0.15 * np.cos(phase * 0.75) - s21 = (s21_envelope * np.cos(phase) + 1j * s21_envelope * np.sin(phase)).astype(np.complex64) - reflected_phase = phase * 0.6 + 0.8 - s11 = ( - s11_envelope * np.cos(reflected_phase) + 1j * s11_envelope * np.sin(reflected_phase) - ).astype(np.complex64) - self._mock_phase += 0.05 - return SweepResult( - x=freq, - traces={ - "s11": s11, - "s21": s21, - }, - ) diff --git a/python_app/hardware_full/librevna_driver/models.py b/python_app/hardware_full/librevna_driver/models.py index ebe0548..d59dfcc 100644 --- a/python_app/hardware_full/librevna_driver/models.py +++ b/python_app/hardware_full/librevna_driver/models.py @@ -201,6 +201,8 @@ class SweepResult: x: np.ndarray traces: dict[str, np.ndarray] = field(default_factory=dict) x_label: str = "frequency_hz" + capture_start_ns: int = 0 + capture_end_ns: int = 0 def __post_init__(self) -> None: """Normalize dtypes and validate shape compatibility.""" diff --git a/python_app/hardware_full/librevna_driver/session.py b/python_app/hardware_full/librevna_driver/session.py index 6d9fdc7..809c459 100644 --- a/python_app/hardware_full/librevna_driver/session.py +++ b/python_app/hardware_full/librevna_driver/session.py @@ -28,6 +28,7 @@ from .protocol import ( from .transport import USBTransport logger = logging.getLogger(__name__) +_PACKET_QUEUE_MAX = 32768 class _IndexedPayload(Protocol): @@ -50,7 +51,9 @@ class LibreVNASession: on_disconnect=self._on_transport_disconnect, ) - self._incoming: dict[PacketType, deque[Packet]] = defaultdict(deque) + # The sweep free-runs between acquisitions, and subscriptions do not + # consume request queues. Retain a bounded tail of stale datapoints. + self._incoming: dict[PacketType, deque[Packet]] = defaultdict(lambda: deque(maxlen=_PACKET_QUEUE_MAX)) self._subscribers: dict[PacketType, set[Callable[[Packet], None]]] = defaultdict(set) self._incoming_cv = threading.Condition() @@ -159,8 +162,8 @@ class LibreVNASession: def disconnect(self) -> None: """Disconnect USB transport and clear session state.""" logger.debug("Closing USB transport and clearing session queues") - self._transport.disconnect() with self._incoming_cv: + self._fatal_error = DeviceDisconnectedError("LibreVNA session is disconnected") self._incoming.clear() self._subscribers.clear() self._incoming_cv.notify_all() @@ -168,6 +171,7 @@ class LibreVNASession: self._awaiting_ack = False self._ack_result = None self._ack_cv.notify_all() + self._transport.disconnect() def send(self, packet: Packet, *, require_ack: bool = True, timeout_s: float = 0.5) -> None: """Send one protocol packet and optionally wait for ACK/NACK.""" @@ -188,26 +192,26 @@ class LibreVNASession: self._awaiting_ack = True self._ack_result = None - self._transport.write(frame, timeout_s=timeout_s) - - if require_ack: - deadline = time.monotonic() + timeout_s - with self._ack_cv: - while self._ack_result is None: - self._raise_if_fatal_locked() - remaining = deadline - time.monotonic() - if remaining <= 0: - self._awaiting_ack = False - raise TimeoutError(f"Timeout waiting for ACK for packet {packet.type.name}") - self._ack_cv.wait(timeout=remaining) - - result = self._ack_result - self._awaiting_ack = False - self._ack_result = None - - if result == PacketType.NACK: - raise NackError(f"Received NACK for packet {packet.type.name}") - logger.debug("ACK received for packet %s", packet.type.name) + try: + self._transport.write(frame, timeout_s=timeout_s) + if require_ack: + deadline = time.monotonic() + timeout_s + with self._ack_cv: + while self._ack_result is None: + self._raise_if_fatal_locked() + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(f"Timeout waiting for ACK for packet {packet.type.name}") + self._ack_cv.wait(timeout=remaining) + result = self._ack_result + if result == PacketType.NACK: + raise NackError(f"Received NACK for packet {packet.type.name}") + logger.debug("ACK received for packet %s", packet.type.name) + finally: + if require_ack: + with self._ack_cv: + self._awaiting_ack = False + self._ack_result = None def request( self, @@ -238,9 +242,10 @@ class LibreVNASession: payload_type: type[TPacketPayload], payload_error: str, ) -> list[TPacketPayload]: - """Collect indexed payloads by `point_number` into ascending order. + """Collect one sweep from point zero, returning points in ascending order. - Missing points are omitted; callers decide whether this is acceptable. + Ignore the tail before the first point zero. A new point zero discards + an incomplete earlier cycle instead of filling its gaps from a later one. """ if expected_points <= 0: raise ValueError("expected_points must be > 0") @@ -265,6 +270,11 @@ class LibreVNASession: f"Received out-of-range point index {point_number}, expected 0..{expected_points - 1}" ) + if point_number == 0: + collected = [None] * expected_points + received = 0 + elif collected[0] is None: + continue if collected[point_number] is None: received += 1 collected[point_number] = payload @@ -391,8 +401,12 @@ class LibreVNASession: callbacks: list[Callable[[Packet], None]] = [] with self._incoming_cv: - self._incoming[packet.type].append(packet) callbacks = list(self._subscribers.get(packet.type, set())) + queue = self._incoming[packet.type] + if len(queue) == queue.maxlen and not callbacks and packet.type != PacketType.VNA_DATAPOINT: + self._set_fatal_error(ParseError(f"Packet queue overflow for {packet.type.name}")) + return + queue.append(packet) self._incoming_cv.notify_all() for callback in callbacks: diff --git a/python_app/hardware_full/librevna_driver/transport/usb.py b/python_app/hardware_full/librevna_driver/transport/usb.py index de82a1b..41f87a1 100644 --- a/python_app/hardware_full/librevna_driver/transport/usb.py +++ b/python_app/hardware_full/librevna_driver/transport/usb.py @@ -94,68 +94,60 @@ class USBTransport: logger.debug("Opening libusb context for connect(serial=%s)", serial) self._ctx = usb1.USBContext() - selected_handle: usb1.USBDeviceHandle | None = None - selected_serial = "" + try: + selected_handle: usb1.USBDeviceHandle | None = None + selected_serial = "" - for device, _, _ in self._iter_matching_devices(self._ctx): - handle: usb1.USBDeviceHandle | None = None + for device, _, _ in self._iter_matching_devices(self._ctx): + handle: usb1.USBDeviceHandle | None = None + try: + handle = device.open() + found_serial = handle.getSerialNumber() or "" + if serial is not None and found_serial != serial: + continue + + selected_handle = handle + selected_serial = found_serial + break + except usb1.USBError as exc: + logger.debug("Skipping USB candidate during connect due to error: %s", exc) + finally: + if handle is not None and handle is not selected_handle: + with suppress(usb1.USBError): + handle.close() + + if selected_handle is None: + serial_msg = f" with serial '{serial}'" if serial else "" + raise DeviceDisconnectedError(f"No compatible LibreVNA USB device found{serial_msg}") + + self._handle = selected_handle try: - handle = device.open() - found_serial = handle.getSerialNumber() or "" - if serial is not None and found_serial != serial: - handle.close() - continue - - selected_handle = handle - selected_serial = found_serial - break + selected_handle.setAutoDetachKernelDriver(True) + if selected_handle.kernelDriverActive(self.INTERFACE): + selected_handle.detachKernelDriver(self.INTERFACE) except usb1.USBError as exc: - if handle is not None: - with suppress(usb1.USBError): - handle.close() - logger.debug("Skipping USB candidate during connect due to error: %s", exc) - continue + raise DeviceDisconnectedError(f"Failed to prepare USB kernel driver state: {exc}") from exc - if selected_handle is None: - if self._ctx is not None: - self._ctx.close() - self._ctx = None - serial_msg = f" with serial '{serial}'" if serial else "" - raise DeviceDisconnectedError(f"No compatible LibreVNA USB device found{serial_msg}") + try: + self._claim_interface_with_busy_recovery(selected_handle) + except usb1.USBError as exc: + raise DeviceDisconnectedError(f"Failed to claim USB interface {self.INTERFACE}: {exc}") from exc - try: - selected_handle.setAutoDetachKernelDriver(True) - if selected_handle.kernelDriverActive(self.INTERFACE): - selected_handle.detachKernelDriver(self.INTERFACE) - except usb1.USBError as exc: - selected_handle.close() - if self._ctx is not None: - self._ctx.close() - self._ctx = None - raise DeviceDisconnectedError(f"Failed to prepare USB kernel driver state: {exc}") from exc + self.connected_serial = selected_serial + self._stop_event.clear() + logger.info( + "USB connected (serial=%s, timeout=%.2fs, endpoint_out=0x%02x, endpoint_in=0x%02x)", + self.connected_serial, + timeout_s, + self.DATA_EP_OUT, + self.DATA_EP_IN, + ) - try: - self._claim_interface_with_busy_recovery(selected_handle) - except usb1.USBError as exc: - selected_handle.close() - if self._ctx is not None: - self._ctx.close() - self._ctx = None - raise DeviceDisconnectedError(f"Failed to claim USB interface {self.INTERFACE}: {exc}") from exc - - self._handle = selected_handle - self.connected_serial = selected_serial - self._stop_event.clear() - logger.info( - "USB connected (serial=%s, timeout=%.2fs, endpoint_out=0x%02x, endpoint_in=0x%02x)", - self.connected_serial, - timeout_s, - self.DATA_EP_OUT, - self.DATA_EP_IN, - ) - - self._rx_thread = threading.Thread(target=self._rx_loop, name="librevna-usb-rx", daemon=True) - self._rx_thread.start() + self._rx_thread = threading.Thread(target=self._rx_loop, name="librevna-usb-rx", daemon=True) + self._rx_thread.start() + except Exception: + self.disconnect() + raise def _claim_interface_with_busy_recovery(self, handle: "usb1.USBDeviceHandle") -> None: # type: ignore[name-defined] """Claim the data interface, clearing a stale BUSY claim once if needed. diff --git a/python_app/hardware_full/librevna_service.py b/python_app/hardware_full/librevna_service.py index 5e37225..5084549 100644 --- a/python_app/hardware_full/librevna_service.py +++ b/python_app/hardware_full/librevna_service.py @@ -1,102 +1,38 @@ -"""High-level orchestration service for configuring and querying LibreVNA.""" +"""Native single-device LibreVNA service used by acquisition and device-limit queries.""" from __future__ import annotations from dataclasses import dataclass, field -import logging -from python_app.hardware_full.librevna_backends import LibreVnaBackend, MockLibreVnaBackend, NativeLibreVnaBackend +from python_app.hardware_full.librevna_backends import NativeLibreVnaBackend from python_app.hardware_full.librevna_driver.models import SweepResult from python_app.models.run_config_model import RadarSweepModel -logger = logging.getLogger(__name__) - @dataclass(slots=True) class LibreVnaService: - """Provide stable API for GUI/workflows while hiding backend details.""" + """Provide the native single-device API; factories select mock acquisition.""" serial: str | None = None strict_protocol_version: int = 14 backend_mode: str = "auto" - _driver_available: bool = field(init=False, default=False, repr=False) - _backend: LibreVnaBackend | None = field(init=False, default=None, repr=False) - _using_mock_backend: bool = field(init=False, default=False, repr=False) + _backend: NativeLibreVnaBackend = field(init=False, repr=False) def __post_init__(self) -> None: - """Initialize selected backend and detect driver availability.""" - self._driver_available = False - self._backend = None - self._using_mock_backend = False - - mode = self.backend_mode.strip().lower() - if mode not in {"auto", "native", "mock"}: - raise ValueError(f"Unsupported LibreVnaService backend mode: {self.backend_mode}") - - if mode == "mock": - logger.info("LibreVNA service using mock backend (mode=mock)") - self._backend = MockLibreVnaBackend() - self._using_mock_backend = True - return - - try: - self._backend = NativeLibreVnaBackend( - serial=self.serial, - strict_protocol_version=self.strict_protocol_version, - ) - self._driver_available = True - logger.info("LibreVNA native backend initialized (serial=%s)", self.serial or "auto") - except Exception as exc: - # 'native' demands real hardware — never substitute synthetic data. - if mode == "native": - raise - # 'auto' falls back to the mock backend ONLY when the native driver - # library itself is unavailable (a dev host without the USB stack) — - # this is not device-absence (that surfaces later from open()). Log it - # loudly so it is never a silent surprise; a deployed appliance should - # set driver_mode='native' to forbid the fallback entirely. - logger.warning("LibreVNA native backend unavailable; falling back to mock (mode=auto): %s", exc) - self._backend = MockLibreVnaBackend() - self._using_mock_backend = True - - @property - def driver_available(self) -> bool: - """Return `True` when native Python LibreVNA driver is available.""" - return self._driver_available + if self.backend_mode.strip().lower() not in {"auto", "native"}: + raise ValueError(f"Unsupported native LibreVNA backend mode: {self.backend_mode}") + self._backend = NativeLibreVnaBackend(self.serial, self.strict_protocol_version) def open(self) -> None: - """Open backend resources.""" - if not self._driver_available and not self._using_mock_backend: - return - if self._backend is None: - return - logger.debug("Opening LibreVNA backend (mock=%s)", self._using_mock_backend) self._backend.open() def close(self) -> None: - """Close backend resources.""" - if self._backend is None: - return - logger.debug("Closing LibreVNA backend") self._backend.close() def configure(self, sweep: RadarSweepModel) -> None: - """Apply sweep settings to active backend.""" - if self._backend is None: - raise RuntimeError("LibreVNA backend is not initialized") - logger.debug( - "Configuring LibreVNA sweep: %s-%s Hz, %s points", - sweep.start_hz, sweep.stop_hz, sweep.points, - ) self._backend.configure(sweep) def read_device_limits(self) -> dict[str, float | int]: - """Read native device limits from connected LibreVNA.""" - if not self._driver_available: - raise RuntimeError("LibreVNA Python driver is not available") - if self._backend is None: - raise RuntimeError("LibreVNA backend is not initialized") - opened_here = not self._backend.is_open try: if opened_here: @@ -107,9 +43,4 @@ class LibreVnaService: self.close() def acquire(self) -> SweepResult: - """Acquire one sweep with all available traces from active backend.""" - if self._backend is None: - raise RuntimeError("LibreVNA backend is not initialized") - if self._using_mock_backend and not self._driver_available and self.backend_mode != "mock": - raise RuntimeError("Device not found") return self._backend.acquire() diff --git a/python_app/hardware_full/matrix_radar_service.py b/python_app/hardware_full/matrix_radar_service.py index 4ca527c..7066c86 100644 --- a/python_app/hardware_full/matrix_radar_service.py +++ b/python_app/hardware_full/matrix_radar_service.py @@ -1,14 +1,4 @@ -"""Factory for matrix-mode (multi-port) radar acquisition services. - -Matrix radars acquire the full virtual switch matrix per call via -``acquire_collection`` instead of one combo per ``acquire`` like single-radar -services. The supported models are LibreVNA in synchronized multi-device mode -and the PLANAR SN9000 multi-port analyzer. - -Service implementations are imported lazily so a config that targets one -model does not pull in transport dependencies (libusb, pyvisa, etc.) needed -only by the other model. -""" +"""Factory for synchronized LibreVNA matrices and the shared physical mock.""" from __future__ import annotations @@ -32,57 +22,51 @@ class MatrixRadarService(Protocol): def acquire_collection(self, collection_id: int = 1) -> SweepCollection: """Acquire one complete virtual switch matrix in canonical combo order.""" + ... def recover(self) -> None: """Reconnect to the radar after a transient acquisition failure.""" -def create_matrix_radar_service(config: RunConfigModel) -> MatrixRadarService: - """Create the matrix radar service implementation for the active config.""" - if not config.is_matrix_radar: +def create_acquisition_service(config: RunConfigModel) -> MatrixRadarService: + """Create a native matrix service or the shared single/multi mock scene.""" + if config.radar.driver_mode == "mock" and config.radar.model in { + RunConfigModel.LIBREVNA_MODEL, RunConfigModel.LIBREVNA_MULTI_MODEL, + }: + from python_app.hardware_full.point_target_mock_service import PointTargetMockService + + return PointTargetMockService(config) + if config.radar.model != RunConfigModel.LIBREVNA_MULTI_MODEL: raise RuntimeError("matrix radar service factory requires a matrix-mode radar.model") - model = config.radar.model - if model == RunConfigModel.LIBREVNA_MULTI_MODEL: - from python_app.hardware_full.multi_device_service import MultiDeviceLibreVnaService + from python_app.hardware_full.multi_device_service import MultiDeviceLibreVnaService - inner = MultiDeviceLibreVnaService( - master_serial=config.radar.serial, - slave_serials=list(config.radar.multi_device.slave_serials), - force_external_reference=config.radar.multi_device.force_external_reference, - recovery_attempts=config.radar.multi_device.recovery_attempts, - backend_mode=config.radar.driver_mode, - switch_substage_count=config.matrix_stimulus_switch_positions, - ) - out_physical = config.matrix_output_switch_positions - in_physical = config.matrix_input_switch_positions - if out_physical <= 1 and in_physical <= 1: - return inner + inner = MultiDeviceLibreVnaService( + master_serial=config.radar.serial, + slave_serials=list(config.radar.multi_device.slave_serials), + force_external_reference=config.radar.multi_device.force_external_reference, + recovery_attempts=config.radar.multi_device.recovery_attempts, + backend_mode=config.radar.driver_mode, + switch_substage_count=config.matrix_stimulus_switch_positions, + ) + out_physical = config.matrix_output_switch_positions + in_physical = config.matrix_input_switch_positions + if out_physical <= 1 and in_physical <= 1: + return inner - from python_app.hardware_full.switched_matrix_radar_service import ( - SwitchedMatrixRadarService, - build_physical_switch, - ) + from python_app.hardware_full.switched_matrix_radar_service import ( + SwitchedMatrixRadarService, + build_physical_switch, + ) - return SwitchedMatrixRadarService( - inner=inner, - output_switch=build_physical_switch(config.output_switch, out_physical, config.radar.driver_mode), - input_switch=build_physical_switch(config.input_switch, in_physical, config.radar.driver_mode), - inner_output_positions=config.matrix_virtual_output_positions, - inner_input_positions=RunConfigModel.MULTI_DEVICE_INPUT_POSITIONS, - settling_ms=config.runtime.settling_ms, - ) + return SwitchedMatrixRadarService( + inner=inner, + output_switch=build_physical_switch(config.output_switch, out_physical, config.radar.driver_mode), + input_switch=build_physical_switch(config.input_switch, in_physical, config.radar.driver_mode), + inner_output_positions=config.matrix_virtual_output_positions, + inner_input_positions=RunConfigModel.MULTI_DEVICE_INPUT_POSITIONS, + settling_ms=config.runtime.settling_ms, + ) - if model == RunConfigModel.SN9000_MODEL: - if config.radar.driver_mode != "native": - raise RuntimeError("SN9000 requires radar.driver_mode='native'") - from python_app.hardware_full.sn9000_service import Sn9000Service - visa_library = config.radar.visa_library or "@ivi" - return Sn9000Service( - host=config.radar.remote_host, - port=config.radar.remote_port, - visa_library=visa_library, - ) - - raise RuntimeError(f"Unsupported matrix radar model: {model}") +create_matrix_radar_service = create_acquisition_service diff --git a/python_app/hardware_full/multi_device_service.py b/python_app/hardware_full/multi_device_service.py index 7eae299..9bf9247 100644 --- a/python_app/hardware_full/multi_device_service.py +++ b/python_app/hardware_full/multi_device_service.py @@ -4,7 +4,6 @@ from __future__ import annotations from dataclasses import dataclass, field import logging -import math import threading import time from typing import TYPE_CHECKING @@ -89,8 +88,6 @@ class MultiDeviceLibreVnaService: switch_substage_count: int = 1 _controller: "MultiDeviceVnaController | None" = field(init=False, default=None, repr=False) _sweep_configuration: SweepConfiguration | None = field(init=False, default=None, repr=False) - _using_mock_backend: bool = field(init=False, default=False, repr=False) - _mock_phase: float = field(init=False, default=0.0, repr=False) def __post_init__(self) -> None: """Validate static topology and backend selection.""" @@ -101,10 +98,9 @@ class MultiDeviceLibreVnaService: self.recovery_attempts = max(0, int(self.recovery_attempts)) mode = self.backend_mode.strip().lower() - if mode not in {"auto", "native", "mock"}: + if mode not in {"auto", "native"}: raise ValueError(f"Unsupported multi-device backend mode: {self.backend_mode}") self.backend_mode = mode - self._using_mock_backend = mode == "mock" self.switch_substage_count = normalize_switch_substage_count(self.switch_substage_count) @property @@ -112,14 +108,9 @@ class MultiDeviceLibreVnaService: """Return how many stimulus outputs one acquisition produces.""" return RunConfigModel.MULTI_DEVICE_MASTER_PORTS * self.switch_substage_count - @property - def using_mock_backend(self) -> bool: - """Return whether this service is generating synthetic data.""" - return self._using_mock_backend - def open(self) -> None: - """Open native device transports when not in mock mode.""" - if self._using_mock_backend or self._controller is not None: + """Open native device transports.""" + if self._controller is not None: return try: from python_app.hardware_full.librevna_multi_device_driver.controller import MultiDeviceVnaController @@ -135,11 +126,6 @@ class MultiDeviceLibreVnaService: self.master_serial, self.slave_serials, ) except Exception as exc: - # Never silently latch to synthetic data: a deployed appliance must wait - # for the real device, not record fakes. Synthetic data requires an - # explicit backend_mode='mock' (selected in __post_init__); both 'auto' - # and 'native' re-raise so the producer's wait-for-device retry keeps - # trying until the hardware appears. logger.warning("Multi-device open failed (backend_mode=%s): %s", self.backend_mode, exc) raise @@ -176,8 +162,6 @@ class MultiDeviceLibreVnaService: sleeping, so a shutdown request aborts the loop immediately; an optional `deadline_monotonic` caps the total wall-time spent here. """ - if self._using_mock_backend: - return self.close() # Try to reopen immediately first, then back off only after a failure: a @@ -261,12 +245,9 @@ class MultiDeviceLibreVnaService: if self._sweep_configuration is None: raise RuntimeError("Multi-device service is not configured") capture_start_ns = time.monotonic_ns() - if self._using_mock_backend: - collection = self._acquire_mock_collection(collection_id, capture_start_ns) - else: - collection = self._acquire_native_collection_with_recovery( - collection_id, capture_start_ns, stop_event=stop_event - ) + collection = self._acquire_native_collection_with_recovery( + collection_id, capture_start_ns, stop_event=stop_event + ) collection.capture_end_ns = time.monotonic_ns() return collection @@ -417,52 +398,6 @@ class MultiDeviceLibreVnaService: capture_start_ns=capture_start_ns, ) - def _acquire_mock_collection(self, collection_id: int, capture_start_ns: int) -> SweepCollection: - assert self._sweep_configuration is not None - mock_sweep_start_ns = time.monotonic_ns() - points = int(self._sweep_configuration.points) - frequencies = np.linspace( - self._sweep_configuration.start_hz, - self._sweep_configuration.stop_hz, - points, - dtype=np.float32, - ) - base_phase = 2.0 * math.pi * np.linspace(0.0, 1.0, points, dtype=np.float32) + self._mock_phase - traces: list[TraceData] = [] - for output_pos in range(self.output_position_count): - reflected_phase = base_phase * (0.55 + 0.05 * output_pos) + 0.7 * (output_pos + 1) - s11 = ( - (0.22 + 0.04 * output_pos) * np.cos(reflected_phase) - + 1j * (0.22 + 0.04 * output_pos) * np.sin(reflected_phase) - ).astype(np.complex64) - for input_pos in range(RunConfigModel.MULTI_DEVICE_INPUT_POSITIONS): - gain = 0.45 + 0.08 * input_pos + 0.03 * output_pos - phase = base_phase * (1.0 + 0.03 * input_pos) + (0.4 * input_pos + 0.9 * output_pos) - s21 = (gain * np.cos(phase) + 1j * gain * np.sin(phase)).astype(np.complex64) - traces.append( - TraceData( - combo=ComboKey(input=input_pos, output=output_pos), - frequency_hz=frequencies, - s11=s11, - s21=s21, - capture_start_ns=mock_sweep_start_ns, - ) - ) - # One shared window, as on real hardware: every combo of a collection comes - # out of the same synchronized cycle. Stamping each trace as it is built - # would instead spread the ends over the generation loop, which never - # happens natively and would mislead anyone reading mock captures. - mock_sweep_end_ns = time.monotonic_ns() - for trace in traces: - trace.capture_end_ns = mock_sweep_end_ns - self._mock_phase += 0.05 - return SweepCollection( - collection_id=int(collection_id), - monotonic_ns=time.monotonic_ns(), - traces=traces, - capture_start_ns=capture_start_ns, - ) - @staticmethod def _required_s_parameter(s_parameters: dict[str, np.ndarray], name: str) -> np.ndarray: values = s_parameters.get(name) diff --git a/python_app/hardware_full/point_target_mock_service.py b/python_app/hardware_full/point_target_mock_service.py new file mode 100644 index 0000000..4b5bab0 --- /dev/null +++ b/python_app/hardware_full/point_target_mock_service.py @@ -0,0 +1,121 @@ +"""One moving point reflector shared by single and multi-radar mock acquisition.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import time + +import numpy as np + +from python_app.hardware_full.librevna_driver.models import SweepResult +from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData +from python_app.models.run_config_model import RadarSweepModel, RunConfigModel + +_WAVE_SPEED_M_S = 3e8 +_FRAME_PERIOD_S = 0.05 +_MOTION_PERIOD_S = 8.0 + + +@dataclass(slots=True) +class PointTargetMockService: + """Simulate bistatic propagation from configured antennas to one moving target. + + The target moves back and forth along the array and between depths 3.25–4.25 m. + One monotonic sample time determines every channel of a collection; channel + numbers never contribute an arbitrary gain, range, or phase offset. + """ + + config: RunConfigModel + _frequency_hz: np.ndarray | None = field(init=False, default=None, repr=False) + _phase_per_m: np.ndarray | None = field(init=False, default=None, repr=False) + _last_capture_ns: int = field(init=False, default=0, repr=False) + _epoch_ns: int = field(init=False, repr=False) + _combos: tuple[ComboKey, ...] = field(init=False, repr=False) + _active_combo: ComboKey = field(init=False, repr=False) + + def __post_init__(self) -> None: + self._epoch_ns = time.monotonic_ns() + self._combos = tuple(ComboKey(c.input, c.output) for c in ( + self.config.combos or self.config.build_runtime_combos() + )) + self._active_combo = self._combos[0] + + def open(self) -> None: + """Mock acquisition owns no hardware resources.""" + + def close(self) -> None: + """Mock acquisition owns no hardware resources.""" + + def recover(self) -> None: + """Mock acquisition has no hardware connection to recover.""" + + def read_device_limits(self) -> dict[str, float | int]: + raise RuntimeError("Mock radar has no hardware device limits") + + def configure(self, sweep: RadarSweepModel) -> None: + self._frequency_hz = np.linspace(sweep.start_hz, sweep.stop_hz, sweep.points, dtype=np.float32) + self._phase_per_m = -2j * np.pi * self._frequency_hz.astype(np.float64) / _WAVE_SPEED_M_S + + def set_active_combo(self, combo: ComboKey) -> None: + if not (0 <= combo.input < self.config.input_switch.positions + and 0 <= combo.output < self.config.output_switch.positions): + raise ValueError(f"Mock combo is outside the antenna matrix: {combo}") + self._active_combo = ComboKey(combo.input, combo.output) + + def acquire(self) -> SweepResult: + collection = self._capture((self._active_combo,), 1) + trace = collection.traces[0] + return SweepResult( + x=trace.frequency_hz, traces={"s11": trace.s11, "s21": trace.s21}, + capture_start_ns=trace.capture_start_ns, capture_end_ns=trace.capture_end_ns, + ) + + def acquire_collection(self, collection_id: int = 1) -> SweepCollection: + return self._capture(self._combos, collection_id) + + def acquire_combo_collection( + self, *, input_pos: int, output_pos: int, collection_id: int = 1, + ) -> SweepCollection: + self.set_active_combo(ComboKey(input_pos, output_pos)) + return self._capture((self._active_combo,), collection_id) + + def _target_position( + self, sample_ns: int, + tx: dict[int, tuple[float, float, float]], rx: dict[int, tuple[float, float, float]], + ) -> np.ndarray: + coordinates = [*tx.values(), *rx.values()] or [(0.0, 0.0, 0.0)] + xs = [position[0] for position in coordinates] + displacement = np.sin(2 * np.pi * (sample_ns - self._epoch_ns) / 1e9 / _MOTION_PERIOD_S) + return np.array([ + (min(xs) + max(xs)) / 2 + max(0.75, (max(xs) - min(xs)) / 2 + 0.25) * displacement, + 0.25, 3.75 + 0.5 * displacement, + ]) + + def _capture(self, combos: tuple[ComboKey, ...], collection_id: int) -> SweepCollection: + if self._frequency_hz is None or self._phase_per_m is None: + raise RuntimeError("Mock radar is not configured") + delay_s = _FRAME_PERIOD_S - (time.monotonic_ns() - self._last_capture_ns) / 1e9 + if delay_s > 0: + time.sleep(delay_s) + capture_start_ns = time.monotonic_ns() + tx = {p.output_pos: (p.x_m, p.y_m, p.z_m) for p in self.config.gpr.tx_geometry} + rx = {p.input_pos: (p.x_m, p.y_m, p.z_m) for p in self.config.gpr.rx_geometry} + target = self._target_position(capture_start_ns, tx, rx) + traces = [] + for combo in combos: + r_tx = float(np.linalg.norm(target - tx.get(combo.output, (0.0, 0.0, 0.0)))) + r_rx = float(np.linalg.norm(target - rx.get(combo.input, (0.0, 0.0, 0.0)))) + traces.append(TraceData( + combo=combo, frequency_hz=self._frequency_hz, + s11=np.asarray(0.5 / max(1.0, r_tx * r_tx) * np.exp(self._phase_per_m * 2 * r_tx), dtype=np.complex64), + s21=np.asarray(0.5 / max(1.0, r_tx * r_rx) * np.exp(self._phase_per_m * (r_tx + r_rx)), dtype=np.complex64), + capture_start_ns=capture_start_ns, + )) + capture_end_ns = time.monotonic_ns() + for trace in traces: + trace.capture_end_ns = capture_end_ns + self._last_capture_ns = capture_start_ns + return SweepCollection( + collection_id=collection_id, monotonic_ns=capture_end_ns, traces=traces, + capture_start_ns=capture_start_ns, capture_end_ns=capture_end_ns, + ) diff --git a/python_app/hardware_full/remote_compact_m_k209_service.py b/python_app/hardware_full/remote_compact_m_k209_service.py deleted file mode 100644 index b591d03..0000000 --- a/python_app/hardware_full/remote_compact_m_k209_service.py +++ /dev/null @@ -1,172 +0,0 @@ -"""TCP client for a Compact-M K209 connected to a remote S2VNA host.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -import socket - -import numpy as np - -from python_app.hardware_full.k209_remote_protocol import ( - COMMAND_ACQUIRE, - COMMAND_CONFIGURE, - COMMAND_IDENTITY, - COMMAND_LIMITS, - CONFIG_STRUCT, - DEFAULT_REMOTE_HOST, - DEFAULT_REMOTE_PORT, - LIMITS_STRUCT, - read_status, - recv_exact, - recv_float32_array, - recv_u32, -) -from python_app.hardware_full.librevna_driver.models import SweepResult -from python_app.models.run_config_model import RadarSweepModel - - -@dataclass(slots=True) -class RemoteCompactMK209Service: - """Acquire K209 sweeps through a persistent TCP connection.""" - - host: str = DEFAULT_REMOTE_HOST - port: int = DEFAULT_REMOTE_PORT - timeout_s: float = 20.0 - _socket: socket.socket | None = field(init=False, default=None, repr=False) - _settings: RadarSweepModel | None = field(init=False, default=None, repr=False) - _frequency_hz: np.ndarray | None = field(init=False, default=None, repr=False) - - def __post_init__(self) -> None: - self.host = str(self.host).strip() - if not self.host: - raise ValueError("K209 remote host must not be empty") - self.port = int(self.port) - if self.port <= 0 or self.port > 65535: - raise ValueError("K209 remote port must be in 1..65535") - self.timeout_s = float(self.timeout_s) - if self.timeout_s <= 0.0: - raise ValueError("K209 remote timeout_s must be > 0") - - def open(self) -> None: - """Open TCP connection and apply stored settings when present.""" - if self._socket is not None: - return - sock = socket.create_connection((self.host, self.port), timeout=self.timeout_s) - sock.settimeout(self.timeout_s) - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - self._socket = sock - try: - if self._settings is not None: - self._apply_configuration(self._settings) - except Exception: - self.close() - raise - - def close(self) -> None: - """Close TCP connection.""" - if self._socket is None: - return - try: - self._socket.close() - finally: - self._socket = None - - def query_identity(self) -> str: - """Read analyzer identity string through the remote server.""" - sock = self._require_socket() - sock.sendall(COMMAND_IDENTITY) - read_status(sock) - return recv_exact(sock, recv_u32(sock)).decode("utf-8", errors="replace").strip() - - def read_device_limits(self) -> dict[str, float | int]: - """Read analyzer limits through the remote server.""" - opened_here = self._socket is None - try: - if opened_here: - self.open() - sock = self._require_socket() - sock.sendall(COMMAND_LIMITS) - read_status(sock) - min_freq, max_freq, min_ifbw, max_ifbw, max_points, min_power, max_power = LIMITS_STRUCT.unpack( - recv_exact(sock, LIMITS_STRUCT.size) - ) - return { - "min_frequency_hz": float(min_freq), - "max_frequency_hz": float(max_freq), - "min_ifbw_hz": float(min_ifbw), - "max_ifbw_hz": float(max_ifbw), - "max_points": int(max_points), - "min_power_dbm": float(min_power), - "max_power_dbm": float(max_power), - } - finally: - if opened_here: - self.close() - - def configure(self, sweep: RadarSweepModel) -> None: - """Store and apply sweep settings.""" - self._validate_sweep(sweep) - self._settings = sweep - self._frequency_hz = None - if self._socket is not None: - self._apply_configuration(sweep) - - def acquire(self) -> SweepResult: - """Acquire one corrected S11/S21 sweep.""" - if self._settings is None or self._frequency_hz is None: - raise RuntimeError("K209 remote service is not configured") - sock = self._require_socket() - points = int(self._settings.points) - sock.sendall(COMMAND_ACQUIRE) - read_status(sock) - returned_points = recv_u32(sock) - if returned_points != points: - raise RuntimeError(f"K209 remote sweep returned {returned_points} points, expected {points}") - s11_values = recv_float32_array(sock, points * 2) - s21_values = recv_float32_array(sock, points * 2) - return SweepResult( - x=self._frequency_hz.copy(), - traces={ - "s11": self._complex_from_interleaved(s11_values), - "s21": self._complex_from_interleaved(s21_values), - }, - ) - - def _apply_configuration(self, sweep: RadarSweepModel) -> None: - sock = self._require_socket() - sock.sendall(COMMAND_CONFIGURE) - sock.sendall( - CONFIG_STRUCT.pack( - float(sweep.start_hz), - float(sweep.stop_hz), - int(sweep.points), - float(sweep.if_bandwidth_hz), - float(sweep.power_dbm), - ) - ) - read_status(sock) - returned_points = recv_u32(sock) - if returned_points != int(sweep.points): - raise RuntimeError(f"K209 remote config returned {returned_points} points, expected {sweep.points}") - self._frequency_hz = recv_float32_array(sock, int(sweep.points)) - - def _require_socket(self) -> socket.socket: - if self._socket is None: - raise RuntimeError("K209 remote socket is not open") - return self._socket - - @staticmethod - def _validate_sweep(sweep: RadarSweepModel) -> None: - if int(sweep.points) < 2: - raise ValueError("K209 sweep points must be >= 2") - if float(sweep.stop_hz) < float(sweep.start_hz): - raise ValueError("K209 sweep stop_hz must be >= start_hz") - if float(sweep.if_bandwidth_hz) <= 0.0: - raise ValueError("K209 IF bandwidth must be > 0") - - @staticmethod - def _complex_from_interleaved(values: np.ndarray) -> np.ndarray: - if values.size % 2 != 0: - raise RuntimeError("K209 remote complex trace payload has odd scalar count") - reshaped = np.asarray(values, dtype=np.float32).reshape((-1, 2)) - return (reshaped[:, 0] + 1j * reshaped[:, 1]).astype(np.complex64) diff --git a/python_app/hardware_full/single_radar_service.py b/python_app/hardware_full/single_radar_service.py index dc6a5d0..9902851 100644 --- a/python_app/hardware_full/single_radar_service.py +++ b/python_app/hardware_full/single_radar_service.py @@ -8,7 +8,6 @@ from typing import Protocol from python_app.hardware_full.kamil_adc import KamilAdcService from python_app.hardware_full.librevna_driver.models import SweepResult from python_app.hardware_full.librevna_service import LibreVnaService -from python_app.hardware_full.remote_compact_m_k209_service import RemoteCompactMK209Service from python_app.models.run_config_model import RadarSweepModel, RunConfigModel logger = logging.getLogger(__name__) @@ -28,9 +27,11 @@ class SingleRadarService(Protocol): def read_device_limits(self) -> dict[str, float | int]: """Read device capability limits.""" + ... def acquire(self) -> SweepResult: """Acquire one sweep.""" + ... def create_single_radar_service(config: RunConfigModel) -> SingleRadarService: @@ -44,17 +45,11 @@ def create_single_radar_service(config: RunConfigModel) -> SingleRadarService: model = config.radar.model or RunConfigModel.LIBREVNA_MODEL logger.debug("Creating single-radar service for model=%s (driver_mode=%s)", model, config.radar.driver_mode) if model == RunConfigModel.LIBREVNA_MODEL: - # Forward driver_mode (mirrors the matrix path): 'native' must require real - # hardware and 'mock' must use the synthetic backend — never silently the wrong one. - return LibreVnaService(serial=config.radar.serial or None, backend_mode=config.radar.driver_mode) + if config.radar.driver_mode == "mock": + from python_app.hardware_full.point_target_mock_service import PointTargetMockService - if model == RunConfigModel.COMPACT_M_K209_MODEL: - if config.radar.driver_mode != "native": - raise RuntimeError("Compact-M K209 requires radar.driver_mode='native'") - return RemoteCompactMK209Service( - host=config.radar.remote_host, - port=config.radar.remote_port, - ) + return PointTargetMockService(config) + return LibreVnaService(serial=config.radar.serial or None, backend_mode=config.radar.driver_mode) if model == RunConfigModel.KAMIL_ADC_MODEL: if config.radar.driver_mode != "native": diff --git a/python_app/hardware_full/sn9000_service.py b/python_app/hardware_full/sn9000_service.py deleted file mode 100644 index 2da22b3..0000000 --- a/python_app/hardware_full/sn9000_service.py +++ /dev/null @@ -1,395 +0,0 @@ -"""VISA HiSLIP driver for the PLANAR SN9000 (Иридиум series) multi-port VNA. - -The SN9000 hardware connects to a host PC over USB 2.0; the PC runs the SNVNA -application, which exposes the SCPI HiSLIP server. This service connects to -that server, configures a 2x4 virtual switch matrix (ports 1 and 2 stimulate, -ports 3..6 receive), and acquires the full matrix per call in a single -synchronized SCPI round trip. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -import logging -import time -from typing import Any - -import numpy as np -import pyvisa - -logger = logging.getLogger(__name__) - -from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData -from python_app.models.run_config_model import RadarSweepModel - -_OUTPUT_PORT_BY_INDEX: tuple[int, ...] = (1, 2) -_INPUT_PORT_BY_INDEX: tuple[int, ...] = (3, 4, 5, 6) -_REFLECTION_BY_OUTPUT: tuple[str, ...] = tuple( - f"S{port}{port}" for port in _OUTPUT_PORT_BY_INDEX -) -_S_PARAMETER_QUERY_ORDER: tuple[str, ...] = tuple( - name - for output_port in _OUTPUT_PORT_BY_INDEX - for name in ( - f"S{output_port}{output_port}", - *(f"S{receiver_port}{output_port}" for receiver_port in _INPUT_PORT_BY_INDEX), - ) -) - - -@dataclass(slots=True) -class Sn9000Service: - """Acquire the full 2x4 corrected sweep matrix from an SN9000 through VISA HiSLIP.""" - - host: str = "127.0.0.1" - port: int = 4880 - timeout_ms: int = 20_000 - preset_on_open: bool = True - visa_library: str = "@ivi" - _resource_manager: pyvisa.ResourceManager | None = field(init=False, default=None, repr=False) - _instrument: Any | None = field(init=False, default=None, repr=False) - _settings: RadarSweepModel | None = field(init=False, default=None, repr=False) - _frequency_hz: np.ndarray | None = field(init=False, default=None, repr=False) - - def __post_init__(self) -> None: - """Normalize and validate constructor values.""" - self.host = str(self.host).strip() - if not self.host: - raise ValueError("SN9000 host must not be empty") - self.port = int(self.port) - if self.port <= 0 or self.port > 65535: - raise ValueError("SN9000 port must be in 1..65535") - self.timeout_ms = int(self.timeout_ms) - if self.timeout_ms <= 0: - raise ValueError("SN9000 timeout_ms must be > 0") - self.visa_library = str(self.visa_library).strip() or "@ivi" - - @property - def resource(self) -> str: - """Return the assembled VISA HiSLIP resource string.""" - return f"TCPIP0::{self.host}::hislip0,{self.port}::INSTR" - - @property - def is_open(self) -> bool: - """Return whether the VISA session is open.""" - return self._instrument is not None - - def open(self) -> None: - """Open the VISA session and apply stored sweep settings when available.""" - if self._instrument is not None: - return - - logger.info("Opening SN9000 VISA session: %s", self.resource) - try: - self._resource_manager = pyvisa.ResourceManager(self.visa_library) - self._instrument = self._resource_manager.open_resource(self.resource) - self._instrument.timeout = self.timeout_ms - self._instrument.write_termination = "\n" - self._instrument.read_termination = None - self._instrument.chunk_size = max(int(getattr(self._instrument, "chunk_size", 20_480)), 8 * 1024 * 1024) - self._instrument.write("*CLS") - - if self._settings is not None: - self._apply_configuration(self._settings) - except Exception: - logger.exception("Failed to open SN9000 VISA session: %s", self.resource) - self.close() - raise - - def close(self) -> None: - """Close VISA sessions.""" - if self._instrument is not None: - logger.debug("Closing SN9000 VISA session") - self._instrument.close() - self._instrument = None - if self._resource_manager is not None: - self._resource_manager.close() - self._resource_manager = None - - def __enter__(self) -> Sn9000Service: - """Open the service and return it.""" - self.open() - return self - - def __exit__(self, exc_type: object, exc: object, traceback: object) -> None: - """Close VISA resources on context exit.""" - self.close() - - def recover(self) -> None: - """Reopen the VISA session after a transient acquisition failure.""" - logger.warning("Recovering SN9000 VISA session (close, wait, reopen)") - self.close() - time.sleep(0.25) - self.open() - - def query_identity(self) -> str: - """Read the analyzer identity string.""" - instrument = self._require_instrument() - return str(instrument.query("*IDN?")).strip() - - def query_system_error(self) -> str: - """Read one entry from the analyzer SCPI error queue.""" - instrument = self._require_instrument() - return str(instrument.query("SYST:ERR?")).strip() - - def configure(self, sweep: RadarSweepModel) -> None: - """Store and apply sweep settings.""" - self._validate_sweep(sweep) - self._settings = sweep - self._frequency_hz = None - logger.debug( - "Configuring SN9000 sweep: %s-%s Hz, %s points, IFBW=%s Hz, %s dBm", - sweep.start_hz, sweep.stop_hz, sweep.points, sweep.if_bandwidth_hz, sweep.power_dbm, - ) - if self._instrument is None: - return - self._apply_configuration(sweep) - - def read_device_limits(self) -> dict[str, float | int]: - """Read analyzer limits through SCPI capability/service queries.""" - opened_here = self._instrument is None - try: - if opened_here: - self.open() - instrument = self._require_instrument() - return { - "min_frequency_hz": float(instrument.query("SERV:SWE:FREQ:MIN?")), - "max_frequency_hz": float(instrument.query("SERV:SWE:FREQ:MAX?")), - # SN9000 SCPI does not expose IFBW capability queries; use the - # documented hardware sequence (1 Hz .. 300 kHz, manual p. 58, 1261). - "min_ifbw_hz": 1.0, - "max_ifbw_hz": 300_000.0, - "max_points": int(float(instrument.query("SERV:SWE:POIN?"))), - "min_power_dbm": float(instrument.query("SERV:SWE:POW:MIN?")), - "max_power_dbm": float(instrument.query("SERV:SWE:POW:MAX?")), - } - finally: - if opened_here: - self.close() - - def frequency_axis(self) -> np.ndarray: - """Return the cached frequency axis from the most recent configuration.""" - if self._frequency_hz is None: - raise RuntimeError("SN9000 frequency axis is not configured") - return self._frequency_hz - - def acquire_collection(self, collection_id: int = 1) -> SweepCollection: - """Acquire one full virtual 2x4 matrix in canonical combo order.""" - if self._settings is None: - raise RuntimeError("SN9000 service is not configured") - if self._frequency_hz is None: - raise RuntimeError("SN9000 frequency axis is not configured") - points = int(self._settings.points) - capture_start_ns = time.monotonic_ns() - - s_parameters = self._query_sweep_s_parameters(points) - traces = self._assemble_traces( - s_parameters, - sweep_start_ns=capture_start_ns, - sweep_end_ns=time.monotonic_ns(), - ) - - return SweepCollection( - collection_id=int(collection_id), - monotonic_ns=time.monotonic_ns(), - traces=traces, - capture_start_ns=capture_start_ns, - capture_end_ns=time.monotonic_ns(), - ) - - def _apply_configuration(self, sweep: RadarSweepModel) -> None: - instrument = self._require_instrument() - if self.preset_on_open: - instrument.write("SYST:PRES") - self._expect_opc("*OPC?", context="SN9000 preset") - - instrument.write(f"CALC:PAR:COUN {len(_S_PARAMETER_QUERY_ORDER)}") - for trace_index, parameter_name in enumerate(_S_PARAMETER_QUERY_ORDER, start=1): - instrument.write(f"CALC:PAR{trace_index}:DEF {parameter_name}") - - instrument.write("SENS:SWE:TYPE LIN") - instrument.write(f"SENS:FREQ:STAR {float(sweep.start_hz):.9f}") - instrument.write(f"SENS:FREQ:STOP {float(sweep.stop_hz):.9f}") - instrument.write(f"SENS:SWE:POIN {int(sweep.points)}") - instrument.write("SENS:SWE:POIN:TIME 0") - instrument.write(f"SENS:BAND {float(sweep.if_bandwidth_hz):.9f}") - instrument.write("SOUR:POW:PORT:COUP ON") - instrument.write(f"SOUR:POW {float(sweep.power_dbm):.3f}") - instrument.write("SENS:AVER OFF") - instrument.write("FORM:DATA REAL32") - instrument.write("FORM:BORD SWAP") - instrument.write("INIT:CONT ON") - instrument.write("TRIG:SOUR BUS") - self._expect_opc("*OPC?", context="SN9000 setup") - self._frequency_hz = self._query_float32_array("SENS:FREQ:DATA?", int(sweep.points)) - logger.info( - "SN9000 configured: %d traces, %d points (%s-%s Hz)", - len(_S_PARAMETER_QUERY_ORDER), int(sweep.points), sweep.start_hz, sweep.stop_hz, - ) - - def _query_sweep_s_parameters(self, points: int) -> dict[str, np.ndarray]: - instrument = self._require_instrument() - if self._uses_pyvisa_py_backend(): - # pyvisa-py HiSLIP loses synchronization when a single packet aggregates - # *OPC? plus multiple binary blocks, so issue trigger and data queries - # one at a time. The corrected-data buffer holds the last completed - # sweep, so reading each S-parameter sequentially is safe. - instrument.write("TRIG:SING") - self._expect_opc("*OPC?", context="SN9000 sweep") - complex_values: dict[str, np.ndarray] = {} - for parameter_name in _S_PARAMETER_QUERY_ORDER: - instrument.write(f"SENS:DATA:CORR? {parameter_name}") - interleaved = self._read_float32_block( - f"SENS:DATA:CORR? {parameter_name}", points * 2 - ) - complex_values[parameter_name] = self._complex_from_interleaved(interleaved) - return complex_values - - data_queries = ";".join(f":SENS:DATA:CORR? {name}" for name in _S_PARAMETER_QUERY_ORDER) - instrument.write(f"TRIG:SING;*OPC?;{data_queries}") - opc_token = self._read_ascii_token() - if opc_token != "1": - raise RuntimeError(f"SN9000 sweep returned unexpected *OPC? response: {opc_token!r}") - - complex_values = {} - for parameter_name in _S_PARAMETER_QUERY_ORDER: - interleaved = self._read_float32_block(f"SENS:DATA:CORR? {parameter_name}", points * 2) - complex_values[parameter_name] = self._complex_from_interleaved(interleaved) - return complex_values - - def _uses_pyvisa_py_backend(self) -> bool: - return self.visa_library == "@py" or self.visa_library.endswith("@py") - - def _assemble_traces( - self, - s_parameters: dict[str, np.ndarray], - *, - sweep_start_ns: int, - sweep_end_ns: int, - ) -> list[TraceData]: - frequency_hz = self._require_frequency_axis() - traces: list[TraceData] = [] - for output_position, output_port in enumerate(_OUTPUT_PORT_BY_INDEX): - reflection = s_parameters[_REFLECTION_BY_OUTPUT[output_position]] - for input_position, receiver_port in enumerate(_INPUT_PORT_BY_INDEX): - transmission = s_parameters[f"S{receiver_port}{output_port}"] - traces.append( - TraceData( - combo=ComboKey(input=input_position, output=output_position), - frequency_hz=frequency_hz, - s11=reflection, - s21=transmission, - # One triggered sweep produces every port pair at once, so - # all combos share the sweep's window. - capture_start_ns=int(sweep_start_ns), - capture_end_ns=int(sweep_end_ns), - ) - ) - return traces - - def _expect_opc(self, command: str, *, context: str) -> None: - instrument = self._require_instrument() - response = str(instrument.query(command)).strip() - if response != "1": - raise RuntimeError(f"{context} returned unexpected *OPC? response: {response!r}") - - def _query_float32_array(self, command: str, expected_values: int) -> np.ndarray: - instrument = self._require_instrument() - instrument.write(command) - return self._read_float32_block(command, expected_values) - - def _read_ascii_token(self) -> str: - token = bytearray() - while True: - byte = self._read_response_bytes(1) - if byte in (b";", b"\n", b"\r"): - if token: - return token.decode("ascii").strip() - continue - token.extend(byte) - if len(token) > 64 * 1024: - raise RuntimeError("SN9000 ASCII response token is too long") - - def _read_float32_block(self, context: str, expected_values: int) -> np.ndarray: - marker = self._read_response_bytes(1) - while marker in (b";", b"\n", b"\r"): - marker = self._read_response_bytes(1) - if marker != b"#": - raise RuntimeError(f"SN9000 response for {context!r} does not start with IEEE block marker") - - width = self._read_response_bytes(1) - if width != b"8": - raise RuntimeError(f"SN9000 response for {context!r} uses unsupported IEEE block header width") - - payload_size = int(self._read_response_bytes(8).decode("ascii")) - expected_size = expected_values * np.dtype(np.float32).itemsize - if payload_size != expected_size: - raise RuntimeError( - f"SN9000 response for {context!r} returned {payload_size} payload bytes, " - f"expected {expected_size}" - ) - - payload = self._read_response_bytes(payload_size) - array = np.frombuffer(payload, dtype=" None: - """Consume the SCPI terminator that follows IEEE binary blocks. - - SCPI responses end with `\\n`, which over HiSLIP closes the DataEnd - message group. pyvisa-py's HiSLIP layer needs the terminator drained - before the next request, otherwise it loses message-frame - synchronization on subsequent reads. - """ - instrument = self._require_instrument() - deadline = time.monotonic() + 0.2 - while time.monotonic() < deadline: - try: - instrument.read_bytes(1, break_on_termchar=True) - return - except pyvisa.errors.VisaIOError as exc: - # Timeouts on a trailing newline are routine; anything else - # likely means HiSLIP framing is out of sync and the next - # request will hang — surface it in the logs. - if exc.error_code != pyvisa.constants.StatusCode.error_timeout: - logger.warning("SN9000 terminator drain failed: %s", exc) - return - - def _read_response_bytes(self, count: int) -> bytes: - instrument = self._require_instrument() - data = instrument.read_bytes(count, break_on_termchar=False) - if len(data) != count: - raise RuntimeError(f"SN9000 response ended after {len(data)} bytes, expected {count}") - return data - - def _require_instrument(self) -> Any: - if self._instrument is None: - raise RuntimeError("SN9000 VISA instrument is not open") - return self._instrument - - def _require_frequency_axis(self) -> np.ndarray: - if self._frequency_hz is None: - raise RuntimeError("SN9000 frequency axis is not configured") - return self._frequency_hz - - @staticmethod - def _validate_sweep(sweep: RadarSweepModel) -> None: - if int(sweep.points) < 2: - raise ValueError("SN9000 sweep points must be >= 2") - if float(sweep.stop_hz) < float(sweep.start_hz): - raise ValueError("SN9000 sweep stop_hz must be >= start_hz") - if float(sweep.if_bandwidth_hz) <= 0.0: - raise ValueError("SN9000 IF bandwidth must be > 0") - - @staticmethod - def _complex_from_interleaved(values: np.ndarray) -> np.ndarray: - if values.size % 2 != 0: - raise RuntimeError("SN9000 complex trace payload has odd scalar count") - reshaped = np.asarray(values, dtype=np.float32).reshape((-1, 2)) - return (reshaped[:, 0] + 1j * reshaped[:, 1]).astype(np.complex64) diff --git a/python_app/hardware_full/switch_drivers/h7992_driver.py b/python_app/hardware_full/switch_drivers/h7992_driver.py index 1902270..7ddc55e 100644 --- a/python_app/hardware_full/switch_drivers/h7992_driver.py +++ b/python_app/hardware_full/switch_drivers/h7992_driver.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from contextlib import suppress from python_app.hardware_full.switch_drivers.gpio_uapi import GpioOutputLines @@ -29,12 +30,6 @@ class H7992Driver: _is_open: bool = field(init=False, default=False, repr=False) _current_position: int = field(init=False, default=0, repr=False) - def __post_init__(self) -> None: - """Initialize runtime state.""" - self._lines: GpioOutputLines | None = None - self._is_open = False - self._current_position = 0 - def open(self) -> None: """Open GPIO lines and switch to default position.""" if self._is_open: @@ -46,16 +41,21 @@ class H7992Driver: offsets=(self.pin_a, self.pin_b), consumer=f"radar_{self.name}", ) - self._lines.open() - self._is_open = True - self.switch_to(self.default_position) + try: + self._lines.open() + self._is_open = True + self.switch_to(self.default_position) + except Exception: + with suppress(Exception): + self.close() + raise def close(self) -> None: """Close GPIO lines.""" - if self._lines is not None: - self._lines.close() - self._lines = None + lines, self._lines = self._lines, None self._is_open = False + if lines is not None: + lines.close() def position_count(self) -> int: """Return number of supported positions.""" diff --git a/python_app/hardware_full/switch_drivers/hmc349a_driver.py b/python_app/hardware_full/switch_drivers/hmc349a_driver.py index 726afa9..9b254b6 100644 --- a/python_app/hardware_full/switch_drivers/hmc349a_driver.py +++ b/python_app/hardware_full/switch_drivers/hmc349a_driver.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from contextlib import suppress from python_app.hardware_full.switch_drivers.gpio_uapi import GpioOutputLines @@ -21,12 +22,6 @@ class HMC349ADriver: _is_open: bool = field(init=False, default=False, repr=False) _current_position: int = field(init=False, default=0, repr=False) - def __post_init__(self) -> None: - """Initialize runtime state.""" - self._lines: GpioOutputLines | None = None - self._is_open = False - self._current_position = 0 - def open(self) -> None: """Open GPIO lines and switch to default position.""" if self._is_open: @@ -40,16 +35,21 @@ class HMC349ADriver: offsets=offsets, consumer=f"radar_{self.name}", ) - self._lines.open() - self._is_open = True - self.switch_to(self.default_position) + try: + self._lines.open() + self._is_open = True + self.switch_to(self.default_position) + except Exception: + with suppress(Exception): + self.close() + raise def close(self) -> None: """Close GPIO lines.""" - if self._lines is not None: - self._lines.close() - self._lines = None + lines, self._lines = self._lines, None self._is_open = False + if lines is not None: + lines.close() def position_count(self) -> int: """Return number of supported positions.""" diff --git a/python_app/hardware_full/switched_matrix_radar_service.py b/python_app/hardware_full/switched_matrix_radar_service.py index 22ff2ad..fdbc546 100644 --- a/python_app/hardware_full/switched_matrix_radar_service.py +++ b/python_app/hardware_full/switched_matrix_radar_service.py @@ -38,21 +38,24 @@ class SwitchedMatrixRadarService: def open(self) -> None: """Open the inner radar and both switches.""" - self.inner.open() - if self.output_switch is not None: - self.output_switch.open() - if self.input_switch is not None: - self.input_switch.open() + try: + self.inner.open() + if self.output_switch is not None: + self.output_switch.open() + if self.input_switch is not None: + self.input_switch.open() + except Exception: + self.close() + raise def close(self) -> None: """Close switches first, then the inner radar; never raises.""" - for switch in (self.input_switch, self.output_switch): - if switch is not None: + for resource in (self.input_switch, self.output_switch, self.inner): + if resource is not None: try: - switch.close() + resource.close() except Exception as exc: # noqa: BLE001 — shutdown path - logger.warning("Switch close ignored error: %s", exc) - self.inner.close() + logger.warning("Switched matrix cleanup failed: %s", exc) def configure(self, sweep: RadarSweepModel) -> None: """Apply sweep settings to the inner radar.""" @@ -211,4 +214,4 @@ def build_physical_switch( driver_mode = "mock" if radar_driver_mode.strip().lower() == "mock" else "native" return SwitchService.from_model( replace(model, positions=physical_positions, driver_mode=driver_mode) - ) \ No newline at end of file + ) diff --git a/python_app/logging_setup.py b/python_app/logging_setup.py index 718e2f5..0d30ab5 100644 --- a/python_app/logging_setup.py +++ b/python_app/logging_setup.py @@ -51,12 +51,6 @@ def coerce_level(value: object) -> int: return resolved if isinstance(resolved, int) else logging.INFO -def normalize_level_name(value: object) -> str: - """Return a canonical UPPERCASE level name from the supported set (default INFO).""" - name = str(value).strip().upper() - return name if name in LOG_LEVELS else DEFAULT_LOG_LEVEL - - def package_logger() -> logging.Logger: """Return the application's root logger.""" return logging.getLogger(PACKAGE_LOGGER_NAME) diff --git a/python_app/models/gui_profile_codec.py b/python_app/models/gui_profile_codec.py index 306b822..ae67abb 100644 --- a/python_app/models/gui_profile_codec.py +++ b/python_app/models/gui_profile_codec.py @@ -6,6 +6,7 @@ import dataclasses import logging from typing import Any +from python_app.models.json_fields import finite_number, json_object as _as_dict from python_app.models.gui_profile_schema import ( GuiBscanStateModel, GuiDataActionsStateModel, @@ -24,15 +25,6 @@ from python_app.models.run_config_model import RunConfigModel logger = logging.getLogger(__name__) -def _as_dict(value: Any, context: str) -> dict[str, Any]: - """Validate payload node is object-like, treating missing values as empty object.""" - if value is None: - return {} - if not isinstance(value, dict): - raise ValueError(f"{context} must be a JSON object") - return value - - def _optional_string(object_payload: dict[str, Any], key: str, fallback: str, context: str) -> str: """Return optional string field with type validation.""" raw_value = object_payload.get(key, fallback) @@ -59,10 +51,7 @@ def _optional_int(object_payload: dict[str, Any], key: str, fallback: int, conte def _optional_float(object_payload: dict[str, Any], key: str, fallback: float, context: str) -> float: """Return optional numeric field with type validation.""" - raw_value = object_payload.get(key, fallback) - if isinstance(raw_value, bool) or not isinstance(raw_value, (int, float)): - raise ValueError(f"{context}.{key} must be a JSON number") - return float(raw_value) + return finite_number(object_payload.get(key, fallback), f"{context}.{key}") def _legacy_gpr_mode_from_algorithm(value: str) -> str | None: @@ -533,12 +522,6 @@ def gui_profile_from_dict(payload: dict[str, Any]) -> GuiProfileModel: data_actions_object = _as_dict(gui_object.get("data_actions"), "gui.data_actions") gui.data_actions = GuiDataActionsStateModel( - save_count=_optional_int( - data_actions_object, - "save_count", - gui.data_actions.save_count, - "gui.data_actions", - ), save_path=_optional_string( data_actions_object, "save_path", @@ -598,102 +581,5 @@ def gui_profile_from_dict(payload: dict[str, Any]) -> GuiProfileModel: def gui_profile_to_dict(model: GuiProfileModel) -> dict[str, Any]: """Encode :class:`GuiProfileModel` into JSON-serializable dictionary.""" payload = model.run_config.to_dict() - gui = model.gui if model.gui is not None else GuiStateModel() - payload["gui"] = { - "version": gui.version, - "switches": { - "combo_mode": gui.switches.combo_mode, - "combos_text": gui.switches.combos_text, - "single_input": gui.switches.single_input, - "single_output": gui.switches.single_output, - }, - "processing": { - "selected_mode": gui.processing.selected_mode, - "pass_through": { - "show_magnitude": gui.processing.pass_through.show_magnitude, - "show_phase": gui.processing.pass_through.show_phase, - "unwrap_phase": gui.processing.pass_through.unwrap_phase, - "combo_filter": gui.processing.pass_through.combo_filter, - "fixed_y_enabled": gui.processing.pass_through.fixed_y_enabled, - "y_min_db": gui.processing.pass_through.y_min_db, - "y_max_db": gui.processing.pass_through.y_max_db, - }, - "bscan": { - "axis": gui.processing.bscan.axis, - "cut_m": gui.processing.bscan.cut_m, - "max_depth_m": gui.processing.bscan.max_depth_m, - "gain": gui.processing.bscan.gain, - "start_freq_mhz": gui.processing.bscan.start_freq_mhz, - "stop_freq_mhz": gui.processing.bscan.stop_freq_mhz, - "subtract_mean_ascan": gui.processing.bscan.subtract_mean_ascan, - "history_window_scans": gui.processing.bscan.history_window_scans, - }, - "gpr": { - "input_positions": gui.processing.gpr.input_positions, - "output_positions": gui.processing.gpr.output_positions, - "min_depth_m": gui.processing.gpr.min_depth_m, - "max_depth_m": gui.processing.gpr.max_depth_m, - "range_comp_power": gui.processing.gpr.range_comp_power, - "object_min_frac": gui.processing.gpr.object_min_frac, - "score_mode": gui.processing.gpr.score_mode, - "motion_mode": gui.processing.gpr.motion_mode, - "look_angle_deg": gui.processing.gpr.look_angle_deg, - "direction_sign": gui.processing.gpr.direction_sign, - "speed_m_s": gui.processing.gpr.speed_m_s, - "ignore_socket_speed_enabled": gui.processing.gpr.ignore_socket_speed_enabled, - "max_detected_objects_to_draw": gui.processing.gpr.max_detected_objects_to_draw, - "draw_top_m_objects": gui.processing.gpr.draw_top_m_objects, - "object_approach_min_frames": gui.processing.gpr.object_approach_min_frames, - "start_freq_mhz": gui.processing.gpr.start_freq_mhz, - "stop_freq_mhz": gui.processing.gpr.stop_freq_mhz, - "background_subtract_enabled": gui.processing.gpr.background_subtract_enabled, - "background_mean_count": gui.processing.gpr.background_mean_count, - "remove_sidelobe_objects_enabled": gui.processing.gpr.remove_sidelobe_objects_enabled, - "imaging_plane_y_m": gui.processing.gpr.imaging_plane_y_m, - "render_mode": gui.processing.gpr.render_mode, - "visible_x_min_m": gui.processing.gpr.visible_x_min_m, - "visible_x_max_m": gui.processing.gpr.visible_x_max_m, - "visible_z_min_m": gui.processing.gpr.visible_z_min_m, - "visible_z_max_m": gui.processing.gpr.visible_z_max_m, - }, - "legacy_gpr": { - "mode": gui.processing.legacy_gpr.mode, - "input_positions": gui.processing.legacy_gpr.input_positions, - "output_positions": gui.processing.legacy_gpr.output_positions, - "min_depth_m": gui.processing.legacy_gpr.min_depth_m, - "max_depth_m": gui.processing.legacy_gpr.max_depth_m, - "comp_power": gui.processing.legacy_gpr.comp_power, - "start_freq_mhz": gui.processing.legacy_gpr.start_freq_mhz, - "stop_freq_mhz": gui.processing.legacy_gpr.stop_freq_mhz, - "speed_m_s": gui.processing.legacy_gpr.speed_m_s, - "ignore_socket_speed_enabled": gui.processing.legacy_gpr.ignore_socket_speed_enabled, - "look_angle_deg": gui.processing.legacy_gpr.look_angle_deg, - "apply_freq_phase_correction": gui.processing.legacy_gpr.apply_freq_phase_correction, - "reference_mode": gui.processing.legacy_gpr.reference_mode, - "snr_thresh": gui.processing.legacy_gpr.snr_thresh, - "snr_comp_max": gui.processing.legacy_gpr.snr_comp_max, - "background_subtract_enabled": gui.processing.legacy_gpr.background_subtract_enabled, - "background_mean_count": gui.processing.legacy_gpr.background_mean_count, - "render_mode": gui.processing.legacy_gpr.render_mode, - "min_visible_pair_count": gui.processing.legacy_gpr.min_visible_pair_count, - "visible_x_min_m": gui.processing.legacy_gpr.visible_x_min_m, - "visible_x_max_m": gui.processing.legacy_gpr.visible_x_max_m, - "visible_z_min_m": gui.processing.legacy_gpr.visible_z_min_m, - "visible_z_max_m": gui.processing.legacy_gpr.visible_z_max_m, - }, - "sar3d": dataclasses.asdict(gui.processing.sar3d), - }, - "data_actions": { - "save_count": gui.data_actions.save_count, - "save_path": gui.data_actions.save_path, - "save_name": gui.data_actions.save_name, - "record_count": gui.data_actions.record_count, - }, - "preprocess_dialog": { - "set_name": gui.preprocess_dialog.set_name, - "radar_config_dir": gui.preprocess_dialog.radar_config_dir, - "use_all_radar_configs": gui.preprocess_dialog.use_all_radar_configs, - "median_sweep_count": gui.preprocess_dialog.median_sweep_count, - }, - } + payload["gui"] = dataclasses.asdict(model.gui if model.gui is not None else GuiStateModel()) return payload diff --git a/python_app/models/gui_profile_schema.py b/python_app/models/gui_profile_schema.py index ca7c722..1477daa 100644 --- a/python_app/models/gui_profile_schema.py +++ b/python_app/models/gui_profile_schema.py @@ -42,8 +42,8 @@ class GuiBscanStateModel: """UI-only defaults for B-scan live settings.""" axis: str = "abs" - cut_m: float = 0.824 - max_depth_m: float = 1.0 + cut_m: float = 0.0 + max_depth_m: float = 8.0 gain: float = 1.0 start_freq_mhz: float = 100.0 stop_freq_mhz: float = 8800.0 @@ -164,7 +164,6 @@ class GuiProcessingStateModel: class GuiDataActionsStateModel: """UI-only defaults for snapshot/export controls.""" - save_count: int = 10 save_path: str = "" save_name: str = "snapshot_manual" # How many freshly acquired measurements the "Start + Record" action writes to diff --git a/python_app/models/json_fields.py b/python_app/models/json_fields.py new file mode 100644 index 0000000..703fa45 --- /dev/null +++ b/python_app/models/json_fields.py @@ -0,0 +1,62 @@ +"""Shared strict scalar and object readers for configuration codecs.""" + +from __future__ import annotations + +import math +from typing import Any + + +def json_object(value: Any, context: str) -> dict[str, Any]: + """Read an object; a missing or null section keeps its defaults.""" + if value is None: + return {} + if not isinstance(value, dict): + raise ValueError(f"{context} must be a JSON object") + return value + + +def _value(payload: dict[str, Any], key: str, default: Any) -> Any: + value = payload.get(key, default) + return default if value is None else value + + +def read_int(payload: dict[str, Any], key: str, default: int) -> int: + """Read a JSON integer without truncation or boolean coercion.""" + value = _value(payload, key, default) + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{key} must be a JSON integer") + return value + + +def read_str(payload: dict[str, Any], key: str, default: str) -> str: + """Read a JSON string without converting other scalar types.""" + value = _value(payload, key, default) + if not isinstance(value, str): + raise ValueError(f"{key} must be a JSON string") + return value + + +def read_bool(payload: dict[str, Any], key: str, default: bool) -> bool: + """Read a JSON boolean.""" + value = _value(payload, key, default) + if not isinstance(value, bool): + raise ValueError(f"{key} must be a JSON boolean") + return value + + +def read_float(payload: dict[str, Any], key: str, default: float) -> float: + """Read a finite JSON number, including overflow-safe integer conversion.""" + return finite_number(_value(payload, key, default), key) + + +def finite_number(value: Any, context: str) -> float: + """Validate a number without accepting booleans, strings or nonfinite values.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{context} must be a JSON number") + try: + result = float(value) + except OverflowError as exc: + raise ValueError(f"{context} must be a finite number") from exc + if not math.isfinite(result): + raise ValueError(f"{context} must be a finite number") + return result diff --git a/python_app/models/run_config_codec.py b/python_app/models/run_config_codec.py index 81aba14..3081ba2 100644 --- a/python_app/models/run_config_codec.py +++ b/python_app/models/run_config_codec.py @@ -3,10 +3,13 @@ from __future__ import annotations import logging -import math from typing import Any from python_app.logging_setup import LOG_LEVELS +from python_app.models.json_fields import ( + finite_number, json_object as _as_dict, read_bool as _read_bool, read_float as _read_float, + read_int as _read_int, read_str as _read_str, +) from python_app.models.run_config_schema import ( ComboModel, @@ -27,15 +30,6 @@ from python_app.models.run_config_validation import ( logger = logging.getLogger(__name__) -def _as_dict(value: Any, context: str) -> dict[str, Any]: - """Validate payload node is object-like, treating missing values as empty object.""" - if value is None: - return {} - if not isinstance(value, dict): - raise ValueError(f"{context} must be a JSON object") - return value - - def _as_list(value: Any, context: str) -> list[Any]: """Validate payload node is array-like, treating missing values as an empty list. @@ -49,71 +43,6 @@ def _as_list(value: Any, context: str) -> list[Any]: return value -def _read_str(payload: dict[str, Any], key: str, default: str) -> str: - """Return a payload string, treating an explicit JSON ``null`` as 'use default'. - - Keeping the default on ``null`` avoids coercing it to the literal string - ``"None"``. JSON arrays/objects reaching a scalar field are rejected as - ValueError to keep the config-error contract uniform. - """ - value = payload.get(key, default) - if value is None: - return default - if isinstance(value, (dict, list)): - raise ValueError(f"{key} must be a JSON string") - return str(value) - - -def _read_int(payload: dict[str, Any], key: str, default: int) -> int: - """Return a payload integer, treating an explicit JSON ``null`` as 'use default'. - - Accepts only a genuine JSON integer (not bool, not float, not numeric - string); silently truncating ``5.7`` or parsing ``"5"`` would hide a - malformed config. Mirrors ``gui_profile_codec._optional_int`` so the two - codecs agree. - """ - value = payload.get(key, default) - if value is None: - return default - if isinstance(value, bool) or not isinstance(value, int): - raise ValueError(f"{key} must be a JSON integer") - return value - - -def _read_float(payload: dict[str, Any], key: str, default: float) -> float: - """Return a payload float, treating an explicit JSON ``null`` as 'use default'. - - Accepts only a genuine JSON number (int/float, not bool, not numeric - string); parsing ``"1e9"`` would hide a malformed config. Non-finite values - (NaN/Infinity) are rejected at decode time so the C++ pipeline never receives - a value it cannot honor. Mirrors ``gui_profile_codec._optional_float``. - """ - value = payload.get(key, default) - if value is None: - return default - if isinstance(value, bool) or not isinstance(value, (int, float)): - raise ValueError(f"{key} must be a JSON number") - result = float(value) - if not math.isfinite(result): - raise ValueError(f"{key} must be a finite number") - return result - - -def _read_bool(payload: dict[str, Any], key: str, default: bool) -> bool: - """Return a payload boolean, treating an explicit JSON ``null`` as 'use default'. - - Keeping the default on ``null`` avoids the silent flip to ``False`` that a - plain ``bool(...)`` coercion would produce. Non-boolean JSON types are - rejected as ValueError. - """ - value = payload.get(key, default) - if value is None: - return default - if not isinstance(value, bool): - raise ValueError(f"{key} must be a JSON boolean") - return value - - def _load_preprocess_asset(payload: dict[str, Any], target: PreprocessAssetModel) -> None: """Load preprocess asset fields into target model.""" target.set_name = _read_str(payload, "set_name", target.set_name) @@ -152,6 +81,8 @@ def _load_string_dict(payload: dict[str, Any], key: str, context: str) -> dict[s def run_config_from_dict(payload: dict[str, Any]) -> RunConfigModel: """Decode JSON-like payload into :class:`RunConfigModel`.""" + if not isinstance(payload, dict): + raise ValueError("Run config root must be a JSON object") # Schema carries only minimal-safe fallbacks; operational defaults live in run_config.json. model = RunConfigModel() @@ -181,11 +112,7 @@ def run_config_from_dict(payload: dict[str, Any]) -> RunConfigModel: model.radar.model = _read_str(radar_payload, "model", model.radar.model) model.radar.serial = _read_str(radar_payload, "serial", model.radar.serial) - model.radar.remote_host = _read_str(radar_payload, "remote_host", model.radar.remote_host) - model.radar.remote_port = _read_int(radar_payload, "remote_port", model.radar.remote_port) model.radar.driver_mode = _read_str(radar_payload, "driver_mode", model.radar.driver_mode) - model.radar.mock_signal_hz = _read_float(radar_payload, "mock_signal_hz", model.radar.mock_signal_hz) - model.radar.visa_library = _read_str(radar_payload, "visa_library", model.radar.visa_library) model.radar.sweep.start_hz = _read_float(sweep_payload, "start_hz", model.radar.sweep.start_hz) model.radar.sweep.stop_hz = _read_float(sweep_payload, "stop_hz", model.radar.sweep.stop_hz) @@ -429,7 +356,14 @@ def run_config_from_dict(payload: dict[str, Any]) -> RunConfigModel: for band in bands_payload: if not (isinstance(band, (list, tuple)) and len(band) == 2): raise ValueError("preprocess.notch.bands_hz entries must be [low_hz, high_hz] pairs") - model.preprocess.notch.bands_hz.append((float(band[0]), float(band[1]))) + low_hz, high_hz = (finite_number(value, "preprocess.notch.bands_hz[]") for value in band) + if high_hz < low_hz or max(abs(low_hz), abs(high_hz)) > 3.4e38: + raise ValueError("preprocess.notch.bands_hz[] must be ordered finite float32 frequencies") + model.preprocess.notch.bands_hz.append((low_hz, high_hz)) + if model.preprocess.notch.taper_width_hz < 0 or model.preprocess.notch.taper_width_hz > 3.4e38: + raise ValueError("preprocess.notch.taper_width_hz must be a nonnegative float32 number") + if model.preprocess.notch.taper_type not in ("cosine", "hard"): + raise ValueError("preprocess.notch.taper_type must be cosine or hard") model.gpr.relative_permittivity = _read_float( gpr_payload, "relative_permittivity", model.gpr.relative_permittivity @@ -514,11 +448,7 @@ def run_config_to_dict(model: RunConfigModel) -> dict[str, Any]: "radar": { "model": model.radar.model, "serial": model.radar.serial, - "remote_host": model.radar.remote_host, - "remote_port": model.radar.remote_port, "driver_mode": model.radar.driver_mode, - "mock_signal_hz": model.radar.mock_signal_hz, - "visa_library": model.radar.visa_library, "multi_device": { "slave_serials": list(model.radar.multi_device.slave_serials), "force_external_reference": model.radar.multi_device.force_external_reference, diff --git a/python_app/models/run_config_schema.py b/python_app/models/run_config_schema.py index 1600648..ef216cb 100644 --- a/python_app/models/run_config_schema.py +++ b/python_app/models/run_config_schema.py @@ -10,6 +10,7 @@ from pathlib import Path from typing import Any logger = logging.getLogger(__name__) +MAX_COMBOS = 4096 @dataclass(slots=True) @@ -171,11 +172,7 @@ class RadarModel: model: str = "librevna" serial: str = "" - remote_host: str = "127.0.0.1" - remote_port: int = 50209 driver_mode: str = "mock" - mock_signal_hz: float = 1_000_000.0 - visa_library: str = "" sweep: RadarSweepModel = field(default_factory=RadarSweepModel) multi_device: RadarMultiDeviceModel = field(default_factory=RadarMultiDeviceModel) replay: RadarReplayModel = field(default_factory=RadarReplayModel) @@ -377,9 +374,7 @@ class RunConfigModel: LIBREVNA_MODEL = "librevna" LIBREVNA_MULTI_MODEL = "librevna_multi" - COMPACT_M_K209_MODEL = "compact_m_k209" KAMIL_ADC_MODEL = "kamil_adc" - SN9000_MODEL = "sn9000" MULTI_DEVICE_INPUT_POSITIONS = 4 MULTI_DEVICE_OUTPUT_POSITIONS = 2 # Stimulus ports the master itself owns. The virtual output axis is this times the @@ -390,39 +385,26 @@ class RunConfigModel: @staticmethod def build_full_combos(input_positions: int, output_positions: int) -> list[ComboModel]: """Build full Cartesian product of input/output switch positions.""" + if any(isinstance(value, bool) or not isinstance(value, int) or value <= 0 + for value in (input_positions, output_positions)): + raise ValueError("Switch positions must be positive integers") + if input_positions * output_positions > MAX_COMBOS: + raise ValueError(f"Too many combos: limit is {MAX_COMBOS}") return [ ComboModel(input=input_pos, output=output_pos) for output_pos in range(output_positions) for input_pos in range(input_positions) ] - @classmethod - def build_multi_device_virtual_combos(cls) -> list[ComboModel]: - """Build fixed virtual combo matrix for one master and two slave devices.""" - return cls.build_full_combos( - cls.MULTI_DEVICE_INPUT_POSITIONS, - cls.MULTI_DEVICE_OUTPUT_POSITIONS, - ) - - @classmethod - def build_matrix_radar_virtual_combos(cls) -> list[ComboModel]: - """Build the canonical 2x4 virtual combo matrix shared by matrix-mode radars.""" - return cls.build_multi_device_virtual_combos() - @property def is_multi_device(self) -> bool: """Return whether this config targets synchronized multi-device acquisition.""" return self.radar.model == self.LIBREVNA_MULTI_MODEL - @property - def is_sn9000(self) -> bool: - """Return whether this config targets the SN9000 multi-port analyzer.""" - return self.radar.model == self.SN9000_MODEL - @property def is_matrix_radar(self) -> bool: """Return whether this config acquires the full virtual switch matrix per sweep.""" - return self.is_multi_device or self.is_sn9000 + return self.is_multi_device @property def matrix_output_switch_positions(self) -> int: diff --git a/python_app/models/run_config_validation.py b/python_app/models/run_config_validation.py index c2ac04d..003f924 100644 --- a/python_app/models/run_config_validation.py +++ b/python_app/models/run_config_validation.py @@ -3,14 +3,20 @@ from __future__ import annotations import logging +import math from typing import Any +from python_app.models.json_fields import ( + read_bool as _require_bool, read_int as _require_int, read_str as _require_str, +) from python_app.models.run_config_schema import ( + MAX_COMBOS, ComboModel, ControlButtonModel, GprModel, RadarSweepModel, RingEndpointModel, + RunConfigModel, SwitchModel, ) @@ -21,45 +27,6 @@ logger = logging.getLogger(__name__) # shared-memory mapping, so reject values the C++ side cannot represent. _UINT32_MAX = (1 << 32) - 1 _RING_SEGMENT_MAX_BYTES = 1 << 40 # 1 TiB upper bound on a single ring mapping. -# Defensive ceiling so a malformed combos string cannot expand into a list that -# stalls the GUI or the downstream acquisition loop. -_MAX_COMBOS = 4096 - - -def _require_int(payload: dict[str, Any], key: str, default: int) -> int: - """Read a strict JSON integer, treating an explicit ``null`` as 'use default'. - - Accepts only a genuine JSON integer (not bool, not float, not numeric string), - because silently truncating ``5.7`` or parsing ``"5"`` would hide a malformed - config. Mirrors ``run_config_codec._read_int`` so every config integer reads - identically. - """ - value = payload.get(key, default) - if value is None: # explicit JSON null -> use the default, never coerce - return default - if isinstance(value, bool) or not isinstance(value, int): - raise ValueError(f"{key} must be a JSON integer") - return value - - -def _require_str(payload: dict[str, Any], key: str, default: str) -> str: - """Read a string field, rejecting JSON arrays/objects with a named ValueError.""" - value = payload.get(key, default) - if value is None: # explicit JSON null -> use the default, never coerce to "None" - return default - if isinstance(value, (dict, list)): - raise ValueError(f"{key} must be a JSON string") - return str(value) - - -def _require_bool(payload: dict[str, Any], key: str, default: bool) -> bool: - """Read a boolean field, rejecting non-boolean JSON types with a named ValueError.""" - value = payload.get(key, default) - if value is None: # explicit JSON null -> use the default - return default - if not isinstance(value, bool): - raise ValueError(f"{key} must be a JSON boolean") - return value def load_switch_payload( @@ -117,18 +84,79 @@ def validate_ring_endpoint(ring: RingEndpointModel) -> None: would exceed the maximum single mapping. """ ring_name = ring.name or "ring" + for field, value in (("capacity", ring.capacity), ("slot_size_bytes", ring.slot_size_bytes)): + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"rings.{ring_name}.{field} must be an integer") if ring.capacity <= 0: raise ValueError(f"rings.{ring_name}.capacity must be > 0") if ring.slot_size_bytes <= 0: raise ValueError(f"rings.{ring_name}.slot_size_bytes must be > 0") if ring.slot_size_bytes > _UINT32_MAX: raise ValueError(f"rings.{ring_name}.slot_size_bytes exceeds the uint32 wire limit") + if ring.slot_size_bytes % 8: + raise ValueError(f"rings.{ring_name}.slot_size_bytes must be a multiple of 8") # Overflow-safe: compare against the ceiling without ever forming the full # product, so an oversized capacity cannot wrap a fixed-width index. - if ring.capacity > _RING_SEGMENT_MAX_BYTES // ring.slot_size_bytes: + if ring.capacity > (_RING_SEGMENT_MAX_BYTES - 64) // (ring.slot_size_bytes + 16): raise ValueError( f"rings.{ring_name} capacity * slot_size_bytes exceeds the maximum ring segment size" ) + if ring.capacity > _UINT32_MAX: + raise ValueError(f"rings.{ring_name}.capacity exceeds the uint32 wire limit") + + +def validate_acquisition_config(config: RunConfigModel) -> None: + """Reject unusable acquisition settings before starting processes or hardware. + + Replay uses the recorded trace dimensions and needs no physical devices; its + producer checks the actual payload size before publishing a frame. + """ + if config.radar.replay.dataset_dir: + return + mode = config.radar.driver_mode + if mode not in ("mock", "native"): + raise ValueError("radar.driver_mode must be mock or native") + if config.radar.model not in ( + config.LIBREVNA_MODEL, config.LIBREVNA_MULTI_MODEL, config.KAMIL_ADC_MODEL, + ): + raise ValueError(f"Unsupported radar model: {config.radar.model}") + if mode != "native" and config.is_kamil_adc: + raise ValueError(f"{config.radar.model} requires native driver mode") + + for switch, physical_positions in ( + (config.input_switch, config.matrix_input_switch_positions), + (config.output_switch, config.matrix_output_switch_positions), + ): + context = f"switches.{switch.name or switch.radar_port}" + if switch.driver_mode not in ("mock", "native"): + raise ValueError(f"{context}.driver_mode must be mock or native") + if switch.driver not in ("h7992", "hmc349a"): + raise ValueError(f"{context}.driver must be h7992 or hmc349a") + if switch.radar_port not in (1, 2): + raise ValueError(f"{context}.radar_port must be 1 or 2") + if not 0 <= switch.default_position < switch.positions: + raise ValueError(f"{context}.default_position must be within switch positions") + positions, switch_mode = switch.positions, switch.driver_mode + if config.is_matrix_radar and physical_positions > 1: + positions, switch_mode = physical_positions, mode + if switch_mode == "native": + limit = 4 if switch.driver == "h7992" else 2 + if not 1 <= positions <= limit: + raise ValueError(f"{context}: native {switch.driver} supports 1..{limit} positions") + if not switch.gpio_chip or switch.pin_a < 0 or ( + switch.driver == "h7992" and (switch.pin_b < 0 or switch.pin_b == switch.pin_a) + ): + raise ValueError(f"{context}: GPIO chip and distinct nonnegative control pins are required") + + points = config.radar.kamil_adc.band.points if config.is_kamil_adc else config.radar.sweep.points + # RAW/PPR header, each combo's frequency/S11/S21 arrays and capture windows. + minimum = 44 + len(config.combos) * (28 + 20 * points) + for name in ("raw", "raw_tap", "preprocessed", "preprocessed_tap"): + ring = getattr(config.rings, name) + if name.endswith("_tap") and not ring.name: + continue + if ring.slot_size_bytes < minimum: + raise ValueError(f"rings.{name}.slot_size_bytes must be at least {minimum} for this sweep") def validate_sweep_model(sweep: RadarSweepModel) -> None: @@ -143,9 +171,17 @@ def validate_sweep_model(sweep: RadarSweepModel) -> None: points = sweep.points if isinstance(points, bool) or not isinstance(points, int): raise ValueError("radar.sweep.points must be an integer") - if points <= 0: - raise ValueError("radar.sweep.points must be > 0") - if float(sweep.stop_hz) <= float(sweep.start_hz): + if not 0 < points <= _UINT32_MAX: + raise ValueError("radar.sweep.points must be in [1, uint32 max]") + for field in ("start_hz", "stop_hz", "if_bandwidth_hz", "power_dbm"): + value = getattr(sweep, field) + if isinstance(value, bool) or not isinstance(value, (int, float)) or abs(value) > 3.4e38 or not math.isfinite(value): + raise ValueError(f"radar.sweep.{field} must be a finite float32 number") + if sweep.start_hz < 0: + raise ValueError("radar.sweep.start_hz must be >= 0") + if sweep.if_bandwidth_hz <= 0: + raise ValueError("radar.sweep.if_bandwidth_hz must be > 0") + if sweep.stop_hz <= sweep.start_hz: raise ValueError("radar.sweep.stop_hz must be > radar.sweep.start_hz") @@ -167,12 +203,16 @@ def validate_gpr_model( if sweep is not None: validate_sweep_model(sweep) - if float(gpr.relative_permittivity) <= 0.0: + if not math.isfinite(gpr.relative_permittivity) or float(gpr.relative_permittivity) <= 0.0: raise ValueError("gpr.relative_permittivity must be > 0") seen_output_positions: set[int] = set() for entry in gpr.tx_geometry: - output_pos = int(entry.output_pos) + if any(not math.isfinite(value) or abs(value) > 3.4e38 for value in (entry.x_m, entry.y_m, entry.z_m)): + raise ValueError("gpr.tx_geometry coordinates must be finite numbers") + output_pos = entry.output_pos + if isinstance(output_pos, bool) or not isinstance(output_pos, int): + raise ValueError("gpr.tx_geometry output_pos must be an integer") if output_pos < 0 or output_pos >= int(output_switch_positions): raise ValueError("gpr.tx_geometry output_pos is out of range") if output_pos in seen_output_positions: @@ -181,7 +221,11 @@ def validate_gpr_model( seen_input_positions: set[int] = set() for entry in gpr.rx_geometry: - input_pos = int(entry.input_pos) + if any(not math.isfinite(value) or abs(value) > 3.4e38 for value in (entry.x_m, entry.y_m, entry.z_m)): + raise ValueError("gpr.rx_geometry coordinates must be finite numbers") + input_pos = entry.input_pos + if isinstance(input_pos, bool) or not isinstance(input_pos, int): + raise ValueError("gpr.rx_geometry input_pos must be an integer") if input_pos < 0 or input_pos >= int(input_switch_positions): raise ValueError("gpr.rx_geometry input_pos is out of range") if input_pos in seen_input_positions: @@ -202,12 +246,18 @@ def validate_combos( config error that would otherwise produce missing or doubled traces downstream. """ seen: set[tuple[int, int]] = set() + if len(combos) > MAX_COMBOS: + raise ValueError(f"Too many combos: limit is {MAX_COMBOS}") for combo in combos: - if not 0 <= int(combo.input) < int(input_positions): + for field in ("input", "output"): + value = getattr(combo, field) + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"run.combos {field} must be an integer") + if not 0 <= combo.input < input_positions: raise ValueError( f"run.combos input {combo.input} is out of range [0, {input_positions})" ) - if not 0 <= int(combo.output) < int(output_positions): + if not 0 <= combo.output < output_positions: raise ValueError( f"run.combos output {combo.output} is out of range [0, {output_positions})" ) @@ -222,7 +272,7 @@ def parse_combos_from_text(text: str) -> list[ComboModel]: Returns an empty list for blank input. Raises ValueError (naming the offending pair) on malformed syntax, empty sides, non-integer values, more - than ``_MAX_COMBOS`` entries, or a non-blank string that yields no combos. + than ``MAX_COMBOS`` entries, or a non-blank string that yields no combos. """ cleaned = text.strip() if not cleaned: @@ -238,8 +288,8 @@ def parse_combos_from_text(text: str) -> list[ComboModel]: # Cap the combo count so a pathological string cannot expand into a list # large enough to stall the GUI or the acquisition loop. - if len(combos) >= _MAX_COMBOS: - raise ValueError(f"Too many combos: limit is {_MAX_COMBOS}") + if len(combos) >= MAX_COMBOS: + raise ValueError(f"Too many combos: limit is {MAX_COMBOS}") input_text, output_text = (side.strip() for side in pair.split(":", 1)) # Reject empty sides and re-raise non-integer values naming the pair/side. diff --git a/python_app/orchestration/config_writer.py b/python_app/orchestration/config_writer.py index 42430b1..8d51a3c 100644 --- a/python_app/orchestration/config_writer.py +++ b/python_app/orchestration/config_writer.py @@ -2,12 +2,10 @@ from __future__ import annotations -from contextlib import suppress -import json import logging -import os from pathlib import Path +from python_app.file_io import atomic_write_json from python_app.models.run_config_model import RunConfigModel, parse_combos_from_text from python_app.orchestration.preprocess_assets import ( PREPROCESS_ASSET_KEYS, @@ -47,32 +45,9 @@ class ConfigWriter: asset.bundle_path = str(bundle_path) def write(self, config: RunConfigModel, output_path: Path) -> Path: - """Atomically write the run configuration JSON file. - - Dump to a sibling ``.tmp``, flush + ``fsync`` to durably commit the bytes on - the SD-card-backed Pi, then ``os.replace()`` atomically onto the destination. - A C++ consumer can therefore never observe a half-written config (which would - abort it with an opaque JSON parse error), even across a crash or power loss. - """ - output_path.parent.mkdir(parents=True, exist_ok=True) - # allow_nan=False: a stray NaN/Infinity must fail loudly here in Python - # rather than serialize to a non-standard token that aborts every C++ - # consumer at startup with an opaque JSON parse error. - serialized = json.dumps(config.to_dict(), indent=2, allow_nan=False) - tmp_path = output_path.with_suffix(output_path.suffix + ".tmp") - try: - with open(tmp_path, "w", encoding="utf-8") as handle: - handle.write(serialized) - handle.flush() - os.fsync(handle.fileno()) - os.replace(tmp_path, output_path) - except Exception: - logger.exception("Failed to write run config to %s; removing temp file", output_path) - # Never leave a half-written .tmp behind on a write/fsync failure. - with suppress(OSError): - tmp_path.unlink() - raise - logger.debug("Wrote run config to %s (%d bytes)", output_path, len(serialized)) + """Atomically write and sync the run configuration JSON file.""" + atomic_write_json(output_path, config.to_dict(), durable=True) + logger.debug("Wrote run config to %s", output_path) return output_path diff --git a/python_app/orchestration/gui_session_state.py b/python_app/orchestration/gui_session_state.py index ca7de48..5324ef1 100644 --- a/python_app/orchestration/gui_session_state.py +++ b/python_app/orchestration/gui_session_state.py @@ -7,6 +7,7 @@ import json import logging from pathlib import Path +from python_app.file_io import atomic_write_json logger = logging.getLogger(__name__) @@ -37,7 +38,7 @@ class GuiSessionStateStore: try: payload = json.loads(self._path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: + except (OSError, UnicodeError, json.JSONDecodeError) as exc: # The state file is GUI-local cache — a corrupted file should not # prevent the app from starting. Reset to defaults and let the # next write overwrite it. @@ -55,11 +56,6 @@ class GuiSessionStateStore: def write(self, state: GuiSessionState) -> Path: """Atomically write session-state JSON file.""" - temp_path = self._path.with_suffix(self._path.suffix + ".tmp") - temp_path.write_text( - json.dumps({"last_profile_path": state.last_profile_path}, indent=2), - encoding="utf-8", - ) - temp_path.replace(self._path) + atomic_write_json(self._path, {"last_profile_path": state.last_profile_path}, durable=True) logger.debug("Wrote GUI session-state to %s", self._path) return self._path diff --git a/python_app/orchestration/live_processing_config.py b/python_app/orchestration/live_processing_config.py index 54d870a..286f39c 100644 --- a/python_app/orchestration/live_processing_config.py +++ b/python_app/orchestration/live_processing_config.py @@ -2,11 +2,11 @@ from __future__ import annotations -from dataclasses import dataclass -import json +from dataclasses import asdict, dataclass import logging from pathlib import Path +from python_app.file_io import atomic_write_json logger = logging.getLogger(__name__) @@ -21,12 +21,15 @@ class ProcessingLiveConfig: pass_through_y_max_db: float = 0.0 bscan_axis: str = "abs" bscan_channel: str = "s21" - bscan_cut_m: float = 0.824 - bscan_max_depth_m: float = 1.0 + bscan_cut_m: float = 0.0 + bscan_max_depth_m: float = 8.0 bscan_gain: float = 1.0 bscan_start_freq_mhz: float = 100.0 bscan_stop_freq_mhz: float = 8800.0 legacy_gpr_mode: str = "point" + relative_permittivity: float | None = None + tx_geometry: list[dict] | None = None + rx_geometry: list[dict] | None = None gpr_input_positions: list[int] | None = None gpr_output_positions: list[int] | None = None gpr_min_depth_m: float = 2.0 @@ -88,6 +91,7 @@ class ProcessingLiveConfig: reprocess_current_result: bool = True history_command_seq: int = 0 history_command: str = "none" + settings_revision: int = 0 def __post_init__(self) -> None: """Normalize optional list fields to concrete integer lists.""" @@ -102,60 +106,10 @@ class ProcessingLiveConfig: def to_dict(self) -> dict[str, object]: """Convert live config to JSON-serializable dictionary.""" + optional_geometry = {"relative_permittivity", "tx_geometry", "rx_geometry"} return { - "processor_mode": str(self.processor_mode), - "pass_through_channel": str(self.pass_through_channel), - "pass_through_fixed_y_enabled": bool(self.pass_through_fixed_y_enabled), - "pass_through_y_min_db": float(self.pass_through_y_min_db), - "pass_through_y_max_db": float(self.pass_through_y_max_db), - "bscan_axis": str(self.bscan_axis), - "bscan_channel": str(self.bscan_channel), - "bscan_cut_m": float(self.bscan_cut_m), - "bscan_max_depth_m": float(self.bscan_max_depth_m), - "bscan_gain": float(self.bscan_gain), - "bscan_start_freq_mhz": float(self.bscan_start_freq_mhz), - "bscan_stop_freq_mhz": float(self.bscan_stop_freq_mhz), - "legacy_gpr_mode": str(self.legacy_gpr_mode), - "gpr_input_positions": [int(value) for value in self.gpr_input_positions], - "gpr_output_positions": [int(value) for value in self.gpr_output_positions], - "gpr_min_depth_m": float(self.gpr_min_depth_m), - "gpr_max_depth_m": float(self.gpr_max_depth_m), - "gpr_range_comp_power": float(self.gpr_range_comp_power), - "gpr_comp_power": float(self.gpr_comp_power), - "gpr_object_min_frac": float(self.gpr_object_min_frac), - "gpr_score_mode": str(self.gpr_score_mode), - "gpr_motion_mode": str(self.gpr_motion_mode), - "gpr_max_detected_objects_to_draw": int(self.gpr_max_detected_objects_to_draw), - "gpr_draw_top_m_objects": int(self.gpr_draw_top_m_objects), - "gpr_object_approach_min_frames": int(self.gpr_object_approach_min_frames), - "gpr_speed_m_s": float(self.gpr_speed_m_s), - "gpr_look_angle_deg": float(self.gpr_look_angle_deg), - "gpr_direction_sign": float(self.gpr_direction_sign), - "gpr_apply_freq_phase_correction": bool(self.gpr_apply_freq_phase_correction), - "gpr_reference_mode": str(self.gpr_reference_mode), - "gpr_snr_thresh": float(self.gpr_snr_thresh), - "gpr_snr_comp_max": float(self.gpr_snr_comp_max), - "gpr_start_freq_mhz": float(self.gpr_start_freq_mhz), - "gpr_stop_freq_mhz": float(self.gpr_stop_freq_mhz), - "gpr_background_subtract_enabled": bool(self.gpr_background_subtract_enabled), - "gpr_background_mean_count": int(self.gpr_background_mean_count), - "gpr_remove_sidelobe_objects_enabled": bool(self.gpr_remove_sidelobe_objects_enabled), - "gpr_imaging_plane_y_m": float(self.gpr_imaging_plane_y_m), - "legacy_gpr_min_visible_pair_count": float(self.legacy_gpr_min_visible_pair_count), - "gpr_visible_x_min_m": float(self.gpr_visible_x_min_m), - "gpr_visible_x_max_m": float(self.gpr_visible_x_max_m), - "gpr_visible_z_min_m": float(self.gpr_visible_z_min_m), - "gpr_visible_z_max_m": float(self.gpr_visible_z_max_m), - "sar3d_frame_radius": int(self.sar3d_frame_radius), - "sar3d_voxel_m": float(self.sar3d_voxel_m), - "sar3d_bp_ground_depth_m": float(self.sar3d_bp_ground_depth_m), - "sar3d_comp_range_power": float(self.sar3d_comp_range_power), - "sar3d_use_compensation": bool(self.sar3d_use_compensation), - "sar3d_normalize_each_frame": bool(self.sar3d_normalize_each_frame), - "ignore_socket_speed": bool(self.ignore_socket_speed), - "reprocess_current_result": bool(self.reprocess_current_result), - "history_command_seq": int(self.history_command_seq), - "history_command": str(self.history_command), + name: value for name, value in asdict(self).items() + if name not in optional_geometry or value is not None } @@ -180,7 +134,4 @@ class ProcessingLiveConfigWriter: config.processor_mode, self._config_path, ) - temp_path = self._config_path.with_suffix(self._config_path.suffix + ".tmp") - temp_path.write_text(json.dumps(config.to_dict(), indent=2), encoding="utf-8") - temp_path.replace(self._config_path) - return self._config_path + return atomic_write_json(self._config_path, config.to_dict()) diff --git a/python_app/orchestration/preprocess_assets.py b/python_app/orchestration/preprocess_assets.py index 09ed818..8c903be 100644 --- a/python_app/orchestration/preprocess_assets.py +++ b/python_app/orchestration/preprocess_assets.py @@ -11,7 +11,6 @@ from python_app.models.run_config_model import PreprocessAssetModel, RunConfigMo class PreprocessAssetSpec: """Stable description of one preprocess asset.""" - key: str display_name: str set_kind: str runtime_filename: str @@ -19,48 +18,12 @@ class PreprocessAssetSpec: PREPROCESS_ASSET_SPECS = { - "s21_calibration": PreprocessAssetSpec( - key="s21_calibration", - display_name="S21 Calibration", - set_kind="s21_calibration", - runtime_filename="s21_calibration_bundle.bin", - channel="s21", - ), - "s21_reference": PreprocessAssetSpec( - key="s21_reference", - display_name="S21 Reference", - set_kind="s21_reference", - runtime_filename="s21_reference_bundle.bin", - channel="s21", - ), - "s11_open": PreprocessAssetSpec( - key="s11_open", - display_name="S11 Open", - set_kind="s11_open", - runtime_filename="s11_open_calibration_bundle.bin", - channel="s11", - ), - "s11_short": PreprocessAssetSpec( - key="s11_short", - display_name="S11 Short", - set_kind="s11_short", - runtime_filename="s11_short_calibration_bundle.bin", - channel="s11", - ), - "s11_load": PreprocessAssetSpec( - key="s11_load", - display_name="S11 Load", - set_kind="s11_load", - runtime_filename="s11_load_calibration_bundle.bin", - channel="s11", - ), - "s11_reference": PreprocessAssetSpec( - key="s11_reference", - display_name="S11 Reference", - set_kind="s11_reference", - runtime_filename="s11_reference_bundle.bin", - channel="s11", - ), + "s21_calibration": PreprocessAssetSpec("S21 Calibration", "s21_calibration", "s21_calibration_bundle.bin", "s21"), + "s21_reference": PreprocessAssetSpec("S21 Reference", "s21_reference", "s21_reference_bundle.bin", "s21"), + "s11_open": PreprocessAssetSpec("S11 Open", "s11_open", "s11_open_calibration_bundle.bin", "s11"), + "s11_short": PreprocessAssetSpec("S11 Short", "s11_short", "s11_short_calibration_bundle.bin", "s11"), + "s11_load": PreprocessAssetSpec("S11 Load", "s11_load", "s11_load_calibration_bundle.bin", "s11"), + "s11_reference": PreprocessAssetSpec("S11 Reference", "s11_reference", "s11_reference_bundle.bin", "s11"), } PREPROCESS_ASSET_KEYS = tuple(PREPROCESS_ASSET_SPECS.keys()) diff --git a/python_app/orchestration/process_supervisor.py b/python_app/orchestration/process_supervisor.py index 198b176..8ad403e 100644 --- a/python_app/orchestration/process_supervisor.py +++ b/python_app/orchestration/process_supervisor.py @@ -3,18 +3,19 @@ from __future__ import annotations from dataclasses import dataclass -import json import logging import os from pathlib import Path import shlex import signal import subprocess -import sys import time from typing import Iterable from typing import Sequence +from python_app.file_io import atomic_write_text +from python_app.orchestration.producer_selection import acquisition_command + logger = logging.getLogger(__name__) # Cap each child log so a long-lived daemon cannot fill the SD card. On reaching @@ -244,31 +245,23 @@ class ProcessSupervisor: # on every write, so a truncate genuinely restarts the file at offset 0. # `_roll_log_to_prev` above already renamed any previous log away, so not # truncating on open costs nothing. - stdout_file = open(stdout_path, "ab") - stderr_file = open(stderr_path, "ab") try: - handle = subprocess.Popen( - command, - cwd=self._project_root, - stdout=stdout_file, - stderr=stderr_file, - # Own session/process group so signalling the group on stop also - # reaches device-I/O grandchildren the producer may have spawned. - start_new_session=True, - ) + with stdout_path.open("ab") as stdout_file, stderr_path.open("ab") as stderr_file: + handle = subprocess.Popen( + command, + cwd=self._project_root, + stdout=stdout_file, + stderr=stderr_file, + # Device-I/O grandchildren share this group and stop with it. + start_new_session=True, + ) except OSError as exc: - stdout_file.close() - stderr_file.close() command_text = shlex.join(command) logger.error("Failed to spawn `%s`: %s: %s", name, type(exc).__name__, exc) raise RuntimeError( f"Failed to spawn {name} with command `{command_text}` from `{self._project_root}`: " f"{type(exc).__name__}: {exc}" ) from exc - finally: - # The child holds its own dup'd fds; the parent's copies are not needed. - stdout_file.close() - stderr_file.close() self._processes[name] = ManagedProcess( name=name, @@ -282,68 +275,8 @@ class ProcessSupervisor: self._write_pidfile() def _acquisition_command(self, config_path: Path) -> list[str]: - """Return acquisition producer command selected by radar.model.""" - # A configured replay dataset overrides the model's normal producer: - # recorded snapshots are replayed into the raw ring instead of - # acquiring from (mock) hardware. - replay_dir = self._read_replay_dataset_dir(config_path) - if replay_dir: - logger.debug("Selecting replay producer for dataset_dir=%s", replay_dir) - return [ - sys.executable, - "-m", - "python_app.scripts.replay_raw_producer", - "--config", - str(config_path), - ] - radar_model = self._read_radar_model(config_path) - logger.debug("Selecting acquisition producer for radar.model=%s", radar_model) - if radar_model in {"librevna_multi", "sn9000"}: - return [ - sys.executable, - "-m", - "python_app.scripts.matrix_raw_producer", - "--config", - str(config_path), - ] - if radar_model == "kamil_adc": - return [ - sys.executable, - "-m", - "python_app.scripts.kamil_adc_raw_producer", - "--config", - str(config_path), - ] - return [ - str(self._project_root / "build/bin/sweep_orchestrator"), - "--config", - str(config_path), - ] - - @staticmethod - def _read_radar_model(config_path: Path) -> str: - """Read `radar.model` cheaply without constructing the full config model.""" - payload = json.loads(config_path.read_text(encoding="utf-8")) - if not isinstance(payload, dict): - return "librevna" - radar_payload = payload.get("radar") - if not isinstance(radar_payload, dict): - return "librevna" - return str(radar_payload.get("model", "librevna")) - - @staticmethod - def _read_replay_dataset_dir(config_path: Path) -> str: - """Read `radar.replay.dataset_dir` cheaply, '' when replay is off.""" - payload = json.loads(config_path.read_text(encoding="utf-8")) - if not isinstance(payload, dict): - return "" - radar_payload = payload.get("radar") - if not isinstance(radar_payload, dict): - return "" - replay_payload = radar_payload.get("replay") - if not isinstance(replay_payload, dict): - return "" - return str(replay_payload.get("dataset_dir", "")).strip() + """Select the same acquisition producer as the shell launcher.""" + return acquisition_command(self._project_root, config_path) def _stop_processes(self, names: Iterable[str]) -> None: """Gracefully terminate processes, then force-kill on per-process timeout.""" @@ -393,7 +326,12 @@ class ProcessSupervisor: if pid is None: return try: - os.killpg(os.getpgid(pid), sig) + # Managed children lead their own sessions. Never signal an unrelated + # shell's group if a stale PID or an external collector does not. + if os.getpgid(pid) == pid: + os.killpg(pid, sig) + else: + os.kill(pid, sig) except (ProcessLookupError, PermissionError): # Group already gone, or could not resolve it; fall back to the child. try: @@ -401,7 +339,8 @@ class ProcessSupervisor: except (ProcessLookupError, PermissionError): pass - def _kill_kamil_collectors(self) -> None: + @staticmethod + def _kill_kamil_collectors() -> None: """SIGKILL any Kamil ADC collector found by name — a guaranteed device backstop. The collector survives a force-killed producer (own session) and can hang @@ -421,8 +360,9 @@ class ProcessSupervisor: cmdline = (Path("/proc") / str(pid) / "cmdline").read_bytes() except OSError: continue - if _KAMIL_COLLECTOR_MARKER in cmdline: - self._signal_group(pid, signal.SIGKILL) + executable = cmdline.split(b"\x00", 1)[0].rsplit(b"/", 1)[-1] + if executable == _KAMIL_COLLECTOR_MARKER: + ProcessSupervisor._signal_group(pid, signal.SIGKILL) killed += 1 if killed: logger.warning("Force-killed %d orphaned Kamil ADC collector process(es)", killed) @@ -569,7 +509,7 @@ class ProcessSupervisor: for process in self._processes.values() if process.handle.poll() is None and process.handle.pid is not None ] - self._pidfile_path.write_text("\n".join(live_pids), encoding="utf-8") + atomic_write_text(self._pidfile_path, "\n".join(live_pids)) except OSError: # Best-effort bookkeeping: failure here must not break start/stop. pass @@ -599,7 +539,7 @@ class ProcessSupervisor: logger.warning("Reaping stale pipeline child from prior run (pid=%d)", pid) self._signal_group(pid, signal.SIGKILL) try: - self._pidfile_path.write_text("", encoding="utf-8") + atomic_write_text(self._pidfile_path, "") except OSError: pass # The ADC collector is a grandchild (spawned by the producer), not in the @@ -612,17 +552,20 @@ class ProcessSupervisor: Reads `/proc//cmdline` so a recycled PID owned by an unrelated process is never killed. """ - markers = ( - "build/bin/data_processor", - "build/bin/data_preprocessor", - "build/bin/sweep_orchestrator", + native_names = {"data_processor", "data_preprocessor", "sweep_orchestrator"} + producer_modules = { "python_app.scripts.matrix_raw_producer", "python_app.scripts.kamil_adc_raw_producer", "python_app.scripts.replay_raw_producer", - ) + } try: raw = (Path("/proc") / str(pid) / "cmdline").read_bytes() except OSError: return False - cmdline = raw.replace(b"\x00", b" ").decode("utf-8", errors="replace") - return any(marker in cmdline for marker in markers) + argv = raw.decode("utf-8", errors="replace").rstrip("\x00").split("\x00") + executable = Path(argv[0]) + if executable.name in native_names: + expected = self._project_root / "build/bin" / executable.name + return executable == expected or executable == Path("build/bin") / executable.name + is_python = executable.name in {"python", "python3"} or executable.name.startswith("python3.") + return is_python and len(argv) >= 3 and argv[1] == "-m" and argv[2] in producer_modules diff --git a/python_app/orchestration/producer_selection.py b/python_app/orchestration/producer_selection.py new file mode 100644 index 0000000..ba1b1c8 --- /dev/null +++ b/python_app/orchestration/producer_selection.py @@ -0,0 +1,35 @@ +"""One acquisition producer selection shared by supervisor and launcher.""" + +from __future__ import annotations + +import json +from pathlib import Path +import sys + +from python_app.models.run_config_schema import RadarModel + + +def acquisition_command(project_root: Path, config_path: Path) -> list[str]: + """Select replay first, then the configured model's acquisition producer.""" + payload = json.loads(config_path.read_text(encoding="utf-8")) + radar = payload.get("radar", {}) if isinstance(payload, dict) else {} + radar = radar if isinstance(radar, dict) else {} + replay = radar.get("replay", {}) + dataset = replay.get("dataset_dir") if isinstance(replay, dict) else None + defaults = RadarModel() + model = radar.get("model") + if model is None and isinstance(payload, dict): + model = defaults.model + driver_mode = radar.get("driver_mode") + if driver_mode is None: + driver_mode = defaults.driver_mode + if model in ("librevna", "librevna_multi") and driver_mode == "mock": + model = "librevna_multi" + module = "replay_raw_producer" if isinstance(dataset, str) and dataset.strip() else { + "librevna_multi": "matrix_raw_producer", + "kamil_adc": "kamil_adc_raw_producer", + }.get(model if isinstance(model, str) else "") + command = [sys.executable, "-m", f"python_app.scripts.{module}"] if module else [ + str(project_root / "build/bin/sweep_orchestrator"), + ] + return [*command, "--config", str(config_path)] diff --git a/python_app/orchestration/shm/atomics.py b/python_app/orchestration/shm/atomics.py new file mode 100644 index 0000000..ece8786 --- /dev/null +++ b/python_app/orchestration/shm/atomics.py @@ -0,0 +1,66 @@ +"""GCC atomic operations on aligned shared-memory fields. + +The libatomic ABI omits the builtin compare-exchange ``weak`` argument: +https://github.com/gcc-mirror/gcc/blob/master/libatomic/cas_n.c +""" + +from __future__ import annotations + +import ctypes +from functools import cache +import mmap + +_ACQUIRE, _RELEASE, _ACQ_REL, _SEQ_CST = 2, 3, 4, 5 + + +@cache +def _library() -> ctypes.CDLL: + try: + library = ctypes.CDLL("libatomic.so.1") + except OSError as exc: + raise RuntimeError("Shared memory requires libatomic.so.1 (install libatomic1)") from exc + library.__atomic_is_lock_free.argtypes = [ctypes.c_size_t, ctypes.c_void_p] + library.__atomic_is_lock_free.restype = ctypes.c_bool + for width, number in ((4, ctypes.c_uint32), (8, ctypes.c_uint64)): + if not library.__atomic_is_lock_free(width, None): + raise RuntimeError(f"Shared memory requires lock-free {width}-byte atomics") + load = getattr(library, f"__atomic_load_{width}") + load.argtypes, load.restype = [ctypes.c_void_p, ctypes.c_int], number + store = getattr(library, f"__atomic_store_{width}") + store.argtypes, store.restype = [ctypes.c_void_p, number, ctypes.c_int], None + library.__atomic_compare_exchange_8.argtypes = [ + ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint64), ctypes.c_uint64, ctypes.c_int, ctypes.c_int, + ] + library.__atomic_compare_exchange_8.restype = ctypes.c_bool + library.atomic_thread_fence.argtypes, library.atomic_thread_fence.restype = [ctypes.c_int], None + return library + + +def _address(buffer: mmap.mmap, offset: int, width: int) -> int: + if offset % width or offset < 0 or offset + width > len(buffer): + raise ValueError("Atomic field must be aligned and inside the mapping") + return ctypes.addressof(ctypes.c_char.from_buffer(buffer, offset)) + + +def load(buffer: mmap.mmap, offset: int, width: int) -> int: + return getattr(_library(), f"__atomic_load_{width}")(_address(buffer, offset, width), _ACQUIRE) + + +def store(buffer: mmap.mmap, offset: int, value: int, width: int) -> None: + if not 0 <= value < 1 << (8 * width): + raise ValueError("Atomic field value exceeds its unsigned wire range") + getattr(_library(), f"__atomic_store_{width}")(_address(buffer, offset, width), value, _RELEASE) + + +def advance(buffer: mmap.mmap, offset: int, desired: int) -> None: + """Advance a shared cursor with CAS; concurrent progress can never be undone.""" + address = _address(buffer, offset, 8) + expected = ctypes.c_uint64(load(buffer, offset, 8)) + while expected.value < desired: + if _library().__atomic_compare_exchange_8(address, ctypes.byref(expected), desired, _ACQ_REL, _ACQUIRE): + break + + +def fence() -> None: + """Order invalidation/copy/validation on CPUs with weak memory ordering.""" + _library().atomic_thread_fence(_SEQ_CST) diff --git a/python_app/orchestration/shm/decoder.py b/python_app/orchestration/shm/decoder.py index 5ab3622..137bfa9 100644 --- a/python_app/orchestration/shm/decoder.py +++ b/python_app/orchestration/shm/decoder.py @@ -119,7 +119,7 @@ def decode_result_collection(payload: bytes) -> ResultCollection: image_y_axis = np.frombuffer(cursor.read_bytes(y_count * 4), dtype=" 0 else np.zeros((0, 0), dtype=np.float32) + image = image_values.reshape((y_count, x_count)) return ResultPayload( processing_name=name, kind=kind, @@ -130,13 +130,11 @@ def decode_result_collection(payload: bytes) -> ResultCollection: if kind == 4: table_columns = cursor.read_u32() table_rows = cursor.read_u32() + if table_rows and not table_columns: + raise ValueError("Result table with rows must have at least one column") value_count = table_columns * table_rows table_values = np.frombuffer(cursor.read_bytes(value_count * 4), dtype=" 0 and table_columns > 0 - else np.zeros((0, 0), dtype=np.float32) - ) + table = table_values.reshape((table_rows, table_columns)) return ResultPayload( processing_name=name, kind=kind, diff --git a/python_app/orchestration/shm/ring_reader.py b/python_app/orchestration/shm/ring_reader.py index 6705c52..6a60cc1 100644 --- a/python_app/orchestration/shm/ring_reader.py +++ b/python_app/orchestration/shm/ring_reader.py @@ -5,11 +5,11 @@ from __future__ import annotations import logging import mmap from pathlib import Path -import struct import time from typing import Final from python_app.models.dataset_model import ResultCollection, SweepCollection +from python_app.orchestration.shm import atomics from python_app.orchestration.shm.decoder import ( PREPROC_MAGIC, RAW_MAGIC, @@ -30,8 +30,8 @@ class ShmRingReader: def __init__(self, ring_name: str, open_timeout_s: float = 2.0, open_poll_s: float = 0.01) -> None: """Open and validate ring by name, for example `/radar_results`.""" - if not ring_name.startswith("/"): - raise ValueError("ring_name must start with '/'") + if not ring_name.startswith("/") or len(ring_name) < 2 or "/" in ring_name[1:] or "\x00" in ring_name: + raise ValueError("ring_name must be '/' followed by a single nonempty filename") self._ring_name = ring_name self._path = Path("/dev/shm") / ring_name[1:] @@ -79,7 +79,7 @@ class ShmRingReader: if sequence != read_seq + 1: # Producer overwrote this slot before we read it. Resync to latest. logger.debug("Ring %s: slot lapped before read, resyncing to write_seq=%d", self._ring_name, write_seq) - self._write_u64(32, write_seq) + atomics.advance(self._mmap, 32, write_seq) return None payload_size = self._read_u32(slot_offset) @@ -92,20 +92,21 @@ class ShmRingReader: payload_size, self.slot_size_bytes, ) - self._write_u64(32, write_seq) + atomics.advance(self._mmap, 32, write_seq) return None payload_offset = slot_offset + _SLOT_HEADER_SIZE payload = bytes(self._mmap[payload_offset : payload_offset + payload_size]) + atomics.fence() # Re-read the slot sequence after the copy; if it changed, the producer # overwrote this slot mid-copy and the payload is torn — discard and resync. if self._read_u64(slot_offset + 8) != read_seq + 1: logger.debug("Ring %s: slot overwritten mid-copy, discarding torn payload", self._ring_name) - self._write_u64(32, write_seq) + atomics.advance(self._mmap, 32, write_seq) return None - self._write_u64(32, read_seq + 1) + atomics.advance(self._mmap, 32, read_seq + 1) return payload def pop_raw_collection(self) -> SweepCollection | None: @@ -157,17 +158,11 @@ class ShmRingReader: return None payload_offset = slot_offset + _SLOT_HEADER_SIZE payload = bytes(self._mmap[payload_offset : payload_offset + payload_size]) + atomics.fence() if self._read_u64(slot_offset + 8) != latest_seq + 1: return None return payload - def peek_latest_result_collection(self) -> ResultCollection | None: - """Return the most recently published result collection without consuming it.""" - payload = self.peek_latest_payload() - if payload is None: - return None - return decode_result_collection(payload) - def drop_all(self) -> int: """Mark all unread slots as consumed and return number of dropped payloads.""" write_seq = self._read_u64(24) @@ -175,7 +170,7 @@ class ShmRingReader: if read_seq >= write_seq: return 0 dropped = int(write_seq - read_seq) - self._write_u64(32, write_seq) + atomics.advance(self._mmap, 32, write_seq) logger.debug("Ring %s: dropped %d unread payload(s)", self._ring_name, dropped) return dropped @@ -196,7 +191,11 @@ class ShmRingReader: magic = self._mmap[:8] version = self._read_u32(8) if magic == _MAGIC and version == _VERSION: - return + capacity = self.capacity + slot_size = self.slot_size_bytes + expected_size = _HEADER_SIZE + capacity * (_SLOT_HEADER_SIZE + slot_size) + if capacity > 0 and slot_size > 0 and slot_size % 8 == 0 and expected_size == len(self._mmap): + return if time.monotonic() >= deadline: if magic != _MAGIC: @@ -206,29 +205,32 @@ class ShmRingReader: f"Shared memory ring magic mismatch for {self._ring_name}: " f"got=0x{got_magic}, expected=0x{expected_magic}" ) - raise RuntimeError( - f"Shared memory ring version mismatch for {self._ring_name}: " - f"got={version}, expected={_VERSION}" - ) + if version != _VERSION: + raise RuntimeError( + f"Shared memory ring version mismatch for {self._ring_name}: " + f"got={version}, expected={_VERSION}" + ) + raise RuntimeError(f"Shared memory ring geometry mismatch for {self._ring_name}") time.sleep(poll_s) def _wait_for_ring_file(self, timeout_s: float, poll_s: float) -> None: - """Wait until ring file appears in `/dev/shm`.""" + """Wait for the creator to allocate the ring before mapping its header.""" deadline = time.monotonic() + timeout_s - while not self._path.exists(): + while True: + try: + if self._path.stat().st_size >= _HEADER_SIZE: + return + except FileNotFoundError: + pass if time.monotonic() >= deadline: raise FileNotFoundError(f"Shared memory ring does not exist: {self._ring_name}") time.sleep(poll_s) def _read_u32(self, offset: int) -> int: """Read little-endian u32 at mmap offset.""" - return struct.unpack_from(" int: """Read little-endian u64 at mmap offset.""" - return struct.unpack_from(" None: - """Write little-endian u64 at mmap offset.""" - struct.pack_into(" None: """Open or create a POSIX SHM ring by name (as the ring owner).""" - if not ring_name.startswith("/"): - raise ValueError("ring_name must start with '/'") - if capacity <= 0: - raise ValueError("capacity must be > 0") - if slot_size_bytes <= 0: - raise ValueError("slot_size_bytes must be > 0") + if not ring_name.startswith("/") or len(ring_name) < 2 or "/" in ring_name[1:] or "\x00" in ring_name: + raise ValueError("ring_name must be '/' followed by a single nonempty filename") + validate_ring_endpoint(RingEndpointModel(ring_name, capacity, slot_size_bytes)) self._ring_name = ring_name self._capacity = int(capacity) self._slot_size_bytes = int(slot_size_bytes) self._mapped_size = _HEADER_SIZE + self._capacity * (_SLOT_HEADER_SIZE + self._slot_size_bytes) self._path = Path("/dev/shm") / ring_name[1:] - self._open_owned() + self._file = self._mmap = None + try: + self._open_owned() + except BaseException: + self.close() + raise def _open_owned(self) -> None: """Open the ring, recreating it from scratch on a geometry/header mismatch. @@ -63,6 +67,10 @@ class ShmRingWriter: # under a producer/consumer that still expects the old layout. if not created: logger.info("Ring %s: stale segment with wrong size, recreating", self._ring_name) + self._file.close() + self._unlink_if_present() + fd = os.open(self._path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o660) + self._file = os.fdopen(fd, "r+b", buffering=0) self._file.truncate(self._mapped_size) created = True @@ -84,7 +92,7 @@ class ShmRingWriter: self._mmap.close() self._file.close() self._unlink_if_present() - fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o660) + fd = os.open(self._path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o660) self._file = os.fdopen(fd, "r+b", buffering=0) self._file.truncate(self._mapped_size) self._mmap = mmap.mmap(self._file.fileno(), self._mapped_size) @@ -101,8 +109,12 @@ class ShmRingWriter: def close(self) -> None: """Close mmap and file handle.""" - self._mmap.close() - self._file.close() + if self._mmap is not None: + self._mmap.close() + self._mmap = None + if self._file is not None: + self._file.close() + self._file = None logger.debug("Closed SHM ring writer %s", self._ring_name) def push(self, payload: bytes) -> bool: @@ -121,13 +133,17 @@ class ShmRingWriter: # re-read it first and move it only forward: a concurrent reader may have # already advanced it, and clobbering that backward would re-deliver an # already-consumed slot as a duplicate. - self._write_u64(32, max(self._read_u64(32), read_seq + 1)) + atomics.advance(self._mmap, 32, read_seq + 1) self._write_u64(40, self._read_u64(40) + 1) index = write_seq % self._capacity slot_offset = _HEADER_SIZE + index * (_SLOT_HEADER_SIZE + self._slot_size_bytes) payload_offset = slot_offset + _SLOT_HEADER_SIZE + # Invalidate the previous publication before changing its bytes. A peek or + # a lapped consumer must not accept the old sequence during this overwrite. + self._write_u64(slot_offset + 8, 0) + atomics.fence() self._write_u32(slot_offset, len(payload)) self._write_u32(slot_offset + 4, 0) self._mmap[payload_offset : payload_offset + len(payload)] = payload @@ -146,7 +162,9 @@ class ShmRingWriter: return self._slot_size_bytes def _initialize_header(self) -> None: - self._mmap[:] = b"\x00" * self._mapped_size + # A new ftruncate-extended segment is already zero-filled by the kernel; + # avoid allocating and copying another complete ring just to initialize it. + self._mmap[:_HEADER_SIZE] = b"\x00" * _HEADER_SIZE self._mmap[:8] = _MAGIC self._write_u32(8, _VERSION) self._write_u32(12, self._capacity) @@ -170,13 +188,13 @@ class ShmRingWriter: ) def _read_u32(self, offset: int) -> int: - return struct.unpack_from(" int: - return struct.unpack_from(" None: - struct.pack_into(" None: - struct.pack_into(" tuple[str, str]: """Return the LibreVNA CSV column names for a trace's real and imaginary parts.""" @@ -185,8 +186,7 @@ def _build_history_payload( def _write_payload(output_path: Path, payload: dict[str, Any]) -> None: """Write a payload as indented UTF-8 JSON, creating parent directories as needed.""" - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + atomic_write_json(output_path, payload, durable=True) def _rmse(left: np.ndarray, right: np.ndarray) -> float: diff --git a/python_app/scripts/convert_snapshot_to_vna_history.py b/python_app/scripts/convert_snapshot_to_vna_history.py index 828d992..17d548d 100644 --- a/python_app/scripts/convert_snapshot_to_vna_history.py +++ b/python_app/scripts/convert_snapshot_to_vna_history.py @@ -4,7 +4,6 @@ from __future__ import annotations import argparse import json -from collections import defaultdict from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path @@ -12,20 +11,10 @@ from typing import Any import numpy as np +from python_app.file_io import atomic_write_json -@dataclass(frozen=True) -class TraceRecord: - """One raw/preprocessed trace extracted from a snapshot collection directory.""" - stage: str - collection_id: int - monotonic_ns: int - stage_index: int - frequency_hz: np.ndarray - samples: np.ndarray - # End of this trace's own sweep, from the snapshot's per-trace metadata; 0 for a - # snapshot recorded before per-trace timing existed. - capture_end_ns: int = 0 +from python_app.storage.npz.vna_history_json import TraceRecord, _build_sweep_history, _normalize_channel @dataclass(frozen=True) @@ -46,10 +35,13 @@ def _load_json(path: Path) -> dict[str, Any]: def _collection_dirs(stage_dir: Path) -> list[Path]: - """Return the stage's collection subdirectories sorted by name (empty if absent).""" + """Return collection subdirectories in numeric recording order (empty if absent).""" if not stage_dir.exists(): return [] - return sorted([path for path in stage_dir.iterdir() if path.is_dir()], key=lambda path: path.name) + return sorted( + (path for path in stage_dir.iterdir() if path.is_dir()), + key=lambda path: (_parse_stage_index(path.name, 0), path.name), + ) def _parse_stage_index(name: str, fallback: int) -> int: @@ -71,14 +63,6 @@ def _pick_trace_meta(meta: dict[str, Any], input_index: int, output_index: int) return None -def _normalize_channel(channel: str) -> str: - """Return a lowercased channel name, accepting only ``s21`` or ``s11``.""" - normalized = str(channel).strip().lower() - if normalized not in {"s21", "s11"}: - raise ValueError("channel must be either 's21' or 's11'") - return normalized - - def _load_stage_records( snapshot_dir: Path, stage: str, @@ -132,6 +116,7 @@ def _load_stage_records( frequency_hz=frequency_hz, samples=samples, capture_end_ns=int(trace_meta.get("capture_end_ns", 0)), + capture_start_ns=int(trace_meta.get("capture_start_ns", 0)), ) ) @@ -158,94 +143,6 @@ def _load_stage_refs(snapshot_dir: Path, stage: str) -> list[CollectionRef]: return refs -def _index_by_collection_occurrence(records: list[TraceRecord]) -> tuple[dict[tuple[int, int], TraceRecord], list[tuple[int, int]]]: - """Key records by (collection_id, occurrence) and return the map plus original order. - - The occurrence counter disambiguates repeated collection ids, so raw and preprocessed - stages can be aligned slot-for-slot even when ids recur. - """ - counters: defaultdict[int, int] = defaultdict(int) - record_map: dict[tuple[int, int], TraceRecord] = {} - order: list[tuple[int, int]] = [] - for record in records: - occurrence = counters[record.collection_id] - counters[record.collection_id] += 1 - key = (record.collection_id, occurrence) - record_map[key] = record - order.append(key) - return record_map, order - - -def _complex_to_points(values: np.ndarray) -> list[list[float]]: - """Return complex samples as ``[real, imag]`` pairs for JSON serialization.""" - return [[float(v.real), float(v.imag)] for v in values] - - -def _build_sweep_history( - raw_records: list[TraceRecord], - preprocessed_records: list[TraceRecord], - *, - channel: str, - primary_stage: str, -) -> list[dict[str, Any]]: - """Merge raw and preprocessed records into vna-system ``sweep_history`` entries. - - Iterates collections in the primary stage's order, pairing each with its counterpart - in the other stage; the raw samples become ``sweep_points`` and the preprocessed - samples ``calibrated_points`` (falling back to whichever stage is present). - """ - raw_map, raw_order = _index_by_collection_occurrence(raw_records) - pre_map, pre_order = _index_by_collection_occurrence(preprocessed_records) - - if primary_stage == "preprocessed": - primary_order = pre_order or raw_order - else: - primary_order = raw_order or pre_order - - history: list[dict[str, Any]] = [] - for fallback_index, key in enumerate(primary_order): - raw = raw_map.get(key) - pre = pre_map.get(key) - base = pre or raw - if base is None: - continue - - # vna_system uses calibrated_data if present, otherwise sweep_data. - sweep_source = raw or pre - calibrated_source = pre or raw - if sweep_source is None or calibrated_source is None: - continue - - start_freq_hz = float(base.frequency_hz[0]) - stop_freq_hz = float(base.frequency_hz[-1]) - # Prefer this trace's own sweep time: with a switching matrix the combos of - # one collection are measured milliseconds apart, so the collection - # timestamp misplaces every combo but the last. - if base.capture_end_ns > 0: - timestamp_sec = float(base.capture_end_ns) / 1_000_000_000.0 - elif base.monotonic_ns > 0: - timestamp_sec = float(base.monotonic_ns) / 1_000_000_000.0 - else: - timestamp_sec = float(fallback_index) - - history.append( - { - "timestamp": timestamp_sec, - "sweep_points": _complex_to_points(sweep_source.samples), - "calibrated_points": _complex_to_points(calibrated_source.samples), - "reference_points": [], - "vna_config": { - "mode": channel, - "start_freq": start_freq_hz, - "stop_freq": stop_freq_hz, - "points": int(base.frequency_hz.size), - }, - } - ) - - return history - - def _stage_alignment_warning(pre_refs: list[CollectionRef], result_refs: list[CollectionRef]) -> str | None: """Return warning text when preprocessed/results stages are not identity-aligned.""" if not pre_refs or not result_refs: @@ -394,8 +291,7 @@ def main() -> None: if alignment_warning is not None: payload["alignment_warning"] = alignment_warning - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + atomic_write_json(output_path, payload, durable=True) print( "Converted snapshot to vna_system history JSON:\n" diff --git a/python_app/scripts/create_neutral_preprocess_sets.py b/python_app/scripts/create_neutral_preprocess_sets.py index fdeb849..2684cce 100644 --- a/python_app/scripts/create_neutral_preprocess_sets.py +++ b/python_app/scripts/create_neutral_preprocess_sets.py @@ -45,9 +45,7 @@ def main() -> int: config = RunConfigModel.load_from_path(args.config) config.ensure_combos() if not supports_neutral_preprocess_sets(config): - raise RuntimeError( - "Neutral S21 sets are available only for kamil_adc and librevna_multi" - ) + raise RuntimeError("Neutral S21 sets are not supported for this radar configuration") radar_key = radar_key_from_config( model_name=config.radar.model, @@ -61,12 +59,19 @@ def main() -> int: ) store = NpzStore(PROJECT_ROOT / "python_app/data") + kinds = ("s21_calibration", "s21_reference") existing = [ kind - for kind in ("s21_calibration", "s21_reference") + for kind in kinds if args.set_name in store.list_sets(kind, radar_key) ] if existing and not args.force: + if len(existing) != len(kinds): + missing = ", ".join(kind for kind in kinds if kind not in existing) + raise RuntimeError( + f"Neutral set pair '{args.set_name}' is incomplete: missing {missing}; " + "use --force to replace both sets." + ) logger.info( "Sets '%s' already exist for %s (radar_key=%s); nothing to do (use --force to overwrite).", args.set_name, ", ".join(existing), radar_key, @@ -74,8 +79,7 @@ def main() -> int: return 0 calibration, reference = build_neutral_s21_sets(config) - store.save_set("s21_calibration", radar_key, args.set_name, calibration) - store.save_set("s21_reference", radar_key, args.set_name, reference) + store.save_sets(radar_key, args.set_name, dict(zip(kinds, (calibration, reference)))) logger.info( "Neutral S21 sets saved: set=%s, radar_key=%s, combos=%d, points=%d", args.set_name, diff --git a/python_app/scripts/hardware_raw_orchestrator_test.py b/python_app/scripts/hardware_raw_orchestrator_test.py index bd7e9f4..8a47d67 100644 --- a/python_app/scripts/hardware_raw_orchestrator_test.py +++ b/python_app/scripts/hardware_raw_orchestrator_test.py @@ -78,14 +78,9 @@ def _prepare_radar_if_needed(config_path: Path, *, strict: bool) -> str | None: if config.is_matrix_radar: return "Radar pre-configuration skipped (matrix radar producer config)." - radar_service = create_single_radar_service(config) - if not getattr(radar_service, "driver_available", True): - message = "LibreVNA Python driver is unavailable: skipping pre-configuration" - if strict: - raise RuntimeError(message) - return message - + radar_service = None try: + radar_service = create_single_radar_service(config) radar_service.open() radar_service.configure(config.radar.sweep) return "Radar pre-configuration completed." @@ -95,7 +90,8 @@ def _prepare_radar_if_needed(config_path: Path, *, strict: bool) -> str | None: raise return f"{message}. Continuing with native C++ configuration." finally: - radar_service.close() + if radar_service is not None: + radar_service.close() class RawOrchestratorViewer(QMainWindow): diff --git a/python_app/scripts/k209_remote_server.py b/python_app/scripts/k209_remote_server.py deleted file mode 100644 index 1a4dfb4..0000000 --- a/python_app/scripts/k209_remote_server.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Run a TCP acquisition server for a locally connected Compact-M K209.""" - -from __future__ import annotations - -import argparse -import socket -import socketserver -import sys -from pathlib import Path - -PROJECT_ROOT = Path(__file__).resolve().parents[2] -if str(PROJECT_ROOT) not in sys.path: - sys.path.insert(0, str(PROJECT_ROOT)) - -from python_app.hardware_full.compact_m_k209_service import CompactMK209Service -from python_app.hardware_full.k209_remote_protocol import ( - COMMAND_ACQUIRE, - COMMAND_CONFIGURE, - COMMAND_IDENTITY, - COMMAND_LIMITS, - CONFIG_STRUCT, - DEFAULT_REMOTE_PORT, - LIMITS_STRUCT, - STATUS_OK, - recv_exact, - send_error, - send_float32_array, - send_u32, -) -from python_app.models.run_config_model import RadarSweepModel - -DEFAULT_RESOURCE = "TCPIP0::127.0.0.1::hislip0,4880::INSTR" - - -class K209RemoteRequestHandler(socketserver.StreamRequestHandler): - """Handle one persistent K209 remote client connection.""" - - def setup(self) -> None: - """Disable Nagle and open the local K209 VISA session for this connection.""" - super().setup() - self.request.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - self.service = CompactMK209Service( - resource=self.server.resource, - timeout_ms=self.server.timeout_ms, - preset_on_open=False, - visa_library="@ivi", - ) - self.service.open() - - def finish(self) -> None: - """Close the K209 VISA session when the client disconnects.""" - try: - self.service.close() - finally: - super().finish() - - def handle(self) -> None: - """Serve command bytes from the client until EOF, reporting errors back inline.""" - while True: - command = self.rfile.read(1) - if not command: - return - try: - if command == COMMAND_IDENTITY: - self._handle_identity() - elif command == COMMAND_LIMITS: - self._handle_limits() - elif command == COMMAND_CONFIGURE: - self._handle_configure() - elif command == COMMAND_ACQUIRE: - self._handle_acquire() - else: - raise RuntimeError(f"unsupported command byte {command!r}") - self.wfile.flush() - except Exception as exc: # noqa: BLE001 - send_error(self.wfile, str(exc)) - self.wfile.flush() - - def _handle_identity(self) -> None: - """Reply with the device identity string (length-prefixed UTF-8).""" - payload = self.service.query_identity().encode("utf-8") - self.wfile.write(STATUS_OK) - send_u32(self.wfile, len(payload)) - self.wfile.write(payload) - - def _handle_limits(self) -> None: - """Reply with the device frequency/IFBW/power/point limits as a packed struct.""" - limits = self.service.read_device_limits() - self.wfile.write(STATUS_OK) - self.wfile.write( - LIMITS_STRUCT.pack( - float(limits["min_frequency_hz"]), - float(limits["max_frequency_hz"]), - float(limits["min_ifbw_hz"]), - float(limits["max_ifbw_hz"]), - int(limits["max_points"]), - float(limits["min_power_dbm"]), - float(limits["max_power_dbm"]), - ) - ) - - def _handle_configure(self) -> None: - """Apply a sweep config from the client and reply with the frequency axis.""" - start_hz, stop_hz, points, ifbw_hz, power_dbm = CONFIG_STRUCT.unpack( - recv_exact(self.rfile, CONFIG_STRUCT.size) - ) - sweep = RadarSweepModel( - start_hz=start_hz, - stop_hz=stop_hz, - points=int(points), - if_bandwidth_hz=ifbw_hz, - power_dbm=power_dbm, - ) - self.service.configure(sweep) - self.wfile.write(STATUS_OK) - send_u32(self.wfile, int(points)) - send_float32_array(self.wfile, self.service.frequency_axis()) - - def _handle_acquire(self) -> None: - """Acquire one interleaved sweep and reply with the S11 and S21 arrays.""" - sweep = self.service.acquire_interleaved() - self.wfile.write(STATUS_OK) - send_u32(self.wfile, int(sweep.frequency_hz.size)) - send_float32_array(self.wfile, sweep.s11_values) - send_float32_array(self.wfile, sweep.s21_values) - - -class K209RemoteServer(socketserver.TCPServer): - """Single-client TCP server with K209 connection settings.""" - - allow_reuse_address = True - - def __init__(self, server_address: tuple[str, int], resource: str, timeout_ms: int) -> None: - """Store the VISA resource and timeout used by each accepted connection.""" - self.resource = resource - self.timeout_ms = timeout_ms - super().__init__(server_address, K209RemoteRequestHandler) - - -def _parse_args() -> argparse.Namespace: - """Parse command-line options for the K209 remote server.""" - parser = argparse.ArgumentParser(description="Serve a locally connected Compact-M K209 over TCP.") - parser.add_argument("--host", default="0.0.0.0", help="Server bind address.") - parser.add_argument("--port", type=int, default=DEFAULT_REMOTE_PORT, help="Server TCP port.") - parser.add_argument("--resource", default=DEFAULT_RESOURCE, help="Local S2VNA VISA resource.") - parser.add_argument("--timeout-ms", type=int, default=20_000, help="K209 VISA timeout.") - return parser.parse_args() - - -def main() -> int: - """Bind the K209 remote server and serve clients until interrupted.""" - args = _parse_args() - with K209RemoteServer((args.host, args.port), resource=args.resource, timeout_ms=args.timeout_ms) as server: - print(f"K209 remote server listening on {args.host}:{args.port}") - print(f"Local S2VNA resource: {args.resource}") - server.serve_forever() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_app/scripts/k209_remote_smoke_test.py b/python_app/scripts/k209_remote_smoke_test.py deleted file mode 100644 index 80738f0..0000000 --- a/python_app/scripts/k209_remote_smoke_test.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Smoke test for a remote Compact-M K209 server.""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path - -import numpy as np - -PROJECT_ROOT = Path(__file__).resolve().parents[2] -if str(PROJECT_ROOT) not in sys.path: - sys.path.insert(0, str(PROJECT_ROOT)) - -from python_app.hardware_full.k209_remote_protocol import DEFAULT_REMOTE_HOST, DEFAULT_REMOTE_PORT -from python_app.hardware_full.remote_compact_m_k209_service import RemoteCompactMK209Service -from python_app.models.run_config_model import RadarSweepModel - - -def _parse_args() -> argparse.Namespace: - """Parse command-line options for the remote K209 smoke test.""" - parser = argparse.ArgumentParser(description="Validate remote K209 connection and one sweep.") - parser.add_argument("--host", default=DEFAULT_REMOTE_HOST, help="K209 remote server host.") - parser.add_argument("--port", type=int, default=DEFAULT_REMOTE_PORT, help="K209 remote server port.") - parser.add_argument("--start-hz", type=float, default=10_000_000.0) - parser.add_argument("--stop-hz", type=float, default=100_000_000.0) - parser.add_argument("--points", type=int, default=11) - parser.add_argument("--ifbw-hz", type=float, default=10_000.0) - parser.add_argument("--power-dbm", type=float, default=-20.0) - return parser.parse_args() - - -def main() -> int: - """Connect to a remote K209, run one sweep, and validate its shape and values. - - Raises on an unexpected point count, non-finite samples, or a non-monotonic frequency - axis; prints a one-line summary on success. - """ - args = _parse_args() - sweep = RadarSweepModel( - start_hz=args.start_hz, - stop_hz=args.stop_hz, - points=args.points, - if_bandwidth_hz=args.ifbw_hz, - power_dbm=args.power_dbm, - ) - service = RemoteCompactMK209Service(host=args.host, port=args.port) - try: - service.open() - print(f"K209 IDN: {service.query_identity()}") - service.configure(sweep) - result = service.acquire() - finally: - service.close() - - s11 = result.trace("s11") - s21 = result.trace("s21") - if result.x.size != args.points or s11.size != args.points or s21.size != args.points: - raise RuntimeError("Remote K209 sweep returned an unexpected point count") - if not np.all(np.isfinite(result.x)) or not np.all(np.isfinite(s11)) or not np.all(np.isfinite(s21)): - raise RuntimeError("Remote K209 sweep contains non-finite values") - if not np.all(np.diff(result.x) >= 0): - raise RuntimeError("Remote K209 frequency axis is not monotonic") - - print( - "Remote K209 sweep OK: " - f"points={args.points}, first_hz={result.x[0]:.3f}, last_hz={result.x[-1]:.3f}, " - f"mean_abs_s11={float(np.mean(np.abs(s11))):.6g}, " - f"mean_abs_s21={float(np.mean(np.abs(s21))):.6g}" - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_app/scripts/k209_smoke_test.py b/python_app/scripts/k209_smoke_test.py deleted file mode 100644 index 4137ba7..0000000 --- a/python_app/scripts/k209_smoke_test.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Standalone smoke test for Compact-M K209 VISA HiSLIP acquisition.""" - -from __future__ import annotations - -import argparse - -import numpy as np - -from python_app.hardware_full.compact_m_k209_service import CompactMK209Service -from python_app.models.run_config_model import RadarSweepModel - - -def _parse_args() -> argparse.Namespace: - """Parse command-line options for the local K209 VISA smoke test.""" - parser = argparse.ArgumentParser(description="Acquire one K209 sweep through VISA HiSLIP") - parser.add_argument( - "--resource", - required=True, - help=( - "S2VNA VISA resource, e.g. TCPIP0::127.0.0.1::hislip0,4880::INSTR " - "when the USB-connected K209 is controlled by local S2VNA" - ), - ) - parser.add_argument("--start-hz", type=float, default=1_000_000.0) - parser.add_argument("--stop-hz", type=float, default=6_000_000_000.0) - parser.add_argument("--points", type=int, default=201) - parser.add_argument("--ifbw-hz", type=float, default=50_000.0) - parser.add_argument("--power-dbm", type=float, default=-10.0) - parser.add_argument("--timeout-ms", type=int, default=20_000) - parser.add_argument("--no-preset", action="store_true") - parser.add_argument( - "--visa-library", - default="@ivi", - help="PyVISA IVI backend specification, e.g. @ivi or /usr/lib/x86_64-linux-gnu/libvisa.so", - ) - return parser.parse_args() - - -def _validate_result(result, expected_points: int) -> None: - """Raise RuntimeError unless the sweep has the expected point count, monotonic axis, and finite S11/S21.""" - if result.x.shape != (expected_points,): - raise RuntimeError(f"Unexpected frequency shape: {result.x.shape}") - s11 = result.trace("s11") - s21 = result.trace("s21") - if s11.shape != (expected_points,) or s21.shape != (expected_points,): - raise RuntimeError(f"Unexpected trace shapes: s11={s11.shape}, s21={s21.shape}") - if not np.all(np.isfinite(result.x)): - raise RuntimeError("Frequency axis contains non-finite values") - if np.any(np.diff(result.x) < 0.0): - raise RuntimeError("Frequency axis is not monotonic") - if not np.all(np.isfinite(s11.real)) or not np.all(np.isfinite(s11.imag)): - raise RuntimeError("S11 contains non-finite values") - if not np.all(np.isfinite(s21.real)) or not np.all(np.isfinite(s21.imag)): - raise RuntimeError("S21 contains non-finite values") - - -def main() -> int: - """Acquire one sweep from a locally connected K209 and validate it end to end. - - Opens the VISA session, configures the sweep, validates the result, checks the SCPI - error queue, and prints a summary; raises on any validation or SCPI failure. - """ - args = _parse_args() - sweep = RadarSweepModel( - start_hz=args.start_hz, - stop_hz=args.stop_hz, - points=args.points, - if_bandwidth_hz=args.ifbw_hz, - power_dbm=args.power_dbm, - ) - - service = CompactMK209Service( - resource=args.resource, - timeout_ms=args.timeout_ms, - preset_on_open=not args.no_preset, - visa_library=args.visa_library, - ) - try: - service.open() - print(f"K209 IDN: {service.query_identity()}") - service.configure(sweep) - result = service.acquire() - _validate_result(result, args.points) - system_error = service.query_system_error() - if not system_error.startswith("0,"): - raise RuntimeError(f"K209 SCPI error after sweep: {system_error}") - print( - "K209 sweep OK: " - f"points={result.x.size}, first_hz={result.x[0]:.3f}, last_hz={result.x[-1]:.3f}, " - f"mean_abs_s11={np.mean(np.abs(result.trace('s11'))):.6g}, " - f"mean_abs_s21={np.mean(np.abs(result.trace('s21'))):.6g}" - ) - finally: - service.close() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_app/scripts/k209_sweep_benchmark.py b/python_app/scripts/k209_sweep_benchmark.py deleted file mode 100644 index 10df26d..0000000 --- a/python_app/scripts/k209_sweep_benchmark.py +++ /dev/null @@ -1,217 +0,0 @@ -"""Measure Compact-M K209 sweep acquisition throughput through VISA HiSLIP.""" - -from __future__ import annotations - -from dataclasses import dataclass -import statistics -import time - -import numpy as np - -from python_app.hardware_full.compact_m_k209_service import CompactMK209Service -from python_app.models.run_config_model import RadarSweepModel - - -RESOURCE = "TCPIP0::127.0.0.1::hislip0,4880::INSTR" -VISA_LIBRARY = "@ivi" - -START_HZ = 100_000_000.0 -STOP_HZ = 6_000_000_000.0 -POINTS = 1501 -IFBW_HZ = 10_000.0 -POWER_DBM = -20.0 - -TIMEOUT_MS = 20_000 -WARMUP_SWEEPS = 3 -TIMED_SWEEPS = 20 -PRESET_ON_OPEN = False - -# False measures the device/transport hot path: one synchronized TRIG:SING/*OPC? -# message plus S11/S21 REAL32 reads. -# True also includes public SweepResult construction and complex array conversion. -INCLUDE_RESULT_CONVERSION = False - - -@dataclass(frozen=True, slots=True) -class BenchmarkResult: - """Timing summary for repeated sweep acquisition.""" - - durations_s: list[float] - points: int - - @property - def total_s(self) -> float: - """Return total timed acquisition duration.""" - return sum(self.durations_s) - - @property - def sweeps_per_s(self) -> float: - """Return completed sweeps per second.""" - return len(self.durations_s) / self.total_s - - @property - def points_per_s(self) -> float: - """Return measured sweep points per second.""" - return (len(self.durations_s) * self.points) / self.total_s - - @property - def binary_payload_mb_per_s(self) -> float: - """Return S11+S21 binary payload throughput, excluding SCPI headers.""" - payload_bytes = len(self.durations_s) * self.points * 2 * 2 * np.dtype(np.float32).itemsize - return payload_bytes / self.total_s / 1_000_000.0 - - -def _validate_config() -> None: - """Raise if any module-level benchmark constant is outside its valid range.""" - if POINTS < 2: - raise ValueError("POINTS must be >= 2") - if WARMUP_SWEEPS < 0: - raise ValueError("WARMUP_SWEEPS must be >= 0") - if TIMED_SWEEPS <= 0: - raise ValueError("TIMED_SWEEPS must be > 0") - if IFBW_HZ <= 0.0: - raise ValueError("IFBW_HZ must be > 0") - if STOP_HZ < START_HZ: - raise ValueError("STOP_HZ must be >= START_HZ") - if TIMEOUT_MS <= 0: - raise ValueError("TIMEOUT_MS must be > 0") - - -def _validate_interleaved(raw, expected_points: int) -> None: - """Raise unless a raw interleaved sweep has the expected shapes, monotonic axis, and finite values.""" - if raw.frequency_hz.shape != (expected_points,): - raise RuntimeError(f"Unexpected frequency shape: {raw.frequency_hz.shape}") - if raw.s11_values.shape != (expected_points * 2,): - raise RuntimeError(f"Unexpected S11 shape: {raw.s11_values.shape}") - if raw.s21_values.shape != (expected_points * 2,): - raise RuntimeError(f"Unexpected S21 shape: {raw.s21_values.shape}") - if not np.all(np.isfinite(raw.frequency_hz)): - raise RuntimeError("Frequency axis contains non-finite values") - if np.any(np.diff(raw.frequency_hz) < 0.0): - raise RuntimeError("Frequency axis is not monotonic") - if not np.all(np.isfinite(raw.s11_values)): - raise RuntimeError("S11 contains non-finite values") - if not np.all(np.isfinite(raw.s21_values)): - raise RuntimeError("S21 contains non-finite values") - - -def _validate_result(result, expected_points: int) -> None: - """Raise unless a converted SweepResult has the expected shapes and finite S11/S21.""" - if result.x.shape != (expected_points,): - raise RuntimeError(f"Unexpected frequency shape: {result.x.shape}") - for name in ("s11", "s21"): - values = result.trace(name) - if values.shape != (expected_points,): - raise RuntimeError(f"Unexpected {name.upper()} shape: {values.shape}") - if not np.all(np.isfinite(values.real)) or not np.all(np.isfinite(values.imag)): - raise RuntimeError(f"{name.upper()} contains non-finite values") - - -def _percentile(values: list[float], percentile: float) -> float: - """Return the nearest-rank percentile (0..1) of ``values``.""" - sorted_values = sorted(values) - index = round((len(sorted_values) - 1) * percentile) - return sorted_values[index] - - -def _run_benchmark(service: CompactMK209Service, *, points: int, warmup: int, sweeps: int, convert: bool) -> BenchmarkResult: - """Time ``sweeps`` acquisitions after ``warmup`` warm-up sweeps and return their durations. - - When ``convert`` is True the timed path includes SweepResult construction; otherwise it - times the raw interleaved REAL32 acquisition. The first and last sweeps are validated. - """ - acquire = service.acquire if convert else service.acquire_interleaved - - first = acquire() - if convert: - _validate_result(first, points) - else: - _validate_interleaved(first, points) - - for _ in range(warmup): - acquire() - - durations_s: list[float] = [] - for _ in range(sweeps): - start_ns = time.perf_counter_ns() - last = acquire() - end_ns = time.perf_counter_ns() - durations_s.append((end_ns - start_ns) / 1_000_000_000.0) - - if convert: - _validate_result(last, points) - else: - _validate_interleaved(last, points) - return BenchmarkResult(durations_s=durations_s, points=points) - - -def _print_limits(limits: dict[str, float | int]) -> None: - """Print the device frequency/IFBW/power/point limits as a single human-readable line.""" - print( - "K209 limits: " - f"frequency={limits['min_frequency_hz']:.0f}..{limits['max_frequency_hz']:.0f} Hz, " - f"IFBW={limits['min_ifbw_hz']:.0f}..{limits['max_ifbw_hz']:.0f} Hz, " - f"power={limits['min_power_dbm']:.1f}..{limits['max_power_dbm']:.1f} dBm, " - f"max_points={limits['max_points']}" - ) - - -def main() -> int: - """Run the K209 sweep-acquisition benchmark and print timing/throughput statistics.""" - _validate_config() - - sweep = RadarSweepModel( - start_hz=START_HZ, - stop_hz=STOP_HZ, - points=POINTS, - if_bandwidth_hz=IFBW_HZ, - power_dbm=POWER_DBM, - ) - service = CompactMK209Service( - resource=RESOURCE, - timeout_ms=TIMEOUT_MS, - preset_on_open=PRESET_ON_OPEN, - visa_library=VISA_LIBRARY, - ) - - try: - service.open() - print(f"K209 IDN: {service.query_identity()}") - _print_limits(service.read_device_limits()) - service.configure(sweep) - print( - "Benchmark settings: " - f"start_hz={START_HZ:.3f}, stop_hz={STOP_HZ:.3f}, " - f"points={POINTS}, ifbw_hz={IFBW_HZ:.3f}, power_dbm={POWER_DBM:.3f}, " - f"warmup={WARMUP_SWEEPS}, sweeps={TIMED_SWEEPS}, " - f"mode={'SweepResult' if INCLUDE_RESULT_CONVERSION else 'raw interleaved REAL32'}" - ) - result = _run_benchmark( - service, - points=POINTS, - warmup=WARMUP_SWEEPS, - sweeps=TIMED_SWEEPS, - convert=INCLUDE_RESULT_CONVERSION, - ) - system_error = service.query_system_error() - if not system_error.startswith("0,"): - raise RuntimeError(f"K209 SCPI error after benchmark: {system_error}") - finally: - service.close() - - durations_ms = [value * 1_000.0 for value in result.durations_s] - print("Benchmark result:") - print(f" total_s={result.total_s:.6f}") - print(f" sweep_mean_ms={statistics.fmean(durations_ms):.3f}") - print(f" sweep_median_ms={statistics.median(durations_ms):.3f}") - print(f" sweep_min_ms={min(durations_ms):.3f}") - print(f" sweep_max_ms={max(durations_ms):.3f}") - print(f" sweep_p95_ms={_percentile(durations_ms, 0.95):.3f}") - print(f" sweeps_per_s={result.sweeps_per_s:.3f}") - print(f" points_per_s={result.points_per_s:.1f}") - print(f" s11_s21_payload_mb_per_s={result.binary_payload_mb_per_s:.3f}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_app/scripts/kamil_adc_raw_producer.py b/python_app/scripts/kamil_adc_raw_producer.py index b3b25dd..1e94747 100644 --- a/python_app/scripts/kamil_adc_raw_producer.py +++ b/python_app/scripts/kamil_adc_raw_producer.py @@ -3,7 +3,7 @@ from __future__ import annotations import argparse -from contextlib import suppress +from contextlib import ExitStack, suppress import logging from pathlib import Path import signal @@ -131,42 +131,46 @@ def main() -> int: args.config, len(config.combos), config.runtime.continuous, ) - raw_writer = ShmRingWriter( - config.rings.raw.name, - config.rings.raw.capacity, - config.rings.raw.slot_size_bytes, - ) - raw_tap_writer = ShmRingWriter( - config.rings.raw_tap.name, - config.rings.raw_tap.capacity, - config.rings.raw_tap.slot_size_bytes, - ) - logger.debug( - "Opened SHM ring writers: raw=%s, raw_tap=%s", - config.rings.raw.name, config.rings.raw_tap.name, - ) - # Switch-aware mode: with native switches the collector drives the RF switches - # itself, in the hardware gap between sweeps, and tags each sweep with its combo - # — no sweep lost at a switch boundary. The producer then only reads tagged - # sweeps and must not touch the GPIO lines the collector owns. With mock switches - # (dev/tests) we keep the Python-driven path, which is fine where speed and the - # in-gap timing do not matter. - collector_driven = ( - config.input_switch.driver_mode == "native" - and config.output_switch.driver_mode == "native" - ) - radar = KamilAdcService( - config, - switch_config_path=str(args.config) if collector_driven else None, - ) - input_switch = None if collector_driven else SwitchService.from_model(config.input_switch) - output_switch = None if collector_driven else SwitchService.from_model(config.output_switch) - logger.info( - "Kamil ADC switch control: %s", - "collector-driven (in-gap, lossless)" if collector_driven else "producer-driven", - ) - + radar = input_switch = output_switch = None + cleanup = ExitStack() try: + raw_writer = ShmRingWriter( + config.rings.raw.name, + config.rings.raw.capacity, + config.rings.raw.slot_size_bytes, + ) + cleanup.callback(raw_writer.close) + raw_tap_writer = ShmRingWriter( + config.rings.raw_tap.name, + config.rings.raw_tap.capacity, + config.rings.raw_tap.slot_size_bytes, + ) + cleanup.callback(raw_tap_writer.close) + logger.debug( + "Opened SHM ring writers: raw=%s, raw_tap=%s", + config.rings.raw.name, config.rings.raw_tap.name, + ) + # Switch-aware mode: with native switches the collector drives the RF switches + # itself, in the hardware gap between sweeps, and tags each sweep with its combo + # — no sweep lost at a switch boundary. The producer then only reads tagged + # sweeps and must not touch the GPIO lines the collector owns. With mock switches + # (dev/tests) we keep the Python-driven path, which is fine where speed and the + # in-gap timing do not matter. + collector_driven = ( + config.input_switch.driver_mode == "native" + and config.output_switch.driver_mode == "native" + ) + radar = KamilAdcService( + config, + switch_config_path=str(args.config) if collector_driven else None, + ) + input_switch = None if collector_driven else SwitchService.from_model(config.input_switch) + output_switch = None if collector_driven else SwitchService.from_model(config.output_switch) + logger.info( + "Kamil ADC switch control: %s", + "collector-driven (in-gap, lossless)" if collector_driven else "producer-driven", + ) + if not _open_radar_with_retry(config, radar, input_switch, output_switch, stop_requested): return 0 # asked to stop before a device became available @@ -187,10 +191,12 @@ def main() -> int: sweep_start_ns = time.monotonic_ns() sweep = radar.acquire(combo=(combo.input, combo.output)) else: + if input_switch is None or output_switch is None: + raise RuntimeError("Producer-driven switches are not initialized") output_switch.switch_to(combo.output) input_switch.switch_to(combo.input) - if config.runtime.settling_ms > 0: - time.sleep(config.runtime.settling_ms / 1000.0) + if config.runtime.settling_ms > 0 and stop_requested.wait(config.runtime.settling_ms / 1000.0): + break # Stamped after switching and settling so the window covers # the sweep alone, not the dead time before it. sweep_start_ns = time.monotonic_ns() @@ -264,10 +270,10 @@ def main() -> int: if input_switch is not None: with suppress(Exception): input_switch.close() - with suppress(Exception): - radar.close() - raw_tap_writer.close() - raw_writer.close() + if radar is not None: + with suppress(Exception): + radar.close() + cleanup.close() logger.info("Kamil ADC raw producer stopped") return 0 diff --git a/python_app/scripts/matrix_raw_producer.py b/python_app/scripts/matrix_raw_producer.py index 7cbf30c..86426e4 100644 --- a/python_app/scripts/matrix_raw_producer.py +++ b/python_app/scripts/matrix_raw_producer.py @@ -1,16 +1,16 @@ -"""Raw acquisition producer for matrix-mode radars (LibreVNA multi-device, SN9000).""" +"""Raw producer for native matrix radars and the shared single/multi mock scene.""" from __future__ import annotations import argparse import logging -from contextlib import suppress +from contextlib import ExitStack, suppress from pathlib import Path import signal import threading import time -from python_app.hardware_full.matrix_radar_service import MatrixRadarService, create_matrix_radar_service +from python_app.hardware_full.matrix_radar_service import MatrixRadarService, create_acquisition_service from python_app.logging_setup import coerce_level from python_app.models.run_config_model import RunConfigModel from python_app.orchestration.shm import ShmRingWriter @@ -48,7 +48,7 @@ def _open_radar_with_retry( attempt = 0 delay = _OPEN_RETRY_MIN_S while not stop_requested.is_set(): - radar = create_matrix_radar_service(config) + radar = create_acquisition_service(config) try: radar.open() radar.configure(config.radar.sweep) @@ -100,33 +100,31 @@ def main() -> int: # turned on from the profile instead of requiring a code edit. basicConfig above # only installed the handler; the package logger owns the level. logging.getLogger("python_app").setLevel(coerce_level(config.logging.level)) - if not config.is_matrix_radar: - raise RuntimeError( - "matrix_raw_producer requires a matrix-mode radar.model " - "(librevna_multi or sn9000)" - ) logger.info( "Matrix radar raw producer starting: config=%s, model=%s, continuous=%s", args.config, config.radar.model, config.runtime.continuous, ) - raw_writer = ShmRingWriter( - config.rings.raw.name, - config.rings.raw.capacity, - config.rings.raw.slot_size_bytes, - ) - raw_tap_writer = ShmRingWriter( - config.rings.raw_tap.name, - config.rings.raw_tap.capacity, - config.rings.raw_tap.slot_size_bytes, - ) - logger.debug( - "Opened SHM ring writers: raw=%s, raw_tap=%s", - config.rings.raw.name, config.rings.raw_tap.name, - ) - radar: MatrixRadarService | None = None + cleanup = ExitStack() try: + raw_writer = ShmRingWriter( + config.rings.raw.name, + config.rings.raw.capacity, + config.rings.raw.slot_size_bytes, + ) + cleanup.callback(raw_writer.close) + raw_tap_writer = ShmRingWriter( + config.rings.raw_tap.name, + config.rings.raw_tap.capacity, + config.rings.raw_tap.slot_size_bytes, + ) + cleanup.callback(raw_tap_writer.close) + logger.debug( + "Opened SHM ring writers: raw=%s, raw_tap=%s", + config.rings.raw.name, config.rings.raw_tap.name, + ) + radar = _open_radar_with_retry(config, previous=None, stop_requested=stop_requested) if radar is None: return 0 # asked to stop before a device became available @@ -174,8 +172,7 @@ def main() -> int: if radar is not None: with suppress(Exception): radar.close() - raw_tap_writer.close() - raw_writer.close() + cleanup.close() logger.info("matrix radar raw producer stopped") return 0 diff --git a/python_app/scripts/replay_raw_producer.py b/python_app/scripts/replay_raw_producer.py index f0c089e..b16c6c2 100644 --- a/python_app/scripts/replay_raw_producer.py +++ b/python_app/scripts/replay_raw_producer.py @@ -20,6 +20,7 @@ never see time jump backwards. ``collection_id`` increases forever. from __future__ import annotations import argparse +from contextlib import ExitStack import json import logging from pathlib import Path @@ -39,7 +40,7 @@ from python_app.storage.npz.serialize import RAW_MAGIC, serialize_trace_collecti logger = logging.getLogger(__name__) -_SNAPSHOT_NAME = re.compile(r"^\d+_id\d+_ns(\d+)$") +_SNAPSHOT_NAME = re.compile(r"^(\d+)_id\d+_ns(\d+)$") class ReplayDataset: @@ -62,11 +63,11 @@ def _load_dataset(dataset_dir: Path, combos: list[ComboKey], expected_points: in if not dataset_dir.is_dir(): raise RuntimeError(f"radar.replay.dataset_dir is not a directory: {dataset_dir}") - named: list[tuple[str, int]] = [] - for entry in sorted(dataset_dir.iterdir()): + named: list[tuple[int, str, int]] = [] + for entry in dataset_dir.iterdir(): match = _SNAPSHOT_NAME.match(entry.name) if entry.is_dir() and match: - named.append((entry.name, int(match.group(1)))) + named.append((int(match.group(1)), entry.name, int(match.group(2)))) if not named: raise RuntimeError( f"No snapshot folders (NNNN_idK_ns) found in {dataset_dir}" @@ -74,7 +75,7 @@ def _load_dataset(dataset_dir: Path, combos: list[ComboKey], expected_points: in snapshots: list[list[TraceData]] = [] timestamps_ns: list[int] = [] - for name, timestamp_ns in named: + for _, name, timestamp_ns in sorted(named): folder = dataset_dir / name # Recorded per-trace capture windows from the snapshot's meta.json: the # sar3d segmented timing model reconstructs the acquisition segments @@ -102,10 +103,12 @@ def _load_dataset(dataset_dir: Path, combos: list[ComboKey], expected_points: in f"input={combo.input} output={combo.output}: expected " f"{s21_path.name} and {freq_path.name}. Check run.combos vs the dataset." ) - s21 = np.load(s21_path).astype(np.complex64, copy=False) - freq = np.load(freq_path).astype(np.float32, copy=False) - if s21.size != freq.size: - raise RuntimeError(f"{s21_path}: s21 size {s21.size} != freq size {freq.size}") + s21 = np.load(s21_path, allow_pickle=False).astype(np.complex64, copy=False) + freq = np.load(freq_path, allow_pickle=False).astype(np.float32, copy=False) + s11_path = folder / f"i{combo.input}_o{combo.output}_s11.npy" + s11 = np.load(s11_path, allow_pickle=False).astype(np.complex64, copy=False) if s11_path.is_file() else np.zeros_like(s21) + if freq.ndim != 1 or s21.shape != freq.shape or s11.shape != freq.shape: + raise RuntimeError(f"Snapshot {name}: frequency, S11 and S21 must be equally sized 1D arrays") if expected_points and s21.size != expected_points: raise RuntimeError( f"{s21_path}: {s21.size} points, but radar.sweep.points = " @@ -116,7 +119,7 @@ def _load_dataset(dataset_dir: Path, combos: list[ComboKey], expected_points: in TraceData( combo=combo, frequency_hz=freq, - s11=np.zeros_like(s21), + s11=s11, s21=s21, capture_start_ns=window[0], capture_end_ns=window[1], @@ -206,25 +209,34 @@ def main() -> int: dataset.median_gap_ns / 1e9, replay.loop, rate_scale, ) - raw_writer = ShmRingWriter( - config.rings.raw.name, - config.rings.raw.capacity, - config.rings.raw.slot_size_bytes, - ) - raw_tap_writer = ShmRingWriter( - config.rings.raw_tap.name, - config.rings.raw_tap.capacity, - config.rings.raw_tap.slot_size_bytes, - ) - + cleanup = ExitStack() try: + raw_writer = ShmRingWriter( + config.rings.raw.name, + config.rings.raw.capacity, + config.rings.raw.slot_size_bytes, + ) + cleanup.callback(raw_writer.close) + raw_tap_writer = ShmRingWriter( + config.rings.raw_tap.name, + config.rings.raw_tap.capacity, + config.rings.raw_tap.slot_size_bytes, + ) + cleanup.callback(raw_tap_writer.close) + + payload_size = len(serialize_trace_collection( + _build_collection(dataset, 0, 1, time.monotonic_ns()), RAW_MAGIC + )) + for ring in (config.rings.raw, config.rings.raw_tap): + if ring.slot_size_bytes < payload_size: + raise ValueError(f"Replay requires {payload_size} bytes per slot; ring {ring.name} has {ring.slot_size_bytes}") + base_ns = time.monotonic_ns() wall_start_s = time.monotonic() first_ts = dataset.timestamps_ns[0] cycle_span_ns = dataset.timestamps_ns[-1] - first_ts + dataset.median_gap_ns collection_id = 1 - publish_failures = 0 cycle = 0 while not stop_requested.is_set(): for index in range(len(dataset)): @@ -246,15 +258,9 @@ def main() -> int: collection = _build_collection(dataset, index, collection_id, frame_ns) payload = serialize_trace_collection(collection, RAW_MAGIC) - if raw_writer.push(payload): - raw_tap_writer.push(payload) # best-effort GUI tap; never fatal - else: - publish_failures += 1 - if publish_failures == 1 or publish_failures % 100 == 0: - logger.error( - "Replay payload %d B exceeds ring slot %d B; dropping collection %d (drops=%d)", - len(payload), raw_writer.slot_size_bytes, collection_id, publish_failures, - ) + if not raw_writer.push(payload): + raise RuntimeError(f"Replay raw ring rejected collection {collection_id}") + raw_tap_writer.push(payload) if collection_id == 1 or collection_id % 50 == 0: logger.info( "replay collection %d published (snapshot %d/%d, cycle %d)", @@ -268,8 +274,7 @@ def main() -> int: break cycle += 1 finally: - raw_tap_writer.close() - raw_writer.close() + cleanup.close() logger.info("replay raw producer stopped") return 0 diff --git a/python_app/scripts/sn9000_smoke_test.py b/python_app/scripts/sn9000_smoke_test.py deleted file mode 100644 index b28f6d6..0000000 --- a/python_app/scripts/sn9000_smoke_test.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Standalone smoke test for the SN9000 multi-port VISA HiSLIP acquisition path.""" - -from __future__ import annotations - -import argparse - -import numpy as np - -from python_app.hardware_full.sn9000_service import Sn9000Service -from python_app.models.dataset_model import SweepCollection -from python_app.models.run_config_model import RadarSweepModel, RunConfigModel - - -def _parse_args() -> argparse.Namespace: - """Parse command-line options for the SN9000 VISA smoke test.""" - parser = argparse.ArgumentParser(description="Acquire one SN9000 collection through VISA HiSLIP") - parser.add_argument( - "--host", - default="127.0.0.1", - help="SNVNA HiSLIP server host (default: 127.0.0.1)", - ) - parser.add_argument( - "--port", - type=int, - default=4880, - help="SNVNA HiSLIP server TCP port (default: 4880)", - ) - parser.add_argument("--start-hz", type=float, default=1_000_000.0) - parser.add_argument("--stop-hz", type=float, default=3_000_000_000.0) - parser.add_argument("--points", type=int, default=201) - parser.add_argument("--ifbw-hz", type=float, default=10_000.0) - parser.add_argument("--power-dbm", type=float, default=-10.0) - parser.add_argument("--timeout-ms", type=int, default=20_000) - parser.add_argument("--no-preset", action="store_true") - parser.add_argument( - "--visa-library", - default="@ivi", - help="PyVISA IVI backend specification, e.g. @ivi or /usr/lib/x86_64-linux-gnu/libvisa.so", - ) - return parser.parse_args() - - -def _validate_collection(collection: SweepCollection, expected_points: int) -> None: - """Raise unless a SN9000 collection has the expected traces, combos, shapes, and finite values.""" - expected_traces = ( - RunConfigModel.MULTI_DEVICE_INPUT_POSITIONS * RunConfigModel.MULTI_DEVICE_OUTPUT_POSITIONS - ) - if len(collection.traces) != expected_traces: - raise RuntimeError( - f"SN9000 collection has {len(collection.traces)} traces, expected {expected_traces}" - ) - - reference_frequency = collection.traces[0].frequency_hz - if reference_frequency.shape != (expected_points,): - raise RuntimeError(f"Unexpected frequency shape: {reference_frequency.shape}") - if not np.all(np.isfinite(reference_frequency)): - raise RuntimeError("Frequency axis contains non-finite values") - if np.any(np.diff(reference_frequency) < 0.0): - raise RuntimeError("Frequency axis is not monotonic") - - expected_combos = RunConfigModel.build_matrix_radar_virtual_combos() - for trace, expected_combo in zip(collection.traces, expected_combos, strict=True): - if ( - int(trace.combo.input) != int(expected_combo.input) - or int(trace.combo.output) != int(expected_combo.output) - ): - raise RuntimeError( - f"Trace combo mismatch: got ({trace.combo.input},{trace.combo.output}), " - f"expected ({expected_combo.input},{expected_combo.output})" - ) - if trace.s11.shape != (expected_points,) or trace.s21.shape != (expected_points,): - raise RuntimeError( - f"Trace ({trace.combo.input},{trace.combo.output}) has shapes " - f"s11={trace.s11.shape}, s21={trace.s21.shape}" - ) - if not np.all(np.isfinite(trace.s11.real)) or not np.all(np.isfinite(trace.s11.imag)): - raise RuntimeError( - f"Trace ({trace.combo.input},{trace.combo.output}) S11 contains non-finite values" - ) - if not np.all(np.isfinite(trace.s21.real)) or not np.all(np.isfinite(trace.s21.imag)): - raise RuntimeError( - f"Trace ({trace.combo.input},{trace.combo.output}) S21 contains non-finite values" - ) - - -def main() -> int: - """Acquire one SN9000 collection and validate it end to end. - - Opens the VISA session, configures the sweep, validates the collection, checks the SCPI - error queue, and prints a summary; raises on any validation or SCPI failure. - """ - args = _parse_args() - sweep = RadarSweepModel( - start_hz=args.start_hz, - stop_hz=args.stop_hz, - points=args.points, - if_bandwidth_hz=args.ifbw_hz, - power_dbm=args.power_dbm, - ) - - service = Sn9000Service( - host=args.host, - port=args.port, - timeout_ms=args.timeout_ms, - preset_on_open=not args.no_preset, - visa_library=args.visa_library, - ) - try: - service.open() - print(f"SN9000 IDN: {service.query_identity()}") - service.configure(sweep) - collection = service.acquire_collection(collection_id=1) - _validate_collection(collection, args.points) - system_error = service.query_system_error() - if not system_error.startswith("0,"): - raise RuntimeError(f"SN9000 SCPI error after sweep: {system_error}") - mean_abs_s21 = np.mean([float(np.mean(np.abs(trace.s21))) for trace in collection.traces]) - print( - "SN9000 collection OK: " - f"traces={len(collection.traces)}, points={args.points}, " - f"first_hz={collection.traces[0].frequency_hz[0]:.3f}, " - f"last_hz={collection.traces[0].frequency_hz[-1]:.3f}, " - f"mean_abs_s21={mean_abs_s21:.6g}" - ) - finally: - service.close() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_app/storage/npz/serialize.py b/python_app/storage/npz/serialize.py index 8f727b0..2dfac1d 100644 --- a/python_app/storage/npz/serialize.py +++ b/python_app/storage/npz/serialize.py @@ -3,6 +3,7 @@ from __future__ import annotations import struct +from numbers import Integral import numpy as np @@ -21,6 +22,41 @@ def _write_interleaved_complex(buffer: bytearray, values: np.ndarray) -> None: buffer.extend(interleaved.tobytes()) +def validate_trace_collection(collection: SweepCollection, *, require_finite: bool = False) -> None: + """Reject ambiguous combinations and malformed traces before saving any files.""" + for field in ("collection_id", "monotonic_ns", "capture_start_ns", "capture_end_ns"): + _validate_u64(getattr(collection, field), field) + if require_finite and not collection.traces: + raise ValueError("Preprocess set must contain at least one trace") + seen: set[tuple[int, int]] = set() + for trace in collection.traces: + _validate_u64(trace.capture_start_ns, "trace.capture_start_ns") + _validate_u64(trace.capture_end_ns, "trace.capture_end_ns") + if any( + isinstance(value, bool) or not isinstance(value, Integral) or not 0 <= value <= 0xFFFFFFFF + for value in (trace.combo.input, trace.combo.output) + ): + raise ValueError("Trace input/output positions must be unsigned 32-bit integers") + combo = (int(trace.combo.input), int(trace.combo.output)) + if combo in seen: + raise ValueError(f"Invalid or duplicate trace combination: {combo}") + seen.add(combo) + arrays = ( + np.asarray(trace.frequency_hz, dtype=np.float32), + np.asarray(trace.s11, dtype=np.complex64), + np.asarray(trace.s21, dtype=np.complex64), + ) + if any(array.ndim != 1 for array in arrays) or len({array.shape for array in arrays}) != 1: + raise ValueError(f"Trace {combo} frequency, S11 and S21 must be equally sized 1D arrays") + if require_finite and (not arrays[0].size or any(not np.isfinite(array).all() for array in arrays)): + raise ValueError(f"Preprocess trace {combo} must contain finite, nonempty arrays") + + +def _validate_u64(value: int, field: str) -> None: + if isinstance(value, bool) or not isinstance(value, Integral) or not 0 <= value < 1 << 64: + raise ValueError(f"{field} must be an unsigned 64-bit integer") + + def serialize_trace_collection(collection: SweepCollection, magic: int) -> bytes: """Serialize one raw/preprocessed trace collection into ring-compatible binary format. @@ -30,6 +66,7 @@ def serialize_trace_collection(collection: SweepCollection, magic: int) -> bytes trace order). See :func:`python_app.orchestration.shm.decoder.decode_trace_collection` and ``read_trace_collection`` in ``common_cpp/ipc/src/shared_types.cpp``. """ + validate_trace_collection(collection) buffer = bytearray() buffer.extend(struct.pack(" bytes freq = np.asarray(trace.frequency_hz, dtype=np.float32) s11 = np.asarray(trace.s11, dtype=np.complex64) s21 = np.asarray(trace.s21, dtype=np.complex64) - if freq.size != s11.size: - raise ValueError("Trace frequency and S11 sizes must match") - if freq.size != s21.size: - raise ValueError("Trace frequency and S21 sizes must match") - buffer.extend(struct.pack(" bytes: if payload.kind == 1: freq = np.asarray(payload.frequency_hz, dtype=np.float32) trace = np.asarray(payload.trace, dtype=np.complex64) - if freq.size != trace.size: - raise ValueError("Result trace frequency and values sizes must match") + if freq.ndim != 1 or trace.ndim != 1 or freq.shape != trace.shape: + raise ValueError("Result trace frequency and values must be equally sized 1D arrays") buffer.extend(struct.pack(" bytes: image_x_axis = np.asarray(payload.image_x_axis, dtype=np.float32) image_y_axis = np.asarray(payload.image_y_axis, dtype=np.float32) image = np.asarray(payload.image, dtype=np.float32) - if image.ndim != 2: - raise ValueError("Result image payload must be a 2D matrix") - if image.shape != (image_y_axis.size, image_x_axis.size): + if image_x_axis.ndim != 1 or image_y_axis.ndim != 1 or image.shape != (image_y_axis.size, image_x_axis.size): raise ValueError("Result image axis sizes must match image matrix shape") buffer.extend(struct.pack(" bytes: table = np.asarray(payload.table, dtype=np.float32) if table.ndim != 2: raise ValueError("Result table payload must be a 2D matrix") + if table.shape[0] and not table.shape[1]: + raise ValueError("Result table with rows must have at least one column") buffer.extend(struct.pack(" Iterator[Path]: + """Publish a complete export directory; discard staging files after a failed write.""" + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists(): + raise FileExistsError(f"Snapshot directory already exists: {destination}") + staging = Path(tempfile.mkdtemp(prefix=f".{destination.name}.", dir=destination.parent)) + try: + yield staging + if destination.exists(): + raise FileExistsError(f"Snapshot directory already exists: {destination}") + staging.rename(destination) + finally: + if staging.exists(): + shutil.rmtree(staging) + + +def _matching_collection( + history: list[SweepCollection], anchor: SweepCollection | ResultCollection, + used: set[int], *, same_timestamp: bool, +) -> SweepCollection | None: + """Match preprocessing identities exactly and raw sweeps by their capture window. + + Producers stamp raw/preprocessed collections at different instants. Their capture + window is copied unchanged; results keep the preprocessed timestamp. Old files + without timing fall back to the latest preceding collection with the same id. + """ + candidates = [ + (index, item) for index, item in enumerate(history) + if index not in used and int(item.collection_id) == int(anchor.collection_id) + ] + timestamp = int(anchor.monotonic_ns) + start_ns = int(getattr(anchor, "capture_start_ns", 0)) + end_ns = int(getattr(anchor, "capture_end_ns", 0)) + if same_timestamp and timestamp: + candidates = [(index, item) for index, item in candidates if int(item.monotonic_ns) == timestamp] + elif start_ns or end_ns: + candidates = [ + (index, item) for index, item in candidates + if (int(item.capture_start_ns), int(item.capture_end_ns)) == (start_ns, end_ns) + ] + elif timestamp: + candidates = [(index, item) for index, item in candidates if int(item.monotonic_ns) <= timestamp] + if not candidates: + return None + index, item = max(candidates, key=lambda pair: (int(pair[1].monotonic_ns), pair[0])) + used.add(index) + return item def select_aligned_histories( @@ -23,89 +74,60 @@ def select_aligned_histories( preprocessed_history: list[SweepCollection], result_history: list[ResultCollection], last_n: int, + *, + include_unaligned: bool = False, ) -> tuple[list[SweepCollection], list[SweepCollection], list[ResultCollection], dict[str, Any]]: - """Select aligned tails with `results` as preferred anchor stage.""" - raw_index, raw_pos = _index_by_collection_sequence(raw_history) - pre_index, pre_pos = _index_by_collection_sequence(preprocessed_history) - result_index, result_pos = _index_by_collection_sequence(result_history) - - if result_index: - ordered_result_keys = sorted(result_index, key=lambda key: result_pos[key]) - selected_keys = ordered_result_keys[-last_n:] - selected_raw = [raw_index[key] for key in selected_keys if key in raw_index] - selected_pre = [pre_index[key] for key in selected_keys if key in pre_index] - selected_results = [result_index[key] for key in selected_keys] - raw_missing = len(selected_keys) - len(selected_raw) - pre_missing = len(selected_keys) - len(selected_pre) + """Select aligned tails, or retain every stage independently for complete exports.""" + if last_n <= 0: + raise ValueError("last_n must be > 0") + if include_unaligned: return ( - selected_raw, - selected_pre, - selected_results, - { - "selection_mode": "result_tail_with_optional_alignment", - "anchor_stage": "results", - "selected_collection_ids": [int(key[0]) for key in selected_keys], - "aligned_key_count": len(selected_keys), - "raw_missing_count": raw_missing, - "preprocessed_missing_count": pre_missing, - }, + raw_history[-last_n:], preprocessed_history[-last_n:], result_history[-last_n:], + {"selection_mode": "independent_tail", "anchor_stage": "none", + "selected_collection_ids": [int(item.collection_id) for item in result_history[-last_n:]], + "aligned_key_count": 0}, ) - if raw_index: - ordered_raw_keys = sorted(raw_index, key=lambda key: raw_pos[key]) - selected_keys = ordered_raw_keys[-last_n:] - selected_raw = [raw_index[key] for key in selected_keys] - selected_pre = [pre_index[key] for key in selected_keys if key in pre_index] - selected_results = [result_index[key] for key in selected_keys if key in result_index] - pre_missing = len(selected_keys) - len(selected_pre) - result_missing = len(selected_keys) - len(selected_results) - return ( - selected_raw, - selected_pre, - selected_results, - { - "selection_mode": "raw_tail_with_optional_alignment", - "anchor_stage": "raw", - "selected_collection_ids": [int(key[0]) for key in selected_keys], - "aligned_key_count": len(selected_keys), - "preprocessed_missing_count": pre_missing, - "result_missing_count": result_missing, - }, - ) + selected_raw: list[SweepCollection] = [] + selected_pre: list[SweepCollection] = [] + selected_results = result_history[-last_n:] + used_raw: set[int] = set() + used_pre: set[int] = set() + if selected_results: + anchors = selected_results + anchor_stage = "results" + for result in reversed(selected_results): + pre = _matching_collection(preprocessed_history, result, used_pre, same_timestamp=True) + raw = _matching_collection(raw_history, pre or result, used_raw, same_timestamp=False) + if pre is not None: + selected_pre.append(pre) + if raw is not None: + selected_raw.append(raw) + selected_pre.reverse() + selected_raw.reverse() + elif preprocessed_history: + selected_pre = preprocessed_history[-last_n:] + anchors = selected_pre + anchor_stage = "preprocessed" + for pre in reversed(selected_pre): + raw = _matching_collection(raw_history, pre, used_raw, same_timestamp=False) + if raw is not None: + selected_raw.append(raw) + selected_raw.reverse() + else: + selected_raw = raw_history[-last_n:] + anchors = selected_raw + anchor_stage = "raw" if anchors else "none" - if pre_index: - ordered_pre_keys = sorted(pre_index, key=lambda key: pre_pos[key]) - selected_keys = ordered_pre_keys[-last_n:] - selected_raw = [raw_index[key] for key in selected_keys if key in raw_index] - selected_pre = [pre_index[key] for key in selected_keys] - selected_results = [result_index[key] for key in selected_keys if key in result_index] - raw_missing = len(selected_keys) - len(selected_raw) - result_missing = len(selected_keys) - len(selected_results) - return ( - selected_raw, - selected_pre, - selected_results, - { - "selection_mode": "preprocessed_tail_with_optional_alignment", - "anchor_stage": "preprocessed", - "selected_collection_ids": [int(key[0]) for key in selected_keys], - "aligned_key_count": len(selected_keys), - "raw_missing_count": raw_missing, - "result_missing_count": result_missing, - }, - ) - - return ( - raw_history[-last_n:], - preprocessed_history[-last_n:], - result_history[-last_n:], - { - "selection_mode": "independent_tail", - "anchor_stage": "none", - "selected_collection_ids": [], - "aligned_key_count": 0, - }, - ) + return selected_raw, selected_pre, selected_results, { + "selection_mode": f"{anchor_stage}_tail_with_optional_alignment", + "anchor_stage": anchor_stage, + "selected_collection_ids": [int(item.collection_id) for item in anchors], + "aligned_key_count": len(anchors), + "raw_missing_count": len(anchors) - len(selected_raw), + "preprocessed_missing_count": len(anchors) - len(selected_pre), + "result_missing_count": len(anchors) - len(selected_results), + } def save_trace_history_binary(stage_dir: Path, history: list[SweepCollection], magic: int) -> None: @@ -155,6 +177,7 @@ def save_result_history_binary(stage_dir: Path, history: list[ResultCollection]) { "collection_id": collection.collection_id, "monotonic_ns": collection.monotonic_ns, + "processing_duration_ns": int(collection.processing_duration_ns), "collection_payload_count": len(collection.collection_payloads), "block_count": len(collection.blocks), }, @@ -173,6 +196,8 @@ def save_trace_history_numpy( streaming recorder can append successive chunks into the same stage directory without colliding or restarting the index. """ + for collection in history: + validate_trace_collection(collection) stage_dir.mkdir(parents=True, exist_ok=True) logger.debug("Writing %d NumPy trace collection(s) to %s", len(history), stage_dir) for index, collection in enumerate(history, start=index_offset): @@ -219,197 +244,84 @@ def save_trace_history_numpy( ) +def _save_result_payload(directory: Path, base_name: str, payload: ResultPayload) -> dict[str, Any]: + """Persist every supported payload kind identically at collection and combo scope.""" + meta: dict[str, Any] = {"kind": int(payload.kind), "name": payload.processing_name} + + def save_array(field: str, array: np.ndarray) -> None: + filename = f"{base_name}_{field}.npy" + np.save(directory / filename, array, allow_pickle=False) + meta[f"{field}_file"] = filename + + if payload.kind == 1: + freq = np.asarray(payload.frequency_hz, dtype=np.float32) + trace = np.asarray(payload.trace, dtype=np.complex64) + if freq.ndim != 1 or trace.ndim != 1 or freq.shape != trace.shape: + raise ValueError("Result trace frequency and values must be equally sized 1D arrays") + save_array("freq", freq) + save_array("trace", trace) + meta["points"] = int(freq.size) + elif payload.kind == 2: + save_array("scalar", np.asarray([payload.scalar_value], dtype=np.float32)) + meta["scalar_value"] = float(payload.scalar_value) + elif payload.kind == 3: + x_axis = np.asarray(payload.image_x_axis, dtype=np.float32) + y_axis = np.asarray(payload.image_y_axis, dtype=np.float32) + image = np.asarray(payload.image, dtype=np.float32) + if x_axis.ndim != 1 or y_axis.ndim != 1 or image.shape != (y_axis.size, x_axis.size): + raise ValueError("Result image axis sizes must match its 2D matrix shape") + save_array("x_axis", x_axis) + save_array("y_axis", y_axis) + save_array("image", image) + meta.update(x_points=int(x_axis.size), y_points=int(y_axis.size)) + elif payload.kind == 4: + table = np.asarray(payload.table, dtype=np.float32) + if table.ndim != 2: + raise ValueError("Result table payload must be a 2D matrix") + save_array("table", table) + meta.update(rows=int(table.shape[0]), columns=int(table.shape[1])) + else: + raise ValueError(f"Unsupported payload kind: {payload.kind}") + return meta + + def save_result_history_numpy( stage_dir: Path, history: list[ResultCollection], *, index_offset: int = 0 ) -> None: - """Write processed result collections as a NumPy directory tree. - - ``index_offset`` continues the per-collection directory numbering across calls (see - :func:`save_trace_history_numpy`) so streamed chunks append cleanly. - """ + """Write result payloads, preserving per-stage indices across streaming chunks.""" stage_dir.mkdir(parents=True, exist_ok=True) logger.debug("Writing %d NumPy result collection(s) to %s", len(history), stage_dir) for index, collection in enumerate(history, start=index_offset): collection_dir = stage_dir / collection_dir_name(index, collection.collection_id, collection.monotonic_ns) collection_dir.mkdir(parents=True, exist_ok=False) - - collection_payload_meta: list[dict[str, int | str | float]] = [] + collection_payload_meta = [] for payload_index, payload in enumerate(collection.collection_payloads): - safe_name = sanitize_path_component(payload.processing_name or "processor") - base_name = f"collection_{payload_index:03d}_{safe_name}_kind{payload.kind}" - if payload.kind == 1: - freq = np.asarray(payload.frequency_hz, dtype=np.float32) - trace = np.asarray(payload.trace, dtype=np.complex64) - np.save(collection_dir / f"{base_name}_freq.npy", freq) - np.save(collection_dir / f"{base_name}_trace.npy", trace) - collection_payload_meta.append( - { - "kind": int(payload.kind), - "name": payload.processing_name, - "points": int(freq.size), - "freq_file": f"{base_name}_freq.npy", - "trace_file": f"{base_name}_trace.npy", - } - ) - elif payload.kind == 2: - scalar = np.asarray([float(payload.scalar_value)], dtype=np.float32) - np.save(collection_dir / f"{base_name}_scalar.npy", scalar) - collection_payload_meta.append( - { - "kind": int(payload.kind), - "name": payload.processing_name, - "scalar_file": f"{base_name}_scalar.npy", - "scalar_value": float(payload.scalar_value), - } - ) - elif payload.kind == 3: - image_x_axis = np.asarray(payload.image_x_axis, dtype=np.float32) - image_y_axis = np.asarray(payload.image_y_axis, dtype=np.float32) - image = np.asarray(payload.image, dtype=np.float32) - np.save(collection_dir / f"{base_name}_x_axis.npy", image_x_axis) - np.save(collection_dir / f"{base_name}_y_axis.npy", image_y_axis) - np.save(collection_dir / f"{base_name}_image.npy", image) - collection_payload_meta.append( - { - "kind": int(payload.kind), - "name": payload.processing_name, - "x_points": int(image_x_axis.size), - "y_points": int(image_y_axis.size), - "x_axis_file": f"{base_name}_x_axis.npy", - "y_axis_file": f"{base_name}_y_axis.npy", - "image_file": f"{base_name}_image.npy", - } - ) - elif payload.kind == 4: - table = np.asarray(payload.table, dtype=np.float32) - np.save(collection_dir / f"{base_name}_table.npy", table) - collection_payload_meta.append( - { - "kind": int(payload.kind), - "name": payload.processing_name, - "rows": int(table.shape[0]) if table.ndim == 2 else 0, - "columns": int(table.shape[1]) if table.ndim == 2 else 0, - "table_file": f"{base_name}_table.npy", - } - ) + name = sanitize_path_component(payload.processing_name or "processor") + base_name = f"collection_{payload_index:03d}_{name}_kind{payload.kind}" + collection_payload_meta.append(_save_result_payload(collection_dir, base_name, payload)) - blocks_meta: list[dict[str, int | str | list[dict[str, int | str | float]]]] = [] + blocks_meta = [] for block_index, block in enumerate(collection.blocks): block_dir = collection_dir / f"block_{block_index:03d}_i{block.combo.input}_o{block.combo.output}" block_dir.mkdir(parents=True, exist_ok=False) - - payload_meta: list[dict[str, int | str | float]] = [] + payload_meta = [] for payload_index, payload in enumerate(block.payloads): - safe_name = sanitize_path_component(payload.processing_name or "processor") - base_name = f"{payload_index:03d}_{safe_name}_kind{payload.kind}" - if payload.kind == 1: - freq = np.asarray(payload.frequency_hz, dtype=np.float32) - trace = np.asarray(payload.trace, dtype=np.complex64) - np.save(block_dir / f"{base_name}_freq.npy", freq) - np.save(block_dir / f"{base_name}_trace.npy", trace) - payload_meta.append( - { - "kind": int(payload.kind), - "name": payload.processing_name, - "points": int(freq.size), - "freq_file": f"{base_name}_freq.npy", - "trace_file": f"{base_name}_trace.npy", - } - ) - elif payload.kind == 2: - scalar = np.asarray([float(payload.scalar_value)], dtype=np.float32) - np.save(block_dir / f"{base_name}_scalar.npy", scalar) - payload_meta.append( - { - "kind": int(payload.kind), - "name": payload.processing_name, - "scalar_file": f"{base_name}_scalar.npy", - "scalar_value": float(payload.scalar_value), - } - ) - elif payload.kind == 3: - image_x_axis = np.asarray(payload.image_x_axis, dtype=np.float32) - image_y_axis = np.asarray(payload.image_y_axis, dtype=np.float32) - image = np.asarray(payload.image, dtype=np.float32) - np.save(block_dir / f"{base_name}_x_axis.npy", image_x_axis) - np.save(block_dir / f"{base_name}_y_axis.npy", image_y_axis) - np.save(block_dir / f"{base_name}_image.npy", image) - payload_meta.append( - { - "kind": int(payload.kind), - "name": payload.processing_name, - "x_points": int(image_x_axis.size), - "y_points": int(image_y_axis.size), - "x_axis_file": f"{base_name}_x_axis.npy", - "y_axis_file": f"{base_name}_y_axis.npy", - "image_file": f"{base_name}_image.npy", - } - ) - elif payload.kind == 4: - table = np.asarray(payload.table, dtype=np.float32) - np.save(block_dir / f"{base_name}_table.npy", table) - payload_meta.append( - { - "kind": int(payload.kind), - "name": payload.processing_name, - "rows": int(table.shape[0]) if table.ndim == 2 else 0, - "columns": int(table.shape[1]) if table.ndim == 2 else 0, - "table_file": f"{base_name}_table.npy", - } - ) - - blocks_meta.append( - { - "input": int(block.combo.input), - "output": int(block.combo.output), - "payload_count": len(block.payloads), - "dir": block_dir.name, - "payloads": payload_meta, - } - ) + name = sanitize_path_component(payload.processing_name or "processor") + base_name = f"{payload_index:03d}_{name}_kind{payload.kind}" + payload_meta.append(_save_result_payload(block_dir, base_name, payload)) + blocks_meta.append({ + "input": int(block.combo.input), "output": int(block.combo.output), + "payload_count": len(block.payloads), "dir": block_dir.name, "payloads": payload_meta, + }) (collection_dir / "meta.json").write_text( - json.dumps( - { - "collection_id": int(collection.collection_id), - "monotonic_ns": int(collection.monotonic_ns), - "collection_payload_count": len(collection.collection_payloads), - "collection_payloads": collection_payload_meta, - "block_count": len(collection.blocks), - "blocks": blocks_meta, - }, - indent=2, - ), + json.dumps({ + "collection_id": int(collection.collection_id), + "monotonic_ns": int(collection.monotonic_ns), + "processing_duration_ns": int(collection.processing_duration_ns), + "collection_payload_count": len(collection.collection_payloads), + "collection_payloads": collection_payload_meta, + "block_count": len(collection.blocks), "blocks": blocks_meta, + }, indent=2), encoding="utf-8", ) - - -def validate_snapshot_name(snapshot_name: str) -> str: - """Normalize snapshot directory stem.""" - return sanitize_path_component(snapshot_name) - - -def _index_by_collection_sequence( - history: list[TCollection], -) -> tuple[dict[tuple[int, int], TCollection], dict[tuple[int, int], int]]: - """Index collections by latest-first `(collection_id, occurrence)` preserving history position. - - Occurrence is counted from the tail (newest item is occurrence `0` for its - collection id). This avoids cross-run misalignment when collection ids are - reused after restarts and one stage keeps longer history than another. - """ - indexed: dict[tuple[int, int], TCollection] = {} - positions: dict[tuple[int, int], int] = {} - seen_count_from_tail_by_id: dict[int, int] = {} - - # Walk from newest to oldest so occurrence=0 always means the most recent - # instance for the given collection id. - for index in range(len(history) - 1, -1, -1): - item = history[index] - collection_id = int(getattr(item, "collection_id")) - occurrence = seen_count_from_tail_by_id.get(collection_id, 0) - key = (collection_id, occurrence) - indexed[key] = item - positions[key] = index - seen_count_from_tail_by_id[collection_id] = occurrence + 1 - return indexed, positions diff --git a/python_app/storage/npz/store.py b/python_app/storage/npz/store.py index 17d5a12..d95130f 100644 --- a/python_app/storage/npz/store.py +++ b/python_app/storage/npz/store.py @@ -6,25 +6,31 @@ from contextlib import suppress from datetime import datetime, timezone import json import logging +import os from pathlib import Path +import shutil +import tempfile from typing import Any +from zipfile import BadZipFile import numpy as np from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection, TraceData from python_app.storage.npz.paths import radar_key_from_config, sanitize_path_component -from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, serialize_trace_collection +from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, serialize_trace_collection, validate_trace_collection from python_app.storage.npz.snapshot_numpy import ( save_result_history_binary, save_result_history_numpy, save_trace_history_binary, save_trace_history_numpy, select_aligned_histories, + atomic_snapshot_directory, ) from python_app.storage.npz.vna_history_json import build_vna_history_payload from python_app.storage.store_api import StoreApi logger = logging.getLogger(__name__) +_SET_METADATA_KEY = "metadata_json" def _compose_snapshot_stem(name: str, name_prefix: str) -> str: @@ -37,7 +43,7 @@ def _compose_snapshot_stem(name: str, name_prefix: str) -> str: class SnapshotStreamWriter: - """Append aligned raw/preprocessed/result collections to a snapshot dir in chunks. + """Append received raw/preprocessed/result collections to a snapshot dir in chunks. A streaming alternative to :meth:`NpzStore.save_runtime_snapshot_numpy` for long recordings: the caller flushes small chunks as measurements arrive and frees them, @@ -69,12 +75,32 @@ class SnapshotStreamWriter: preprocessed: list[SweepCollection], results: list[ResultCollection], ) -> None: - """Write one chunk of (already aligned) collections, continuing each stage's index.""" - save_trace_history_numpy(self._dir / "raw", raw, index_offset=self._raw_written) - save_trace_history_numpy( - self._dir / "preprocessed", preprocessed, index_offset=self._preprocessed_written - ) - save_result_history_numpy(self._dir / "results", results, index_offset=self._result_written) + """Write a chunk without requiring equal stage lengths, continuing each index.""" + # Stage a whole chunk before publishing complete collection directories. A + # failed write leaves earlier chunks intact and the next retry uses the same indices. + chunk_dir = Path(tempfile.mkdtemp(prefix=".chunk-", dir=self._dir)) + published: list[Path] = [] + try: + save_trace_history_numpy(chunk_dir / "raw", raw, index_offset=self._raw_written) + save_trace_history_numpy( + chunk_dir / "preprocessed", preprocessed, index_offset=self._preprocessed_written + ) + save_result_history_numpy(chunk_dir / "results", results, index_offset=self._result_written) + for stage in ("raw", "preprocessed", "results"): + stage_dir = self._dir / stage + stage_dir.mkdir(exist_ok=True) + for source in sorted((chunk_dir / stage).iterdir()): + destination = stage_dir / source.name + if destination.exists(): + raise FileExistsError(f"Collection directory already exists: {destination}") + source.rename(destination) + published.append(destination) + except BaseException: + for destination in published: + shutil.rmtree(destination) + raise + finally: + shutil.rmtree(chunk_dir) self._raw_written += len(raw) self._preprocessed_written += len(preprocessed) self._result_written += len(results) @@ -96,13 +122,15 @@ class NpzStore(StoreApi): def save_set(self, kind: str, radar_key: str, set_name: str, collection: SweepCollection) -> None: """Persist named preprocess set as NPZ and metadata JSON.""" + self._validate_set_name(set_name) + validate_trace_collection(collection, require_finite=True) set_dir = self._set_dir(kind, radar_key) set_dir.mkdir(parents=True, exist_ok=True) npz_path = set_dir / f"{set_name}.npz" meta_path = set_dir / f"{set_name}.json" - payload: dict[str, np.ndarray] = {} + payload: dict[str, Any] = {} combo_records: list[dict[str, str | int]] = [] for trace in collection.traces: @@ -133,67 +161,137 @@ class NpzStore(StoreApi): "combos": combo_records, } - # Write both files to temporary paths first, then atomically rename so a - # crash never leaves an .npz without its meta (or vice versa). - npz_tmp = npz_path.with_name(npz_path.name + ".tmp") - meta_tmp = meta_path.with_name(meta_path.name + ".tmp") + # Metadata in the NPZ makes a single atomic replacement authoritative. The + # JSON remains a readable sidecar and legacy NPZ/JSON pairs remain loadable. + metadata_json = json.dumps(meta, indent=2) + payload[_SET_METADATA_KEY] = np.asarray(metadata_json) + npz_tmp = meta_tmp = backup_path = None + archive_replaced = False try: + npz_tmp = self._temporary_file(npz_path) + meta_tmp = self._temporary_file(meta_path) + backup_path = self._temporary_file(npz_path) if npz_path.exists() else None + if backup_path is not None: + backup_path.unlink() + os.link(npz_path, backup_path) with npz_tmp.open("wb") as npz_file: np.savez(npz_file, **payload) - meta_tmp.write_text(json.dumps(meta, indent=2), encoding="utf-8") + meta_tmp.write_text(metadata_json, encoding="utf-8") npz_tmp.replace(npz_path) + archive_replaced = True meta_tmp.replace(meta_path) except BaseException: logger.exception("Failed to save set %s/%s/%s; rolling back temp files", kind, radar_key, set_name) - for tmp_path in (npz_tmp, meta_tmp): + if archive_replaced: + if backup_path is None: + npz_path.unlink(missing_ok=True) + else: + try: + backup_path.replace(npz_path) + except OSError: + logger.exception("Set rollback failed; previous archive retained at %s", backup_path) + backup_path = None # retain the recovery copy rather than deleting it below + raise + raise + finally: + for tmp_path in (npz_tmp, meta_tmp, backup_path): + if tmp_path is None: + continue with suppress(OSError): tmp_path.unlink(missing_ok=True) - raise logger.info( "Saved set %s/%s/%s (%d traces) to %s", kind, radar_key, set_name, len(combo_records), npz_path ) + def save_sets(self, radar_key: str, set_name: str, collections: dict[str, SweepCollection]) -> None: + """Save related sets together, restoring their previous files if any save fails.""" + self._validate_set_name(set_name) + for collection in collections.values(): + validate_trace_collection(collection, require_finite=True) + backup_dir = Path(tempfile.mkdtemp(prefix=".sets-", dir=self._root_dir)) + backups: dict[Path, Path | None] = {} + retain_backups = False + try: + for kind in collections: + set_dir = self._set_dir(kind, radar_key) + for extension in ("npz", "json"): + destination = set_dir / f"{set_name}.{extension}" + backup = backup_dir / f"{kind}.{extension}" if destination.exists() else None + if backup is not None: + os.link(destination, backup) + backups[destination] = backup + try: + for kind, collection in collections.items(): + self.save_set(kind, radar_key, set_name, collection) + except BaseException as save_error: + retain_backups = True # preserve recovery files if rollback itself is interrupted + rollback_failed = False + for destination, backup in backups.items(): + try: + if backup is None: + destination.unlink(missing_ok=True) + else: + backup.replace(destination) + except OSError: + rollback_failed = True + logger.exception("Failed to restore previous set file %s", destination) + retain_backups = rollback_failed + if retain_backups: + raise RuntimeError( + f"Saving set group '{set_name}' failed and rollback was incomplete; " + f"previous files are retained at {backup_dir}" + ) from save_error + raise + finally: + if not retain_backups: + with suppress(OSError): + shutil.rmtree(backup_dir) + def load_set(self, kind: str, radar_key: str, set_name: str) -> SweepCollection: """Load named preprocess set from NPZ representation.""" + self._validate_set_name(set_name) set_dir = self._set_dir(kind, radar_key) npz_path = set_dir / f"{set_name}.npz" meta_path = set_dir / f"{set_name}.json" - if not npz_path.exists() or not meta_path.exists(): + if not npz_path.exists(): logger.error("Missing set files for %s/%s/%s", kind, radar_key, set_name) raise FileNotFoundError(f"Missing set files for {kind}/{radar_key}/{set_name}") set_label = f"{kind}/{radar_key}/{set_name}" try: - meta = json.loads(meta_path.read_text(encoding="utf-8")) - arrays = np.load(npz_path, allow_pickle=False) - - traces: list[TraceData] = [] - for combo in meta["combos"]: - freq = np.asarray(arrays[combo["freq_key"]], dtype=np.float32) - s11 = np.asarray(arrays[combo["s11_key"]], dtype=np.complex64) - s21 = np.asarray(arrays[combo["s21_key"]], dtype=np.complex64) - traces.append( - TraceData( - combo=ComboKey(input=int(combo["input"]), output=int(combo["output"])), - frequency_hz=freq, - s11=s11, - s21=s21, - capture_start_ns=int(combo.get("capture_start_ns", 0)), - capture_end_ns=int(combo.get("capture_end_ns", 0)), - ) + with np.load(npz_path, allow_pickle=False) as arrays: + meta = json.loads( + str(arrays[_SET_METADATA_KEY].item()) + if _SET_METADATA_KEY in arrays + else meta_path.read_text(encoding="utf-8") ) + if not isinstance(meta, dict): + raise ValueError("Set metadata must be a JSON object") + traces: list[TraceData] = [] + for combo in meta["combos"]: + traces.append( + TraceData( + combo=ComboKey(input=combo["input"], output=combo["output"]), + frequency_hz=np.asarray(arrays[combo["freq_key"]], dtype=np.float32), + s11=np.asarray(arrays[combo["s11_key"]], dtype=np.complex64), + s21=np.asarray(arrays[combo["s21_key"]], dtype=np.complex64), + capture_start_ns=combo.get("capture_start_ns", 0), + capture_end_ns=combo.get("capture_end_ns", 0), + ) + ) collection = SweepCollection( - collection_id=int(meta["collection_id"]), - monotonic_ns=int(meta["monotonic_ns"]), + collection_id=meta["collection_id"], + monotonic_ns=meta["monotonic_ns"], traces=traces, - capture_start_ns=int(meta.get("capture_start_ns", 0)), - capture_end_ns=int(meta.get("capture_end_ns", 0)), + capture_start_ns=meta.get("capture_start_ns", 0), + capture_end_ns=meta.get("capture_end_ns", 0), ) + validate_trace_collection(collection, require_finite=True) logger.debug("Loaded set %s (%d traces)", set_label, len(traces)) return collection - except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: + except (OSError, BadZipFile, EOFError, KeyError, TypeError, ValueError) as exc: logger.exception("Corrupted preprocess set %s", set_label) raise RuntimeError(f"Corrupted preprocess set {set_label}: {exc}") from exc @@ -215,7 +313,12 @@ class NpzStore(StoreApi): """Export named set as binary collection bundle for C++ preprocessing stage.""" collection = self.load_set(kind, radar_key, set_name) output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_bytes(serialize_trace_collection(collection, RAW_MAGIC)) + temporary = self._temporary_file(output_path) + try: + temporary.write_bytes(serialize_trace_collection(collection, RAW_MAGIC)) + temporary.replace(output_path) + finally: + temporary.unlink(missing_ok=True) logger.info("Exported set %s/%s/%s bundle to %s", kind, radar_key, set_name, output_path) return output_path @@ -234,11 +337,10 @@ class NpzStore(StoreApi): output_dir.mkdir(parents=True, exist_ok=True) timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") snapshot_dir = output_dir / f"snapshot_{timestamp}" - snapshot_dir.mkdir(parents=True, exist_ok=True) - - save_trace_history_binary(snapshot_dir / "raw", raw_history[-last_n:], RAW_MAGIC) - save_trace_history_binary(snapshot_dir / "preprocessed", preprocessed_history[-last_n:], PREPROC_MAGIC) - save_result_history_binary(snapshot_dir / "results", result_history[-last_n:]) + with atomic_snapshot_directory(snapshot_dir) as staging: + save_trace_history_binary(staging / "raw", raw_history[-last_n:], RAW_MAGIC) + save_trace_history_binary(staging / "preprocessed", preprocessed_history[-last_n:], PREPROC_MAGIC) + save_result_history_binary(staging / "results", result_history[-last_n:]) logger.info("Saved binary runtime snapshot (last_n=%d) to %s", last_n, snapshot_dir) return snapshot_dir @@ -281,6 +383,7 @@ class NpzStore(StoreApi): last_n: int, *, name_prefix: str = "", + include_unaligned: bool = False, ) -> tuple[Path, dict[str, Any]]: """Save historical runtime collections in NumPy tree format.""" if last_n <= 0: @@ -292,19 +395,15 @@ class NpzStore(StoreApi): snapshot_dir = output_root_dir / snapshot_stem if snapshot_dir.exists(): raise FileExistsError(f"Snapshot directory already exists: {snapshot_dir}") - snapshot_dir.mkdir(parents=True, exist_ok=False) selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories( raw_history, preprocessed_history, result_history, last_n, + include_unaligned=include_unaligned, ) - save_trace_history_numpy(snapshot_dir / "raw", selected_raw) - save_trace_history_numpy(snapshot_dir / "preprocessed", selected_preprocessed) - save_result_history_numpy(snapshot_dir / "results", selected_results) - if selection_summary.get("anchor_stage") == "results": expected = min(int(last_n), len(result_history)) if len(selected_results) != expected: @@ -313,6 +412,11 @@ class NpzStore(StoreApi): f"expected={expected}, selected={len(selected_results)}" ) + with atomic_snapshot_directory(snapshot_dir) as staging: + save_trace_history_numpy(staging / "raw", selected_raw) + save_trace_history_numpy(staging / "preprocessed", selected_preprocessed) + save_result_history_numpy(staging / "results", selected_results) + selection_summary["raw_count"] = len(selected_raw) selection_summary["preprocessed_count"] = len(selected_preprocessed) selection_summary["result_count"] = len(selected_results) @@ -341,6 +445,7 @@ class NpzStore(StoreApi): output_index: int = 0, channel: str = "s21", primary_stage: str = "preprocessed", + include_unaligned: bool = False, ) -> tuple[Path, dict[str, Any]]: """Save runtime history as vna_system-compatible JSON file.""" if last_n <= 0: @@ -351,16 +456,14 @@ class NpzStore(StoreApi): output_root_dir.mkdir(parents=True, exist_ok=True) output_dir = self._vna_json_output_dir(output_root_dir, output_stem) - output_dir.mkdir(parents=True, exist_ok=True) output_path = output_dir / f"{output_stem}_vna_bscan_history.json" - if output_path.exists(): - raise FileExistsError(f"Output JSON file already exists: {output_path}") selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories( raw_history, preprocessed_history, result_history, last_n, + include_unaligned=include_unaligned, ) payload = build_vna_history_payload( selected_raw, @@ -371,10 +474,8 @@ class NpzStore(StoreApi): channel=channel, primary_stage=primary_stage, ) - output_path.write_text( - json.dumps(payload, ensure_ascii=False, indent=2), - encoding="utf-8", - ) + with atomic_snapshot_directory(output_dir) as staging: + (staging / output_path.name).write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") summary: dict[str, Any] = dict(selection_summary) summary["raw_count"] = len(selected_raw) @@ -412,6 +513,7 @@ class NpzStore(StoreApi): channel: str = "s21", primary_stage: str = "preprocessed", name_prefix: str = "", + include_unaligned: bool = False, ) -> tuple[list[Path], dict[str, Any]]: """Save one runtime VNA-history JSON per available combo.""" if last_n <= 0: @@ -420,13 +522,13 @@ class NpzStore(StoreApi): output_stem = _compose_snapshot_stem(output_name, name_prefix) output_root_dir.mkdir(parents=True, exist_ok=True) output_dir = self._vna_json_output_dir(output_root_dir, output_stem) - output_dir.mkdir(parents=True, exist_ok=True) selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories( raw_history, preprocessed_history, result_history, last_n, + include_unaligned=include_unaligned, ) combos = sorted( @@ -461,11 +563,11 @@ class NpzStore(StoreApi): ) output_paths.append(output_path) - for output_path, payload in zip(output_paths, payloads, strict=True): - output_path.write_text( - json.dumps(payload, ensure_ascii=False, indent=2), - encoding="utf-8", - ) + with atomic_snapshot_directory(output_dir) as staging: + for output_path, payload in zip(output_paths, payloads, strict=True): + (staging / output_path.name).write_text( + json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8" + ) summary: dict[str, Any] = dict(selection_summary) summary["raw_count"] = len(selected_raw) @@ -486,8 +588,25 @@ class NpzStore(StoreApi): def _set_dir(self, kind: str, radar_key: str) -> Path: """Return directory for set kind and radar key.""" + self._validate_set_name(kind) + self._validate_set_name(radar_key) return self._root_dir / kind / radar_key + @staticmethod + def _validate_set_name(value: str) -> None: + """Keep caller-provided set components inside their storage directory.""" + if not value or value in {".", ".."} or Path(value).name != value or "\\" in value: + raise ValueError(f"Invalid set path component: {value!r}") + + @staticmethod + def _temporary_file(destination: Path) -> Path: + """Create a unique temporary file beside its atomic-replacement destination.""" + with tempfile.NamedTemporaryFile( + dir=destination.parent, prefix=f".{destination.name}.", suffix=".tmp", delete=False + ) as handle: + path = Path(handle.name) + return path + def preview_png_dir(self, kind: str, set_name: str, radar_key: str) -> Path: """Return directory for preview PNGs of one set/radar variant. diff --git a/python_app/storage/npz/vna_history_json.py b/python_app/storage/npz/vna_history_json.py index 37884e5..edae0e2 100644 --- a/python_app/storage/npz/vna_history_json.py +++ b/python_app/storage/npz/vna_history_json.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections import defaultdict from dataclasses import dataclass from datetime import datetime, timezone import logging @@ -29,6 +28,7 @@ class TraceRecord: # timing. Preferred over the collection timestamp for the exported sweep time: # in a switched matrix each combo is measured at a different instant. capture_end_ns: int = 0 + capture_start_ns: int = 0 def _normalize_channel(channel: str) -> str: @@ -90,24 +90,12 @@ def _build_stage_records( frequency_hz=frequency_hz, samples=samples, capture_end_ns=int(trace.capture_end_ns), + capture_start_ns=int(trace.capture_start_ns), ) ) return records -def _index_by_collection_occurrence(records: list[TraceRecord]) -> tuple[dict[tuple[int, int], TraceRecord], list[tuple[int, int]]]: - counters: defaultdict[int, int] = defaultdict(int) - record_map: dict[tuple[int, int], TraceRecord] = {} - order: list[tuple[int, int]] = [] - for record in records: - occurrence = counters[record.collection_id] - counters[record.collection_id] += 1 - key = (record.collection_id, occurrence) - record_map[key] = record - order.append(key) - return record_map, order - - def _complex_to_points(values: np.ndarray) -> list[list[float]]: return [[float(value.real), float(value.imag)] for value in values] @@ -119,22 +107,41 @@ def _build_sweep_history( channel: str, primary_stage: str, ) -> list[dict[str, Any]]: - raw_map, raw_order = _index_by_collection_occurrence(raw_records) - pre_map, pre_order = _index_by_collection_occurrence(preprocessed_records) - - if primary_stage == "preprocessed": - primary_order = pre_order or raw_order - else: - primary_order = raw_order or pre_order + primary = preprocessed_records if primary_stage == "preprocessed" else raw_records + secondary = raw_records if primary_stage == "preprocessed" else preprocessed_records + used: set[int] = set() + pairs: list[tuple[TraceRecord, TraceRecord | None]] = [] + for base in primary: + candidates = [ + (index, record) for index, record in enumerate(secondary) + if index not in used and record.collection_id == base.collection_id + ] + if base.capture_start_ns or base.capture_end_ns: + candidates = [ + (index, record) for index, record in candidates + if (record.capture_start_ns, record.capture_end_ns) == (base.capture_start_ns, base.capture_end_ns) + ] + elif base.monotonic_ns: + # Raw is produced before preprocessing; never pair it to a newer run. + candidates = [ + (index, record) for index, record in candidates + if (record.monotonic_ns <= base.monotonic_ns if primary_stage == "preprocessed" + else record.monotonic_ns >= base.monotonic_ns) + ] + match = None + if candidates: + index, match = min(candidates, key=lambda pair: abs(pair[1].monotonic_ns - base.monotonic_ns)) + used.add(index) + pairs.append((base, match)) + # A tap can deliver a sweep while another stage drops it or is still processing. + # Preserve those records as well instead of silently discarding the extra stage. + pairs.extend((record, None) for index, record in enumerate(secondary) if index not in used) + pairs.sort(key=lambda pair: (pair[0].monotonic_ns, pair[0].stage_index)) history: list[dict[str, Any]] = [] - for fallback_index, key in enumerate(primary_order): - raw = raw_map.get(key) - pre = pre_map.get(key) - base = pre or raw - if base is None: - continue - + for fallback_index, (base, match) in enumerate(pairs): + raw = base if base.stage == "raw" else match + pre = base if base.stage == "preprocessed" else match sweep_source = raw or pre calibrated_source = pre or raw if sweep_source is None or calibrated_source is None: diff --git a/python_app/tests/gui_test_support.py b/python_app/tests/gui_test_support.py new file mode 100644 index 0000000..2bfb5b5 --- /dev/null +++ b/python_app/tests/gui_test_support.py @@ -0,0 +1,29 @@ +"""Construct desktop fixtures without touching a running application's resources.""" + +from pathlib import Path +import json +import os +import tempfile +from unittest import mock + +from python_app.gui.app_window import AppWindow + + +def isolated_window(add_cleanup) -> AppWindow: + temporary = tempfile.TemporaryDirectory(prefix="radar_gui_test_") + add_cleanup(temporary.cleanup) + root = Path(temporary.name) + source = Path(__file__).resolve().parents[2] + payload = json.loads((source / "run_config_examples/run_config_simulator.example.json").read_text(encoding="utf-8")) + for name, ring in payload["rings"].items(): + ring["name"] = f"/{root.name}_{name}" + payload["run"]["processing_live_config_path"] = str(root / "python_app/runtime/processing_live.json") + payload["control_button"]["enabled"] = False + profile = root / "run_config.json" + profile.write_text(json.dumps(payload), encoding="utf-8") + with mock.patch.dict(os.environ, { + "RADAR_SYSTEM_PROFILE": str(profile), "RADAR_SYSTEM_AUTO_START": "0", + "RADAR_SYSTEM_AUTO_APPLY_RADAR": "0", + }), mock.patch.object(AppWindow, "_init_web_ui"): + window = AppWindow(root) + return window diff --git a/python_app/tests/test_antenna_geometry.py b/python_app/tests/test_antenna_geometry.py new file mode 100644 index 0000000..9f98afa --- /dev/null +++ b/python_app/tests/test_antenna_geometry.py @@ -0,0 +1,134 @@ +"""Geometry edits preserve XYZ and reject stale or invalid processing results.""" + +from __future__ import annotations + +from dataclasses import asdict +import os +from pathlib import Path +import unittest +from unittest import mock + +import numpy as np + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") +from PyQt6.QtCore import QSignalBlocker +from PyQt6.QtWidgets import QApplication + +from python_app.gui.antenna_geometry_editor import AntennaGeometryEditor +from python_app.tests.gui_test_support import isolated_window +from python_app.models.dataset_model import ResultCollection, ResultPayload +from python_app.models.gui_profile_model import GuiProfileModel +from python_app.models.run_config_model import GprTxGeometryModel, GprRxGeometryModel + + +class AntennaGeometryTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.app = QApplication.instance() or QApplication([]) + + def test_coordinates_keep_full_precision_and_default_missing_axes(self) -> None: + entry = GprTxGeometryModel(output_pos=2, x_m=0.12345678912345, y_m=-2.5, z_m=3.5) + editor = AntennaGeometryEditor("output_pos", [entry], position_count=4) + self.assertEqual(editor.records(), [asdict(entry)]) + editor.set_records([{"output_pos": 1, "x_m": -1}]) + self.assertEqual(editor.records(), [{"output_pos": 1, "x_m": -1.0, "y_m": 0.0, "z_m": 0.0}]) + + def test_invalid_edit_rolls_back_without_emitting_a_change(self) -> None: + editor = AntennaGeometryEditor("input_pos", [GprRxGeometryModel(input_pos=0, x_m=1.0)], position_count=2) + changed = mock.Mock() + editor.changed.connect(changed) + with mock.patch("python_app.gui.antenna_geometry_editor.QMessageBox.warning") as warning: + editor._table.item(0, 1).setText("nan") + warning.assert_called_once() + changed.assert_not_called() + self.assertEqual(editor.records()[0]["x_m"], 1.0) + + def test_full_table_disables_add_and_removal_frees_a_position(self) -> None: + editor = AntennaGeometryEditor("input_pos", [GprRxGeometryModel(input_pos=0, x_m=1.0)], position_count=1) + self.assertFalse(editor._add_button.isEnabled()) + editor._add_row() + self.assertEqual(len(editor.records()), 1) + editor._table.selectRow(0) + editor._remove_button.click() + self.assertEqual(editor.records(), []) + self.assertTrue(editor._add_button.isEnabled()) + editor._add_button.click() + self.assertEqual(editor.records()[0]["input_pos"], 0) + + def test_web_geometry_rejects_duplicates_and_out_of_range_before_mutating(self) -> None: + editor = AntennaGeometryEditor("output_pos", [], position_count=2) + for records in ([{"output_pos": 2}], [{"output_pos": 0}, {"output_pos": 0}], + [{"output_pos": True}], [{"output_pos": 0, "x_m": True}]): + with self.subTest(records=records), self.assertRaises(ValueError): + editor.set_records(records) + self.assertEqual(editor.records(), []) + + +class WindowGeometryTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.app = QApplication.instance() or QApplication([]) + cls.window = isolated_window(cls.addClassCleanup) + cls.window._timer.stop() + cls.window._shutdown_web_ui() + cls.initial_config = cls.window._build_config() + + @classmethod + def tearDownClass(cls) -> None: + cls.window.close() + + def setUp(self) -> None: + self.w = self.window + self.w._apply_loaded_profile(GuiProfileModel(run_config=self.initial_config.clone()), Path("run_config.json")) + + def test_profile_and_live_config_preserve_all_coordinates(self) -> None: + config = self.initial_config.clone() + config.gpr.tx_geometry = [GprTxGeometryModel(output_pos=0, x_m=1.25, y_m=-2.0, z_m=3.0)] + config.gpr.rx_geometry = [GprRxGeometryModel(input_pos=0, x_m=-1.25, y_m=4.0, z_m=-5.0)] + self.w._apply_loaded_profile(GuiProfileModel(run_config=config), Path("run_config.json")) + rebuilt = self.w._build_config() + self.assertEqual(rebuilt.gpr, config.gpr) + live = self.w._live_processing_config().to_dict() + self.assertEqual(live["tx_geometry"], [asdict(entry) for entry in config.gpr.tx_geometry]) + self.assertEqual(live["rx_geometry"], [asdict(entry) for entry in config.gpr.rx_geometry]) + schema = self.w._web_settings_schema() + self.assertFalse(any(entry.get("applies_on_start") for entry in schema)) + + def test_malformed_web_batch_restores_every_changed_widget(self) -> None: + original_gain = self.w._bscan_gain.value() + original_mode = self.w._processing_mode.currentText() + with mock.patch.object(self.w, "_show_exception") as error, mock.patch.object(self.w._live_config_writer, "write") as write: + self.w._apply_web_live_settings({ + "processor_mode": "bscan", "bscan_gain": original_gain + 1, + "tx_geometry": [{"output_pos": 0}, {"output_pos": 0}], + }) + error.assert_called_once() + write.assert_not_called() + self.assertEqual(self.w._bscan_gain.value(), original_gain) + self.assertEqual(self.w._processing_mode.currentText(), original_mode) + + def test_stale_results_cannot_draw_over_new_geometry(self) -> None: + revision = (1 << 63) + 123456789 + self.w._required_processing_revision = revision + with QSignalBlocker(self.w._processing_mode): + self.w._processing_mode.setCurrentText("gpr") + words = [(revision >> shift) & 0xffff for shift in range(0, 64, 16)] + fresh = ResultCollection(collection_id=1, monotonic_ns=1, collection_payloads=[ResultPayload( + processing_name="processing_revision", kind=4, table=np.asarray([words], dtype=np.float32), + )]) + with mock.patch.object(self.w, "_draw_gpr_map", return_value=True) as draw: + self.assertFalse(self.w._draw_results(ResultCollection(collection_id=1, monotonic_ns=1))) + self.assertTrue(self.w._draw_results(fresh)) + draw.assert_called_once_with(fresh) + self.assertFalse(self.w._result_collection_has_trace(fresh)) + + def test_invalid_or_mixed_history_commands_do_not_clear_data(self) -> None: + with mock.patch.object(self.w, "_show_exception") as error, mock.patch.object(self.w, "_clear_all_runtime_history") as clear: + self.w._apply_web_live_settings({"history_command": "bogus"}) + self.w._apply_web_live_settings({"history_command": "clear_all", "bscan_gain": 2}) + self.assertEqual(error.call_count, 2) + clear.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/python_app/tests/test_bscan_history_window.py b/python_app/tests/test_bscan_history_window.py index c3547b4..85a3696 100644 --- a/python_app/tests/test_bscan_history_window.py +++ b/python_app/tests/test_bscan_history_window.py @@ -15,7 +15,6 @@ from __future__ import annotations import os import unittest -from pathlib import Path import numpy as np @@ -23,6 +22,7 @@ os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") from PyQt6.QtWidgets import QApplication # noqa: E402 from python_app.gui.app_window import AppWindow # noqa: E402 +from python_app.tests.gui_test_support import isolated_window from collections import deque # noqa: E402 from python_app.gui.runtime.history import record_result_history # noqa: E402 @@ -40,7 +40,7 @@ _window: AppWindow | None = None def setUpModule() -> None: global _app, _window _app = QApplication.instance() or QApplication([]) - _window = AppWindow(Path(".")) + _window = isolated_window(unittest.addModuleCleanup) def tearDownModule() -> None: diff --git a/python_app/tests/test_bscan_replay.py b/python_app/tests/test_bscan_replay.py index a9c6805..aaf006d 100644 --- a/python_app/tests/test_bscan_replay.py +++ b/python_app/tests/test_bscan_replay.py @@ -26,6 +26,7 @@ from PyQt6.QtCore import QSignalBlocker # noqa: E402 from PyQt6.QtWidgets import QApplication # noqa: E402 from python_app.gui.app_window import AppWindow # noqa: E402 +from python_app.tests.gui_test_support import isolated_window from python_app.gui.runtime.history import collection_key # noqa: E402 from python_app.models.dataset_model import ( # noqa: E402 ComboKey, @@ -50,7 +51,7 @@ _window: AppWindow | None = None def setUpModule() -> None: global _app, _window _app = QApplication.instance() or QApplication([]) - _window = AppWindow(Path(".")) + _window = isolated_window(unittest.addModuleCleanup) def tearDownModule() -> None: diff --git a/python_app/tests/test_cart_firmware.py b/python_app/tests/test_cart_firmware.py new file mode 100644 index 0000000..25f3af2 --- /dev/null +++ b/python_app/tests/test_cart_firmware.py @@ -0,0 +1,88 @@ +"""Run firmware logic on the host; timing and physical I/O still need the board.""" + +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +PROJECT = Path(__file__).resolve().parents[2] + +ARDUINO_STUB = r''' +#pragma once +#include +#include +#include +#include +inline uint32_t test_time = 0; +inline int transmissions = 0; +inline uint32_t millis() { return test_time; } +inline int analogRead(uint32_t pin) { return pin == 0 ? 2014 : 1981; } +inline void analogReadResolution(int) {} +inline void pinMode(uint32_t, int) {} +inline float constrain(float v, float lo, float hi) { return std::clamp(v, lo, hi); } +struct SerialStub { + void begin(int) {} + template void print(T, int = 0) {} + template void println(T) {} +}; +inline SerialStub Serial; +struct GPIO_InitTypeDef { int Pin, Mode, Pull, Speed, Alternate; }; +struct UART_HandleTypeDef { + int Instance; + struct { int BaudRate, WordLength, StopBits, Parity, Mode, HwFlowCtl, OverSampling; } Init; + struct { int AdvFeatureInit, TxPinLevelInvert; } AdvancedInit; +}; +enum { PA0, PA1, GPIO_PIN_9, GPIO_MODE_AF_PP, GPIO_NOPULL, GPIO_SPEED_FREQ_LOW, + GPIO_AF7_USART1, GPIOA, USART1, UART_WORDLENGTH_9B, UART_STOPBITS_2, + UART_PARITY_EVEN, UART_MODE_TX, UART_HWCONTROL_NONE, UART_OVERSAMPLING_16, + UART_ADVFEATURE_TXINVERT_INIT, UART_ADVFEATURE_TXINV_ENABLE, INPUT_ANALOG }; +inline void __HAL_RCC_GPIOA_CLK_ENABLE() {} +inline void __HAL_RCC_USART1_CLK_ENABLE() {} +inline void HAL_GPIO_Init(int, GPIO_InitTypeDef*) {} +inline void HAL_UART_Init(UART_HandleTypeDef*) {} +inline void HAL_UART_Transmit(UART_HandleTypeDef*, uint8_t*, int, int) { ++transmissions; } +''' + + +@unittest.skipUnless(shutil.which('g++'), 'requires a host C++ compiler') +class CartFirmwareTest(unittest.TestCase): + def test_schedule_rollover_and_sbus_encoding(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'Arduino.h').write_text(ARDUINO_STUB) + source = root / 'check.cpp' + firmware = PROJECT / 'device_firmware/cart_firmware/src/main.cpp' + source.write_text(f'#include "{firmware}"\n' + r''' +#include +int main() { + test_time = UINT32_MAX - 100; + loop(); assert(transmissions == 1); + test_time = 1; + loop(); assert(transmissions == 2); + test_time += 49; + loop(); assert(transmissions == 2); + ++test_time; + loop(); assert(transmissions == 3); + assert(axisNorm(X_MID, X_FWD, X_MID, X_BACK) == 0); + assert(axisNorm(X_FWD, X_FWD, X_MID, X_BACK) == 1); + assert(driveToSbus(0, CH2_SPAN, CH2_DB, POWER_FWD, POWER_BACK) == SBUS_MID); + uint16_t channels[16]; + for (int n = 0; n < 16; ++n) channels[n] = n * 127; + uint8_t packed[25]; sbusPack(packed, channels); + assert(packed[0] == 0x0f && packed[23] == 0 && packed[24] == 0); + for (int n = 0; n < 16; ++n) { + uint16_t value = 0; + for (int b = 0; b < 11; ++b) { + const int bit = n * 11 + b; + value |= ((packed[1 + bit / 8] >> (bit % 8)) & 1) << b; + } + assert(value == channels[n]); + } +} +''') + binary = root / 'check' + built = subprocess.run(['g++', '-std=c++20', '-Wall', '-Wextra', '-Werror', f'-I{root}', + str(source), '-o', str(binary)], capture_output=True, text=True, timeout=15) + self.assertEqual(built.returncode, 0, built.stderr) + subprocess.run([str(binary)], check=True, timeout=5) diff --git a/python_app/tests/test_combo_controls.py b/python_app/tests/test_combo_controls.py new file mode 100644 index 0000000..a07dba9 --- /dev/null +++ b/python_app/tests/test_combo_controls.py @@ -0,0 +1,131 @@ +"""Desktop channel navigation and fixed-size manual saves.""" + +from __future__ import annotations + +import os +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest import mock + +import numpy as np + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") +from PyQt6.QtCore import QSignalBlocker # noqa: E402 +from PyQt6.QtWidgets import QApplication # noqa: E402 + +from python_app.tests.gui_test_support import isolated_window +from python_app.gui.controllers.app_window_snapshot_mixin import AppWindowSnapshotMixin # noqa: E402 +from python_app.models.dataset_model import ComboKey, ResultBlock, ResultCollection, ResultPayload # noqa: E402 +from python_app.models.run_config_model import RunConfigModel # noqa: E402 + + +class ComboNavigationTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.app = QApplication.instance() or QApplication([]) + cls.window = isolated_window(cls.addClassCleanup) + cls.window._timer.stop() + cls.window._bscan_reprocess_timer.stop() + + @classmethod + def tearDownClass(cls) -> None: + cls.window.close() + + def setUp(self) -> None: + self.w = self.window + config = RunConfigModel() + config.radar.model = "librevna_multi" + config.apply_device_model_constraints() + self.w._defaults_config = config + self.w._set_combo_selection_mode("text") + with QSignalBlocker(self.w._processing_mode): + self.w._processing_mode.setCurrentText("bscan") + self.w._result_history.clear() + self.w._bscan_replay_results = [] + self.w._bscan_render_signature = None + self.w._single_combo_input.setText("0") + self.w._single_combo_output.setText("0") + + def test_next_button_draws_the_next_cached_combo_without_reprocessing(self) -> None: + self.w._result_history.append(ResultCollection( + collection_id=1, + monotonic_ns=1, + blocks=[ + ResultBlock(combo=ComboKey(input=0, output=output), payloads=[ResultPayload( + processing_name="bscan", kind=1, + frequency_hz=np.array([0.0, 1.0], dtype=np.float32), + trace=np.array([output + 1, output + 2], dtype=np.complex64), + )]) + for output in (0, 1) + ], + )) + with mock.patch.object(self.w, "_write_live_processing_config") as write_config: + self.w._next_combo_button.click() + + self.assertEqual(self.w._single_combo_input.text(), "0") + self.assertEqual(self.w._single_combo_output.text(), "1") + self.assertIn("in0/out1", self.w._bscan_plot.getPlotItem().titleLabel.text) + write_config.assert_not_called() + self.assertFalse(self.w._bscan_reprocess_timer.isActive()) + + def test_navigation_wraps_with_output_changing_first(self) -> None: + keys = self.w._available_display_combo_keys() + visited = [(0, 0)] + for _ in keys: + self.w._next_combo_button.click() + visited.append((int(self.w._single_combo_input.text()), int(self.w._single_combo_output.text()))) + self.assertEqual(visited, keys + [keys[0]]) + + def test_invalid_current_pair_recovers_to_the_first_pair(self) -> None: + self.w._single_combo_input.setText("invalid") + self.w._single_combo_output.setText("99") + self.w._next_combo_button.click() + self.assertEqual(self.w._single_combo_input.text(), "0") + self.assertEqual(self.w._single_combo_output.text(), "0") + + def test_unknown_selection_keeps_the_previous_fields(self) -> None: + with self.assertRaisesRegex(ValueError, "Unknown channel combination"): + self.w._select_display_combo(999, 999) + self.assertEqual(self.w._single_combo_input.text(), "0") + self.assertEqual(self.w._single_combo_output.text(), "0") + + +class ManualSaveLimitTest(unittest.TestCase): + def setUp(self) -> None: + self.w = AppWindowSnapshotMixin() + self.w._raw_history = ["raw"] + self.w._pre_history = ["pre"] + self.w._result_history = ["result"] + self.w._drain_runtime_rings_for_snapshot = mock.Mock() + self.w._show_exception = mock.Mock() + + def test_snapshot_always_saves_1000_and_preserves_unaligned_stages(self) -> None: + self.w._write_snapshot_dataset = mock.Mock() + self.w._save_snapshot() + self.w._write_snapshot_dataset.assert_called_once_with( + ["raw"], ["pre"], ["result"], 1000, include_unaligned=True, + ) + + def test_json_export_uses_the_same_fixed_limit(self) -> None: + self.w._save_path_input = SimpleNamespace(text=lambda: "/tmp/measurements") + self.w._save_name_input = SimpleNamespace(text=lambda: "scene") + self.w._radar_config_name_prefix = lambda: "radar" + save_json = mock.Mock(return_value=([Path("/tmp/measurements/scene/trace.json")], { + "output_stem": "scene", "output_dir": "/tmp/measurements/scene", + })) + self.w._store = SimpleNamespace(save_runtime_vna_history_json_batch=save_json) + self.w._write_gui_profile_to_path = mock.Mock() + self.w._log = mock.Mock() + + self.w._save_vna_history_json() + + self.w._show_exception.assert_not_called() + save_json.assert_called_once_with( + Path("/tmp/measurements"), "scene", ["raw"], ["pre"], ["result"], 1000, + channel="s21", primary_stage="preprocessed", name_prefix="radar", include_unaligned=True, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/python_app/tests/test_deploy_scripts.py b/python_app/tests/test_deploy_scripts.py new file mode 100644 index 0000000..02b1554 --- /dev/null +++ b/python_app/tests/test_deploy_scripts.py @@ -0,0 +1,66 @@ +"""Exercise updates and launch guards using isolated files and command stubs.""" + +import os +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest + +PROJECT = Path(__file__).resolve().parents[2] + + +class DeploymentTest(unittest.TestCase): + def test_update_preserves_state_and_an_independent_rollback_environment(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + target = root / 'radar_system' + release = root / 'release/radar_system' + for tree in (target, release): + (tree / 'python_app').mkdir(parents=True) + (tree / 'run_config_examples').mkdir(parents=True) + (tree / 'run_config_examples/device.json').write_text(tree.name) + (tree / 'run_config.json').write_text(str(tree)) + (release / 'start.sh').write_text('#!/bin/bash\n') + (target / 'run_config_examples/device.json').write_text('device settings') + (target / 'loose_recording.bin').write_bytes(b'original data') + (target / 'python_app/data/calibration').mkdir(parents=True) + (target / 'python_app/data/calibration/set.npz').write_bytes(b'calibration') + (target / 'python_app/runtime/logs').mkdir(parents=True) + (target / 'python_app/runtime/gui_session_state.json').write_text('session') + (target / 'python_app/runtime/logs/old.log').write_text('log') + (target / '.venv/bin').mkdir(parents=True) + (target / '.venv/state').write_text('old environment') + interpreter = target / '.venv/bin/python' + interpreter.write_text('''#!/bin/bash +if [[ "$1" == --version ]]; then echo Python-test; exit; fi +printf 'updated environment' > "$(dirname -- "$0")/../state" +''') + interpreter.chmod(0o755) + commands = root / 'commands' + commands.mkdir() + for name, text in {'systemctl': 'exit 3', 'make': 'printf "%s\\n" "$@" > "$RADAR_TEST_BUILD_LOG"'}.items(): + path = commands / name + path.write_text('#!/bin/bash\n' + text + '\n') + path.chmod(0o755) + archive = root / 'release.tar.gz' + with tarfile.open(archive, 'w:gz') as handle: + handle.add(release, arcname='radar_system') + log = root / 'build.log' + completed = subprocess.run( + ['bash', str(PROJECT / 'deploy/update.sh'), archive.name], cwd=root, + env={**os.environ, 'RADAR_DIR': str(target), 'PATH': f'{commands}:{os.environ["PATH"]}', + 'RADAR_TEST_BUILD_LOG': str(log)}, capture_output=True, text=True, timeout=10, + ) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + backup = root / 'radar_system1' + self.assertEqual((target / '.venv/state').read_text(), 'updated environment') + self.assertEqual((backup / '.venv/state').read_text(), 'old environment') + self.assertEqual((target / 'loose_recording.bin').read_bytes(), b'original data') + self.assertEqual((target / 'python_app/data/calibration/set.npz').read_bytes(), b'calibration') + self.assertEqual((target / 'run_config.json').read_text(), str(target)) + self.assertEqual((target / 'run_config_examples/device.json').read_text(), 'device settings') + self.assertTrue((target / 'run_config_examples/device.json.new').exists()) + self.assertEqual((target / 'python_app/runtime/gui_session_state.json').read_text(), 'session') + self.assertFalse((target / 'python_app/runtime/logs').exists()) + self.assertIn('kamil_adc_collector', log.read_text().splitlines()) diff --git a/python_app/tests/test_file_io.py b/python_app/tests/test_file_io.py new file mode 100644 index 0000000..7ab2d05 --- /dev/null +++ b/python_app/tests/test_file_io.py @@ -0,0 +1,44 @@ +"""Settings commits retain valid files on failure and concurrent writes.""" + +from concurrent.futures import ThreadPoolExecutor +import json +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from python_app.file_io import atomic_write_json +from python_app.orchestration.gui_session_state import GuiSessionStateStore + + +class AtomicJsonTest(unittest.TestCase): + def setUp(self) -> None: + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.path = Path(directory.name) / "settings.json" + atomic_write_json(self.path, {"old": True}) + + def test_failed_sync_or_commit_preserves_previous_file(self) -> None: + for operation in ('fsync', 'replace'): + with self.subTest(operation=operation), \ + mock.patch(f'python_app.file_io.os.{operation}', side_effect=OSError('disk failure')), \ + self.assertRaises(OSError): + atomic_write_json(self.path, {'new': True}, durable=True) + self.assertEqual(json.loads(self.path.read_text()), {'old': True}) + self.assertEqual(list(self.path.parent.glob('*.tmp')), []) + + def test_nonfinite_json_cannot_overwrite_settings(self) -> None: + with self.assertRaises(ValueError): + atomic_write_json(self.path, {'bad': float('nan')}) + self.assertEqual(json.loads(self.path.read_text()), {'old': True}) + + def test_concurrent_writers_publish_complete_json_and_leave_no_temp_files(self) -> None: + payloads = [{'index': i, 'values': [i] * 1000} for i in range(32)] + with ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(lambda payload: atomic_write_json(self.path, payload), payloads)) + self.assertIn(json.loads(self.path.read_text()), payloads) + self.assertEqual(list(self.path.parent.glob('*.tmp')), []) + + def test_invalid_utf8_session_cache_uses_defaults(self) -> None: + self.path.write_bytes(b'\xff') + self.assertEqual(GuiSessionStateStore(self.path).load().last_profile_path, '') diff --git a/python_app/tests/test_generator_sweep.py b/python_app/tests/test_generator_sweep.py new file mode 100644 index 0000000..3b88ee1 --- /dev/null +++ b/python_app/tests/test_generator_sweep.py @@ -0,0 +1,45 @@ +"""Standalone generator configuration and independent resource cleanup.""" + +from dataclasses import replace +import unittest +from unittest import mock + +from python_app.generator_sweep.config import GeneratorSweepConfig, resolve_generator_sweep_config +from python_app.generator_sweep.runner import GeneratorSweepRunner + + +class GeneratorSweepTest(unittest.TestCase): + def test_frequency_grid_includes_stop_with_and_without_an_exact_step(self) -> None: + for step, expected in ((2, (10, 12, 14, 15)), (5, (10, 15))): + config = GeneratorSweepConfig(start_hz=10, stop_hz=15, step_hz=step) + self.assertEqual(resolve_generator_sweep_config(config).frequencies_hz, expected) + config = GeneratorSweepConfig(start_hz=10, stop_hz=15, points=3) + self.assertEqual(resolve_generator_sweep_config(config).frequencies_hz, (10, 12, 15)) + + def test_malformed_types_nonfinite_values_and_excessive_grids_fail_before_open(self) -> None: + valid = GeneratorSweepConfig(points=3) + for changes in ({'points': 3.5}, {'port': True}, {'sweep_start_pin': 5.5}, + {'power_dbm': float('inf')}, {'hold_time_ms': float('nan')}, + {'points': 1_000_001}, {'points': 0, 'step_hz': 1, 'stop_hz': 10 ** 10}): + with self.subTest(changes=changes), self.assertRaises(ValueError): + resolve_generator_sweep_config(replace(valid, **changes)) + + def test_each_cleanup_failure_still_releases_the_other_resources(self) -> None: + for failure in ('pwm_stop', 'device_idle', 'device_close', 'marker_reset', 'marker_close'): + with self.subTest(failure=failure): + runner = GeneratorSweepRunner(GeneratorSweepConfig(points=3)) + runner._pwm = mock.Mock() + runner._markers = mock.Mock() + device = runner._device = mock.Mock(is_connected=True) + actions = { + 'pwm_stop': runner._pwm.disable, 'device_idle': device.generator.set_idle, + 'device_close': device.disconnect, 'marker_reset': runner._markers.reset_to_idle, + 'marker_close': runner._markers.close, + } + actions[failure].side_effect = OSError('cleanup failure') + with self.assertLogs('python_app.generator_sweep.runner', level='WARNING'): + runner._close() + for action in actions.values(): + action.assert_called_once() + runner._pwm.close.assert_called_once() + self.assertIsNone(runner._device) diff --git a/python_app/tests/test_gui_action_lifecycle.py b/python_app/tests/test_gui_action_lifecycle.py new file mode 100644 index 0000000..687ca75 --- /dev/null +++ b/python_app/tests/test_gui_action_lifecycle.py @@ -0,0 +1,297 @@ +"""Regression coverage for saves, full history drains and interruptible GUI actions.""" + +from __future__ import annotations + +from collections import deque +import itertools +import os +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest import mock + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") +from PyQt6.QtWidgets import QApplication # noqa: E402 +from PyQt6.QtCore import QTimer # noqa: E402 + +from python_app.gui.app_window import AppWindow # noqa: E402 +from python_app.gui.controllers.app_window_config.profile_io_mixin import AppWindowConfigProfileIOMixin # noqa: E402 +from python_app.gui.controllers.app_window_pipeline_mixin import AppWindowPipelineMixin # noqa: E402 +from python_app.gui.controllers.app_window_preprocess_mixin import AppWindowPreprocessMixin # noqa: E402 +from python_app.gui.controllers.app_window_snapshot_mixin import AppWindowSnapshotMixin # noqa: E402 +from python_app.models.dataset_model import ResultCollection, SweepCollection # noqa: E402 +from python_app.models.gui_profile_model import GuiProfileModel # noqa: E402 +from python_app.models.run_config_model import RunConfigModel # noqa: E402 +from python_app.storage.npz_store import NpzStore # noqa: E402 +from python_app.workflows.kamil_adc_neutral_preprocess import build_neutral_s21_sets # noqa: E402 + + +def setUpModule() -> None: + global _app + _app = QApplication.instance() or QApplication([]) + + +def _reader(method: str, collections: list): + pending = deque(collections) + return SimpleNamespace(**{method: lambda: pending.popleft() if pending else None, "close": mock.Mock()}) + + +class _Runtime(AppWindowPipelineMixin, AppWindowSnapshotMixin, AppWindowPreprocessMixin): + def __init__(self) -> None: + self._raw_reader = self._pre_reader = self._result_reader = None + self._raw_history = deque(maxlen=2) + self._pre_history = deque(maxlen=2) + self._result_history = deque(maxlen=2) + self._max_pop_per_poll = self._max_pop_per_snapshot_drain = 1 + self._single_capture_active = True + self._single_capture_start_ns = 10 + self._single_capture_seen_raw = False + self._single_capture_target_collection_id = None + self._capture_session = None + self._preprocess_capture_busy = self._closing = False + self._resume_pipeline_after_capture = True + self._stop_request_seq = 0 + self._pipeline_metrics = mock.Mock() + self._record_collection = mock.Mock() + self._supervisor = mock.Mock() + self._supervisor.is_running.return_value = False + self._supervisor.is_processor_running.return_value = True + self._status_label = mock.Mock() + for name in ("_log", "_log_debug", "_log_exception", "_show_error", "_show_exception", + "_update_history_indicator", "_finalize_recording_on_stop", "_web_update_snapshot", + "_build_config", "_update_capture_dialog_state", "close"): + setattr(self, name, mock.Mock()) + self._pump_events_during_drain = mock.Mock() + + +class NeutralSetCreationTest(unittest.TestCase): + def test_failed_second_set_preserves_gui_selection_and_previous_data(self) -> None: + w = _Runtime() + config = RunConfigModel.from_dict({"radar": { + "model": "librevna", "driver_mode": "mock", "sweep": {"points": 7}, + }}) + w._build_config.return_value = config + w._ensure_preprocess_dialog = mock.Mock(return_value=SimpleNamespace(set_name=lambda: "new_neutral")) + w._radar_key = mock.Mock(return_value="radar") + w._selected_preprocess_sets = {"s21_calibration": "old_cal", "s21_reference": "old_ref"} + w._selected_preprocess_radar_key = "previous_radar" + w._processor_run_signature = "processor_signature" + w._history_run_signature = "history_signature" + w._refresh_sets = mock.Mock() + w._reset_runtime_history = mock.Mock() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + w._store = NpzStore(root) + calibration, reference = build_neutral_s21_sets(config) + w._store.save_set("s21_calibration", "radar", "old_cal", calibration) + w._store.save_set("s21_reference", "radar", "old_ref", reference) + before = {path: path.read_bytes() for path in root.rglob("*") if path.is_file()} + save = w._store.save_set + + def save_set(kind, *args): + if kind == "s21_reference": + raise OSError("reference write failed") + return save(kind, *args) + + with mock.patch.object(w._store, "save_set", side_effect=save_set): + w._create_neutral_sets() + self.assertEqual({path: path.read_bytes() for path in root.rglob("*") if path.is_file()}, before) + self.assertEqual(w._selected_preprocess_sets, {"s21_calibration": "old_cal", "s21_reference": "old_ref"}) + self.assertEqual(w._selected_preprocess_radar_key, "previous_radar") + self.assertEqual((w._processor_run_signature, w._history_run_signature), ("processor_signature", "history_signature")) + w._show_exception.assert_called_once() + w._refresh_sets.assert_not_called() + w._reset_runtime_history.assert_not_called() + + +class SnapshotDrainTest(unittest.TestCase): + def test_save_drain_preserves_recording_and_single_capture_tracking(self) -> None: + w = _Runtime() + raw = SweepCollection(collection_id=5, monotonic_ns=20, traces=[], capture_start_ns=1, capture_end_ns=8) + preprocessed = SweepCollection(collection_id=5, monotonic_ns=20, traces=[]) + result = ResultCollection(collection_id=5, monotonic_ns=20, blocks=[]) + w._raw_reader = _reader("pop_raw_collection", [raw]) + w._pre_reader = _reader("pop_preprocessed_collection", [preprocessed]) + w._result_reader = _reader("pop_result_collection", [result, result]) + + w._drain_runtime_rings_for_snapshot() + + self.assertEqual(w._record_collection.call_args_list, [ + mock.call("raw", raw), mock.call("preprocessed", preprocessed), mock.call("results", result), + ]) + self.assertTrue(w._single_capture_seen_raw) + self.assertEqual(w._single_capture_target_collection_id, 5) + w._pipeline_metrics.record.assert_any_call("acquisition", 7) + self.assertEqual(list(w._result_history), [result]) + w._log_exception.assert_not_called() + + def test_continuous_producer_cannot_prevent_snapshot_timeout(self) -> None: + w = _Runtime() + collection = SweepCollection(collection_id=1, monotonic_ns=20, traces=[]) + pop = mock.Mock(return_value=collection) + w._raw_reader = SimpleNamespace(pop_raw_collection=pop) + ticks = itertools.count(0.0, 0.1) + with mock.patch("python_app.gui.controllers.app_window_snapshot_mixin.time.monotonic", side_effect=lambda: next(ticks)): + w._drain_runtime_rings_for_snapshot() + self.assertGreater(pop.call_count, 1) + self.assertLess(pop.call_count, 15) + w._log_exception.assert_not_called() + + def test_stop_drain_reads_the_whole_tail_when_history_is_full(self) -> None: + w = _Runtime() + w._raw_history = deque([SweepCollection(collection_id=0, monotonic_ns=0, traces=[])], maxlen=1) + tail = [SweepCollection(collection_id=i, monotonic_ns=i, traces=[]) for i in range(1, 11)] + w._raw_reader = _reader("pop_raw_collection", tail) + clock = [0.0] + w._pump_events_during_drain.side_effect = lambda pause: clock.__setitem__(0, clock[0] + pause) + with mock.patch("python_app.gui.controllers.app_window_pipeline_mixin.time.monotonic", side_effect=lambda: clock[0]): + w._drain_rings_until_quiet(timeout_s=1.0, poll_s=0.01) + self.assertEqual([call.args[1].collection_id for call in w._record_collection.call_args_list], list(range(1, 11))) + self.assertEqual(w._raw_history[-1].collection_id, 10) + + +class ShutdownReentryTest(unittest.TestCase): + def test_reader_failure_on_stop_is_reported_and_cancels_automatic_resume(self) -> None: + w = _Runtime() + w._supervisor.is_running.return_value = True + w._drain_rings_until_quiet = mock.Mock(side_effect=ValueError("corrupt ring payload")) + raw = w._raw_reader = SimpleNamespace(close=mock.Mock()) + preprocessed = w._pre_reader = SimpleNamespace(close=mock.Mock()) + result = w._result_reader = SimpleNamespace(close=mock.Mock(), _ring_name="/results") + expected_stop_seq = w._stop_request_seq + 1 + with mock.patch("python_app.gui.controllers.app_window_pipeline_mixin.ShmRingReader") as reopen: + w._stop_run() + w._supervisor.stop.assert_called_once() + w._supervisor.stop_all.assert_not_called() + w._finalize_recording_on_stop.assert_called_once() + for reader in (raw, preprocessed, result): + reader.close.assert_called_once() + self.assertIsNone(w._raw_reader) + self.assertIsNone(w._pre_reader) + self.assertIs(w._result_reader, reopen.return_value) + reopen.assert_called_once_with("/results") + w._show_exception.assert_called_once() + self.assertIn("corrupt ring", str(w._show_exception.call_args.args[1])) + self.assertFalse(w._stopping_run) + self.assertFalse(w._single_capture_active) + self.assertEqual(w._status_label.setText.call_args.args[0], "Status: error") + self.assertFalse(w._pipeline_resume_allowed(expected_stop_seq)) + + def test_unsupported_kamil_adc_mock_fails_before_starting_any_process(self) -> None: + w = _Runtime() + config = RunConfigModel() + config.radar.model = "kamil_adc" + config.apply_device_model_constraints() + w._build_config.return_value = config + w._stop_all_processes = mock.Mock() + w._start_run(single_capture=True) + w._supervisor.start.assert_not_called() + w._show_exception.assert_called_once() + self.assertIn("native", str(w._show_exception.call_args.args[1])) + + def test_stop_blocks_nested_start_poll_and_duplicate_stop(self) -> None: + w = _Runtime() + w._supervisor.is_running.return_value = True + + def reenter(**_kwargs): + w._start_run() + w._poll_rings() + w._stop_run() + + w._drain_rings_until_quiet = mock.Mock(side_effect=reenter) + w._stop_run() + w._supervisor.stop_orchestrator.assert_called_once() + w._supervisor.stop_preprocessor.assert_called_once() + w._finalize_recording_on_stop.assert_called_once() + w._build_config.assert_not_called() + w._web_update_snapshot.assert_not_called() + self.assertFalse(w._stopping_run) + self.assertFalse(w._resume_pipeline_after_capture) + + def test_close_during_stop_waits_until_tail_is_finalized(self) -> None: + w = _Runtime() + w._supervisor.is_running.return_value = True + event = mock.Mock() + ordering = [] + + def close_during_drain(**_kwargs): + AppWindow.closeEvent(w, event) + + w._drain_rings_until_quiet = mock.Mock(side_effect=close_during_drain) + w._finalize_recording_on_stop.side_effect = lambda: ordering.append("finalize") + w.close.side_effect = lambda: ordering.append("close") + w._stop_run() + event.ignore.assert_called() + w._supervisor.stop_all.assert_called_once() + self.assertEqual(ordering, ["finalize", "close"]) + self.assertFalse(w._close_after_stop) + + def test_explicit_stop_during_apply_pause_cancels_apply_and_restart(self) -> None: + w = _Runtime() + w._supervisor.is_running.return_value = True + w._start_run = mock.Mock() + w._drain_rings_until_quiet = mock.Mock(side_effect=lambda **_kwargs: w._stop_run()) + w._apply_radar_settings() + w._build_config.assert_not_called() + w._start_run.assert_not_called() + + +class CaptureBatchControlTest(unittest.TestCase): + def test_stop_between_combos_keeps_captures_and_allows_later_resume(self) -> None: + w = _Runtime() + captured = [] + session = SimpleNamespace( + kind="s21_reference", is_complete=lambda: len(captured) == 3, + state=lambda: SimpleNamespace(total_count=3, captured_count=len(captured)), + capture_current_combo=lambda: captured.append(len(captured)), + ) + w._capture_session = session + dialog = mock.Mock() + w._ensure_preprocess_dialog = lambda: dialog + w._record_preprocess_capture = mock.Mock() + w._pump_events_during_drain = lambda pause: AppWindowPipelineMixin._pump_events_during_drain(w, pause) + QTimer.singleShot(0, w._stop_run) + with mock.patch("python_app.gui.controllers.app_window_preprocess_mixin.QTimer.singleShot", side_effect=lambda _delay, fn: fn()): + w._capture_all_remaining() + self.assertEqual(captured, [0]) + self.assertIs(w._capture_session, session) + self.assertFalse(w._preprocess_capture_busy) + self.assertIn("paused", dialog.set_status.call_args.args[0]) + w._capture_all_remaining() + self.assertEqual(captured, [0, 1, 2]) + self.assertEqual(w._record_preprocess_capture.call_count, 3) + + +class GuiProfileWriteTest(unittest.TestCase): + def test_failed_profile_replace_preserves_existing_profile_and_cleans_temporary(self) -> None: + writer = AppWindowConfigProfileIOMixin() + writer._build_gui_profile = lambda: GuiProfileModel() + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "profile.json" + previous = b'{"previous": "profile"}' + path.write_bytes(previous) + with mock.patch("python_app.file_io.os.replace", side_effect=OSError("disk failure")): + with self.assertRaisesRegex(OSError, "disk failure"): + writer._write_gui_profile_to_path(path) + self.assertEqual(path.read_bytes(), previous) + self.assertEqual(list(path.parent.iterdir()), [path]) + + +class HistoryCommandRestoreTest(unittest.TestCase): + def test_malformed_or_exhausted_counter_cannot_prevent_gui_start(self) -> None: + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "live.json" + for content in ('[]', 'null', 'invalid', '{"history_command_seq": NaN}', + '{"history_command_seq": Infinity}', '{"history_command_seq": true}', + '{"history_command_seq": 1.5}', '{"history_command_seq": -1}', + '{"history_command_seq": 18446744073709551615}'): + path.write_text(content) + with self.subTest(content=content): + self.assertEqual(AppWindow._load_history_command_seq(path), 0) + path.write_text('{"history_command_seq": 9876543210}') + self.assertEqual(AppWindow._load_history_command_seq(path), 9876543210) + + +if __name__ == "__main__": + unittest.main() diff --git a/python_app/tests/test_gui_profile_codec.py b/python_app/tests/test_gui_profile_codec.py index cba565c..b6a84ad 100644 --- a/python_app/tests/test_gui_profile_codec.py +++ b/python_app/tests/test_gui_profile_codec.py @@ -54,6 +54,22 @@ class GuiProfileCodecTest(unittest.TestCase): twice = GuiProfileModel.from_dict(once).to_dict() self.assertEqual(once["gui"], twice["gui"]) + def test_legacy_save_count_is_ignored_without_losing_save_destination(self) -> None: + decoded = GuiProfileModel.from_dict({ + "gui": {"data_actions": { + "save_count": 7, + "save_path": "/tmp/measurements", + "save_name": "scene", + "record_count": 250, + }}, + }) + + encoded = decoded.to_dict()["gui"]["data_actions"] + self.assertNotIn("save_count", encoded) + self.assertEqual(encoded, { + "save_path": "/tmp/measurements", "save_name": "scene", "record_count": 250, + }) + def test_sar3d_section_round_trips(self) -> None: sar3d = GuiSar3dStateModel( input_positions="0,2", diff --git a/python_app/tests/test_hardware_lifecycle.py b/python_app/tests/test_hardware_lifecycle.py new file mode 100644 index 0000000..8d538e2 --- /dev/null +++ b/python_app/tests/test_hardware_lifecycle.py @@ -0,0 +1,324 @@ +"""Failure-path coverage without physical instruments or protocol changes.""" + +from __future__ import annotations + +import json +import os +import pty +from pathlib import Path +import tempfile +import threading +import tty +import unittest +from dataclasses import dataclass +from unittest.mock import Mock, patch + +import numpy as np + +from python_app.hardware_full.kamil_adc.tty_reader import KamilAdcTtyReader +from python_app.hardware_full.laser_control.protocol import Protocol +from python_app.hardware_full.librevna_backends import NativeLibreVnaBackend +from python_app.hardware_full.librevna_driver.enums import PacketType +from python_app.hardware_full.librevna_driver.exceptions import DeviceDisconnectedError +from python_app.hardware_full.librevna_driver.models import Packet +from python_app.hardware_full.librevna_driver.session import LibreVNASession +from python_app.hardware_full.librevna_driver.transport.usb import USBTransport +from python_app.hardware_full.switch_drivers.h7992_driver import H7992Driver +from python_app.hardware_full.switch_drivers.hmc349a_driver import HMC349ADriver +from python_app.hardware_full.switched_matrix_radar_service import SwitchedMatrixRadarService +from python_app.models.run_config_model import RunConfigModel +from python_app.models.dataset_model import ComboKey, TraceData +from python_app.scripts import kamil_adc_raw_producer, matrix_raw_producer, replay_raw_producer +from python_app.workflows.radar_config_variants import scan_radar_config_variants + + +@dataclass +class _SweepPoint: + point_number: int + cycle: str + + +class HardwareCleanupTest(unittest.TestCase): + def test_partial_matrix_open_closes_every_component(self) -> None: + inner, output, input_ = Mock(), Mock(), Mock() + failure = OSError("input open failed") + input_.open.side_effect = failure + output.close.side_effect = OSError("output close failed") + service = SwitchedMatrixRadarService(inner, output, input_, 2, 4) + with self.assertRaises(OSError) as raised: + service.open() + self.assertIs(raised.exception, failure) + for resource in (inner, output, input_): + resource.close.assert_called_once() + + def test_native_open_failure_allows_fresh_retry(self) -> None: + backend = NativeLibreVnaBackend(None, 14) + failed, working = Mock(), Mock() + failed.connect.side_effect = OSError("USB unavailable") + failed.disconnect.side_effect = OSError("disconnect failed") + backend._libre_vna_device = Mock(side_effect=[failed, working]) + with self.assertRaisesRegex(OSError, "USB unavailable"): + backend.open() + self.assertFalse(backend.is_open) + backend.open() + self.assertIs(backend._device, working) + working.disconnect.side_effect = OSError("device removed") + with self.assertRaises(OSError): + backend.close() + self.assertFalse(backend.is_open) + + def test_gpio_initialization_failure_releases_lines(self) -> None: + for driver_type, module in ((H7992Driver, "h7992_driver"), (HMC349ADriver, "hmc349a_driver")): + with self.subTest(driver=driver_type.__name__): + lines = Mock(set_values=Mock(side_effect=OSError("GPIO write failed"))) + driver = driver_type("test") + with patch(f"python_app.hardware_full.switch_drivers.{module}.GpioOutputLines", return_value=lines): + with self.assertRaisesRegex(OSError, "GPIO write failed"): + driver.open() + lines.close.assert_called_once() + self.assertFalse(driver._is_open) + self.assertIsNone(driver._lines) + + def test_serial_connect_is_idempotent_and_failed_close_clears_state(self) -> None: + connection = Mock(is_open=True) + protocol = Protocol("/dev/test") + with patch("serial.Serial", return_value=connection) as factory: + protocol.connect() + protocol.connect() + factory.assert_called_once() + connection.close.side_effect = OSError("serial removed") + with self.assertRaises(OSError): + protocol.disconnect() + self.assertFalse(protocol.is_connected) + + +class SessionLifecycleTest(unittest.TestCase): + def test_acquisition_discards_tail_before_first_point_zero(self) -> None: + self._assert_complete_cycle([(1, "old"), (2, "old"), (0, "new"), (1, "new"), (2, "new")]) + + def test_acquisition_discards_incomplete_cycle_on_next_point_zero(self) -> None: + self._assert_complete_cycle([(0, "old"), (1, "old"), (0, "new"), (1, "new"), (2, "new")]) + + def _assert_complete_cycle(self, sequence) -> None: + session = LibreVNASession() + packets = [Packet(PacketType.VNA_DATAPOINT, _SweepPoint(point_number=index, cycle=cycle)) for index, cycle in sequence] + with patch.object(session, "wait_for_packet", side_effect=packets): + points = session.collect_indexed_payloads( + packet_type=PacketType.VNA_DATAPOINT, expected_points=3, timeout_s=1, + payload_type=_SweepPoint, payload_error="Unexpected datapoint", + ) + self.assertEqual([p.point_number for p in points], [0, 1, 2]) + self.assertEqual([p.cycle for p in points], ["new"] * 3) + + def test_disconnect_wakes_packet_and_ack_waiters(self) -> None: + session = LibreVNASession() + write_started = threading.Event() + session._transport = Mock(write=Mock(side_effect=lambda *_, **__: write_started.set())) + errors = [] + + def waiting(action): + try: + action() + except Exception as exc: + errors.append(exc) + + workers = [ + threading.Thread(target=waiting, args=(lambda: session.wait_for_packet(PacketType.DEVICE_INFO, timeout_s=20),)), + threading.Thread(target=waiting, args=(lambda: session.send(Packet(PacketType.REQUEST_DEVICE_INFO), timeout_s=20),)), + ] + for worker in workers: + worker.start() + self.assertTrue(write_started.wait(1)) + session.disconnect() + for worker in workers: + worker.join(timeout=1) + self.assertFalse(worker.is_alive()) + self.assertEqual(len(errors), 2) + self.assertTrue(all(isinstance(exc, DeviceDisconnectedError) for exc in errors)) + self.assertFalse(session._awaiting_ack) + + def test_write_failure_resets_ack_state(self) -> None: + session = LibreVNASession() + session._transport = Mock(write=Mock(side_effect=OSError("write failed"))) + with self.assertRaises(OSError): + session.send(Packet(PacketType.REQUEST_DEVICE_INFO)) + self.assertFalse(session._awaiting_ack) + self.assertIsNone(session._ack_result) + + def test_stream_subscription_has_bounded_unused_queue(self) -> None: + with patch("python_app.hardware_full.librevna_driver.session._PACKET_QUEUE_MAX", 4): + session = LibreVNASession() + callback = Mock() + session.subscribe(PacketType.VNA_DATAPOINT, callback) + for index in range(20): + session._dispatch_packet(Packet(PacketType.VNA_DATAPOINT, index)) + self.assertEqual(callback.call_count, 20) + self.assertEqual([p.payload for p in session._incoming[PacketType.VNA_DATAPOINT]], [16, 17, 18, 19]) + self.assertIsNone(session._fatal_error) + + def test_idle_free_running_sweep_does_not_poison_next_acquisition(self) -> None: + with patch("python_app.hardware_full.librevna_driver.session._PACKET_QUEUE_MAX", 2): + session = LibreVNASession() + for index in range(10): + session._dispatch_packet(Packet(PacketType.VNA_DATAPOINT, index)) + self.assertEqual(len(session._incoming[PacketType.VNA_DATAPOINT]), 2) + self.assertIsNone(session._fatal_error) + session.clear_queue(PacketType.VNA_DATAPOINT) + session._dispatch_packet(Packet(PacketType.VNA_DATAPOINT, 0)) + self.assertEqual(session.wait_for_packet(PacketType.VNA_DATAPOINT).payload, 0) + + def test_synchronous_queue_overflow_is_reported(self) -> None: + with patch("python_app.hardware_full.librevna_driver.session._PACKET_QUEUE_MAX", 1): + session = LibreVNASession() + session._dispatch_packet(Packet(PacketType.DEVICE_INFO)) + session._dispatch_packet(Packet(PacketType.DEVICE_INFO)) + with self.assertRaisesRegex(DeviceDisconnectedError, "queue overflow"): + session.wait_for_packet(PacketType.DEVICE_INFO) + + +class UsbLifecycleTest(unittest.TestCase): + def test_failed_enumeration_releases_context(self) -> None: + transport = USBTransport(on_data=Mock()) + context = Mock() + with patch("usb1.USBContext", return_value=context), patch.object(transport, "_iter_matching_devices", side_effect=OSError("enumeration failed")): + with self.assertRaisesRegex(OSError, "enumeration failed"): + transport.connect() + context.close.assert_called_once() + self.assertIsNone(transport._ctx) + self.assertFalse(transport.is_connected) + + def test_failed_candidate_identity_read_closes_handle_and_context(self) -> None: + transport = USBTransport(on_data=Mock()) + context, handle = Mock(), Mock(getSerialNumber=Mock(side_effect=OSError("identity failed"))) + device = Mock(open=Mock(return_value=handle)) + with patch("usb1.USBContext", return_value=context), patch.object(transport, "_iter_matching_devices", return_value=[(device, 0, 0)]): + with self.assertRaisesRegex(OSError, "identity failed"): + transport.connect() + handle.close.assert_called_once() + context.close.assert_called_once() + self.assertFalse(transport.is_connected) + + def test_failed_thread_start_releases_claimed_handle_and_context(self) -> None: + transport = USBTransport(on_data=Mock()) + context, handle = Mock(), Mock(kernelDriverActive=Mock(return_value=False)) + device = Mock(open=Mock(return_value=handle)) + with patch("usb1.USBContext", return_value=context), patch.object(transport, "_iter_matching_devices", return_value=[(device, 0, 0)]), patch("threading.Thread.start", side_effect=RuntimeError("thread failed")): + with self.assertRaisesRegex(RuntimeError, "thread failed"): + transport.connect() + handle.releaseInterface.assert_called_once_with(transport.INTERFACE) + handle.close.assert_called_once() + context.close.assert_called_once() + self.assertIsNone(transport._rx_thread) + self.assertIsNone(transport._ctx) + self.assertFalse(transport.is_connected) + + +class TtyLifecycleTest(unittest.TestCase): + def test_close_interrupts_both_waiting_read_methods(self) -> None: + for combination in (None, (0, 0)): + with self.subTest(combination=combination): + master, slave = pty.openpty() + tty.setraw(slave) + reader = KamilAdcTtyReader(os.ttyname(slave)) + errors = [] + started = threading.Event() + reader.open() + + def wait(): + started.set() + try: + if combination is None: + reader.read_sweep(timeout_s=20) + else: + reader.read_sweep_for(combination, timeout_s=20) + except Exception as exc: + errors.append(exc) + + worker = threading.Thread(target=wait) + try: + worker.start() + self.assertTrue(started.wait(1)) + reader.close() + worker.join(timeout=1) + self.assertFalse(worker.is_alive()) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], RuntimeError) + finally: + reader.close() + os.close(master) + os.close(slave) + + def test_failed_reader_thread_start_closes_descriptor(self) -> None: + read_fd, write_fd = os.pipe() + reader = KamilAdcTtyReader("/dev/test") + try: + with patch("os.open", return_value=read_fd), patch("threading.Thread.start", side_effect=RuntimeError("thread failed")): + with self.assertRaises(RuntimeError): + reader.open() + self.assertIsNone(reader._fd) + self.assertIsNone(reader._thread) + with self.assertRaises(OSError): + os.fstat(read_fd) + finally: + os.close(write_fd) + + +class ProducerCleanupTest(unittest.TestCase): + def test_replay_rejects_small_slots_before_publishing_any_data(self) -> None: + module = replay_raw_producer + config = RunConfigModel() + config.radar.replay.dataset_dir = "/test/data" + config.rings.raw.slot_size_bytes = 1 + trace = TraceData( + ComboKey(0, 0), np.array([1, 2], dtype=np.float32), + np.array([0, 0], dtype=np.complex64), np.array([1, 2], dtype=np.complex64), + ) + dataset = module.ReplayDataset([[trace]], [20]) + raw, tap = Mock(), Mock() + with patch.object(module.RunConfigModel, "load_from_path", return_value=config), patch.object(module.signal, "signal"), patch.object(module, "_load_dataset", return_value=dataset), patch.object(module, "ShmRingWriter", side_effect=[raw, tap]), patch("sys.argv", [module.__name__, "--config", "/test/config.json"]): + with self.assertRaisesRegex(ValueError, "Replay requires .* bytes per slot"): + module.main() + raw.push.assert_not_called() + tap.push.assert_not_called() + raw.close.assert_called_once() + tap.close.assert_called_once() + + def test_partial_ring_initialization_closes_first_writer(self) -> None: + for module, model in ((matrix_raw_producer, "librevna_multi"), (kamil_adc_raw_producer, "kamil_adc"), (replay_raw_producer, "librevna")): + with self.subTest(producer=module.__name__): + config = RunConfigModel() + config.radar.model = model + config.radar.replay.dataset_dir = "/test/data" + writer = Mock() + with patch.object(module.RunConfigModel, "load_from_path", return_value=config), patch.object(module.signal, "signal"), patch.object(module, "ShmRingWriter", side_effect=[writer, OSError("second ring failed")]), patch("sys.argv", [module.__name__, "--config", "/test/config.json"]): + if module is replay_raw_producer: + with patch.object(module, "_load_dataset", return_value=Mock(median_gap_ns=1, __len__=Mock(return_value=1))): + with self.assertRaisesRegex(OSError, "second ring failed"): + module.main() + else: + with self.assertRaisesRegex(OSError, "second ring failed"): + module.main() + writer.close.assert_called_once() + + +class RadarVariantValidationTest(unittest.TestCase): + def test_scan_rejects_malformed_sweeps_and_keeps_valid_overlay(self) -> None: + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) + for index, sweep in enumerate(( + {"points": 1.5}, {"points": True}, {"if_bandwidth_hz": -1}, + {"start_hz": 10, "stop_hz": 5}, {"start_hz": float("nan")}, + {"points": 2, "stimulus_power_dbm": -20}, + )): + (source / f"variant{index}.json").write_text(json.dumps({"radar": {"sweep": sweep}})) + base = RunConfigModel() + variants, summary = scan_radar_config_variants(directory, base_config=base) + self.assertEqual(summary.valid_variant_count, 1) + self.assertEqual(summary.skipped_file_count, 5) + self.assertEqual(variants[0].config.radar.sweep.points, 2) + self.assertEqual(variants[0].config.radar.sweep.power_dbm, -20) + self.assertEqual(base.radar.sweep.points, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/python_app/tests/test_kamil_adc_neutral_preprocess.py b/python_app/tests/test_kamil_adc_neutral_preprocess.py index e0f9dbd..8bc42b8 100644 --- a/python_app/tests/test_kamil_adc_neutral_preprocess.py +++ b/python_app/tests/test_kamil_adc_neutral_preprocess.py @@ -1,4 +1,4 @@ -"""Tests for Kamil ADC neutral preprocessing-set generation.""" +"""Neutral preprocessing grids must preserve the actual radar data unchanged.""" from __future__ import annotations @@ -6,8 +6,9 @@ import unittest import numpy as np +from python_app.hardware_full.point_target_mock_service import PointTargetMockService from python_app.models.run_config_model import RunConfigModel -from python_app.workflows.kamil_adc_neutral_preprocess import build_kamil_adc_neutral_s21_sets +from python_app.workflows.kamil_adc_neutral_preprocess import build_neutral_s21_sets, supports_neutral_preprocess_sets def _kamil_config(*, points: int = 5) -> RunConfigModel: @@ -27,7 +28,7 @@ def _kamil_config(*, points: int = 5) -> RunConfigModel: class KamilAdcNeutralPreprocessTest(unittest.TestCase): def test_builds_passthrough_s21_sets_on_band_grid(self) -> None: - calibration, reference = build_kamil_adc_neutral_s21_sets(_kamil_config(points=5)) + calibration, reference = build_neutral_s21_sets(_kamil_config(points=5)) expected_frequency = np.linspace(2_100_000_000.0, 5_500_000_000.0, 5).astype(np.float32) self.assertEqual(len(calibration.traces), 2) @@ -53,20 +54,15 @@ class KamilAdcNeutralPreprocessTest(unittest.TestCase): config = _kamil_config(points=17) processor = KamilAdcSweepProcessor(KamilAdcProcessingParams.from_kamil_model(config.radar.kamil_adc)) - calibration, _reference = build_kamil_adc_neutral_s21_sets(config) + calibration, _reference = build_neutral_s21_sets(config) np.testing.assert_array_equal(calibration.traces[0].frequency_hz, processor.grid_hz) - def test_rejects_non_kamil_config(self) -> None: - config = RunConfigModel.from_dict({"radar": {"model": "librevna"}}) - with self.assertRaisesRegex(ValueError, "kamil_adc"): - build_kamil_adc_neutral_s21_sets(config) - def test_rejects_degenerate_band(self) -> None: with self.assertRaisesRegex(ValueError, "points must be >= 2"): - build_kamil_adc_neutral_s21_sets(_kamil_config(points=1)) + build_neutral_s21_sets(_kamil_config(points=1)) def test_dtypes_are_float32_and_complex64(self) -> None: - calibration, _reference = build_kamil_adc_neutral_s21_sets(_kamil_config()) + calibration, _reference = build_neutral_s21_sets(_kamil_config()) trace = calibration.traces[0] self.assertEqual(trace.frequency_hz.dtype, np.float32) self.assertEqual(trace.s21.dtype, np.complex64) @@ -75,10 +71,29 @@ class KamilAdcNeutralPreprocessTest(unittest.TestCase): def test_calibration_s21_is_a_nonzero_divisor(self) -> None: # The C++ through-calibrator divides measured/calibration, so calibration S21 # must never be zero — that is the whole point of the '1+0j neutral' contract. - calibration, _reference = build_kamil_adc_neutral_s21_sets(_kamil_config()) + calibration, _reference = build_neutral_s21_sets(_kamil_config()) for trace in calibration.traces: self.assertTrue(bool(np.all(trace.s21 != 0))) + def test_single_mock_grid_is_exact_and_neutral_s21_is_passthrough(self) -> None: + config = RunConfigModel.from_dict({"radar": { + "model": "librevna", "driver_mode": "mock", + "sweep": {"start_hz": 1e6, "stop_hz": 6e9, "points": 201}, + }}) + self.assertTrue(supports_neutral_preprocess_sets(config)) + radar = PointTargetMockService(config) + radar.configure(config.radar.sweep) + measured = radar.acquire_collection() + calibration, reference = build_neutral_s21_sets(config) + for trace, cal, ref in zip(measured.traces, calibration.traces, reference.traces): + np.testing.assert_array_equal(trace.frequency_hz, cal.frequency_hz) + np.testing.assert_array_equal(trace.frequency_hz, ref.frequency_hz) + np.testing.assert_array_equal(trace.s21 / cal.s21 - ref.s21, trace.s21) + + def test_native_single_axis_remains_outside_verified_shortcut(self) -> None: + config = RunConfigModel.from_dict({"radar": {"model": "librevna", "driver_mode": "native"}}) + self.assertFalse(supports_neutral_preprocess_sets(config)) + if __name__ == "__main__": unittest.main() diff --git a/python_app/tests/test_librevna_multidevice.py b/python_app/tests/test_librevna_multidevice.py index f9bbffb..e134e97 100644 --- a/python_app/tests/test_librevna_multidevice.py +++ b/python_app/tests/test_librevna_multidevice.py @@ -346,7 +346,7 @@ class TransientRetryTierTest(unittest.TestCase): @staticmethod def _stub(behavior) -> _StubService: - service = _StubService(master_serial="M", slave_serials=["A", "B"], backend_mode="mock") + service = _StubService(master_serial="M", slave_serials=["A", "B"], backend_mode="native") service.behavior = behavior service.collection_calls = 0 service.reopen_calls = 0 @@ -517,15 +517,9 @@ class PerPointStimulusSwitchingTest(unittest.TestCase): def test_service_reports_four_outputs_when_switching(self) -> None: service = MultiDeviceLibreVnaService( - master_serial="M", slave_serials=["A", "B"], backend_mode="mock", switch_substage_count=2 + master_serial="M", slave_serials=["A", "B"], backend_mode="native", switch_substage_count=2 ) self.assertEqual(service.output_position_count, 4) - service.configure(RadarSweepModel(start_hz=1e6, stop_hz=2e6, points=4, if_bandwidth_hz=1e3)) - collection = service.acquire_collection(collection_id=1) - self.assertEqual(len(collection.traces), 4 * 4) - self.assertEqual( - sorted({trace.combo.output for trace in collection.traces}), [0, 1, 2, 3] - ) if __name__ == "__main__": diff --git a/python_app/tests/test_live_settings_schema.py b/python_app/tests/test_live_settings_schema.py index d49ad13..62b42b9 100644 --- a/python_app/tests/test_live_settings_schema.py +++ b/python_app/tests/test_live_settings_schema.py @@ -16,7 +16,6 @@ from python_app.gui.controllers.app_window_config.live_processing_mixin import ( _NON_WIDGET_LIVE_FIELDS, _WEB_DISPLAY_SCHEMA, _WEB_LIVE_GETTERS, - _WEB_STABLE_SCHEMA, web_apply_field_names, ) from python_app.orchestration.live_processing_config import ProcessingLiveConfig @@ -44,10 +43,8 @@ class LiveSettingsSchemaCoverageTest(unittest.TestCase): # A field is either widget-backed (in the schema) or explicitly non-widget — never both. self.assertEqual(set(_WEB_LIVE_GETTERS) & _NON_WIDGET_LIVE_FIELDS, set()) - def test_display_and_stable_fields_are_not_live_config(self) -> None: - # Display toggles (GUI profile) and stable fields (run_config) must never collide - # with live ProcessingLiveConfig fields — they take different write paths. - extra = {f for f, *_ in _WEB_DISPLAY_SCHEMA} | {f for f, *_ in _WEB_STABLE_SCHEMA} + def test_display_fields_are_not_live_config(self) -> None: + extra = {f for f, *_ in _WEB_DISPLAY_SCHEMA} self.assertEqual(extra & self._config_fields(), set()) self.assertEqual(extra & set(_WEB_LIVE_GETTERS), set()) @@ -55,7 +52,6 @@ class LiveSettingsSchemaCoverageTest(unittest.TestCase): form_fields = ( set(_WEB_LIVE_GETTERS) | {f for f, *_ in _WEB_DISPLAY_SCHEMA} - | {f for f, *_ in _WEB_STABLE_SCHEMA} ) # The web may apply every form field plus the history command, and nothing else. self.assertEqual(web_apply_field_names(), form_fields | {"history_command"}) diff --git a/python_app/tests/test_locator_client.py b/python_app/tests/test_locator_client.py new file mode 100644 index 0000000..c534eed --- /dev/null +++ b/python_app/tests/test_locator_client.py @@ -0,0 +1,36 @@ +"""The locator test client's framing survives fragmented timeout polls.""" + +import socket +import threading +import unittest +from unittest.mock import Mock + +from locator_test_client import encode_packet, read_packet, recv_exactly + + +class LocatorClientTest(unittest.TestCase): + def test_partial_header_and_payload_survive_timeouts(self) -> None: + payload = {"vlc": 5.5} + packet = encode_packet(payload, 7) + sock = Mock() + sock.recv.side_effect = [packet[:3], socket.timeout(), packet[3:8], + packet[8:10], socket.timeout(), packet[10:]] + self.assertEqual(read_packet(sock), (7, payload)) + + def test_timeout_wait_can_be_stopped_without_losing_thread(self) -> None: + stop = threading.Event() + sock = Mock() + + def receive(_size): + stop.set() + raise socket.timeout() + + sock.recv.side_effect = receive + with self.assertRaisesRegex(ConnectionError, "stopped"): + recv_exactly(sock, 8, stop) + + def test_eof_in_the_middle_of_packet_is_reported(self) -> None: + sock = Mock() + sock.recv.side_effect = [b"123", b""] + with self.assertRaisesRegex(ConnectionError, "closed"): + read_packet(sock) diff --git a/python_app/tests/test_neutral_preprocess_cli.py b/python_app/tests/test_neutral_preprocess_cli.py new file mode 100644 index 0000000..ad7f73f --- /dev/null +++ b/python_app/tests/test_neutral_preprocess_cli.py @@ -0,0 +1,126 @@ +"""Neutral pair creation must report incomplete sets and roll back failed writes.""" + +from __future__ import annotations + +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +import numpy as np + +from python_app.models.run_config_model import RunConfigModel +from python_app.scripts import create_neutral_preprocess_sets as cli +from python_app.storage.npz_store import NpzStore +from python_app.workflows.kamil_adc_neutral_preprocess import build_neutral_s21_sets + + +class NeutralPreprocessCliTest(unittest.TestCase): + def setUp(self) -> None: + self.directory = self.enterContext(tempfile.TemporaryDirectory()) + self.root = Path(self.directory) + self.store = NpzStore(self.root) + self.config = RunConfigModel.from_dict({"radar": { + "model": "librevna", "driver_mode": "mock", "sweep": {"points": 7}, + }}) + calibration, reference = build_neutral_s21_sets(self.config) + calibration.traces[0].s21[:] = 3 + 1j + reference.traces[0].s21[:] = 2 - 1j + self.previous = {"s21_calibration": calibration, "s21_reference": reference} + + def _main(self, *options: str) -> int: + with ( + mock.patch.object(cli.RunConfigModel, "load_from_path", return_value=self.config), + mock.patch.object(cli, "radar_key_from_config", return_value="radar"), + mock.patch.object(cli, "NpzStore", return_value=self.store), + mock.patch("sys.argv", ["neutral", "--config", "profile.json", "--set-name", "neutral", *options]), + ): + return cli.main() + + def _files(self) -> dict[str, bytes]: + return {str(path.relative_to(self.root)): path.read_bytes() for path in self.root.rglob("*") if path.is_file()} + + def _fail_second_archive(self): + save_archive = np.savez + count = 0 + + def write(*args, **kwargs): + nonlocal count + count += 1 + if count == 2: + raise OSError("reference write failed") + return save_archive(*args, **kwargs) + + return mock.patch("python_app.storage.npz.store.np.savez", side_effect=write) + + def test_partial_existing_pair_reports_missing_kind_without_overwriting(self) -> None: + for present, missing in [("s21_calibration", "s21_reference"), ("s21_reference", "s21_calibration")]: + with self.subTest(present=present): + self.store = NpzStore(self.root / present) + self.store.save_set(present, "radar", "neutral", self.previous[present]) + before = self._files() + with self.assertRaisesRegex(RuntimeError, f"missing {missing}; use --force to replace both"): + self._main() + self.assertEqual(self._files(), before) + + def test_complete_existing_pair_is_skipped(self) -> None: + self.store.save_sets("radar", "neutral", self.previous) + before = self._files() + with mock.patch.object(self.store, "save_sets", wraps=self.store.save_sets) as save: + self.assertEqual(self._main(), 0) + save.assert_not_called() + self.assertEqual(self._files(), before) + + def test_force_replaces_both_existing_sets_with_neutral_data(self) -> None: + self.store.save_sets("radar", "neutral", self.previous) + self.assertEqual(self._main("--force"), 0) + for kind, value in [("s21_calibration", 1), ("s21_reference", 0)]: + np.testing.assert_array_equal(self.store.load_set(kind, "radar", "neutral").traces[0].s21, value) + self.assertEqual(list(self.root.glob(".sets-*")), []) + + def test_force_completes_either_partial_pair(self) -> None: + for present in self.previous: + with self.subTest(present=present): + self.store = NpzStore(self.root / present) + self.store.save_set(present, "radar", "neutral", self.previous[present]) + self.assertEqual(self._main("--force"), 0) + for kind, value in [("s21_calibration", 1), ("s21_reference", 0)]: + np.testing.assert_array_equal(self.store.load_set(kind, "radar", "neutral").traces[0].s21, value) + + def test_second_archive_failure_restores_previous_pair_byte_for_byte(self) -> None: + self.store.save_sets("radar", "neutral", self.previous) + before = self._files() + with self._fail_second_archive(), self.assertRaisesRegex(OSError, "reference write failed"): + self._main("--force") + self.assertEqual(self._files(), before) + + def test_second_archive_failure_removes_new_pair(self) -> None: + with self._fail_second_archive(), self.assertRaisesRegex(OSError, "reference write failed"): + self._main() + self.assertEqual(self._files(), {}) + + def test_failed_rollback_keeps_the_previous_archive_for_recovery(self) -> None: + self.store.save_sets("radar", "neutral", self.previous) + previous_archive = (self.root / "s21_calibration/radar/neutral.npz").read_bytes() + replace = Path.replace + + def replace_file(path: Path, destination: Path): + if path.parent.name.startswith(".sets-") and path.name == "s21_calibration.npz": + raise OSError("restore failed") + return replace(path, destination) + + with ( + self._fail_second_archive(), mock.patch.object(Path, "replace", replace_file), + self.assertRaisesRegex(RuntimeError, "rollback was incomplete; previous files are retained") as error, + ): + self._main("--force") + backups = list(self.root.glob(".sets-*/s21_calibration.npz")) + self.assertEqual(len(backups), 1) + self.assertEqual(backups[0].read_bytes(), previous_archive) + self.assertIn(str(backups[0].parent), str(error.exception)) + + def test_native_single_is_rejected_before_any_write(self) -> None: + self.config.radar.driver_mode = "native" + with self.assertRaisesRegex(RuntimeError, "not supported for this radar configuration"): + self._main() + self.assertEqual(self._files(), {}) diff --git a/python_app/tests/test_orchestration_recovery.py b/python_app/tests/test_orchestration_recovery.py index 13a720c..f812949 100644 --- a/python_app/tests/test_orchestration_recovery.py +++ b/python_app/tests/test_orchestration_recovery.py @@ -13,13 +13,16 @@ from __future__ import annotations import json import os +import signal import sys import tempfile import time import unittest from pathlib import Path +from unittest import mock from python_app.models.run_config_model import RunConfigModel +from python_app.gui.controllers.app_window_pipeline_mixin import AppWindowPipelineMixin from python_app.orchestration.config_writer import ConfigWriter from python_app.orchestration.process_supervisor import ProcessExitReport, ProcessSupervisor from python_app.orchestration.restart_policy import RestartPolicy @@ -53,6 +56,64 @@ class RestartPolicyTest(unittest.TestCase): self.assertTrue(RestartPolicy().should_restart_now(now_s=5_000.0, last_restart_s=0.0, consecutive_failures=0)) +class PipelineRecoveryPollingTest(unittest.TestCase): + def setUp(self) -> None: + self.window = mock.Mock(spec=AppWindowPipelineMixin) + self.window._pipeline_should_run = True + self.window._supervisor = mock.Mock() + self.window._supervisor.collect_exit_reports.return_value = [] + + def test_missing_child_keeps_retrying_without_fresh_exit_reports(self) -> None: + self.window._supervisor.pids.return_value = {"data_processor": 1, "data_preprocessor": 2} + for _ in range(3): + AppWindowPipelineMixin._handle_process_exit_reports(self.window) + self.assertEqual(self.window._recover_pipeline_after_crash.call_count, 3) + + def test_failed_start_with_no_children_keeps_retrying(self) -> None: + self.window._supervisor.pids.return_value = {} + for _ in range(3): + AppWindowPipelineMixin._handle_process_exit_reports(self.window) + self.assertEqual(self.window._recover_pipeline_after_crash.call_count, 3) + + def test_failed_restarts_retry_after_backoff_without_new_exit_reports(self) -> None: + self.window._supervisor.pids.return_value = {} + self.window._supervisor.start.side_effect = RuntimeError("device unavailable") + self.window._active_run_config = RunConfigModel() + self.window._active_run_config_path = Path("run_config.json") + self.window._pipeline_restart_count = 0 + self.window._last_pipeline_restart_s = 0.0 + self.window._RESTART_POLICY = RestartPolicy() + self.window._log_error = mock.Mock() + self.window._log_exception = mock.Mock() + self.window._recover_pipeline_after_crash = ( + lambda: AppWindowPipelineMixin._recover_pipeline_after_crash(self.window) + ) + + with mock.patch( + "python_app.gui.controllers.app_window_pipeline_mixin.time.monotonic", + side_effect=[100.0, 101.0, 106.0], + ): + for _ in range(3): + AppWindowPipelineMixin._handle_process_exit_reports(self.window) + + self.assertEqual(self.window._supervisor.start.call_count, 2) + self.assertEqual(self.window._pipeline_restart_count, 2) + self.assertTrue(self.window._pipeline_should_run) + + def test_healthy_pipeline_does_not_restart(self) -> None: + self.window._supervisor.pids.return_value = { + "data_processor": 1, "data_preprocessor": 2, "sweep_orchestrator": 3, + } + AppWindowPipelineMixin._handle_process_exit_reports(self.window) + self.window._recover_pipeline_after_crash.assert_not_called() + + def test_explicit_stop_does_not_restart_missing_children(self) -> None: + self.window._pipeline_should_run = False + self.window._supervisor.pids.return_value = {} + AppWindowPipelineMixin._handle_process_exit_reports(self.window) + self.window._recover_pipeline_after_crash.assert_not_called() + + class ConfigWriterTest(unittest.TestCase): def setUp(self) -> None: self._dir = tempfile.TemporaryDirectory() @@ -88,23 +149,32 @@ class RadarModelCommandTest(unittest.TestCase): self.root = Path(self._dir.name) self.supervisor = ProcessSupervisor(self.root) - def _config_with_model(self, model: str) -> Path: + def _config_with_model(self, model: str, driver_mode: str = "native") -> Path: path = self.root / "cfg.json" - path.write_text(json.dumps({"radar": {"model": model}})) + path.write_text(json.dumps({"radar": {"model": model, "driver_mode": driver_mode}})) return path - def test_read_radar_model(self) -> None: - self.assertEqual(ProcessSupervisor._read_radar_model(self._config_with_model("kamil_adc")), "kamil_adc") - - def test_read_radar_model_defaults_on_malformed(self) -> None: + def test_malformed_root_falls_back_to_native_producer(self) -> None: path = self.root / "bad.json" - path.write_text("[]") # not an object - self.assertEqual(ProcessSupervisor._read_radar_model(path), "librevna") + path.write_text("[]") + self.assertTrue(self.supervisor._acquisition_command(path)[0].endswith("sweep_orchestrator")) - def test_matrix_producer_for_multi_and_sn9000(self) -> None: - for model in ("librevna_multi", "sn9000"): - cmd = self.supervisor._acquisition_command(self._config_with_model(model)) - self.assertEqual(cmd[:3], [sys.executable, "-m", "python_app.scripts.matrix_raw_producer"]) + def test_null_replay_directory_does_not_select_replay(self) -> None: + path = self._config_with_model("librevna") + path.write_text(json.dumps({"radar": {"replay": {"dataset_dir": None}}})) + self.assertEqual(self.supervisor._acquisition_command(path)[:3], + [sys.executable, "-m", "python_app.scripts.matrix_raw_producer"]) + + def test_replay_overrides_every_model(self) -> None: + for model in ("librevna", "librevna_multi", "kamil_adc"): + path = self._config_with_model(model) + path.write_text(json.dumps({"radar": {"model": model, "replay": {"dataset_dir": "capture"}}})) + self.assertEqual(self.supervisor._acquisition_command(path)[:3], + [sys.executable, "-m", "python_app.scripts.replay_raw_producer"]) + + def test_matrix_producer_for_multi(self) -> None: + cmd = self.supervisor._acquisition_command(self._config_with_model("librevna_multi")) + self.assertEqual(cmd[:3], [sys.executable, "-m", "python_app.scripts.matrix_raw_producer"]) def test_kamil_producer_for_kamil_adc(self) -> None: cmd = self.supervisor._acquisition_command(self._config_with_model("kamil_adc")) @@ -114,6 +184,30 @@ class RadarModelCommandTest(unittest.TestCase): cmd = self.supervisor._acquisition_command(self._config_with_model("librevna")) self.assertTrue(cmd[0].endswith("build/bin/sweep_orchestrator")) + def test_single_and_multi_mock_share_the_simulator(self) -> None: + for model in ("librevna", "librevna_multi"): + command = self.supervisor._acquisition_command(self._config_with_model(model, "mock")) + self.assertEqual(command[:3], [sys.executable, "-m", "python_app.scripts.matrix_raw_producer"]) + + def test_omitted_driver_mode_uses_the_default_simulator(self) -> None: + path = self._config_with_model("librevna") + path.write_text(json.dumps({"radar": {"model": "librevna"}})) + self.assertEqual(self.supervisor._acquisition_command(path)[2], "python_app.scripts.matrix_raw_producer") + + def test_missing_or_null_radar_fields_keep_schema_defaults(self) -> None: + path = self.root / "defaults.json" + for payload in ({}, {"radar": None}, {"radar": {}}, {"radar": {"model": None}}, + {"radar": {"driver_mode": None}}, {"radar": {"driver_mode": "mock"}}): + with self.subTest(payload=payload): + path.write_text(json.dumps(payload)) + self.assertEqual(self.supervisor._acquisition_command(path)[:3], + [sys.executable, "-m", "python_app.scripts.matrix_raw_producer"]) + + def test_omitted_model_with_native_mode_uses_native_producer(self) -> None: + path = self.root / "native.json" + path.write_text(json.dumps({"radar": {"driver_mode": "native"}})) + self.assertTrue(self.supervisor._acquisition_command(path)[0].endswith("sweep_orchestrator")) + class ProcessExitReportTest(unittest.TestCase): @staticmethod @@ -192,6 +286,46 @@ class SupervisorLifecycleTest(unittest.TestCase): self.assertFalse(self.supervisor._is_stale_pipeline_pid(2_000_000_000)) # no such pid self.assertFalse(self.supervisor._is_stale_pipeline_pid(os.getpid())) # the test runner + def test_stale_pid_guard_matches_executable_or_module_arguments(self) -> None: + processor = str(self.supervisor._project_root / "build/bin/data_processor") + cases = ( + (f"{processor}\x00--config\x00config.json\x00", True), + ("python\x00-m\x00python_app.scripts.matrix_raw_producer\x00", True), + (f"bash\x00-c\x00echo {processor}\x00", False), + ("python\x00-c\x00print('python_app.scripts.matrix_raw_producer')\x00", False), + ("/other/project/build/bin/data_processor\x00", False), + ) + for cmdline, expected in cases: + with ( + self.subTest(cmdline=cmdline), + mock.patch.object(Path, "read_bytes", return_value=cmdline.encode()), + ): + self.assertEqual(self.supervisor._is_stale_pipeline_pid(123), expected) + + def test_signal_group_does_not_kill_a_non_leaders_group(self) -> None: + with ( + mock.patch.object(os, "getpgid", return_value=456), + mock.patch.object(os, "killpg") as group_kill, + mock.patch.object(os, "kill") as kill, + ): + self.supervisor._signal_group(123, signal.SIGTERM) + group_kill.assert_not_called() + kill.assert_called_once_with(123, signal.SIGTERM) + + def test_collector_reaper_does_not_match_command_text(self) -> None: + entries = [Path("/proc/123"), Path("/proc/456")] + cmdlines = [ + b"bash\x00-c\x00echo kamil_adc_collector\x00", + b"/project/build/bin/kamil_adc_collector\x00--config\x00config.json\x00", + ] + with ( + mock.patch.object(Path, "iterdir", return_value=entries), + mock.patch.object(Path, "read_bytes", side_effect=cmdlines), + mock.patch.object(ProcessSupervisor, "_signal_group") as signal_group, + ): + self.supervisor._kill_kamil_collectors() + signal_group.assert_called_once_with(456, signal.SIGKILL) + if __name__ == "__main__": unittest.main() diff --git a/python_app/tests/test_persistence_integrity.py b/python_app/tests/test_persistence_integrity.py new file mode 100644 index 0000000..3880c18 --- /dev/null +++ b/python_app/tests/test_persistence_integrity.py @@ -0,0 +1,349 @@ +"""Persistence invariants: complete data, compatible sets and recoverable write failures.""" +from __future__ import annotations + +import json +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +import numpy as np + +from python_app.models.dataset_model import ( + ComboKey, ResultBlock, ResultCollection, ResultPayload, SweepCollection, TraceData, +) +from python_app.storage.npz.store import NpzStore +from python_app.storage.npz.snapshot_numpy import select_aligned_histories +from python_app.storage.npz.vna_history_json import build_vna_history_payload +from python_app.scripts.replay_raw_producer import _load_dataset +from python_app.scripts.convert_snapshot_to_vna_history import _collection_dirs, _load_stage_records, _build_sweep_history +from python_app.workflows.sequential_capture_workflow import combine_collections_via_median, combine_traces_via_median + + +def _sweep(cid: int = 1, ns: int = 20, *, start: int = 10, end: int = 15) -> SweepCollection: + return SweepCollection( + collection_id=cid, monotonic_ns=ns, capture_start_ns=start, capture_end_ns=end, + traces=[TraceData( + combo=ComboKey(0, 0), frequency_hz=np.array([1e9, 2e9], dtype=np.float32), + s11=np.array([1 + 2j, 3 - 4j], dtype=np.complex64), + s21=np.array([5 - 6j, 7 + 8j], dtype=np.complex64), + capture_start_ns=start, capture_end_ns=end, + )], + ) + + +def _payloads() -> list[ResultPayload]: + return [ + ResultPayload('trace', 1, frequency_hz=np.array([1, 2], dtype=np.float32), + trace=np.array([1 + 2j, 3 - 4j], dtype=np.complex64)), + ResultPayload('scalar', 2, scalar_value=3.5), + ResultPayload('image', 3, image_x_axis=np.array([1, 2], dtype=np.float32), + image_y_axis=np.array([3], dtype=np.float32), image=np.array([[4, 5]], dtype=np.float32)), + ResultPayload('table', 4, table=np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32)), + ] + + +class PersistenceIntegrityTest(unittest.TestCase): + def setUp(self) -> None: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.store = NpzStore(self.root / 'sets') + self.output = self.root / 'output' + + def test_all_calibration_and_reference_kinds_preserve_arrays_and_timing(self) -> None: + for kind in ('s21_calibration', 's21_reference', 's11_open', 's11_short', 's11_load', 's11_reference'): + with self.subTest(kind=kind): + original = _sweep(7, 100) + second = _sweep().traces[0] + second.combo = ComboKey(2, 3) + original.traces.append(second) + self.store.save_set(kind, 'radar', 'full', original) + loaded = self.store.load_set(kind, 'radar', 'full') + self.assertEqual((loaded.collection_id, loaded.monotonic_ns), (7, 100)) + self.assertEqual((loaded.capture_start_ns, loaded.capture_end_ns), (10, 15)) + self.assertTrue(self.store.has_combo_coverage(kind, 'radar', 'full', [ComboKey(0, 0), ComboKey(2, 3)])) + self.assertFalse(self.store.has_combo_coverage(kind, 'radar', 'full', [ComboKey(9, 9)])) + for before, after in zip(original.traces, loaded.traces, strict=True): + self.assertEqual(before.combo, after.combo) + self.assertEqual((after.capture_start_ns, after.capture_end_ns), (10, 15)) + for field in ('frequency_hz', 's11', 's21'): + np.testing.assert_array_equal(getattr(before, field), getattr(after, field)) + + def test_new_set_remains_loadable_without_json_sidecar(self) -> None: + self.store.save_set('s21_reference', 'radar', 'full', _sweep()) + (self.root / 'sets/s21_reference/radar/full.json').unlink() + loaded = self.store.load_set('s21_reference', 'radar', 'full') + np.testing.assert_array_equal(loaded.traces[0].s21, _sweep().traces[0].s21) + + def test_legacy_npz_json_pair_is_still_loadable(self) -> None: + self.store.save_set('s21_reference', 'radar', 'full', _sweep()) + archive = self.root / 'sets/s21_reference/radar/full.npz' + with np.load(archive, allow_pickle=False) as arrays: + legacy = {key: arrays[key] for key in arrays.files if key != 'metadata_json'} + np.savez(archive, **legacy) + self.assertEqual(self.store.load_set('s21_reference', 'radar', 'full').monotonic_ns, 20) + + def test_failed_set_commit_preserves_previous_data_and_sidecar(self) -> None: + self.store.save_set('s21_reference', 'radar', 'full', _sweep(1, 20)) + original_meta = (self.root / 'sets/s21_reference/radar/full.json').read_bytes() + replace = Path.replace + + def fail_archive_commit(path, destination): + if Path(destination).suffix == '.npz': + raise OSError('disk full during commit') + return replace(path, destination) + + with mock.patch.object(Path, 'replace', fail_archive_commit), self.assertRaises(OSError): + self.store.save_set('s21_reference', 'radar', 'full', _sweep(2, 200)) + self.assertEqual(self.store.load_set('s21_reference', 'radar', 'full').collection_id, 1) + self.assertEqual((self.root / 'sets/s21_reference/radar/full.json').read_bytes(), original_meta) + self.assertEqual(list(self.root.rglob('*.tmp')), []) + + def test_failed_new_set_commit_does_not_publish_a_sidecar_only_set(self) -> None: + replace = Path.replace + + def fail_archive_commit(path, destination): + if Path(destination).suffix == '.npz': + raise OSError('commit failed') + return replace(path, destination) + + with mock.patch.object(Path, 'replace', fail_archive_commit), self.assertRaises(OSError): + self.store.save_set('s21_reference', 'radar', 'full', _sweep()) + self.assertEqual(self.store.list_sets('s21_reference', 'radar'), []) + self.assertEqual(list(self.root.rglob('*.json')), []) + self.assertEqual(list(self.root.rglob('*.tmp')), []) + + def test_failed_sidecar_commit_restores_previous_archive_without_rewriting_it(self) -> None: + self.store.save_set('s21_reference', 'radar', 'full', _sweep(1, 20)) + original = (self.root / 'sets/s21_reference/radar/full.npz').read_bytes() + replace = Path.replace + + def fail_metadata_commit(path, destination): + if Path(destination).suffix == '.json': + raise OSError('metadata commit failed') + return replace(path, destination) + + with mock.patch.object(Path, 'replace', fail_metadata_commit), self.assertRaises(OSError): + self.store.save_set('s21_reference', 'radar', 'full', _sweep(2, 200)) + self.assertEqual((self.root / 'sets/s21_reference/radar/full.npz').read_bytes(), original) + self.assertEqual(self.store.load_set('s21_reference', 'radar', 'full').collection_id, 1) + self.assertEqual(list(self.root.rglob('*.tmp')), []) + + def test_invalid_sets_are_rejected_before_overwriting_valid_data(self) -> None: + self.store.save_set('s21_calibration', 'radar', 'full', _sweep()) + invalid = [_sweep(), _sweep(), _sweep(), _sweep()] + invalid[0].traces.append(invalid[0].traces[0]) + invalid[1].traces[0].s21 = np.array([1], dtype=np.complex64) + invalid[2].traces[0].frequency_hz = np.array([[1, 2]], dtype=np.float32) + invalid[3].traces[0].s11[0] = complex(float('nan'), 0) + for collection in invalid: + with self.subTest(collection=collection), self.assertRaises(ValueError): + self.store.save_set('s21_calibration', 'radar', 'full', collection) + self.assertEqual(self.store.load_set('s21_calibration', 'radar', 'full').monotonic_ns, 20) + + def test_set_names_cannot_escape_store(self) -> None: + for name in ('../escape', 'sub/name', '..', '.', '', r'sub\name'): + with self.subTest(name=name), self.assertRaises(ValueError): + self.store.save_set('s21_reference', 'radar', name, _sweep()) + + def test_invalid_channel_identifiers_are_rejected_without_truncation(self) -> None: + for position in (0.5, True, -1, 2 ** 32): + sweep = _sweep() + sweep.traces[0].combo = ComboKey(position, 0) + with self.subTest(position=position), self.assertRaises(ValueError): + self.store.save_set('s21_reference', 'radar', 'invalid', sweep) + self.assertEqual(self.store.list_sets('s21_reference', 'radar'), []) + + def test_corrupted_metadata_types_are_rejected_without_truncation(self) -> None: + self.store.save_set('s21_reference', 'radar', 'full', _sweep()) + archive = self.root / 'sets/s21_reference/radar/full.npz' + with np.load(archive, allow_pickle=False) as stored: + arrays = {key: stored[key] for key in stored.files} + original = json.loads(str(arrays['metadata_json'].item())) + for malformed in ([], {**original, 'collection_id': 1.5}, + {**original, 'monotonic_ns': True}, + {**original, 'combos': [{**original['combos'][0], 'input': 0.5}]}): + with self.subTest(metadata=malformed): + np.savez(archive, **{**arrays, 'metadata_json': np.asarray(json.dumps(malformed))}) + with self.assertRaisesRegex(RuntimeError, 'Corrupted preprocess set'): + self.store.load_set('s21_reference', 'radar', 'full') + + def test_complete_snapshot_keeps_unmatched_stage_tails(self) -> None: + raw = [_sweep(1, 10), _sweep(2, 30)] + pre = [_sweep(1, 20), _sweep(3, 40)] + result = [ResultCollection(1, 20)] + directory, summary = self.store.save_runtime_snapshot_numpy( + self.output, 'full', raw, pre, result, 1000, include_unaligned=True, + ) + self.assertEqual((summary['raw_count'], summary['preprocessed_count'], summary['result_count']), (2, 2, 1)) + for stage, ids in [('raw', [1, 2]), ('preprocessed', [1, 3]), ('results', [1])]: + metadata = [json.loads(path.read_text()) for path in sorted((directory / stage).glob('*/meta.json'))] + self.assertEqual([item['collection_id'] for item in metadata], ids) + + def test_snapshot_result_payloads_preserve_every_kind_at_both_scopes(self) -> None: + payloads = _payloads() + result = ResultCollection(1, 20, processing_duration_ns=12345, + collection_payloads=payloads, blocks=[ResultBlock(ComboKey(0, 0), payloads)]) + directory, _ = self.store.save_runtime_snapshot_numpy(self.output, 'full', [], [], [result], 1000) + collection_dir = next((directory / 'results').iterdir()) + meta = json.loads((collection_dir / 'meta.json').read_text()) + self.assertEqual(meta['processing_duration_ns'], 12345) + for base, entries in [(collection_dir, meta['collection_payloads']), + (collection_dir / meta['blocks'][0]['dir'], meta['blocks'][0]['payloads'])]: + self.assertEqual([entry['kind'] for entry in entries], [1, 2, 3, 4]) + np.testing.assert_array_equal(np.load(base / entries[0]['freq_file']), payloads[0].frequency_hz) + np.testing.assert_array_equal(np.load(base / entries[0]['trace_file']), payloads[0].trace) + np.testing.assert_array_equal(np.load(base / entries[1]['scalar_file']), np.array([3.5], dtype=np.float32)) + np.testing.assert_array_equal(np.load(base / entries[2]['x_axis_file']), payloads[2].image_x_axis) + np.testing.assert_array_equal(np.load(base / entries[2]['y_axis_file']), payloads[2].image_y_axis) + np.testing.assert_array_equal(np.load(base / entries[2]['image_file']), payloads[2].image) + np.testing.assert_array_equal(np.load(base / entries[3]['table_file']), payloads[3].table) + + def test_bad_result_payload_rolls_back_entire_snapshot(self) -> None: + for payload in (ResultPayload('unknown', 9), ResultPayload('bad_table', 4, table=np.array([1, 2]))): + with self.subTest(kind=payload.kind), self.assertRaises(ValueError): + self.store.save_runtime_snapshot_numpy( + self.output, 'failed', [_sweep()], [], [ResultCollection(1, 20, collection_payloads=[payload])], 1000, + ) + self.assertFalse((self.output / 'failed').exists()) + self.assertEqual(list(self.output.iterdir()), []) + + def test_snapshot_write_failure_can_be_retried_with_same_name(self) -> None: + with mock.patch('numpy.save', side_effect=OSError('disk full')), self.assertRaises(OSError): + self.store.save_runtime_snapshot_numpy(self.output, 'retry', [_sweep()], [], [], 1000) + directory, _ = self.store.save_runtime_snapshot_numpy(self.output, 'retry', [_sweep()], [], [], 1000) + self.assertTrue(directory.exists()) + with self.assertRaises(FileExistsError): + self.store.save_runtime_snapshot_numpy(self.output, 'retry', [_sweep()], [], [], 1000) + + def test_stream_failure_keeps_prior_chunks_and_retries_without_partial_data(self) -> None: + writer = self.store.create_snapshot_stream(self.output, 'stream') + writer.append([_sweep()], [], [ResultCollection(1, 20)]) + with mock.patch('numpy.save', side_effect=OSError('disk full')), self.assertRaises(OSError): + writer.append([_sweep(2, 30)], [], [ResultCollection(2, 40)]) + self.assertEqual(writer.result_count, 1) + self.assertEqual(len(list((writer.directory / 'raw').iterdir())), 1) + self.assertEqual(list(writer.directory.glob('.chunk-*')), []) + writer.append([_sweep(2, 30)], [], [ResultCollection(2, 40)]) + self.assertEqual(writer.result_count, 2) + self.assertEqual([path.name.split('_')[0] for path in sorted((writer.directory / 'raw').iterdir())], ['0000', '0001']) + + def test_json_export_preserves_unmatched_raw_sweeps_and_matches_reused_ids_by_timing(self) -> None: + raw_old = _sweep(1, 10, start=1, end=5) + raw_new = _sweep(1, 100, start=90, end=95) + raw_new.traces[0].s21[:] = 99 + pre_new = _sweep(1, 110, start=90, end=95) + payload = build_vna_history_payload([raw_old, raw_new], [pre_new], [], input_index=0, output_index=0) + self.assertEqual(len(payload['sweep_history']), 2) + self.assertEqual(payload['sweep_history'][-1]['sweep_points'], [[99.0, 0.0], [99.0, 0.0]]) + self.assertEqual(payload['sweep_history'][-1]['calibrated_points'], [[5.0, -6.0], [7.0, 8.0]]) + + def test_json_batch_failure_does_not_publish_partial_export(self) -> None: + sweep = _sweep() + second = _sweep().traces[0] + second.combo = ComboKey(0, 1) + sweep.traces.append(second) + write_text = Path.write_text + writes = 0 + + def fail_second_file(path, *args, **kwargs): + nonlocal writes + writes += 1 + if writes == 2: + raise OSError('disk full') + return write_text(path, *args, **kwargs) + + with mock.patch.object(Path, 'write_text', fail_second_file), self.assertRaises(OSError): + self.store.save_runtime_vna_history_json_batch(self.output, 'export', [sweep], [], [], 1000) + self.assertEqual(list(self.output.iterdir()), []) + paths, summary = self.store.save_runtime_vna_history_json_batch(self.output, 'export', [sweep], [], [], 1000, channel='s11') + self.assertEqual(summary['combo_count'], 2) + self.assertEqual(len(paths), 2) + for path in paths: + self.assertEqual(json.loads(path.read_text())['channel'], 's11') + + def test_saved_dataset_replay_preserves_s11_and_accepts_legacy_s21_only(self) -> None: + directory, _ = self.store.save_runtime_snapshot_numpy(self.output, 'replay', [_sweep()], [], [], 1000) + dataset = _load_dataset(directory / 'raw', [ComboKey(0, 0)], 2) + np.testing.assert_array_equal(dataset.snapshots[0][0].s11, _sweep().traces[0].s11) + self.assertEqual(dataset.snapshots[0][0].capture_end_ns, 15) + next((directory / 'raw').glob('*/i0_o0_s11.npy')).unlink() + legacy = _load_dataset(directory / 'raw', [ComboKey(0, 0)], 2) + np.testing.assert_array_equal(legacy.snapshots[0][0].s11, np.zeros(2, dtype=np.complex64)) + + def test_long_recording_replay_and_converter_use_numeric_directory_order(self) -> None: + writer = self.store.create_snapshot_stream(self.output, 'long') + writer.append([_sweep(1, 10), _sweep(2, 20), _sweep(3, 30)], [], []) + folders = sorted((writer.directory / 'raw').iterdir()) + for folder, index in zip(folders, (9998, 9999, 10000), strict=True): + folder.rename(folder.with_name(f'{index}_{folder.name.split("_", 1)[1]}')) + dataset = _load_dataset(writer.directory / 'raw', [ComboKey(0, 0)], 2) + self.assertEqual(dataset.timestamps_ns, [10, 20, 30]) + self.assertEqual([int(path.name.split('_')[0]) for path in _collection_dirs(writer.directory / 'raw')], [9998, 9999, 10000]) + + def test_offline_and_runtime_json_exports_share_matching_and_preserve_orphan_sweeps(self) -> None: + raw = [_sweep(1, 10, start=1, end=5), _sweep(1, 100, start=90, end=95)] + pre = [_sweep(1, 110, start=90, end=95)] + directory, _ = self.store.save_runtime_snapshot_numpy( + self.output, 'convert', raw, pre, [], 1000, include_unaligned=True, + ) + records = { + stage: _load_stage_records(directory, stage, input_index=0, output_index=0, channel='s21') + for stage in ('raw', 'preprocessed') + } + offline = _build_sweep_history(records['raw'], records['preprocessed'], channel='s21', primary_stage='preprocessed') + runtime = build_vna_history_payload(raw, pre, [], input_index=0, output_index=0) + self.assertEqual(offline, runtime['sweep_history']) + self.assertEqual(len(offline), 2) + + +class SnapshotAlignmentTest(unittest.TestCase): + def test_reused_ids_and_missing_frames_do_not_cross_align_runs(self) -> None: + raw_old = _sweep(1, 10, start=1, end=5) + pre_old = _sweep(1, 20, start=1, end=5) + pre_new = _sweep(1, 200, start=150, end=175) + selected_raw, selected_pre, results, summary = select_aligned_histories( + [raw_old], [pre_old, pre_new], [ResultCollection(1, 200)], 1, + ) + self.assertEqual(selected_raw, []) + self.assertEqual(selected_pre, [pre_new]) + self.assertEqual(summary['raw_missing_count'], 1) + self.assertEqual(results[0].monotonic_ns, 200) + + def test_result_does_not_pair_with_wrong_preprocessed_timestamp(self) -> None: + _, pre, _, summary = select_aligned_histories([], [_sweep(1, 20)], [ResultCollection(1, 200)], 1000) + self.assertEqual(pre, []) + self.assertEqual(summary['preprocessed_missing_count'], 1) + + def test_independent_stage_limit_is_exactly_requested_tail(self) -> None: + raw = [_sweep(cid=index, ns=index) for index in range(1005)] + selected_raw, _, _, _ = select_aligned_histories(raw, [], [], 1000, include_unaligned=True) + self.assertEqual(len(selected_raw), 1000) + self.assertEqual((selected_raw[0].collection_id, selected_raw[-1].collection_id), (5, 1004)) + + +class CalibrationMedianIntegrityTest(unittest.TestCase): + def test_rejects_frequency_axes_that_changed_between_captures(self) -> None: + first, second = _sweep(), _sweep() + second.traces[0].frequency_hz[1] += 1e8 + with self.assertRaisesRegex(RuntimeError, 'frequency-axis'): + combine_traces_via_median([first.traces[0], second.traces[0]]) + + def test_rejects_extra_combinations_instead_of_silently_discarding_them(self) -> None: + first, second = _sweep(), _sweep() + extra = _sweep().traces[0] + extra.combo = ComboKey(0, 1) + second.traces.append(extra) + with self.assertRaisesRegex(RuntimeError, 'trace counts'): + combine_collections_via_median([first, second]) + + def test_collection_and_trace_windows_span_all_median_captures(self) -> None: + first, second = _sweep(start=10, end=15), _sweep(start=30, end=35) + combined = combine_collections_via_median([first, second]) + self.assertEqual((combined.capture_start_ns, combined.capture_end_ns), (10, 35)) + self.assertEqual((combined.traces[0].capture_start_ns, combined.traces[0].capture_end_ns), (10, 35)) + + +if __name__ == '__main__': + unittest.main() diff --git a/python_app/tests/test_point_target_mock.py b/python_app/tests/test_point_target_mock.py new file mode 100644 index 0000000..ac77aa2 --- /dev/null +++ b/python_app/tests/test_point_target_mock.py @@ -0,0 +1,198 @@ +"""Analytical checks for one coherent moving reflector across mock radar modes.""" + +from __future__ import annotations + +from pathlib import Path +import unittest +from unittest import mock + +import numpy as np + +from python_app.hardware_full.matrix_radar_service import create_acquisition_service +from python_app.hardware_full.point_target_mock_service import PointTargetMockService +from python_app.hardware_full.single_radar_service import create_single_radar_service +from python_app.models.dataset_model import ComboKey +from python_app.models.run_config_model import RunConfigModel +from python_app.models.run_config_schema import GprRxGeometryModel, GprTxGeometryModel +from python_app.workflows.multi_radar_capture_workflow import MultiRadarSequentialCaptureSession +from python_app.workflows.radar_config_variants import RadarConfigVariant +from python_app.workflows.sequential_capture_workflow import SequentialCaptureSession + +_EPOCH_NS = 10_000_000_000 +_CLOCK = "python_app.hardware_full.point_target_mock_service.time.monotonic_ns" +_SLEEP = "python_app.hardware_full.point_target_mock_service.time.sleep" + + +def _config() -> RunConfigModel: + config = RunConfigModel() + config.radar.model = RunConfigModel.LIBREVNA_MODEL + config.radar.driver_mode = "mock" + config.radar.sweep.start_hz = 300e6 + config.radar.sweep.stop_hz = 900e6 + config.radar.sweep.points = 32 + config.input_switch.positions = config.output_switch.positions = 2 + config.input_switch.driver_mode = config.output_switch.driver_mode = "mock" + config.runtime.settling_ms = 0 + config.combos = config.build_runtime_combos() + config.gpr.tx_geometry = [GprTxGeometryModel(0, -1), GprTxGeometryModel(1, 1)] + config.gpr.rx_geometry = [GprRxGeometryModel(0, -.4, .1, .2), GprRxGeometryModel(1, .5, -.1, .3)] + return config + + +class PointTargetMockTest(unittest.TestCase): + def setUp(self) -> None: + self.clock = self.enterContext(mock.patch(_CLOCK, return_value=_EPOCH_NS)) + self.sleep = self.enterContext(mock.patch(_SLEEP)) + self.config = _config() + self.service = PointTargetMockService(self.config) + self.service.configure(self.config.radar.sweep) + + def test_all_channels_follow_bistatic_and_monostatic_distance(self) -> None: + collection = self.service.acquire_collection(collection_id=43) + target = np.array([0, .25, 3.75]) + self.assertEqual(collection.collection_id, 43) + self.assertEqual([(t.combo.input, t.combo.output) for t in collection.traces], [(0, 0), (1, 0), (0, 1), (1, 1)]) + for trace in collection.traces: + tx = self.config.gpr.tx_geometry[trace.combo.output] + rx = self.config.gpr.rx_geometry[trace.combo.input] + r_tx = np.linalg.norm(target - [tx.x_m, tx.y_m, tx.z_m]) + r_rx = np.linalg.norm(target - [rx.x_m, rx.y_m, rx.z_m]) + phase_per_m = -2j * np.pi * trace.frequency_hz.astype(np.float64) / 3e8 + np.testing.assert_allclose(trace.s21, .5 / (r_tx * r_rx) * np.exp(phase_per_m * (r_tx + r_rx)), rtol=1e-6) + np.testing.assert_allclose(trace.s11, .5 / r_tx**2 * np.exp(phase_per_m * 2 * r_tx), rtol=1e-6) + self.assertEqual((trace.capture_start_ns, trace.capture_end_ns), (_EPOCH_NS, _EPOCH_NS)) + self.assertEqual(trace.s21.dtype, np.dtype(np.complex64)) + self.assertEqual((collection.capture_start_ns, collection.capture_end_ns, collection.monotonic_ns), (_EPOCH_NS,) * 3) + + def test_coincident_antennas_have_identical_traces_regardless_of_channel_number(self) -> None: + self.config.gpr.tx_geometry = [GprTxGeometryModel(0, 1), GprTxGeometryModel(1, 1)] + self.config.gpr.rx_geometry = [GprRxGeometryModel(0, -.5), GprRxGeometryModel(1, -.5)] + traces = self.service.acquire_collection().traces + for trace in traces[1:]: + np.testing.assert_array_equal(trace.s21, traces[0].s21) + np.testing.assert_array_equal(trace.s11, traces[0].s11) + + def test_geometry_changes_receiver_path_without_changing_transmitter_echo(self) -> None: + before = self.service.acquire_collection().traces[0] + self.config.gpr.rx_geometry[0].z_m = 1 + after = self.service.acquire_collection().traces[0] + self.assertFalse(np.array_equal(before.s21, after.s21)) + np.testing.assert_array_equal(before.s11, after.s11) + + def test_motion_uses_elapsed_time_and_returns_after_eight_seconds(self) -> None: + tx = {0: (-1., 0., 0.), 1: (1., 0., 0.)} + rx = {0: (-.4, .1, .2), 1: (.5, -.1, .3)} + for seconds, displacement in [(0, 0), (2, 1), (4, 0), (6, -1), (8, 0)]: + np.testing.assert_allclose( + self.service._target_position(_EPOCH_NS + seconds * 1_000_000_000, tx, rx), + [1.25 * displacement, .25, 3.75 + .5 * displacement], atol=1e-12, + ) + first = self.service.acquire_collection(1) + repeated = self.service.acquire_collection(999) + for a, b in zip(first.traces, repeated.traces): + np.testing.assert_array_equal(a.s21, b.s21) + self.clock.return_value = _EPOCH_NS + 2_000_000_000 + moved = self.service.acquire_collection(1) + self.assertFalse(np.array_equal(first.traces[0].s21, moved.traces[0].s21)) + + def test_pacing_excludes_wait_from_measurement_window(self) -> None: + current_ns = _EPOCH_NS + + def sleep(seconds: float) -> None: + nonlocal current_ns + current_ns += round(seconds * 1e9) + + self.clock.side_effect = lambda: current_ns + self.sleep.side_effect = sleep + first = self.service.acquire_collection() + current_ns += 20_000_000 + second = self.service.acquire_collection() + self.sleep.assert_called_once() + self.assertAlmostEqual(self.sleep.call_args.args[0], .03) + self.assertEqual(second.capture_start_ns - first.capture_start_ns, 50_000_000) + self.assertEqual(second.capture_start_ns, second.capture_end_ns) + + def test_full_motion_stays_inside_original_sweep_unambiguous_range(self) -> None: + # The existing 201-point 1 MHz–6 GHz profile aliases paths beyond about 5 m. + # Check its actual 4x4 geometry throughout the cycle, including both extremes. + self.config.input_switch.positions = self.config.output_switch.positions = 4 + self.config.combos = self.config.build_runtime_combos() + tx = {i: (x, 0., 0.) for i, x in enumerate([.905, -.905, .905, -.905])} + rx = {i: (x, 0., 0.) for i, x in enumerate([-.18, .485, -.49, .185])} + self.config.gpr.tx_geometry = [GprTxGeometryModel(i, *position) for i, position in tx.items()] + self.config.gpr.rx_geometry = [GprRxGeometryModel(i, *position) for i, position in rx.items()] + service = PointTargetMockService(self.config) + unambiguous_range_m = 3e8 / (2 * ((6e9 - 1e6) / (201 - 1))) + ranges = [] + for seconds in np.linspace(0, 8, 81): + target = service._target_position(_EPOCH_NS + round(seconds * 1e9), tx, rx) + for transmitter in tx.values(): + for receiver in rx.values(): + ranges.append((np.linalg.norm(target - transmitter) + np.linalg.norm(target - receiver)) / 2) + self.assertLess(max(ranges), unambiguous_range_m) + + def test_single_manual_capture_selects_actual_antenna_and_keeps_capture_time(self) -> None: + session = SequentialCaptureSession(self.config, "s21_calibration", "mock_geometry") + session.open() + try: + first = session.capture_current_combo() + second = session.capture_current_combo() + finally: + session.close() + self.assertEqual((first.combo, second.combo), (ComboKey(0, 0), ComboKey(1, 0))) + self.assertFalse(np.array_equal(first.s21, second.s21)) + self.assertEqual((second.capture_start_ns, second.capture_end_ns), (_EPOCH_NS, _EPOCH_NS)) + + def test_factories_share_scene_and_mock_widened_matrix_needs_no_hardware(self) -> None: + single = create_single_radar_service(self.config) + self.assertIsInstance(single, PointTargetMockService) + single.configure(self.config.radar.sweep) + self.assertIsInstance(create_acquisition_service(self.config), PointTargetMockService) + self.config.radar.model = RunConfigModel.LIBREVNA_MULTI_MODEL + self.config.radar.multi_device.stimulus_switch_positions = 2 + self.config.radar.multi_device.input_switch_positions = 2 + self.config.radar.multi_device.output_switch_positions = 2 + self.config.apply_device_model_constraints() + matrix = create_acquisition_service(self.config) + self.assertIsInstance(matrix, PointTargetMockService) + matrix.open() + matrix.configure(self.config.radar.sweep) + collection = matrix.acquire_collection() + matrix.close() + self.assertEqual(len(collection.traces), 64) + self.assertEqual([(t.combo.input, t.combo.output) for t in collection.traces], [(i, o) for o in range(8) for i in range(8)]) + np.testing.assert_array_equal(collection.traces[0].s21, single.acquire().s21) + + def test_multi_variant_single_capture_drains_each_variant_before_sampling(self) -> None: + radar = mock.Mock(wraps=self.service) + events = [] + radar.drain_after_switch = mock.Mock(side_effect=lambda: events.append("drain")) + radar.acquire.side_effect = lambda: (events.append("acquire"), self.service.acquire())[1] + variants = [RadarConfigVariant(Path(name), name, self.config, name) for name in ("a", "b")] + with mock.patch( + "python_app.workflows.multi_radar_capture_workflow.create_single_radar_service", + return_value=radar, + ): + session = MultiRadarSequentialCaptureSession( + base_config=self.config, kind="s21_calibration", set_name="variants", radar_variants=variants, + ) + session.open() + try: + session.capture_current_combo() + batch = session.capture_current_combo() + finally: + session.close() + self.assertEqual(events, ["drain", "acquire"] * 4) + self.assertEqual([trace.combo for trace in batch.traces], [ComboKey(1, 0)] * 2) + for trace in batch.traces: + self.assertEqual((trace.capture_start_ns, trace.capture_end_ns), (_EPOCH_NS, _EPOCH_NS)) + + def test_invalid_or_unconfigured_mock_fails_clearly(self) -> None: + with self.assertRaisesRegex(ValueError, "outside the antenna matrix"): + self.service.acquire_combo_collection(input_pos=2, output_pos=0) + with self.assertRaisesRegex(RuntimeError, "not configured"): + PointTargetMockService(self.config).acquire_collection() + self.config.input_switch.positions = 0 + self.config.combos = [] + with self.assertRaisesRegex(ValueError, "Switch positions must be positive integers"): + PointTargetMockService(self.config) diff --git a/python_app/tests/test_processor_history.py b/python_app/tests/test_processor_history.py index cff24e3..ac8b617 100644 --- a/python_app/tests/test_processor_history.py +++ b/python_app/tests/test_processor_history.py @@ -39,17 +39,24 @@ def _sweep(collection_id: int, monotonic_ns: int) -> SweepCollection: frequency_hz=np.linspace(1e9, 6e9, points, dtype=np.float32), s11=np.zeros(points, dtype=np.complex64), s21=np.full(points, collection_id, dtype=np.complex64), + capture_start_ns=monotonic_ns, + capture_end_ns=monotonic_ns + 1000, ) - return SweepCollection(collection_id=collection_id, monotonic_ns=monotonic_ns, traces=[trace]) + return SweepCollection(collection_id=collection_id, monotonic_ns=monotonic_ns, traces=[trace], + capture_start_ns=monotonic_ns, capture_end_ns=monotonic_ns + 1000) @unittest.skipUnless(_PROCESSOR.exists(), "data_processor is not built (run make)") -class FedBackSweepsTest(unittest.TestCase): +class _ProcessorTestCase(unittest.TestCase): + result_slot_size_bytes: int | None = None + def setUp(self) -> None: workdir = Path(tempfile.mkdtemp(prefix="processor_history_")) config = json.loads((_ROOT / "run_config.json").read_text(encoding="utf-8")) for name, ring in config["rings"].items(): ring["name"] = f"/test_processor_history_{os.getpid()}_{name}" + if self.result_slot_size_bytes is not None: + config["rings"]["results"]["slot_size_bytes"] = self.result_slot_size_bytes config["run"]["combos"] = [{"input": 0, "output": 0}] config["run"]["processing_live_config_path"] = str(workdir / "processing_live.json") config["run"]["locator_server"]["enabled"] = False @@ -61,11 +68,14 @@ class FedBackSweepsTest(unittest.TestCase): rings = config["rings"] shm_files = [Path("/dev/shm" + ring["name"]) for ring in rings.values()] + self._stderr_path = workdir / "processor.log" + stderr_log = self._stderr_path.open("w") + self.addCleanup(stderr_log.close) processor = subprocess.Popen( [str(_PROCESSOR), "--config", str(config_path)], cwd=_ROOT, stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + stderr=stderr_log, ) def cleanup() -> None: @@ -78,6 +88,7 @@ class FedBackSweepsTest(unittest.TestCase): workdir.rmdir() self.addCleanup(cleanup) + self._processor = processor # The processor creates both rings at start-up, the results ring last. self._results = ShmRingReader(rings["results"]["name"], open_timeout_s=_TIMEOUT_S) self.addCleanup(self._results.close) @@ -103,6 +114,22 @@ class FedBackSweepsTest(unittest.TestCase): deadline = min(deadline, time.monotonic() + 0.3) return ids + def _result_for_revision(self, revision: int, collection_id: int | None = None): + deadline = time.monotonic() + _TIMEOUT_S + while time.monotonic() < deadline: + result = self._results.pop_result_collection() + if result is not None: + payload = next((item for item in result.collection_payloads + if item.processing_name == "processing_revision"), None) + if payload is not None: + actual = sum(int(word) << (index * 16) for index, word in enumerate(payload.table[0])) + if actual == revision and (collection_id is None or result.collection_id == collection_id): + return result + time.sleep(0.01) + self.fail(f"No processor result for settings revision {revision}: {self._stderr_path.read_text()}") + + +class FedBackSweepsTest(_ProcessorTestCase): def test_fed_back_sweeps_are_processed_but_not_retained(self) -> None: start_ns = time.monotonic_ns() sweeps = [_sweep(index, start_ns + index * 1_000_000) for index in range(1, _SWEEPS + 1)] @@ -119,6 +146,139 @@ class FedBackSweepsTest(unittest.TestCase): self._live_writer.write(self._live) self.assertEqual(self._published_ids(_SWEEPS), expected) + def test_live_geometry_recomputes_retained_gpr_without_restarting(self) -> None: + self._live.processor_mode = "gpr" + self._live.gpr_input_positions = [0] + self._live.gpr_output_positions = [0] + self._live.gpr_background_subtract_enabled = False + self._live.tx_geometry = [{"output_pos": 0, "x_m": 0.0, "y_m": 0.0, "z_m": 0.0}] + self._live.rx_geometry = [{"input_pos": 0, "x_m": 0.0, "y_m": 0.0, "z_m": 0.0}] + self._live.settings_revision = (1 << 63) + 987654321 + self._live_writer.write(self._live) + self._push([_sweep(1, time.monotonic_ns())]) + original = self._result_for_revision(self._live.settings_revision) + original_image = next(payload for payload in original.collection_payloads if payload.processing_name == "gpr_accumulator") + # A malformed live file retains the last accepted geometry and revision. + accepted_revision = self._live.settings_revision + self._live.settings_revision += 1 + self._live.tx_geometry.append(dict(self._live.tx_geometry[0])) + self._live_writer.write(self._live) + self._push([_sweep(2, time.monotonic_ns())]) + retained = self._result_for_revision(accepted_revision, collection_id=2) + retained_image = next(payload for payload in retained.collection_payloads if payload.processing_name == "gpr_accumulator") + np.testing.assert_array_equal(retained_image.image_x_axis, original_image.image_x_axis) + self._live.tx_geometry = self._live.tx_geometry[:1] + self._live.tx_geometry[0]["x_m"] = 2.0 + self._live.rx_geometry[0]["x_m"] = 2.0 + self._live.settings_revision += 1 + self._live_writer.write(self._live) + updated = self._result_for_revision(self._live.settings_revision) + updated_image = next(payload for payload in updated.collection_payloads if payload.processing_name == "gpr_accumulator") + self.assertEqual(updated.collection_id, retained.collection_id) + self.assertEqual(updated.monotonic_ns, retained.monotonic_ns) + np.testing.assert_allclose(updated_image.image_x_axis, original_image.image_x_axis + 2.0, atol=1e-6) + self.assertIsNone(self._processor.poll()) + + +class OversizedResultsTest(_ProcessorTestCase): + result_slot_size_bytes = 4096 + + def test_valid_mode_recovers_after_an_oversized_result(self) -> None: + self._live.processor_mode = "gpr" + self._live.gpr_input_positions = [0] + self._live.gpr_output_positions = [0] + self._live.gpr_background_subtract_enabled = False + self._live.settings_revision = 17 + self._live_writer.write(self._live) + self._push([_sweep(1, time.monotonic_ns())]) + deadline = time.monotonic() + _TIMEOUT_S + while "requires" not in self._stderr_path.read_text(): + self.assertIsNone(self._processor.poll(), self._stderr_path.read_text()) + self.assertLess(time.monotonic(), deadline, self._stderr_path.read_text()) + time.sleep(0.01) + self.assertIn("increase rings.results.slot_size_bytes", self._stderr_path.read_text()) + self._live.processor_mode = "pass_through" + self._live.settings_revision = 18 + self._live_writer.write(self._live) + self.assertEqual(self._result_for_revision(18).collection_id, 1) + self._push([_sweep(2, time.monotonic_ns())]) + self.assertEqual(self._result_for_revision(18, collection_id=2).collection_id, 2) + self.assertIsNone(self._processor.poll()) + + +class Sar3dRecoveryTest(_ProcessorTestCase): + def test_sar3d_failure_recovers_without_restarting(self) -> None: + self._live.processor_mode = "sar3d" + self._live.gpr_input_positions = [0] + self._live.gpr_output_positions = [0] + self._live.gpr_background_subtract_enabled = False + self._live.gpr_min_depth_m = 2.0 + self._live.gpr_max_depth_m = 5.0 + self._live.gpr_speed_m_s = 0.0 + self._live.gpr_start_freq_mhz = 1500.0 + self._live.gpr_stop_freq_mhz = 6000.0 + self._live.sar3d_voxel_m = 0.4 + self._live.sar3d_frame_radius = 0 + self._live.tx_geometry = [{"output_pos": 0, "x_m": -0.3, "y_m": 0.0, "z_m": 0.0}] + self._live.rx_geometry = [{"input_pos": 0, "x_m": 0.2, "y_m": 0.4, "z_m": 0.0}] + self._live.settings_revision = 31 + self._live_writer.write(self._live) + sweep = _sweep(1, time.monotonic_ns()) + # df=25 MHz gives a 6 m unambiguous one-way range. The history tests' + # 64-point sweep aliases before this test's 2..5 m depth gate. + sweep.traces[0].frequency_hz = np.linspace(1e9, 6e9, 201, dtype=np.float32) + sweep.traces[0].s11 = np.zeros(201, dtype=np.complex64) + sweep.traces[0].s21 = np.exp(-4j * np.pi * sweep.traces[0].frequency_hz * 4.0 / 3e8).astype(np.complex64) + self._push([sweep]) + + def ready_images(result): + payloads = {item.processing_name: item for item in result.collection_payloads} + self.assertEqual(payloads["sar3d_status"].table[0, 0], 0) + images = [payloads[name].image for name in ("sar3d_front", "sar3d_side", "sar3d_top", "sar3d_oblique")] + for image in images: + self.assertTrue(np.all(np.isfinite(image))) + self.assertGreater(float(image.max()), 0.0) + self.assertIn("sar3d_peak", payloads) + return images + + first = self._result_for_revision(31) + status = next(item for item in first.collection_payloads if item.processing_name == "sar3d_status") + if status.table[0, 0] == 3: + self.assertIn("CUDA", bytes(map(int, status.table[0, 1:])).decode("utf-8")) + self.assertFalse(any(item.processing_name in {"sar3d_front", "sar3d_side", "sar3d_top", "sar3d_oblique"} + for item in first.collection_payloads)) + self._push([_sweep(2, sweep.monotonic_ns + 1_000_000)]) + repeated = self._result_for_revision(31, collection_id=2) + repeated_status = next(item for item in repeated.collection_payloads if item.processing_name == "sar3d_status") + np.testing.assert_array_equal(status.table, repeated_status.table) + self._live.processor_mode = "pass_through" + self._live.settings_revision = 34 + self._live_writer.write(self._live) + recovered = self._result_for_revision(34, collection_id=2) + self.assertEqual(len(recovered.blocks), 1) + self.assertIsNone(self._processor.poll()) + return + original = ready_images(first) + self._live.gpr_start_freq_mhz = 8000.0 + self._live.gpr_stop_freq_mhz = 9000.0 + self._live.settings_revision = 32 + self._live_writer.write(self._live) + failed = self._result_for_revision(32) + status = next(item for item in failed.collection_payloads if item.processing_name == "sar3d_status") + self.assertEqual(status.table[0, 0], 2) + self.assertIn("frequency points", bytes(map(int, status.table[0, 1:])).decode("utf-8")) + self.assertIsNone(self._processor.poll()) + + self._live.gpr_start_freq_mhz = 1500.0 + self._live.gpr_stop_freq_mhz = 6000.0 + self._live.settings_revision = 33 + self._live_writer.write(self._live) + recovered = self._result_for_revision(33) + self.assertEqual(recovered.collection_id, sweep.collection_id) + for before, after in zip(original, ready_images(recovered)): + np.testing.assert_array_equal(before, after) + self.assertIsNone(self._processor.poll()) + if __name__ == "__main__": unittest.main() diff --git a/python_app/tests/test_recording_mixin.py b/python_app/tests/test_recording_mixin.py index 72dd4fd..2f88dbd 100644 --- a/python_app/tests/test_recording_mixin.py +++ b/python_app/tests/test_recording_mixin.py @@ -1,8 +1,8 @@ """Tests for the streaming disk-recording mixin. -The recorder pairs each result with its raw/preprocessed collection by id, buffers a -small chunk, and flushes it to a stream writer — so memory stays flat regardless of how -many measurements are recorded. These tests pin that behaviour (chunking, finish-at-N, +The recorder preserves each received stage independently, buffers a small chunk, +and flushes it to a stream writer — so memory stays flat regardless of how many +measurements are recorded. These tests pin that behaviour (chunking, finish-at-N, arm guards, only-new gating) against light stubs of the AppWindow collaborators — no Qt, hardware, or disk needed. """ @@ -10,11 +10,17 @@ hardware, or disk needed. from __future__ import annotations from pathlib import Path +import threading +import tempfile import unittest from unittest import mock +import numpy as np + from python_app.gui.controllers import app_window_recording_mixin as rec from python_app.gui.controllers.app_window_recording_mixin import AppWindowRecordingMixin +from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData +from python_app.storage.npz.store import NpzStore class _Collection: @@ -112,6 +118,7 @@ class _Harness(AppWindowRecordingMixin): def _start_run(self) -> None: self.started = True + self._supervisor._running = True def _show_error(self, message, *, details=None) -> None: self.errors.append(message) @@ -148,6 +155,46 @@ class _RecordingTestBase(unittest.TestCase): class RecordingArmTest(_RecordingTestBase): + def test_writer_thread_start_failure_cleans_new_dataset_and_allows_retry(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + h = self._make(count=5, running=True) + h._store = NpzStore(Path(temporary) / "sets") + h._save_path_input = _Text(str(Path(temporary) / "datasets")) + destination = Path(temporary) / "datasets/pref_run" + + def write_profile(path, allow_overwrite): + path.write_text("fresh profile", encoding="utf-8") + + with mock.patch.object(h, "_write_gui_profile_to_path", side_effect=write_profile): + with mock.patch.object(rec.threading.Thread, "start", side_effect=RuntimeError("cannot start new thread")): + h._start_run_with_recording() + self.assertFalse(destination.exists()) + self.assertFalse(h._recording_active) + self.assertFalse(h._recording_collecting) + self.assertIsNone(h._recording_thread) + self.assertIsNone(h._recording_writer) + self.assertIsNone(h._recording_queue) + self.assertEqual(h._recording_target, 0) + self.assertEqual(len(h.exceptions), 1) + h._start_run_with_recording() + self.assertTrue(h._recording_active) + self.assertEqual((destination / "config_profile.json").read_text(), "fresh profile") + + def test_existing_profile_is_preserved_when_recording_destination_collides(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + h = self._make(count=5, running=True) + h._store = NpzStore(Path(temporary) / "sets") + h._save_path_input = _Text(str(Path(temporary) / "datasets")) + destination = Path(temporary) / "datasets/pref_run" + destination.mkdir(parents=True) + profile = destination / "config_profile.json" + profile.write_bytes(b"existing profile") + with mock.patch.object(rec.threading.Thread, "start", side_effect=RuntimeError("cannot start new thread")) as start: + h._start_run_with_recording() + start.assert_not_called() + self.assertEqual(profile.read_bytes(), b"existing profile") + self.assertFalse(h._recording_active) + def test_arm_starts_a_stopped_run_and_opens_a_stream(self) -> None: h = self._make(count=3, running=False) h._start_run_with_recording() @@ -178,6 +225,43 @@ class RecordingArmTest(_RecordingTestBase): self.assertEqual(len(h.errors), 1) self.assertTrue(h._recording_active) + def test_failed_pipeline_start_does_not_leave_an_empty_recorder_armed(self) -> None: + h = self._make(count=5, running=False) + with mock.patch.object(h, '_start_run'), mock.patch.object(rec.shutil, 'rmtree') as remove: + h._start_run_with_recording() + self.assertFalse(h._recording_active) + self.assertFalse(h._recording_collecting) + self.assertIsNone(h._recording_thread) + remove.assert_called_once_with(h._store.writer.directory) + self.assertTrue(any('acquisition did not start' in message for message in h.logs)) + + def test_failed_pipeline_start_retains_received_raw_data_and_profile(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + h = self._make(count=5, running=False) + h._store = NpzStore(Path(temporary) / 'sets') + h._save_path_input = _Text(str(Path(temporary) / 'datasets')) + samples = np.array([1 + 2j], dtype=np.complex64) + + def failed_start(): + h._record_collection('raw', SweepCollection( + collection_id=1, monotonic_ns=h._recording_since_ns + 1, + traces=[TraceData(ComboKey(0, 0), np.array([1e9], dtype=np.float32), samples, samples)], + )) + + def write_profile(path, allow_overwrite): + path.write_text('retained configuration', encoding='utf-8') + + with mock.patch.object(h, '_start_run', side_effect=failed_start), mock.patch.object( + h, '_write_gui_profile_to_path', side_effect=write_profile, + ): + h._start_run_with_recording() + destination = Path(temporary) / 'datasets/pref_run' + self.assertFalse(h._recording_active) + self.assertEqual((destination / 'config_profile.json').read_text(), 'retained configuration') + files = list((destination / 'raw').glob('*/i0_o0_s21.npy')) + self.assertEqual(len(files), 1) + np.testing.assert_array_equal(np.load(files[0]), samples) + class RecordingStreamTest(_RecordingTestBase): def test_ignores_collections_from_before_arming(self) -> None: @@ -208,8 +292,8 @@ class RecordingStreamTest(_RecordingTestBase): for cid in range(1, 6): h.feed(cid) self._drain_and_finalize(h) - # 2 + 2 + 1: flushed at the chunk boundaries and the final remainder. - self.assertEqual(writer.appends, [(2, 2, 2), (2, 2, 2), (1, 1, 1)]) + self.assertEqual(tuple(sum(chunk[stage] for chunk in writer.appends) for stage in range(3)), (5, 5, 5)) + self.assertTrue(all(max(chunk) <= 2 for chunk in writer.appends)) self.assertEqual(writer.result_count, 5) self.assertFalse(h._recording_active) @@ -252,13 +336,104 @@ class RecordingStreamTest(_RecordingTestBase): self.assertEqual(len(h._store.created), 0) self.assertFalse(h._recording_active) - def test_pending_maps_are_bounded(self) -> None: - with mock.patch.object(rec, "_RECORDING_MAX_PENDING", 4): + def test_raw_stage_streams_when_processor_has_not_returned_any_results(self) -> None: + with mock.patch.object(rec, "_RECORDING_CHUNK_SIZE", 4): h = self._make(count=1000, running=True) h._start_run_with_recording() for cid in range(50): # raws whose results never arrive h._record_collection("raw", _Collection(cid, monotonic_ns=h._recording_since_ns + 1)) - self.assertLessEqual(len(h._recording_pending_raw), 4) + self.assertLess(len(h._recording_chunk_raw), 4) + h._finalize_recording_on_stop() + self.assertEqual(sum(chunk[0] for chunk in h._store.writer.appends), 50) + self.assertEqual(h._store.writer.result_count, 0) + + def test_shutdown_flushes_data_without_results(self) -> None: + h = self._make(count=1000, running=True) + h._start_run_with_recording() + h._record_collection("raw", _Collection(1, h._recording_since_ns + 1)) + h._record_collection("preprocessed", _Collection(1, h._recording_since_ns + 2)) + h._shutdown_recording() + self.assertEqual(h._store.writer.appends, [(1, 1, 0)]) + self.assertFalse(h._recording_active) + + def test_full_queue_does_not_drop_the_finish_sentinel(self) -> None: + with mock.patch.object(rec, "_RECORDING_QUEUE_CHUNKS", 1), mock.patch.object(rec, "_RECORDING_CHUNK_SIZE", 1): + h = self._make(count=1000, running=True) + h._start_run_with_recording() + entered, release = threading.Event(), threading.Event() + self.addCleanup(release.set) + append = h._store.writer.append + + def slow_append(*args): + entered.set() + release.wait(timeout=2) + append(*args) + + h._store.writer.append = slow_append + h._record_collection("raw", _Collection(1, h._recording_since_ns + 1)) + self.assertTrue(entered.wait(timeout=1)) + h._record_collection("preprocessed", _Collection(1, h._recording_since_ns + 2)) + self.assertTrue(h._recording_queue.full()) + timer = threading.Timer(0.05, release.set) + timer.start() + h._stop_collecting() + timer.join(timeout=1) + self._drain_and_finalize(h) + self.assertEqual(h._store.writer.appends, [(1, 0, 0), (0, 1, 0)]) + + def test_stop_keeps_a_slow_writer_alive_until_accepted_queue_is_written(self) -> None: + h = self._make(count=1000, running=True) + h._start_run_with_recording() + entered, release = threading.Event(), threading.Event() + self.addCleanup(release.set) + append = h._store.writer.append + + def slow_append(*args): + entered.set() + release.wait(timeout=2) + append(*args) + + h._store.writer.append = slow_append + h.feed(1) + h._flush_recording_chunk() + self.assertTrue(entered.wait(timeout=1)) + h.feed(2) + thread = h._recording_thread + with mock.patch.object(thread, 'join'): + h._finalize_recording_on_stop() + self.assertTrue(h._recording_active) + self.assertIs(h._recording_thread, thread) + self.assertFalse(h._recording_stop.is_set()) + release.set() + self._drain_and_finalize(h) + self.assertEqual(h._store.writer.result_count, 2) + + def test_write_failure_with_full_queue_does_not_block_the_producer(self) -> None: + with mock.patch.object(rec, "_RECORDING_QUEUE_CHUNKS", 1), mock.patch.object(rec, "_RECORDING_CHUNK_SIZE", 1): + h = self._make(count=1000, running=True) + h._start_run_with_recording() + entered, release = threading.Event(), threading.Event() + self.addCleanup(release.set) + + def fail_append(*args): + entered.set() + release.wait(timeout=2) + raise OSError('disk full') + + h._store.writer.append = fail_append + h._record_collection("raw", _Collection(1, h._recording_since_ns + 1)) + self.assertTrue(entered.wait(timeout=1)) + h._record_collection("preprocessed", _Collection(1, h._recording_since_ns + 2)) + producer = threading.Thread( + target=h._record_collection, + args=("results", _Collection(1, h._recording_since_ns + 3)), daemon=True, + ) + producer.start() + release.set() + producer.join(timeout=1) + self.assertFalse(producer.is_alive()) + self.assertFalse(h._recording_active) + self.assertEqual(len(h.errors), 1) if __name__ == "__main__": diff --git a/python_app/tests/test_run_config_codec.py b/python_app/tests/test_run_config_codec.py index e8ba92e..f1fa474 100644 --- a/python_app/tests/test_run_config_codec.py +++ b/python_app/tests/test_run_config_codec.py @@ -25,6 +25,7 @@ from python_app.models.run_config_schema import ( ) from python_app.models.run_config_validation import ( parse_combos_from_text, + validate_acquisition_config, validate_combos, validate_gpr_model, validate_ring_endpoint, @@ -78,6 +79,11 @@ class StrictNumericTypingTest(unittest.TestCase): class StructuralTypingTest(unittest.TestCase): """A present-but-wrong-shaped section fails loudly instead of being dropped.""" + def test_root_must_be_an_object(self) -> None: + for payload in (None, [], 'bad', 42): + with self.subTest(payload=payload), self.assertRaisesRegex(ValueError, 'root'): + RunConfigModel.from_dict(payload) + def test_combos_must_be_an_array(self) -> None: payload = _valid_payload() payload["run"]["combos"] = {"input": 0, "output": 0} @@ -85,7 +91,103 @@ class StructuralTypingTest(unittest.TestCase): RunConfigModel.from_dict(payload) +class AcquisitionValidationTest(unittest.TestCase): + def test_default_combo_expansion_is_bounded_before_allocation(self) -> None: + payload = _valid_payload() + payload["switches"]["port1"]["positions"] = 65536 + payload["run"]["combos"] = [] + with self.assertRaisesRegex(ValueError, "Too many combos"): + RunConfigModel.from_dict(payload) + config = self.config("librevna_multi") + config.radar.multi_device.input_switch_positions = 1 << 32 + with self.assertRaisesRegex(ValueError, "Too many combos"): + config.apply_device_model_constraints() + + def config(self, model: str = "librevna") -> RunConfigModel: + config = RunConfigModel() + config.radar.model = model + config.radar.sweep.points = 32 + for port, switch in enumerate((config.output_switch, config.input_switch), 1): + switch.radar_port = port + switch.driver = "h7992" + switch.gpio_chip = "/dev/gpiochip0" + switch.pin_a, switch.pin_b = 17, 27 + for name in ("raw", "preprocessed", "raw_tap", "preprocessed_tap"): + getattr(config.rings, name).slot_size_bytes = 1 << 20 + config.ensure_combos() + return config + + def test_undersized_trace_rings_fail_before_start(self) -> None: + config = self.config("librevna_multi") + for name in ("raw", "raw_tap", "preprocessed", "preprocessed_tap"): + ring = getattr(config.rings, name) + ring.name = "/test" + original = ring.slot_size_bytes + ring.slot_size_bytes = 4096 + with self.subTest(ring=name), self.assertRaisesRegex(ValueError, f"rings.{name}"): + validate_acquisition_config(config) + ring.slot_size_bytes = original + validate_acquisition_config(config) + + def test_kamil_requires_native_mode_except_for_replay(self) -> None: + config = self.config("kamil_adc") + with self.assertRaisesRegex(ValueError, "native"): + validate_acquisition_config(config) + config.radar.replay.dataset_dir = "recorded" + validate_acquisition_config(config) + + def test_matrix_validates_physical_switch_instead_of_virtual_axis(self) -> None: + config = self.config("librevna_multi") + config.radar.driver_mode = "native" + config.radar.multi_device.output_switch_positions = 4 + config.radar.multi_device.input_switch_positions = 4 + config.ensure_combos() + self.assertEqual(config.output_switch.positions, 8) + self.assertEqual(config.input_switch.positions, 16) + validate_acquisition_config(config) + config.radar.multi_device.output_switch_positions = 5 + config.ensure_combos() + with self.assertRaisesRegex(ValueError, "supports 1..4"): + validate_acquisition_config(config) + config.radar.driver_mode = "mock" + validate_acquisition_config(config) + + def test_bad_switch_settings_are_reported_before_hardware_open(self) -> None: + for field, value in ( + ("driver_mode", "typo"), ("driver", "typo"), ("default_position", 1), + ("radar_port", 0), ("pin_a", -1), ("pin_b", 17), ("gpio_chip", ""), + ): + config = self.config() + config.output_switch.driver_mode = "native" + setattr(config.output_switch, field, value) + with self.subTest(field=field), self.assertRaises(ValueError): + validate_acquisition_config(config) + + +class NotchValidationTest(unittest.TestCase): + def test_malformed_bands_and_tapers_match_native_rejection(self) -> None: + for band in (("1", 2), (True, 2), (None, 2), (1, float("nan")), (2, 1), (1, 1e100)): + payload = _valid_payload() + payload["preprocess"]["notch"]["bands_hz"] = [list(band)] + with self.subTest(band=band), self.assertRaises(ValueError): + RunConfigModel.from_dict(payload) + for field, value in (("taper_width_hz", -1), ("taper_type", "typo")): + payload = _valid_payload() + payload["preprocess"]["notch"][field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + RunConfigModel.from_dict(payload) + + class SweepValidationTest(unittest.TestCase): + def test_nonfinite_fields_and_wire_overflow_are_rejected(self) -> None: + for field in ('start_hz', 'stop_hz', 'if_bandwidth_hz', 'power_dbm'): + for value in (float('nan'), float('inf')): + sweep = RadarSweepModel() + setattr(sweep, field, value) + with self.subTest(field=field, value=value), self.assertRaisesRegex(ValueError, field): + validate_sweep_model(sweep) + with self.assertRaisesRegex(ValueError, 'points'): + validate_sweep_model(RadarSweepModel(points=2 ** 32)) def test_points_must_be_positive(self) -> None: for points in (0, -1): with self.subTest(points=points), self.assertRaisesRegex(ValueError, "points"): @@ -102,6 +204,10 @@ class SweepValidationTest(unittest.TestCase): class ComboValidationTest(unittest.TestCase): + def test_direct_model_positions_cannot_be_silently_truncated(self) -> None: + for value in (0.5, True, '0'): + with self.subTest(value=value), self.assertRaisesRegex(ValueError, 'integer'): + validate_combos([ComboModel(value, 0)], input_positions=2, output_positions=2) def test_within_bounds_passes(self) -> None: validate_combos( [ComboModel(input=0, output=0), ComboModel(input=3, output=1)], @@ -136,6 +242,9 @@ class ComboValidationTest(unittest.TestCase): class RingValidationTest(unittest.TestCase): + def test_slot_stride_must_keep_atomic_fields_aligned(self) -> None: + with self.assertRaisesRegex(ValueError, 'multiple of 8'): + validate_ring_endpoint(RingEndpointModel(name='r', slot_size_bytes=9)) def test_capacity_and_slot_must_be_positive(self) -> None: with self.assertRaisesRegex(ValueError, "capacity"): validate_ring_endpoint(RingEndpointModel(name="r", capacity=0, slot_size_bytes=16)) @@ -189,6 +298,11 @@ class GprValidationTest(unittest.TestCase): rx_geometry=[GprRxGeometryModel(input_pos=0, x_m=0.0, y_m=0.0, z_m=0.0)], )) + def test_direct_geometry_positions_cannot_be_silently_truncated(self) -> None: + for value in (0.5, True, '0'): + with self.subTest(value=value), self.assertRaisesRegex(ValueError, 'integer'): + self._validate(self._gpr(tx_geometry=[GprTxGeometryModel(output_pos=value)])) + class ParseCombosFromTextTest(unittest.TestCase): def test_parses_pairs(self) -> None: diff --git a/python_app/tests/test_sar3d_plot.py b/python_app/tests/test_sar3d_plot.py new file mode 100644 index 0000000..0da19bd --- /dev/null +++ b/python_app/tests/test_sar3d_plot.py @@ -0,0 +1,157 @@ +"""SAR3D projection rendering and visible processor failures, without live resources.""" + +from __future__ import annotations + +import os +from types import SimpleNamespace +import unittest +from unittest import mock + +import numpy as np + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") +from PyQt6.QtCore import Qt # noqa: E402 +from PyQt6.QtWidgets import QApplication, QStackedWidget # noqa: E402 + +from python_app.gui.controllers.app_window_pipeline_mixin import AppWindowPipelineMixin # noqa: E402 +from python_app.gui.controllers.app_window_config.live_processing_mixin import AppWindowLiveProcessingMixin # noqa: E402 +from python_app.gui.controllers.app_window_plot_mixin import AppWindowPlotMixin # noqa: E402 +from python_app.gui.controllers.app_window_plot.sar3d_plot_mixin import SAR3D_VIEWS, sar3d_status # noqa: E402 +from python_app.gui.controllers.app_window_ui_mixin import AppWindowUiMixin # noqa: E402 +from python_app.models.dataset_model import ResultCollection, ResultPayload # noqa: E402 + + +def _status(code: int, message: str) -> ResultPayload: + return ResultPayload(processing_name="sar3d_status", kind=4, + table=np.array([[code, *message.encode("utf-8")]], dtype=np.float32)) + + +def _collection(*payloads: ResultPayload) -> ResultCollection: + return ResultCollection(collection_id=5, monotonic_ns=1500, collection_payloads=list(payloads)) + + +def _projections() -> ResultCollection: + x, y, z = np.array([-1, 0, 1]), np.array([-0.5, 0.5]), np.array([1, 2, 3]) + axes = [(x, y), (z, y), (x, z), (x, y)] + images = [ResultPayload(processing_name=name, kind=3, image_x_axis=xx, image_y_axis=yy, + image=np.ones((len(yy), len(xx)), dtype=np.float32)) + for name, (xx, yy) in zip(SAR3D_VIEWS, axes)] + return _collection(*images, _status(0, "CUDA"), + ResultPayload(processing_name="sar3d_oblique_basis", kind=4, + table=np.array([[1, 0, 0], [0, 1, 0]], dtype=np.float32)), + ResultPayload(processing_name="sar3d_peak", kind=4, + table=np.array([[2, 0.5, -0.2, 2]], dtype=np.float32))) + + +class _View(AppWindowPlotMixin, AppWindowUiMixin, AppWindowPipelineMixin, AppWindowLiveProcessingMixin): + def __init__(self) -> None: + self._processing_mode = SimpleNamespace(currentText=lambda: "sar3d") + self._plot_stack = QStackedWidget() + self._build_sar3d_plot_page() + + +class Sar3dPlotTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.app = QApplication.instance() or QApplication([]) + + def setUp(self) -> None: + self.view = _View() + self.addCleanup(self.view._plot_stack.close) + + def test_all_four_projections_have_visible_images_and_correct_peak_axes(self) -> None: + self.assertTrue(self.view._draw_results(_projections())) + self.assertEqual(self.view._sar3d_status_label.text(), "SAR3D: ready — CUDA") + positions = {"sar3d_front": (0.5, -0.2), "sar3d_side": (2, -0.2), + "sar3d_top": (0.5, 2), "sar3d_oblique": (0.5, -0.2)} + for name, position in positions.items(): + image = self.view._sar3d_images[name] + self.assertTrue(image.isVisible(), name) + self.assertGreater(image.boundingRect().width(), 0) + self.assertGreater(image.boundingRect().height(), 0) + xx, yy = self.view._sar3d_peaks[name].getData() + np.testing.assert_allclose([xx[0], yy[0]], position) + self.assertTrue(all(label.isVisible() for label in self.view._sar3d_axis_labels.values())) + + def test_unavailable_warning_clears_previous_images_and_labels_and_shows_reason(self) -> None: + self.view._draw_results(_projections()) + message = "CUDA unavailable: драйвер " + self.assertFalse(self.view._draw_results(_collection(_status(3, message)))) + self.assertEqual(self.view._sar3d_status_label.text(), f"SAR3D: warning — {message}") + self.assertEqual(self.view._sar3d_status_label.textFormat(), Qt.TextFormat.PlainText) + self.assertTrue(all(not image.isVisible() for image in self.view._sar3d_images.values())) + self.assertTrue(all(not label.isVisible() for label in self.view._sar3d_axis_labels.values())) + + def test_no_signal_and_missing_projections_explain_empty_panels(self) -> None: + self.assertFalse(self.view._draw_results(_collection(_status(1, "No signal in selected range")))) + self.assertIn("no signal", self.view._sar3d_status_label.text()) + self.assertFalse(self.view._draw_results(_collection())) + self.assertEqual(self.view._sar3d_status_label.text(), "SAR3D: waiting for projection data") + + def test_ready_cuda_status_and_clear_restores_waiting(self) -> None: + self.assertTrue(self.view._draw_results(_projections())) + self.assertEqual(self.view._sar3d_status_label.text(), "SAR3D: ready — CUDA") + self.view._clear_sar3d_plot() + self.assertEqual(self.view._sar3d_status_label.text(), "SAR3D: waiting for measurements") + + def test_malformed_status_is_a_visible_error_instead_of_a_qt_exception(self) -> None: + for table in (np.empty((0, 0)), np.array([[2, np.nan]]), np.array([[2, 256]]), + np.array([[2, 0.5]]), np.array([[99]]), np.array([2])): + with self.subTest(table=table): + payload = _status(2, "failure") + payload.table = table + self.assertEqual(sar3d_status(_collection(payload)), (2, "Invalid SAR3D status payload")) + self.assertFalse(self.view._draw_results(_collection(payload))) + + def test_single_capture_finishes_reported_failure_without_losing_its_result(self) -> None: + for code in (1, 2, 3): + with self.subTest(code=code): + result = _collection(_status(code, "processor explanation")) + self.view._result_history = [result] + self.view._single_capture_active = True + self.view._single_capture_start_ns = 1000 + self.view._single_capture_seen_raw = True + self.view._single_capture_target_collection_id = 5 + self.view._log = mock.Mock() + self.view._log_error = mock.Mock() + self.view._log_warning = mock.Mock() + self.view._status_label = mock.Mock() + self.view._stop_run = mock.Mock(side_effect=lambda: setattr(self.view, "_single_capture_active", False)) + self.assertTrue(self.view._finish_single_capture_if_ready()) + self.assertFalse(self.view._single_capture_active) + self.assertEqual(self.view._result_history, [result]) + self.assertIn("processor explanation", self.view._sar3d_status_label.text()) + state = {1: "no signal in selected range", 2: "error", 3: "warning"}[code] + self.view._status_label.setText.assert_called_once_with(f"Status: {state}") + log = self.view._log_error if code == 2 else self.view._log_warning + log.assert_called_once_with( + f"Single capture: SAR3D {state}: processor explanation") + + def test_live_edit_retains_failure_reason_then_recovers_to_ready_images(self) -> None: + self.view._result_history = [] + self.view._write_live_processing_config = mock.Mock() + self.view._show_exception = mock.Mock() + for result, expected in ((_collection(_status(2, "CUDA kernel failure")), "error"), + (_collection(_status(3, "Driver initialization unavailable")), "warning"), + (_collection(_status(2, "fewer than two frequency points")), "error"), + (_collection(_status(1, "No signal in selected depth range")), "no signal"), + (_projections(), "ready — CUDA")): + self.view._drain_results_until_quiet = mock.Mock(side_effect=[None, result]) + self.view._on_processing_live_settings_changed() + self.assertIn(expected, self.view._sar3d_status_label.text()) + self.view._show_exception.assert_not_called() + self.assertTrue(all(image.isVisible() for image in self.view._sar3d_images.values())) + + def test_history_redraw_preserves_error_but_stale_revision_clears_old_images(self) -> None: + self.view._result_history = [_collection(_status(2, "processor error"))] + self.view._redraw_after_history_deletion() + self.assertEqual(self.view._sar3d_status_label.text(), "SAR3D: error — processor error") + self.view._draw_results(_projections()) + self.view._required_processing_revision = 10 + self.view._redraw_after_history_deletion() + self.assertEqual(self.view._sar3d_status_label.text(), "SAR3D: waiting for measurements") + self.assertTrue(all(not image.isVisible() for image in self.view._sar3d_images.values())) + + +if __name__ == "__main__": + unittest.main() diff --git a/python_app/tests/test_shm_ipc.py b/python_app/tests/test_shm_ipc.py index 89f4a31..d44a36b 100644 --- a/python_app/tests/test_shm_ipc.py +++ b/python_app/tests/test_shm_ipc.py @@ -13,9 +13,12 @@ Pins the agreed contract: from __future__ import annotations import struct +import threading +import time import unittest from contextlib import suppress from pathlib import Path +from unittest import mock import numpy as np @@ -119,6 +122,28 @@ class TraceCollectionRoundTripTest(unittest.TestCase): class ResultCollectionRoundTripTest(unittest.TestCase): + def test_empty_tables_and_images_retain_their_declared_dimensions(self) -> None: + for shape in ((0, 3), (2, 0), (0, 0)): + for kind in (3, 4): + with self.subTest(shape=shape, kind=kind): + payload = ResultPayload('empty', kind, table=np.empty(shape, dtype=np.float32), + image=np.empty(shape, dtype=np.float32), + image_x_axis=np.zeros(shape[1], dtype=np.float32), + image_y_axis=np.zeros(shape[0], dtype=np.float32)) + if kind == 4 and shape == (2, 0): + with self.assertRaises(ValueError): + serialize_result_collection(ResultCollection(1, 2, collection_payloads=[payload])) + continue + result = decode_result_collection(serialize_result_collection( + ResultCollection(1, 2, collection_payloads=[payload]))) + decoded = result.collection_payloads[0] + self.assertEqual((decoded.image if kind == 3 else decoded.table).shape, shape) + + def test_multidimensional_trace_cannot_be_serialized_as_a_flat_trace(self) -> None: + payload = ResultPayload('bad', 1, frequency_hz=np.ones((1, 2)), trace=np.ones((1, 2))) + with self.assertRaises(ValueError): + serialize_result_collection(ResultCollection(1, 2, collection_payloads=[payload])) + def test_all_payload_kinds_round_trip(self) -> None: image = np.arange(6, dtype=np.float32).reshape((2, 3)) table = np.array([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], dtype=np.float32) @@ -240,6 +265,26 @@ class RingRoundTripTest(_RingTestCase): class RingOverflowTest(_RingTestCase): + def test_reader_cannot_rewind_cursor_after_producer_laps_during_commit(self) -> None: + from python_app.orchestration.shm import atomics + + name = self._ring_name() + writer = self._writer(name, capacity=4, slot_size=64) + reader = self._reader(name) + writer.push(b"f0") + advance = atomics.advance + + def lap_before_reader_commit(buffer, offset, desired): + if buffer is reader._mmap and desired == 1: + for i in range(1, 6): + writer.push(f"f{i}".encode()) + advance(buffer, offset, desired) + + with mock.patch.object(atomics, "advance", side_effect=lap_before_reader_commit): + self.assertEqual(reader.pop_payload(), b"f0") + self.assertEqual(reader._read_u64(32), 2) + self.assertEqual([reader.pop_payload() for _ in range(4)], [b"f2", b"f3", b"f4", b"f5"]) + def test_latest_wins_drops_oldest(self) -> None: name = self._ring_name() writer = self._writer(name, capacity=4, slot_size=64) @@ -254,6 +299,26 @@ class RingOverflowTest(_RingTestCase): class PeekLatestTest(_RingTestCase): + def test_peek_rejects_a_slot_while_it_is_being_overwritten(self) -> None: + name = self._ring_name() + writer = self._writer(name, capacity=1, slot_size=8) + reader = self._reader(name) + writer.push(b"aaaaaaaa") + observations = [] + write_size = writer._write_u32 + + def overwrite_in_progress(offset: int, value: int) -> None: + write_size(offset, value) + if offset == 64: + writer._mmap[80:84] = b"bbbb" + observations.append(reader.peek_latest_payload()) + + with mock.patch.object(writer, "_write_u32", side_effect=overwrite_in_progress): + writer.push(b"bbbbbbbb") + + self.assertEqual(observations, [None]) + self.assertEqual(reader.peek_latest_payload(), b"bbbbbbbb") + def test_peek_is_latest_and_non_consuming(self) -> None: name = self._ring_name() writer = self._writer(name, capacity=8, slot_size=64) @@ -269,6 +334,85 @@ class PeekLatestTest(_RingTestCase): class RingOpenTest(_RingTestCase): + def test_invalid_geometry_and_names_fail_before_opening_files(self) -> None: + for capacity, slot in ((True, 8), (1.5, 8), (1, 9), (2 ** 32, 8)): + with self.subTest(capacity=capacity, slot=slot), self.assertRaises(ValueError): + ShmRingWriter(self._ring_name(), capacity, slot) + for name in ("/", "/../tmp/ring", "/nested/ring", "/bad\x00ring"): + with self.subTest(name=name): + with self.assertRaises(ValueError): + ShmRingWriter(name, 1, 8) + with self.assertRaises(ValueError): + ShmRingReader(name, open_timeout_s=0) + + def test_failed_mapping_closes_opened_file(self) -> None: + from python_app.orchestration.shm import ring_writer + + name = self._ring_name() + opened = [] + fdopen = ring_writer.os.fdopen + + def track_fd(fd, *args, **kwargs): + handle = fdopen(fd, *args, **kwargs) + opened.append(handle) + return handle + + self.addCleanup(lambda: (Path('/dev/shm') / name[1:]).unlink(missing_ok=True)) + with mock.patch.object(ring_writer.os, 'fdopen', side_effect=track_fd), \ + mock.patch.object(ring_writer.mmap, 'mmap', side_effect=OSError('mapping failed')), \ + self.assertRaises(OSError): + ShmRingWriter(name, 1, 8) + self.assertTrue(opened[0].closed) + + def test_owner_replaces_wrong_size_without_truncating_existing_mappings(self) -> None: + name = self._ring_name() + writer = self._writer(name, capacity=8, slot_size=64) + reader = self._reader(name) + writer.push(b"old frame") + path = Path("/dev/shm") / name[1:] + old_inode = path.stat().st_ino + + replacement = ShmRingWriter(name, capacity=1, slot_size_bytes=8) + self.addCleanup(self._safe_close, replacement) + + self.assertNotEqual(path.stat().st_ino, old_inode) + self.assertEqual(reader.pop_payload(), b"old frame") + self.assertTrue(replacement.push(b"new")) + self.assertEqual(self._reader(name).pop_payload(), b"new") + + def test_waits_for_created_but_unallocated_ring(self) -> None: + name = self._ring_name() + path = Path("/dev/shm") / name[1:] + path.touch() + self.addCleanup(lambda: path.unlink(missing_ok=True)) + writers = [] + + def initialize() -> None: + time.sleep(0.02) + writers.append(ShmRingWriter(name, capacity=1, slot_size_bytes=8)) + + creator = threading.Thread(target=initialize) + creator.start() + + def cleanup_creator() -> None: + creator.join() + for writer in writers: + writer.close() + + self.addCleanup(cleanup_creator) + reader = self._reader(name) + creator.join() + + self.assertEqual(reader.capacity, 1) + self.assertEqual(reader.slot_size_bytes, 8) + + def test_header_geometry_outside_mapping_is_rejected(self) -> None: + name = self._ring_name() + writer = self._writer(name, capacity=1, slot_size=8) + writer._write_u32(12, 100) + with self.assertRaisesRegex(RuntimeError, "geometry mismatch"): + ShmRingReader(name) + def test_missing_ring_fails_fast(self) -> None: with self.assertRaises(FileNotFoundError): ShmRingReader("/radar_test_definitely_missing", open_timeout_s=0.1, open_poll_s=0.02) diff --git a/python_app/tests/test_single_capture.py b/python_app/tests/test_single_capture.py index f628902..137f6b8 100644 --- a/python_app/tests/test_single_capture.py +++ b/python_app/tests/test_single_capture.py @@ -11,7 +11,6 @@ from __future__ import annotations import os import unittest -from pathlib import Path import numpy as np @@ -19,6 +18,7 @@ os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") from PyQt6.QtWidgets import QApplication # noqa: E402 from python_app.gui.app_window import AppWindow # noqa: E402 +from python_app.tests.gui_test_support import isolated_window from python_app.models.dataset_model import ( # noqa: E402 ComboKey, ResultBlock, @@ -33,7 +33,7 @@ _window: AppWindow | None = None def setUpModule() -> None: global _app, _window _app = QApplication.instance() or QApplication([]) - _window = AppWindow(Path(".")) + _window = isolated_window(unittest.addModuleCleanup) def tearDownModule() -> None: @@ -89,6 +89,15 @@ class SingleCaptureBoundaryTest(unittest.TestCase): self.assertTrue(self.w._finish_single_capture_if_ready()) self.assertFalse(self.w._single_capture_active) # _stop_run cleared the flag + def test_empty_processed_trace_completes_capture_instead_of_waiting_for_timeout(self) -> None: + result = _result(5, 1500) + payload = result.blocks[0].payloads[0] + payload.frequency_hz = np.empty(0, dtype=np.float32) + payload.trace = np.empty(0, dtype=np.complex64) + self.w._result_history.append(result) + self.assertTrue(self.w._finish_single_capture_if_ready()) + self.assertFalse(self.w._single_capture_active) + if __name__ == "__main__": unittest.main() diff --git a/python_app/tests/test_socket_speed_controls.py b/python_app/tests/test_socket_speed_controls.py index d0e6bd1..2355d6f 100644 --- a/python_app/tests/test_socket_speed_controls.py +++ b/python_app/tests/test_socket_speed_controls.py @@ -10,12 +10,12 @@ from __future__ import annotations import os import unittest -from pathlib import Path os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") from PyQt6.QtWidgets import QApplication # noqa: E402 from python_app.gui.app_window import AppWindow # noqa: E402 +from python_app.tests.gui_test_support import isolated_window _app: QApplication | None = None _window: AppWindow | None = None @@ -31,7 +31,7 @@ _PAGES = { def setUpModule() -> None: global _app, _window _app = QApplication.instance() or QApplication([]) - _window = AppWindow(Path(".")) + _window = isolated_window(unittest.addModuleCleanup) def tearDownModule() -> None: diff --git a/python_app/tests/test_storage_webui.py b/python_app/tests/test_storage_webui.py index 9b0e5df..6472692 100644 --- a/python_app/tests/test_storage_webui.py +++ b/python_app/tests/test_storage_webui.py @@ -15,16 +15,19 @@ import os import tempfile import unittest from pathlib import Path +from types import SimpleNamespace from unittest import mock import numpy as np os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") # before any Qt import from PyQt6.QtWidgets import QApplication # noqa: E402 +from fastapi import HTTPException, WebSocketDisconnect # noqa: E402 from python_app.gui.controllers.app_window_web_mixin import ( # noqa: E402 AppWindowWebController, AppWindowWebMixin, + _WebActionCall, ) from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData # noqa: E402 from python_app.storage.npz.paths import radar_config_filename_prefix # noqa: E402 @@ -35,6 +38,7 @@ from python_app.storage.npz.vna_history_json import ( # noqa: E402 build_vna_history_payload, ) from python_app.webui.controller import WebActionError # noqa: E402 +from python_app.webui.routes import post_live_settings, ws # noqa: E402 from python_app.webui.streaming import RingBroadcaster # noqa: E402 @@ -160,10 +164,24 @@ class WebControllerTest(unittest.TestCase): def test_known_field_emits_and_returns_snapshot(self) -> None: received: list[dict] = [] - self.controller.apply_settings_requested.connect(received.append) + updated = [{"name": "gpr_object_min_frac", "value": 0.5}] + + def apply(fields, call): + received.append(fields) + self.controller.update_snapshot(status={}, live_settings=updated, frame=None) + call.done.set() + + self.controller.apply_settings_requested.connect(apply) out = self.controller.apply_live_settings({"gpr_object_min_frac": 0.5}) self.assertEqual(received, [{"gpr_object_min_frac": 0.5}]) - self.assertIsInstance(out, list) + self.assertEqual(out, updated) + + def test_live_settings_error_is_raised_to_the_caller(self) -> None: + self.controller.apply_settings_requested.connect( + lambda fields, call: (setattr(call, "error", "Duplicate antenna position: 0"), call.done.set()) + ) + with self.assertRaisesRegex(WebActionError, "Duplicate antenna position"): + self.controller.apply_live_settings({"tx_geometry": []}) def test_snapshot_is_replaced_and_returned_as_copy(self) -> None: self.controller.update_snapshot(status={"running": True}, live_settings=[{"name": "x"}], frame={"seq": 1}) @@ -238,6 +256,85 @@ class WebControllerTest(unittest.TestCase): self.controller.save_dataset("/tmp/out", "run1") self.assertEqual(received, [("/tmp/out", "run1")]) + def test_combo_controls_forward_selection_and_next_to_gui(self) -> None: + received: list = [] + self.controller.select_combo_requested.connect( + lambda inp, out, call: (received.append((inp, out)), call.done.set()) + ) + self.controller.next_combo_requested.connect( + lambda call: (received.append("next"), call.done.set()) + ) + + self.controller.select_combo(0, 1) + self.controller.next_combo() + + self.assertEqual(received, [(0, 1), "next"]) + + def test_invalid_recording_count_is_rejected_without_dispatch(self) -> None: + fired = mock.Mock() + self.controller.start_recording_requested.connect(fired) + for count in (0, -1, True, 1.5, "2"): + with self.subTest(count=count), self.assertRaisesRegex(ValueError, "positive integer"): + self.controller.start_recording("/tmp", "run", count) + fired.assert_not_called() + + def test_timed_out_queued_action_cannot_run_later(self) -> None: + queued = [] + self.controller.start_requested.connect(queued.append) + with mock.patch("python_app.gui.controllers.app_window_web_mixin._WEB_ACTION_TIMEOUT_S", 0): + with self.assertRaisesRegex(WebActionError, "cancelled"): + self.controller.start() + w = AppWindowWebMixin() + action = mock.Mock() + w._run_web_action(queued[0], action) + action.assert_not_called() + self.assertFalse(self.controller._pending_calls) + + def test_timeout_reports_an_already_running_action_without_repeating_it(self) -> None: + started = [] + self.controller.start_requested.connect(lambda call: (call.begin(), started.append(call))) + with mock.patch("python_app.gui.controllers.app_window_web_mixin._WEB_ACTION_TIMEOUT_S", 0): + with self.assertRaisesRegex(WebActionError, "still running"): + self.controller.start() + self.assertFalse(started[0].done.is_set()) + started[0].finish() + self.assertTrue(started[0].done.is_set()) + + def test_shutdown_releases_waiting_requests_and_rejects_new_controls(self) -> None: + queued = [] + self.controller.start_requested.connect(lambda call: (queued.append(call), self.controller.shutdown())) + with self.assertRaisesRegex(WebActionError, "closing"): + self.controller.start() + self.assertTrue(queued[0].done.is_set()) + self.assertFalse(queued[0].begin()) + with self.assertRaisesRegex(WebActionError, "closing"): + self.controller.next_combo() + + +class WebActionLifecycleTest(unittest.TestCase): + def test_closed_window_does_not_run_queued_control(self) -> None: + w = AppWindowWebMixin() + w._closing = True + call = _WebActionCall() + action = mock.Mock() + w._run_web_action(call, action) + action.assert_not_called() + self.assertTrue(call.done.is_set()) + self.assertEqual(call.error, "The application is closing") + + def test_out_of_range_recording_count_does_not_change_destination_or_arm(self) -> None: + w = AppWindowWebMixin() + w._record_count = SimpleNamespace(minimum=lambda: 1, maximum=lambda: 100, setValue=mock.Mock()) + w._save_path_input = SimpleNamespace(setText=mock.Mock()) + w._save_name_input = SimpleNamespace(setText=mock.Mock()) + w._start_run_with_recording = mock.Mock() + with self.assertRaisesRegex(ValueError, "from 1 to 100"): + w._start_web_recording("/tmp/new", "run", 101) + w._save_path_input.setText.assert_not_called() + w._save_name_input.setText.assert_not_called() + w._record_count.setValue.assert_not_called() + w._start_run_with_recording.assert_not_called() + class WebPortTest(unittest.TestCase): def _port_with_env(self, value: str | None) -> int: @@ -257,9 +354,59 @@ class WebPortTest(unittest.TestCase): self.assertEqual(self._port_with_env("9000"), 9000) +class WebLiveSettingsRouteTest(unittest.TestCase): + def setUp(self) -> None: + self.controller = mock.Mock() + self.request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(controller=self.controller))) + self.enterContext(mock.patch( + "python_app.webui.routes.asyncio.to_thread", + new=mock.AsyncMock(side_effect=lambda func, *args: func(*args)), + )) + + def test_response_contains_schema_after_the_action_finishes(self) -> None: + updated = [{"name": "bscan_gain", "value": 2.0}] + self.controller.apply_live_settings.return_value = updated + + response = asyncio.run(post_live_settings(self.request, {"bscan_gain": 2.0})) + + self.assertEqual(response, updated) + self.controller.apply_live_settings.assert_called_once_with({"bscan_gain": 2.0}) + + def test_gui_failure_is_returned_as_http_error(self) -> None: + self.controller.apply_live_settings.side_effect = WebActionError("Invalid antenna geometry") + + with self.assertRaises(HTTPException) as caught: + asyncio.run(post_live_settings(self.request, {"tx_geometry": []})) + self.assertEqual(caught.exception.status_code, 400) + self.assertEqual(caught.exception.detail, "Invalid antenna geometry") + + # --------------------------------------------------------------------------- # # Latest-wins fan-out # --------------------------------------------------------------------------- # +class WebSocketInitialFrameTest(unittest.TestCase): + def test_new_client_receives_cached_plot_even_when_no_new_frame_is_published(self) -> None: + self.enterContext(mock.patch( + "python_app.webui.routes.asyncio.to_thread", + new=mock.AsyncMock(side_effect=lambda func, *args: func(*args)), + )) + for frame in ({"type": "frame", "seq": 7, "png_b64": "cached"}, None): + with self.subTest(frame=frame): + controller = SimpleNamespace(peek_frame=lambda: frame) + broadcaster = mock.Mock() + status = {"type": "status", "running": False} + broadcaster.register.side_effect = lambda queue: queue.put_nowait(status) + socket = SimpleNamespace( + app=SimpleNamespace(state=SimpleNamespace(controller=controller, broadcaster=broadcaster)), + accept=mock.AsyncMock(), close=mock.AsyncMock(), + send_json=mock.AsyncMock(side_effect=WebSocketDisconnect), + ) + asyncio.run(ws(socket)) + socket.send_json.assert_awaited_once_with(frame if frame is not None else status) + broadcaster.unregister.assert_called_once() + socket.close.assert_awaited_once() + + class RingBroadcasterFanOutTest(unittest.TestCase): def test_full_client_queue_drops_oldest(self) -> None: async def scenario() -> None: diff --git a/python_app/tests/test_switched_matrix_capture.py b/python_app/tests/test_switched_matrix_capture.py index 98cb499..7a84bab 100644 --- a/python_app/tests/test_switched_matrix_capture.py +++ b/python_app/tests/test_switched_matrix_capture.py @@ -187,31 +187,23 @@ class ManualComboCaptureUsesTargetedAcquisitionTest(unittest.TestCase): def test_manual_capture_runs_one_inner_collection_per_median_sweep(self) -> None: config = self._switched_mock_config() - session = SequentialCaptureSession( - config=config, - kind="s21_calibration", - set_name="targeted_test", - median_sweep_count=2, - ) - with mock.patch.object( - MultiDeviceLibreVnaService, - "acquire_collection", - autospec=True, - side_effect=MultiDeviceLibreVnaService.acquire_collection, - ) as inner_acquire: - session.open() - try: - trace = session.capture_current_combo() - finally: - session.close() - + service, inner, _ = _switched_service(input_steps=3) + with mock.patch( + "python_app.workflows.sequential_capture_workflow.create_matrix_radar_service", + return_value=service, + ): + session = SequentialCaptureSession( + config=config, kind="s21_calibration", set_name="targeted_test", median_sweep_count=2, + ) + session.open() + try: + trace = session.capture_current_combo() + finally: + session.close() first_combo = config.combos[0] - self.assertEqual( - (trace.combo.input, trace.combo.output), - (first_combo.input, first_combo.output), - ) + self.assertEqual((trace.combo.input, trace.combo.output), (first_combo.input, first_combo.output)) # 2 median sweeps of ONE physical step — not 2 x 3 full-matrix steps. - self.assertEqual(inner_acquire.call_count, 2) + self.assertEqual(inner.acquire_count, 2) class PerPointStimulusSwitchingConfigTest(unittest.TestCase): @@ -238,7 +230,9 @@ class PerPointStimulusSwitchingConfigTest(unittest.TestCase): # The switch is stepped by the controller in the hardware trigger loop, so the # host must acquire the whole 4x4 matrix in ONE sweep and never wrap the # service in the sweep-per-step widener. - service = create_matrix_radar_service(self._config(2)) + config = self._config(2) + config.radar.driver_mode = "native" + service = create_matrix_radar_service(config) self.assertIsInstance(service, MultiDeviceLibreVnaService) self.assertNotIsInstance(service, SwitchedMatrixRadarService) self.assertEqual(service.output_position_count, 4) diff --git a/python_app/webui/controller.py b/python_app/webui/controller.py index e85b4c9..0d9b0ba 100644 --- a/python_app/webui/controller.py +++ b/python_app/webui/controller.py @@ -64,7 +64,13 @@ class WebController(Protocol): """Load a run-config profile by file name (the desktop "Load Config" button).""" def save_dataset(self, path: str, name: str) -> None: - """Save the runtime dataset to ``path``/``name`` (the desktop "Save Dataset" button).""" + """Save up to 1000 recent measurements to ``path``/``name``.""" + + def select_combo(self, input_pos: int, output_pos: int) -> None: + """Select the matrix B-scan input/output pair shown by the desktop.""" + + def next_combo(self) -> None: + """Advance the matrix B-scan display to the next input/output pair.""" def current_live_settings(self) -> list: """Return the live-settings schema (built from the Qt widgets).""" diff --git a/python_app/webui/routes.py b/python_app/webui/routes.py index a3dc1d3..b77d91d 100644 --- a/python_app/webui/routes.py +++ b/python_app/webui/routes.py @@ -26,7 +26,7 @@ def _controller(request: Request) -> WebController: return request.app.state.controller -async def _run_action(func, *args) -> None: +async def _run_action(func, *args): """Run a (blocking) controller control call off the event loop, mapping failures to 400. The control methods run the desktop action synchronously and raise ``ValueError`` @@ -35,7 +35,7 @@ async def _run_action(func, *args) -> None: keeps the async event loop free while the GUI thread does the work. """ try: - await asyncio.to_thread(func, *args) + return await asyncio.to_thread(func, *args) except (ValueError, WebActionError) as exc: logger.warning("Web UI action failed: %s", exc) raise HTTPException(status_code=400, detail=str(exc)) from exc @@ -75,7 +75,7 @@ async def post_start_recording( request: Request, path: str = Body("", embed=True), name: str = Body("", embed=True), - count: int = Body(..., embed=True), + count: int = Body(..., embed=True, strict=True, ge=1), ) -> dict: logger.info("Web UI request: start with disk recording (count=%s)", count) controller = _controller(request) @@ -129,13 +129,27 @@ async def get_live_settings(request: Request) -> list: return _controller(request).current_live_settings() +@router.post("/api/display_combo") +async def post_display_combo( + request: Request, + input_pos: int = Body(..., embed=True, strict=True, ge=0), + output_pos: int = Body(..., embed=True, strict=True, ge=0), +) -> dict: + controller = _controller(request) + await _run_action(controller.select_combo, input_pos, output_pos) + return controller.status() + + +@router.post("/api/next_combo") +async def post_next_combo(request: Request) -> dict: + controller = _controller(request) + await _run_action(controller.next_combo) + return controller.status() + + @router.post("/api/live_settings") async def post_live_settings(request: Request, fields: dict = Body(default={})) -> list: - try: - return _controller(request).apply_live_settings(fields) - except ValueError as exc: - logger.warning("Web UI rejected live settings update: %s", exc) - raise HTTPException(status_code=400, detail=str(exc)) from exc + return await _run_action(_controller(request).apply_live_settings, fields) @router.websocket("/ws") @@ -147,6 +161,9 @@ async def ws(websocket: WebSocket) -> None: broadcaster.register(queue) logger.info("Web UI client connected") try: + frame = await asyncio.to_thread(websocket.app.state.controller.peek_frame) + if frame is not None: + await websocket.send_json(frame) while True: await websocket.send_json(await queue.get()) except WebSocketDisconnect: diff --git a/python_app/webui/static/app.js b/python_app/webui/static/app.js index 5cf0b98..2bc894c 100644 --- a/python_app/webui/static/app.js +++ b/python_app/webui/static/app.js @@ -8,6 +8,10 @@ /* ---- DOM handles ------------------------------------------------- */ const plotImg = document.getElementById("plot"); +const sar3dStatusEl = document.getElementById("sar3d-status"); +const displayComboControls = document.getElementById("display-combo-controls"); +const displayComboSelect = document.getElementById("display-combo"); +const btnNextCombo = document.getElementById("btn-next-combo"); const btnStart = document.getElementById("btn-start"); const btnSingle = document.getElementById("btn-single"); @@ -89,6 +93,25 @@ bindControl(btnStop, "/api/stop", {}); bindControl(btnTmpRef, "/api/tmp_reference", {}); bindControl(btnRemoveLast, "/api/remove_last", {}); +let comboOptionsSignature = ""; + +async function changeDisplayCombo(path, body) { + displayComboSelect.disabled = btnNextCombo.disabled = true; + try { + applyStatus(await api(path, body)); + } catch (err) { + toast(err.message, true); + } finally { + displayComboSelect.disabled = btnNextCombo.disabled = false; + } +} + +displayComboSelect.addEventListener("change", () => { + const [input_pos, output_pos] = displayComboSelect.value.split(":").map(Number); + changeDisplayCombo("/api/display_combo", { input_pos, output_pos }); +}); +btnNextCombo.addEventListener("click", () => changeDisplayCombo("/api/next_combo", {})); + /* ---- settings panel --------------------------------------------- */ settingsToggle.addEventListener("click", () => sidePanel.classList.toggle("collapsed")); @@ -98,24 +121,88 @@ settingsToggle.addEventListener("click", () => sidePanel.classList.toggle("colla const fieldInputs = {}; // field name -> { el, kind, dirty } let formSignature = ""; // field names currently in the form (detect mode/structure change) +function settingsFormSignature(schema) { + return JSON.stringify(schema.map((entry) => [entry.name, entry.kind, entry.position_count])); +} + +function makeGeometryEditor(entry) { + const el = document.createElement("div"); + el.className = "geometry-editor"; + const table = document.createElement("table"); + const head = table.createTHead().insertRow(); + const keys = [entry.position_field, "x_m", "y_m", "z_m"]; + for (const title of [entry.position_field === "output_pos" ? "Output" : "Input", "X (m)", "Y (m)", "Z (m)", ""]) { + const cell = document.createElement("th"); + cell.textContent = title; + head.appendChild(cell); + } + const body = table.createTBody(); + const add = document.createElement("button"); + const appendRow = (record) => { + const row = body.insertRow(); + for (const [index, key] of keys.entries()) { + const input = document.createElement("input"); + input.type = "number"; + input.step = index === 0 ? "1" : "any"; + input.required = true; + input.setAttribute("aria-label", `${entry.name} ${key}`); + if (index === 0) { input.min = "0"; input.max = entry.position_count - 1; } + input.value = record[key] ?? 0; + input.disabled = entry.enabled === false; + row.insertCell().appendChild(input); + } + const remove = document.createElement("button"); + remove.type = "button"; + remove.textContent = "−"; + remove.title = "Remove antenna"; + remove.disabled = entry.enabled === false; + remove.addEventListener("click", () => { + row.remove(); + add.disabled = entry.enabled === false; + el.dispatchEvent(new Event("input")); + }); + row.insertCell().appendChild(remove); + }; + el.getRecords = () => Array.from(body.rows, (row) => Object.fromEntries( + Array.from(row.querySelectorAll("input"), (input, index) => { + if (!input.reportValidity()) throw new Error("Enter valid antenna coordinates"); + return [keys[index], Number(input.value)]; + }) + )); + el.setRecords = (records) => { + body.replaceChildren(); + for (const record of records) appendRow(record); + add.disabled = entry.enabled === false || records.length >= entry.position_count; + }; + el.setRecords(entry.value); + add.type = "button"; + add.textContent = "Add antenna"; + add.addEventListener("click", () => { + if (body.rows.length >= entry.position_count) return; + const positions = new Set(Array.from(body.rows, (row) => Number(row.querySelector("input").value))); + let position = 0; + while (positions.has(position)) position++; + appendRow({ [entry.position_field]: position }); + add.disabled = body.rows.length >= entry.position_count; + el.dispatchEvent(new Event("input")); + }); + el.append(table, add); + return el; +} + function makeFieldRow(entry) { const row = document.createElement("div"); row.className = "field"; + if (entry.kind === "geometry") row.classList.add("geometry-field"); const label = document.createElement("label"); - label.textContent = entry.name; + label.textContent = entry.kind === "geometry" ? (entry.name === "tx_geometry" ? "Tx antenna positions" : "Rx antenna positions") : entry.name; label.htmlFor = "f_" + entry.name; - if (entry.applies_on_start) { - // Stable run_config field: editing it here takes effect on the next pipeline start. - const hint = document.createElement("span"); - hint.className = "on-start-hint"; - hint.textContent = " (on Start)"; - hint.title = "Applied when the pipeline next starts"; - label.appendChild(hint); - } row.appendChild(label); let el; - if (entry.kind === "bool") { + if (entry.kind === "geometry") { + el = makeGeometryEditor(entry); + } else if (entry.kind === "bool") { el = document.createElement("input"); el.type = "checkbox"; el.checked = !!entry.value; @@ -173,23 +260,23 @@ function buildSettingsForm(schema) { settingsFields.appendChild(heading); } settingsFields.appendChild(makeFieldRow(entry)); - if (entry.name === "rx_geometry") { - const note = document.createElement("div"); - note.className = "config-active"; - note.textContent = "To apply Tx/Rx geometry or permittivity changes: Save Config and restart the app"; - settingsFields.appendChild(note); - } } - formSignature = schema.map((e) => e.name).join(","); + formSignature = settingsFormSignature(schema); } function collectFields() { const out = {}; for (const key in fieldInputs) { const { el, kind } = fieldInputs[key]; - if (kind === "bool") out[key] = el.checked; - else if (kind === "int") out[key] = parseInt(el.value, 10); - else if (kind === "float") out[key] = parseFloat(el.value); + if (kind === "geometry") out[key] = el.getRecords(); + else if (kind === "bool") out[key] = el.checked; + else if (kind === "int" || kind === "float") { + const value = Number(el.value); + if (!el.value.trim() || !Number.isFinite(value) || (kind === "int" && !Number.isSafeInteger(value))) { + throw new Error("Enter valid numeric settings"); + } + out[key] = value; + } else out[key] = el.value; // select + text (positions are sent as CSV text) } return out; @@ -197,7 +284,8 @@ function collectFields() { function setFieldValue(input, value) { const { el, kind } = input; - if (kind === "bool") el.checked = !!value; + if (kind === "geometry") el.setRecords(value); + else if (kind === "bool") el.checked = !!value; else if (kind === "select") el.value = String(value); else el.value = value == null ? "" : value; } @@ -218,13 +306,13 @@ async function applyOne(field, value) { // changed (e.g. the mode switched); otherwise update values in place without // clobbering a field the operator is editing here. function applySettings(schema) { - if (schema.map((e) => e.name).join(",") !== formSignature) { + if (settingsFormSignature(schema) !== formSignature) { buildSettingsForm(schema); return; } for (const entry of schema) { const input = fieldInputs[entry.name]; - if (!input || input.el === document.activeElement || input.dirty) continue; + if (!input || input.el.contains(document.activeElement) || input.dirty) continue; setFieldValue(input, entry.value); if (entry.enabled !== undefined) input.el.disabled = entry.enabled === false; } @@ -333,8 +421,8 @@ function updateRecordButtonState() { } btnStartRecording.addEventListener("click", async () => { - const count = parseInt(recordCountInput.value, 10); - if (!Number.isFinite(count) || count < 1) { + const count = Number(recordCountInput.value); + if (!Number.isSafeInteger(count) || count < 1) { toast("Enter how many sweeps to record (>= 1)", true); return; } @@ -361,6 +449,22 @@ function setStat(el, label, value, cls) { } function applyStatus(s) { + if (s.display_combo) { + const combo = s.display_combo; + displayComboControls.hidden = !combo.enabled; + const options = combo.options || []; + const signature = JSON.stringify(options); + if (signature !== comboOptionsSignature) { + displayComboSelect.replaceChildren(...options.map(([inp, out]) => { + const option = document.createElement("option"); + option.value = `${inp}:${out}`; + option.textContent = `Input ${inp} / Output ${out}`; + return option; + })); + comboOptionsSignature = signature; + } + displayComboSelect.value = `${combo.input}:${combo.output}`; + } if ("running" in s) { setStat(runningEl, "running", s.running ? "yes" : "no", s.running ? "ok" : "off"); pipelineRunning = !!s.running; @@ -393,6 +497,8 @@ function applyStatus(s) { if ("raw_count" in s) setStat(scansEl, "scans", `raw ${s.raw_count} / preprocessed ${s.preprocessed_count} / results ${s.result_count}`); + sar3dStatusEl.textContent = s.sar3d_status || ""; + sar3dStatusEl.hidden = !s.sar3d_status; } /* ---- frame rendering (latest-wins; the frame IS the Qt plot image) - */ diff --git a/python_app/webui/static/index.html b/python_app/webui/static/index.html index 62da8f9..9d5ce9d 100644 --- a/python_app/webui/static/index.html +++ b/python_app/webui/static/index.html @@ -22,7 +22,12 @@
Processor output +
+
Live processor plot
@@ -45,7 +50,7 @@
- +
tuple[str, SweepCollection]: - """Capture all switch combinations and persist them as calibration set.""" - logger.info("Starting one-shot calibration capture: set=%s", set_name) - if config.is_matrix_radar: - raise RuntimeError( - "Matrix-radar S21 through calibration is not supported by this one-shot full-set helper. " - "Use the sequential preprocess capture flow so each virtual combo can be connected through " - "and captured explicitly." - ) - - session = SequentialCaptureSession( - config=config, - kind="s21_calibration", - set_name=set_name, - median_sweep_count=median_sweep_count, - ) - try: - session.open() - while not session.is_complete(): - session.capture_current_combo() - return session.finalize(store) - finally: - session.close() diff --git a/python_app/workflows/kamil_adc_neutral_preprocess.py b/python_app/workflows/kamil_adc_neutral_preprocess.py index fa6ab7b..1b80e73 100644 --- a/python_app/workflows/kamil_adc_neutral_preprocess.py +++ b/python_app/workflows/kamil_adc_neutral_preprocess.py @@ -6,9 +6,8 @@ reference is subtracted, so applying both leaves the measured S21 untouched. Tha lets an operator start the pipeline before any real calibration exists, which the required-asset check in `_start_run` would otherwise refuse. -Supported models: Kamil ADC (axis from the ADC processing grid) and every -VNA-style model, including synchronized multi-device LibreVNA (axis from the -configured linear sweep grid). +The shortcut is enabled for verified axes: Kamil ADC's processing grid, +synchronized multi-device LibreVNA, and the physical single LibreVNA mock. """ from __future__ import annotations @@ -29,26 +28,14 @@ logger = logging.getLogger(__name__) def supports_neutral_preprocess_sets(config: RunConfigModel) -> bool: - """Return whether neutral S21 sets can be generated for this radar model. - - Enabled for the Kamil ADC and for synchronized multi-device LibreVNA, the two - models whose emitted frequency axis is fully derivable from the config alone. - Other models still work through `build_neutral_s21_sets`, but are kept out of the - UI shortcut until their axis has been verified against real hardware. - """ - return bool(config.is_kamil_adc or config.is_multi_device) + """Enable the shortcut only when the emitted frequency grid is verified.""" + return bool(config.is_kamil_adc or config.is_multi_device or ( + config.radar.model == RunConfigModel.LIBREVNA_MODEL and config.radar.driver_mode == "mock" + )) def neutral_frequency_grid_hz(config: RunConfigModel) -> np.ndarray: - """Return the exact per-trace frequency axis the configured radar emits. - - Neutral sets must line up sample-for-sample with live sweeps, so the axis comes - from the same source the acquisition path uses: the ADC processing grid for Kamil - ADC, and the configured linear sweep grid for every VNA-style model (LibreVNA - single and multi-device, SN9000, Compact-M). The C++ preprocessor re-checks this - axis against the measured one within a tolerance, so a mismatch fails loudly - instead of silently corrupting the correction. - """ + """Derive the neutral axis; the preprocessor verifies it against measured data.""" if config.is_kamil_adc: # Single source of truth for the axis: the same grid the processor emits. processor = KamilAdcSweepProcessor( @@ -59,8 +46,6 @@ def neutral_frequency_grid_hz(config: RunConfigModel) -> np.ndarray: points = int(config.radar.sweep.points) if points < 1: raise ValueError("Neutral sets require radar.sweep.points >= 1") - if points == 1: - return np.array([float(config.radar.sweep.start_hz)], dtype=np.float32) # Mirrors both acquisition paths: the native collector seeds this same linspace # and the mock backend generates it outright. return np.linspace( @@ -110,19 +95,6 @@ def build_neutral_s21_sets( return calibration, reference -def build_kamil_adc_neutral_s21_sets( - config: RunConfigModel, -) -> tuple[SweepCollection, SweepCollection]: - """Build neutral S21 sets, rejecting anything but the Kamil ADC radar. - - Kept as the model-checked entry point for the ADC path; new callers that must - work for several radar models should use `build_neutral_s21_sets` instead. - """ - if not config.is_kamil_adc: - raise ValueError("Neutral Kamil ADC sets require radar.model='kamil_adc'") - return build_neutral_s21_sets(config) - - def _neutral_collection( *, combos: list[ComboModel], diff --git a/python_app/workflows/multi_radar_capture_workflow.py b/python_app/workflows/multi_radar_capture_workflow.py index db03007..246ba31 100644 --- a/python_app/workflows/multi_radar_capture_workflow.py +++ b/python_app/workflows/multi_radar_capture_workflow.py @@ -6,13 +6,12 @@ from contextlib import suppress from dataclasses import dataclass import logging import time - -import numpy as np +from typing import cast from python_app.hardware_full.matrix_radar_service import MatrixRadarService, create_matrix_radar_service -from python_app.hardware_full.single_radar_service import create_single_radar_service +from python_app.hardware_full.single_radar_service import SingleRadarService, create_single_radar_service from python_app.hardware_full.switch_service import SwitchService -from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData +from python_app.models.dataset_model import SweepCollection, TraceData from python_app.models.run_config_model import ComboModel, RunConfigModel from python_app.storage.npz_store import NpzStore from python_app.workflows.radar_config_variants import RadarConfigVariant @@ -20,9 +19,11 @@ from python_app.workflows.sequential_capture_workflow import ( MATRIX_RADAR_MANUAL_CAPTURE_KINDS, SequentialCaptureState, acquire_matrix_combo_collection, + capture_single_radar_combo, combine_collections_via_median, combine_traces_via_median, select_trace_for_combo, + trace_capture_window, ) logger = logging.getLogger(__name__) @@ -103,8 +104,9 @@ class MultiRadarSequentialCaptureSession: self._is_matrix_radar, ) + self._radar: MatrixRadarService | SingleRadarService if self._is_matrix_radar: - self._radar: MatrixRadarService = create_matrix_radar_service(base_config) + self._radar = create_matrix_radar_service(base_config) self._input_switch = None self._output_switch = None else: @@ -145,14 +147,10 @@ class MultiRadarSequentialCaptureSession: def close(self) -> None: """Close all opened hardware resources.""" was_open = self._opened - with suppress(Exception): - if self._output_switch is not None: - self._output_switch.close() - with suppress(Exception): - if self._input_switch is not None: - self._input_switch.close() - with suppress(Exception): - self._radar.close() + for resource in (self._output_switch, self._input_switch, self._radar): + if resource is not None: + with suppress(Exception): + resource.close() self._opened = False if was_open: logger.info("Multi-radar capture session closed (kind=%s set=%s)", self._kind, self._set_name) @@ -189,6 +187,7 @@ class MultiRadarSequentialCaptureSession: variant_labels: list[str] = [] if self._is_matrix_radar: + radar = cast(MatrixRadarService, self._radar) for variant in self._radar_variants: self._radar.configure(variant.config.radar.sweep) if self._base_config.runtime.settling_ms > 0: @@ -198,7 +197,7 @@ class MultiRadarSequentialCaptureSession: # that contains it instead of the full (switch-widened) matrix. per_sweep_traces = [ select_trace_for_combo( - acquire_matrix_combo_collection(self._radar, combo), combo + acquire_matrix_combo_collection(radar, combo), combo ) for _ in range(self._median_sweep_count) ] @@ -208,7 +207,7 @@ class MultiRadarSequentialCaptureSession: else: collections: list[SweepCollection] = [] for _ in range(self._median_sweep_count): - collection = self._radar.acquire_collection(collection_id=1) + collection = radar.acquire_collection(collection_id=1) if not collection.traces: raise RuntimeError( f"Matrix radar variant {variant.display_name} returned no traces" @@ -230,21 +229,9 @@ class MultiRadarSequentialCaptureSession: self._radar.configure(variant.config.radar.sweep) if self._base_config.runtime.settling_ms > 0: time.sleep(self._base_config.runtime.settling_ms / 1000.0) - sweep_traces: list[TraceData] = [] - for _ in range(self._median_sweep_count): - sweep_start_ns = time.monotonic_ns() - sweep = self._radar.acquire() - sweep_traces.append( - TraceData( - combo=ComboKey(input=combo.input, output=combo.output), - frequency_hz=np.asarray(sweep.x, dtype=np.float32), - s11=np.asarray(sweep.trace("s11"), dtype=np.complex64), - s21=np.asarray(sweep.trace("s21"), dtype=np.complex64), - capture_start_ns=sweep_start_ns, - capture_end_ns=time.monotonic_ns(), - ) - ) - trace = combine_traces_via_median(sweep_traces) + trace = capture_single_radar_combo( + cast(SingleRadarService, self._radar), combo, self._median_sweep_count, + ) pending_traces_by_radar_key[variant.radar_key] = [trace] display_traces.append(trace) variant_labels.append(variant.display_name) @@ -343,10 +330,13 @@ class MultiRadarSequentialCaptureSession: saved_sets: list[MultiRadarSavedSet] = [] for variant in self._radar_variants: traces = list(self._traces_by_radar_key[variant.radar_key]) + capture_start_ns, capture_end_ns = trace_capture_window(traces) collection = SweepCollection( collection_id=1, monotonic_ns=monotonic_ns, traces=traces, + capture_start_ns=capture_start_ns, + capture_end_ns=capture_end_ns, ) store.save_set(self._kind, variant.radar_key, self._set_name, collection) saved_sets.append( diff --git a/python_app/workflows/radar_config_variants.py b/python_app/workflows/radar_config_variants.py index a971051..eb92f81 100644 --- a/python_app/workflows/radar_config_variants.py +++ b/python_app/workflows/radar_config_variants.py @@ -143,18 +143,9 @@ def _load_radar_config_variant(path: Path, *, base_config: RunConfigModel) -> Ra if not any(key in sweep_payload for key in _RADAR_SWEEP_KEYS): raise ValueError("radar.sweep does not contain any supported sweep keys") - config = base_config.clone() - sweep = config.radar.sweep - if "start_hz" in sweep_payload: - sweep.start_hz = float(sweep_payload["start_hz"]) - if "stop_hz" in sweep_payload: - sweep.stop_hz = float(sweep_payload["stop_hz"]) - if "points" in sweep_payload: - sweep.points = int(sweep_payload["points"]) - if "if_bandwidth_hz" in sweep_payload: - sweep.if_bandwidth_hz = float(sweep_payload["if_bandwidth_hz"]) - if "stimulus_power_dbm" in sweep_payload: - sweep.power_dbm = float(sweep_payload["stimulus_power_dbm"]) + config_payload = base_config.to_dict() + config_payload["radar"]["sweep"].update(sweep_payload) + config = RunConfigModel.from_dict(config_payload) radar_key = radar_key_from_config( model_name=config.radar.model, diff --git a/python_app/workflows/sequential_capture_workflow.py b/python_app/workflows/sequential_capture_workflow.py index bdd67f2..5b65959 100644 --- a/python_app/workflows/sequential_capture_workflow.py +++ b/python_app/workflows/sequential_capture_workflow.py @@ -6,11 +6,12 @@ from contextlib import suppress from dataclasses import dataclass import logging import time +from typing import cast import numpy as np from python_app.hardware_full.matrix_radar_service import MatrixRadarService, create_matrix_radar_service -from python_app.hardware_full.single_radar_service import create_single_radar_service +from python_app.hardware_full.single_radar_service import SingleRadarService, create_single_radar_service from python_app.hardware_full.switch_service import SwitchService from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData from python_app.models.run_config_model import ComboModel, RunConfigModel @@ -22,6 +23,13 @@ MATRIX_RADAR_MANUAL_CAPTURE_KINDS = frozenset({"s21_calibration", "s11_open", "s DEFAULT_CALIBRATION_MEDIAN_SWEEP_COUNT = 5 +def trace_capture_window(traces: list[TraceData]) -> tuple[int, int]: + """Return the total measured window, retaining zero when timing was not reported.""" + starts = [int(trace.capture_start_ns) for trace in traces if int(trace.capture_start_ns) > 0] + ends = [int(trace.capture_end_ns) for trace in traces if int(trace.capture_end_ns) > 0] + return min(starts, default=0), max(ends, default=0) + + @dataclass(slots=True) class SequentialCaptureState: """Snapshot of sequential capture progress for the GUI/controller.""" @@ -81,8 +89,9 @@ class SequentialCaptureSession: self._median_sweep_count, ) + self._radar: MatrixRadarService | SingleRadarService if self._is_matrix_radar: - self._radar: MatrixRadarService = create_matrix_radar_service(config) + self._radar = create_matrix_radar_service(config) self._input_switch = None self._output_switch = None else: @@ -121,14 +130,10 @@ class SequentialCaptureSession: def close(self) -> None: """Close all opened hardware resources.""" was_open = self._opened - with suppress(Exception): - if self._output_switch is not None: - self._output_switch.close() - with suppress(Exception): - if self._input_switch is not None: - self._input_switch.close() - with suppress(Exception): - self._radar.close() + for resource in (self._output_switch, self._input_switch, self._radar): + if resource is not None: + with suppress(Exception): + resource.close() self._opened = False if was_open: logger.info("Sequential capture session closed (kind=%s set=%s)", self._kind, self._set_name) @@ -156,12 +161,13 @@ class SequentialCaptureSession: raise RuntimeError("Capture session is already complete") if self._is_matrix_radar: + radar = cast(MatrixRadarService, self._radar) if self._manual_matrix_radar_capture: # Only this combo is kept, so acquire the smallest collection that # contains it instead of the full (switch-widened) matrix. per_sweep_traces = [ select_trace_for_combo( - acquire_matrix_combo_collection(self._radar, combo), combo + acquire_matrix_combo_collection(radar, combo), combo ) for _ in range(self._median_sweep_count) ] @@ -173,7 +179,7 @@ class SequentialCaptureSession: collections: list[SweepCollection] = [] for _ in range(self._median_sweep_count): - collection = self._radar.acquire_collection(collection_id=1) + collection = radar.acquire_collection(collection_id=1) if not collection.traces: raise RuntimeError("Matrix radar capture returned no traces") collections.append(collection) @@ -189,32 +195,7 @@ class SequentialCaptureSession: self._input_switch.switch_to(combo.input) if self._config.runtime.settling_ms > 0: time.sleep(self._config.runtime.settling_ms / 1000.0) - - # A free-running streaming radar (Kamil ADC) keeps a sweep captured in the - # previous combination buffered, and may have a transition-straddling sweep - # in flight. Drop those so this capture holds data from the new switch state - # — otherwise the trace is labelled with this combo but carries the previous - # one's data (an off-by-one across the sequence). Discrete radars (LibreVNA) - # acquire a fresh sweep per call and expose no such method, so skip them. - drain_after_switch = getattr(self._radar, "drain_after_switch", None) - if callable(drain_after_switch): - drain_after_switch() - - sweep_traces: list[TraceData] = [] - for _ in range(self._median_sweep_count): - sweep_start_ns = time.monotonic_ns() - sweep = self._radar.acquire() - sweep_traces.append( - TraceData( - combo=ComboKey(input=combo.input, output=combo.output), - frequency_hz=np.asarray(sweep.x, dtype=np.float32), - s11=np.asarray(sweep.trace("s11"), dtype=np.complex64), - s21=np.asarray(sweep.trace("s21"), dtype=np.complex64), - capture_start_ns=sweep_start_ns, - capture_end_ns=time.monotonic_ns(), - ) - ) - trace = combine_traces_via_median(sweep_traces) + trace = capture_single_radar_combo(cast(SingleRadarService, self._radar), combo, self._median_sweep_count) self._traces.append(trace) self._next_index += 1 logger.debug( @@ -279,10 +260,13 @@ class SequentialCaptureSession: if not self.is_complete(): raise RuntimeError("Capture session is not complete") + capture_start_ns, capture_end_ns = trace_capture_window(self._traces) collection = SweepCollection( collection_id=1, monotonic_ns=time.monotonic_ns(), traces=list(self._traces), + capture_start_ns=capture_start_ns, + capture_end_ns=capture_end_ns, ) radar_key = radar_key_from_config( model_name=self._config.radar.model, @@ -311,6 +295,32 @@ class SequentialCaptureSession: return self._combos[self._next_index] +def capture_single_radar_combo( + radar: SingleRadarService, combo: ComboModel, median_sweep_count: int, +) -> TraceData: + """Capture the selected antenna, discarding buffered data from the previous one.""" + set_active_combo = getattr(radar, "set_active_combo", None) + if callable(set_active_combo): + set_active_combo(ComboKey(combo.input, combo.output)) + # Streaming Kamil sweeps may predate or straddle the switch transition. + drain_after_switch = getattr(radar, "drain_after_switch", None) + if callable(drain_after_switch): + drain_after_switch() + traces = [] + for _ in range(median_sweep_count): + sweep_start_ns = time.monotonic_ns() + sweep = radar.acquire() + traces.append(TraceData( + combo=ComboKey(combo.input, combo.output), + frequency_hz=np.asarray(sweep.x, dtype=np.float32), + s11=np.asarray(sweep.trace("s11"), dtype=np.complex64), + s21=np.asarray(sweep.trace("s21"), dtype=np.complex64), + capture_start_ns=sweep.capture_start_ns or sweep_start_ns, + capture_end_ns=sweep.capture_end_ns or time.monotonic_ns(), + )) + return combine_traces_via_median(traces) + + def acquire_matrix_combo_collection( radar: MatrixRadarService, combo: ComboModel, @@ -325,11 +335,11 @@ def acquire_matrix_combo_collection( """ acquire_combo = getattr(radar, "acquire_combo_collection", None) if callable(acquire_combo): - collection = acquire_combo( + collection = cast(SweepCollection, acquire_combo( input_pos=int(combo.input), output_pos=int(combo.output), collection_id=collection_id, - ) + )) else: collection = radar.acquire_collection(collection_id=collection_id) if not collection.traces: @@ -379,6 +389,8 @@ def combine_traces_via_median(traces: list[TraceData]) -> TraceData: f"Median combine point-count mismatch at sweep {index}: " f"{trace.frequency_hz.size} vs {point_count}" ) + if not np.array_equal(trace.frequency_hz, first.frequency_hz): + raise RuntimeError(f"Median combine frequency-axis mismatch at sweep {index}") s11_stack = np.stack([np.asarray(t.s11, dtype=np.complex64) for t in traces], axis=0) s21_stack = np.stack([np.asarray(t.s21, dtype=np.complex64) for t in traces], axis=0) @@ -389,15 +401,14 @@ def combine_traces_via_median(traces: list[TraceData]) -> TraceData: np.median(s21_stack.real, axis=0) + 1j * np.median(s21_stack.imag, axis=0) ).astype(np.complex64) # The median is built from every input sweep, so its window spans all of them. - capture_starts = [int(t.capture_start_ns) for t in traces if int(t.capture_start_ns) > 0] - capture_ends = [int(t.capture_end_ns) for t in traces if int(t.capture_end_ns) > 0] + capture_start_ns, capture_end_ns = trace_capture_window(traces) return TraceData( combo=ComboKey(input=int(combo.input), output=int(combo.output)), frequency_hz=np.asarray(first.frequency_hz, dtype=np.float32), s11=s11_median, s21=s21_median, - capture_start_ns=min(capture_starts) if capture_starts else 0, - capture_end_ns=max(capture_ends) if capture_ends else 0, + capture_start_ns=capture_start_ns, + capture_end_ns=capture_end_ns, ) @@ -409,15 +420,14 @@ def combine_collections_via_median(collections: list[SweepCollection]) -> SweepC return collections[0] reference = collections[0] + for index, collection in enumerate(collections[1:], start=1): + if len(collection.traces) != len(reference.traces): + raise RuntimeError(f"Median collections have mismatched trace counts at sweep {index}") expected_combos = [(trace.combo.input, trace.combo.output) for trace in reference.traces] medianed_traces: list[TraceData] = [] for combo_index, (input_pos, output_pos) in enumerate(expected_combos): per_sweep_traces: list[TraceData] = [] for collection_index, collection in enumerate(collections): - if combo_index >= len(collection.traces): - raise RuntimeError( - f"Median collections have mismatched trace counts at sweep {collection_index}" - ) trace = collection.traces[combo_index] if ( int(trace.combo.input) != int(input_pos) @@ -434,6 +444,6 @@ def combine_collections_via_median(collections: list[SweepCollection]) -> SweepC collection_id=int(reference.collection_id), monotonic_ns=int(reference.monotonic_ns), traces=medianed_traces, - capture_start_ns=int(reference.capture_start_ns), - capture_end_ns=int(reference.capture_end_ns), + capture_start_ns=min((int(item.capture_start_ns) for item in collections if item.capture_start_ns > 0), default=0), + capture_end_ns=max((int(item.capture_end_ns) for item in collections if item.capture_end_ns > 0), default=0), ) diff --git a/requirements-cuda.txt b/requirements-cuda.txt new file mode 100644 index 0000000..c489ba1 --- /dev/null +++ b/requirements-cuda.txt @@ -0,0 +1,3 @@ +# Required for SAR3D on x86_64. Jetson uses JetPack's system CUDA instead. +nvidia-cuda-nvrtc==13.3.33; platform_machine == "x86_64" +nvidia-cuda-runtime==13.3.29; platform_machine == "x86_64" diff --git a/requirements.txt b/requirements.txt index 773ef96..71fc52a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,6 @@ numpy>=1.26,<3 libusb1>=3.1 pyserial>=3.5,<4 -pyvisa>=1.14 -PyVISA-py>=0.7 # pure-Python VISA backend ("@py"), required by the SN9000 (and any visa_library="@py") path # PyQt6 6.8+ ships only manylinux_2_39 aarch64 wheels (glibc >= 2.39); Jetson / # Ubuntu 22.04 has glibc 2.35, so pip would fall back to a source build that # dies on qmake. 6.7.1 is the last release with a manylinux_2_28 aarch64 wheel. @@ -12,8 +10,3 @@ pyqtgraph>=0.13.7 rpi-hardware-pwm>=0.2.2,<1 fastapi>=0.110 # embedded web UI (browser view + controls), served in-process uvicorn[standard]>=0.29 # ASGI server for the web UI, run on a background thread -# nvrtc.h, cuda.h and libnvrtc for the sar3d processor's GPU code (kernels are -# compiled at run time, no nvcc). x86_64 only: on Jetson (aarch64) CUDA comes -# with JetPack at /usr/local/cuda, where the Makefile finds it. -nvidia-cuda-nvrtc==13.3.33; platform_machine == "x86_64" # nvrtc.h + libnvrtc -nvidia-cuda-runtime==13.3.29; platform_machine == "x86_64" # cuda.h (Driver API) diff --git a/run_config.json b/run_config.json index a5eaa58..b8134f4 100644 --- a/run_config.json +++ b/run_config.json @@ -2,11 +2,8 @@ "radar": { "model": "librevna_multi", "serial": "206830985532", - "remote_host": "127.0.0.1", - "remote_port": 50209, "driver_mode": "mock", "mock_signal_hz": 5000000.0, - "visa_library": "", "multi_device": { "slave_serials": [ "206930965532", diff --git a/run_config_examples/run_config_compact_m_k209.example.json b/run_config_examples/run_config_compact_m_k209.example.json deleted file mode 100644 index 42655e0..0000000 --- a/run_config_examples/run_config_compact_m_k209.example.json +++ /dev/null @@ -1,252 +0,0 @@ -{ - "radar": { - "model": "compact_m_k209", - "serial": "", - "remote_host": "192.168.1.10", - "remote_port": 50209, - "driver_mode": "native", - "mock_signal_hz": 5000000.0, - "visa_library": "", - "multi_device": { - "slave_serials": [], - "force_external_reference": false, - "recovery_attempts": 3 - }, - "kamil_adc": { - "project_dir": "", - "executable_path": "", - "tty_path": "", - "args": [], - "env": {}, - "startup_timeout_s": 5.0, - "sweep_timeout_s": 5.0, - "stop_timeout_s": 2.0 - }, - "laser_control": { - "enabled": false, - "port": "", - "mode": "manual", - "pi_coeff1_p": 2560, - "pi_coeff1_i": 128, - "pi_coeff2_p": 2560, - "pi_coeff2_i": 128, - "manual": { - "temp1": 25.0, - "temp2": 25.0, - "current1": 30.0, - "current2": 30.0 - }, - "variation": { - "variation_type": "CHANGE_CURRENT_LD1", - "static_temp1": 25.0, - "static_temp2": 25.0, - "static_current1": 30.0, - "static_current2": 30.0, - "min_value": 30.0, - "max_value": 35.0, - "step": 0.1, - "time_step": 20, - "delay_time": 3 - } - }, - "sweep": { - "start_hz": 1000000.0, - "stop_hz": 6000000000.0, - "if_bandwidth_hz": 50000.0, - "stimulus_power_dbm": -10.0, - "points": 201 - } - }, - "switches": { - "port1": { - "name": "port1", - "driver_mode": "native", - "driver": "h7992", - "radar_port": 1, - "positions": 2, - "default_position": 0, - "gpio_chip": "/dev/gpiochip0", - "pin_a": 17, - "pin_b": 27, - "invert_logic": false - }, - "port2": { - "name": "port2", - "driver_mode": "native", - "driver": "h7992", - "radar_port": 2, - "positions": 4, - "default_position": 0, - "gpio_chip": "/dev/gpiochip0", - "pin_a": 22, - "pin_b": 23, - "invert_logic": false - } - }, - "control_button": { - "enabled": false, - "gpio_chip": "/dev/gpiochip0", - "pin": -1, - "active_low": true, - "bias": "", - "debounce_ms": 50, - "action": "capture_tmp_reference" - }, - "run": { - "settling_ms": 0, - "idle_sleep_ms": 2, - "continuous": true, - "processing_live_config_path": "python_app/runtime/processing_live.json", - "locator_server": { - "device_id": 3, - "protocol_version": 1, - "host": "0.0.0.0", - "port": 8888, - "max_payload_bytes": 65536, - "client_queue_size": 32, - "logger_name": "locator_runtime" - }, - "combos": [ - { - "input": 0, - "output": 0 - }, - { - "input": 1, - "output": 0 - }, - { - "input": 2, - "output": 0 - }, - { - "input": 3, - "output": 0 - }, - { - "input": 0, - "output": 1 - }, - { - "input": 1, - "output": 1 - }, - { - "input": 2, - "output": 1 - }, - { - "input": 3, - "output": 1 - } - ] - }, - "preprocess": { - "s21": { - "calibration": { - "set_name": "", - "bundle_path": "" - }, - "reference": { - "set_name": "", - "bundle_path": "" - } - }, - "s11": { - "calibration": { - "open": { - "set_name": "", - "bundle_path": "" - }, - "short": { - "set_name": "", - "bundle_path": "" - }, - "load": { - "set_name": "", - "bundle_path": "" - } - }, - "reference": { - "set_name": "", - "bundle_path": "" - } - }, - "notch": { - "enabled": true, - "bands_hz": [], - "taper_width_hz": 40000000.0, - "taper_type": "cosine" - } - }, - "gpr": { - "relative_permittivity": 1.0, - "tx_geometry": [ - { - "output_pos": 0, - "x_m": 0.905, - "y_m": 0.0, - "z_m": 0.0 - }, - { - "output_pos": 1, - "x_m": -0.905, - "y_m": 0.0, - "z_m": 0.0 - } - ], - "rx_geometry": [ - { - "input_pos": 0, - "x_m": -0.18, - "y_m": 0.0, - "z_m": 0.0 - }, - { - "input_pos": 1, - "x_m": 0.485, - "y_m": 0.0, - "z_m": 0.0 - }, - { - "input_pos": 2, - "x_m": -0.49, - "y_m": 0.0, - "z_m": 0.0 - }, - { - "input_pos": 3, - "x_m": 0.185, - "y_m": 0.0, - "z_m": 0.0 - } - ] - }, - "rings": { - "raw": { - "name": "/radar_raw", - "capacity": 50, - "slot_size_bytes": 2097152 - }, - "raw_tap": { - "name": "/radar_raw_tap", - "capacity": 50, - "slot_size_bytes": 2097152 - }, - "preprocessed": { - "name": "/radar_preprocessed", - "capacity": 50, - "slot_size_bytes": 2097152 - }, - "preprocessed_tap": { - "name": "/radar_preprocessed_tap", - "capacity": 50, - "slot_size_bytes": 2097152 - }, - "results": { - "name": "/radar_results", - "capacity": 50, - "slot_size_bytes": 2097152 - } - } -} diff --git a/run_config_examples/run_config_compact_m_k209_local_mock_switches.example.json b/run_config_examples/run_config_compact_m_k209_local_mock_switches.example.json deleted file mode 100644 index 851dd3a..0000000 --- a/run_config_examples/run_config_compact_m_k209_local_mock_switches.example.json +++ /dev/null @@ -1,252 +0,0 @@ -{ - "radar": { - "model": "compact_m_k209", - "serial": "", - "remote_host": "192.168.8.102", - "remote_port": 50209, - "driver_mode": "native", - "mock_signal_hz": 5000000.0, - "visa_library": "", - "multi_device": { - "slave_serials": [], - "force_external_reference": false, - "recovery_attempts": 3 - }, - "kamil_adc": { - "project_dir": "", - "executable_path": "", - "tty_path": "", - "args": [], - "env": {}, - "startup_timeout_s": 5.0, - "sweep_timeout_s": 5.0, - "stop_timeout_s": 2.0 - }, - "laser_control": { - "enabled": false, - "port": "", - "mode": "manual", - "pi_coeff1_p": 2560, - "pi_coeff1_i": 128, - "pi_coeff2_p": 2560, - "pi_coeff2_i": 128, - "manual": { - "temp1": 25.0, - "temp2": 25.0, - "current1": 30.0, - "current2": 30.0 - }, - "variation": { - "variation_type": "CHANGE_CURRENT_LD1", - "static_temp1": 25.0, - "static_temp2": 25.0, - "static_current1": 30.0, - "static_current2": 30.0, - "min_value": 30.0, - "max_value": 35.0, - "step": 0.1, - "time_step": 20, - "delay_time": 3 - } - }, - "sweep": { - "start_hz": 1000000.0, - "stop_hz": 6000000000.0, - "if_bandwidth_hz": 50000.0, - "stimulus_power_dbm": -10.0, - "points": 201 - } - }, - "switches": { - "port1": { - "name": "port1", - "driver_mode": "mock", - "driver": "h7992", - "radar_port": 1, - "positions": 2, - "default_position": 0, - "gpio_chip": "/dev/gpiochip0", - "pin_a": 17, - "pin_b": 27, - "invert_logic": false - }, - "port2": { - "name": "port2", - "driver_mode": "mock", - "driver": "h7992", - "radar_port": 2, - "positions": 4, - "default_position": 0, - "gpio_chip": "/dev/gpiochip0", - "pin_a": 22, - "pin_b": 23, - "invert_logic": false - } - }, - "control_button": { - "enabled": false, - "gpio_chip": "/dev/gpiochip0", - "pin": -1, - "active_low": true, - "bias": "", - "debounce_ms": 50, - "action": "capture_tmp_reference" - }, - "run": { - "settling_ms": 0, - "idle_sleep_ms": 2, - "continuous": false, - "processing_live_config_path": "python_app/runtime/processing_live.json", - "locator_server": { - "device_id": 3, - "protocol_version": 1, - "host": "0.0.0.0", - "port": 8888, - "max_payload_bytes": 65536, - "client_queue_size": 32, - "logger_name": "locator_runtime" - }, - "combos": [ - { - "input": 0, - "output": 0 - }, - { - "input": 1, - "output": 0 - }, - { - "input": 2, - "output": 0 - }, - { - "input": 3, - "output": 0 - }, - { - "input": 0, - "output": 1 - }, - { - "input": 1, - "output": 1 - }, - { - "input": 2, - "output": 1 - }, - { - "input": 3, - "output": 1 - } - ] - }, - "preprocess": { - "s21": { - "calibration": { - "set_name": "", - "bundle_path": "" - }, - "reference": { - "set_name": "", - "bundle_path": "" - } - }, - "s11": { - "calibration": { - "open": { - "set_name": "", - "bundle_path": "" - }, - "short": { - "set_name": "", - "bundle_path": "" - }, - "load": { - "set_name": "", - "bundle_path": "" - } - }, - "reference": { - "set_name": "", - "bundle_path": "" - } - }, - "notch": { - "enabled": true, - "bands_hz": [], - "taper_width_hz": 40000000.0, - "taper_type": "cosine" - } - }, - "gpr": { - "relative_permittivity": 1.0, - "tx_geometry": [ - { - "output_pos": 0, - "x_m": 0.905, - "y_m": 0.0, - "z_m": 0.0 - }, - { - "output_pos": 1, - "x_m": -0.905, - "y_m": 0.0, - "z_m": 0.0 - } - ], - "rx_geometry": [ - { - "input_pos": 0, - "x_m": -0.18, - "y_m": 0.0, - "z_m": 0.0 - }, - { - "input_pos": 1, - "x_m": 0.485, - "y_m": 0.0, - "z_m": 0.0 - }, - { - "input_pos": 2, - "x_m": -0.49, - "y_m": 0.0, - "z_m": 0.0 - }, - { - "input_pos": 3, - "x_m": 0.185, - "y_m": 0.0, - "z_m": 0.0 - } - ] - }, - "rings": { - "raw": { - "name": "/radar_k209_local_raw", - "capacity": 50, - "slot_size_bytes": 2097152 - }, - "raw_tap": { - "name": "/radar_k209_local_raw_tap", - "capacity": 50, - "slot_size_bytes": 2097152 - }, - "preprocessed": { - "name": "/radar_k209_local_preprocessed", - "capacity": 50, - "slot_size_bytes": 2097152 - }, - "preprocessed_tap": { - "name": "/radar_k209_local_preprocessed_tap", - "capacity": 50, - "slot_size_bytes": 2097152 - }, - "results": { - "name": "/radar_k209_local_results", - "capacity": 50, - "slot_size_bytes": 2097152 - } - } -} diff --git a/run_config_examples/run_config_kamil_adc.example.json b/run_config_examples/run_config_kamil_adc.example.json index bcc5d65..902b6d2 100644 --- a/run_config_examples/run_config_kamil_adc.example.json +++ b/run_config_examples/run_config_kamil_adc.example.json @@ -2,11 +2,7 @@ "radar": { "model": "kamil_adc", "serial": "kamil_adc", - "remote_host": "127.0.0.1", - "remote_port": 50209, "driver_mode": "native", - "mock_signal_hz": 5000000.0, - "visa_library": "", "multi_device": { "slave_serials": [], "force_external_reference": false, diff --git a/run_config_examples/run_config_librevna.example.json b/run_config_examples/run_config_librevna.example.json index a82eddd..ba2086a 100644 --- a/run_config_examples/run_config_librevna.example.json +++ b/run_config_examples/run_config_librevna.example.json @@ -2,11 +2,7 @@ "radar": { "model": "librevna", "serial": "", - "remote_host": "127.0.0.1", - "remote_port": 50209, "driver_mode": "native", - "mock_signal_hz": 5000000.0, - "visa_library": "", "multi_device": { "slave_serials": [], "force_external_reference": false, diff --git a/run_config_examples/run_config_librevna_multi.example.json b/run_config_examples/run_config_librevna_multi.example.json index 19fecca..e59cba7 100644 --- a/run_config_examples/run_config_librevna_multi.example.json +++ b/run_config_examples/run_config_librevna_multi.example.json @@ -2,11 +2,7 @@ "radar": { "model": "librevna_multi", "serial": "209E307A5532", - "remote_host": "127.0.0.1", - "remote_port": 50209, "driver_mode": "native", - "mock_signal_hz": 5000000.0, - "visa_library": "", "multi_device": { "slave_serials": [ "20A1307D5532", diff --git a/run_config_examples/run_config_librevna_multi.example2.json b/run_config_examples/run_config_librevna_multi.example2.json index 098cc90..2b13b39 100644 --- a/run_config_examples/run_config_librevna_multi.example2.json +++ b/run_config_examples/run_config_librevna_multi.example2.json @@ -2,11 +2,7 @@ "radar": { "model": "librevna_multi", "serial": "206830985532", - "remote_host": "127.0.0.1", - "remote_port": 50209, "driver_mode": "native", - "mock_signal_hz": 5000000.0, - "visa_library": "", "multi_device": { "slave_serials": [ "206930965532", diff --git a/run_config_examples/run_config_librevna_multi_per_point.example.json b/run_config_examples/run_config_librevna_multi_per_point.example.json index 533cba1..d315de7 100644 --- a/run_config_examples/run_config_librevna_multi_per_point.example.json +++ b/run_config_examples/run_config_librevna_multi_per_point.example.json @@ -2,11 +2,7 @@ "radar": { "model": "librevna_multi", "serial": "206830985532", - "remote_host": "127.0.0.1", - "remote_port": 50209, "driver_mode": "native", - "mock_signal_hz": 5000000.0, - "visa_library": "", "multi_device": { "slave_serials": [ "206930965532", diff --git a/run_config_examples/run_config_librevna_multi_per_point_lowband.example.json b/run_config_examples/run_config_librevna_multi_per_point_lowband.example.json index a279d05..fe5a6ea 100644 --- a/run_config_examples/run_config_librevna_multi_per_point_lowband.example.json +++ b/run_config_examples/run_config_librevna_multi_per_point_lowband.example.json @@ -2,11 +2,7 @@ "radar": { "model": "librevna_multi", "serial": "206830985532", - "remote_host": "127.0.0.1", - "remote_port": 50209, "driver_mode": "native", - "mock_signal_hz": 5000000.0, - "visa_library": "", "multi_device": { "slave_serials": [ "206930965532", diff --git a/run_config_examples/run_config_replay_vna.example.json b/run_config_examples/run_config_replay_vna.example.json index da0dddb..6677359 100644 --- a/run_config_examples/run_config_replay_vna.example.json +++ b/run_config_examples/run_config_replay_vna.example.json @@ -3,11 +3,7 @@ "radar": { "model": "librevna_multi", "serial": "206830985532", - "remote_host": "127.0.0.1", - "remote_port": 50209, "driver_mode": "mock", - "mock_signal_hz": 5000000.0, - "visa_library": "", "multi_device": { "slave_serials": [ "206930965532", diff --git a/run_config_examples/run_config_simulator.example.json b/run_config_examples/run_config_simulator.example.json index 364621d..1ba15ad 100644 --- a/run_config_examples/run_config_simulator.example.json +++ b/run_config_examples/run_config_simulator.example.json @@ -2,11 +2,7 @@ "radar": { "model": "librevna", "serial": "", - "remote_host": "127.0.0.1", - "remote_port": 50209, "driver_mode": "mock", - "mock_signal_hz": 5000000.0, - "visa_library": "", "multi_device": { "slave_serials": [], "force_external_reference": false, @@ -270,7 +266,7 @@ "bscan": { "axis": "abs", "cut_m": 0.0, - "max_depth_m": 3.0, + "max_depth_m": 8.0, "gain": 1.0, "start_freq_mhz": 100.0, "stop_freq_mhz": 6000.0, diff --git a/run_configs/run_config.json b/run_configs/run_config.json index 364621d..28c09d8 100644 --- a/run_configs/run_config.json +++ b/run_configs/run_config.json @@ -2,11 +2,8 @@ "radar": { "model": "librevna", "serial": "", - "remote_host": "127.0.0.1", - "remote_port": 50209, "driver_mode": "mock", "mock_signal_hz": 5000000.0, - "visa_library": "", "multi_device": { "slave_serials": [], "force_external_reference": false, diff --git a/start.sh b/start.sh index e280110..54c72c5 100755 --- a/start.sh +++ b/start.sh @@ -133,37 +133,6 @@ except Exception: echo "[start.sh] Detected radar model: ${RADAR_MODEL} (config: ${config_path})" } -# radar.replay.dataset_dir of a config, empty unless it replays a recording. -replay_dataset_dir() { - python3 -c ' -import json, sys -try: - radar = json.load(open(sys.argv[1])).get("radar", {}) - print(str(radar.get("replay", {}).get("dataset_dir", "")).strip()) -except Exception: - print("") -' "$1" 2>/dev/null -} - -# Acquisition producer for the active model, mirroring the process supervisor's -# selection so --producer-only behaves identically to a full pipeline launch: a -# replay dataset takes precedence over the model's producer. -producer_command() { - local config_path="$1" - if [[ -n "$(replay_dataset_dir "${config_path}")" ]]; then - printf '%s\0' "${PYTHON_CMD}" -m python_app.scripts.replay_raw_producer --config "${config_path}" - return - fi - case "${RADAR_MODEL}" in - kamil_adc) - printf '%s\0' "${PYTHON_CMD}" -m python_app.scripts.kamil_adc_raw_producer --config "${config_path}" ;; - librevna_multi|sn9000) - printf '%s\0' "${PYTHON_CMD}" -m python_app.scripts.matrix_raw_producer --config "${config_path}" ;; - *) - printf '%s\0' "${PROJECT_ROOT}/build/bin/sweep_orchestrator" --config "${config_path}" ;; - esac -} - check_environment() { if ! command -v python3 >/dev/null 2>&1; then echo "python3 is not installed or not found in PATH." >&2 @@ -182,13 +151,7 @@ check_environment() { } ensure_python_dependencies() { - local dependency_check - if [[ "${RADAR_MODEL}" == "kamil_adc" ]]; then - # Kamil ADC has no VISA dependency (it talks to its own L-Card collector). - dependency_check='import numpy, serial, PyQt6, pyqtgraph, usb1' - else - dependency_check='import numpy, serial, PyQt6, pyqtgraph, usb1, pyvisa' - fi + local dependency_check='import numpy, serial, PyQt6, pyqtgraph, usb1' if [[ ! -x "${VENV_PYTHON}" ]]; then echo "[start.sh] Creating virtual environment..." @@ -257,24 +220,25 @@ ensure_system_dependencies() { fi if command -v pkg-config >/dev/null 2>&1 && pkg-config --exists libusb-1.0 \ - && ((qt_gui_packages_missing == 0)); then + && ((qt_gui_packages_missing == 0)) \ + && python3 -c 'import ctypes; ctypes.CDLL("libatomic.so.1")' >/dev/null 2>&1; then return fi - echo "[start.sh] Installing system dependencies: libusb-1.0 dev headers (and Qt xcb libraries on apt)..." + echo "[start.sh] Installing system dependencies: libusb headers, libatomic and Qt xcb libraries..." if command -v apt-get >/dev/null 2>&1; then run_privileged apt-get update - run_privileged apt-get install -y pkg-config libusb-1.0-0-dev libxcb-cursor0 libxkbcommon-x11-0 + run_privileged apt-get install -y pkg-config libusb-1.0-0-dev libatomic1 libxcb-cursor0 libxkbcommon-x11-0 elif command -v dnf >/dev/null 2>&1; then - run_privileged dnf install -y pkgconf-pkg-config libusb1-devel + run_privileged dnf install -y pkgconf-pkg-config libusb1-devel libatomic elif command -v yum >/dev/null 2>&1; then - run_privileged yum install -y pkgconfig libusb1-devel + run_privileged yum install -y pkgconfig libusb1-devel libatomic elif command -v pacman >/dev/null 2>&1; then - run_privileged pacman -Sy --needed pkgconf libusb + run_privileged pacman -Sy --needed pkgconf libusb gcc-libs elif command -v zypper >/dev/null 2>&1; then - run_privileged zypper --non-interactive install pkg-config libusb-1_0-devel + run_privileged zypper --non-interactive install pkg-config libusb-1_0-devel libatomic1 elif command -v brew >/dev/null 2>&1; then - brew install pkg-config libusb + brew install pkg-config libusb gcc else echo "Could not detect supported package manager." >&2 echo "Install manually: pkg-config and libusb development package." >&2 @@ -354,14 +318,11 @@ cleanup_known_shm() { } kill_stale_adc_collector() { - # The L-Card ADC collector runs in its own session and can outlive a crashed - # or force-killed run, holding the E-502 device and hanging the next start. - # Kill any leftover hard before launching so acquisition always opens cleanly. - if command -v pkill >/dev/null 2>&1; then - if pkill -9 -f kamil_adc_collector 2>/dev/null; then - echo "[start.sh] Killed leftover ADC collector process(es)." - fi - fi + # Exact executable matching: an argument mentioning the collector is not a process to kill. + PYTHONPATH="${PROJECT_ROOT}${PYTHONPATH:+:${PYTHONPATH}}" "${PYTHON_CMD}" -c ' +from python_app.orchestration.process_supervisor import ProcessSupervisor +ProcessSupervisor._kill_kamil_collectors() +' } run_gui() { @@ -396,13 +357,14 @@ run_producer_only() { fi export PYTHONPATH="${PROJECT_ROOT}${PYTHONPATH:+:${PYTHONPATH}}" - # Run the acquisition producer the supervisor would pick for this model. - local -a command=() - while IFS= read -r -d '' token; do - command+=("${token}") - done < <(producer_command "${config_path}") - echo "[start.sh] Starting ${RADAR_MODEL} producer: ${command[*]}" - exec "${command[@]}" + exec "${PYTHON_CMD}" -c ' +import os, sys +from pathlib import Path +from python_app.orchestration.producer_selection import acquisition_command +command = acquisition_command(Path(sys.argv[1]), Path(sys.argv[2])) +print("[start.sh] Starting producer:", command, flush=True) +os.execv(command[0], command) +' "${PROJECT_ROOT}" "${config_path}" } stop_headless_service() { @@ -426,11 +388,16 @@ verify_cpp_binaries() { # binaries were never built, instead of failing obscurely at spawn time. local missing=0 bin local required_bins="data_processor data_preprocessor" - # Kamil ADC also needs its acquisition collector; other models use the C++ - # sweep_orchestrator, which `all` always builds. - if [[ "${RADAR_MODEL}" == "kamil_adc" ]]; then - required_bins="${required_bins} kamil_adc_collector" - fi + local acquisition_binary + acquisition_binary="$(PYTHONPATH="${PROJECT_ROOT}${PYTHONPATH:+:${PYTHONPATH}}" "${PYTHON_CMD}" -c ' +import sys +from pathlib import Path +from python_app.orchestration.producer_selection import acquisition_command +command = acquisition_command(Path(sys.argv[1]), Path(sys.argv[2])) +print("kamil_adc_collector" if "python_app.scripts.kamil_adc_raw_producer" in command + else "sweep_orchestrator" if len(command) == 3 else "") +' "${PROJECT_ROOT}" "${PROFILE_PATH:-${PROJECT_ROOT}/run_config.json}")" + required_bins="${required_bins} ${acquisition_binary}" for bin in ${required_bins}; do if [[ ! -x "${PROJECT_ROOT}/build/bin/${bin}" ]]; then echo "Required binary missing or not executable: build/bin/${bin}" >&2 @@ -463,18 +430,14 @@ main() { # Skip first-time provisioning (build headers, USB udev rule) in headless # mode: the daemon runs unattended at boot as a non-root user and must not # block on sudo. A fresh machine is provisioned by one interactive launch. - # Also skip it for non-LibreVNA devices (e.g. Kamil ADC), which neither use - # libusb directly nor need the LibreVNA USB access rule. - if [[ "${RADAR_MODEL}" != "kamil_adc" ]] && ((HEADLESS == 0)); then + # `all` builds the libusb orchestrator for every model; only the USB access + # rule is device-specific. Python/C++ SHM also needs libatomic. + if ((HEADLESS == 0)); then ensure_system_dependencies - ensure_usb_access_rules + if [[ "${RADAR_MODEL}" != "kamil_adc" ]]; then ensure_usb_access_rules; fi fi ensure_python_dependencies - if ((CLEAN_SHM == 1)); then - cleanup_known_shm - fi - if ((SKIP_BUILD == 0)); then build_cpp_binaries fi @@ -493,6 +456,9 @@ main() { stop_headless_service fi acquire_single_instance_lock + if ((CLEAN_SHM == 1)); then + cleanup_known_shm + fi # Clear any ADC collector wedged by a previous run before we launch a new one. kill_stale_adc_collector